diff --git a/.clang-tidy b/.clang-tidy index ddd0ee6d911..0400b500e5c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -33,6 +33,8 @@ Checks: '-*, performance-no-automatic-move, performance-trivially-destructible, performance-unnecessary-copy-initialization, + performance-noexcept-move-constructor, + performance-move-const-arg, readability-avoid-const-params-in-decls, readability-const-return-type, @@ -206,3 +208,5 @@ CheckOptions: value: CamelCase - key: modernize-loop-convert.UseCxx20ReverseRanges value: false + - key: performance-move-const-arg.CheckTriviallyCopyableMove + value: false diff --git a/.gitattributes b/.gitattributes index bcc7d57b904..a23f027122b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ contrib/* linguist-vendored *.h linguist-language=C++ +tests/queries/0_stateless/data_json/* binary diff --git a/.github/ISSUE_TEMPLATE/10_question.md b/.github/ISSUE_TEMPLATE/10_question.md index a112b9599d5..5b3d00a3180 100644 --- a/.github/ISSUE_TEMPLATE/10_question.md +++ b/.github/ISSUE_TEMPLATE/10_question.md @@ -7,6 +7,6 @@ assignees: '' --- -> Make sure to check documentation https://clickhouse.yandex/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse +> Make sure to check documentation https://clickhouse.com/docs/en/ first. If the question is concise and probably has a short answer, asking it in Telegram chat https://telegram.me/clickhouse_en is probably the fastest way to find the answer. For more complicated questions, consider asking them on StackOverflow with "clickhouse" tag https://stackoverflow.com/questions/tagged/clickhouse > If you still prefer GitHub issues, remove all this text and ask your question here. diff --git a/.github/ISSUE_TEMPLATE/50_build-issue.md b/.github/ISSUE_TEMPLATE/50_build-issue.md index a358575cd7c..9b05fbbdd13 100644 --- a/.github/ISSUE_TEMPLATE/50_build-issue.md +++ b/.github/ISSUE_TEMPLATE/50_build-issue.md @@ -7,7 +7,7 @@ assignees: '' --- -> Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.yandex/docs/en/development/build/ +> Make sure that `git diff` result is empty and you've just pulled fresh master. Try cleaning up cmake cache. Just in case, official build instructions are published here: https://clickhouse.com/docs/en/development/build/ **Operating system** diff --git a/.github/codecov.yml b/.github/codecov.yml deleted file mode 100644 index f185c5e2dcc..00000000000 --- a/.github/codecov.yml +++ /dev/null @@ -1,17 +0,0 @@ -codecov: - max_report_age: "off" - strict_yaml_branch: "master" - -ignore: - - "contrib" - - "docs" - - "benchmark" - - "tests" - - "docker" - - "debian" - - "cmake" - -comment: false - -github_checks: - annotations: false diff --git a/.github/workflows/anchore-analysis.yml b/.github/workflows/anchore-analysis.yml deleted file mode 100644 index 9f3f944c696..00000000000 --- a/.github/workflows/anchore-analysis.yml +++ /dev/null @@ -1,43 +0,0 @@ -# This workflow checks out code, performs an Anchore container image -# vulnerability and compliance scan, and integrates the results with -# GitHub Advanced Security code scanning feature. For more information on -# the Anchore scan action usage and parameters, see -# https://github.com/anchore/scan-action. For more information on -# Anchore container image scanning in general, see -# https://docs.anchore.com. - -name: Docker Container Scan (clickhouse-server) - -env: - # Force the stdout and stderr streams to be unbuffered - PYTHONUNBUFFERED: 1 - -"on": - pull_request: - paths: - - docker/server/Dockerfile - - .github/workflows/anchore-analysis.yml - schedule: - - cron: '0 21 * * *' - -jobs: - Anchore-Build-Scan: - runs-on: ubuntu-latest - steps: - - name: Checkout the code - uses: actions/checkout@v2 - - name: Build the Docker image - run: | - cd docker/server - perl -pi -e 's|=\$version||g' Dockerfile - docker build . --file Dockerfile --tag localbuild/testimage:latest - - name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled - uses: anchore/scan-action@v2 - id: scan - with: - image: "localbuild/testimage:latest" - acs-report-enable: true - - name: Upload Anchore Scan Report - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: ${{ steps.scan.outputs.sarif }} diff --git a/.github/workflows/debug.yml b/.github/workflows/debug.yml new file mode 100644 index 00000000000..7bb5ac65140 --- /dev/null +++ b/.github/workflows/debug.yml @@ -0,0 +1,11 @@ +# The CI for each commit, prints envs and content of GITHUB_EVENT_PATH +name: Debug + +'on': + [push, pull_request, release] + +jobs: + DebugInfo: + runs-on: ubuntu-latest + steps: + - uses: hmarr/debug-action@1201a20fc9d278ddddd5f0f46922d06513892491 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c32896205d5..5816a58081d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1065,6 +1065,41 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseS3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_s3_storage + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, s3 storage, actions) + REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] @@ -2844,6 +2879,7 @@ jobs: - FunctionalStatefulTestDebug - FunctionalStatefulTestRelease - FunctionalStatefulTestReleaseDatabaseOrdinary + - FunctionalStatelessTestReleaseS3 - FunctionalStatefulTestAarch64 - FunctionalStatefulTestAsan - FunctionalStatefulTestTsan diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2602b9c28d5..5b47f94a324 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,8 @@ env: "on": schedule: - - cron: '0 0 * * *' + - cron: '13 3 * * *' + workflow_dispatch: jobs: DockerHubPushAarch64: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 81a0cb68bd9..d50a2151f2f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -168,35 +168,6 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - PVSCheck: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/pvs_check - REPO_COPY=${{runner.temp}}/pvs_check/ClickHouse - EOF - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - with: - submodules: 'true' - - name: PVS Check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 pvs_check.py - - name: Cleanup - if: always() - run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: - sudo rm -fr "$TEMP_PATH" CompatibilityCheck: needs: [BuilderDebRelease] runs-on: [self-hosted, style-checker] @@ -1215,6 +1186,41 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseS3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_s3_storage + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, s3 storage, actions) + REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] @@ -1727,6 +1733,51 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + TestsBugfixCheck: + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/tests_bugfix_check + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Tests bugfix validate check (actions) + KILL_TIMEOUT=3600 + REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Bugfix test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + + TEMP_PATH="${TEMP_PATH}/integration" \ + REPORTS_PATH="${REPORTS_PATH}/integration" \ + python3 integration_test_check.py "Integration tests bugfix validate check" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + TEMP_PATH="${TEMP_PATH}/stateless" \ + REPORTS_PATH="${REPORTS_PATH}/stateless" \ + python3 functional_test_check.py "Stateless tests bugfix validate check" "$KILL_TIMEOUT" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### ############################################################################################## @@ -3037,6 +3088,7 @@ jobs: - FunctionalStatefulTestTsan - FunctionalStatefulTestMsan - FunctionalStatefulTestUBsan + - FunctionalStatelessTestReleaseS3 - StressTestDebug - StressTestAsan - StressTestTsan @@ -3060,7 +3112,6 @@ jobs: - PerformanceComparison1 - PerformanceComparison2 - PerformanceComparison3 - - PVSCheck - UnitTestsAsan - UnitTestsTsan - UnitTestsMsan diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 46e36c846d0..bd62e64409f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,8 +23,8 @@ jobs: uses: actions/checkout@v2 - name: Download packages and push to Artifactory run: | - rm -rf "$TEMP_PATH" && mkdir -p "$REPO_COPY" - cp -r "$GITHUB_WORKSPACE" "$REPO_COPY" + rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY" python3 ./tests/ci/push_to_artifactory.py --release "${{ github.ref }}" \ --commit '${{ github.sha }}' --all @@ -32,7 +32,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{runner.temp}}/release_packages/* + file: ${{runner.temp}}/push_to_artifactory/* overwrite: true tag: ${{ github.ref }} file_glob: true diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml index 30b6bfb027e..7a7eddf444d 100644 --- a/.github/workflows/tags_stable.yml +++ b/.github/workflows/tags_stable.yml @@ -27,6 +27,8 @@ jobs: - name: Create Pull Request uses: peter-evans/create-pull-request@v3 with: + author: "robot-clickhouse " + committer: "robot-clickhouse " commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }} branch: auto/${{ env.GITHUB_TAG }} delete-branch: true diff --git a/.gitmodules b/.gitmodules index 91f4ddb2007..6c9e66f9cbc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "contrib/poco"] path = contrib/poco - url = https://github.com/ClickHouse-Extras/poco.git + url = https://github.com/ClickHouse/poco.git branch = clickhouse [submodule "contrib/zstd"] path = contrib/zstd @@ -10,13 +10,13 @@ url = https://github.com/lz4/lz4.git [submodule "contrib/librdkafka"] path = contrib/librdkafka - url = https://github.com/ClickHouse-Extras/librdkafka.git + url = https://github.com/ClickHouse/librdkafka.git [submodule "contrib/cctz"] path = contrib/cctz - url = https://github.com/ClickHouse-Extras/cctz.git + url = https://github.com/ClickHouse/cctz.git [submodule "contrib/zlib-ng"] path = contrib/zlib-ng - url = https://github.com/ClickHouse-Extras/zlib-ng.git + url = https://github.com/ClickHouse/zlib-ng.git branch = clickhouse-2.0.x [submodule "contrib/googletest"] path = contrib/googletest @@ -32,51 +32,51 @@ url = https://github.com/google/re2.git [submodule "contrib/llvm"] path = contrib/llvm - url = https://github.com/ClickHouse-Extras/llvm + url = https://github.com/ClickHouse/llvm [submodule "contrib/mariadb-connector-c"] path = contrib/mariadb-connector-c - url = https://github.com/ClickHouse-Extras/mariadb-connector-c.git + url = https://github.com/ClickHouse/mariadb-connector-c.git [submodule "contrib/jemalloc"] path = contrib/jemalloc - url = https://github.com/ClickHouse-Extras/jemalloc.git + url = https://github.com/jemalloc/jemalloc.git [submodule "contrib/unixodbc"] path = contrib/unixodbc - url = https://github.com/ClickHouse-Extras/UnixODBC.git + url = https://github.com/ClickHouse/UnixODBC.git [submodule "contrib/protobuf"] path = contrib/protobuf - url = https://github.com/ClickHouse-Extras/protobuf.git + url = https://github.com/ClickHouse/protobuf.git branch = v3.13.0.1 [submodule "contrib/boost"] path = contrib/boost - url = https://github.com/ClickHouse-Extras/boost.git + url = https://github.com/ClickHouse/boost.git [submodule "contrib/base64"] path = contrib/base64 - url = https://github.com/ClickHouse-Extras/Turbo-Base64.git + url = https://github.com/ClickHouse/Turbo-Base64.git [submodule "contrib/arrow"] path = contrib/arrow - url = https://github.com/ClickHouse-Extras/arrow.git + url = https://github.com/ClickHouse/arrow.git branch = blessed/release-6.0.1 [submodule "contrib/thrift"] path = contrib/thrift url = https://github.com/apache/thrift.git [submodule "contrib/libhdfs3"] path = contrib/libhdfs3 - url = https://github.com/ClickHouse-Extras/libhdfs3.git + url = https://github.com/ClickHouse/libhdfs3.git [submodule "contrib/libxml2"] path = contrib/libxml2 url = https://github.com/GNOME/libxml2.git [submodule "contrib/libgsasl"] path = contrib/libgsasl - url = https://github.com/ClickHouse-Extras/libgsasl.git + url = https://github.com/ClickHouse/libgsasl.git [submodule "contrib/libcxx"] path = contrib/libcxx - url = https://github.com/ClickHouse-Extras/libcxx.git + url = https://github.com/ClickHouse/libcxx.git [submodule "contrib/libcxxabi"] path = contrib/libcxxabi - url = https://github.com/ClickHouse-Extras/libcxxabi.git + url = https://github.com/ClickHouse/libcxxabi.git [submodule "contrib/snappy"] path = contrib/snappy - url = https://github.com/ClickHouse-Extras/snappy.git + url = https://github.com/ClickHouse/snappy.git [submodule "contrib/cppkafka"] path = contrib/cppkafka url = https://github.com/mfontanini/cppkafka.git @@ -85,95 +85,95 @@ url = https://github.com/google/brotli.git [submodule "contrib/h3"] path = contrib/h3 - url = https://github.com/ClickHouse-Extras/h3 + url = https://github.com/ClickHouse/h3 [submodule "contrib/hyperscan"] path = contrib/hyperscan - url = https://github.com/ClickHouse-Extras/hyperscan.git + url = https://github.com/ClickHouse/hyperscan.git [submodule "contrib/libunwind"] path = contrib/libunwind - url = https://github.com/ClickHouse-Extras/libunwind.git + url = https://github.com/ClickHouse/libunwind.git [submodule "contrib/simdjson"] path = contrib/simdjson url = https://github.com/simdjson/simdjson.git [submodule "contrib/rapidjson"] path = contrib/rapidjson - url = https://github.com/ClickHouse-Extras/rapidjson + url = https://github.com/ClickHouse/rapidjson [submodule "contrib/fastops"] path = contrib/fastops - url = https://github.com/ClickHouse-Extras/fastops + url = https://github.com/ClickHouse/fastops [submodule "contrib/orc"] path = contrib/orc - url = https://github.com/ClickHouse-Extras/orc + url = https://github.com/ClickHouse/orc [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11.git [submodule "contrib/grpc"] path = contrib/grpc - url = https://github.com/ClickHouse-Extras/grpc.git + url = https://github.com/ClickHouse/grpc.git branch = v1.33.2 [submodule "contrib/aws"] path = contrib/aws - url = https://github.com/ClickHouse-Extras/aws-sdk-cpp.git + url = https://github.com/ClickHouse/aws-sdk-cpp.git [submodule "aws-c-event-stream"] path = contrib/aws-c-event-stream - url = https://github.com/ClickHouse-Extras/aws-c-event-stream.git + url = https://github.com/ClickHouse/aws-c-event-stream.git [submodule "aws-c-common"] path = contrib/aws-c-common - url = https://github.com/ClickHouse-Extras/aws-c-common.git + url = https://github.com/ClickHouse/aws-c-common.git [submodule "aws-checksums"] path = contrib/aws-checksums - url = https://github.com/ClickHouse-Extras/aws-checksums.git + url = https://github.com/ClickHouse/aws-checksums.git [submodule "contrib/curl"] path = contrib/curl url = https://github.com/curl/curl.git [submodule "contrib/icudata"] path = contrib/icudata - url = https://github.com/ClickHouse-Extras/icudata.git + url = https://github.com/ClickHouse/icudata.git [submodule "contrib/icu"] path = contrib/icu url = https://github.com/unicode-org/icu.git [submodule "contrib/flatbuffers"] path = contrib/flatbuffers - url = https://github.com/ClickHouse-Extras/flatbuffers.git + url = https://github.com/ClickHouse/flatbuffers.git [submodule "contrib/replxx"] path = contrib/replxx - url = https://github.com/ClickHouse-Extras/replxx.git + url = https://github.com/ClickHouse/replxx.git [submodule "contrib/avro"] path = contrib/avro - url = https://github.com/ClickHouse-Extras/avro.git + url = https://github.com/ClickHouse/avro.git ignore = untracked [submodule "contrib/msgpack-c"] path = contrib/msgpack-c url = https://github.com/msgpack/msgpack-c [submodule "contrib/libcpuid"] path = contrib/libcpuid - url = https://github.com/ClickHouse-Extras/libcpuid.git + url = https://github.com/ClickHouse/libcpuid.git [submodule "contrib/openldap"] path = contrib/openldap - url = https://github.com/ClickHouse-Extras/openldap.git + url = https://github.com/ClickHouse/openldap.git [submodule "contrib/AMQP-CPP"] path = contrib/AMQP-CPP - url = https://github.com/ClickHouse-Extras/AMQP-CPP.git + url = https://github.com/ClickHouse/AMQP-CPP.git [submodule "contrib/cassandra"] path = contrib/cassandra - url = https://github.com/ClickHouse-Extras/cpp-driver.git + url = https://github.com/ClickHouse/cpp-driver.git branch = clickhouse [submodule "contrib/libuv"] path = contrib/libuv - url = https://github.com/ClickHouse-Extras/libuv.git + url = https://github.com/ClickHouse/libuv.git branch = clickhouse [submodule "contrib/fmtlib"] path = contrib/fmtlib url = https://github.com/fmtlib/fmt.git [submodule "contrib/sentry-native"] path = contrib/sentry-native - url = https://github.com/ClickHouse-Extras/sentry-native.git + url = https://github.com/ClickHouse/sentry-native.git [submodule "contrib/krb5"] path = contrib/krb5 - url = https://github.com/ClickHouse-Extras/krb5 + url = https://github.com/ClickHouse/krb5 [submodule "contrib/cyrus-sasl"] path = contrib/cyrus-sasl - url = https://github.com/ClickHouse-Extras/cyrus-sasl + url = https://github.com/ClickHouse/cyrus-sasl branch = cyrus-sasl-2.1 [submodule "contrib/croaring"] path = contrib/croaring @@ -184,7 +184,7 @@ url = https://github.com/danlark1/miniselect [submodule "contrib/rocksdb"] path = contrib/rocksdb - url = https://github.com/ClickHouse-Extras/rocksdb.git + url = https://github.com/ClickHouse/rocksdb.git [submodule "contrib/xz"] path = contrib/xz url = https://github.com/xz-mirror/xz @@ -194,53 +194,53 @@ branch = lts_2021_11_02 [submodule "contrib/dragonbox"] path = contrib/dragonbox - url = https://github.com/ClickHouse-Extras/dragonbox.git + url = https://github.com/ClickHouse/dragonbox.git [submodule "contrib/fast_float"] path = contrib/fast_float url = https://github.com/fastfloat/fast_float [submodule "contrib/libpq"] path = contrib/libpq - url = https://github.com/ClickHouse-Extras/libpq + url = https://github.com/ClickHouse/libpq [submodule "contrib/boringssl"] path = contrib/boringssl - url = https://github.com/ClickHouse-Extras/boringssl.git + url = https://github.com/ClickHouse/boringssl.git branch = MergeWithUpstream [submodule "contrib/NuRaft"] path = contrib/NuRaft - url = https://github.com/ClickHouse-Extras/NuRaft.git + url = https://github.com/ClickHouse/NuRaft.git [submodule "contrib/nanodbc"] path = contrib/nanodbc - url = https://github.com/ClickHouse-Extras/nanodbc.git + url = https://github.com/ClickHouse/nanodbc.git [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp - url = https://github.com/ClickHouse-Extras/datasketches-cpp.git + url = https://github.com/ClickHouse/datasketches-cpp.git [submodule "contrib/yaml-cpp"] path = contrib/yaml-cpp - url = https://github.com/ClickHouse-Extras/yaml-cpp.git + url = https://github.com/ClickHouse/yaml-cpp.git [submodule "contrib/cld2"] path = contrib/cld2 - url = https://github.com/ClickHouse-Extras/cld2.git + url = https://github.com/ClickHouse/cld2.git [submodule "contrib/libstemmer_c"] path = contrib/libstemmer_c - url = https://github.com/ClickHouse-Extras/libstemmer_c.git + url = https://github.com/ClickHouse/libstemmer_c.git [submodule "contrib/wordnet-blast"] path = contrib/wordnet-blast - url = https://github.com/ClickHouse-Extras/wordnet-blast.git + url = https://github.com/ClickHouse/wordnet-blast.git [submodule "contrib/lemmagen-c"] path = contrib/lemmagen-c - url = https://github.com/ClickHouse-Extras/lemmagen-c.git + url = https://github.com/ClickHouse/lemmagen-c.git [submodule "contrib/libpqxx"] path = contrib/libpqxx - url = https://github.com/ClickHouse-Extras/libpqxx.git + url = https://github.com/ClickHouse/libpqxx.git [submodule "contrib/sqlite-amalgamation"] path = contrib/sqlite-amalgamation url = https://github.com/azadkuh/sqlite-amalgamation [submodule "contrib/s2geometry"] path = contrib/s2geometry - url = https://github.com/ClickHouse-Extras/s2geometry.git + url = https://github.com/ClickHouse/s2geometry.git [submodule "contrib/bzip2"] path = contrib/bzip2 - url = https://github.com/ClickHouse-Extras/bzip2.git + url = https://github.com/ClickHouse/bzip2.git [submodule "contrib/magic_enum"] path = contrib/magic_enum url = https://github.com/Neargye/magic_enum @@ -249,16 +249,16 @@ url = https://github.com/google/libprotobuf-mutator [submodule "contrib/sysroot"] path = contrib/sysroot - url = https://github.com/ClickHouse-Extras/sysroot.git + url = https://github.com/ClickHouse/sysroot.git [submodule "contrib/nlp-data"] path = contrib/nlp-data - url = https://github.com/ClickHouse-Extras/nlp-data.git + url = https://github.com/ClickHouse/nlp-data.git [submodule "contrib/hive-metastore"] path = contrib/hive-metastore - url = https://github.com/ClickHouse-Extras/hive-metastore + url = https://github.com/ClickHouse/hive-metastore [submodule "contrib/azure"] path = contrib/azure - url = https://github.com/ClickHouse-Extras/azure-sdk-for-cpp.git + url = https://github.com/ClickHouse/azure-sdk-for-cpp.git [submodule "contrib/minizip-ng"] path = contrib/minizip-ng url = https://github.com/zlib-ng/minizip-ng diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e4ea95c08c..61724ab2d0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,314 @@ -### ClickHouse release v22.1, 2022-01-18 +### Table of Contents +**[ClickHouse release v22.3-lts, 2022-03-17](#223)**
+**[ClickHouse release v22.2, 2022-02-17](#222)**
+**[ClickHouse release v22.1, 2022-01-18](#221)**
+**[Changelog for 2021](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/whats-new/changelog/2021.md)**
+ + +## ClickHouse release v22.3-lts, 2022-03-17 + +#### Backward Incompatible Change + +* Make `arrayCompact` function behave as other higher-order functions: perform compaction not of lambda function results but on the original array. If you're using nontrivial lambda functions in arrayCompact you may restore old behaviour by wrapping `arrayCompact` arguments into `arrayMap`. Closes [#34010](https://github.com/ClickHouse/ClickHouse/issues/34010) [#18535](https://github.com/ClickHouse/ClickHouse/issues/18535) [#14778](https://github.com/ClickHouse/ClickHouse/issues/14778). [#34795](https://github.com/ClickHouse/ClickHouse/pull/34795) ([Alexandre Snarskii](https://github.com/snar)). +* Change implementation specific behavior on overflow of function `toDatetime`. It will be saturated to the nearest min/max supported instant of datetime instead of wraparound. This change is highlighted as "backward incompatible" because someone may unintentionally rely on the old behavior. [#32898](https://github.com/ClickHouse/ClickHouse/pull/32898) ([HaiBo Li](https://github.com/marising)). + +#### New Feature + +* Support for caching data locally for remote filesystems. It can be enabled for `s3` disks. Closes [#28961](https://github.com/ClickHouse/ClickHouse/issues/28961). [#33717](https://github.com/ClickHouse/ClickHouse/pull/33717) ([Kseniia Sumarokova](https://github.com/kssenii)). In the meantime, we enabled the test suite on s3 filesystem and no more known issues exist, so it is started to be production ready. +* Add new table function `hive`. It can be used as follows `hive('', '', '', '', '')` for example `SELECT * FROM hive('thrift://hivetest:9083', 'test', 'demo', 'id Nullable(String), score Nullable(Int32), day Nullable(String)', 'day')`. [#34946](https://github.com/ClickHouse/ClickHouse/pull/34946) ([lgbo](https://github.com/lgbo-ustc)). +* Support authentication of users connected via SSL by their X.509 certificate. [#31484](https://github.com/ClickHouse/ClickHouse/pull/31484) ([eungenue](https://github.com/eungenue)). +* Support schema inference for inserting into table functions `file`/`hdfs`/`s3`/`url`. [#34732](https://github.com/ClickHouse/ClickHouse/pull/34732) ([Kruglov Pavel](https://github.com/Avogar)). +* Now you can read `system.zookeeper` table without restrictions on path or using `like` expression. This reads can generate quite heavy load for zookeeper so to enable this ability you have to enable setting `allow_unrestricted_reads_from_keeper`. [#34609](https://github.com/ClickHouse/ClickHouse/pull/34609) ([Sergei Trifonov](https://github.com/serxa)). +* Display CPU and memory metrics in clickhouse-local. Close [#34545](https://github.com/ClickHouse/ClickHouse/issues/34545). [#34605](https://github.com/ClickHouse/ClickHouse/pull/34605) ([李扬](https://github.com/taiyang-li)). +* Implement `startsWith` and `endsWith` function for arrays, closes [#33982](https://github.com/ClickHouse/ClickHouse/issues/33982). [#34368](https://github.com/ClickHouse/ClickHouse/pull/34368) ([usurai](https://github.com/usurai)). +* Add three functions for Map data type: 1. `mapReplace(map1, map2)` - replaces values for keys in map1 with the values of the corresponding keys in map2; adds keys from map2 that don't exist in map1. 2. `mapFilter` 3. `mapMap`. mapFilter and mapMap are higher order functions, accepting two arguments, the first argument is a lambda function with k, v pair as arguments, the second argument is a column of type Map. [#33698](https://github.com/ClickHouse/ClickHouse/pull/33698) ([hexiaoting](https://github.com/hexiaoting)). +* Allow getting default user and password for clickhouse-client from the `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` environment variables. Close [#34538](https://github.com/ClickHouse/ClickHouse/issues/34538). [#34947](https://github.com/ClickHouse/ClickHouse/pull/34947) ([DR](https://github.com/freedomDR)). + +#### Experimental Feature + +* New data type `Object()`, which supports storing of semi-structured data (for now JSON only). Data is written to such types as string. Then all paths are extracted according to format of semi-structured data and written as separate columns in most optimal types, that can store all their values. Those columns can be queried by names that match paths in source data. E.g `data.key1.key2` or with cast operator `data.key1.key2::Int64`. +* Add `database_replicated_allow_only_replicated_engine` setting. When enabled, it only allowed to only create `Replicated` tables or tables with stateless engines in `Replicated` databases. [#35214](https://github.com/ClickHouse/ClickHouse/pull/35214) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). Note that `Replicated` database is still an experimental feature. + +#### Performance Improvement + +* Improve performance of insertion into `MergeTree` tables by optimizing sorting. Up to 2x improvement is observed on realistic benchmarks. [#34750](https://github.com/ClickHouse/ClickHouse/pull/34750) ([Maksim Kita](https://github.com/kitaisreal)). +* Columns pruning when reading Parquet, ORC and Arrow files from URL and S3. Closes [#34163](https://github.com/ClickHouse/ClickHouse/issues/34163). [#34849](https://github.com/ClickHouse/ClickHouse/pull/34849) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Columns pruning when reading Parquet, ORC and Arrow files from Hive. [#34954](https://github.com/ClickHouse/ClickHouse/pull/34954) ([lgbo](https://github.com/lgbo-ustc)). +* A bunch of performance optimizations from a performance superhero. Improve performance of processing queries with large `IN` section. Improve performance of `direct` dictionary if its source is `ClickHouse`. Improve performance of `detectCharset `, `detectLanguageUnknown ` functions. [#34888](https://github.com/ClickHouse/ClickHouse/pull/34888) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve performance of `any` aggregate function by using more batching. [#34760](https://github.com/ClickHouse/ClickHouse/pull/34760) ([Raúl Marín](https://github.com/Algunenano)). +* Multiple improvements for performance of `clickhouse-keeper`: less locking [#35010](https://github.com/ClickHouse/ClickHouse/pull/35010) ([zhanglistar](https://github.com/zhanglistar)), lower memory usage by streaming reading and writing of snapshot instead of full copy. [#34584](https://github.com/ClickHouse/ClickHouse/pull/34584) ([zhanglistar](https://github.com/zhanglistar)), optimizing compaction of log store in the RAFT implementation. [#34534](https://github.com/ClickHouse/ClickHouse/pull/34534) ([zhanglistar](https://github.com/zhanglistar)), versioning of the internal data structure [#34486](https://github.com/ClickHouse/ClickHouse/pull/34486) ([zhanglistar](https://github.com/zhanglistar)). + +#### Improvement + +* Allow asynchronous inserts to table functions. Fixes [#34864](https://github.com/ClickHouse/ClickHouse/issues/34864). [#34866](https://github.com/ClickHouse/ClickHouse/pull/34866) ([Anton Popov](https://github.com/CurtizJ)). +* Implicit type casting of the key argument for functions `dictGetHierarchy`, `dictIsIn`, `dictGetChildren`, `dictGetDescendants`. Closes [#34970](https://github.com/ClickHouse/ClickHouse/issues/34970). [#35027](https://github.com/ClickHouse/ClickHouse/pull/35027) ([Maksim Kita](https://github.com/kitaisreal)). +* `EXPLAIN AST` query can output AST in form of a graph in Graphviz format: `EXPLAIN AST graph = 1 SELECT * FROM system.parts`. [#35173](https://github.com/ClickHouse/ClickHouse/pull/35173) ([李扬](https://github.com/taiyang-li)). +* When large files were written with `s3` table function or table engine, the content type on the files was mistakenly set to `application/xml` due to a bug in the AWS SDK. This closes [#33964](https://github.com/ClickHouse/ClickHouse/issues/33964). [#34433](https://github.com/ClickHouse/ClickHouse/pull/34433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change restrictive row policies a bit to make them an easier alternative to permissive policies in easy cases. If for a particular table only restrictive policies exist (without permissive policies) users will be able to see some rows. Also `SHOW CREATE ROW POLICY` will always show `AS permissive` or `AS restrictive` in row policy's definition. [#34596](https://github.com/ClickHouse/ClickHouse/pull/34596) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve schema inference with globs in File/S3/HDFS/URL engines. Try to use the next path for schema inference in case of error. [#34465](https://github.com/ClickHouse/ClickHouse/pull/34465) ([Kruglov Pavel](https://github.com/Avogar)). +* Play UI now correctly detects the preferred light/dark theme from the OS. [#35068](https://github.com/ClickHouse/ClickHouse/pull/35068) ([peledni](https://github.com/peledni)). +* Added `date_time_input_format = 'best_effort_us'`. Closes [#34799](https://github.com/ClickHouse/ClickHouse/issues/34799). [#34982](https://github.com/ClickHouse/ClickHouse/pull/34982) ([WenYao](https://github.com/Cai-Yao)). +* A new settings called `allow_plaintext_password` and `allow_no_password` are added in server configuration which turn on/off authentication types that can be potentially insecure in some environments. They are allowed by default. [#34738](https://github.com/ClickHouse/ClickHouse/pull/34738) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Support for `DateTime64` data type in `Arrow` format, closes [#8280](https://github.com/ClickHouse/ClickHouse/issues/8280) and closes [#28574](https://github.com/ClickHouse/ClickHouse/issues/28574). [#34561](https://github.com/ClickHouse/ClickHouse/pull/34561) ([李扬](https://github.com/taiyang-li)). +* Reload `remote_url_allow_hosts` (filtering of outgoing connections) on config update. [#35294](https://github.com/ClickHouse/ClickHouse/pull/35294) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support `--testmode` parameter for `clickhouse-local`. This parameter enables interpretation of test hints that we use in functional tests. [#35264](https://github.com/ClickHouse/ClickHouse/pull/35264) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `distributed_depth` to query log. It is like a more detailed variant of `is_initial_query` [#35207](https://github.com/ClickHouse/ClickHouse/pull/35207) ([李扬](https://github.com/taiyang-li)). +* Respect `remote_url_allow_hosts` for `MySQL` and `PostgreSQL` table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Added `disk_name` field to `system.part_log`. [#35178](https://github.com/ClickHouse/ClickHouse/pull/35178) ([Artyom Yurkov](https://github.com/Varinara)). +* Do not retry non-rertiable errors when querying remote URLs. Closes [#35161](https://github.com/ClickHouse/ClickHouse/issues/35161). [#35172](https://github.com/ClickHouse/ClickHouse/pull/35172) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support distributed INSERT SELECT queries (the setting `parallel_distributed_insert_select`) table function `view()`. [#35132](https://github.com/ClickHouse/ClickHouse/pull/35132) ([Azat Khuzhin](https://github.com/azat)). +* More precise memory tracking during `INSERT` into `Buffer` with `AggregateFunction`. [#35072](https://github.com/ClickHouse/ClickHouse/pull/35072) ([Azat Khuzhin](https://github.com/azat)). +* Avoid division by zero in Query Profiler if Linux kernel has a bug. Closes [#34787](https://github.com/ClickHouse/ClickHouse/issues/34787). [#35032](https://github.com/ClickHouse/ClickHouse/pull/35032) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more sanity checks for keeper configuration: now mixing of localhost and non-local servers is not allowed, also add checks for same value of internal raft port and keeper client port. [#35004](https://github.com/ClickHouse/ClickHouse/pull/35004) ([alesapin](https://github.com/alesapin)). +* Currently, if the user changes the settings of the system tables there will be tons of logs and ClickHouse will rename the tables every minute. This fixes [#34929](https://github.com/ClickHouse/ClickHouse/issues/34929). [#34949](https://github.com/ClickHouse/ClickHouse/pull/34949) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use connection pool for Hive metastore client. [#34940](https://github.com/ClickHouse/ClickHouse/pull/34940) ([lgbo](https://github.com/lgbo-ustc)). +* Ignore per-column `TTL` in `CREATE TABLE AS` if new table engine does not support it (i.e. if the engine is not of `MergeTree` family). [#34938](https://github.com/ClickHouse/ClickHouse/pull/34938) ([Azat Khuzhin](https://github.com/azat)). +* Allow `LowCardinality` strings for `ngrambf_v1`/`tokenbf_v1` indexes. Closes [#21865](https://github.com/ClickHouse/ClickHouse/issues/21865). [#34911](https://github.com/ClickHouse/ClickHouse/pull/34911) ([Lars Hiller Eidnes](https://github.com/larspars)). +* Allow opening empty sqlite db if the file doesn't exist. Closes [#33367](https://github.com/ClickHouse/ClickHouse/issues/33367). [#34907](https://github.com/ClickHouse/ClickHouse/pull/34907) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Implement memory statistics for FreeBSD - this is required for `max_server_memory_usage` to work correctly. [#34902](https://github.com/ClickHouse/ClickHouse/pull/34902) ([Alexandre Snarskii](https://github.com/snar)). +* In previous versions the progress bar in clickhouse-client can jump forward near 50% for no reason. This closes [#34324](https://github.com/ClickHouse/ClickHouse/issues/34324). [#34801](https://github.com/ClickHouse/ClickHouse/pull/34801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now `ALTER TABLE DROP COLUMN columnX` queries for `MergeTree` table engines will work instantly when `columnX` is an `ALIAS` column. Fixes [#34660](https://github.com/ClickHouse/ClickHouse/issues/34660). [#34786](https://github.com/ClickHouse/ClickHouse/pull/34786) ([alesapin](https://github.com/alesapin)). +* Show hints when user mistyped the name of a data skipping index. Closes [#29698](https://github.com/ClickHouse/ClickHouse/issues/29698). [#34764](https://github.com/ClickHouse/ClickHouse/pull/34764) ([flynn](https://github.com/ucasfl)). +* Support `remote()`/`cluster()` table functions for `parallel_distributed_insert_select`. [#34728](https://github.com/ClickHouse/ClickHouse/pull/34728) ([Azat Khuzhin](https://github.com/azat)). +* Do not reset logging that configured via `--log-file`/`--errorlog-file` command line options in case of empty configuration in the config file. [#34718](https://github.com/ClickHouse/ClickHouse/pull/34718) ([Amos Bird](https://github.com/amosbird)). +* Extract schema only once on table creation and prevent reading from local files/external sources to extract schema on each server startup. [#34684](https://github.com/ClickHouse/ClickHouse/pull/34684) ([Kruglov Pavel](https://github.com/Avogar)). +* Allow specifying argument names for executable UDFs. This is necessary for formats where argument name is part of serialization, like `Native`, `JSONEachRow`. Closes [#34604](https://github.com/ClickHouse/ClickHouse/issues/34604). [#34653](https://github.com/ClickHouse/ClickHouse/pull/34653) ([Maksim Kita](https://github.com/kitaisreal)). +* `MaterializedMySQL` (experimental feature) now supports `materialized_mysql_tables_list` (a comma-separated list of MySQL database tables, which will be replicated by the MaterializedMySQL database engine. Default value: empty list — means all the tables will be replicated), mentioned at [#32977](https://github.com/ClickHouse/ClickHouse/issues/32977). [#34487](https://github.com/ClickHouse/ClickHouse/pull/34487) ([zzsmdfj](https://github.com/zzsmdfj)). +* Improve OpenTelemetry span logs for INSERT operation on distributed table. [#34480](https://github.com/ClickHouse/ClickHouse/pull/34480) ([Frank Chen](https://github.com/FrankChen021)). +* Make the znode `ctime` and `mtime` consistent between servers in ClickHouse Keeper. [#33441](https://github.com/ClickHouse/ClickHouse/pull/33441) ([小路](https://github.com/nicelulu)). + +#### Build/Testing/Packaging Improvement + +* Package repository is migrated to JFrog Artifactory (**Mikhail f. Shiryaev**). +* Randomize some settings in functional tests, so more possible combinations of settings will be tested. This is yet another fuzzing method to ensure better test coverage. This closes [#32268](https://github.com/ClickHouse/ClickHouse/issues/32268). [#34092](https://github.com/ClickHouse/ClickHouse/pull/34092) ([Kruglov Pavel](https://github.com/Avogar)). +* Drop PVS-Studio from our CI. [#34680](https://github.com/ClickHouse/ClickHouse/pull/34680) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add an ability to build stripped binaries with CMake. In previous versions it was performed by dh-tools. [#35196](https://github.com/ClickHouse/ClickHouse/pull/35196) ([alesapin](https://github.com/alesapin)). +* Smaller "fat-free" `clickhouse-keeper` build. [#35031](https://github.com/ClickHouse/ClickHouse/pull/35031) ([alesapin](https://github.com/alesapin)). +* Use @robot-clickhouse as an author and committer for PRs like https://github.com/ClickHouse/ClickHouse/pull/34685. [#34793](https://github.com/ClickHouse/ClickHouse/pull/34793) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Limit DWARF version for debug info by 4 max, because our internal stack symbolizer cannot parse DWARF version 5. This makes sense if you compile ClickHouse with clang-15. [#34777](https://github.com/ClickHouse/ClickHouse/pull/34777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `clickhouse-test` debian package as unneeded complication. CI use tests from repository and standalone testing via deb package is no longer supported. [#34606](https://github.com/ClickHouse/ClickHouse/pull/34606) ([Ilya Yatsishin](https://github.com/qoega)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* A fix for HDFS integration: When the inner buffer size is too small, NEED_MORE_INPUT in `HadoopSnappyDecoder` will run multi times (>=3) for one compressed block. This makes the input data be copied into the wrong place in `HadoopSnappyDecoder::buffer`. [#35116](https://github.com/ClickHouse/ClickHouse/pull/35116) ([lgbo](https://github.com/lgbo-ustc)). +* Ignore obsolete grants in ATTACH GRANT statements. This PR fixes [#34815](https://github.com/ClickHouse/ClickHouse/issues/34815). [#34855](https://github.com/ClickHouse/ClickHouse/pull/34855) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix segfault in Postgres database when getting create table query if database was created using named collections. Closes [#35312](https://github.com/ClickHouse/ClickHouse/issues/35312). [#35313](https://github.com/ClickHouse/ClickHouse/pull/35313) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix partial merge join duplicate rows bug, close [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009). [#35311](https://github.com/ClickHouse/ClickHouse/pull/35311) ([Vladimir C](https://github.com/vdimir)). +* Fix possible `Assertion 'position() != working_buffer.end()' failed` while using bzip2 compression with small `max_read_buffer_size` setting value. The bug was found in https://github.com/ClickHouse/ClickHouse/pull/35047. [#35300](https://github.com/ClickHouse/ClickHouse/pull/35300) ([Kruglov Pavel](https://github.com/Avogar)). While using lz4 compression with a small max_read_buffer_size setting value. [#35296](https://github.com/ClickHouse/ClickHouse/pull/35296) ([Kruglov Pavel](https://github.com/Avogar)). While using lzma compression with small `max_read_buffer_size` setting value. [#35295](https://github.com/ClickHouse/ClickHouse/pull/35295) ([Kruglov Pavel](https://github.com/Avogar)). While using `brotli` compression with a small `max_read_buffer_size` setting value. The bug was found in https://github.com/ClickHouse/ClickHouse/pull/35047. [#35281](https://github.com/ClickHouse/ClickHouse/pull/35281) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible segfault in `JSONEachRow` schema inference. [#35291](https://github.com/ClickHouse/ClickHouse/pull/35291) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix `CHECK TABLE` query in case when sparse columns are enabled in table. [#35274](https://github.com/ClickHouse/ClickHouse/pull/35274) ([Anton Popov](https://github.com/CurtizJ)). +* Avoid std::terminate in case of exception in reading from remote VFS. [#35257](https://github.com/ClickHouse/ClickHouse/pull/35257) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading port from config, close [#34776](https://github.com/ClickHouse/ClickHouse/issues/34776). [#35193](https://github.com/ClickHouse/ClickHouse/pull/35193) ([Vladimir C](https://github.com/vdimir)). +* Fix error in query with `WITH TOTALS` in case if `HAVING` returned empty result. This fixes [#33711](https://github.com/ClickHouse/ClickHouse/issues/33711). [#35186](https://github.com/ClickHouse/ClickHouse/pull/35186) ([Amos Bird](https://github.com/amosbird)). +* Fix a corner case of `replaceRegexpAll`, close [#35117](https://github.com/ClickHouse/ClickHouse/issues/35117). [#35182](https://github.com/ClickHouse/ClickHouse/pull/35182) ([Vladimir C](https://github.com/vdimir)). +* Schema inference didn't work properly on case of `INSERT INTO FUNCTION s3(...) FROM ...`, it tried to read schema from s3 file instead of from select query. [#35176](https://github.com/ClickHouse/ClickHouse/pull/35176) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix MaterializedPostgreSQL (experimental feature) `table overrides` for partition by, etc. Closes [#35048](https://github.com/ClickHouse/ClickHouse/issues/35048). [#35162](https://github.com/ClickHouse/ClickHouse/pull/35162) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix MaterializedPostgreSQL (experimental feature) adding new table to replication (ATTACH TABLE) after manually removing (DETACH TABLE). Closes [#33800](https://github.com/ClickHouse/ClickHouse/issues/33800). Closes [#34922](https://github.com/ClickHouse/ClickHouse/issues/34922). Closes [#34315](https://github.com/ClickHouse/ClickHouse/issues/34315). [#35158](https://github.com/ClickHouse/ClickHouse/pull/35158) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix partition pruning error when non-monotonic function is used with IN operator. This fixes [#35136](https://github.com/ClickHouse/ClickHouse/issues/35136). [#35146](https://github.com/ClickHouse/ClickHouse/pull/35146) ([Amos Bird](https://github.com/amosbird)). +* Fixed slightly incorrect translation of YAML configs to XML. [#35135](https://github.com/ClickHouse/ClickHouse/pull/35135) ([Miel Donkers](https://github.com/mdonkers)). +* Fix `optimize_skip_unused_shards_rewrite_in` for signed columns and negative values. [#35134](https://github.com/ClickHouse/ClickHouse/pull/35134) ([Azat Khuzhin](https://github.com/azat)). +* The `update_lag` external dictionary configuration option was unusable showing the error message ``Unexpected key `update_lag` in dictionary source configuration``. [#35089](https://github.com/ClickHouse/ClickHouse/pull/35089) ([Jason Chu](https://github.com/1lann)). +* Avoid possible deadlock on server shutdown. [#35081](https://github.com/ClickHouse/ClickHouse/pull/35081) ([Azat Khuzhin](https://github.com/azat)). +* Fix missing alias after function is optimized to a subcolumn when setting `optimize_functions_to_subcolumns` is enabled. Closes [#33798](https://github.com/ClickHouse/ClickHouse/issues/33798). [#35079](https://github.com/ClickHouse/ClickHouse/pull/35079) ([qieqieplus](https://github.com/qieqieplus)). +* Fix reading from `system.asynchronous_inserts` table if there exists asynchronous insert into table function. [#35050](https://github.com/ClickHouse/ClickHouse/pull/35050) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible exception `Reading for MergeTree family tables must be done with last position boundary` (relevant to operation on remote VFS). Closes [#34979](https://github.com/ClickHouse/ClickHouse/issues/34979). [#35001](https://github.com/ClickHouse/ClickHouse/pull/35001) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix unexpected result when use -State type aggregate function in window frame. [#34999](https://github.com/ClickHouse/ClickHouse/pull/34999) ([metahys](https://github.com/metahys)). +* Fix possible segfault in FileLog (experimental feature). Closes [#30749](https://github.com/ClickHouse/ClickHouse/issues/30749). [#34996](https://github.com/ClickHouse/ClickHouse/pull/34996) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible rare error `Cannot push block to port which already has data`. [#34993](https://github.com/ClickHouse/ClickHouse/pull/34993) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix wrong schema inference for unquoted dates in CSV. Closes [#34768](https://github.com/ClickHouse/ClickHouse/issues/34768). [#34961](https://github.com/ClickHouse/ClickHouse/pull/34961) ([Kruglov Pavel](https://github.com/Avogar)). +* Integration with Hive: Fix unexpected result when use `in` in `where` in hive query. [#34945](https://github.com/ClickHouse/ClickHouse/pull/34945) ([lgbo](https://github.com/lgbo-ustc)). +* Avoid busy polling in ClickHouse Keeper while searching for changelog files to delete. [#34931](https://github.com/ClickHouse/ClickHouse/pull/34931) ([Azat Khuzhin](https://github.com/azat)). +* Fix DateTime64 conversion from PostgreSQL. Closes [#33364](https://github.com/ClickHouse/ClickHouse/issues/33364). [#34910](https://github.com/ClickHouse/ClickHouse/pull/34910) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible "Part directory doesn't exist" during `INSERT` into MergeTree table backed by VFS over s3. [#34876](https://github.com/ClickHouse/ClickHouse/pull/34876) ([Azat Khuzhin](https://github.com/azat)). +* Support DDLs like CREATE USER to be executed on cross replicated cluster. [#34860](https://github.com/ClickHouse/ClickHouse/pull/34860) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Fix bugs for multiple columns group by in `WindowView` (experimental feature). [#34859](https://github.com/ClickHouse/ClickHouse/pull/34859) ([vxider](https://github.com/Vxider)). +* Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix bug for H3 funcs containing const columns which cause queries to fail. [#34743](https://github.com/ClickHouse/ClickHouse/pull/34743) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix `No such file or directory` with enabled `fsync_part_directory` and vertical merge. [#34739](https://github.com/ClickHouse/ClickHouse/pull/34739) ([Azat Khuzhin](https://github.com/azat)). +* Fix serialization/printing for system queries `RELOAD MODEL`, `RELOAD FUNCTION`, `RESTART DISK` when used `ON CLUSTER`. Closes [#34514](https://github.com/ClickHouse/ClickHouse/issues/34514). [#34696](https://github.com/ClickHouse/ClickHouse/pull/34696) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix `allow_experimental_projection_optimization` with `enable_global_with_statement` (before it may lead to `Stack size too large` error in case of multiple expressions in `WITH` clause, and also it executes scalar subqueries again and again, so not it will be more optimal). [#34650](https://github.com/ClickHouse/ClickHouse/pull/34650) ([Azat Khuzhin](https://github.com/azat)). +* Stop to select part for mutate when the other replica has already updated the transaction log for `ReplatedMergeTree` engine. [#34633](https://github.com/ClickHouse/ClickHouse/pull/34633) ([Jianmei Zhang](https://github.com/zhangjmruc)). +* Fix incorrect result of trivial count query when part movement feature is used [#34089](https://github.com/ClickHouse/ClickHouse/issues/34089). [#34385](https://github.com/ClickHouse/ClickHouse/pull/34385) ([nvartolomei](https://github.com/nvartolomei)). +* Fix inconsistency of `max_query_size` limitation in distributed subqueries. [#34078](https://github.com/ClickHouse/ClickHouse/pull/34078) ([Chao Ma](https://github.com/godliness)). + + +### ClickHouse release v22.2, 2022-02-17 + +#### Upgrade Notes + +* Applying data skipping indexes for queries with FINAL may produce incorrect result. In this release we disabled data skipping indexes by default for queries with FINAL (a new setting `use_skip_indexes_if_final` is introduced and disabled by default). [#34243](https://github.com/ClickHouse/ClickHouse/pull/34243) ([Azat Khuzhin](https://github.com/azat)). + +#### New Feature + +* Projections are production ready. Set `allow_experimental_projection_optimization` by default and deprecate this setting. [#34456](https://github.com/ClickHouse/ClickHouse/pull/34456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* An option to create a new files on insert for `File`/`S3`/`HDFS` engines. Allow to overwrite a file in `HDFS`. Throw an exception in attempt to overwrite a file in `S3` by default. Throw an exception in attempt to append data to file in formats that have a suffix (and thus don't support appends, like `Parquet`, `ORC`). Closes [#31640](https://github.com/ClickHouse/ClickHouse/issues/31640) Closes [#31622](https://github.com/ClickHouse/ClickHouse/issues/31622) Closes [#23862](https://github.com/ClickHouse/ClickHouse/issues/23862) Closes [#15022](https://github.com/ClickHouse/ClickHouse/issues/15022) Closes [#16674](https://github.com/ClickHouse/ClickHouse/issues/16674). [#33302](https://github.com/ClickHouse/ClickHouse/pull/33302) ([Kruglov Pavel](https://github.com/Avogar)). +* Add a setting that allows a user to provide own deduplication semantic in `MergeTree`/`ReplicatedMergeTree` If provided, it's used instead of data digest to generate block ID. So, for example, by providing a unique value for the setting in each INSERT statement, the user can avoid the same inserted data being deduplicated. This closes: [#7461](https://github.com/ClickHouse/ClickHouse/issues/7461). [#32304](https://github.com/ClickHouse/ClickHouse/pull/32304) ([Igor Nikonov](https://github.com/devcrafter)). +* Add support of `DEFAULT` keyword for INSERT statements. Closes [#6331](https://github.com/ClickHouse/ClickHouse/issues/6331). [#33141](https://github.com/ClickHouse/ClickHouse/pull/33141) ([Andrii Buriachevskyi](https://github.com/1over)). +* `EPHEMERAL` column specifier is added to `CREATE TABLE` query. Closes [#9436](https://github.com/ClickHouse/ClickHouse/issues/9436). [#34424](https://github.com/ClickHouse/ClickHouse/pull/34424) ([yakov-olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Support `IF EXISTS` clause for `TTL expr TO [DISK|VOLUME] [IF EXISTS] 'xxx'` feature. Parts will be moved to disk or volume only if it exists on replica, so `MOVE TTL` rules will be able to behave differently on replicas according to the existing storage policies. Resolves [#34455](https://github.com/ClickHouse/ClickHouse/issues/34455). [#34504](https://github.com/ClickHouse/ClickHouse/pull/34504) ([Anton Popov](https://github.com/CurtizJ)). +* Allow set default table engine and to create tables without specifying ENGINE. [#34187](https://github.com/ClickHouse/ClickHouse/pull/34187) ([Ilya Yatsishin](https://github.com/qoega)). +* Add table function `format(format_name, data)`. [#34125](https://github.com/ClickHouse/ClickHouse/pull/34125) ([Kruglov Pavel](https://github.com/Avogar)). +* Detect format in `clickhouse-local` by file name even in the case when it is passed to stdin. [#33829](https://github.com/ClickHouse/ClickHouse/pull/33829) ([Kruglov Pavel](https://github.com/Avogar)). +* Add schema inference for `values` table function. Closes [#33811](https://github.com/ClickHouse/ClickHouse/issues/33811). [#34017](https://github.com/ClickHouse/ClickHouse/pull/34017) ([Kruglov Pavel](https://github.com/Avogar)). +* Dynamic reload of server TLS certificates on config reload. Closes [#15764](https://github.com/ClickHouse/ClickHouse/issues/15764). [#15765](https://github.com/ClickHouse/ClickHouse/pull/15765) ([johnskopis](https://github.com/johnskopis)). [#31257](https://github.com/ClickHouse/ClickHouse/pull/31257) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Now ReplicatedMergeTree can recover data when some of its disks are broken. [#13544](https://github.com/ClickHouse/ClickHouse/pull/13544) ([Amos Bird](https://github.com/amosbird)). +* Fault-tolerant connections in clickhouse-client: `clickhouse-client ... --host host1 --host host2 --port port2 --host host3 --port port --host host4`. [#34490](https://github.com/ClickHouse/ClickHouse/pull/34490) ([Kruglov Pavel](https://github.com/Avogar)). [#33824](https://github.com/ClickHouse/ClickHouse/pull/33824) ([Filippov Denis](https://github.com/DF5HSE)). +* Add `DEGREES` and `RADIANS` functions for MySQL compatibility. [#33769](https://github.com/ClickHouse/ClickHouse/pull/33769) ([Bharat Nallan](https://github.com/bharatnc)). +* Add `h3ToCenterChild` function. [#33313](https://github.com/ClickHouse/ClickHouse/pull/33313) ([Bharat Nallan](https://github.com/bharatnc)). Add new h3 miscellaneous functions: `edgeLengthKm`,`exactEdgeLengthKm`,`exactEdgeLengthM`,`exactEdgeLengthRads`,`numHexagons`. [#33621](https://github.com/ClickHouse/ClickHouse/pull/33621) ([Bharat Nallan](https://github.com/bharatnc)). +* Add function `bitSlice` to extract bit subsequences from String/FixedString. [#33360](https://github.com/ClickHouse/ClickHouse/pull/33360) ([RogerYK](https://github.com/RogerYK)). +* Implemented `meanZTest` aggregate function. [#33354](https://github.com/ClickHouse/ClickHouse/pull/33354) ([achimbab](https://github.com/achimbab)). +* Add confidence intervals to T-tests aggregate functions. [#33260](https://github.com/ClickHouse/ClickHouse/pull/33260) ([achimbab](https://github.com/achimbab)). +* Add function `addressToLineWithInlines`. Close [#26211](https://github.com/ClickHouse/ClickHouse/issues/26211). [#33467](https://github.com/ClickHouse/ClickHouse/pull/33467) ([SuperDJY](https://github.com/cmsxbc)). +* Added `#!` and `# ` as a recognised start of a single line comment. Closes [#34138](https://github.com/ClickHouse/ClickHouse/issues/34138). [#34230](https://github.com/ClickHouse/ClickHouse/pull/34230) ([Aaron Katz](https://github.com/aaronstephenkatz)). + +#### Experimental Feature + +* Functions for text classification: language and charset detection. See [#23271](https://github.com/ClickHouse/ClickHouse/issues/23271). [#33314](https://github.com/ClickHouse/ClickHouse/pull/33314) ([Nikolay Degterinsky](https://github.com/evillique)). +* Add memory overcommit to `MemoryTracker`. Added `guaranteed` settings for memory limits which represent soft memory limits. In case when hard memory limit is reached, `MemoryTracker` tries to cancel the most overcommited query. New setting `memory_usage_overcommit_max_wait_microseconds` specifies how long queries may wait another query to stop. Closes [#28375](https://github.com/ClickHouse/ClickHouse/issues/28375). [#31182](https://github.com/ClickHouse/ClickHouse/pull/31182) ([Dmitry Novik](https://github.com/novikd)). +* Enable stream to table join in WindowView. [#33729](https://github.com/ClickHouse/ClickHouse/pull/33729) ([vxider](https://github.com/Vxider)). +* Support `SET`, `YEAR`, `TIME` and `GEOMETRY` data types in `MaterializedMySQL` (experimental feature). Fixes [#18091](https://github.com/ClickHouse/ClickHouse/issues/18091), [#21536](https://github.com/ClickHouse/ClickHouse/issues/21536), [#26361](https://github.com/ClickHouse/ClickHouse/issues/26361). [#33429](https://github.com/ClickHouse/ClickHouse/pull/33429) ([zzsmdfj](https://github.com/zzsmdfj)). +* Fix various issues when projection is enabled by default. Each issue is described in separate commit. This is for [#33678](https://github.com/ClickHouse/ClickHouse/issues/33678) . This fixes [#34273](https://github.com/ClickHouse/ClickHouse/issues/34273). [#34305](https://github.com/ClickHouse/ClickHouse/pull/34305) ([Amos Bird](https://github.com/amosbird)). + +#### Performance Improvement + +* Support `optimize_read_in_order` if prefix of sorting key is already sorted. E.g. if we have sorting key `ORDER BY (a, b)` in table and query with `WHERE a = const ORDER BY b` clauses, now it will be applied reading in order of sorting key instead of full sort. [#32748](https://github.com/ClickHouse/ClickHouse/pull/32748) ([Anton Popov](https://github.com/CurtizJ)). +* Improve performance of partitioned insert into table functions `URL`, `S3`, `File`, `HDFS`. Closes [#34348](https://github.com/ClickHouse/ClickHouse/issues/34348). [#34510](https://github.com/ClickHouse/ClickHouse/pull/34510) ([Maksim Kita](https://github.com/kitaisreal)). +* Multiple performance improvements of clickhouse-keeper. [#34484](https://github.com/ClickHouse/ClickHouse/pull/34484) [#34587](https://github.com/ClickHouse/ClickHouse/pull/34587) ([zhanglistar](https://github.com/zhanglistar)). +* `FlatDictionary` improve performance of dictionary data load. [#33871](https://github.com/ClickHouse/ClickHouse/pull/33871) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve performance of `mapPopulateSeries` function. Closes [#33944](https://github.com/ClickHouse/ClickHouse/issues/33944). [#34318](https://github.com/ClickHouse/ClickHouse/pull/34318) ([Maksim Kita](https://github.com/kitaisreal)). +* `_file` and `_path` virtual columns (in file-like table engines) are made `LowCardinality` - it will make queries for multiple files faster. Closes [#34300](https://github.com/ClickHouse/ClickHouse/issues/34300). [#34317](https://github.com/ClickHouse/ClickHouse/pull/34317) ([flynn](https://github.com/ucasfl)). +* Speed up loading of data parts. It was not parallelized before: the setting `part_loading_threads` did not have effect. See [#4699](https://github.com/ClickHouse/ClickHouse/issues/4699). [#34310](https://github.com/ClickHouse/ClickHouse/pull/34310) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve performance of `LineAsString` format. This closes [#34303](https://github.com/ClickHouse/ClickHouse/issues/34303). [#34306](https://github.com/ClickHouse/ClickHouse/pull/34306) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Optimize `quantilesExact{Low,High}` to use `nth_element` instead of `sort`. [#34287](https://github.com/ClickHouse/ClickHouse/pull/34287) ([Danila Kutenin](https://github.com/danlark1)). +* Slightly improve performance of `Regexp` format. [#34202](https://github.com/ClickHouse/ClickHouse/pull/34202) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Minor improvement for analysis of scalar subqueries. [#34128](https://github.com/ClickHouse/ClickHouse/pull/34128) ([Federico Rodriguez](https://github.com/fedrod)). +* Make ORDER BY tuple almost as fast as ORDER BY columns. We have special optimizations for multiple column ORDER BY: https://github.com/ClickHouse/ClickHouse/pull/10831 . It's beneficial to also apply to tuple columns. [#34060](https://github.com/ClickHouse/ClickHouse/pull/34060) ([Amos Bird](https://github.com/amosbird)). +* Rework and reintroduce the scalar subqueries cache to Materialized Views execution. [#33958](https://github.com/ClickHouse/ClickHouse/pull/33958) ([Raúl Marín](https://github.com/Algunenano)). +* Slightly improve performance of `ORDER BY` by adding x86-64 AVX-512 support for `memcmpSmall` functions to accelerate memory comparison. It works only if you compile ClickHouse by yourself. [#33706](https://github.com/ClickHouse/ClickHouse/pull/33706) ([hanqf-git](https://github.com/hanqf-git)). +* Improve `range_hashed` dictionary performance if for key there are a lot of intervals. Fixes [#23821](https://github.com/ClickHouse/ClickHouse/issues/23821). [#33516](https://github.com/ClickHouse/ClickHouse/pull/33516) ([Maksim Kita](https://github.com/kitaisreal)). +* For inserts and merges into S3, write files in parallel whenever possible (TODO: check if it's merged). [#33291](https://github.com/ClickHouse/ClickHouse/pull/33291) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Improve `clickhouse-keeper` performance and fix several memory leaks in NuRaft library. [#33329](https://github.com/ClickHouse/ClickHouse/pull/33329) ([alesapin](https://github.com/alesapin)). + +#### Improvement + +* Support asynchronous inserts in `clickhouse-client` for queries with inlined data. [#34267](https://github.com/ClickHouse/ClickHouse/pull/34267) ([Anton Popov](https://github.com/CurtizJ)). +* Functions `dictGet`, `dictHas` implicitly cast key argument to dictionary key structure, if they are different. [#33672](https://github.com/ClickHouse/ClickHouse/pull/33672) ([Maksim Kita](https://github.com/kitaisreal)). +* Improvements for `range_hashed` dictionaries. Improve performance of load time if there are multiple attributes. Allow to create a dictionary without attributes. Added option to specify strategy when intervals `start` and `end` have `Nullable` type `convert_null_range_bound_to_open` by default is `true`. Closes [#29791](https://github.com/ClickHouse/ClickHouse/issues/29791). Allow to specify `Float`, `Decimal`, `DateTime64`, `Int128`, `Int256`, `UInt128`, `UInt256` as range types. `RangeHashedDictionary` added support for range values that extend `Int64` type. Closes [#28322](https://github.com/ClickHouse/ClickHouse/issues/28322). Added option `range_lookup_strategy` to specify range lookup type `min`, `max` by default is `min` . Closes [#21647](https://github.com/ClickHouse/ClickHouse/issues/21647). Fixed allocated bytes calculations. Fixed type name in `system.dictionaries` in case of `ComplexKeyHashedDictionary`. [#33927](https://github.com/ClickHouse/ClickHouse/pull/33927) ([Maksim Kita](https://github.com/kitaisreal)). +* `flat`, `hashed`, `hashed_array` dictionaries now support creating with empty attributes, with support of reading the keys and using `dictHas`. Fixes [#33820](https://github.com/ClickHouse/ClickHouse/issues/33820). [#33918](https://github.com/ClickHouse/ClickHouse/pull/33918) ([Maksim Kita](https://github.com/kitaisreal)). +* Added support for `DateTime64` data type in dictionaries. [#33914](https://github.com/ClickHouse/ClickHouse/pull/33914) ([Maksim Kita](https://github.com/kitaisreal)). +* Allow to write `s3(url, access_key_id, secret_access_key)` (autodetect of data format and table structure, but with explicit credentials). [#34503](https://github.com/ClickHouse/ClickHouse/pull/34503) ([Kruglov Pavel](https://github.com/Avogar)). +* Added sending of the output format back to client like it's done in HTTP protocol as suggested in [#34362](https://github.com/ClickHouse/ClickHouse/issues/34362). Closes [#34362](https://github.com/ClickHouse/ClickHouse/issues/34362). [#34499](https://github.com/ClickHouse/ClickHouse/pull/34499) ([Vitaly Baranov](https://github.com/vitlibar)). +* Send ProfileEvents statistics in case of INSERT SELECT query (to display query metrics in `clickhouse-client` for this type of queries). [#34498](https://github.com/ClickHouse/ClickHouse/pull/34498) ([Dmitry Novik](https://github.com/novikd)). +* Recognize `.jsonl` extension for JSONEachRow format. [#34496](https://github.com/ClickHouse/ClickHouse/pull/34496) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve schema inference in clickhouse-local. Allow to write just `clickhouse-local -q "select * from table" < data.format`. [#34495](https://github.com/ClickHouse/ClickHouse/pull/34495) ([Kruglov Pavel](https://github.com/Avogar)). +* Privileges CREATE/ALTER/DROP ROW POLICY now can be granted on a table or on `database.*` as well as globally `*.*`. [#34489](https://github.com/ClickHouse/ClickHouse/pull/34489) ([Vitaly Baranov](https://github.com/vitlibar)). +* Allow to export arbitrary large files to `s3`. Add two new settings: `s3_upload_part_size_multiply_factor` and `s3_upload_part_size_multiply_parts_count_threshold`. Now each time `s3_upload_part_size_multiply_parts_count_threshold` uploaded to S3 from a single query `s3_min_upload_part_size` multiplied by `s3_upload_part_size_multiply_factor`. Fixes [#34244](https://github.com/ClickHouse/ClickHouse/issues/34244). [#34422](https://github.com/ClickHouse/ClickHouse/pull/34422) ([alesapin](https://github.com/alesapin)). +* Allow to skip not found (404) URLs for globs when using URL storage / table function. Also closes [#34359](https://github.com/ClickHouse/ClickHouse/issues/34359). [#34392](https://github.com/ClickHouse/ClickHouse/pull/34392) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Default input and output formats for `clickhouse-local` that can be overriden by --input-format and --output-format. Close [#30631](https://github.com/ClickHouse/ClickHouse/issues/30631). [#34352](https://github.com/ClickHouse/ClickHouse/pull/34352) ([李扬](https://github.com/taiyang-li)). +* Add options for `clickhouse-format`. Which close [#30528](https://github.com/ClickHouse/ClickHouse/issues/30528) - `max_query_size` - `max_parser_depth`. [#34349](https://github.com/ClickHouse/ClickHouse/pull/34349) ([李扬](https://github.com/taiyang-li)). +* Better handling of pre-inputs before client start. This is for [#34308](https://github.com/ClickHouse/ClickHouse/issues/34308). [#34336](https://github.com/ClickHouse/ClickHouse/pull/34336) ([Amos Bird](https://github.com/amosbird)). +* `REGEXP_MATCHES` and `REGEXP_REPLACE` function aliases for compatibility with PostgreSQL. Close [#30885](https://github.com/ClickHouse/ClickHouse/issues/30885). [#34334](https://github.com/ClickHouse/ClickHouse/pull/34334) ([李扬](https://github.com/taiyang-li)). +* Some servers expect a User-Agent header in their HTTP requests. A `User-Agent` header entry has been added to HTTP requests of the form: User-Agent: ClickHouse/VERSION_STRING. [#34330](https://github.com/ClickHouse/ClickHouse/pull/34330) ([Saad Ur Rahman](https://github.com/surahman)). +* Cancel merges before acquiring table lock for `TRUNCATE` query to avoid `DEADLOCK_AVOIDED` error in some cases. Fixes [#34302](https://github.com/ClickHouse/ClickHouse/issues/34302). [#34304](https://github.com/ClickHouse/ClickHouse/pull/34304) ([tavplubix](https://github.com/tavplubix)). +* Change severity of the "Cancelled merging parts" message in logs, because it's not an error. This closes [#34148](https://github.com/ClickHouse/ClickHouse/issues/34148). [#34232](https://github.com/ClickHouse/ClickHouse/pull/34232) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add ability to compose PostgreSQL-style cast operator `::` with expressions using `[]` and `.` operators (array and tuple indexing). [#34229](https://github.com/ClickHouse/ClickHouse/pull/34229) ([Nikolay Degterinsky](https://github.com/evillique)). +* Recognize `YYYYMMDD-hhmmss` format in `parseDateTimeBestEffort` function. This closes [#34206](https://github.com/ClickHouse/ClickHouse/issues/34206). [#34208](https://github.com/ClickHouse/ClickHouse/pull/34208) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow carriage return in the middle of the line while parsing by `Regexp` format. This closes [#34200](https://github.com/ClickHouse/ClickHouse/issues/34200). [#34205](https://github.com/ClickHouse/ClickHouse/pull/34205) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow to parse dictionary's `PRIMARY KEY` as `PRIMARY KEY (id, value)`; previously supported only `PRIMARY KEY id, value`. Closes [#34135](https://github.com/ClickHouse/ClickHouse/issues/34135). [#34141](https://github.com/ClickHouse/ClickHouse/pull/34141) ([Maksim Kita](https://github.com/kitaisreal)). +* An optional argument for `splitByChar` to limit the number of resulting elements. close [#34081](https://github.com/ClickHouse/ClickHouse/issues/34081). [#34140](https://github.com/ClickHouse/ClickHouse/pull/34140) ([李扬](https://github.com/taiyang-li)). +* Improving the experience of multiple line editing for clickhouse-client. This is a follow-up of [#31123](https://github.com/ClickHouse/ClickHouse/pull/31123). [#34114](https://github.com/ClickHouse/ClickHouse/pull/34114) ([Amos Bird](https://github.com/amosbird)). +* Add `UUID` suport in `MsgPack` input/output format. [#34065](https://github.com/ClickHouse/ClickHouse/pull/34065) ([Kruglov Pavel](https://github.com/Avogar)). +* Tracing context (for OpenTelemetry) is now propagated from GRPC client metadata (this change is relevant for GRPC client-server protocol). [#34064](https://github.com/ClickHouse/ClickHouse/pull/34064) ([andremarianiello](https://github.com/andremarianiello)). +* Supports all types of `SYSTEM` queries with `ON CLUSTER` clause. [#34005](https://github.com/ClickHouse/ClickHouse/pull/34005) ([小路](https://github.com/nicelulu)). +* Improve memory accounting for queries that are using less than `max_untracker_memory`. [#34001](https://github.com/ClickHouse/ClickHouse/pull/34001) ([Azat Khuzhin](https://github.com/azat)). +* Fixed UTF-8 string case-insensitive search when lowercase and uppercase characters are represented by different number of bytes. Example is `ẞ` and `ß`. This closes [#7334](https://github.com/ClickHouse/ClickHouse/issues/7334). [#33992](https://github.com/ClickHouse/ClickHouse/pull/33992) ([Harry Lee](https://github.com/HarryLeeIBM)). +* Detect format and schema from stdin in `clickhouse-local`. [#33960](https://github.com/ClickHouse/ClickHouse/pull/33960) ([Kruglov Pavel](https://github.com/Avogar)). +* Correctly handle the case of misconfiguration when multiple disks are using the same path on the filesystem. [#29072](https://github.com/ClickHouse/ClickHouse/issues/29072). [#33905](https://github.com/ClickHouse/ClickHouse/pull/33905) ([zhongyuankai](https://github.com/zhongyuankai)). +* Try every resolved IP address while getting S3 proxy. S3 proxies are rarely used, mostly in Yandex Cloud. [#33862](https://github.com/ClickHouse/ClickHouse/pull/33862) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support EXPLAIN AST CREATE FUNCTION query `EXPLAIN AST CREATE FUNCTION mycast AS (n) -> cast(n as String)` will return `EXPLAIN AST CREATE FUNCTION mycast AS n -> CAST(n, 'String')`. [#33819](https://github.com/ClickHouse/ClickHouse/pull/33819) ([李扬](https://github.com/taiyang-li)). +* Added support for cast from `Map(Key, Value)` to `Array(Tuple(Key, Value))`. [#33794](https://github.com/ClickHouse/ClickHouse/pull/33794) ([Maksim Kita](https://github.com/kitaisreal)). +* Add some improvements and fixes for `Bool` data type. Fixes [#33244](https://github.com/ClickHouse/ClickHouse/issues/33244). [#33737](https://github.com/ClickHouse/ClickHouse/pull/33737) ([Kruglov Pavel](https://github.com/Avogar)). +* Parse and store OpenTelemetry trace-id in big-endian order. [#33723](https://github.com/ClickHouse/ClickHouse/pull/33723) ([Frank Chen](https://github.com/FrankChen021)). +* Improvement for `fromUnixTimestamp64` family functions.. They now accept any integer value that can be converted to `Int64`. This closes: [#14648](https://github.com/ClickHouse/ClickHouse/issues/14648). [#33505](https://github.com/ClickHouse/ClickHouse/pull/33505) ([Andrey Zvonov](https://github.com/zvonand)). +* Reimplement `_shard_num` from constants (see [#7624](https://github.com/ClickHouse/ClickHouse/issues/7624)) with `shardNum()` function (seee [#27020](https://github.com/ClickHouse/ClickHouse/issues/27020)), to avoid possible issues (like those that had been found in [#16947](https://github.com/ClickHouse/ClickHouse/issues/16947)). [#33392](https://github.com/ClickHouse/ClickHouse/pull/33392) ([Azat Khuzhin](https://github.com/azat)). +* Enable binary arithmetic (plus, minus, multiply, division, least, greatest) between Decimal and Float. [#33355](https://github.com/ClickHouse/ClickHouse/pull/33355) ([flynn](https://github.com/ucasfl)). +* Respect cgroups limits in max_threads autodetection. [#33342](https://github.com/ClickHouse/ClickHouse/pull/33342) ([JaySon](https://github.com/JaySon-Huang)). +* Add new clickhouse-keeper setting `min_session_timeout_ms`. Now clickhouse-keeper will determine client session timeout according to `min_session_timeout_ms` and `session_timeout_ms` settings. [#33288](https://github.com/ClickHouse/ClickHouse/pull/33288) ([JackyWoo](https://github.com/JackyWoo)). +* Added `UUID` data type support for functions `hex` and `bin`. [#32170](https://github.com/ClickHouse/ClickHouse/pull/32170) ([Frank Chen](https://github.com/FrankChen021)). +* Fix reading of subcolumns with dots in their names. In particular fixed reading of `Nested` columns, if their element names contain dots (e.g ```Nested(`keys.name` String, `keys.id` UInt64, values UInt64)```). [#34228](https://github.com/ClickHouse/ClickHouse/pull/34228) ([Anton Popov](https://github.com/CurtizJ)). +* Fixes `parallel_view_processing = 0` not working when inserting into a table using `VALUES`. - Fixes `view_duration_ms` in the `query_views_log` not being set correctly for materialized views. [#34067](https://github.com/ClickHouse/ClickHouse/pull/34067) ([Raúl Marín](https://github.com/Algunenano)). +* Fix parsing tables structure from ZooKeeper: now metadata from ZooKeeper compared with local metadata in canonical form. It helps when canonical function names can change between ClickHouse versions. [#33933](https://github.com/ClickHouse/ClickHouse/pull/33933) ([sunny](https://github.com/sunny19930321)). +* Properly escape some characters for interaction with LDAP. [#33401](https://github.com/ClickHouse/ClickHouse/pull/33401) ([IlyaTsoi](https://github.com/IlyaTsoi)). + +#### Build/Testing/Packaging Improvement + +* Remove unbundled build support. [#33690](https://github.com/ClickHouse/ClickHouse/pull/33690) ([Azat Khuzhin](https://github.com/azat)). +* Ensure that tests don't depend on the result of non-stable sorting of equal elements. Added equal items ranges randomization in debug after sort to prevent issues when we rely on equal items sort order. [#34393](https://github.com/ClickHouse/ClickHouse/pull/34393) ([Maksim Kita](https://github.com/kitaisreal)). +* Add verbosity to a style check. [#34289](https://github.com/ClickHouse/ClickHouse/pull/34289) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove `clickhouse-test` debian package because it's obsolete. [#33948](https://github.com/ClickHouse/ClickHouse/pull/33948) ([Ilya Yatsishin](https://github.com/qoega)). +* Multiple improvements for build system to remove the possibility of occasionally using packages from the OS and to enforce hermetic builds. [#33695](https://github.com/ClickHouse/ClickHouse/pull/33695) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Fixed the assertion in case of using `allow_experimental_parallel_reading_from_replicas` with `max_parallel_replicas` equals to 1. This fixes [#34525](https://github.com/ClickHouse/ClickHouse/issues/34525). [#34613](https://github.com/ClickHouse/ClickHouse/pull/34613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix rare bug while reading of empty arrays, which could lead to `Data compressed with different methods` error. It can reproduce if you have mostly empty arrays, but not always. And reading is performed in backward direction with ORDER BY ... DESC. This error is extremely unlikely to happen. [#34327](https://github.com/ClickHouse/ClickHouse/pull/34327) ([Anton Popov](https://github.com/CurtizJ)). +* Fix wrong result of `round`/`roundBankers` if integer values of small types are rounded. Closes [#33267](https://github.com/ClickHouse/ClickHouse/issues/33267). [#34562](https://github.com/ClickHouse/ClickHouse/pull/34562) ([李扬](https://github.com/taiyang-li)). +* Sometimes query cancellation did not work immediately when we were reading multiple files from s3 or HDFS. Fixes [#34301](https://github.com/ClickHouse/ClickHouse/issues/34301) Relates to [#34397](https://github.com/ClickHouse/ClickHouse/issues/34397). [#34539](https://github.com/ClickHouse/ClickHouse/pull/34539) ([Dmitry Novik](https://github.com/novikd)). +* Fix exception `Chunk should have AggregatedChunkInfo in MergingAggregatedTransform` (in case of `optimize_aggregation_in_order = 1` and `distributed_aggregation_memory_efficient = 0`). Fixes [#34526](https://github.com/ClickHouse/ClickHouse/issues/34526). [#34532](https://github.com/ClickHouse/ClickHouse/pull/34532) ([Anton Popov](https://github.com/CurtizJ)). +* Fix comparison between integers and floats in index analysis. Previously it could lead to skipping some granules for reading by mistake. Fixes [#34493](https://github.com/ClickHouse/ClickHouse/issues/34493). [#34528](https://github.com/ClickHouse/ClickHouse/pull/34528) ([Anton Popov](https://github.com/CurtizJ)). +* Fix compression support in URL engine. [#34524](https://github.com/ClickHouse/ClickHouse/pull/34524) ([Frank Chen](https://github.com/FrankChen021)). +* Fix possible error 'file_size: Operation not supported' in files' schema autodetection. [#34479](https://github.com/ClickHouse/ClickHouse/pull/34479) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixes possible race with table deletion. [#34416](https://github.com/ClickHouse/ClickHouse/pull/34416) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible error `Cannot convert column Function to mask` in short circuit function evaluation. Closes [#34171](https://github.com/ClickHouse/ClickHouse/issues/34171). [#34415](https://github.com/ClickHouse/ClickHouse/pull/34415) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix potential crash when doing schema inference from url source. Closes [#34147](https://github.com/ClickHouse/ClickHouse/issues/34147). [#34405](https://github.com/ClickHouse/ClickHouse/pull/34405) ([Kruglov Pavel](https://github.com/Avogar)). +* For UDFs access permissions were checked for database level instead of global level as it should be. Closes [#34281](https://github.com/ClickHouse/ClickHouse/issues/34281). [#34404](https://github.com/ClickHouse/ClickHouse/pull/34404) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix wrong engine syntax in result of `SHOW CREATE DATABASE` query for databases with engine `Memory`. This closes [#34335](https://github.com/ClickHouse/ClickHouse/issues/34335). [#34345](https://github.com/ClickHouse/ClickHouse/pull/34345) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed a couple of extremely rare race conditions that might lead to broken state of replication queue and "intersecting parts" error. [#34297](https://github.com/ClickHouse/ClickHouse/pull/34297) ([tavplubix](https://github.com/tavplubix)). +* Fix progress bar width. It was incorrectly rounded to integer number of characters. [#34275](https://github.com/ClickHouse/ClickHouse/pull/34275) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix current_user/current_address client information fields for inter-server communication (before this patch current_user/current_address will be preserved from the previous query). [#34263](https://github.com/ClickHouse/ClickHouse/pull/34263) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory leak in case of some Exception during query processing with `optimize_aggregation_in_order=1`. [#34234](https://github.com/ClickHouse/ClickHouse/pull/34234) ([Azat Khuzhin](https://github.com/azat)). +* Fix metric `Query`, which shows the number of executing queries. In last several releases it was always 0. [#34224](https://github.com/ClickHouse/ClickHouse/pull/34224) ([Anton Popov](https://github.com/CurtizJ)). +* Fix schema inference for table runction `s3`. [#34186](https://github.com/ClickHouse/ClickHouse/pull/34186) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix rare and benign race condition in `HDFS`, `S3` and `URL` storage engines which can lead to additional connections. [#34172](https://github.com/ClickHouse/ClickHouse/pull/34172) ([alesapin](https://github.com/alesapin)). +* Fix bug which can rarely lead to error "Cannot read all data" while reading LowCardinality columns of MergeTree table engines family which stores data on remote file system like S3 (virtual filesystem over s3 is an experimental feature that is not ready for production). [#34139](https://github.com/ClickHouse/ClickHouse/pull/34139) ([alesapin](https://github.com/alesapin)). +* Fix inserts to distributed tables in case of a change of native protocol. The last change was in the version 22.1, so there may be some failures of inserts to distributed tables after upgrade to that version. [#34132](https://github.com/ClickHouse/ClickHouse/pull/34132) ([Anton Popov](https://github.com/CurtizJ)). +* Fix possible data race in `File` table engine that was introduced in [#33960](https://github.com/ClickHouse/ClickHouse/pull/33960). Closes [#34111](https://github.com/ClickHouse/ClickHouse/issues/34111). [#34113](https://github.com/ClickHouse/ClickHouse/pull/34113) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed minor race condition that might cause "intersecting parts" error in extremely rare cases after ZooKeeper connection loss. [#34096](https://github.com/ClickHouse/ClickHouse/pull/34096) ([tavplubix](https://github.com/tavplubix)). +* Fix asynchronous inserts with `Native` format. [#34068](https://github.com/ClickHouse/ClickHouse/pull/34068) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug which lead to inability for server to start when both replicated access storage and keeper (embedded in clickhouse-server) are used. Introduced two settings for keeper socket timeout instead of settings from default user: `keeper_server.socket_receive_timeout_sec` and `keeper_server.socket_send_timeout_sec`. Fixes [#33973](https://github.com/ClickHouse/ClickHouse/issues/33973). [#33988](https://github.com/ClickHouse/ClickHouse/pull/33988) ([alesapin](https://github.com/alesapin)). +* Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix parsing IPv6 from query parameter (prepared statements) and fix IPv6 to string conversion. Closes [#33928](https://github.com/ClickHouse/ClickHouse/issues/33928). [#33971](https://github.com/ClickHouse/ClickHouse/pull/33971) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash while reading of nested tuples. Fixes [#33838](https://github.com/ClickHouse/ClickHouse/issues/33838). [#33956](https://github.com/ClickHouse/ClickHouse/pull/33956) ([Anton Popov](https://github.com/CurtizJ)). +* Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). +* Aggregate function combinator `-If` did not correctly process `Nullable` filter argument. This closes [#27073](https://github.com/ClickHouse/ClickHouse/issues/27073). [#33920](https://github.com/ClickHouse/ClickHouse/pull/33920) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix potential race condition when doing remote disk read (virtual filesystem over s3 is an experimental feature that is not ready for production). [#33912](https://github.com/ClickHouse/ClickHouse/pull/33912) ([Amos Bird](https://github.com/amosbird)). +* Fix crash if SQL UDF is created with lambda with non identifier arguments. Closes [#33866](https://github.com/ClickHouse/ClickHouse/issues/33866). [#33868](https://github.com/ClickHouse/ClickHouse/pull/33868) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed `replica is not readonly` logical error on `SYSTEM RESTORE REPLICA` query when replica is actually readonly. Fixes [#33806](https://github.com/ClickHouse/ClickHouse/issues/33806). [#33847](https://github.com/ClickHouse/ClickHouse/pull/33847) ([tavplubix](https://github.com/tavplubix)). +* Fix memory leak in `clickhouse-keeper` in case of compression is used (default). [#33840](https://github.com/ClickHouse/ClickHouse/pull/33840) ([Azat Khuzhin](https://github.com/azat)). +* Fix index analysis with no common types available. [#33833](https://github.com/ClickHouse/ClickHouse/pull/33833) ([Amos Bird](https://github.com/amosbird)). +* Fix schema inference for `JSONEachRow` and `JSONCompactEachRow`. [#33830](https://github.com/ClickHouse/ClickHouse/pull/33830) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix usage of external dictionaries with `redis` source and large number of keys. [#33804](https://github.com/ClickHouse/ClickHouse/pull/33804) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bug in client that led to 'Connection reset by peer' in server. Closes [#33309](https://github.com/ClickHouse/ClickHouse/issues/33309). [#33790](https://github.com/ClickHouse/ClickHouse/pull/33790) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix parsing query INSERT INTO ... VALUES SETTINGS ... (...), ... [#33776](https://github.com/ClickHouse/ClickHouse/pull/33776) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix bug of check table when creating data part with wide format and projection. [#33774](https://github.com/ClickHouse/ClickHouse/pull/33774) ([李扬](https://github.com/taiyang-li)). +* Fix tiny race between count() and INSERT/merges/... in MergeTree (it is possible to return incorrect number of rows for SELECT with optimize_trivial_count_query). [#33753](https://github.com/ClickHouse/ClickHouse/pull/33753) ([Azat Khuzhin](https://github.com/azat)). +* Throw exception when directory listing request has failed in storage HDFS. [#33724](https://github.com/ClickHouse/ClickHouse/pull/33724) ([LiuNeng](https://github.com/liuneng1994)). +* Fix mutation when table contains projections. This fixes [#33010](https://github.com/ClickHouse/ClickHouse/issues/33010). This fixes [#33275](https://github.com/ClickHouse/ClickHouse/issues/33275). [#33679](https://github.com/ClickHouse/ClickHouse/pull/33679) ([Amos Bird](https://github.com/amosbird)). +* Correctly determine current database if `CREATE TEMPORARY TABLE AS SELECT` is queried inside a named HTTP session. This is a very rare use case. This closes [#8340](https://github.com/ClickHouse/ClickHouse/issues/8340). [#33676](https://github.com/ClickHouse/ClickHouse/pull/33676) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow some queries with sorting, LIMIT BY, ARRAY JOIN and lambda functions. This closes [#7462](https://github.com/ClickHouse/ClickHouse/issues/7462). [#33675](https://github.com/ClickHouse/ClickHouse/pull/33675) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix bug in "zero copy replication" (a feature that is under development and should not be used in production) which lead to data duplication in case of TTL move. Fixes [#33643](https://github.com/ClickHouse/ClickHouse/issues/33643). [#33642](https://github.com/ClickHouse/ClickHouse/pull/33642) ([alesapin](https://github.com/alesapin)). +* Fix `Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform` (in case of `optimize_aggregation_in_order = 1`). [#33637](https://github.com/ClickHouse/ClickHouse/pull/33637) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `Bad cast from type ... to DB::DataTypeArray` which may happen when table has `Nested` column with dots in name, and default value is generated for it (e.g. during insert, when column is not listed). Continuation of [#28762](https://github.com/ClickHouse/ClickHouse/issues/28762). [#33588](https://github.com/ClickHouse/ClickHouse/pull/33588) ([Alexey Pavlenko](https://github.com/alexeypavlenko)). +* Export into `lz4` files has been fixed. Closes [#31421](https://github.com/ClickHouse/ClickHouse/issues/31421). [#31862](https://github.com/ClickHouse/ClickHouse/pull/31862) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix potential crash if `group_by_overflow_mode` was set to `any` (approximate GROUP BY) and aggregation was performed by single column of type `LowCardinality`. [#34506](https://github.com/ClickHouse/ClickHouse/pull/34506) ([DR](https://github.com/freedomDR)). +* Fix inserting to temporary tables via gRPC client-server protocol. Fixes [#34347](https://github.com/ClickHouse/ClickHouse/issues/34347), issue `#2`. [#34364](https://github.com/ClickHouse/ClickHouse/pull/34364) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix issue [#19429](https://github.com/ClickHouse/ClickHouse/issues/19429). [#34225](https://github.com/ClickHouse/ClickHouse/pull/34225) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix issue [#18206](https://github.com/ClickHouse/ClickHouse/issues/18206). [#33977](https://github.com/ClickHouse/ClickHouse/pull/33977) ([Vitaly Baranov](https://github.com/vitlibar)). +* This PR allows using multiple LDAP storages in the same list of user directories. It worked earlier but was broken because LDAP tests are disabled (they are part of the testflows tests). [#33574](https://github.com/ClickHouse/ClickHouse/pull/33574) ([Vitaly Baranov](https://github.com/vitlibar)). + + +### ClickHouse release v22.1, 2022-01-18 #### Upgrade Notes diff --git a/CMakeLists.txt b/CMakeLists.txt index f27e9cdbea4..7ed3872fd6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,8 +62,8 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# Check that submodules are present only if source was downloaded with git -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") +# Check that submodules are present +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md") message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") endif () @@ -248,7 +248,9 @@ endif() if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") set(USE_DEBUG_HELPERS ON) endif() + option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS}) +option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF) # Create BuildID when using lld. For other linkers it is created by default. if (LINKER_NAME MATCHES "lld$") @@ -263,6 +265,14 @@ if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE)) set (USE_BINARY_HASH 1) endif () +# Allows to build stripped binary in a separate directory +if (OBJCOPY_PATH AND READELF_PATH) + option(INSTALL_STRIPPED_BINARIES "Build stripped binaries with debug info in separate directory" OFF) + if (INSTALL_STRIPPED_BINARIES) + set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information") + endif() +endif() + cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd @@ -285,8 +295,13 @@ include(cmake/cpu_features.cmake) set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables") # Reproducible builds -set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") -set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") +# If turned `ON`, remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE(). +option(ENABLE_BUILD_PATH_MAPPING "Enable remap file source paths in debug info, predefined preprocessor macros and __builtin_FILE(). It's to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ON) + +if (ENABLE_BUILD_PATH_MAPPING) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") + set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") +endif() if (${CMAKE_VERSION} VERSION_LESS "3.12.4") # CMake < 3.12 doesn't support setting 20 as a C++ standard version. @@ -340,16 +355,19 @@ if (WITH_COVERAGE AND COMPILER_GCC) set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage") endif() -set(COMPILER_FLAGS "${COMPILER_FLAGS}") +set (COMPILER_FLAGS "${COMPILER_FLAGS}") + +# Our built-in unwinder only supports DWARF version up to 4. +set (DEBUG_INFO_FLAGS "-g -gdwarf-4") set (CMAKE_BUILD_COLOR_MAKEFILE ON) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") -set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}") -set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") +set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}") +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") -set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") -set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}") if (COMPILER_CLANG) if (OS_DARWIN) diff --git a/README.md b/README.md index f433b457861..e6dc9f1e6fc 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,6 @@ ClickHouse® is an open-source column-oriented database management system that a * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. * [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. * [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. -* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation. +* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation. * [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev. * [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. diff --git a/SECURITY.md b/SECURITY.md index ca3c8b439fd..6c03a6bb945 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,7 +15,7 @@ The following versions of ClickHouse server are currently being supported with s | 20.x | :x: | | 21.1 | :x: | | 21.2 | :x: | -| 21.3 | ✅ | +| 21.3 | :x: | | 21.4 | :x: | | 21.5 | :x: | | 21.6 | :x: | @@ -23,9 +23,11 @@ The following versions of ClickHouse server are currently being supported with s | 21.8 | ✅ | | 21.9 | :x: | | 21.10 | :x: | -| 21.11 | ✅ | -| 21.12 | ✅ | +| 21.11 | :x: | +| 21.12 | :x: | | 22.1 | ✅ | +| 22.2 | ✅ | +| 22.3 | ✅ | ## Reporting a Vulnerability diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index de7ace7c178..3cfd2f6906a 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -18,6 +18,7 @@ set (SRCS terminalColors.cpp errnoToString.cpp StringRef.cpp + safeExit.cpp ) if (ENABLE_REPLXX) diff --git a/base/base/LineReader.cpp b/base/base/LineReader.cpp index 686d70f247d..f4e741a54e7 100644 --- a/base/base/LineReader.cpp +++ b/base/base/LineReader.cpp @@ -109,10 +109,10 @@ void LineReader::Suggest::addWords(Words && new_words) std::lock_guard lock(mutex); addNewWords(words, new_words, std::less{}); addNewWords(words_no_case, new_words_no_case, NoCaseCompare{}); - } - assert(std::is_sorted(words.begin(), words.end())); - assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{})); + assert(std::is_sorted(words.begin(), words.end())); + assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{})); + } } LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) diff --git a/base/base/StringRef.h b/base/base/StringRef.h index eefc87121fc..f300a2d63df 100644 --- a/base/base/StringRef.h +++ b/base/base/StringRef.h @@ -19,6 +19,12 @@ #if defined(__SSE4_2__) #include #include + #define CRC_INT _mm_crc32_u64 +#endif + +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) + #include + #define CRC_INT __crc32cd #endif @@ -40,9 +46,9 @@ struct StringRef constexpr StringRef(const char * data_, size_t size_) : data(data_), size(size_) {} - StringRef(const std::string & s) : data(s.data()), size(s.size()) {} + StringRef(const std::string & s) : data(s.data()), size(s.size()) {} /// NOLINT constexpr explicit StringRef(std::string_view s) : data(s.data()), size(s.size()) {} - constexpr StringRef(const char * data_) : StringRef(std::string_view{data_}) {} + constexpr StringRef(const char * data_) : StringRef(std::string_view{data_}) {} /// NOLINT constexpr StringRef() = default; std::string toString() const { return std::string(data, size); } @@ -205,7 +211,7 @@ struct StringRefHash64 } }; -#if defined(__SSE4_2__) +#if defined(CRC_INT) /// Parts are taken from CityHash. @@ -281,13 +287,13 @@ struct CRC32Hash do { UInt64 word = unalignedLoad(pos); - res = _mm_crc32_u64(res, word); + res = CRC_INT(res, word); pos += 8; } while (pos + 8 < end); UInt64 word = unalignedLoad(end - 8); /// I'm not sure if this is normal. - res = _mm_crc32_u64(res, word); + res = CRC_INT(res, word); return res; } diff --git a/base/base/insertAtEnd.h b/base/base/insertAtEnd.h index c4fef664511..abb2aa7d563 100644 --- a/base/base/insertAtEnd.h +++ b/base/base/insertAtEnd.h @@ -26,3 +26,27 @@ void insertAtEnd(std::vector & dest, std::vector && src) dest.insert(dest.end(), std::make_move_iterator(src.begin()), std::make_move_iterator(src.end())); src.clear(); } + +template +void insertAtEnd(Container & dest, const Container & src) +{ + if (src.empty()) + return; + + dest.insert(dest.end(), src.begin(), src.end()); +} + +template +void insertAtEnd(Container & dest, Container && src) +{ + if (src.empty()) + return; + if (dest.empty()) + { + dest.swap(src); + return; + } + + dest.insert(dest.end(), std::make_move_iterator(src.begin()), std::make_move_iterator(src.end())); + src.clear(); +} diff --git a/base/base/safeExit.cpp b/base/base/safeExit.cpp new file mode 100644 index 00000000000..4ccfee80643 --- /dev/null +++ b/base/base/safeExit.cpp @@ -0,0 +1,18 @@ +#if defined(OS_LINUX) +# include +#endif +#include +#include +#include + +[[noreturn]] void safeExit(int code) +{ +#if defined(THREAD_SANITIZER) && defined(OS_LINUX) + /// Thread sanitizer tries to do something on exit that we don't need if we want to exit immediately, + /// while connection handling threads are still run. + (void)syscall(SYS_exit_group, code); + __builtin_unreachable(); +#else + _exit(code); +#endif +} diff --git a/base/base/safeExit.h b/base/base/safeExit.h new file mode 100644 index 00000000000..f999ccfac18 --- /dev/null +++ b/base/base/safeExit.h @@ -0,0 +1,4 @@ +#pragma once + +/// _exit() with a workaround for TSan. +[[noreturn]] void safeExit(int code); diff --git a/base/base/sort.h b/base/base/sort.h index 592a899a291..589469fffaa 100644 --- a/base/base/sort.h +++ b/base/base/sort.h @@ -2,6 +2,63 @@ #include +#ifndef NDEBUG + +#include +#include + +/** Same as libcxx std::__debug_less. Just without dependency on private part of standard library. + * Check that Comparator induce strict weak ordering. + */ +template +class DebugLessComparator +{ +public: + constexpr DebugLessComparator(Comparator & cmp_) + : cmp(cmp_) + {} + + template + constexpr bool operator()(const LhsType & lhs, const RhsType & rhs) + { + bool lhs_less_than_rhs = cmp(lhs, rhs); + if (lhs_less_than_rhs) + assert(!cmp(rhs, lhs)); + + return lhs_less_than_rhs; + } + + template + constexpr bool operator()(LhsType & lhs, RhsType & rhs) + { + bool lhs_less_than_rhs = cmp(lhs, rhs); + if (lhs_less_than_rhs) + assert(!cmp(rhs, lhs)); + + return lhs_less_than_rhs; + } + +private: + Comparator & cmp; +}; + +template +using ComparatorWrapper = DebugLessComparator; + +template +void shuffle(RandomIt first, RandomIt last) +{ + static thread_local pcg64 rng(getThreadId()); + std::shuffle(first, last, rng); +} + +#else + +template +using ComparatorWrapper = Comparator; + +#endif + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" @@ -10,19 +67,48 @@ template void nth_element(RandomIt first, RandomIt nth, RandomIt last) { - ::miniselect::floyd_rivest_select(first, nth, last); -} + using value_type = typename std::iterator_traits::value_type; + using comparator = std::less; -template -void partial_sort(RandomIt first, RandomIt middle, RandomIt last) -{ - ::miniselect::floyd_rivest_partial_sort(first, middle, last); + comparator compare; + ComparatorWrapper compare_wrapper = compare; + +#ifndef NDEBUG + ::shuffle(first, last); +#endif + + ::miniselect::floyd_rivest_select(first, nth, last, compare_wrapper); + +#ifndef NDEBUG + ::shuffle(first, nth); + + if (nth != last) + ::shuffle(nth + 1, last); +#endif } template void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare) { - ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare); +#ifndef NDEBUG + ::shuffle(first, last); +#endif + + ComparatorWrapper compare_wrapper = compare; + ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare_wrapper); + +#ifndef NDEBUG + ::shuffle(middle, last); +#endif +} + +template +void partial_sort(RandomIt first, RandomIt middle, RandomIt last) +{ + using value_type = typename std::iterator_traits::value_type; + using comparator = std::less; + + ::partial_sort(first, middle, last, comparator()); } #pragma GCC diagnostic pop @@ -30,7 +116,12 @@ void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compar template void sort(RandomIt first, RandomIt last, Compare compare) { - ::pdqsort(first, last, compare); +#ifndef NDEBUG + ::shuffle(first, last); +#endif + + ComparatorWrapper compare_wrapper = compare; + ::pdqsort(first, last, compare_wrapper); } template @@ -38,5 +129,5 @@ void sort(RandomIt first, RandomIt last) { using value_type = typename std::iterator_traits::value_type; using comparator = std::less; - ::pdqsort(first, last, comparator()); + ::sort(first, last, comparator()); } diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index ec146fd5821..ed2c2972cfe 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -12,6 +12,18 @@ #include #include +#include +#include + +/// Use same extended double for all platforms +#if (LDBL_MANT_DIG == 64) +#define CONSTEXPR_FROM_DOUBLE constexpr +using FromDoubleIntermediateType = long double; +#else +/// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended` +#define CONSTEXPR_FROM_DOUBLE +using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended; +#endif namespace wide { @@ -265,12 +277,23 @@ struct integer::_impl constexpr static void set_multiplier(integer & self, T t) noexcept { constexpr uint64_t max_int = std::numeric_limits::max(); - + static_assert(std::is_same_v || std::is_same_v); /// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast). - if (!std::isfinite(t)) + if constexpr (std::is_same_v) { - self = 0; - return; + if (!std::isfinite(t)) + { + self = 0; + return; + } + } + else + { + if (!boost::math::isfinite(t)) + { + self = 0; + return; + } } const T alpha = t / static_cast(max_int); @@ -278,13 +301,13 @@ struct integer::_impl if (alpha <= static_cast(max_int)) self = static_cast(alpha); else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations. - set_multiplier(self, alpha); + set_multiplier(self, static_cast(alpha)); self *= max_int; self += static_cast(t - floor(alpha) * static_cast(max_int)); // += b_i } - constexpr static void wide_integer_from_builtin(integer & self, double rhs) noexcept + CONSTEXPR_FROM_DOUBLE static void wide_integer_from_builtin(integer & self, double rhs) noexcept { constexpr int64_t max_int = std::numeric_limits::max(); constexpr int64_t min_int = std::numeric_limits::lowest(); @@ -294,24 +317,17 @@ struct integer::_impl /// the result may not fit in 64 bits. /// The example of such a number is 9.22337e+18. /// As to_Integral does a static_cast to int64_t, it may result in UB. - /// The necessary check here is that long double has enough significant (mantissa) bits to store the + /// The necessary check here is that FromDoubleIntermediateType has enough significant (mantissa) bits to store the /// int64_t max value precisely. - // TODO Be compatible with Apple aarch64 -#if not (defined(__APPLE__) && defined(__aarch64__)) - static_assert(LDBL_MANT_DIG >= 64, - "On your system long double has less than 64 precision bits, " - "which may result in UB when initializing double from int64_t"); -#endif - - if (rhs > static_cast(min_int) && rhs < static_cast(max_int)) + if (rhs > static_cast(min_int) && rhs < static_cast(max_int)) { self = static_cast(rhs); return; } - const long double rhs_long_double = (static_cast(rhs) < 0) - ? -static_cast(rhs) + const FromDoubleIntermediateType rhs_long_double = (static_cast(rhs) < 0) + ? -static_cast(rhs) : rhs; set_multiplier(self, rhs_long_double); diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 6996b63b5dd..0260c6380f4 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -18,7 +18,7 @@ #include "Common/config_version.h" #include -#if USE_SENTRY +#if USE_SENTRY && !defined(KEEPER_STANDALONE_BUILD) # include # include diff --git a/base/loggers/CMakeLists.txt b/base/loggers/CMakeLists.txt index 22be002e069..148c4f84f68 100644 --- a/base/loggers/CMakeLists.txt +++ b/base/loggers/CMakeLists.txt @@ -1,5 +1,13 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(loggers .) + +# Standard version depends on DBMS and works with text log add_library(loggers ${loggers_sources} ${loggers_headers}) +target_compile_definitions(loggers PUBLIC WITH_TEXT_LOG=1) target_link_libraries(loggers PRIVATE dbms clickhouse_common_io) target_include_directories(loggers PUBLIC ..) + +# Lightweight version doesn't work with textlog and also doesn't depend on DBMS +add_library(loggers_no_text_log ${loggers_sources} ${loggers_headers}) +target_link_libraries(loggers_no_text_log PRIVATE clickhouse_common_io) +target_include_directories(loggers PUBLIC ..) diff --git a/base/loggers/Loggers.cpp b/base/loggers/Loggers.cpp index 3c350c834e5..7c627ad2272 100644 --- a/base/loggers/Loggers.cpp +++ b/base/loggers/Loggers.cpp @@ -9,7 +9,11 @@ #include #include #include -#include + +#ifdef WITH_TEXT_LOG + #include +#endif + #include namespace fs = std::filesystem; @@ -30,17 +34,21 @@ static std::string createDirectory(const std::string & file) return path; }; +#ifdef WITH_TEXT_LOG void Loggers::setTextLog(std::shared_ptr log, int max_priority) { text_log = log; text_log_max_priority = max_priority; } +#endif void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name) { +#ifdef WITH_TEXT_LOG if (split) if (auto log = text_log.lock()) split->addTextLog(log, text_log_max_priority); +#endif auto current_logger = config.getString("logger", ""); if (config_logger == current_logger) //-V1051 @@ -247,11 +255,8 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log if (log_level > max_log_level) max_log_level = log_level; - const auto log_path = config.getString("logger.log", ""); - if (!log_path.empty()) + if (log_file) split->setLevel("log", log_level); - else - split->setLevel("log", 0); // Set level to console bool is_daemon = config.getBool("application.runAsDaemon", false); @@ -263,15 +268,13 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log split->setLevel("console", 0); // Set level to errorlog - int errorlog_level = 0; - const auto errorlog_path = config.getString("logger.errorlog", ""); - if (!errorlog_path.empty()) + if (error_log_file) { - errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice")); + int errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice")); if (errorlog_level > max_log_level) max_log_level = errorlog_level; + split->setLevel("errorlog", errorlog_level); } - split->setLevel("errorlog", errorlog_level); // Set level to syslog int syslog_level = 0; diff --git a/base/loggers/Loggers.h b/base/loggers/Loggers.h index a859c32fa89..22b2b5e2c69 100644 --- a/base/loggers/Loggers.h +++ b/base/loggers/Loggers.h @@ -7,10 +7,12 @@ #include #include "OwnSplitChannel.h" +#ifdef WITH_TEXT_LOG namespace DB { class TextLog; } +#endif namespace Poco::Util { @@ -27,7 +29,9 @@ public: /// Close log files. On next log write files will be reopened. void closeLogs(Poco::Logger & logger); +#ifdef WITH_TEXT_LOG void setTextLog(std::shared_ptr log, int max_priority); +#endif private: Poco::AutoPtr log_file; @@ -37,8 +41,10 @@ private: /// Previous value of logger element in config. It is used to reinitialize loggers whenever the value changed. std::string config_logger; +#ifdef WITH_TEXT_LOG std::weak_ptr text_log; int text_log_max_priority = -1; +#endif Poco::AutoPtr split; }; diff --git a/base/loggers/OwnSplitChannel.cpp b/base/loggers/OwnSplitChannel.cpp index 2267b8f425d..b255d89f124 100644 --- a/base/loggers/OwnSplitChannel.cpp +++ b/base/loggers/OwnSplitChannel.cpp @@ -20,10 +20,13 @@ namespace DB { void OwnSplitChannel::log(const Poco::Message & msg) { + +#ifdef WITH_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); if (channels.empty() && (logs_queue == nullptr || msg.getPriority() > logs_queue->max_priority)) return; +#endif if (auto * masker = SensitiveDataMasker::getInstance()) { @@ -86,6 +89,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg) channel.first->log(msg); // ordinary child } +#ifdef WITH_TEXT_LOG auto logs_queue = CurrentThread::getInternalTextLogsQueue(); /// Log to "TCP queue" if message is not too noisy @@ -137,6 +141,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg) if (text_log_locked) text_log_locked->add(elem); } +#endif } @@ -145,12 +150,14 @@ void OwnSplitChannel::addChannel(Poco::AutoPtr channel, const std channels.emplace(name, ExtendedChannelPtrPair(std::move(channel), dynamic_cast(channel.get()))); } +#ifdef WITH_TEXT_LOG void OwnSplitChannel::addTextLog(std::shared_ptr log, int max_priority) { std::lock_guard lock(text_log_mutex); text_log = log; text_log_max_priority.store(max_priority, std::memory_order_relaxed); } +#endif void OwnSplitChannel::setLevel(const std::string & name, int level) { diff --git a/base/loggers/OwnSplitChannel.h b/base/loggers/OwnSplitChannel.h index 364a6346ede..72027f66afd 100644 --- a/base/loggers/OwnSplitChannel.h +++ b/base/loggers/OwnSplitChannel.h @@ -7,10 +7,12 @@ #include #include "ExtendedLogChannel.h" +#ifdef WITH_TEXT_LOG namespace DB { class TextLog; } +#endif namespace DB { @@ -25,7 +27,9 @@ public: /// Adds a child channel void addChannel(Poco::AutoPtr channel, const std::string & name); +#ifdef WITH_TEXT_LOG void addTextLog(std::shared_ptr log, int max_priority); +#endif void setLevel(const std::string & name, int level); @@ -40,8 +44,10 @@ private: std::mutex text_log_mutex; +#ifdef WITH_TEXT_LOG std::weak_ptr text_log; std::atomic text_log_max_priority = -1; +#endif }; } diff --git a/benchmark/greenplum/result_parser.py b/benchmark/greenplum/result_parser.py index 8af20d265a0..4ed1aa5c4a5 100755 --- a/benchmark/greenplum/result_parser.py +++ b/benchmark/greenplum/result_parser.py @@ -4,11 +4,12 @@ import sys import json + def parse_block(block=[], options=[]): - #print('block is here', block) - #show_query = False - #show_query = options.show_query + # print('block is here', block) + # show_query = False + # show_query = options.show_query result = [] query = block[0].strip() if len(block) > 4: @@ -20,9 +21,9 @@ def parse_block(block=[], options=[]): timing2 = block[2].strip().split()[1] timing3 = block[3].strip().split()[1] if options.show_queries: - result.append( query ) + result.append(query) if not options.show_first_timings: - result += [ timing1 , timing2, timing3 ] + result += [timing1, timing2, timing3] else: result.append(timing1) return result @@ -37,12 +38,12 @@ def read_stats_file(options, fname): for line in f.readlines(): - if 'SELECT' in line: + if "SELECT" in line: if len(block) > 1: - result.append( parse_block(block, options) ) - block = [ line ] - elif 'Time:' in line: - block.append( line ) + result.append(parse_block(block, options)) + block = [line] + elif "Time:" in line: + block.append(line) return result @@ -50,7 +51,7 @@ def read_stats_file(options, fname): def compare_stats_files(options, arguments): result = [] file_output = [] - pyplot_colors = ['y', 'b', 'g', 'r'] + pyplot_colors = ["y", "b", "g", "r"] for fname in arguments[1:]: file_output.append((read_stats_file(options, fname))) if len(file_output[0]) > 0: @@ -58,65 +59,92 @@ def compare_stats_files(options, arguments): for idx, data_set in enumerate(file_output): int_result = [] for timing in data_set: - int_result.append(float(timing[0])) #y values - result.append([[x for x in range(0, len(int_result)) ], int_result, -pyplot_colors[idx] + '^' ] ) -# result.append([x for x in range(1, len(int_result)) ]) #x values -# result.append( pyplot_colors[idx] + '^' ) + int_result.append(float(timing[0])) # y values + result.append( + [ + [x for x in range(0, len(int_result))], + int_result, + pyplot_colors[idx] + "^", + ] + ) + # result.append([x for x in range(1, len(int_result)) ]) #x values + # result.append( pyplot_colors[idx] + '^' ) return result + def parse_args(): from optparse import OptionParser - parser = OptionParser(usage='usage: %prog [options] [result_file_path]..') - parser.add_option("-q", "--show-queries", help="Show statements along with timings", action="store_true", dest="show_queries") - parser.add_option("-f", "--show-first-timings", help="Show only first tries timings", action="store_true", dest="show_first_timings") - parser.add_option("-c", "--compare-mode", help="Prepare output for pyplot comparing result files.", action="store", dest="compare_mode") + + parser = OptionParser(usage="usage: %prog [options] [result_file_path]..") + parser.add_option( + "-q", + "--show-queries", + help="Show statements along with timings", + action="store_true", + dest="show_queries", + ) + parser.add_option( + "-f", + "--show-first-timings", + help="Show only first tries timings", + action="store_true", + dest="show_first_timings", + ) + parser.add_option( + "-c", + "--compare-mode", + help="Prepare output for pyplot comparing result files.", + action="store", + dest="compare_mode", + ) (options, arguments) = parser.parse_args(sys.argv) if len(arguments) < 2: parser.print_usage() sys.exit(1) - return ( options, arguments ) + return (options, arguments) + def gen_pyplot_code(options, arguments): - result = '' + result = "" data_sets = compare_stats_files(options, arguments) for idx, data_set in enumerate(data_sets, start=0): x_values, y_values, line_style = data_set - result += '\nplt.plot(' - result += '%s, %s, \'%s\'' % ( x_values, y_values, line_style ) - result += ', label=\'%s try\')' % idx - print('import matplotlib.pyplot as plt') + result += "\nplt.plot(" + result += "%s, %s, '%s'" % (x_values, y_values, line_style) + result += ", label='%s try')" % idx + print("import matplotlib.pyplot as plt") print(result) - print( 'plt.xlabel(\'Try number\')' ) - print( 'plt.ylabel(\'Timing\')' ) - print( 'plt.title(\'Benchmark query timings\')' ) - print('plt.legend()') - print('plt.show()') + print("plt.xlabel('Try number')") + print("plt.ylabel('Timing')") + print("plt.title('Benchmark query timings')") + print("plt.legend()") + print("plt.show()") def gen_html_json(options, arguments): tuples = read_stats_file(options, arguments[1]) - print('{') + print("{") print('"system: GreenPlum(x2),') - print(('"version": "%s",' % '4.3.9.1')) + print(('"version": "%s",' % "4.3.9.1")) print('"data_size": 10000000,') print('"time": "",') print('"comments": "",') print('"result":') - print('[') + print("[") for s in tuples: print(s) - print(']') - print('}') + print("]") + print("}") def main(): - ( options, arguments ) = parse_args() + (options, arguments) = parse_args() if len(arguments) > 2: gen_pyplot_code(options, arguments) else: gen_html_json(options, arguments) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 69e05cf804b..f6206d0257c 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -11,10 +11,6 @@ DATASET="${TABLE}_v1.tar.xz" QUERIES_FILE="queries.sql" TRIES=3 -AMD64_BIN_URL="https://builds.clickhouse.com/master/amd64/clickhouse" -AARCH64_BIN_URL="https://builds.clickhouse.com/master/aarch64/clickhouse" -POWERPC64_BIN_URL="https://builds.clickhouse.com/master/ppc64le/clickhouse" - # Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'. FASTER_DOWNLOAD=wget @@ -33,20 +29,60 @@ fi mkdir -p clickhouse-benchmark-$SCALE pushd clickhouse-benchmark-$SCALE -if [[ ! -f clickhouse ]]; then - CPU=$(uname -m) - if [[ ($CPU == x86_64) || ($CPU == amd64) ]]; then - $FASTER_DOWNLOAD "$AMD64_BIN_URL" - elif [[ $CPU == aarch64 ]]; then - $FASTER_DOWNLOAD "$AARCH64_BIN_URL" - elif [[ $CPU == powerpc64le ]]; then - $FASTER_DOWNLOAD "$POWERPC64_BIN_URL" - else - echo "Unsupported CPU type: $CPU" - exit 1 +OS=$(uname -s) +ARCH=$(uname -m) + +DIR= + +if [ "${OS}" = "Linux" ] +then + if [ "${ARCH}" = "x86_64" ] + then + DIR="amd64" + elif [ "${ARCH}" = "aarch64" ] + then + DIR="aarch64" + elif [ "${ARCH}" = "powerpc64le" ] + then + DIR="powerpc64le" + fi +elif [ "${OS}" = "FreeBSD" ] +then + if [ "${ARCH}" = "x86_64" ] + then + DIR="freebsd" + elif [ "${ARCH}" = "aarch64" ] + then + DIR="freebsd-aarch64" + elif [ "${ARCH}" = "powerpc64le" ] + then + DIR="freebsd-powerpc64le" + fi +elif [ "${OS}" = "Darwin" ] +then + if [ "${ARCH}" = "x86_64" ] + then + DIR="macos" + elif [ "${ARCH}" = "aarch64" -o "${ARCH}" = "arm64" ] + then + DIR="macos-aarch64" fi fi +if [ -z "${DIR}" ] +then + echo "The '${OS}' operating system with the '${ARCH}' architecture is not supported." + exit 1 +fi + +URL="https://builds.clickhouse.com/master/${DIR}/clickhouse" +echo +echo "Will download ${URL}" +echo +curl -O "${URL}" && chmod a+x clickhouse || exit 1 +echo +echo "Successfully downloaded the ClickHouse binary" + chmod a+x clickhouse if [[ ! -f $QUERIES_FILE ]]; then @@ -88,7 +124,12 @@ echo cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do sync - echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null + if [ "${OS}" = "Darwin" ] + then + sudo purge > /dev/null + else + echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null + fi echo -n "[" for i in $(seq 1 $TRIES); do @@ -104,27 +145,45 @@ echo echo "Benchmark complete. System info:" echo -echo '----Version, build id-----------' -./clickhouse local --query "SELECT format('Version: {}, build id: {}', version(), buildId())" -./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw -./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" -echo '----CPU-------------------------' -cat /proc/cpuinfo | grep -i -F 'model name' | uniq -lscpu -echo '----Block Devices---------------' -lsblk -echo '----Disk Free and Total--------' -df -h . -echo '----Memory Free and Total-------' -free -h -echo '----Physical Memory Amount------' -cat /proc/meminfo | grep MemTotal -echo '----RAID Info-------------------' -cat /proc/mdstat -#echo '----PCI-------------------------' -#lspci -#echo '----All Hardware Info-----------' -#lshw -echo '--------------------------------' - +if [ "${OS}" = "Darwin" ] +then + echo '----Version, build id-----------' + ./clickhouse local --query "SELECT format('Version: {}', version())" + sw_vers | grep BuildVersion + ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw + ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" + echo '----CPU-------------------------' + sysctl hw.model + sysctl -a | grep -E 'hw.activecpu|hw.memsize|hw.byteorder|cachesize' + echo '----Disk Free and Total--------' + df -h . + echo '----Memory Free and Total-------' + vm_stat + echo '----Physical Memory Amount------' + ls -l /var/vm + echo '--------------------------------' +else + echo '----Version, build id-----------' + ./clickhouse local --query "SELECT format('Version: {}, build id: {}', version(), buildId())" + ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw + ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" + echo '----CPU-------------------------' + cat /proc/cpuinfo | grep -i -F 'model name' | uniq + lscpu + echo '----Block Devices---------------' + lsblk + echo '----Disk Free and Total--------' + df -h . + echo '----Memory Free and Total-------' + free -h + echo '----Physical Memory Amount------' + cat /proc/meminfo | grep MemTotal + echo '----RAID Info-------------------' + cat /proc/mdstat + #echo '----PCI-------------------------' + #lspci + #echo '----All Hardware Info-----------' + #lshw + echo '--------------------------------' +fi echo diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 39aa6c88c4e..28ce5c82a92 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54459) +SET(VERSION_REVISION 54461) SET(VERSION_MAJOR 22) -SET(VERSION_MINOR 2) +SET(VERSION_MINOR 4) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH dfe64a2789bbf51046bb6b5476f874f7b59d124c) -SET(VERSION_DESCRIBE v22.2.1.1-prestable) -SET(VERSION_STRING 22.2.1.1) +SET(VERSION_GITHASH 92ab33f560e638d1989c5ca543021ab53d110f5c) +SET(VERSION_DESCRIBE v22.4.1.1-testing) +SET(VERSION_STRING 22.4.1.1) # end of autochange diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index b85260e6c76..96c6b75bc43 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -55,5 +55,5 @@ endif () if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS) message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. - Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}") + Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS} (system has ${NUMBER_OF_LOGICAL_CORES} logical cores)") endif () diff --git a/cmake/strip.sh b/cmake/strip.sh new file mode 100755 index 00000000000..f85d82fab31 --- /dev/null +++ b/cmake/strip.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +BINARY_PATH=$1 +BINARY_NAME=$(basename "$BINARY_PATH") +DESTINATION_STRIPPED_DIR=$2 +OBJCOPY_PATH=${3:objcopy} +READELF_PATH=${4:readelf} + +BUILD_ID=$($READELF_PATH -n "$1" | sed -n '/Build ID/ { s/.*: //p; q; }') +BUILD_ID_PREFIX=${BUILD_ID:0:2} +BUILD_ID_SUFFIX=${BUILD_ID:2} + +DESTINATION_DEBUG_INFO_DIR="$DESTINATION_STRIPPED_DIR/lib/debug/.build-id" +DESTINATION_STRIP_BINARY_DIR="$DESTINATION_STRIPPED_DIR/bin" + +mkdir -p "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX" +mkdir -p "$DESTINATION_STRIP_BINARY_DIR" + + +cp "$BINARY_PATH" "$DESTINATION_STRIP_BINARY_DIR/$BINARY_NAME" + +$OBJCOPY_PATH --only-keep-debug --compress-debug-sections "$DESTINATION_STRIP_BINARY_DIR/$BINARY_NAME" "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX/$BUILD_ID_SUFFIX.debug" +chmod 0644 "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX/$BUILD_ID_SUFFIX.debug" +chown 0:0 "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX/$BUILD_ID_SUFFIX.debug" + +strip --remove-section=.comment --remove-section=.note "$DESTINATION_STRIP_BINARY_DIR/$BINARY_NAME" + +$OBJCOPY_PATH --add-gnu-debuglink "$DESTINATION_DEBUG_INFO_DIR/$BUILD_ID_PREFIX/$BUILD_ID_SUFFIX.debug" "$DESTINATION_STRIP_BINARY_DIR/$BINARY_NAME" diff --git a/cmake/strip_binary.cmake b/cmake/strip_binary.cmake new file mode 100644 index 00000000000..e430807772d --- /dev/null +++ b/cmake/strip_binary.cmake @@ -0,0 +1,26 @@ +macro(clickhouse_strip_binary) + set(oneValueArgs TARGET DESTINATION_DIR BINARY_PATH) + + cmake_parse_arguments(STRIP "" "${oneValueArgs}" "" ${ARGN}) + + if (NOT DEFINED STRIP_TARGET) + message(FATAL_ERROR "A target name must be provided for stripping binary") + endif() + + if (NOT DEFINED STRIP_BINARY_PATH) + message(FATAL_ERROR "A binary path name must be provided for stripping binary") + endif() + + + if (NOT DEFINED STRIP_DESTINATION_DIR) + message(FATAL_ERROR "Destination directory for stripped binary must be provided") + endif() + + add_custom_command(TARGET ${STRIP_TARGET} POST_BUILD + COMMAND bash ${ClickHouse_SOURCE_DIR}/cmake/strip.sh ${STRIP_BINARY_PATH} ${STRIP_DESTINATION_DIR} ${OBJCOPY_PATH} ${READELF_PATH} + COMMENT "Stripping clickhouse binary" VERBATIM + ) + + install(PROGRAMS ${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET} DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + install(DIRECTORY ${STRIP_DESTINATION_DIR}/lib/debug DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) +endmacro() diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 69a37304f58..d6fddd0509e 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -169,3 +169,33 @@ if (OBJCOPY_PATH) else () message (FATAL_ERROR "Cannot find objcopy.") endif () + +# Readelf (FIXME copypaste) + +if (COMPILER_GCC) + find_program (READELF_PATH NAMES "llvm-readelf" "llvm-readelf-13" "llvm-readelf-12" "llvm-readelf-11" "readelf") +else () + find_program (READELF_PATH NAMES "llvm-readelf-${COMPILER_VERSION_MAJOR}" "llvm-readelf" "readelf") +endif () + +if (NOT READELF_PATH AND OS_DARWIN) + find_program (BREW_PATH NAMES "brew") + if (BREW_PATH) + execute_process (COMMAND ${BREW_PATH} --prefix llvm ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE LLVM_PREFIX) + if (LLVM_PREFIX) + find_program (READELF_PATH NAMES "llvm-readelf" PATHS "${LLVM_PREFIX}/bin" NO_DEFAULT_PATH) + endif () + if (NOT READELF_PATH) + execute_process (COMMAND ${BREW_PATH} --prefix binutils ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE BINUTILS_PREFIX) + if (BINUTILS_PREFIX) + find_program (READELF_PATH NAMES "readelf" PATHS "${BINUTILS_PREFIX}/bin" NO_DEFAULT_PATH) + endif () + endif () + endif () +endif () + +if (READELF_PATH) + message (STATUS "Using readelf: ${READELF_PATH}") +else () + message (FATAL_ERROR "Cannot find readelf.") +endif () diff --git a/contrib/icu b/contrib/icu index faa2f9f9e1f..a56dde820dc 160000 --- a/contrib/icu +++ b/contrib/icu @@ -1 +1 @@ -Subproject commit faa2f9f9e1fe74c5ed00eba371d2830134cdbea1 +Subproject commit a56dde820dc35665a66f2e9ee8ba58e75049b668 diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index ae19ef20e38..9c34228e2a0 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -212,7 +212,9 @@ set(ICUUC_SOURCES "${ICU_SOURCE_DIR}/common/ubiditransform.cpp" "${ICU_SOURCE_DIR}/common/pluralmap.cpp" "${ICU_SOURCE_DIR}/common/static_unicode_sets.cpp" -"${ICU_SOURCE_DIR}/common/restrace.cpp") +"${ICU_SOURCE_DIR}/common/restrace.cpp" +"${ICU_SOURCE_DIR}/common/emojiprops.cpp" +"${ICU_SOURCE_DIR}/common/lstmbe.cpp") set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/ucln_in.cpp" @@ -398,7 +400,6 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/sharedbreakiterator.cpp" "${ICU_SOURCE_DIR}/i18n/scientificnumberformatter.cpp" "${ICU_SOURCE_DIR}/i18n/dayperiodrules.cpp" -"${ICU_SOURCE_DIR}/i18n/nounit.cpp" "${ICU_SOURCE_DIR}/i18n/number_affixutils.cpp" "${ICU_SOURCE_DIR}/i18n/number_compact.cpp" "${ICU_SOURCE_DIR}/i18n/number_decimalquantity.cpp" @@ -446,12 +447,21 @@ set(ICUI18N_SOURCES "${ICU_SOURCE_DIR}/i18n/formattedvalue.cpp" "${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp" "${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp" -"${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp") +"${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp" +"${ICU_SOURCE_DIR}/i18n/measunit_extra.cpp" +"${ICU_SOURCE_DIR}/i18n/number_symbolswrapper.cpp" +"${ICU_SOURCE_DIR}/i18n/number_usageprefs.cpp" +"${ICU_SOURCE_DIR}/i18n/numrange_capi.cpp" +"${ICU_SOURCE_DIR}/i18n/pluralranges.cpp" +"${ICU_SOURCE_DIR}/i18n/units_complexconverter.cpp" +"${ICU_SOURCE_DIR}/i18n/units_converter.cpp" +"${ICU_SOURCE_DIR}/i18n/units_data.cpp" +"${ICU_SOURCE_DIR}/i18n/units_router.cpp") file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" CONTENT " ") enable_language(ASM) set(ICUDATA_SOURCES - "${ICUDATA_SOURCE_DIR}/icudt66l_dat.S" + "${ICUDATA_SOURCE_DIR}/icudt70l_dat.S" "${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" # Without this cmake can incorrectly detects library type (OBJECT) instead of SHARED/STATIC ) diff --git a/contrib/icudata b/contrib/icudata index f020820388e..72d9a4a7feb 160000 --- a/contrib/icudata +++ b/contrib/icudata @@ -1 +1 @@ -Subproject commit f020820388e3faafb44cc643574a2d563dfde572 +Subproject commit 72d9a4a7febc904e2b0a534ccb25ae40fac5f1e5 diff --git a/contrib/jemalloc b/contrib/jemalloc index a1404807211..78b58379c85 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit a1404807211b1612539f840b3dcb1bf38d1a269e +Subproject commit 78b58379c854a639df79beb3289351129d863d4b diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index b3845c7d56b..41b042df95b 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -53,43 +53,75 @@ set (SRCS "${LIBRARY_DIR}/src/background_thread.c" "${LIBRARY_DIR}/src/base.c" "${LIBRARY_DIR}/src/bin.c" + "${LIBRARY_DIR}/src/bin_info.c" "${LIBRARY_DIR}/src/bitmap.c" + "${LIBRARY_DIR}/src/buf_writer.c" + "${LIBRARY_DIR}/src/cache_bin.c" "${LIBRARY_DIR}/src/ckh.c" + "${LIBRARY_DIR}/src/counter.c" "${LIBRARY_DIR}/src/ctl.c" + "${LIBRARY_DIR}/src/decay.c" "${LIBRARY_DIR}/src/div.c" + "${LIBRARY_DIR}/src/ecache.c" + "${LIBRARY_DIR}/src/edata.c" + "${LIBRARY_DIR}/src/edata_cache.c" + "${LIBRARY_DIR}/src/ehooks.c" + "${LIBRARY_DIR}/src/emap.c" + "${LIBRARY_DIR}/src/eset.c" + "${LIBRARY_DIR}/src/exp_grow.c" "${LIBRARY_DIR}/src/extent.c" "${LIBRARY_DIR}/src/extent_dss.c" "${LIBRARY_DIR}/src/extent_mmap.c" - "${LIBRARY_DIR}/src/hash.c" + "${LIBRARY_DIR}/src/fxp.c" "${LIBRARY_DIR}/src/hook.c" + "${LIBRARY_DIR}/src/hpa.c" + "${LIBRARY_DIR}/src/hpa_hooks.c" + "${LIBRARY_DIR}/src/hpdata.c" + "${LIBRARY_DIR}/src/inspect.c" "${LIBRARY_DIR}/src/jemalloc.c" "${LIBRARY_DIR}/src/large.c" "${LIBRARY_DIR}/src/log.c" "${LIBRARY_DIR}/src/malloc_io.c" "${LIBRARY_DIR}/src/mutex.c" - "${LIBRARY_DIR}/src/mutex_pool.c" "${LIBRARY_DIR}/src/nstime.c" + "${LIBRARY_DIR}/src/pa.c" + "${LIBRARY_DIR}/src/pac.c" + "${LIBRARY_DIR}/src/pa_extra.c" "${LIBRARY_DIR}/src/pages.c" - "${LIBRARY_DIR}/src/prng.c" + "${LIBRARY_DIR}/src/pai.c" + "${LIBRARY_DIR}/src/peak_event.c" "${LIBRARY_DIR}/src/prof.c" + "${LIBRARY_DIR}/src/prof_data.c" + "${LIBRARY_DIR}/src/prof_log.c" + "${LIBRARY_DIR}/src/prof_recent.c" + "${LIBRARY_DIR}/src/prof_stats.c" + "${LIBRARY_DIR}/src/prof_sys.c" + "${LIBRARY_DIR}/src/psset.c" "${LIBRARY_DIR}/src/rtree.c" + "${LIBRARY_DIR}/src/safety_check.c" + "${LIBRARY_DIR}/src/san_bump.c" + "${LIBRARY_DIR}/src/san.c" "${LIBRARY_DIR}/src/sc.c" + "${LIBRARY_DIR}/src/sec.c" "${LIBRARY_DIR}/src/stats.c" "${LIBRARY_DIR}/src/sz.c" "${LIBRARY_DIR}/src/tcache.c" "${LIBRARY_DIR}/src/test_hooks.c" + "${LIBRARY_DIR}/src/thread_event.c" "${LIBRARY_DIR}/src/ticker.c" "${LIBRARY_DIR}/src/tsd.c" "${LIBRARY_DIR}/src/witness.c" - "${LIBRARY_DIR}/src/safety_check.c" ) if (OS_DARWIN) list(APPEND SRCS "${LIBRARY_DIR}/src/zone.c") endif () add_library(_jemalloc ${SRCS}) +# First include jemalloc-cmake files, to override anything that jemalloc has. +# (for example if you were trying to build jemalloc inside contrib/jemalloc you +# will have some files that may be out of date) +target_include_directories(_jemalloc PUBLIC include) target_include_directories(_jemalloc PRIVATE "${LIBRARY_DIR}/include") -target_include_directories(_jemalloc SYSTEM PUBLIC include) set (JEMALLOC_INCLUDE_PREFIX) # OS_ diff --git a/contrib/jemalloc-cmake/README b/contrib/jemalloc-cmake/README index 0af9c4f0e45..8d27e7844c5 100644 --- a/contrib/jemalloc-cmake/README +++ b/contrib/jemalloc-cmake/README @@ -1 +1,6 @@ It allows to integrate JEMalloc into CMake project. + +- Remove JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. +- Added JEMALLOC_CONFIG_MALLOC_CONF substitution +- Add musl support (USE_MUSL) +- Also note, that darwin build requires JEMALLOC_PREFIX, while others don not diff --git a/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h index e5e34925b55..6169b385f46 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h +++ b/contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h @@ -4,8 +4,14 @@ #include "jemalloc_internal_defs.h" #include "jemalloc/internal/jemalloc_internal_decls.h" -#ifdef JEMALLOC_UTRACE +#if defined(JEMALLOC_UTRACE) || defined(JEMALLOC_UTRACE_LABEL) #include +# if defined(JEMALLOC_UTRACE) +# define UTRACE_CALL(p, l) utrace(p, l) +# else +# define UTRACE_CALL(p, l) utrace("jemalloc_process", p, l) +# define JEMALLOC_UTRACE +# endif #endif #define JEMALLOC_NO_DEMANGLE @@ -180,6 +186,35 @@ static const bool config_opt_safety_checks = #endif ; +/* + * Extra debugging of sized deallocations too onerous to be included in the + * general safety checks. + */ +static const bool config_opt_size_checks = +#if defined(JEMALLOC_OPT_SIZE_CHECKS) || defined(JEMALLOC_DEBUG) + true +#else + false +#endif + ; + +static const bool config_uaf_detection = +#if defined(JEMALLOC_UAF_DETECTION) || defined(JEMALLOC_DEBUG) + true +#else + false +#endif + ; + +/* Whether or not the C++ extensions are enabled. */ +static const bool config_enable_cxx = +#ifdef JEMALLOC_ENABLE_CXX + true +#else + false +#endif +; + #if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA @@ -209,5 +244,20 @@ static const bool have_background_thread = false #endif ; +static const bool config_high_res_timer = +#ifdef JEMALLOC_HAVE_CLOCK_REALTIME + true +#else + false +#endif + ; + +static const bool have_memcntl = +#ifdef JEMALLOC_HAVE_MEMCNTL + true +#else + false +#endif + ; #endif /* JEMALLOC_PREAMBLE_H */ diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h index d06243c5239..64c4f4956b6 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h @@ -4,12 +4,21 @@ extern "C" { #endif +#if !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#endif + #include #include #include #include #include +#if !defined(__clang__) +#pragma GCC diagnostic pop +#endif + #ifdef __cplusplus } #endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h index 0aa4033f859..47569b16a88 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h @@ -9,11 +9,17 @@ #define JEMALLOC_HAVE_ATTR_FORMAT_ARG /* Defined if format(gnu_printf, ...) attribute is supported. */ -#define JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF +/* #undef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF */ /* Defined if format(printf, ...) attribute is supported. */ #define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF +/* Defined if fallthrough attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FALLTHROUGH + +/* Defined if cold attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_COLD + /* * Define overrides for non-standard allocator-related functions if they are * present on the system. diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h index 34235894285..a594a945351 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h @@ -4,13 +4,13 @@ #include #include -#define JEMALLOC_VERSION "5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756" +#define JEMALLOC_VERSION "5.3-RC" #define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 2 -#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_MINOR 3 +#define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "ea6b3e973b477b8061e0076bb257dbd7f3faa756" -#define JEMALLOC_VERSION_GID_IDENT ea6b3e973b477b8061e0076bb257dbd7f3faa756 +#define JEMALLOC_VERSION_GID "ca709c3139f77f4c00a903cdee46d71e9028f6c6" +#define JEMALLOC_VERSION_GID_IDENT ca709c3139f77f4c00a903cdee46d71e9028f6c6 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -71,6 +71,7 @@ # endif # define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus # define JEMALLOC_NOTHROW __declspec(nothrow) @@ -84,6 +85,7 @@ # else # define JEMALLOC_ALLOCATOR # endif +# define JEMALLOC_COLD #elif defined(JEMALLOC_HAVE_ATTR) # define JEMALLOC_ATTR(s) __attribute__((s)) # define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) @@ -109,11 +111,21 @@ # else # define JEMALLOC_FORMAT_PRINTF(s, i) # endif +# ifdef JEMALLOC_HAVE_ATTR_FALLTHROUGH +# define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough) +# else +# define JEMALLOC_FALLTHROUGH +# endif # define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) # define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) # define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) # define JEMALLOC_RESTRICT_RETURN # define JEMALLOC_ALLOCATOR +# ifdef JEMALLOC_HAVE_ATTR_COLD +# define JEMALLOC_COLD JEMALLOC_ATTR(__cold__) +# else +# define JEMALLOC_COLD +# endif #else # define JEMALLOC_ATTR(s) # define JEMALLOC_ALIGNED(s) @@ -121,9 +133,17 @@ # define JEMALLOC_ALLOC_SIZE2(s1, s2) # define JEMALLOC_EXPORT # define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_FALLTHROUGH # define JEMALLOC_NOINLINE # define JEMALLOC_NOTHROW # define JEMALLOC_SECTION(s) # define JEMALLOC_RESTRICT_RETURN # define JEMALLOC_ALLOCATOR +# define JEMALLOC_COLD +#endif + +#if (defined(__APPLE__) || defined(__FreeBSD__)) && !defined(JEMALLOC_NO_RENAME) +# define JEMALLOC_SYS_NOTHROW +#else +# define JEMALLOC_SYS_NOTHROW JEMALLOC_NOTHROW #endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h index 2e35e7b6249..e1e5b0575f3 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h @@ -4,6 +4,9 @@ # undef JEMALLOC_NOTHROW # define JEMALLOC_NOTHROW +# undef JEMALLOC_SYS_NOTHROW +# define JEMALLOC_SYS_NOTHROW + # undef JEMALLOC_CXX_THROW # define JEMALLOC_CXX_THROW #endif @@ -18,21 +21,22 @@ extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, const char *s); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_malloc(size_t size) + void JEMALLOC_SYS_NOTHROW *je_malloc(size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_calloc(size_t num, size_t size) + void JEMALLOC_SYS_NOTHROW *je_calloc(size_t num, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); -JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_posix_memalign(void **memptr, - size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW je_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_aligned_alloc(size_t alignment, + void JEMALLOC_SYS_NOTHROW *je_aligned_alloc(size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2); JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_realloc(void *ptr, size_t size) + void JEMALLOC_SYS_NOTHROW *je_realloc(void *ptr, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); -JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr) +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW je_free(void *ptr) JEMALLOC_CXX_THROW; JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN @@ -62,15 +66,19 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( const char *opts); JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_size( + const void *ptr); +#endif #ifdef JEMALLOC_OVERRIDE_MEMALIGN JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_memalign(size_t alignment, size_t size) + void JEMALLOC_SYS_NOTHROW *je_memalign(size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); #endif #ifdef JEMALLOC_OVERRIDE_VALLOC JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW + void JEMALLOC_SYS_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); #endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h new file mode 100644 index 00000000000..72182727a6f --- /dev/null +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos_jet.h @@ -0,0 +1,71 @@ +/* + * The jet_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle@install_suffix@.h). + */ +extern JEMALLOC_EXPORT const char *jet_malloc_conf; +extern JEMALLOC_EXPORT void (*jet_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_SYS_NOTHROW jet_posix_memalign( + void **memptr, size_t alignment, size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_SYS_NOTHROW jet_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *jet_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *jet_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW jet_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW jet_malloc_stats_print( + void (*write_cb)(void *, const char *), void *jet_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +#ifdef JEMALLOC_HAVE_MALLOC_SIZE +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW jet_malloc_size( + const void *ptr); +#endif + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_SYS_NOTHROW *jet_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h index 2e94f7a0cc3..9bdbcf49e61 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h @@ -13,11 +13,12 @@ # define je_mallctlnametomib mallctlnametomib # define je_malloc malloc # define je_malloc_conf malloc_conf +# define je_malloc_conf_2_conf_harder malloc_conf_2_conf_harder # define je_malloc_message malloc_message # define je_malloc_stats_print malloc_stats_print # define je_malloc_usable_size malloc_usable_size # define je_mallocx mallocx -# define je_smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 +# define je_smallocx_ca709c3139f77f4c00a903cdee46d71e9028f6c6 smallocx_ca709c3139f77f4c00a903cdee46d71e9028f6c6 # define je_nallocx nallocx # define je_posix_memalign posix_memalign # define je_rallocx rallocx diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h index 5f94f16f937..1a58874306e 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h @@ -65,13 +65,13 @@ typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, bool, unsigned); struct extent_hooks_s { - extent_alloc_t *alloc; - extent_dalloc_t *dalloc; - extent_destroy_t *destroy; - extent_commit_t *commit; - extent_decommit_t *decommit; - extent_purge_t *purge_lazy; - extent_purge_t *purge_forced; - extent_split_t *split; - extent_merge_t *merge; + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; }; diff --git a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 5c0407db24a..8ad95c51560 100644 --- a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 64 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -86,6 +86,12 @@ /* Defined if pthread_setname_np(3) is available. */ /* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -99,7 +105,12 @@ /* * Defined if mach_absolute_time() is available. */ -#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME 1 +#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME + +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -163,6 +174,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +192,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 14 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +308,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +#define JEMALLOC_HAVE_MALLOC_SIZE + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -351,7 +397,7 @@ /* #undef JEMALLOC_EXPORT */ /* config.malloc_conf options string. */ -#define JEMALLOC_CONFIG_MALLOC_CONF "" +#define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ /* #undef JEMALLOC_IS_MALLOC */ @@ -364,4 +410,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +#define JEMALLOC_HAVE_VM_MAKE_TAG + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 11fa5c4d727..8671da5db69 100644 --- a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -86,6 +86,12 @@ /* Defined if pthread_setname_np(3) is available. */ /* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -99,12 +105,12 @@ /* * Defined if mach_absolute_time() is available. */ -#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME 1 +#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME /* * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_REALTIME 1 +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -168,6 +174,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -183,6 +192,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 12 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -296,17 +308,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +#define JEMALLOC_HAVE_MALLOC_SIZE + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -369,4 +410,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +#define JEMALLOC_HAVE_VM_MAKE_TAG + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 3db0e14b268..0f61417d65f 100644 --- a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -83,9 +83,16 @@ /* Defined if pthread_atfork(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_ATFORK +/* Only since 12.1-STABLE */ /* Defined if pthread_setname_np(3) is available. */ -// Only since 12.1-STABLE -// #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ + +/* Only since 12.1-STABLE */ +/* Defined if pthread_getname_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GETNAME_NP */ + +/* Defined if pthread_get_name_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GET_NAME_NP /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. @@ -95,7 +102,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. @@ -105,7 +112,7 @@ /* * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_REALTIME 1 +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -128,7 +135,7 @@ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order * to avoid recursive allocation during mutex initialization. */ -#define JEMALLOC_MUTEX_INIT_CB 1 +#define JEMALLOC_MUTEX_INIT_CB /* Non-empty if the tls_model attribute is supported. */ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) @@ -161,7 +168,7 @@ * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage * segment (DSS). */ -/* #undef JEMALLOC_DSS */ +#define JEMALLOC_DSS /* Support memory filling (junk/zero). */ #define JEMALLOC_FILL @@ -169,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -184,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -297,17 +310,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_NOCORE + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -348,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -360,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -370,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index dbf55f3f6e0..32cad025f5f 100644 --- a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -45,17 +45,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -83,9 +83,16 @@ /* Defined if pthread_atfork(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_ATFORK +/* Only since 12.1-STABLE */ /* Defined if pthread_setname_np(3) is available. */ -// Only since 12.1-STABLE -// #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* #undef JEMALLOC_HAVE_PTHREAD_SETNAME_NP */ + +/* Only since 12.1-STABLE */ +/* Defined if pthread_getname_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GETNAME_NP */ + +/* Defined if pthread_get_name_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GET_NAME_NP /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. @@ -95,7 +102,7 @@ /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. @@ -105,7 +112,7 @@ /* * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_REALTIME 1 +#define JEMALLOC_HAVE_CLOCK_REALTIME /* * Defined if _malloc_thread_cleanup() exists. At least in the case of @@ -128,7 +135,7 @@ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order * to avoid recursive allocation during mutex initialization. */ -#define JEMALLOC_MUTEX_INIT_CB 1 +#define JEMALLOC_MUTEX_INIT_CB /* Non-empty if the tls_model attribute is supported. */ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) @@ -169,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -184,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 12 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -297,17 +310,46 @@ */ /* #undef JEMALLOC_MADVISE_DONTDUMP */ +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_NOCORE + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ /* #undef JEMALLOC_HAS_ALLOCA_H */ /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -348,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -360,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -370,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/README b/contrib/jemalloc-cmake/include_linux_aarch64/README deleted file mode 100644 index 3cecf7fa36d..00000000000 --- a/contrib/jemalloc-cmake/include_linux_aarch64/README +++ /dev/null @@ -1,8 +0,0 @@ -Here are pre-generated files from jemalloc on Linux aarch64. -You can obtain these files by running ./autogen.sh inside jemalloc source directory. - -Added #define GNU_SOURCE -Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. -Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. -Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. -Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 5e0135cc0d0..ad535e6d773 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -13,12 +13,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#define JEMALLOC_OVERRIDE___LIBC_CALLOC -#define JEMALLOC_OVERRIDE___LIBC_FREE -#define JEMALLOC_OVERRIDE___LIBC_MALLOC -#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN -#define JEMALLOC_OVERRIDE___LIBC_REALLOC -#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#if !defined(USE_MUSL) + #define JEMALLOC_OVERRIDE___LIBC_CALLOC + #define JEMALLOC_OVERRIDE___LIBC_FREE + #define JEMALLOC_OVERRIDE___LIBC_MALLOC + #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN + #define JEMALLOC_OVERRIDE___LIBC_REALLOC + #define JEMALLOC_OVERRIDE___LIBC_VALLOC +#endif /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* @@ -45,17 +47,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -73,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -86,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -163,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -342,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -354,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -364,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 97d0d4d8471..12890f80ef1 100644 --- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -13,12 +13,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#define JEMALLOC_OVERRIDE___LIBC_CALLOC -#define JEMALLOC_OVERRIDE___LIBC_FREE -#define JEMALLOC_OVERRIDE___LIBC_MALLOC -#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN -#define JEMALLOC_OVERRIDE___LIBC_REALLOC -#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#if !defined(USE_MUSL) + #define JEMALLOC_OVERRIDE___LIBC_CALLOC + #define JEMALLOC_OVERRIDE___LIBC_FREE + #define JEMALLOC_OVERRIDE___LIBC_MALLOC + #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN + #define JEMALLOC_OVERRIDE___LIBC_REALLOC + #define JEMALLOC_OVERRIDE___LIBC_VALLOC +#endif /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* @@ -45,17 +47,17 @@ #define LG_VADDR 64 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -73,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -86,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -163,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -342,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -354,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -364,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/README b/contrib/jemalloc-cmake/include_linux_riscv64/README deleted file mode 100644 index 01b65655c55..00000000000 --- a/contrib/jemalloc-cmake/include_linux_riscv64/README +++ /dev/null @@ -1,8 +0,0 @@ -Here are pre-generated files from jemalloc on Linux risc-v. -You can obtain these files by running ./autogen.sh inside jemalloc source directory. - -Added #define GNU_SOURCE -Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. -Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. -Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. -Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in index 5e0135cc0d0..ad535e6d773 100644 --- a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -13,12 +13,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#define JEMALLOC_OVERRIDE___LIBC_CALLOC -#define JEMALLOC_OVERRIDE___LIBC_FREE -#define JEMALLOC_OVERRIDE___LIBC_MALLOC -#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN -#define JEMALLOC_OVERRIDE___LIBC_REALLOC -#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#if !defined(USE_MUSL) + #define JEMALLOC_OVERRIDE___LIBC_CALLOC + #define JEMALLOC_OVERRIDE___LIBC_FREE + #define JEMALLOC_OVERRIDE___LIBC_MALLOC + #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN + #define JEMALLOC_OVERRIDE___LIBC_REALLOC + #define JEMALLOC_OVERRIDE___LIBC_VALLOC +#endif /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* @@ -45,17 +47,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -73,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -86,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -163,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -178,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 16 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -291,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -342,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -354,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -364,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/README b/contrib/jemalloc-cmake/include_linux_x86_64/README deleted file mode 100644 index 8b93e0d4dcd..00000000000 --- a/contrib/jemalloc-cmake/include_linux_x86_64/README +++ /dev/null @@ -1,8 +0,0 @@ -Here are pre-generated files from jemalloc on Linux x86_64. -You can obtain these files by running ./autogen.sh inside jemalloc source directory. - -Added #define GNU_SOURCE -Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. -Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. -Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. -Added JEMALLOC_CONFIG_MALLOC_CONF substitution diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 44ff2d9fad1..99ab2d53ca9 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -47,17 +47,17 @@ #define LG_VADDR 48 /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11_ATOMICS 1 +#define JEMALLOC_C11_ATOMICS /* Defined if GCC __atomic atomics are available. */ -#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_ATOMIC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ -#define JEMALLOC_GCC_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_SYNC_ATOMICS /* and the 8-bit variant support. */ -#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 +#define JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -75,7 +75,7 @@ /* * Defined if secure_getenv(3) is available. */ -// #define JEMALLOC_HAVE_SECURE_GETENV +/* #undef JEMALLOC_HAVE_SECURE_GETENV */ /* * Defined if issetugid(2) is available. @@ -88,21 +88,32 @@ /* Defined if pthread_setname_np(3) is available. */ #define JEMALLOC_HAVE_PTHREAD_SETNAME_NP +/* Defined if pthread_getname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_GETNAME_NP + +/* Defined if pthread_get_name_np(3) is available. */ +/* #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP */ + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE /* * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. */ -#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 +#define JEMALLOC_HAVE_CLOCK_MONOTONIC /* * Defined if mach_absolute_time() is available. */ /* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ +/* + * Defined if clock_gettime(CLOCK_REALTIME, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_REALTIME + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc @@ -165,6 +176,9 @@ /* Support utrace(2)-based tracing. */ /* #undef JEMALLOC_UTRACE */ +/* Support utrace(2)-based tracing (label based signature). */ +/* #undef JEMALLOC_UTRACE_LABEL */ + /* Support optional abort() on OOM. */ /* #undef JEMALLOC_XMALLOC */ @@ -180,6 +194,9 @@ /* One page is 2^LG_PAGE bytes. */ #define LG_PAGE 12 +/* Maximum number of regions in a slab. */ +/* #undef CONFIG_LG_SLAB_MAXREGS */ + /* * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the * system does not explicitly support huge pages; system calls that require @@ -293,17 +310,46 @@ */ #define JEMALLOC_MADVISE_DONTDUMP +/* + * Defined if MADV_[NO]CORE is supported as an argument to madvise. + */ +/* #undef JEMALLOC_MADVISE_NOCORE */ + +/* Defined if mprotect(2) is available. */ +#define JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. */ /* #undef JEMALLOC_THP */ +/* Defined if posix_madvise is available. */ +/* #undef JEMALLOC_HAVE_POSIX_MADVISE */ + +/* + * Method for purging unused pages using posix_madvise. + * + * posix_madvise(..., POSIX_MADV_DONTNEED) + */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED */ +/* #undef JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS */ + +/* + * Defined if memcntl page admin call is supported + */ +/* #undef JEMALLOC_HAVE_MEMCNTL */ + +/* + * Defined if malloc_size is supported + */ +/* #undef JEMALLOC_HAVE_MALLOC_SIZE */ + /* Define if operating system has alloca.h header. */ -#define JEMALLOC_HAS_ALLOCA_H 1 +#define JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ -#define JEMALLOC_HAS_RESTRICT 1 +#define JEMALLOC_HAS_RESTRICT /* For use by hash code. */ /* #undef JEMALLOC_BIG_ENDIAN */ @@ -344,7 +390,7 @@ /* * If defined, all the features necessary for background threads are present. */ -#define JEMALLOC_BACKGROUND_THREAD 1 +#define JEMALLOC_BACKGROUND_THREAD /* * If defined, jemalloc symbols are not exported (doesn't work when @@ -356,7 +402,7 @@ #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@" /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ -#define JEMALLOC_IS_MALLOC 1 +#define JEMALLOC_IS_MALLOC /* * Defined if strerror_r returns char * if _GNU_SOURCE is defined. @@ -366,4 +412,16 @@ /* Performs additional safety checks when defined. */ /* #undef JEMALLOC_OPT_SAFETY_CHECKS */ +/* Is C++ support being built? */ +/* #undef JEMALLOC_ENABLE_CXX */ + +/* Performs additional size checks when defined. */ +/* #undef JEMALLOC_OPT_SIZE_CHECKS */ + +/* Allows sampled junk and stash for checking use-after-free when defined. */ +/* #undef JEMALLOC_UAF_DETECTION */ + +/* Darwin VM_MAKE_TAG support */ +/* #undef JEMALLOC_HAVE_VM_MAKE_TAG */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/llvm-cmake/CMakeLists.txt b/contrib/llvm-cmake/CMakeLists.txt index d240924cac3..6ff07f0e016 100644 --- a/contrib/llvm-cmake/CMakeLists.txt +++ b/contrib/llvm-cmake/CMakeLists.txt @@ -1,4 +1,8 @@ -if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") +# During cross-compilation in our CI we have to use llvm-tblgen and other building tools +# tools to be build for host architecture and everything else for target architecture (e.g. AArch64) +# Possible workaround is to use llvm-tblgen from some package... +# But lets just enable LLVM for native builds +if (CMAKE_CROSSCOMPILING OR SANITIZE STREQUAL "undefined") set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF) else() set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON) @@ -6,7 +10,7 @@ endif() option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT}) if (NOT ENABLE_EMBEDDED_COMPILER) - set (USE_EMBEDDED_COMPILER 0) + message(STATUS "Not using LLVM") return() endif() @@ -22,9 +26,6 @@ set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm") set (REQUIRED_LLVM_LIBRARIES LLVMExecutionEngine LLVMRuntimeDyld - LLVMX86CodeGen - LLVMX86Desc - LLVMX86Info LLVMAsmPrinter LLVMDebugInfoDWARF LLVMGlobalISel @@ -56,6 +57,12 @@ set (REQUIRED_LLVM_LIBRARIES LLVMDemangle ) +if (ARCH_AMD64) + list(APPEND REQUIRED_LLVM_LIBRARIES LLVMX86Info LLVMX86Desc LLVMX86CodeGen) +elseif (ARCH_AARCH64) + list(APPEND REQUIRED_LLVM_LIBRARIES LLVMAArch64Info LLVMAArch64Desc LLVMAArch64CodeGen) +endif () + #function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) # llvm_map_components_to_libnames (result all) # if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result) diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt index e74f488643d..19939c11ebf 100644 --- a/contrib/re2-cmake/CMakeLists.txt +++ b/contrib/re2-cmake/CMakeLists.txt @@ -71,7 +71,6 @@ foreach (FILENAME mutex.h) add_dependencies (re2_st transform_${FILENAME}) endforeach () -# NOTE: you should not change name of library here, since it is used for PVS -# (see docker/test/pvs/Dockerfile), to generate required header (see above) +# NOTE: you should not change name of library here, since it is used to generate required header (see above) add_library(ch_contrib::re2 ALIAS re2) add_library(ch_contrib::re2_st ALIAS re2_st) diff --git a/debian/.gitignore b/debian/.gitignore index 7d927d3a70b..b4432556de7 100644 --- a/debian/.gitignore +++ b/debian/.gitignore @@ -10,7 +10,6 @@ clickhouse-common-static/ clickhouse-server-base/ clickhouse-server-common/ clickhouse-server/ -clickhouse-test/ debhelper-build-stamp files *.debhelper.log diff --git a/debian/clickhouse-test.install b/debian/clickhouse-test.install deleted file mode 100644 index 042a4e02be1..00000000000 --- a/debian/clickhouse-test.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/bin/clickhouse-test -usr/share/clickhouse-test/* diff --git a/debian/control b/debian/control index f22980fdbc4..c5d98d98f41 100644 --- a/debian/control +++ b/debian/control @@ -56,9 +56,3 @@ Replaces: clickhouse-common-dbg Conflicts: clickhouse-common-dbg Description: debugging symbols for clickhouse-common-static This package contains the debugging symbols for clickhouse-common. - -Package: clickhouse-test -Priority: optional -Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-client, bash, expect, python3, python3-lxml, python3-termcolor, python3-requests, curl, perl, sudo, openssl, netcat-openbsd, telnet, brotli, bsdutils -Description: ClickHouse tests diff --git a/docker/images.json b/docker/images.json index 01284d4de69..06d689e8f7c 100644 --- a/docker/images.json +++ b/docker/images.json @@ -7,7 +7,6 @@ "name": "clickhouse/binary-builder", "dependent": [ "docker/test/split_build_smoke_test", - "docker/test/pvs", "docker/test/codebrowser" ] }, @@ -31,11 +30,6 @@ "name": "clickhouse/performance-comparison", "dependent": [] }, - "docker/test/pvs": { - "only_amd64": true, - "name": "clickhouse/pvs-test", - "dependent": [] - }, "docker/test/util": { "name": "clickhouse/test-util", "dependent": [ diff --git a/docker/packager/README.md b/docker/packager/README.md index a745f6225fa..a78feb8d7fc 100644 --- a/docker/packager/README.md +++ b/docker/packager/README.md @@ -3,26 +3,25 @@ compilers and build settings. Correctly configured Docker daemon is single depen Usage: -Build deb package with `clang-11` in `debug` mode: +Build deb package with `clang-14` in `debug` mode: ``` $ mkdir deb/test_output -$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=clang-11 --build-type=debug +$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=clang-14 --build-type=debug $ ls -l deb/test_output --rw-r--r-- 1 root root 3730 clickhouse-client_18.14.2+debug_all.deb --rw-r--r-- 1 root root 84221888 clickhouse-common-static_18.14.2+debug_amd64.deb --rw-r--r-- 1 root root 255967314 clickhouse-common-static-dbg_18.14.2+debug_amd64.deb --rw-r--r-- 1 root root 14940 clickhouse-server_18.14.2+debug_all.deb --rw-r--r-- 1 root root 340206010 clickhouse-server-base_18.14.2+debug_amd64.deb --rw-r--r-- 1 root root 7900 clickhouse-server-common_18.14.2+debug_all.deb --rw-r--r-- 1 root root 2880432 clickhouse-test_18.14.2+debug_all.deb +-rw-r--r-- 1 root root 3730 clickhouse-client_22.2.2+debug_all.deb +-rw-r--r-- 1 root root 84221888 clickhouse-common-static_22.2.2+debug_amd64.deb +-rw-r--r-- 1 root root 255967314 clickhouse-common-static-dbg_22.2.2+debug_amd64.deb +-rw-r--r-- 1 root root 14940 clickhouse-server_22.2.2+debug_all.deb +-rw-r--r-- 1 root root 340206010 clickhouse-server-base_22.2.2+debug_amd64.deb +-rw-r--r-- 1 root root 7900 clickhouse-server-common_22.2.2+debug_all.deb ``` -Build ClickHouse binary with `clang-11` and `address` sanitizer in `relwithdebuginfo` +Build ClickHouse binary with `clang-14` and `address` sanitizer in `relwithdebuginfo` mode: ``` $ mkdir $HOME/some_clickhouse -$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-11 --sanitizer=address +$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-14 --sanitizer=address $ ls -l $HOME/some_clickhouse -rwxr-xr-x 1 root root 787061952 clickhouse lrwxrwxrwx 1 root root 10 clickhouse-benchmark -> clickhouse diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index e3e2e689b17..a57a734e3df 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -95,6 +95,14 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ && apt-get install gcc-11 g++-11 --yes \ && apt-get clean +# Architecture of the image when BuildKit/buildx is used +ARG TARGETARCH +ARG NFPM_VERSION=2.15.0 + +RUN arch=${TARGETARCH:-amd64} \ + && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \ + && dpkg -i /tmp/nfpm.deb \ + && rm /tmp/nfpm.deb COPY build.sh / -CMD ["bash", "-c", "/build.sh 2>&1 | ts"] +CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 2f18b07ffe1..31416e1a0ee 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash +exec &> >(ts) set -x -e +cache_status () { + ccache --show-config ||: + ccache --show-stats ||: +} + mkdir -p build/cmake/toolchain/darwin-x86_64 tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1 ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64 @@ -19,15 +25,23 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}" env cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. -ccache --show-config ||: -ccache --show-stats ||: +cache_status +# clear cache stats ccache --zero-stats ||: -# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. +# No quotes because I want it to expand to nothing if empty. +# shellcheck disable=SC2086 ninja $NINJA_FLAGS clickhouse-bundle -ccache --show-config ||: -ccache --show-stats ||: +cache_status + +if [ -n "$MAKE_DEB" ]; then + rm -rf /build/packages/root + # No quotes because I want it to expand to nothing if empty. + # shellcheck disable=SC2086 + DESTDIR=/build/packages/root ninja $NINJA_FLAGS install + bash -x /build/packages/build +fi mv ./programs/clickhouse* /output mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds @@ -84,8 +98,7 @@ fi # ../docker/packager/other/fuzzer.sh # fi -ccache --show-config ||: -ccache --show-stats ||: +cache_status if [ "${CCACHE_DEBUG:-}" == "1" ] then diff --git a/docker/packager/packager b/docker/packager/packager index 05b2e02df96..a5763273f5f 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- import subprocess import os import argparse @@ -8,36 +8,39 @@ import sys SCRIPT_PATH = os.path.realpath(__file__) -IMAGE_MAP = { - "deb": "clickhouse/deb-builder", - "binary": "clickhouse/binary-builder", -} def check_image_exists_locally(image_name): try: - output = subprocess.check_output("docker images -q {} 2> /dev/null".format(image_name), shell=True) + output = subprocess.check_output( + f"docker images -q {image_name} 2> /dev/null", shell=True + ) return output != "" - except subprocess.CalledProcessError as ex: + except subprocess.CalledProcessError: return False + def pull_image(image_name): try: - subprocess.check_call("docker pull {}".format(image_name), shell=True) + subprocess.check_call(f"docker pull {image_name}", shell=True) return True - except subprocess.CalledProcessError as ex: - logging.info("Cannot pull image {}".format(image_name)) + except subprocess.CalledProcessError: + logging.info(f"Cannot pull image {image_name}".format()) return False + def build_image(image_name, filepath): context = os.path.dirname(filepath) - build_cmd = "docker build --network=host -t {} -f {} {}".format(image_name, filepath, context) - logging.info("Will build image with cmd: '{}'".format(build_cmd)) + build_cmd = f"docker build --network=host -t {image_name} -f {filepath} {context}" + logging.info("Will build image with cmd: '%s'", build_cmd) subprocess.check_call( build_cmd, shell=True, ) -def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir, docker_image_version): + +def run_docker_image_with_env( + image_name, output, env_variables, ch_root, ccache_dir, docker_image_version +): env_part = " -e ".join(env_variables) if env_part: env_part = " -e " + env_part @@ -47,28 +50,52 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache else: interactive = "" - cmd = "docker run --network=host --rm --volume={output_path}:/output --volume={ch_root}:/build --volume={ccache_dir}:/ccache {env} {interactive} {img_name}".format( - output_path=output, - ch_root=ch_root, - ccache_dir=ccache_dir, - env=env_part, - img_name=image_name + ":" + docker_image_version, - interactive=interactive + cmd = ( + f"docker run --network=host --rm --volume={output}:/output " + f"--volume={ch_root}:/build --volume={ccache_dir}:/ccache {env_part} " + f"{interactive} {image_name}:{docker_image_version}" ) - logging.info("Will build ClickHouse pkg with cmd: '{}'".format(cmd)) + logging.info("Will build ClickHouse pkg with cmd: '%s'", cmd) subprocess.check_call(cmd, shell=True) -def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries): + +def is_release_build(build_type, package_type, sanitizer, split_binary): + return ( + build_type == "" + and package_type == "deb" + and sanitizer == "" + and not split_binary + ) + + +def parse_env_variables( + build_type, + compiler, + sanitizer, + package_type, + image_type, + cache, + distcc_hosts, + split_binary, + clang_tidy, + version, + author, + official, + additional_pkgs, + with_coverage, + with_binaries, +): DARWIN_SUFFIX = "-darwin" DARWIN_ARM_SUFFIX = "-darwin-aarch64" ARM_SUFFIX = "-aarch64" FREEBSD_SUFFIX = "-freebsd" - PPC_SUFFIX = '-ppc64le' + PPC_SUFFIX = "-ppc64le" result = [] - cmake_flags = ['$CMAKE_FLAGS'] + result.append("OUTPUT_DIR=/output") + cmake_flags = ["$CMAKE_FLAGS"] is_cross_darwin = compiler.endswith(DARWIN_SUFFIX) is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX) @@ -77,46 +104,72 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX) if is_cross_darwin: - cc = compiler[:-len(DARWIN_SUFFIX)] + cc = compiler[: -len(DARWIN_SUFFIX)] cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar") - cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/x86_64-apple-darwin-install_name_tool") - cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib") + cmake_flags.append( + "-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/" + "x86_64-apple-darwin-install_name_tool" + ) + cmake_flags.append( + "-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib" + ) cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld") - cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake") + cmake_flags.append( + "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake" + ) elif is_cross_darwin_arm: - cc = compiler[:-len(DARWIN_ARM_SUFFIX)] + cc = compiler[: -len(DARWIN_ARM_SUFFIX)] cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar") - cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/aarch64-apple-darwin-install_name_tool") - cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib") + cmake_flags.append( + "-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/" + "aarch64-apple-darwin-install_name_tool" + ) + cmake_flags.append( + "-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib" + ) cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld") - cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake") + cmake_flags.append( + "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake" + ) elif is_cross_arm: - cc = compiler[:-len(ARM_SUFFIX)] - cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake") - result.append("DEB_ARCH_FLAG=-aarm64") + cc = compiler[: -len(ARM_SUFFIX)] + cmake_flags.append( + "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake" + ) + result.append("DEB_ARCH=arm64") elif is_cross_freebsd: - cc = compiler[:-len(FREEBSD_SUFFIX)] - cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake") + cc = compiler[: -len(FREEBSD_SUFFIX)] + cmake_flags.append( + "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake" + ) elif is_cross_ppc: - cc = compiler[:-len(PPC_SUFFIX)] - cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake") + cc = compiler[: -len(PPC_SUFFIX)] + cmake_flags.append( + "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake" + ) else: cc = compiler - result.append("DEB_ARCH_FLAG=-aamd64") + result.append("DEB_ARCH=amd64") - cxx = cc.replace('gcc', 'g++').replace('clang', 'clang++') + cxx = cc.replace("gcc", "g++").replace("clang", "clang++") if image_type == "deb": - result.append("DEB_CC={}".format(cc)) - result.append("DEB_CXX={}".format(cxx)) - # For building fuzzers - result.append("CC={}".format(cc)) - result.append("CXX={}".format(cxx)) - elif image_type == "binary": - result.append("CC={}".format(cc)) - result.append("CXX={}".format(cxx)) - cmake_flags.append('-DCMAKE_C_COMPILER=`which {}`'.format(cc)) - cmake_flags.append('-DCMAKE_CXX_COMPILER=`which {}`'.format(cxx)) + result.append("MAKE_DEB=true") + cmake_flags.append("-DENABLE_TESTS=0") + cmake_flags.append("-DENABLE_UTILS=0") + cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON") + cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON") + cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON") + cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") + cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") + cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") + if is_release_build(build_type, package_type, sanitizer, split_binary): + cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON") + + result.append(f"CC={cc}") + result.append(f"CXX={cxx}") + cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}") + cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}") # Create combined output archive for split build and for performance tests. if package_type == "performance": @@ -126,12 +179,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ result.append("COMBINED_OUTPUT=shared_build") if sanitizer: - result.append("SANITIZER={}".format(sanitizer)) + result.append(f"SANITIZER={sanitizer}") if build_type: - result.append("BUILD_TYPE={}".format(build_type)) + result.append(f"BUILD_TYPE={build_type.capitalize()}") + else: + result.append("BUILD_TYPE=None") - if cache == 'distcc': - result.append("CCACHE_PREFIX={}".format(cache)) + if cache == "distcc": + result.append(f"CCACHE_PREFIX={cache}") if cache: result.append("CCACHE_DIR=/ccache") @@ -142,109 +197,188 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ # result.append("CCACHE_UMASK=777") if distcc_hosts: - hosts_with_params = ["{}/24,lzo".format(host) for host in distcc_hosts] + ["localhost/`nproc`"] - result.append('DISTCC_HOSTS="{}"'.format(" ".join(hosts_with_params))) + hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [ + "localhost/`nproc`" + ] + result.append('DISTCC_HOSTS="' + " ".join(hosts_with_params) + '"') elif cache == "distcc": - result.append('DISTCC_HOSTS="{}"'.format("localhost/`nproc`")) + result.append('DISTCC_HOSTS="localhost/`nproc`"') - if alien_pkgs: - result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") + if additional_pkgs: + result.append("MAKE_APK=true") + result.append("MAKE_RPM=true") + result.append("MAKE_TGZ=true") if with_binaries == "programs": - result.append('BINARY_OUTPUT=programs') + result.append("BINARY_OUTPUT=programs") elif with_binaries == "tests": - result.append('ENABLE_TESTS=1') - result.append('BINARY_OUTPUT=tests') - cmake_flags.append('-DENABLE_TESTS=1') + result.append("ENABLE_TESTS=1") + result.append("BINARY_OUTPUT=tests") + cmake_flags.append("-DENABLE_TESTS=1") if split_binary: - cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') + cmake_flags.append( + "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 " + "-DCLICKHOUSE_SPLIT_BINARY=1" + ) # We can't always build utils because it requires too much space, but # we have to build them at least in some way in CI. The split build is # probably the least heavy disk-wise. - cmake_flags.append('-DENABLE_UTILS=1') + cmake_flags.append("-DENABLE_UTILS=1") if clang_tidy: - cmake_flags.append('-DENABLE_CLANG_TIDY=1') - cmake_flags.append('-DENABLE_UTILS=1') - cmake_flags.append('-DENABLE_TESTS=1') - cmake_flags.append('-DENABLE_EXAMPLES=1') + cmake_flags.append("-DENABLE_CLANG_TIDY=1") + cmake_flags.append("-DENABLE_UTILS=1") + cmake_flags.append("-DENABLE_TESTS=1") + cmake_flags.append("-DENABLE_EXAMPLES=1") # Don't stop on first error to find more clang-tidy errors in one run. - result.append('NINJA_FLAGS=-k0') + result.append("NINJA_FLAGS=-k0") if with_coverage: - cmake_flags.append('-DWITH_COVERAGE=1') + cmake_flags.append("-DWITH_COVERAGE=1") if version: - result.append("VERSION_STRING='{}'".format(version)) + result.append(f"VERSION_STRING='{version}'") if author: - result.append("AUTHOR='{}'".format(author)) + result.append(f"AUTHOR='{author}'") if official: - cmake_flags.append('-DYANDEX_OFFICIAL_BUILD=1') + cmake_flags.append("-DYANDEX_OFFICIAL_BUILD=1") - result.append('CMAKE_FLAGS="' + ' '.join(cmake_flags) + '"') + result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"') return result + if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse building script using prebuilt Docker image") - # 'performance' creates a combined .tgz with server and configs to be used for performance test. - parser.add_argument("--package-type", choices=['deb', 'binary', 'performance'], required=True) - parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="ClickHouse building script using prebuilt Docker image", + ) + # 'performance' creates a combined .tgz with server + # and configs to be used for performance test. + parser.add_argument( + "--package-type", + choices=("deb", "binary", "performance"), + required=True, + help="a build type", + ) + parser.add_argument( + "--clickhouse-repo-path", + default=os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir + ), + help="ClickHouse git repository", + ) parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64", - "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64", - "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", "clang-13-ppc64le", - "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13") - parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") + parser.add_argument( + "--compiler", + choices=( + "clang-11", + "clang-11-darwin", + "clang-11-darwin-aarch64", + "clang-11-aarch64", + "clang-12", + "clang-12-darwin", + "clang-12-darwin-aarch64", + "clang-12-aarch64", + "clang-13", + "clang-13-darwin", + "clang-13-darwin-aarch64", + "clang-13-aarch64", + "clang-13-ppc64le", + "clang-11-freebsd", + "clang-12-freebsd", + "clang-13-freebsd", + "gcc-11", + ), + default="clang-13", + help="a compiler to use", + ) + parser.add_argument( + "--sanitizer", + choices=("address", "thread", "memory", "undefined", ""), + default="", + ) parser.add_argument("--split-binary", action="store_true") parser.add_argument("--clang-tidy", action="store_true") - parser.add_argument("--cache", choices=("", "ccache", "distcc"), default="") - parser.add_argument("--ccache_dir", default= os.getenv("HOME", "") + '/.ccache') + parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="") + parser.add_argument( + "--ccache_dir", + default=os.getenv("HOME", "") + "/.ccache", + help="a directory with ccache", + ) parser.add_argument("--distcc-hosts", nargs="+") parser.add_argument("--force-build-image", action="store_true") parser.add_argument("--version") - parser.add_argument("--author", default="clickhouse") + parser.add_argument("--author", default="clickhouse", help="a package author") parser.add_argument("--official", action="store_true") - parser.add_argument("--alien-pkgs", nargs='+', default=[]) + parser.add_argument("--additional-pkgs", action="store_true") parser.add_argument("--with-coverage", action="store_true") - parser.add_argument("--with-binaries", choices=("programs", "tests", ""), default="") - parser.add_argument("--docker-image-version", default="latest") + parser.add_argument( + "--with-binaries", choices=("programs", "tests", ""), default="" + ) + parser.add_argument( + "--docker-image-version", default="latest", help="docker image tag to use" + ) args = parser.parse_args() if not os.path.isabs(args.output_dir): args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir)) - image_type = 'binary' if args.package_type == 'performance' else args.package_type - image_name = IMAGE_MAP[image_type] + image_type = "binary" if args.package_type == "performance" else args.package_type + image_name = "clickhouse/binary-builder" if not os.path.isabs(args.clickhouse_repo_path): ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path)) else: ch_root = args.clickhouse_repo_path - if args.alien_pkgs and not image_type == "deb": - raise Exception("Can add alien packages only in deb build") + if args.additional_pkgs and image_type != "deb": + raise Exception("Can build additional packages only in deb build") - if args.with_binaries != "" and not image_type == "deb": + if args.with_binaries != "" and image_type != "deb": raise Exception("Can add additional binaries only in deb build") if args.with_binaries != "" and image_type == "deb": - logging.info("Should place {} to output".format(args.with_binaries)) + logging.info("Should place %s to output", args.with_binaries) dockerfile = os.path.join(ch_root, "docker/packager", image_type, "Dockerfile") image_with_version = image_name + ":" + args.docker_image_version - if image_type != "freebsd" and not check_image_exists_locally(image_name) or args.force_build_image: + if ( + image_type != "freebsd" + and not check_image_exists_locally(image_name) + or args.force_build_image + ): if not pull_image(image_with_version) or args.force_build_image: build_image(image_with_version, dockerfile) env_prepared = parse_env_variables( - args.build_type, args.compiler, args.sanitizer, args.package_type, image_type, - args.cache, args.distcc_hosts, args.split_binary, args.clang_tidy, - args.version, args.author, args.official, args.alien_pkgs, args.with_coverage, args.with_binaries) + args.build_type, + args.compiler, + args.sanitizer, + args.package_type, + image_type, + args.cache, + args.distcc_hosts, + args.split_binary, + args.clang_tidy, + args.version, + args.author, + args.official, + args.additional_pkgs, + args.with_coverage, + args.with_binaries, + ) - run_docker_image_with_env(image_name, args.output_dir, env_prepared, ch_root, args.ccache_dir, args.docker_image_version) - logging.info("Output placed into {}".format(args.output_dir)) + run_docker_image_with_env( + image_name, + args.output_dir, + env_prepared, + ch_root, + args.ccache_dir, + args.docker_image_version, + ) + logging.info("Output placed into %s", args.output_dir) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 5b10d1fc490..5b7990ab030 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:20.04 ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/" +ARG repository="deb https://packages.clickhouse.com/deb stable main" ARG version=22.1.1.* # set non-empty deb_location_url url to create a docker image @@ -58,7 +58,7 @@ RUN groupadd -r clickhouse --gid=101 \ wget \ tzdata \ && mkdir -p /etc/apt/sources.list.d \ - && apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv 8919F6BD2B48D754 \ && echo $repository > /etc/apt/sources.list.d/clickhouse.list \ && if [ -n "$deb_location_url" ]; then \ echo "installing from custom url with deb packages: $deb_location_url" \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile deleted file mode 100644 index 66d1afb309b..00000000000 --- a/docker/test/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:18.04 - -ARG repository="deb https://repo.clickhouse.com/deb/stable/ main/" -ARG version=22.1.1.* - -RUN apt-get update && \ - apt-get install -y apt-transport-https dirmngr && \ - mkdir -p /etc/apt/sources.list.d && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 && \ - echo $repository | tee /etc/apt/sources.list.d/clickhouse.list && \ - apt-get update && \ - env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y clickhouse-test && \ - rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ - apt-get clean - -ENTRYPOINT ["/usr/bin/clickhouse-test"] diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 24168cea330..bd1e0292636 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -263,9 +263,20 @@ function run_tests if [[ $NPROC == 0 ]]; then NPROC=1 fi - time clickhouse-test --hung-check -j "${NPROC}" --order=random \ - --fast-tests-only --no-long --testname --shard --zookeeper --check-zookeeper-session \ - -- "$FASTTEST_FOCUS" 2>&1 \ + + local test_opts=( + --hung-check + --fast-tests-only + --no-long + --testname + --shard + --zookeeper + --check-zookeeper-session + --order random + --print-time + --jobs "${NPROC}" + ) + time clickhouse-test "${test_opts[@]}" -- "$FASTTEST_FOCUS" 2>&1 \ | ts '%Y-%m-%d %H:%M:%S' \ | tee "$FASTTEST_OUTPUT/test_result.txt" set -e diff --git a/docker/test/fuzzer/generate-test-j2.py b/docker/test/fuzzer/generate-test-j2.py index bcc1bf6bc84..11525163ed8 100755 --- a/docker/test/fuzzer/generate-test-j2.py +++ b/docker/test/fuzzer/generate-test-j2.py @@ -11,7 +11,7 @@ def removesuffix(text, suffix): https://www.python.org/dev/peps/pep-0616/ """ if suffix and text.endswith(suffix): - return text[:-len(suffix)] + return text[: -len(suffix)] else: return text[:] diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 91b26735fe5..eaf0f01e36d 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -60,5 +60,5 @@ clientPort=2181 \n\ maxClientCnxns=80' > /opt/zookeeper/conf/zoo.cfg RUN mkdir /zookeeper && chmod -R 777 /zookeeper -ENV TZ=Europe/Moscow +ENV TZ=Etc/UTC RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/docker/test/integration/hive_server/Dockerfile b/docker/test/integration/hive_server/Dockerfile index fa6e4bf6313..391f9a5e22f 100644 --- a/docker/test/integration/hive_server/Dockerfile +++ b/docker/test/integration/hive_server/Dockerfile @@ -42,6 +42,9 @@ COPY prepare_hive_data.sh / COPY demo_data.txt / ENV PATH=/apache-hive-2.3.9-bin/bin:/hadoop-3.1.0/bin:/hadoop-3.1.0/sbin:$PATH - +RUN service ssh start && sed s/HOSTNAME/$HOSTNAME/ /hadoop-3.1.0/etc/hadoop/core-site.xml.template > /hadoop-3.1.0/etc/hadoop/core-site.xml && hdfs namenode -format +RUN apt install -y python3 python3-pip +RUN pip3 install flask requests +COPY http_api_server.py / COPY start.sh / diff --git a/docker/test/integration/hive_server/http_api_server.py b/docker/test/integration/hive_server/http_api_server.py new file mode 100644 index 00000000000..8a9d3da4846 --- /dev/null +++ b/docker/test/integration/hive_server/http_api_server.py @@ -0,0 +1,73 @@ +import os +import subprocess +import datetime +from flask import Flask, flash, request, redirect, url_for + + +def run_command(command, wait=False): + print("{} - execute shell command:{}".format(datetime.datetime.now(), command)) + lines = [] + p = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True + ) + if wait: + for l in iter(p.stdout.readline, b""): + lines.append(l) + p.poll() + return (lines, p.returncode) + else: + return (iter(p.stdout.readline, b""), 0) + + +UPLOAD_FOLDER = "./" +ALLOWED_EXTENSIONS = {"txt", "sh"} +app = Flask(__name__) +app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER + + +@app.route("/") +def hello_world(): + return "Hello World" + + +def allowed_file(filename): + return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS + + +@app.route("/upload", methods=["GET", "POST"]) +def upload_file(): + if request.method == "POST": + # check if the post request has the file part + if "file" not in request.files: + flash("No file part") + return redirect(request.url) + file = request.files["file"] + # If the user does not select a file, the browser submits an + # empty file without a filename. + if file.filename == "": + flash("No selected file") + return redirect(request.url) + if file and allowed_file(file.filename): + filename = file.filename + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + return redirect(url_for("upload_file", name=filename)) + return """ + + Upload new File +

Upload new File

+
+ + +
+ """ + + +@app.route("/run", methods=["GET", "POST"]) +def parse_request(): + data = request.data # data is empty + run_command(data, wait=True) + return "Ok" + + +if __name__ == "__main__": + app.run(port=5011) diff --git a/docker/test/integration/hive_server/prepare_hive_data.sh b/docker/test/integration/hive_server/prepare_hive_data.sh index afecbb91c5d..8126b975612 100755 --- a/docker/test/integration/hive_server/prepare_hive_data.sh +++ b/docker/test/integration/hive_server/prepare_hive_data.sh @@ -2,5 +2,9 @@ hive -e "create database test" hive -e "create table test.demo(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; create table test.demo_orc(id string, score int) PARTITIONED BY(day string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; " +hive -e "create table test.parquet_demo(id string, score int) PARTITIONED BY(day string, hour string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" hive -e "create table test.demo_text(id string, score int, day string)row format delimited fields terminated by ','; load data local inpath '/demo_data.txt' into table test.demo_text " - hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo partition(day) select * from test.demo_text; insert into test.demo_orc partition(day) select * from test.demo_text" +hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.demo partition(day) select * from test.demo_text; insert into test.demo_orc partition(day) select * from test.demo_text" + +hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '00' as hour from test.demo;" +hive -e "set hive.exec.dynamic.partition.mode=nonstrict;insert into test.parquet_demo partition(day, hour) select id, score, day, '01' as hour from test.demo;" diff --git a/docker/test/integration/hive_server/start.sh b/docker/test/integration/hive_server/start.sh index e01f28542af..4224b8126e6 100755 --- a/docker/test/integration/hive_server/start.sh +++ b/docker/test/integration/hive_server/start.sh @@ -1,6 +1,5 @@ service ssh start sed s/HOSTNAME/$HOSTNAME/ /hadoop-3.1.0/etc/hadoop/core-site.xml.template > /hadoop-3.1.0/etc/hadoop/core-site.xml -hadoop namenode -format start-all.sh service mysql start mysql -u root -e "CREATE USER \"test\"@\"localhost\" IDENTIFIED BY \"test\"" @@ -9,4 +8,4 @@ schematool -initSchema -dbType mysql #nohup hiveserver2 & nohup hive --service metastore & bash /prepare_hive_data.sh -while true; do sleep 1000; done +python3 http_api_server.py diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index e42d115999a..592c3e36ef7 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -15,9 +15,10 @@ RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault. rm -fr *.rpm RUN cd /tmp && \ - curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \ - tar xzf commons-daemon-1.0.15-src.tar.gz && \ - cd commons-daemon-1.0.15-src/src/native/unix && \ - ./configure && \ - make && \ - cp ./jsvc /usr/local/hadoop-2.7.0/sbin + curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \ + tar xzf commons-daemon-1.0.15-src.tar.gz && \ + cd commons-daemon-1.0.15-src/src/native/unix && \ + ./configure && \ + make && \ + cp ./jsvc /usr/local/hadoop-2.7.0/sbin && \ + [ -e /usr/local/hadoop ] || ln -s ./hadoop-2.7.0 /usr/local/hadoop diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 22dd2e14456..b5c6a39a965 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -40,7 +40,7 @@ RUN apt-get update \ /tmp/* \ && apt-get clean -ENV TZ=Europe/Moscow +ENV TZ=Etc/UTC RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV DOCKER_CHANNEL stable diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml index 88be3e45085..e1b4d393169 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml @@ -4,7 +4,7 @@ services: kerberizedhdfs1: cap_add: - DAC_READ_SEARCH - image: clickhouse/kerberized-hadoop + image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest} hostname: kerberizedhdfs1 restart: always volumes: diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 8109ef7ae64..34414abc3f5 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -45,6 +45,7 @@ export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest} export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest} export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest} export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest} +export DOCKER_KERBERIZED_HADOOP_TAG=${DOCKER_KERBERIZED_HADOOP_TAG:=latest} cd /ClickHouse/tests/integration exec "$@" diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 61987d34299..2266641397b 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -19,58 +19,126 @@ import xml.etree.ElementTree as et from threading import Thread from scipy import stats -logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', level='WARNING') +logging.basicConfig( + format="%(asctime)s: %(levelname)s: %(module)s: %(message)s", level="WARNING" +) total_start_seconds = time.perf_counter() stage_start_seconds = total_start_seconds + def reportStageEnd(stage): global stage_start_seconds, total_start_seconds current = time.perf_counter() - print(f'stage\t{stage}\t{current - stage_start_seconds:.3f}\t{current - total_start_seconds:.3f}') + print( + f"stage\t{stage}\t{current - stage_start_seconds:.3f}\t{current - total_start_seconds:.3f}" + ) stage_start_seconds = current def tsv_escape(s): - return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') + return ( + s.replace("\\", "\\\\") + .replace("\t", "\\t") + .replace("\n", "\\n") + .replace("\r", "") + ) -parser = argparse.ArgumentParser(description='Run performance test.') +parser = argparse.ArgumentParser(description="Run performance test.") # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. -parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') -parser.add_argument('--host', nargs='*', default=['localhost'], help="Space-separated list of server hostname(s). Corresponds to '--port' options.") -parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated list of server port(s). Corresponds to '--host' options.") -parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.') -parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.') -parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.') -parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') -parser.add_argument('--prewarm-max-query-seconds', type=int, default=180, help='For how many seconds at most a prewarm (cold storage) query is allowed to run. The script finishes with error if this time is exceeded.') -parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.') -parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.') -parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.') -parser.add_argument('--print-settings', action='store_true', help='Print test settings and exit.') -parser.add_argument('--keep-created-tables', action='store_true', help="Don't drop the created tables after the test.") -parser.add_argument('--use-existing-tables', action='store_true', help="Don't create or drop the tables, use the existing ones instead.") +parser.add_argument( + "file", + metavar="FILE", + type=argparse.FileType("r", encoding="utf-8"), + nargs=1, + help="test description file", +) +parser.add_argument( + "--host", + nargs="*", + default=["localhost"], + help="Space-separated list of server hostname(s). Corresponds to '--port' options.", +) +parser.add_argument( + "--port", + nargs="*", + default=[9000], + help="Space-separated list of server port(s). Corresponds to '--host' options.", +) +parser.add_argument( + "--runs", type=int, default=1, help="Number of query runs per server." +) +parser.add_argument( + "--max-queries", + type=int, + default=None, + help="Test no more than this number of queries, chosen at random.", +) +parser.add_argument( + "--queries-to-run", + nargs="*", + type=int, + default=None, + help="Space-separated list of indexes of queries to test.", +) +parser.add_argument( + "--max-query-seconds", + type=int, + default=15, + help="For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.", +) +parser.add_argument( + "--prewarm-max-query-seconds", + type=int, + default=180, + help="For how many seconds at most a prewarm (cold storage) query is allowed to run. The script finishes with error if this time is exceeded.", +) +parser.add_argument( + "--profile-seconds", + type=int, + default=0, + help="For how many seconds to profile a query for which the performance has changed.", +) +parser.add_argument( + "--long", action="store_true", help="Do not skip the tests tagged as long." +) +parser.add_argument( + "--print-queries", action="store_true", help="Print test queries and exit." +) +parser.add_argument( + "--print-settings", action="store_true", help="Print test settings and exit." +) +parser.add_argument( + "--keep-created-tables", + action="store_true", + help="Don't drop the created tables after the test.", +) +parser.add_argument( + "--use-existing-tables", + action="store_true", + help="Don't create or drop the tables, use the existing ones instead.", +) args = parser.parse_args() -reportStageEnd('start') +reportStageEnd("start") test_name = os.path.splitext(os.path.basename(args.file[0].name))[0] tree = et.parse(args.file[0]) root = tree.getroot() -reportStageEnd('parse') +reportStageEnd("parse") # Process query parameters -subst_elems = root.findall('substitutions/substitution') -available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... } +subst_elems = root.findall("substitutions/substitution") +available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... } for e in subst_elems: - name = e.find('name').text - values = [v.text for v in e.findall('values/value')] + name = e.find("name").text + values = [v.text for v in e.findall("values/value")] if not values: - raise Exception(f'No values given for substitution {{{name}}}') + raise Exception(f"No values given for substitution {{{name}}}") available_parameters[name] = values @@ -78,7 +146,7 @@ for e in subst_elems: # parameters. The set of parameters is determined based on the first list. # Note: keep the order of queries -- sometimes we have DROP IF EXISTS # followed by CREATE in create queries section, so the order matters. -def substitute_parameters(query_templates, other_templates = []): +def substitute_parameters(query_templates, other_templates=[]): query_results = [] other_results = [[]] * (len(other_templates)) for i, q in enumerate(query_templates): @@ -103,17 +171,21 @@ def substitute_parameters(query_templates, other_templates = []): # and reporting the queries marked as short. test_queries = [] is_short = [] -for e in root.findall('query'): - new_queries, [new_is_short] = substitute_parameters([e.text], [[e.attrib.get('short', '0')]]) +for e in root.findall("query"): + new_queries, [new_is_short] = substitute_parameters( + [e.text], [[e.attrib.get("short", "0")]] + ) test_queries += new_queries is_short += [eval(s) for s in new_is_short] -assert(len(test_queries) == len(is_short)) +assert len(test_queries) == len(is_short) # If we're given a list of queries to run, check that it makes sense. for i in args.queries_to_run or []: if i < 0 or i >= len(test_queries): - print(f'There is no query no. {i} in this test, only [{0}-{len(test_queries) - 1}] are present') + print( + f"There is no query no. {i} in this test, only [{0}-{len(test_queries) - 1}] are present" + ) exit(1) # If we're only asked to print the queries, do that and exit. @@ -125,60 +197,65 @@ if args.print_queries: # Print short queries for i, s in enumerate(is_short): if s: - print(f'short\t{i}') + print(f"short\t{i}") # If we're only asked to print the settings, do that and exit. These are settings # for clickhouse-benchmark, so we print them as command line arguments, e.g. # '--max_memory_usage=10000000'. if args.print_settings: - for s in root.findall('settings/*'): - print(f'--{s.tag}={s.text}') + for s in root.findall("settings/*"): + print(f"--{s.tag}={s.text}") exit(0) # Skip long tests if not args.long: - for tag in root.findall('.//tag'): - if tag.text == 'long': - print('skipped\tTest is tagged as long.') + for tag in root.findall(".//tag"): + if tag.text == "long": + print("skipped\tTest is tagged as long.") sys.exit(0) # Print report threshold for the test if it is set. ignored_relative_change = 0.05 -if 'max_ignored_relative_change' in root.attrib: +if "max_ignored_relative_change" in root.attrib: ignored_relative_change = float(root.attrib["max_ignored_relative_change"]) - print(f'report-threshold\t{ignored_relative_change}') + print(f"report-threshold\t{ignored_relative_change}") -reportStageEnd('before-connect') +reportStageEnd("before-connect") # Open connections -servers = [{'host': host or args.host[0], 'port': port or args.port[0]} for (host, port) in itertools.zip_longest(args.host, args.port)] +servers = [ + {"host": host or args.host[0], "port": port or args.port[0]} + for (host, port) in itertools.zip_longest(args.host, args.port) +] # Force settings_is_important to fail queries on unknown settings. -all_connections = [clickhouse_driver.Client(**server, settings_is_important=True) for server in servers] +all_connections = [ + clickhouse_driver.Client(**server, settings_is_important=True) for server in servers +] for i, s in enumerate(servers): print(f'server\t{i}\t{s["host"]}\t{s["port"]}') -reportStageEnd('connect') +reportStageEnd("connect") if not args.use_existing_tables: # Run drop queries, ignoring errors. Do this before all other activity, # because clickhouse_driver disconnects on error (this is not configurable), # and the new connection loses the changes in settings. - drop_query_templates = [q.text for q in root.findall('drop_query')] + drop_query_templates = [q.text for q in root.findall("drop_query")] drop_queries = substitute_parameters(drop_query_templates) for conn_index, c in enumerate(all_connections): for q in drop_queries: try: c.execute(q) - print(f'drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}') + print(f"drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}") except: pass - reportStageEnd('drop-1') + reportStageEnd("drop-1") # Apply settings. -settings = root.findall('settings/*') +settings = root.findall("settings/*") for conn_index, c in enumerate(all_connections): for s in settings: # requires clickhouse-driver >= 1.1.5 to accept arbitrary new settings @@ -189,48 +266,52 @@ for conn_index, c in enumerate(all_connections): # the test, which is wrong. c.execute("select 1") -reportStageEnd('settings') +reportStageEnd("settings") # Check tables that should exist. If they don't exist, just skip this test. -tables = [e.text for e in root.findall('preconditions/table_exists')] +tables = [e.text for e in root.findall("preconditions/table_exists")] for t in tables: for c in all_connections: try: res = c.execute("select 1 from {} limit 1".format(t)) except: exception_message = traceback.format_exception_only(*sys.exc_info()[:2])[-1] - skipped_message = ' '.join(exception_message.split('\n')[:2]) - print(f'skipped\t{tsv_escape(skipped_message)}') + skipped_message = " ".join(exception_message.split("\n")[:2]) + print(f"skipped\t{tsv_escape(skipped_message)}") sys.exit(0) -reportStageEnd('preconditions') +reportStageEnd("preconditions") if not args.use_existing_tables: # Run create and fill queries. We will run them simultaneously for both # servers, to save time. The weird XML search + filter is because we want to # keep the relative order of elements, and etree doesn't support the # appropriate xpath query. - create_query_templates = [q.text for q in root.findall('./*') - if q.tag in ('create_query', 'fill_query')] + create_query_templates = [ + q.text for q in root.findall("./*") if q.tag in ("create_query", "fill_query") + ] create_queries = substitute_parameters(create_query_templates) # Disallow temporary tables, because the clickhouse_driver reconnects on # errors, and temporary tables are destroyed. We want to be able to continue # after some errors. for q in create_queries: - if re.search('create temporary table', q, flags=re.IGNORECASE): - print(f"Temporary tables are not allowed in performance tests: '{q}'", - file = sys.stderr) + if re.search("create temporary table", q, flags=re.IGNORECASE): + print( + f"Temporary tables are not allowed in performance tests: '{q}'", + file=sys.stderr, + ) sys.exit(1) def do_create(connection, index, queries): for q in queries: connection.execute(q) - print(f'create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}') + print(f"create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}") threads = [ - Thread(target = do_create, args = (connection, index, create_queries)) - for index, connection in enumerate(all_connections)] + Thread(target=do_create, args=(connection, index, create_queries)) + for index, connection in enumerate(all_connections) + ] for t in threads: t.start() @@ -238,14 +319,16 @@ if not args.use_existing_tables: for t in threads: t.join() - reportStageEnd('create') + reportStageEnd("create") # By default, test all queries. queries_to_run = range(0, len(test_queries)) if args.max_queries: # If specified, test a limited number of queries chosen at random. - queries_to_run = random.sample(range(0, len(test_queries)), min(len(test_queries), args.max_queries)) + queries_to_run = random.sample( + range(0, len(test_queries)), min(len(test_queries), args.max_queries) + ) if args.queries_to_run: # Run the specified queries. @@ -255,16 +338,16 @@ if args.queries_to_run: profile_total_seconds = 0 for query_index in queries_to_run: q = test_queries[query_index] - query_prefix = f'{test_name}.query{query_index}' + query_prefix = f"{test_name}.query{query_index}" # We have some crazy long queries (about 100kB), so trim them to a sane # length. This means we can't use query text as an identifier and have to # use the test name + the test-wide query index. query_display_name = q if len(query_display_name) > 1000: - query_display_name = f'{query_display_name[:1000]}...({query_index})' + query_display_name = f"{query_display_name[:1000]}...({query_index})" - print(f'display-name\t{query_index}\t{tsv_escape(query_display_name)}') + print(f"display-name\t{query_index}\t{tsv_escape(query_display_name)}") # Prewarm: run once on both servers. Helps to bring the data into memory, # precompile the queries, etc. @@ -272,10 +355,10 @@ for query_index in queries_to_run: # new one. We want to run them on the new server only, so that the PR author # can ensure that the test works properly. Remember the errors we had on # each server. - query_error_on_connection = [None] * len(all_connections); + query_error_on_connection = [None] * len(all_connections) for conn_index, c in enumerate(all_connections): try: - prewarm_id = f'{query_prefix}.prewarm0' + prewarm_id = f"{query_prefix}.prewarm0" try: # During the warmup runs, we will also: @@ -283,25 +366,30 @@ for query_index in queries_to_run: # * collect profiler traces, which might be helpful for analyzing # test coverage. We disable profiler for normal runs because # it makes the results unstable. - res = c.execute(q, query_id = prewarm_id, - settings = { - 'max_execution_time': args.prewarm_max_query_seconds, - 'query_profiler_real_time_period_ns': 10000000, - 'memory_profiler_step': '4Mi', - }) + res = c.execute( + q, + query_id=prewarm_id, + settings={ + "max_execution_time": args.prewarm_max_query_seconds, + "query_profiler_real_time_period_ns": 10000000, + "memory_profiler_step": "4Mi", + }, + ) except clickhouse_driver.errors.Error as e: # Add query id to the exception to make debugging easier. e.args = (prewarm_id, *e.args) - e.message = prewarm_id + ': ' + e.message + e.message = prewarm_id + ": " + e.message raise - print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') + print( + f"prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}" + ) except KeyboardInterrupt: raise except: # FIXME the driver reconnects on error and we lose settings, so this # might lead to further errors or unexpected behavior. - query_error_on_connection[conn_index] = traceback.format_exc(); + query_error_on_connection[conn_index] = traceback.format_exc() continue # Report all errors that ocurred during prewarm and decide what to do next. @@ -311,14 +399,14 @@ for query_index in queries_to_run: no_errors = [] for i, e in enumerate(query_error_on_connection): if e: - print(e, file = sys.stderr) + print(e, file=sys.stderr) else: no_errors.append(i) if len(no_errors) == 0: continue elif len(no_errors) < len(all_connections): - print(f'partial\t{query_index}\t{no_errors}') + print(f"partial\t{query_index}\t{no_errors}") this_query_connections = [all_connections[index] for index in no_errors] @@ -337,27 +425,34 @@ for query_index in queries_to_run: all_server_times.append([]) while True: - run_id = f'{query_prefix}.run{run}' + run_id = f"{query_prefix}.run{run}" for conn_index, c in enumerate(this_query_connections): try: - res = c.execute(q, query_id = run_id, settings = {'max_execution_time': args.max_query_seconds}) + res = c.execute( + q, + query_id=run_id, + settings={"max_execution_time": args.max_query_seconds}, + ) except clickhouse_driver.errors.Error as e: # Add query id to the exception to make debugging easier. e.args = (run_id, *e.args) - e.message = run_id + ': ' + e.message + e.message = run_id + ": " + e.message raise elapsed = c.last_query.elapsed all_server_times[conn_index].append(elapsed) server_seconds += elapsed - print(f'query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}') + print(f"query\t{query_index}\t{run_id}\t{conn_index}\t{elapsed}") if elapsed > args.max_query_seconds: # Do not stop processing pathologically slow queries, # since this may hide errors in other queries. - print(f'The query no. {query_index} is taking too long to run ({elapsed} s)', file=sys.stderr) + print( + f"The query no. {query_index} is taking too long to run ({elapsed} s)", + file=sys.stderr, + ) # Be careful with the counter, after this line it's the next iteration # already. @@ -386,7 +481,7 @@ for query_index in queries_to_run: break client_seconds = time.perf_counter() - start_seconds - print(f'client-time\t{query_index}\t{client_seconds}\t{server_seconds}') + print(f"client-time\t{query_index}\t{client_seconds}\t{server_seconds}") # Run additional profiling queries to collect profile data, but only if test times appeared to be different. # We have to do it after normal runs because otherwise it will affect test statistics too much @@ -397,13 +492,15 @@ for query_index in queries_to_run: # Don't fail if for some reason there are not enough measurements. continue - pvalue = stats.ttest_ind(all_server_times[0], all_server_times[1], equal_var = False).pvalue + pvalue = stats.ttest_ind( + all_server_times[0], all_server_times[1], equal_var=False + ).pvalue median = [statistics.median(t) for t in all_server_times] # Keep this consistent with the value used in report. Should eventually move # to (median[1] - median[0]) / min(median), which is compatible with "times" # difference we use in report (max(median) / min(median)). relative_diff = (median[1] - median[0]) / median[0] - print(f'diff\t{query_index}\t{median[0]}\t{median[1]}\t{relative_diff}\t{pvalue}') + print(f"diff\t{query_index}\t{median[0]}\t{median[1]}\t{relative_diff}\t{pvalue}") if abs(relative_diff) < ignored_relative_change or pvalue > 0.05: continue @@ -412,25 +509,31 @@ for query_index in queries_to_run: profile_start_seconds = time.perf_counter() run = 0 while time.perf_counter() - profile_start_seconds < args.profile_seconds: - run_id = f'{query_prefix}.profile{run}' + run_id = f"{query_prefix}.profile{run}" for conn_index, c in enumerate(this_query_connections): try: - res = c.execute(q, query_id = run_id, settings = {'query_profiler_real_time_period_ns': 10000000}) - print(f'profile\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}') + res = c.execute( + q, + query_id=run_id, + settings={"query_profiler_real_time_period_ns": 10000000}, + ) + print( + f"profile\t{query_index}\t{run_id}\t{conn_index}\t{c.last_query.elapsed}" + ) except clickhouse_driver.errors.Error as e: # Add query id to the exception to make debugging easier. e.args = (run_id, *e.args) - e.message = run_id + ': ' + e.message + e.message = run_id + ": " + e.message raise run += 1 profile_total_seconds += time.perf_counter() - profile_start_seconds -print(f'profile-total\t{profile_total_seconds}') +print(f"profile-total\t{profile_total_seconds}") -reportStageEnd('run') +reportStageEnd("run") # Run drop queries if not args.keep_created_tables and not args.use_existing_tables: @@ -438,6 +541,6 @@ if not args.keep_created_tables and not args.use_existing_tables: for conn_index, c in enumerate(all_connections): for q in drop_queries: c.execute(q) - print(f'drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}') + print(f"drop\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}") - reportStageEnd('drop-2') + reportStageEnd("drop-2") diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 4cff6b41949..0cb8481ee6e 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -12,9 +12,13 @@ import pprint import sys import traceback -parser = argparse.ArgumentParser(description='Create performance test report') -parser.add_argument('--report', default='main', choices=['main', 'all-queries'], - help='Which report to build') +parser = argparse.ArgumentParser(description="Create performance test report") +parser.add_argument( + "--report", + default="main", + choices=["main", "all-queries"], + help="Which report to build", +) args = parser.parse_args() tables = [] @@ -31,8 +35,8 @@ unstable_partial_queries = 0 # max seconds to run one query by itself, not counting preparation allowed_single_run_time = 2 -color_bad='#ffb0c0' -color_good='#b0d050' +color_bad = "#ffb0c0" +color_good = "#b0d050" header_template = """ @@ -151,24 +155,29 @@ tr:nth-child(odd) td {{filter: brightness(90%);}} table_anchor = 0 row_anchor = 0 + def currentTableAnchor(): global table_anchor - return f'{table_anchor}' + return f"{table_anchor}" + def newTableAnchor(): global table_anchor table_anchor += 1 return currentTableAnchor() + def currentRowAnchor(): global row_anchor global table_anchor - return f'{table_anchor}.{row_anchor}' + return f"{table_anchor}.{row_anchor}" + def nextRowAnchor(): global row_anchor global table_anchor - return f'{table_anchor}.{row_anchor + 1}' + return f"{table_anchor}.{row_anchor + 1}" + def advanceRowAnchor(): global row_anchor @@ -178,43 +187,58 @@ def advanceRowAnchor(): def tr(x, anchor=None): - #return '{x}'.format(a=a, x=str(x)) + # return '{x}'.format(a=a, x=str(x)) anchor = anchor if anchor else advanceRowAnchor() - return f'{x}' + return f"{x}" -def td(value, cell_attributes = ''): - return '{value}'.format( - cell_attributes = cell_attributes, - value = value) -def th(value, cell_attributes = ''): - return '{value}'.format( - cell_attributes = cell_attributes, - value = value) +def td(value, cell_attributes=""): + return "{value}".format( + cell_attributes=cell_attributes, value=value + ) -def tableRow(cell_values, cell_attributes = [], anchor=None): + +def th(value, cell_attributes=""): + return "{value}".format( + cell_attributes=cell_attributes, value=value + ) + + +def tableRow(cell_values, cell_attributes=[], anchor=None): return tr( - ''.join([td(v, a) - for v, a in itertools.zip_longest( - cell_values, cell_attributes, - fillvalue = '') - if a is not None and v is not None]), - anchor) + "".join( + [ + td(v, a) + for v, a in itertools.zip_longest( + cell_values, cell_attributes, fillvalue="" + ) + if a is not None and v is not None + ] + ), + anchor, + ) -def tableHeader(cell_values, cell_attributes = []): + +def tableHeader(cell_values, cell_attributes=[]): return tr( - ''.join([th(v, a) - for v, a in itertools.zip_longest( - cell_values, cell_attributes, - fillvalue = '') - if a is not None and v is not None])) + "".join( + [ + th(v, a) + for v, a in itertools.zip_longest( + cell_values, cell_attributes, fillvalue="" + ) + if a is not None and v is not None + ] + ) + ) + def tableStart(title): - cls = '-'.join(title.lower().split(' ')[:3]); + cls = "-".join(title.lower().split(" ")[:3]) global table_anchor table_anchor = cls anchor = currentTableAnchor() - help_anchor = '-'.join(title.lower().split(' ')); + help_anchor = "-".join(title.lower().split(" ")) return f"""

{title} @@ -223,12 +247,14 @@ def tableStart(title): """ + def tableEnd(): - return '
' + return "" + def tsvRows(n): try: - with open(n, encoding='utf-8') as fd: + with open(n, encoding="utf-8") as fd: result = [] for row in csv.reader(fd, delimiter="\t", quoting=csv.QUOTE_NONE): new_row = [] @@ -237,27 +263,32 @@ def tsvRows(n): # The second one (encode('latin1').decode('utf-8')) fixes the changes with unicode vs utf-8 chars, so # 'Чем зÐ�нимаеÑ�ЬÑ�Ñ�' is transformed back into 'Чем зАнимаешЬся'. - new_row.append(e.encode('utf-8').decode('unicode-escape').encode('latin1').decode('utf-8')) + new_row.append( + e.encode("utf-8") + .decode("unicode-escape") + .encode("latin1") + .decode("utf-8") + ) result.append(new_row) return result except: - report_errors.append( - traceback.format_exception_only( - *sys.exc_info()[:2])[-1]) + report_errors.append(traceback.format_exception_only(*sys.exc_info()[:2])[-1]) pass return [] + def htmlRows(n): rawRows = tsvRows(n) - result = '' + result = "" for row in rawRows: result += tableRow(row) return result + def addSimpleTable(caption, columns, rows, pos=None): global tables - text = '' + text = "" if not rows: return @@ -268,51 +299,63 @@ def addSimpleTable(caption, columns, rows, pos=None): text += tableEnd() tables.insert(pos if pos else len(tables), text) + def add_tested_commits(): global report_errors try: - addSimpleTable('Tested Commits', ['Old', 'New'], - [['
{}
'.format(x) for x in - [open('left-commit.txt').read(), - open('right-commit.txt').read()]]]) + addSimpleTable( + "Tested Commits", + ["Old", "New"], + [ + [ + "
{}
".format(x) + for x in [ + open("left-commit.txt").read(), + open("right-commit.txt").read(), + ] + ] + ], + ) except: # Don't fail if no commit info -- maybe it's a manual run. - report_errors.append( - traceback.format_exception_only( - *sys.exc_info()[:2])[-1]) + report_errors.append(traceback.format_exception_only(*sys.exc_info()[:2])[-1]) pass + def add_report_errors(): global tables global report_errors # Add the errors reported by various steps of comparison script try: - report_errors += [l.strip() for l in open('report/errors.log')] + report_errors += [l.strip() for l in open("report/errors.log")] except: - report_errors.append( - traceback.format_exception_only( - *sys.exc_info()[:2])[-1]) + report_errors.append(traceback.format_exception_only(*sys.exc_info()[:2])[-1]) pass if not report_errors: return - text = tableStart('Errors while Building the Report') - text += tableHeader(['Error']) + text = tableStart("Errors while Building the Report") + text += tableHeader(["Error"]) for x in report_errors: text += tableRow([x]) text += tableEnd() # Insert after Tested Commits tables.insert(1, text) - errors_explained.append([f'There were some errors while building the report']); + errors_explained.append( + [ + f'There were some errors while building the report' + ] + ) + def add_errors_explained(): if not errors_explained: return text = '' - text += tableStart('Error Summary') - text += tableHeader(['Description']) + text += tableStart("Error Summary") + text += tableHeader(["Description"]) for row in errors_explained: text += tableRow(row) text += tableEnd() @@ -321,59 +364,81 @@ def add_errors_explained(): tables.insert(1, text) -if args.report == 'main': +if args.report == "main": print((header_template.format())) add_tested_commits() - - run_error_rows = tsvRows('run-errors.tsv') + run_error_rows = tsvRows("run-errors.tsv") error_tests += len(run_error_rows) - addSimpleTable('Run Errors', ['Test', 'Error'], run_error_rows) + addSimpleTable("Run Errors", ["Test", "Error"], run_error_rows) if run_error_rows: - errors_explained.append([f'There were some errors while running the tests']); + errors_explained.append( + [ + f'There were some errors while running the tests' + ] + ) - - slow_on_client_rows = tsvRows('report/slow-on-client.tsv') + slow_on_client_rows = tsvRows("report/slow-on-client.tsv") error_tests += len(slow_on_client_rows) - addSimpleTable('Slow on Client', - ['Client time, s', 'Server time, s', 'Ratio', 'Test', 'Query'], - slow_on_client_rows) + addSimpleTable( + "Slow on Client", + ["Client time, s", "Server time, s", "Ratio", "Test", "Query"], + slow_on_client_rows, + ) if slow_on_client_rows: - errors_explained.append([f'Some queries are taking noticeable time client-side (missing `FORMAT Null`?)']); + errors_explained.append( + [ + f'Some queries are taking noticeable time client-side (missing `FORMAT Null`?)' + ] + ) - unmarked_short_rows = tsvRows('report/unexpected-query-duration.tsv') + unmarked_short_rows = tsvRows("report/unexpected-query-duration.tsv") error_tests += len(unmarked_short_rows) - addSimpleTable('Unexpected Query Duration', - ['Problem', 'Marked as "short"?', 'Run time, s', 'Test', '#', 'Query'], - unmarked_short_rows) + addSimpleTable( + "Unexpected Query Duration", + ["Problem", 'Marked as "short"?', "Run time, s", "Test", "#", "Query"], + unmarked_short_rows, + ) if unmarked_short_rows: - errors_explained.append([f'Some queries have unexpected duration']); + errors_explained.append( + [ + f'Some queries have unexpected duration' + ] + ) def add_partial(): - rows = tsvRows('report/partial-queries-report.tsv') + rows = tsvRows("report/partial-queries-report.tsv") if not rows: return global unstable_partial_queries, slow_average_tests, tables - text = tableStart('Partial Queries') - columns = ['Median time, s', 'Relative time variance', 'Test', '#', 'Query'] + text = tableStart("Partial Queries") + columns = ["Median time, s", "Relative time variance", "Test", "#", "Query"] text += tableHeader(columns) - attrs = ['' for c in columns] + attrs = ["" for c in columns] for row in rows: - anchor = f'{currentTableAnchor()}.{row[2]}.{row[3]}' + anchor = f"{currentTableAnchor()}.{row[2]}.{row[3]}" if float(row[1]) > 0.10: attrs[1] = f'style="background: {color_bad}"' unstable_partial_queries += 1 - errors_explained.append([f'The query no. {row[3]} of test \'{row[2]}\' has excessive variance of run time. Keep it below 10%']) + errors_explained.append( + [ + f"The query no. {row[3]} of test '{row[2]}' has excessive variance of run time. Keep it below 10%" + ] + ) else: - attrs[1] = '' + attrs[1] = "" if float(row[0]) > allowed_single_run_time: attrs[0] = f'style="background: {color_bad}"' - errors_explained.append([f'The query no. {row[3]} of test \'{row[2]}\' is taking too long to run. Keep the run time below {allowed_single_run_time} seconds"']) + errors_explained.append( + [ + f'The query no. {row[3]} of test \'{row[2]}\' is taking too long to run. Keep the run time below {allowed_single_run_time} seconds"' + ] + ) slow_average_tests += 1 else: - attrs[0] = '' + attrs[0] = "" text += tableRow(row, attrs, anchor) text += tableEnd() tables.append(text) @@ -381,41 +446,45 @@ if args.report == 'main': add_partial() def add_changes(): - rows = tsvRows('report/changed-perf.tsv') + rows = tsvRows("report/changed-perf.tsv") if not rows: return global faster_queries, slower_queries, tables - text = tableStart('Changes in Performance') + text = tableStart("Changes in Performance") columns = [ - 'Old, s', # 0 - 'New, s', # 1 - 'Ratio of speedup (-) or slowdown (+)', # 2 - 'Relative difference (new − old) / old', # 3 - 'p < 0.01 threshold', # 4 - '', # Failed # 5 - 'Test', # 6 - '#', # 7 - 'Query', # 8 - ] - attrs = ['' for c in columns] + "Old, s", # 0 + "New, s", # 1 + "Ratio of speedup (-) or slowdown (+)", # 2 + "Relative difference (new − old) / old", # 3 + "p < 0.01 threshold", # 4 + "", # Failed # 5 + "Test", # 6 + "#", # 7 + "Query", # 8 + ] + attrs = ["" for c in columns] attrs[5] = None text += tableHeader(columns, attrs) for row in rows: - anchor = f'{currentTableAnchor()}.{row[6]}.{row[7]}' + anchor = f"{currentTableAnchor()}.{row[6]}.{row[7]}" if int(row[5]): - if float(row[3]) < 0.: + if float(row[3]) < 0.0: faster_queries += 1 attrs[2] = attrs[3] = f'style="background: {color_good}"' else: slower_queries += 1 attrs[2] = attrs[3] = f'style="background: {color_bad}"' - errors_explained.append([f'The query no. {row[7]} of test \'{row[6]}\' has slowed down']) + errors_explained.append( + [ + f"The query no. {row[7]} of test '{row[6]}' has slowed down" + ] + ) else: - attrs[2] = attrs[3] = '' + attrs[2] = attrs[3] = "" text += tableRow(row, attrs, anchor) @@ -427,35 +496,35 @@ if args.report == 'main': def add_unstable_queries(): global unstable_queries, very_unstable_queries, tables - unstable_rows = tsvRows('report/unstable-queries.tsv') + unstable_rows = tsvRows("report/unstable-queries.tsv") if not unstable_rows: return unstable_queries += len(unstable_rows) columns = [ - 'Old, s', #0 - 'New, s', #1 - 'Relative difference (new - old)/old', #2 - 'p < 0.01 threshold', #3 - '', # Failed #4 - 'Test', #5 - '#', #6 - 'Query' #7 + "Old, s", # 0 + "New, s", # 1 + "Relative difference (new - old)/old", # 2 + "p < 0.01 threshold", # 3 + "", # Failed #4 + "Test", # 5 + "#", # 6 + "Query", # 7 ] - attrs = ['' for c in columns] + attrs = ["" for c in columns] attrs[4] = None - text = tableStart('Unstable Queries') + text = tableStart("Unstable Queries") text += tableHeader(columns, attrs) for r in unstable_rows: - anchor = f'{currentTableAnchor()}.{r[5]}.{r[6]}' + anchor = f"{currentTableAnchor()}.{r[5]}.{r[6]}" if int(r[4]): very_unstable_queries += 1 attrs[3] = f'style="background: {color_bad}"' else: - attrs[3] = '' + attrs[3] = "" # Just don't add the slightly unstable queries we don't consider # errors. It's not clear what the user should do with them. continue @@ -470,53 +539,70 @@ if args.report == 'main': add_unstable_queries() - skipped_tests_rows = tsvRows('analyze/skipped-tests.tsv') - addSimpleTable('Skipped Tests', ['Test', 'Reason'], skipped_tests_rows) + skipped_tests_rows = tsvRows("analyze/skipped-tests.tsv") + addSimpleTable("Skipped Tests", ["Test", "Reason"], skipped_tests_rows) - addSimpleTable('Test Performance Changes', - ['Test', 'Ratio of speedup (-) or slowdown (+)', 'Queries', 'Total not OK', 'Changed perf', 'Unstable'], - tsvRows('report/test-perf-changes.tsv')) + addSimpleTable( + "Test Performance Changes", + [ + "Test", + "Ratio of speedup (-) or slowdown (+)", + "Queries", + "Total not OK", + "Changed perf", + "Unstable", + ], + tsvRows("report/test-perf-changes.tsv"), + ) def add_test_times(): global slow_average_tests, tables - rows = tsvRows('report/test-times.tsv') + rows = tsvRows("report/test-times.tsv") if not rows: return columns = [ - 'Test', #0 - 'Wall clock time, entire test, s', #1 - 'Total client time for measured query runs, s', #2 - 'Queries', #3 - 'Longest query, total for measured runs, s', #4 - 'Wall clock time per query, s', #5 - 'Shortest query, total for measured runs, s', #6 - '', # Runs #7 - ] - attrs = ['' for c in columns] + "Test", # 0 + "Wall clock time, entire test, s", # 1 + "Total client time for measured query runs, s", # 2 + "Queries", # 3 + "Longest query, total for measured runs, s", # 4 + "Wall clock time per query, s", # 5 + "Shortest query, total for measured runs, s", # 6 + "", # Runs #7 + ] + attrs = ["" for c in columns] attrs[7] = None - text = tableStart('Test Times') + text = tableStart("Test Times") text += tableHeader(columns, attrs) - allowed_average_run_time = 3.75 # 60 seconds per test at (7 + 1) * 2 runs + allowed_average_run_time = 3.75 # 60 seconds per test at (7 + 1) * 2 runs for r in rows: - anchor = f'{currentTableAnchor()}.{r[0]}' + anchor = f"{currentTableAnchor()}.{r[0]}" total_runs = (int(r[7]) + 1) * 2 # one prewarm run, two servers - if r[0] != 'Total' and float(r[5]) > allowed_average_run_time * total_runs: + if r[0] != "Total" and float(r[5]) > allowed_average_run_time * total_runs: # FIXME should be 15s max -- investigate parallel_insert slow_average_tests += 1 attrs[5] = f'style="background: {color_bad}"' - errors_explained.append([f'The test \'{r[0]}\' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up']) + errors_explained.append( + [ + f"The test '{r[0]}' is too slow to run as a whole. Investigate whether the create and fill queries can be sped up" + ] + ) else: - attrs[5] = '' + attrs[5] = "" - if r[0] != 'Total' and float(r[4]) > allowed_single_run_time * total_runs: + if r[0] != "Total" and float(r[4]) > allowed_single_run_time * total_runs: slow_average_tests += 1 attrs[4] = f'style="background: {color_bad}"' - errors_explained.append([f'Some query of the test \'{r[0]}\' is too slow to run. See the all queries report']) + errors_explained.append( + [ + f"Some query of the test '{r[0]}' is too slow to run. See the all queries report" + ] + ) else: - attrs[4] = '' + attrs[4] = "" text += tableRow(r, attrs, anchor) @@ -525,10 +611,17 @@ if args.report == 'main': add_test_times() - addSimpleTable('Metric Changes', - ['Metric', 'Old median value', 'New median value', - 'Relative difference', 'Times difference'], - tsvRows('metrics/changes.tsv')) + addSimpleTable( + "Metric Changes", + [ + "Metric", + "Old median value", + "New median value", + "Relative difference", + "Times difference", + ], + tsvRows("metrics/changes.tsv"), + ) add_report_errors() add_errors_explained() @@ -536,7 +629,8 @@ if args.report == 'main': for t in tables: print(t) - print(f""" + print( + f"""

All queries @@ -546,104 +640,111 @@ if args.report == 'main':

- """) + """ + ) - status = 'success' - message = 'See the report' + status = "success" + message = "See the report" message_array = [] if slow_average_tests: - status = 'failure' - message_array.append(str(slow_average_tests) + ' too long') + status = "failure" + message_array.append(str(slow_average_tests) + " too long") if faster_queries: - message_array.append(str(faster_queries) + ' faster') + message_array.append(str(faster_queries) + " faster") if slower_queries: if slower_queries > 3: - status = 'failure' - message_array.append(str(slower_queries) + ' slower') + status = "failure" + message_array.append(str(slower_queries) + " slower") if unstable_partial_queries: very_unstable_queries += unstable_partial_queries - status = 'failure' + status = "failure" # Don't show mildly unstable queries, only the very unstable ones we # treat as errors. if very_unstable_queries: if very_unstable_queries > 5: error_tests += very_unstable_queries - status = 'failure' - message_array.append(str(very_unstable_queries) + ' unstable') + status = "failure" + message_array.append(str(very_unstable_queries) + " unstable") error_tests += slow_average_tests if error_tests: - status = 'failure' - message_array.insert(0, str(error_tests) + ' errors') + status = "failure" + message_array.insert(0, str(error_tests) + " errors") if message_array: - message = ', '.join(message_array) + message = ", ".join(message_array) if report_errors: - status = 'failure' - message = 'Errors while building the report.' + status = "failure" + message = "Errors while building the report." - print((""" + print( + ( + """ - """.format(status=status, message=message))) + """.format( + status=status, message=message + ) + ) + ) -elif args.report == 'all-queries': +elif args.report == "all-queries": print((header_template.format())) add_tested_commits() def add_all_queries(): - rows = tsvRows('report/all-queries.tsv') + rows = tsvRows("report/all-queries.tsv") if not rows: return columns = [ - '', # Changed #0 - '', # Unstable #1 - 'Old, s', #2 - 'New, s', #3 - 'Ratio of speedup (-) or slowdown (+)', #4 - 'Relative difference (new − old) / old', #5 - 'p < 0.01 threshold', #6 - 'Test', #7 - '#', #8 - 'Query', #9 - ] - attrs = ['' for c in columns] + "", # Changed #0 + "", # Unstable #1 + "Old, s", # 2 + "New, s", # 3 + "Ratio of speedup (-) or slowdown (+)", # 4 + "Relative difference (new − old) / old", # 5 + "p < 0.01 threshold", # 6 + "Test", # 7 + "#", # 8 + "Query", # 9 + ] + attrs = ["" for c in columns] attrs[0] = None attrs[1] = None - text = tableStart('All Query Times') + text = tableStart("All Query Times") text += tableHeader(columns, attrs) for r in rows: - anchor = f'{currentTableAnchor()}.{r[7]}.{r[8]}' + anchor = f"{currentTableAnchor()}.{r[7]}.{r[8]}" if int(r[1]): attrs[6] = f'style="background: {color_bad}"' else: - attrs[6] = '' + attrs[6] = "" if int(r[0]): - if float(r[5]) > 0.: + if float(r[5]) > 0.0: attrs[4] = attrs[5] = f'style="background: {color_bad}"' else: attrs[4] = attrs[5] = f'style="background: {color_good}"' else: - attrs[4] = attrs[5] = '' + attrs[4] = attrs[5] = "" if (float(r[2]) + float(r[3])) / 2 > allowed_single_run_time: attrs[2] = f'style="background: {color_bad}"' attrs[3] = f'style="background: {color_bad}"' else: - attrs[2] = '' - attrs[3] = '' + attrs[2] = "" + attrs[3] = "" text += tableRow(r, attrs, anchor) @@ -655,7 +756,8 @@ elif args.report == 'all-queries': for t in tables: print(t) - print(f""" + print( + f"""

Main report @@ -665,4 +767,5 @@ elif args.report == 'all-queries':

- """) + """ + ) diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile deleted file mode 100644 index 01cc7c97548..00000000000 --- a/docker/test/pvs/Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -# rebuild in #33610 -# docker build -t clickhouse/pvs-test . - -ARG FROM_TAG=latest -FROM clickhouse/binary-builder:$FROM_TAG - -RUN apt-get update --yes \ - && apt-get install \ - bash \ - wget \ - software-properties-common \ - gpg-agent \ - debsig-verify \ - strace \ - protobuf-compiler \ - protobuf-compiler-grpc \ - libprotoc-dev \ - libgrpc++-dev \ - libc-ares-dev \ - --yes --no-install-recommends - -#RUN wget -nv -O - http://files.viva64.com/etc/pubkey.txt | sudo apt-key add - -#RUN sudo wget -nv -O /etc/apt/sources.list.d/viva64.list http://files.viva64.com/etc/viva64.list -# -#RUN apt-get --allow-unauthenticated update -y \ -# && env DEBIAN_FRONTEND=noninteractive \ -# apt-get --allow-unauthenticated install --yes --no-install-recommends \ -# pvs-studio - -ENV PKG_VERSION="pvs-studio-latest" - -RUN set -x \ - && export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \ - && wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \ - && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \ - && apt-key add /tmp/pubkey.txt \ - && wget -nv "https://files.viva64.com/${PKG_VERSION}.deb" \ - && { debsig-verify ${PKG_VERSION}.deb \ - || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \ - && dpkg -i "${PKG_VERSION}.deb" - -ENV CCACHE_DIR=/test_output/ccache - -CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \ - && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \ - && ninja re2_st clickhouse_grpc_protos \ - && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j "$(nproc)" -l ./licence.lic; \ - cp /repo_folder/pvs-studio.log /test_output; \ - plog-converter -a GA:1,2 -t fullhtml -o /test_output/pvs-studio-html-report pvs-studio.log; \ - plog-converter -a GA:1,2 -t tasklist -o /test_output/pvs-studio-task-report.txt pvs-studio.log diff --git a/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py b/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py index 58d6ba8c62a..b5bc82e6818 100755 --- a/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py +++ b/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py @@ -7,18 +7,19 @@ import csv RESULT_LOG_NAME = "run.log" + def process_result(result_folder): status = "success" - description = 'Server started and responded' + description = "Server started and responded" summary = [("Smoke test", "OK")] - with open(os.path.join(result_folder, RESULT_LOG_NAME), 'r') as run_log: - lines = run_log.read().split('\n') - if not lines or lines[0].strip() != 'OK': + with open(os.path.join(result_folder, RESULT_LOG_NAME), "r") as run_log: + lines = run_log.read().split("\n") + if not lines or lines[0].strip() != "OK": status = "failure" - logging.info("Lines is not ok: %s", str('\n'.join(lines))) + logging.info("Lines is not ok: %s", str("\n".join(lines))) summary = [("Smoke test", "FAIL")] - description = 'Server failed to respond, see result in logs' + description = "Server failed to respond, see result in logs" result_logs = [] server_log_path = os.path.join(result_folder, "clickhouse-server.log") @@ -38,20 +39,22 @@ def process_result(result_folder): def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of split build smoke test") - parser.add_argument("--in-results-dir", default='/test_output/') - parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') - parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + parser = argparse.ArgumentParser( + description="ClickHouse script for parsing results of split build smoke test" + ) + parser.add_argument("--in-results-dir", default="/test_output/") + parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") + parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") args = parser.parse_args() state, description, test_results, logs = process_result(args.in_results_dir) diff --git a/docker/test/sqlancer/process_sqlancer_result.py b/docker/test/sqlancer/process_sqlancer_result.py index ede3cabc1c5..37b8f465498 100755 --- a/docker/test/sqlancer/process_sqlancer_result.py +++ b/docker/test/sqlancer/process_sqlancer_result.py @@ -10,11 +10,18 @@ def process_result(result_folder): status = "success" summary = [] paths = [] - tests = ["TLPWhere", "TLPGroupBy", "TLPHaving", "TLPWhereGroupBy", "TLPDistinct", "TLPAggregate"] + tests = [ + "TLPWhere", + "TLPGroupBy", + "TLPHaving", + "TLPWhereGroupBy", + "TLPDistinct", + "TLPAggregate", + ] for test in tests: - err_path = '{}/{}.err'.format(result_folder, test) - out_path = '{}/{}.out'.format(result_folder, test) + err_path = "{}/{}.err".format(result_folder, test) + out_path = "{}/{}.out".format(result_folder, test) if not os.path.exists(err_path): logging.info("No output err on path %s", err_path) summary.append((test, "SKIPPED")) @@ -23,24 +30,24 @@ def process_result(result_folder): else: paths.append(err_path) paths.append(out_path) - with open(err_path, 'r') as f: - if 'AssertionError' in f.read(): + with open(err_path, "r") as f: + if "AssertionError" in f.read(): summary.append((test, "FAIL")) - status = 'failure' + status = "failure" else: summary.append((test, "OK")) - logs_path = '{}/logs.tar.gz'.format(result_folder) + logs_path = "{}/logs.tar.gz".format(result_folder) if not os.path.exists(logs_path): logging.info("No logs tar on path %s", logs_path) else: paths.append(logs_path) - stdout_path = '{}/stdout.log'.format(result_folder) + stdout_path = "{}/stdout.log".format(result_folder) if not os.path.exists(stdout_path): logging.info("No stdout log on path %s", stdout_path) else: paths.append(stdout_path) - stderr_path = '{}/stderr.log'.format(result_folder) + stderr_path = "{}/stderr.log".format(result_folder) if not os.path.exists(stderr_path): logging.info("No stderr log on path %s", stderr_path) else: @@ -52,20 +59,22 @@ def process_result(result_folder): def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of sqlancer test") - parser.add_argument("--in-results-dir", default='/test_output/') - parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') - parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + parser = argparse.ArgumentParser( + description="ClickHouse script for parsing results of sqlancer test" + ) + parser.add_argument("--in-results-dir", default="/test_output/") + parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") + parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") args = parser.parse_args() state, description, test_results, logs = process_result(args.in_results_dir) diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 7c16e69a99b..93e7cebb857 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -13,6 +13,17 @@ COPY s3downloader /s3downloader ENV S3_URL="https://clickhouse-datasets.s3.yandex.net" ENV DATASETS="hits visits" +ENV EXPORT_S3_STORAGE_POLICIES=1 + +# Download Minio-related binaries +RUN arch=${TARGETARCH:-amd64} \ + && wget "https://dl.min.io/server/minio/release/linux-${arch}/minio" \ + && chmod +x ./minio \ + && wget "https://dl.min.io/client/mc/release/linux-${arch}/mc" \ + && chmod +x ./mc +ENV MINIO_ROOT_USER="clickhouse" +ENV MINIO_ROOT_PASSWORD="clickhouse" +COPY setup_minio.sh / COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 8202a07f017..77dc61e6cd0 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -11,11 +11,14 @@ dpkg -i package_folder/clickhouse-common-static_*.deb; dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb + +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # install test configs /usr/share/clickhouse-test/config/install.sh +./setup_minio.sh + function start() { if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then @@ -92,6 +95,8 @@ else clickhouse-client --query "SHOW TABLES FROM test" clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" + clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'" + clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits" fi clickhouse-client --query "SHOW TABLES FROM test" diff --git a/docker/test/stateful/setup_minio.sh b/docker/test/stateful/setup_minio.sh new file mode 100755 index 00000000000..5758d905197 --- /dev/null +++ b/docker/test/stateful/setup_minio.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# TODO: Make this file shared with stateless tests +# +# Usage for local run: +# +# ./docker/test/stateful/setup_minio.sh ./tests/ +# + +set -e -x -a -u + +ls -lha + +mkdir -p ./minio_data + +if [ ! -f ./minio ]; then + echo 'MinIO binary not found, downloading...' + + BINARY_TYPE=$(uname -s | tr '[:upper:]' '[:lower:]') + + wget "https://dl.min.io/server/minio/release/${BINARY_TYPE}-amd64/minio" \ + && chmod +x ./minio \ + && wget "https://dl.min.io/client/mc/release/${BINARY_TYPE}-amd64/mc" \ + && chmod +x ./mc +fi + +MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse} +MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse} + +./minio server --address ":11111" ./minio_data & + +while ! curl -v --silent http://localhost:11111 2>&1 | grep AccessDenied +do + echo "Trying to connect to minio" + sleep 1 +done + +lsof -i :11111 + +sleep 5 + +./mc alias set clickminio http://localhost:11111 clickhouse clickhouse +./mc admin user add clickminio test testtest +./mc admin policy set clickminio readwrite user=test +./mc mb clickminio/test + + +# Upload data to Minio. By default after unpacking all tests will in +# /usr/share/clickhouse-test/queries + +TEST_PATH=${1:-/usr/share/clickhouse-test} +MINIO_DATA_PATH=${TEST_PATH}/queries/1_stateful/data_minio + +# Iterating over globs will cause redudant FILE variale to be a path to a file, not a filename +# shellcheck disable=SC2045 +for FILE in $(ls "${MINIO_DATA_PATH}"); do + echo "$FILE"; + ./mc cp "${MINIO_DATA_PATH}"/"$FILE" clickminio/test/"$FILE"; +done + +mkdir -p ~/.aws +cat <> ~/.aws/credentials +[default] +aws_access_key_id=clickhouse +aws_secret_access_key=clickhouse +EOT diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 9b7fde7d542..68c08c23b3f 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -31,7 +31,8 @@ RUN apt-get update -y \ wget \ mysql-client=8.0* \ postgresql-client \ - sqlite3 + sqlite3 \ + awscli RUN pip3 install numpy scipy pandas Jinja2 @@ -59,6 +60,7 @@ RUN arch=${TARGETARCH:-amd64} \ ENV MINIO_ROOT_USER="clickhouse" ENV MINIO_ROOT_PASSWORD="clickhouse" +ENV EXPORT_S3_STORAGE_POLICIES=1 COPY run.sh / COPY setup_minio.sh / diff --git a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile index 562141ba147..dc83e8b8d2e 100644 --- a/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile +++ b/docker/test/stateless/clickhouse-statelest-test-runner.Dockerfile @@ -2,7 +2,7 @@ # 1. build base container # 2. run base conatiner with mounted volumes # 3. commit container as image -FROM ubuntu:18.10 as clickhouse-test-runner-base +FROM ubuntu:20.04 as clickhouse-test-runner-base # A volume where directory with clickhouse packages to be mounted, # for later installing. @@ -11,5 +11,4 @@ VOLUME /packages CMD apt-get update ;\ DEBIAN_FRONTEND=noninteractive \ apt install -y /packages/clickhouse-common-static_*.deb \ - /packages/clickhouse-client_*.deb \ - /packages/clickhouse-test_*.deb + /packages/clickhouse-client_*.deb diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index bd0fc494826..5fd78502337 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -12,11 +12,8 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then - dpkg -i package_folder/clickhouse-test_*.deb -else - ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test -fi + +ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test # install test configs /usr/share/clickhouse-test/config/install.sh @@ -89,6 +86,10 @@ function run_tests() # everything in parallel except DatabaseReplicated. See below. fi + if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--s3-storage') + fi + if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ADDITIONAL_OPTIONS+=('--replicated-database') ADDITIONAL_OPTIONS+=('--jobs') diff --git a/docker/test/stateless/setup_minio.sh b/docker/test/stateless/setup_minio.sh index 7f8b90ee741..df27b21b05b 100755 --- a/docker/test/stateless/setup_minio.sh +++ b/docker/test/stateless/setup_minio.sh @@ -55,3 +55,10 @@ for FILE in $(ls "${MINIO_DATA_PATH}"); do echo "$FILE"; ./mc cp "${MINIO_DATA_PATH}"/"$FILE" clickminio/test/"$FILE"; done + +mkdir -p ~/.aws +cat <> ~/.aws/credentials +[default] +aws_access_key_id=clickhouse +aws_secret_access_key=clickhouse +EOT diff --git a/docker/test/stateless_pytest/Dockerfile b/docker/test/stateless_pytest/Dockerfile index f692f8f39f0..789ee0e9b30 100644 --- a/docker/test/stateless_pytest/Dockerfile +++ b/docker/test/stateless_pytest/Dockerfile @@ -30,5 +30,4 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \ dpkg -i package_folder/clickhouse-server_*.deb; \ dpkg -i package_folder/clickhouse-client_*.deb; \ - dpkg -i package_folder/clickhouse-test_*.deb; \ python3 -m pytest /usr/share/clickhouse-test/queries -n $(nproc) --reruns=1 --timeout=600 --json=test_output/report.json --html=test_output/report.html --self-contained-html diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 4e0b6741061..495c12f4f83 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -25,9 +25,11 @@ RUN apt-get update -y \ brotli COPY ./stress /stress +COPY ./download_previous_release /download_previous_release COPY run.sh / ENV DATASETS="hits visits" ENV S3_URL="https://clickhouse-datasets.s3.yandex.net" +ENV EXPORT_S3_STORAGE_POLICIES=1 CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stress/README.md b/docker/test/stress/README.md index b1519e7968d..96807b9f9a6 100644 --- a/docker/test/stress/README.md +++ b/docker/test/stress/README.md @@ -5,7 +5,7 @@ This allows to find problems like segmentation fault which cause shutdown of ser Usage: ``` $ ls $HOME/someclickhouse -clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb +clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb $ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test Selecting previously unselected package clickhouse-common-static. (Reading database ... 14442 files and directories currently installed.) diff --git a/docker/test/stress/download_previous_release b/docker/test/stress/download_previous_release new file mode 100755 index 00000000000..ea3d376ad90 --- /dev/null +++ b/docker/test/stress/download_previous_release @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import requests +import re +import os + +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" + +CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_amd64.deb" + + +CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = "clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" + +PACKETS_DIR = "previous_release_package_folder/" +VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" + + +class Version: + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return list(map(int, self.version.split('.'))) < list(map(int, other.version.split('.'))) + + def __str__(self): + return self.version + + +class ReleaseInfo: + def __init__(self, version, release_type): + self.version = version + self.type = release_type + + +def find_previous_release(server_version, releases): + releases.sort(key=lambda x: x.version, reverse=True) + for release in releases: + if release.version < server_version: + return True, release + + return False, None + + +def get_previous_release(server_version): + page = 1 + found = False + while not found: + response = requests.get(CLICKHOUSE_TAGS_URL, {'page': page, 'per_page': 100}) + if not response.ok: + raise Exception('Cannot load the list of tags from github: ' + response.reason) + + releases_str = set(re.findall(VERSION_PATTERN, response.text)) + if len(releases_str) == 0: + raise Exception('Cannot find previous release for ' + str(server_version) + ' server version') + + releases = list(map(lambda x: ReleaseInfo(Version(x.split('-')[0]), x.split('-')[1]), releases_str)) + found, previous_release = find_previous_release(server_version, releases) + page += 1 + + return previous_release + + +def download_packet(url, local_file_name, retries=10, backoff_factor=0.3): + session = requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + response = session.get(url) + print(url) + if response.ok: + open(PACKETS_DIR + local_file_name, 'wb').write(response.content) + + +def download_packets(release): + if not os.path.exists(PACKETS_DIR): + os.makedirs(PACKETS_DIR) + + download_packet(CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type), + CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version)) + + download_packet(CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(version=release.version, type=release.type), + CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version)) + + download_packet(CLICKHOUSE_SERVER_DOWNLOAD_URL.format(version=release.version, type=release.type), + CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)) + + download_packet(CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(version=release.version, type=release.type), + CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)) + + +if __name__ == '__main__': + server_version = Version(input()) + previous_release = get_previous_release(server_version) + download_packets(previous_release) + diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index e57dbc38ded..3cef5b008db 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -22,22 +22,28 @@ export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001 export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 + export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000 export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000 -dpkg -i package_folder/clickhouse-common-static_*.deb -dpkg -i package_folder/clickhouse-common-static-dbg_*.deb -dpkg -i package_folder/clickhouse-server_*.deb -dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb +function install_packages() +{ + dpkg -i $1/clickhouse-common-static_*.deb + dpkg -i $1/clickhouse-common-static-dbg_*.deb + dpkg -i $1/clickhouse-server_*.deb + dpkg -i $1/clickhouse-client_*.deb +} function configure() { # install test configs /usr/share/clickhouse-test/config/install.sh + # we mount tests folder from repo to /usr/share + ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test + # avoid too slow startup sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|100000|10000|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml @@ -114,7 +120,7 @@ function start() counter=0 until clickhouse-client --query "SELECT 1" do - if [ "$counter" -gt 240 ] + if [ "$counter" -gt ${1:-240} ] then echo "Cannot start clickhouse-server" cat /var/log/clickhouse-server/stdout.log @@ -169,8 +175,12 @@ quit time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||: } +install_packages package_folder + configure +./setup_minio.sh + start # shellcheck disable=SC2086 # No quotes because I want to split it into words. @@ -186,6 +196,8 @@ clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "SHOW TABLES FROM test" clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" +clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'" +clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits" clickhouse-client --query "SHOW TABLES FROM test" ./stress --hung-check --drop-databases --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" \ @@ -196,14 +208,12 @@ stop start clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \ - || echo -e 'Server failed to start\tFAIL' >> /test_output/test_results.tsv + || (echo -e 'Server failed to start (see application_errors.txt)\tFAIL' >> /test_output/test_results.tsv \ + && grep -Fa ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt) [ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL" [ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL" -# Print Fatal log messages to stdout -zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log* - # Grep logs for sanitizer asserts, crashes and other critical errors # Sanitizer asserts @@ -220,26 +230,155 @@ zgrep -Fa " Application: Child process was terminated by signal 9" /var/ || echo -e 'No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv # Logical errors -zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ - && echo -e 'Logical error thrown (see clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ +zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log* > /test_output/logical_errors.txt \ + && echo -e 'Logical error thrown (see clickhouse-server.log or logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'No logical errors\tOK' >> /test_output/test_results.tsv +# Remove file logical_errors.txt if it's empty +[ -s /test_output/logical_errors.txt ] || rm /test_output/logical_errors.txt + # Crash zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv # It also checks for crash without stacktrace (printed by watchdog) -zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ - && echo -e 'Fatal message in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \ +zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log* > /test_output/fatal_messages.txt \ + && echo -e 'Fatal message in clickhouse-server.log (see fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv +# Remove file fatal_messages.txt if it's empty +[ -s /test_output/fatal_messages.txt ] || rm /test_output/fatal_messages.txt + zgrep -Fa "########################################" /test_output/* > /dev/null \ && echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv zgrep -Fa " received signal " /test_output/gdb.log > /dev/null \ && echo -e 'Found signal in gdb.log\tFAIL' >> /test_output/test_results.tsv +echo -e "Backward compatibility check\n" + +echo "Download previous release server" +mkdir previous_release_package_folder +clickhouse-client --query="SELECT version()" | ./download_previous_release && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \ + || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv + +if [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ] +then + echo -e "Successfully downloaded previous release packets\tOK" >> /test_output/test_results.tsv + stop + + # Uninstall current packages + dpkg --remove clickhouse-client + dpkg --remove clickhouse-server + dpkg --remove clickhouse-common-static-dbg + dpkg --remove clickhouse-common-static + + rm -rf /var/lib/clickhouse/* + + # Install previous release packages + install_packages previous_release_package_folder + + # Start server from previous release + configure + start + + clickhouse-client --query="SELECT 'Server version: ', version()" + + # Install new package before running stress test because we should use new clickhouse-client and new clickhouse-test + install_packages package_folder + + mkdir tmp_stress_output + + ./stress --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \ + && echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv + rm -rf tmp_stress_output + + clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables" + + stop + + # Start new server + configure + start 500 + clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \ + || (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \ + && grep -Fa ".*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt) + + clickhouse-client --query="SELECT 'Server version: ', version()" + + # Let the server run for a while before checking log. + sleep 60 + + stop + + # Error messages (we should ignore some errors) + echo "Check for Error messages in server log:" + zgrep -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \ + -e "Code: 236. DB::Exception: Cancelled mutating parts" \ + -e "REPLICA_IS_ALREADY_ACTIVE" \ + -e "REPLICA_IS_ALREADY_EXIST" \ + -e "ALL_REPLICAS_LOST" \ + -e "DDLWorker: Cannot parse DDL task query" \ + -e "RaftInstance: failed to accept a rpc connection due to error 125" \ + -e "UNKNOWN_DATABASE" \ + -e "NETWORK_ERROR" \ + -e "UNKNOWN_TABLE" \ + -e "ZooKeeperClient" \ + -e "KEEPER_EXCEPTION" \ + -e "DirectoryMonitor" \ + -e "TABLE_IS_READ_ONLY" \ + -e "Code: 1000, e.code() = 111, Connection refused" \ + -e "UNFINISHED" \ + -e "Renaming unexpected part" \ + /var/log/clickhouse-server/clickhouse-server.log | zgrep -Fa "" > /test_output/bc_check_error_messages.txt \ + && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv + + # Remove file bc_check_error_messages.txt if it's empty + [ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt + + # Sanitizer asserts + zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp + zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp + zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" /test_output/tmp > /dev/null \ + && echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv + rm -f /test_output/tmp + + # OOM + zgrep -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv + + # Logical errors + echo "Check for Logical errors in server log:" + zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /test_output/bc_check_logical_errors.txt \ + && echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv + + # Remove file bc_check_logical_errors.txt if it's empty + [ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt + + # Crash + zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + && echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv + + # It also checks for crash without stacktrace (printed by watchdog) + echo "Check for Fatal message in server log:" + zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log > /test_output/bc_check_fatal_messages.txt \ + && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv + + # Remove file bc_check_fatal_messages.txt if it's empty + [ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt + +else + echo -e "Backward compatibility check: Failed to download previous release packets\tFAIL" >> /test_output/test_results.tsv +fi + # Put logs into /test_output/ for log_file in /var/log/clickhouse-server/clickhouse-server.log* do diff --git a/docker/test/stress/stress b/docker/test/stress/stress index c89c5ff5e27..86f8edf5980 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -47,7 +47,8 @@ def get_options(i): return ' '.join(options) -def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit): +def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit, backward_compatibility_check): + backward_compatibility_check_option = '--backward-compatibility-check' if backward_compatibility_check else '' global_time_limit_option = '' if global_time_limit: global_time_limit_option = "--global_time_limit={}".format(global_time_limit) @@ -56,7 +57,7 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t pipes = [] for i in range(0, len(output_paths)): f = open(output_paths[i], 'w') - full_command = "{} {} {} {}".format(cmd, get_options(i), global_time_limit_option, skip_tests_option) + full_command = "{} {} {} {} {}".format(cmd, get_options(i), global_time_limit_option, skip_tests_option, backward_compatibility_check_option) logging.info("Run func tests '%s'", full_command) p = Popen(full_command, shell=True, stdout=f, stderr=f) pipes.append(p) @@ -168,6 +169,7 @@ if __name__ == "__main__": parser.add_argument("--output-folder") parser.add_argument("--global-time-limit", type=int, default=1800) parser.add_argument("--num-parallel", type=int, default=cpu_count()) + parser.add_argument('--backward-compatibility-check', action='store_true') parser.add_argument('--hung-check', action='store_true', default=False) # make sense only for hung check parser.add_argument('--drop-databases', action='store_true', default=False) @@ -176,7 +178,7 @@ if __name__ == "__main__": if args.drop_databases and not args.hung_check: raise Exception("--drop-databases only used in hung check (--hung-check)") func_pipes = [] - func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit) + func_pipes = run_func_test(args.test_cmd, args.output_folder, args.num_parallel, args.skip_func_tests, args.global_time_limit, args.backward_compatibility_check) logging.info("Will wait functests to finish") while True: diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 85c751edfbe..3101ab84c40 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ shellcheck \ yamllint \ - && pip3 install codespell PyGithub boto3 unidiff dohq-artifactory + && pip3 install black boto3 codespell dohq-artifactory PyGithub unidiff # Architecture of the image when BuildKit/buildx is used ARG TARGETARCH diff --git a/docker/test/style/process_style_check_result.py b/docker/test/style/process_style_check_result.py index 655b7d70243..6472ff21f5e 100755 --- a/docker/test/style/process_style_check_result.py +++ b/docker/test/style/process_style_check_result.py @@ -14,6 +14,7 @@ def process_result(result_folder): ("header duplicates", "duplicate_output.txt"), ("shellcheck", "shellcheck_output.txt"), ("style", "style_output.txt"), + ("black", "black_output.txt"), ("typos", "typos_output.txt"), ("whitespaces", "whitespaces_output.txt"), ("workflows", "workflows_output.txt"), diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh index ce3ea4e50a6..651883511e8 100755 --- a/docker/test/style/run.sh +++ b/docker/test/style/run.sh @@ -7,11 +7,13 @@ echo "Check duplicates" | ts ./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt echo "Check style" | ts ./check-style -n |& tee /test_output/style_output.txt +echo "Check python formatting with black" | ts +./check-black -n |& tee /test_output/black_output.txt echo "Check typos" | ts ./check-typos |& tee /test_output/typos_output.txt echo "Check whitespaces" | ts ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt -echo "Check sorkflows" | ts +echo "Check workflows" | ts ./check-workflows |& tee /test_output/workflows_output.txt echo "Check shell scripts with shellcheck" | ts ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt diff --git a/docker/test/testflows/runner/process_testflows_result.py b/docker/test/testflows/runner/process_testflows_result.py index 37d0b6a69d1..8bfc4ac0b0f 100755 --- a/docker/test/testflows/runner/process_testflows_result.py +++ b/docker/test/testflows/runner/process_testflows_result.py @@ -22,9 +22,9 @@ def process_result(result_folder): total_other = 0 test_results = [] for test in results["tests"]: - test_name = test['test']['test_name'] - test_result = test['result']['result_type'].upper() - test_time = str(test['result']['message_rtime']) + test_name = test["test"]["test_name"] + test_result = test["result"]["result_type"].upper() + test_time = str(test["result"]["message_rtime"]) total_tests += 1 if test_result == "OK": total_ok += 1 @@ -39,24 +39,29 @@ def process_result(result_folder): else: status = "success" - description = "failed: {}, passed: {}, other: {}".format(total_fail, total_ok, total_other) + description = "failed: {}, passed: {}, other: {}".format( + total_fail, total_ok, total_other + ) return status, description, test_results, [json_path, test_binary_log] def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) + if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of Testflows tests") - parser.add_argument("--in-results-dir", default='./') - parser.add_argument("--out-results-file", default='./test_results.tsv') - parser.add_argument("--out-status-file", default='./check_status.tsv') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + parser = argparse.ArgumentParser( + description="ClickHouse script for parsing results of Testflows tests" + ) + parser.add_argument("--in-results-dir", default="./") + parser.add_argument("--out-results-file", default="./test_results.tsv") + parser.add_argument("--out-status-file", default="./check_status.tsv") args = parser.parse_args() state, description, test_results, logs = process_result(args.in_results_dir) @@ -64,4 +69,3 @@ if __name__ == "__main__": status = (state, description) write_results(args.out_results_file, args.out_status_file, test_results, status) logging.info("Result written") - diff --git a/docker/test/unit/process_unit_tests_result.py b/docker/test/unit/process_unit_tests_result.py index 7219aa13b82..0550edc7c25 100755 --- a/docker/test/unit/process_unit_tests_result.py +++ b/docker/test/unit/process_unit_tests_result.py @@ -5,24 +5,26 @@ import logging import argparse import csv -OK_SIGN = 'OK ]' -FAILED_SIGN = 'FAILED ]' -SEGFAULT = 'Segmentation fault' -SIGNAL = 'received signal SIG' -PASSED = 'PASSED' +OK_SIGN = "OK ]" +FAILED_SIGN = "FAILED ]" +SEGFAULT = "Segmentation fault" +SIGNAL = "received signal SIG" +PASSED = "PASSED" + def get_test_name(line): - elements = reversed(line.split(' ')) + elements = reversed(line.split(" ")) for element in elements: - if '(' not in element and ')' not in element: + if "(" not in element and ")" not in element: return element raise Exception("No test name in line '{}'".format(line)) + def process_result(result_folder): summary = [] total_counter = 0 failed_counter = 0 - result_log_path = '{}/test_result.txt'.format(result_folder) + result_log_path = "{}/test_result.txt".format(result_folder) if not os.path.exists(result_log_path): logging.info("No output log on path %s", result_log_path) return "exception", "No output log", [] @@ -30,7 +32,7 @@ def process_result(result_folder): status = "success" description = "" passed = False - with open(result_log_path, 'r') as test_result: + with open(result_log_path, "r") as test_result: for line in test_result: if OK_SIGN in line: logging.info("Found ok line: '%s'", line) @@ -38,7 +40,7 @@ def process_result(result_folder): logging.info("Test name: '%s'", test_name) summary.append((test_name, "OK")) total_counter += 1 - elif FAILED_SIGN in line and 'listed below' not in line and 'ms)' in line: + elif FAILED_SIGN in line and "listed below" not in line and "ms)" in line: logging.info("Found fail line: '%s'", line) test_name = get_test_name(line.strip()) logging.info("Test name: '%s'", test_name) @@ -67,25 +69,30 @@ def process_result(result_folder): status = "failure" if not description: - description += "fail: {}, passed: {}".format(failed_counter, total_counter - failed_counter) + description += "fail: {}, passed: {}".format( + failed_counter, total_counter - failed_counter + ) return status, description, summary def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) + if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of unit tests") - parser.add_argument("--in-results-dir", default='/test_output/') - parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') - parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + parser = argparse.ArgumentParser( + description="ClickHouse script for parsing results of unit tests" + ) + parser.add_argument("--in-results-dir", default="/test_output/") + parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") + parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") args = parser.parse_args() state, description, test_results = process_result(args.in_results_dir) @@ -93,4 +100,3 @@ if __name__ == "__main__": status = (state, description) write_results(args.out_results_file, args.out_status_file, test_results, status) logging.info("Result written") - diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index 82df170686d..dadda55c830 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -16,6 +16,7 @@ NO_TASK_TIMEOUT_SIGNS = ["All tests have finished", "No tests were run"] RETRIES_SIGN = "Some tests were restarted" + def process_test_log(log_path): total = 0 skipped = 0 @@ -26,7 +27,7 @@ def process_test_log(log_path): retries = False task_timeout = True test_results = [] - with open(log_path, 'r') as test_file: + with open(log_path, "r") as test_file: for line in test_file: original_line = line line = line.strip() @@ -36,12 +37,15 @@ def process_test_log(log_path): hung = True if RETRIES_SIGN in line: retries = True - if any(sign in line for sign in (OK_SIGN, FAIL_SIGN, UNKNOWN_SIGN, SKIPPED_SIGN)): - test_name = line.split(' ')[2].split(':')[0] + if any( + sign in line + for sign in (OK_SIGN, FAIL_SIGN, UNKNOWN_SIGN, SKIPPED_SIGN) + ): + test_name = line.split(" ")[2].split(":")[0] - test_time = '' + test_time = "" try: - time_token = line.split(']')[1].strip().split()[0] + time_token = line.split("]")[1].strip().split()[0] float(time_token) test_time = time_token except: @@ -66,9 +70,22 @@ def process_test_log(log_path): elif len(test_results) > 0 and test_results[-1][1] == "FAIL": test_results[-1][3].append(original_line) - test_results = [(test[0], test[1], test[2], ''.join(test[3])) for test in test_results] + test_results = [ + (test[0], test[1], test[2], "".join(test[3])) for test in test_results + ] + + return ( + total, + skipped, + unknown, + failed, + success, + hung, + task_timeout, + retries, + test_results, + ) - return total, skipped, unknown, failed, success, hung, task_timeout, retries, test_results def process_result(result_path): test_results = [] @@ -76,16 +93,26 @@ def process_result(result_path): description = "" files = os.listdir(result_path) if files: - logging.info("Find files in result folder %s", ','.join(files)) - result_path = os.path.join(result_path, 'test_result.txt') + logging.info("Find files in result folder %s", ",".join(files)) + result_path = os.path.join(result_path, "test_result.txt") else: result_path = None description = "No output log" state = "error" if result_path and os.path.exists(result_path): - total, skipped, unknown, failed, success, hung, task_timeout, retries, test_results = process_test_log(result_path) - is_flacky_check = 1 < int(os.environ.get('NUM_TRIES', 1)) + ( + total, + skipped, + unknown, + failed, + success, + hung, + task_timeout, + retries, + test_results, + ) = process_test_log(result_path) + is_flacky_check = 1 < int(os.environ.get("NUM_TRIES", 1)) logging.info("Is flacky check: %s", is_flacky_check) # If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately) # But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped. @@ -120,20 +147,22 @@ def process_result(result_path): def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of functional tests") - parser.add_argument("--in-results-dir", default='/test_output/') - parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') - parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + parser = argparse.ArgumentParser( + description="ClickHouse script for parsing results of functional tests" + ) + parser.add_argument("--in-results-dir", default="/test_output/") + parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") + parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") args = parser.parse_args() state, description, test_results = process_result(args.in_results_dir) diff --git a/docs/README.md b/docs/README.md index cd5c1af0cbd..b328a3ee125 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,7 +38,7 @@ Writing the docs is extremely useful for project's users and developers, and gro The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries. -At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, Chinese, Japanese, and Farsi. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs). +At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, Chinese, Japanese. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs). Each language lays in the corresponding folder. Files that are not translated from English are the symbolic links to the English ones. diff --git a/docs/_includes/install/deb.sh b/docs/_includes/install/deb.sh index 21106e9fc47..0daf12a132f 100644 --- a/docs/_includes/install/deb.sh +++ b/docs/_includes/install/deb.sh @@ -1,11 +1,11 @@ -sudo apt-get install apt-transport-https ca-certificates dirmngr -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 +sudo apt-get install -y apt-transport-https ca-certificates dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 -echo "deb https://repo.clickhouse.com/deb/stable/ main/" | sudo tee \ +echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \ /etc/apt/sources.list.d/clickhouse.list sudo apt-get update sudo apt-get install -y clickhouse-server clickhouse-client sudo service clickhouse-server start -clickhouse-client # or "clickhouse-client --password" if you set up a password. +clickhouse-client # or "clickhouse-client --password" if you've set up a password. diff --git a/docs/_includes/install/deb_repo.sh b/docs/_includes/install/deb_repo.sh new file mode 100644 index 00000000000..21106e9fc47 --- /dev/null +++ b/docs/_includes/install/deb_repo.sh @@ -0,0 +1,11 @@ +sudo apt-get install apt-transport-https ca-certificates dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 + +echo "deb https://repo.clickhouse.com/deb/stable/ main/" | sudo tee \ + /etc/apt/sources.list.d/clickhouse.list +sudo apt-get update + +sudo apt-get install -y clickhouse-server clickhouse-client + +sudo service clickhouse-server start +clickhouse-client # or "clickhouse-client --password" if you set up a password. diff --git a/docs/_includes/install/rpm.sh b/docs/_includes/install/rpm.sh index e3fd1232047..ff99018f872 100644 --- a/docs/_includes/install/rpm.sh +++ b/docs/_includes/install/rpm.sh @@ -1,7 +1,6 @@ -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/clickhouse.repo -sudo yum install clickhouse-server clickhouse-client +sudo yum install -y yum-utils +sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo +sudo yum install -y clickhouse-server clickhouse-client sudo /etc/init.d/clickhouse-server start clickhouse-client # or "clickhouse-client --password" if you set up a password. diff --git a/docs/_includes/install/rpm_repo.sh b/docs/_includes/install/rpm_repo.sh new file mode 100644 index 00000000000..e3fd1232047 --- /dev/null +++ b/docs/_includes/install/rpm_repo.sh @@ -0,0 +1,7 @@ +sudo yum install yum-utils +sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/clickhouse.repo +sudo yum install clickhouse-server clickhouse-client + +sudo /etc/init.d/clickhouse-server start +clickhouse-client # or "clickhouse-client --password" if you set up a password. diff --git a/docs/_includes/install/tgz.sh b/docs/_includes/install/tgz.sh index 0994510755b..4ba5890b32b 100644 --- a/docs/_includes/install/tgz.sh +++ b/docs/_includes/install/tgz.sh @@ -1,19 +1,20 @@ -export LATEST_VERSION=$(curl -s https://repo.clickhouse.com/tgz/stable/ | \ +LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \ grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) -curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz +export LATEST_VERSION +curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz" +curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz" +curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz" -tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz -sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh +tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz" +sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh" -tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz -sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh +tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz" +sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh" -tar -xzvf clickhouse-server-$LATEST_VERSION.tgz -sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh +tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz" +sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" sudo /etc/init.d/clickhouse-server start -tar -xzvf clickhouse-client-$LATEST_VERSION.tgz -sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh +tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz" +sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh" diff --git a/docs/_includes/install/tgz_repo.sh b/docs/_includes/install/tgz_repo.sh new file mode 100644 index 00000000000..0994510755b --- /dev/null +++ b/docs/_includes/install/tgz_repo.sh @@ -0,0 +1,19 @@ +export LATEST_VERSION=$(curl -s https://repo.clickhouse.com/tgz/stable/ | \ + grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1) +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz + +tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz +sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh + +tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz +sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh + +tar -xzvf clickhouse-server-$LATEST_VERSION.tgz +sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh +sudo /etc/init.d/clickhouse-server start + +tar -xzvf clickhouse-client-$LATEST_VERSION.tgz +sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh diff --git a/docs/en/development/browse-code.md b/docs/en/development/browse-code.md index fa57d2289b3..0fe8a46873c 100644 --- a/docs/en/development/browse-code.md +++ b/docs/en/development/browse-code.md @@ -5,7 +5,7 @@ toc_title: Source Code Browser # Browse ClickHouse Source Code {#browse-clickhouse-source-code} -You can use **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/html_report/ClickHouse/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. +You can use **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 982ba0556a9..5379fc37937 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -23,7 +23,7 @@ $ sudo apt-get install git cmake python ninja-build Or cmake3 instead of cmake on older systems. -### Install clang-13 (recommended) {#install-clang-13} +### Install the latest clang (recommended) On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) @@ -33,13 +33,15 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). -#### Use clang-13 for Builds +#### Use the latest clang for Builds ``` bash -$ export CC=clang-13 -$ export CXX=clang++-13 +$ export CC=clang-14 +$ export CXX=clang++-14 ``` +In this example we use version 14 that is the latest as of Feb 2022. + Gcc can also be used though it is discouraged. ### Checkout ClickHouse Sources {#checkout-clickhouse-sources} @@ -76,7 +78,6 @@ The build requires the following components: - Ninja - C++ compiler: clang-13 or newer - Linker: lld -- Python (is only used inside LLVM build and it is optional) If all the components are installed, you may build in the same way as the steps above. @@ -109,6 +110,29 @@ cmake ../ClickHouse make -j $(nproc) ``` +Here is an example of how to build `clang` and all the llvm infrastructure from sources: + +``` + git clone git@github.com:llvm/llvm-project.git + mkdir llvm-build && cd llvm-build + cmake -DCMAKE_BUILD_TYPE:STRING=Release -DLLVM_ENABLE_PROJECTS=all ../llvm-project/llvm/ + make -j16 + sudo make install + hash clang + clang --version +``` + +You can install the older clang like clang-11 from packages and then use it to build the new clang from sources. + +Here is an example of how to install the new `cmake` from the official website: + +``` +wget https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.sh +chmod +x cmake-3.22.2-linux-x86_64.sh +./cmake-3.22.2-linux-x86_64.sh +export PATH=/home/milovidov/work/cmake-3.22.2-linux-x86_64/bin/:${PATH} +hash cmake +``` ## How to Build ClickHouse Debian Package {#how-to-build-clickhouse-debian-package} @@ -132,14 +156,6 @@ $ cd ClickHouse $ ./release ``` -## Faster builds for development - -Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. One common way to improve build time is to use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable this tweak, pass the following flags to `cmake`: - -``` --DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -``` - ## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse} ClickHouse is available in pre-built binaries and packages. Binaries are portable and can be run on any Linux flavour. @@ -148,9 +164,9 @@ They are built for stable, prestable and testing releases as long as for every c To find the freshest build from `master`, go to [commits page](https://github.com/ClickHouse/ClickHouse/commits/master), click on the first green checkmark or red cross near commit, and click to the “Details” link right after “ClickHouse Build Check”. -## Split build configuration {#split-build} +## Faster builds for development: Split build configuration {#split-build} -Normally ClickHouse is statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that on every change the entire binary is linked again, which is slow and may be inconvenient for development. There is an alternative configuration which creates dynamically loaded shared libraries instead, allowing faster incremental builds. To use it, add the following flags to your `cmake` invocation: +Normally, ClickHouse is statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that on every change the entire binary needs to be linked, which is slow and may be inconvenient for development. There is an alternative configuration which instead creates dynamically loaded shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation: ``` -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 ``` diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index d2745c0080b..f9dfebff3f9 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -71,20 +71,13 @@ This check means that the CI system started to process the pull request. When it Performs some simple regex-based checks of code style, using the [`utils/check-style/check-style`](https://github.com/ClickHouse/ClickHouse/blob/master/utils/check-style/check-style) binary (note that it can be run locally). If it fails, fix the style errors following the [code style guide](style.md). +Python code is checked with [black](https://github.com/psf/black/). + ### Report Details - [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) - `output.txt` contains the check resulting errors (invalid tabulation etc), blank page means no errors. [Successful result example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt). -## PVS Check -Check the code with [PVS-studio](https://www.viva64.com/en/pvs-studio/), a static analysis tool. Look at the report to see the exact errors. Fix them if you can, if not -- ask a ClickHouse maintainer for help. - -### Report Details -- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) -- `test_run.txt.out.log` contains the building and analyzing log file. It includes only parsing or not-found errors. -- `HTML report` contains the analysis results. For its description visit PVS's [official site](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD). - - ## Fast Test Normally this is the first check that is ran for a PR. It builds ClickHouse and runs most of [stateless functional tests](tests.md#functional-tests), omitting @@ -135,7 +128,6 @@ Builds ClickHouse in various configurations for use in further steps. You have t - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - `clickhouse_XXX_amd64.buildinfo` - `clickhouse_XXX_amd64.changes` - `clickhouse`: Main built binary. @@ -200,15 +192,3 @@ Runs randomly generated queries to catch program errors. If it fails, ask a main ## Performance Tests Measure changes in query performance. This is the longest check that takes just below 6 hours to run. The performance test report is described in detail [here](https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison#how-to-read-the-report). - - - -# QA - -> What is a `Task (private network)` item on status pages? - -It's a link to the Yandex's internal job system. Yandex employees can see the check's start time and its more verbose status. - -> Where the tests are run - -Somewhere on Yandex internal infrastructure. diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 07969f8ef6a..6c12a3d9055 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -40,8 +40,8 @@ The list of third-party libraries: | grpc | [Apache](https://github.com/ClickHouse-Extras/grpc/blob/60c986e15cae70aade721d26badabab1f822fdd6/LICENSE) | | h3 | [Apache](https://github.com/ClickHouse-Extras/h3/blob/c7f46cfd71fb60e2fefc90e28abe81657deff735/LICENSE) | | hyperscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/e9f08df0213fc637aac0a5bbde9beeaeba2fe9fa/LICENSE) | -| icu | [Public Domain](https://github.com/unicode-org/icu/blob/faa2f9f9e1fe74c5ed00eba371d2830134cdbea1/icu4c/LICENSE) | -| icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/f020820388e3faafb44cc643574a2d563dfde572/LICENSE) | +| icu | [Public Domain](https://github.com/unicode-org/icu/blob/a56dde820dc35665a66f2e9ee8ba58e75049b668/icu4c/LICENSE) | +| icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/72d9a4a7febc904e2b0a534ccb25ae40fac5f1e5/LICENSE) | | jemalloc | [BSD 2-clause](https://github.com/ClickHouse-Extras/jemalloc/blob/e6891d9746143bf2cf617493d880ba5a0b9a3efd/COPYING) | | krb5 | [MIT](https://github.com/ClickHouse-Extras/krb5/blob/5149dea4e2be0f67707383d2682b897c14631374/src/lib/gssapi/LICENSE) | | libc-headers | [LGPL](https://github.com/ClickHouse-Extras/libc-headers/blob/a720b7105a610acbd7427eea475a5b6810c151eb/LICENSE) | diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index f7d7100d181..db78637f104 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -229,6 +229,25 @@ As simple code editors, you can use Sublime Text or Visual Studio Code, or Kate Just in case, it is worth mentioning that CLion creates `build` path on its own, it also on its own selects `debug` for build type, for configuration it uses a version of CMake that is defined in CLion and not the one installed by you, and finally, CLion will use `make` to run build tasks instead of `ninja`. This is normal behaviour, just keep that in mind to avoid confusion. +## Debugging + +Many graphical IDEs offer with an integrated debugger but you can also use a standalone debugger. + +### GDB + +### LLDB + + # tell LLDB where to find the source code + settings set target.source-map /path/to/build/dir /path/to/source/dir + + # configure LLDB to display code before/after currently executing line + settings set stop-line-count-before 10 + settings set stop-line-count-after 10 + + target create ./clickhouse-client + # + process launch -- --query="SELECT * FROM TAB" + ## Writing Code {#writing-code} The description of ClickHouse architecture can be found here: https://clickhouse.com/docs/en/development/architecture/ @@ -243,7 +262,7 @@ List of tasks: https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3A ## Test Data {#test-data} -Developing ClickHouse often requires loading realistic datasets. It is particularly important for performance testing. We have a specially prepared set of anonymized data from Yandex.Metrica. It requires additionally some 3GB of free disk space. Note that this data is not required to accomplish most of the development tasks. +Developing ClickHouse often requires loading realistic datasets. It is particularly important for performance testing. We have a specially prepared set of anonymized data of web analytics. It requires additionally some 3GB of free disk space. Note that this data is not required to accomplish most of the development tasks. sudo apt install wget xz-utils @@ -270,7 +289,7 @@ Navigate to your fork repository in GitHub’s UI. If you have been developing i A pull request can be created even if the work is not completed yet. In this case please put the word “WIP” (work in progress) at the beginning of the title, it can be changed later. This is useful for cooperative reviewing and discussion of changes as well as for running all of the available tests. It is important that you provide a brief description of your changes, it will later be used for generating release changelogs. -Testing will commence as soon as Yandex employees label your PR with a tag “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. +Testing will commence as soon as ClickHouse employees label your PR with a tag “can be tested”. The results of some first checks (e.g. code style) will come in within several minutes. Build check results will arrive within half an hour. And the main set of tests will report itself within an hour. The system will prepare ClickHouse binary builds for your pull request individually. To retrieve these builds click the “Details” link next to “ClickHouse build check” entry in the list of checks. There you will find direct links to the built .deb packages of ClickHouse which you can deploy even on your production servers (if you have no fear). diff --git a/docs/en/development/style.md b/docs/en/development/style.md index 49b2f68b9f3..03121880555 100644 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -322,7 +322,7 @@ std::string getName() const override { return "Memory"; } class StorageMemory : public IStorage ``` -**4.** `using` are named the same way as classes, or with `_t` on the end. +**4.** `using` are named the same way as classes. **5.** Names of template type arguments: in simple cases, use `T`; `T`, `U`; `T1`, `T2`. @@ -404,9 +404,9 @@ enum class CompressionMethod }; ``` -**15.** All names must be in English. Transliteration of Russian words is not allowed. +**15.** All names must be in English. Transliteration of Hebrew words is not allowed. - not Stroka + not T_PAAMAYIM_NEKUDOTAYIM **16.** Abbreviations are acceptable if they are well known (when you can easily find the meaning of the abbreviation in Wikipedia or in a search engine). @@ -490,7 +490,7 @@ if (0 != close(fd)) throwFromErrno("Cannot close file " + file_name, ErrorCodes::CANNOT_CLOSE_FILE); ``` -`Do not use assert`. +You can use assert to check invariants in code. **4.** Exception types. @@ -571,7 +571,7 @@ Don’t use these types for numbers: `signed/unsigned long`, `long long`, `short **13.** Passing arguments. -Pass complex values by reference (including `std::string`). +Pass complex values by value if they are going to be moved and use std::move; pass by reference if you want to update value in a loop. If a function captures ownership of an object created in the heap, make the argument type `shared_ptr` or `unique_ptr`. @@ -581,7 +581,7 @@ In most cases, just use `return`. Do not write `return std::move(res)`. If the function allocates an object on heap and returns it, use `shared_ptr` or `unique_ptr`. -In rare cases you might need to return the value via an argument. In this case, the argument should be a reference. +In rare cases (updating a value in a loop) you might need to return the value via an argument. In this case, the argument should be a reference. ``` cpp using AggregateFunctionPtr = std::shared_ptr; diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 5f3245c4d60..be9fc7907af 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -11,7 +11,7 @@ Functional tests are the most simple and convenient to use. Most of ClickHouse f Each functional test sends one or multiple queries to the running ClickHouse server and compares the result with reference. -Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and it is available to general public. +Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from CLickHouse and it is available to general public. Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`. @@ -133,44 +133,6 @@ If the system clickhouse-server is already running and you do not want to stop i `clickhouse` binary has almost no dependencies and works across wide range of Linux distributions. To quick and dirty test your changes on a server, you can simply `scp` your fresh built `clickhouse` binary to your server and then run it as in examples above. -## Testing Environment {#testing-environment} - -Before publishing release as stable we deploy it on testing environment. Testing environment is a cluster that process 1/39 part of [Yandex.Metrica](https://metrica.yandex.com/) data. We share our testing environment with Yandex.Metrica team. ClickHouse is upgraded without downtime on top of existing data. We look at first that data is processed successfully without lagging from realtime, the replication continue to work and there is no issues visible to Yandex.Metrica team. First check can be done in the following way: - -``` sql -SELECT hostName() AS h, any(version()), any(uptime()), max(UTCEventTime), count() FROM remote('example01-01-{1..3}t', merge, hits) WHERE EventDate >= today() - 2 GROUP BY h ORDER BY h; -``` - -In some cases we also deploy to testing environment of our friend teams in Yandex: Market, Cloud, etc. Also we have some hardware servers that are used for development purposes. - -## Load Testing {#load-testing} - -After deploying to testing environment we run load testing with queries from production cluster. This is done manually. - -Make sure you have enabled `query_log` on your production cluster. - -Collect query log for a day or more: - -``` bash -$ clickhouse-client --query="SELECT DISTINCT query FROM system.query_log WHERE event_date = today() AND query LIKE '%ym:%' AND query NOT LIKE '%system.query_log%' AND type = 2 AND is_initial_query" > queries.tsv -``` - -This is a way complicated example. `type = 2` will filter queries that are executed successfully. `query LIKE '%ym:%'` is to select relevant queries from Yandex.Metrica. `is_initial_query` is to select only queries that are initiated by client, not by ClickHouse itself (as parts of distributed query processing). - -`scp` this log to your testing cluster and run it as following: - -``` bash -$ clickhouse benchmark --concurrency 16 < queries.tsv -``` - -(probably you also want to specify a `--user`) - -Then leave it for a night or weekend and go take a rest. - -You should check that `clickhouse-server` does not crash, memory footprint is bounded and performance not degrading over time. - -Precise query execution timings are not recorded and not compared due to high variability of queries and environment. - ## Build Tests {#build-tests} Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. These tests are automated as well. @@ -259,13 +221,13 @@ Thread Fuzzer (please don't mix up with Thread Sanitizer) is another kind of fuz ## Security Audit -People from Yandex Security Team did some basic overview of ClickHouse capabilities from the security standpoint. +Our Security Team did some basic overview of ClickHouse capabilities from the security standpoint. ## Static Analyzers {#static-analyzers} -We run `clang-tidy` and `PVS-Studio` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks. +We run `clang-tidy` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks. -We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). +We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of the box. @@ -310,12 +272,6 @@ Alternatively you can try `uncrustify` tool to reformat your code. Configuration We also use `codespell` to find typos in code. It is automated as well. -## Metrica B2B Tests {#metrica-b2b-tests} - -Each ClickHouse release is tested with Yandex Metrica and AppMetrica engines. Testing and stable versions of ClickHouse are deployed on VMs and run with a small copy of Metrica engine that is processing fixed sample of input data. Then results of two instances of Metrica engine are compared together. - -These tests are automated by separate team. Due to high number of moving parts, tests are fail most of the time by completely unrelated reasons, that are very difficult to figure out. Most likely these tests have negative value for us. Nevertheless these tests was proved to be useful in about one or two times out of hundreds. - ## Test Coverage {#test-coverage} We also track test coverage but only for functional tests and only for clickhouse-server. It is performed on daily basis. diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index bcb026aa0dc..d7dcf21cb02 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -36,6 +36,7 @@ ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'passwo - `max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`. - `max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`. - `allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`). +- `materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated. ```sql CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***') @@ -75,7 +76,7 @@ When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree]( | FLOAT | [Float32](../../sql-reference/data-types/float.md) | | DOUBLE | [Float64](../../sql-reference/data-types/float.md) | | DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) | -| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) | +| DATE, NEWDATE | [Date32](../../sql-reference/data-types/date32.md) | | DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | | DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) | | YEAR | [UInt16](../../sql-reference/data-types/int-uint.md) | diff --git a/docs/en/engines/database-engines/mysql.md b/docs/en/engines/database-engines/mysql.md index c5a1bba44b2..df4965b1f8c 100644 --- a/docs/en/engines/database-engines/mysql.md +++ b/docs/en/engines/database-engines/mysql.md @@ -49,6 +49,8 @@ ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') All other MySQL data types are converted into [String](../../sql-reference/data-types/string.md). +Because of the ClickHouse date type has a different range from the MySQL date range,If the MySQL date type is out of the range of ClickHouse date, you can use the setting mysql_datatypes_support_level to modify the mapping from the MySQL date type to the Clickhouse date type: date2Date32 (convert MySQL's date type to ClickHouse Date32) or date2String(convert MySQL's date type to ClickHouse String,this is usually used when your mysql data is less than 1925) or default(convert MySQL's date type to ClickHouse Date). + [Nullable](../../sql-reference/data-types/nullable.md) is supported. ## Global Variables Support {#global-variables-support} diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 9fdc43f3f18..1d80f143098 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -186,6 +186,7 @@ Example: - `_key` — Key of the message. - `_offset` — Offset of the message. - `_timestamp` — Timestamp of the message. +- `_timestamp_ms` — Timestamp in milliseconds of the message. - `_partition` — Partition of Kafka topic. **See Also** diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 691666cffef..c7301a55bf0 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -26,7 +26,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) ``` sql CREATE TABLE s3_engine_table (name String, value UInt32) - ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'gzip') + ENGINE=S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'gzip') SETTINGS input_format_with_names_use_header = 0; INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3); @@ -75,19 +75,19 @@ Create table with files named `file-000.csv`, `file-001.csv`, … , `file-999.cs ``` sql CREATE TABLE big_table (name String, value UInt32) - ENGINE = S3('https://storage.yandexcloud.net/my-bucket/my_folder/file-{000..999}.csv', 'CSV'); + ENGINE = S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/my_folder/file-{000..999}.csv', 'CSV'); ``` **Example with wildcards 2** Suppose we have several files in CSV format with the following URIs on S3: -- 'https://storage.yandexcloud.net/my-bucket/some_folder/some_file_1.csv' -- 'https://storage.yandexcloud.net/my-bucket/some_folder/some_file_2.csv' -- 'https://storage.yandexcloud.net/my-bucket/some_folder/some_file_3.csv' -- 'https://storage.yandexcloud.net/my-bucket/another_folder/some_file_1.csv' -- 'https://storage.yandexcloud.net/my-bucket/another_folder/some_file_2.csv' -- 'https://storage.yandexcloud.net/my-bucket/another_folder/some_file_3.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/some_folder/some_file_1.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/some_folder/some_file_2.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/some_folder/some_file_3.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/another_folder/some_file_1.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/another_folder/some_file_2.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/another_folder/some_file_3.csv' There are several ways to make a table consisting of all six files: @@ -96,21 +96,21 @@ There are several ways to make a table consisting of all six files: ``` sql CREATE TABLE table_with_range (name String, value UInt32) - ENGINE = S3('https://storage.yandexcloud.net/my-bucket/{some,another}_folder/some_file_{1..3}', 'CSV'); + ENGINE = S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/{some,another}_folder/some_file_{1..3}', 'CSV'); ``` 2. Take all files with `some_file_` prefix (there should be no extra files with such prefix in both folders): ``` sql CREATE TABLE table_with_question_mark (name String, value UInt32) - ENGINE = S3('https://storage.yandexcloud.net/my-bucket/{some,another}_folder/some_file_?', 'CSV'); + ENGINE = S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/{some,another}_folder/some_file_?', 'CSV'); ``` 3. Take all the files in both folders (all files should satisfy format and schema described in query): ``` sql CREATE TABLE table_with_asterisk (name String, value UInt32) - ENGINE = S3('https://storage.yandexcloud.net/my-bucket/{some,another}_folder/*', 'CSV'); + ENGINE = S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/{some,another}_folder/*', 'CSV'); ``` ## S3-related Settings {#settings} @@ -142,7 +142,7 @@ The following settings can be specified in configuration file for given endpoint ``` xml - https://storage.yandexcloud.net/my-test-bucket-768/ + https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/ diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index 5ac2105e9fd..b58e90a3d92 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -55,27 +55,28 @@ WHERE table = 'visits' ``` ``` text -┌─partition─┬─name───────────┬─active─┐ -│ 201901 │ 201901_1_3_1 │ 0 │ -│ 201901 │ 201901_1_9_2 │ 1 │ -│ 201901 │ 201901_8_8_0 │ 0 │ -│ 201901 │ 201901_9_9_0 │ 0 │ -│ 201902 │ 201902_4_6_1 │ 1 │ -│ 201902 │ 201902_10_10_0 │ 1 │ -│ 201902 │ 201902_11_11_0 │ 1 │ -└───────────┴────────────────┴────────┘ +┌─partition─┬─name──────────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2_11 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1_11 │ 1 │ +│ 201902 │ 201902_10_10_0_11 │ 1 │ +│ 201902 │ 201902_11_11_0_11 │ 1 │ +└───────────┴───────────────────┴────────┘ ``` The `partition` column contains the names of the partitions. There are two partitions in this example: `201901` and `201902`. You can use this column value to specify the partition name in [ALTER … PARTITION](../../../sql-reference/statements/alter/partition.md) queries. The `name` column contains the names of the partition data parts. You can use this column to specify the name of the part in the [ALTER ATTACH PART](../../../sql-reference/statements/alter/partition.md#alter_attach-partition) query. -Let’s break down the name of the first part: `201901_1_3_1`: +Let’s break down the name of the part: `201901_1_9_2_11`: - `201901` is the partition name. - `1` is the minimum number of the data block. -- `3` is the maximum number of the data block. -- `1` is the chunk level (the depth of the merge tree it is formed from). +- `9` is the maximum number of the data block. +- `2` is the chunk level (the depth of the merge tree it is formed from). +- `11` is the mutation version (if a part mutated) !!! info "Info" The parts of old-type tables have the name: `20190117_20190123_2_2_0` (minimum date - maximum date - minimum block number - maximum block number - level). @@ -89,16 +90,16 @@ OPTIMIZE TABLE visits PARTITION 201902; ``` ``` text -┌─partition─┬─name───────────┬─active─┐ -│ 201901 │ 201901_1_3_1 │ 0 │ -│ 201901 │ 201901_1_9_2 │ 1 │ -│ 201901 │ 201901_8_8_0 │ 0 │ -│ 201901 │ 201901_9_9_0 │ 0 │ -│ 201902 │ 201902_4_6_1 │ 0 │ -│ 201902 │ 201902_4_11_2 │ 1 │ -│ 201902 │ 201902_10_10_0 │ 0 │ -│ 201902 │ 201902_11_11_0 │ 0 │ -└───────────┴────────────────┴────────┘ +┌─partition─┬─name─────────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2_11 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1 │ 0 │ +│ 201902 │ 201902_4_11_2_11 │ 1 │ +│ 201902 │ 201902_10_10_0 │ 0 │ +│ 201902 │ 201902_11_11_0 │ 0 │ +└───────────┴──────────────────┴────────┘ ``` Inactive parts will be deleted approximately 10 minutes after merging. @@ -109,12 +110,12 @@ Another way to view a set of parts and partitions is to go into the directory of /var/lib/clickhouse/data/default/visits$ ls -l total 40 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 201901_1_3_1 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2_11 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_8_8_0 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_9_9_0 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_10_10_0 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_11_11_0 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:19 201902_4_11_2 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:19 201902_4_11_2_11 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 12:09 201902_4_6_1 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached ``` diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 92865c94475..a0acda5d5c6 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -802,7 +802,7 @@ Configuration markup: s3 - https://storage.yandexcloud.net/my-bucket/root-path/ + https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/root-path/ your_access_key_id your_secret_access_key @@ -856,7 +856,7 @@ S3 disk can be configured as `main` or `cold` storage: s3 - https://storage.yandexcloud.net/my-bucket/root-path/ + https://clickhouse-public-datasets.s3.amazonaws.com/my-bucket/root-path/ your_access_key_id your_secret_access_key @@ -887,6 +887,57 @@ S3 disk can be configured as `main` or `cold` storage: In case of `cold` option a data can be moved to S3 if local disk free size will be smaller than `move_factor * disk_size` or by TTL move rule. +## Using Azure Blob Storage for Data Storage {#table_engine-mergetree-azure-blob-storage} + +`MergeTree` family table engines can store data to [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) using a disk with type `azure_blob_storage`. + +As of February 2022, this feature is still a fresh addition, so expect that some Azure Blob Storage functionalities might be unimplemented. + +Configuration markup: +``` xml + + ... + + + azure_blob_storage + http://account.blob.core.windows.net + container + account + pass123 + /var/lib/clickhouse/disks/blob_storage_disk/ + true + /var/lib/clickhouse/disks/blob_storage_disk/cache/ + false + + + ... + +``` + +Connection parameters: +* `storage_account_url` - **Required**, Azure Blob Storage account URL, like `http://account.blob.core.windows.net` or `http://azurite1:10000/devstoreaccount1`. +* `container_name` - Target container name, defaults to `default-container`. +* `container_already_exists` - If set to `false`, a new container `container_name` is created in the storage account, if set to `true`, disk connects to the container directly, and if left unset, disk connects to the account, checks if the container `container_name` exists, and creates it if it doesn't exist yet. + +Authentication parameters (the disk will try all available methods **and** Managed Identity Credential): +* `connection_string` - For authentication using a connection string. +* `account_name` and `account_key` - For authentication using Shared Key. + +Limit parameters (mainly for internal usage): +* `max_single_part_upload_size` - Limits the size of a single block upload to Blob Storage. +* `min_bytes_for_seek` - Limits the size of a seekable region. +* `max_single_read_retries` - Limits the number of attempts to read a chunk of data from Blob Storage. +* `max_single_download_retries` - Limits the number of attempts to download a readable buffer from Blob Storage. +* `thread_pool_size` - Limits the number of threads with which `IDiskRemote` is instantiated. + +Other parameters: +* `metadata_path` - Path on local FS to store metadata files for Blob Storage. Default value is `/var/lib/clickhouse/disks//`. +* `cache_enabled` - Allows to cache mark and index files on local FS. Default value is `true`. +* `cache_path` - Path on local FS where to store cached mark and index files. Default value is `/var/lib/clickhouse/disks//cache/`. +* `skip_access_check` - If true, disk access checks will not be performed on disk start-up. Default value is `false`. + +Examples of working configurations can be found in integration tests directory (see e.g. [test_merge_tree_azure_blob_storage](https://github.com/ClickHouse/ClickHouse/blob/master/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml) or [test_azure_blob_storage_zero_copy_replication](https://github.com/ClickHouse/ClickHouse/blob/master/tests/integration/test_azure_blob_storage_zero_copy_replication/configs/config.d/storage_conf.xml)). + ## Virtual Columns {#virtual-columns} - `_part` — Name of a part. diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 7cd58d35362..d574bd9449e 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -97,7 +97,7 @@ ZooKeeper is not used in `SELECT` queries because replication does not affect th For each `INSERT` query, approximately ten entries are added to ZooKeeper through several transactions. (To be more precise, this is for each inserted block of data; an INSERT query contains one block or one block per `max_insert_block_size = 1048576` rows.) This leads to slightly longer latencies for `INSERT` compared to non-replicated tables. But if you follow the recommendations to insert data in batches of no more than one `INSERT` per second, it does not create any problems. The entire ClickHouse cluster used for coordinating one ZooKeeper cluster has a total of several hundred `INSERTs` per second. The throughput on data inserts (the number of rows per second) is just as high as for non-replicated data. -For very large clusters, you can use different ZooKeeper clusters for different shards. However, this hasn’t proven necessary on the Yandex.Metrica cluster (approximately 300 servers). +For very large clusters, you can use different ZooKeeper clusters for different shards. However, from our experience this has not proven necessary based on production clusters with approximately 300 servers. Replication is asynchronous and multi-master. `INSERT` queries (as well as `ALTER`) can be sent to any available server. Data is inserted on the server where the query is run, and then it is copied to the other servers. Because it is asynchronous, recently inserted data appears on the other replicas with some latency. If part of the replicas are not available, the data is written when they become available. If a replica is available, the latency is the amount of time it takes to transfer the block of compressed data over the network. The number of threads performing background tasks for replicated tables can be set by [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) setting. @@ -111,7 +111,7 @@ Data blocks are deduplicated. For multiple writes of the same data block (data b During replication, only the source data to insert is transferred over the network. Further data transformation (merging) is coordinated and performed on all the replicas in the same way. This minimizes network usage, which means that replication works well when replicas reside in different datacenters. (Note that duplicating data in different datacenters is the main goal of replication.) -You can have any number of replicas of the same data. Yandex.Metrica uses double replication in production. Each server uses RAID-5 or RAID-6, and RAID-10 in some cases. This is a relatively reliable and convenient solution. +You can have any number of replicas of the same data. Based on our experiences, a relatively reliable and convenient solution could use double replication in production, with each server using RAID-5 or RAID-6 (and RAID-10 in some cases). The system monitors data synchronicity on replicas and is able to recover after a failure. Failover is automatic (for small differences in data) or semi-automatic (when data differs too much, which may indicate a configuration error). @@ -163,7 +163,7 @@ Example: 05 02 - example05-02-1.yandex.ru + example05-02-1 ``` @@ -172,7 +172,7 @@ In this case, the path consists of the following parts: `/clickhouse/tables/` is the common prefix. We recommend using exactly this one. -`{layer}-{shard}` is the shard identifier. In this example it consists of two parts, since the Yandex.Metrica cluster uses bi-level sharding. For most tasks, you can leave just the {shard} substitution, which will be expanded to the shard identifier. +`{layer}-{shard}` is the shard identifier. In this example it consists of two parts, since the example cluster uses bi-level sharding. For most tasks, you can leave just the {shard} substitution, which will be expanded to the shard identifier. `table_name` is the name of the node for the table in ZooKeeper. It is a good idea to make it the same as the table name. It is defined explicitly, because in contrast to the table name, it does not change after a RENAME query. *HINT*: you could add a database name in front of `table_name` as well. E.g. `db_name.table_name` diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index faa1026b919..5072465687e 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -197,7 +197,7 @@ A simple remainder from the division is a limited solution for sharding and isn You should be concerned about the sharding scheme in the following cases: - Queries are used that require joining data (`IN` or `JOIN`) by a specific key. If data is sharded by this key, you can use local `IN` or `JOIN` instead of `GLOBAL IN` or `GLOBAL JOIN`, which is much more efficient. -- A large number of servers is used (hundreds or more) with a large number of small queries, for example, queries for data of individual clients (e.g. websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, as we’ve done in Yandex.Metrica, you can set up bi-level sharding: divide the entire cluster into “layers”, where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. `Distributed` tables are created for each layer, and a single shared distributed table is created for global queries. +- A large number of servers is used (hundreds or more) with a large number of small queries, for example, queries for data of individual clients (e.g. websites, advertisers, or partners). In order for the small queries to not affect the entire cluster, it makes sense to locate data for a single client on a single shard. Alternatively, you can set up bi-level sharding: divide the entire cluster into “layers”, where a layer may consist of multiple shards. Data for a single client is located on a single layer, but shards can be added to a layer as necessary, and data is randomly distributed within them. `Distributed` tables are created for each layer, and a single shared distributed table is created for global queries. Data is written asynchronously. When inserted in the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The periodicity for sending data is managed by the [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) and [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) settings. The `Distributed` engine sends each file with inserted data separately, but you can enable batch sending of files with the [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) setting. This setting improves cluster performance by better utilizing local server and network resources. You should check whether data is sent successfully by checking the list of files (data waiting to be sent) in the table directory: `/var/lib/clickhouse/data/database/table/`. The number of threads performing background tasks can be set by [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) setting. @@ -209,6 +209,8 @@ When querying a `Distributed` table, `SELECT` queries are sent to all shards and When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). +To learn more about how distibuted `in` and `global in` queries are processed, refer to [this](../../../sql-reference/operators/in.md#select-distributed-subqueries) documentation. + ## Virtual Columns {#virtual-columns} - `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/en/engines/table-engines/special/url.md b/docs/en/engines/table-engines/special/url.md index 04f035206b5..26d928085ce 100644 --- a/docs/en/engines/table-engines/special/url.md +++ b/docs/en/engines/table-engines/special/url.md @@ -7,18 +7,29 @@ toc_title: URL Queries data to/from a remote HTTP/HTTPS server. This engine is similar to the [File](../../../engines/table-engines/special/file.md) engine. -Syntax: `URL(URL, Format)` +Syntax: `URL(URL [,Format] [,CompressionMethod])` + +- The `URL` parameter must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server that uses HTTP or HTTPS. This does not require any additional headers for getting a response from the server. + +- The `Format` must be one that ClickHouse can use in `SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see [Formats](../../../interfaces/formats.md#formats). + +- `CompressionMethod` indicates that whether the HTTP body should be compressed. If the compression is enabled, the HTTP packets sent by the URL engine contain 'Content-Encoding' header to indicate which compression method is used. + +To enable compression, please first make sure the remote HTTP endpoint indicated by the `URL` parameter supports corresponding compression algorithm. + +The supported `CompressionMethod` should be one of following: +- gzip or gz +- deflate +- brotli or br +- lzma or xz +- zstd or zst +- lz4 +- bz2 +- snappy +- none ## Usage {#using-the-engine-in-the-clickhouse-server} -The `format` must be one that ClickHouse can use in -`SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see -[Formats](../../../interfaces/formats.md#formats). - -The `URL` must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server -that uses HTTP or HTTPS. This does not require any -additional headers for getting a response from the server. - `INSERT` and `SELECT` queries are transformed to `POST` and `GET` requests, respectively. For processing `POST` requests, the remote server must support [Chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). diff --git a/docs/en/faq/general/columnar-database.md b/docs/en/faq/general/columnar-database.md index cbc5f77d0ba..11bbd2e63f6 100644 --- a/docs/en/faq/general/columnar-database.md +++ b/docs/en/faq/general/columnar-database.md @@ -22,4 +22,4 @@ Here is the illustration of the difference between traditional row-oriented syst **Columnar** ![Columnar](https://clickhouse.com/docs/en/images/column-oriented.gif#) -A columnar database is a preferred choice for analytical applications because it allows to have many columns in a table just in case, but do not pay the cost for unused columns on read query execution time. Column-oriented databases are designed for big data processing because and data warehousing, they often natively scale using distributed clusters of low-cost hardware to increase throughput. ClickHouse does it with combination of [distributed](../../engines/table-engines/special/distributed.md) and [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. +A columnar database is a preferred choice for analytical applications because it allows to have many columns in a table just in case, but do not pay the cost for unused columns on read query execution time. Column-oriented databases are designed for big data processing and data warehousing, because they often natively scale using distributed clusters of low-cost hardware to increase throughput. ClickHouse does it with combination of [distributed](../../engines/table-engines/special/distributed.md) and [replicated](../../engines/table-engines/mergetree-family/replication.md) tables. diff --git a/docs/en/faq/general/mapreduce.md b/docs/en/faq/general/mapreduce.md index 7d25d308d14..30cae65cba2 100644 --- a/docs/en/faq/general/mapreduce.md +++ b/docs/en/faq/general/mapreduce.md @@ -6,7 +6,7 @@ toc_priority: 110 # Why Not Use Something Like MapReduce? {#why-not-use-something-like-mapreduce} -We can refer to systems like MapReduce as distributed computing systems in which the reduce operation is based on distributed sorting. The most common open-source solution in this class is [Apache Hadoop](http://hadoop.apache.org). Yandex uses its in-house solution, YT. +We can refer to systems like MapReduce as distributed computing systems in which the reduce operation is based on distributed sorting. The most common open-source solution in this class is [Apache Hadoop](http://hadoop.apache.org). Large IT companies often have proprietary in-house solutions. These systems aren’t appropriate for online queries due to their high latency. In other words, they can’t be used as the back-end for a web interface. These types of systems aren’t useful for real-time data updates. Distributed sorting isn’t the best way to perform reduce operations if the result of the operation and all the intermediate results (if there are any) are located in the RAM of a single server, which is usually the case for online queries. In such a case, a hash table is an optimal way to perform reduce operations. A common approach to optimizing map-reduce tasks is pre-aggregation (partial reduce) using a hash table in RAM. The user performs this optimization manually. Distributed sorting is one of the main causes of reduced performance when running simple map-reduce tasks. diff --git a/docs/en/faq/general/ne-tormozit.md b/docs/en/faq/general/ne-tormozit.md index 26ae741216d..e8dc7388eff 100644 --- a/docs/en/faq/general/ne-tormozit.md +++ b/docs/en/faq/general/ne-tormozit.md @@ -9,7 +9,7 @@ toc_priority: 11 This question usually arises when people see official ClickHouse t-shirts. They have large words **“ClickHouse не тормозит”** on the front. -Before ClickHouse became open-source, it has been developed as an in-house storage system by the largest Russian IT company, [Yandex](https://yandex.com/company/). That’s why it initially got its slogan in Russian, which is “не тормозит” (pronounced as “ne tormozit”). After the open-source release we first produced some of those t-shirts for events in Russia and it was a no-brainer to use the slogan as-is. +Before ClickHouse became open-source, it has been developed as an in-house storage system by the largest Russian IT company, Yandex. That’s why it initially got its slogan in Russian, which is “не тормозит” (pronounced as “ne tormozit”). After the open-source release we first produced some of those t-shirts for events in Russia and it was a no-brainer to use the slogan as-is. One of the following batches of those t-shirts was supposed to be given away on events outside of Russia and we tried to make the English version of the slogan. Unfortunately, the Russian language is kind of elegant in terms of expressing stuff and there was a restriction of limited space on a t-shirt, so we failed to come up with good enough translation (most options appeared to be either long or inaccurate) and decided to keep the slogan in Russian even on t-shirts produced for international events. It appeared to be a great decision because people all over the world get positively surprised and curious when they see it. diff --git a/docs/en/getting-started/example-datasets/index.md b/docs/en/getting-started/example-datasets/index.md index 6dae6c20073..d4c9bab2441 100644 --- a/docs/en/getting-started/example-datasets/index.md +++ b/docs/en/getting-started/example-datasets/index.md @@ -11,7 +11,7 @@ This section describes how to obtain example datasets and import them into Click The list of documented datasets: - [GitHub Events](../../getting-started/example-datasets/github-events.md) -- [Anonymized Yandex.Metrica Dataset](../../getting-started/example-datasets/metrica.md) +- [Anonymized Web Analytics Dataset](../../getting-started/example-datasets/metrica.md) - [Recipes](../../getting-started/example-datasets/recipes.md) - [Star Schema Benchmark](../../getting-started/example-datasets/star-schema.md) - [WikiStat](../../getting-started/example-datasets/wikistat.md) diff --git a/docs/en/getting-started/example-datasets/metrica.md b/docs/en/getting-started/example-datasets/metrica.md index 483220d12ee..d9d8beb0181 100644 --- a/docs/en/getting-started/example-datasets/metrica.md +++ b/docs/en/getting-started/example-datasets/metrica.md @@ -1,11 +1,11 @@ --- toc_priority: 15 -toc_title: Yandex.Metrica Data +toc_title: Web Analytics Data --- -# Anonymized Yandex.Metrica Data {#anonymized-yandex-metrica-data} +# Anonymized Web Analytics Data {#anonymized-web-analytics-data} -Dataset consists of two tables containing anonymized data about hits (`hits_v1`) and visits (`visits_v1`) of Yandex.Metrica. You can read more about Yandex.Metrica in [ClickHouse history](../../introduction/history.md) section. +Dataset consists of two tables containing anonymized web analytics data with hits (`hits_v1`) and visits (`visits_v1`). The dataset consists of two tables, either of them can be downloaded as a compressed `tsv.xz` file or as prepared partitions. In addition to that, an extended version of the `hits` table containing 100 million rows is available as TSV at https://datasets.clickhouse.com/hits/tsv/hits_100m_obfuscated_v1.tsv.xz and as prepared partitions at https://datasets.clickhouse.com/hits/partitions/hits_100m_obfuscated_v1.tar.xz. @@ -73,6 +73,6 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ## Example Queries {#example-queries} -[ClickHouse tutorial](../../getting-started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. +[The ClickHouse tutorial](../../getting-started/tutorial.md) is based on this web analytics dataset, and the recommended way to get started with this dataset is to go through the tutorial. Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hits` and `test.visits` there). diff --git a/docs/en/getting-started/example-datasets/nyc-taxi.md b/docs/en/getting-started/example-datasets/nyc-taxi.md index 64810d3fa37..a7825988695 100644 --- a/docs/en/getting-started/example-datasets/nyc-taxi.md +++ b/docs/en/getting-started/example-datasets/nyc-taxi.md @@ -375,7 +375,7 @@ Q3: 0.051 sec. Q4: 0.072 sec. In this case, the query processing time is determined above all by network latency. -We ran queries using a client located in a Yandex datacenter in Finland on a cluster in Russia, which added about 20 ms of latency. +We ran queries using a client located in a different datacenter than where the cluster was located, which added about 20 ms of latency. ## Summary {#summary} diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 70a1b8349ff..cd734d4dc8b 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -27,9 +27,17 @@ It is recommended to use official pre-compiled `deb` packages for Debian or Ubun {% include 'install/deb.sh' %} ``` +
+ +Deprecated Method for installing deb-packages +``` bash +{% include 'install/deb_repo.sh' %} +``` +
+ You can replace `stable` with `lts` or `testing` to use different [release trains](../faq/operations/production.md) based on your needs. -You can also download and install packages manually from [here](https://repo.clickhouse.com/deb/stable/main/). +You can also download and install packages manually from [here](https://packages.clickhouse.com/deb/pool/stable). #### Packages {#packages} @@ -49,11 +57,17 @@ It is recommended to use official pre-compiled `rpm` packages for CentOS, RedHat First, you need to add the official repository: ``` bash -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 +{% include 'install/rpm.sh' %} ``` +
+ +Deprecated Method for installing rpm-packages +``` bash +{% include 'install/rpm_repo.sh' %} +``` +
+ If you want to use the most recent version, replace `stable` with `testing` (this is recommended for your testing environments). `prestable` is sometimes also available. Then run these commands to install packages: @@ -62,36 +76,27 @@ Then run these commands to install packages: sudo yum install clickhouse-server clickhouse-client ``` -You can also download and install packages manually from [here](https://repo.clickhouse.com/rpm/stable/x86_64). +You can also download and install packages manually from [here](https://packages.clickhouse.com/rpm/stable). ### From Tgz Archives {#from-tgz-archives} It is recommended to use official pre-compiled `tgz` archives for all Linux distributions, where installation of `deb` or `rpm` packages is not possible. -The required version can be downloaded with `curl` or `wget` from repository https://repo.clickhouse.com/tgz/. -After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest version: +The required version can be downloaded with `curl` or `wget` from repository https://packages.clickhouse.com/tgz/. +After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-client-$LATEST_VERSION.tgz - -tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz -sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz -sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-server-$LATEST_VERSION.tgz -sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh -sudo /etc/init.d/clickhouse-server start - -tar -xzvf clickhouse-client-$LATEST_VERSION.tgz -sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh +{% include 'install/tgz.sh' %} ``` +
+ +Deprecated Method for installing tgz archives +``` bash +{% include 'install/tgz_repo.sh' %} +``` +
+ For production environments, it’s recommended to use the latest `stable`-version. You can find its number on GitHub page https://github.com/ClickHouse/ClickHouse/tags with postfix `-stable`. ### From Docker Image {#from-docker-image} @@ -215,6 +220,6 @@ SELECT 1 **Congratulations, the system works!** -To continue experimenting, you can download one of the test data sets or go through [tutorial](https://clickhouse.com/tutorial.html). +To continue experimenting, you can download one of the test data sets or go through [tutorial](./tutorial.md). [Original article](https://clickhouse.com/docs/en/getting_started/install/) diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index 90e3eedb764..01d7dd5b69f 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -5,29 +5,19 @@ toc_title: Playground # ClickHouse Playground {#clickhouse-playground} -[ClickHouse Playground](https://play.clickhouse.com) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. -Several example datasets are available in Playground as well as sample queries that show ClickHouse features. There’s also a selection of ClickHouse LTS releases to experiment with. - -ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance (4 vCPU, 32 GB RAM) hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md). +[ClickHouse Playground](https://play.clickhouse.com/play?user=play) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. +Several example datasets are available in Playground. You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). ## Credentials {#credentials} -| Parameter | Value | -|:--------------------|:----------------------------------------| -| HTTPS endpoint | `https://play-api.clickhouse.com:8443` | -| Native TCP endpoint | `play-api.clickhouse.com:9440` | -| User | `playground` | -| Password | `clickhouse` | - -There are additional endpoints with specific ClickHouse releases to experiment with their differences (ports and user/password are the same as above): - -- 20.3 LTS: `play-api-v20-3.clickhouse.com` -- 19.14 LTS: `play-api-v19-14.clickhouse.com` - -!!! note "Note" - All these endpoints require a secure TLS connection. +| Parameter | Value | +|:--------------------|:-----------------------------------| +| HTTPS endpoint | `https://play.clickhouse.com:443/` | +| Native TCP endpoint | `play.clickhouse.com:9440` | +| User | `explorer` or `play` | +| Password | (empty) | ## Limitations {#limitations} @@ -36,31 +26,18 @@ The queries are executed as a read-only user. It implies some limitations: - DDL queries are not allowed - INSERT queries are not allowed -The following settings are also enforced: - -- [max_result_bytes=10485760](../operations/settings/query-complexity/#max-result-bytes) -- [max_result_rows=2000](../operations/settings/query-complexity/#setting-max_result_rows) -- [result_overflow_mode=break](../operations/settings/query-complexity/#result-overflow-mode) -- [max_execution_time=60000](../operations/settings/query-complexity/#max-execution-time) +The service also have quotas on its usage. ## Examples {#examples} HTTPS endpoint example with `curl`: ``` bash -curl "https://play-api.clickhouse.com:8443/?query=SELECT+'Play+ClickHouse\!';&user=playground&password=clickhouse&database=datasets" +curl "https://play.clickhouse.com/?user=explorer" --data-binary "SELECT 'Play ClickHouse'" ``` TCP endpoint example with [CLI](../interfaces/cli.md): ``` bash -clickhouse client --secure -h play-api.clickhouse.com --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse\!'" +clickhouse client --secure --host play.clickhouse.com --user explorer ``` - -## Implementation Details {#implementation-details} - -ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md). -The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add an extra layer of protection and improved global connectivity. - -!!! warning "Warning" - Exposing the ClickHouse server to the public internet in any other situation is **strongly not recommended**. Make sure it listens only on a private network and is covered by a properly configured firewall. diff --git a/docs/en/getting-started/tutorial.md b/docs/en/getting-started/tutorial.md index e08b319f2a4..9f43cc8769d 100644 --- a/docs/en/getting-started/tutorial.md +++ b/docs/en/getting-started/tutorial.md @@ -80,7 +80,7 @@ clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv ## Import Sample Dataset {#import-sample-dataset} -Now it’s time to fill our ClickHouse server with some sample data. In this tutorial, we’ll use the anonymized data of Yandex.Metrica, the first service that runs ClickHouse in production way before it became open-source (more on that in [history section](../introduction/history.md)). There are [multiple ways to import Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md), and for the sake of the tutorial, we’ll go with the most realistic one. +Now it’s time to fill our ClickHouse server with some sample data. In this tutorial, we’ll use some anonymized web analytics data. There are [multiple ways to import the dataset](../getting-started/example-datasets/metrica.md), and for the sake of the tutorial, we’ll go with the most realistic one. ### Download and Extract Table Data {#download-and-extract-table-data} @@ -105,7 +105,7 @@ Syntax for creating tables is way more complicated compared to databases (see [r 2. Table schema, i.e. list of columns and their [data types](../sql-reference/data-types/index.md). 3. [Table engine](../engines/table-engines/index.md) and its settings, which determines all the details on how queries to this table will be physically executed. -Yandex.Metrica is a web analytics service, and sample dataset does not cover its full functionality, so there are only two tables to create: +There are two tables to create: - `hits` is a table with each action done by all users on all websites covered by the service. - `visits` is a table that contains pre-built sessions instead of individual actions. @@ -533,19 +533,19 @@ Example config for a cluster with three shards, one replica each: - example-perftest01j.yandex.ru + example-perftest01j 9000 - example-perftest02j.yandex.ru + example-perftest02j 9000 - example-perftest03j.yandex.ru + example-perftest03j 9000 @@ -591,15 +591,15 @@ Example config for a cluster of one shard containing three replicas: - example-perftest01j.yandex.ru + example-perftest01j 9000 - example-perftest02j.yandex.ru + example-perftest02j 9000 - example-perftest03j.yandex.ru + example-perftest03j 9000 @@ -617,15 +617,15 @@ ZooKeeper locations are specified in the configuration file: ``` xml - zoo01.yandex.ru + zoo01 2181 - zoo02.yandex.ru + zoo02 2181 - zoo03.yandex.ru + zoo03 2181 diff --git a/docs/en/guides/apply-catboost-model.md b/docs/en/guides/apply-catboost-model.md index 9fd48fcc62d..859703a31df 100644 --- a/docs/en/guides/apply-catboost-model.md +++ b/docs/en/guides/apply-catboost-model.md @@ -5,7 +5,7 @@ toc_title: Applying CatBoost Models # Applying a Catboost Model in ClickHouse {#applying-catboost-model-in-clickhouse} -[CatBoost](https://catboost.ai) is a free and open-source gradient boosting library developed at [Yandex](https://yandex.com/company/) for machine learning. +[CatBoost](https://catboost.ai) is a free and open-source gradient boosting library developed at Yandex for machine learning. With this instruction, you will learn to apply pre-trained models in ClickHouse by running model inference from SQL. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index f266d0e6354..a7066fca087 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -51,6 +51,7 @@ The supported formats are: | [PrettySpace](#prettyspace) | ✗ | ✔ | | [Protobuf](#protobuf) | ✔ | ✔ | | [ProtobufSingle](#protobufsingle) | ✔ | ✔ | +| [ProtobufList](#protobuflist) | ✔ | ✔ | | [Avro](#data-format-avro) | ✔ | ✔ | | [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | | [Parquet](#data-format-parquet) | ✔ | ✔ | @@ -64,7 +65,7 @@ The supported formats are: | [Null](#null) | ✗ | ✔ | | [XML](#xml) | ✗ | ✔ | | [CapnProto](#capnproto) | ✔ | ✔ | -| [LineAsString](#lineasstring) | ✔ | ✗ | +| [LineAsString](#lineasstring) | ✔ | ✔ | | [Regexp](#data-format-regexp) | ✔ | ✗ | | [RawBLOB](#rawblob) | ✔ | ✔ | | [MsgPack](#msgpack) | ✔ | ✔ | @@ -300,7 +301,7 @@ Result: Search phrase Count 8267016 bathroom interior design 2166 - yandex 1655 + clickhouse 1655 spring 2014 fashion 1549 freeform photos 1480 @@ -371,7 +372,7 @@ Similar to TabSeparated, but outputs a value in name=value format. Names are esc ``` text SearchPhrase= count()=8267016 SearchPhrase=bathroom interior design count()=2166 -SearchPhrase=yandex count()=1655 +SearchPhrase=clickhouse count()=1655 SearchPhrase=2014 spring fashion count()=1549 SearchPhrase=freeform photos count()=1480 SearchPhrase=angelina jolie count()=1245 @@ -401,7 +402,7 @@ Parsing allows the presence of the additional field `tskv` without the equal sig Comma Separated Values format ([RFC](https://tools.ietf.org/html/rfc4180)). -When formatting, rows are enclosed in double-quotes. A double quote inside a string is output as two double quotes in a row. There are no other rules for escaping characters. Date and date-time are enclosed in double-quotes. Numbers are output without quotes. Values are separated by a delimiter character, which is `,` by default. The delimiter character is defined in the setting [format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Rows are separated using the Unix line feed (LF). Arrays are serialized in CSV as follows: first, the array is serialized to a string as in TabSeparated format, and then the resulting string is output to CSV in double-quotes. Tuples in CSV format are serialized as separate columns (that is, their nesting in the tuple is lost). +When formatting, strings are enclosed in double-quotes. A double quote inside a string is output as two double quotes in a row. There are no other rules for escaping characters. Date and date-time are enclosed in double-quotes. Numbers are output without quotes. Values are separated by a delimiter character, which is `,` by default. The delimiter character is defined in the setting [format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter). Rows are separated using the Unix line feed (LF). Arrays are serialized in CSV as follows: first, the array is serialized to a string as in TabSeparated format, and then the resulting string is output to CSV in double-quotes. Tuples in CSV format are serialized as separate columns (that is, their nesting in the tuple is lost). ``` bash $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMAT CSV" < data.csv @@ -409,7 +410,7 @@ $ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FOR \*By default, the delimiter is `,`. See the [format_csv_delimiter](../operations/settings/settings.md#settings-format_csv_delimiter) setting for more information. -When parsing, all values can be parsed either with or without quotes. Both double and single quotes are supported. Rows can also be arranged without quotes. In this case, they are parsed up to the delimiter character or line feed (CR or LF). In violation of the RFC, when parsing rows without quotes, the leading and trailing spaces and tabs are ignored. For the line feed, Unix (LF), Windows (CR LF) and Mac OS Classic (CR LF) types are all supported. +When parsing, all values can be parsed either with or without quotes. Both double and single quotes are supported. Strings can also be arranged without quotes. In this case, they are parsed up to the delimiter character or line feed (CR or LF). In violation of the RFC, when parsing strings without quotes, the leading and trailing spaces and tabs are ignored. For the line feed, Unix (LF), Windows (CR LF) and Mac OS Classic (CR LF) types are all supported. If setting [input_format_csv_empty_as_default](../operations/settings/settings.md#settings-input_format_csv_empty_as_default) is enabled, empty unquoted input values are replaced with default values. For complex default expressions [input_format_defaults_for_omitted_fields](../operations/settings/settings.md#settings-input_format_defaults_for_omitted_fields) must be enabled too. @@ -1060,7 +1061,7 @@ XML format is suitable only for output, not for parsing. Example: 2166 - yandex + clickhouse 1655 @@ -1230,7 +1231,38 @@ See also [how to read/write length-delimited protobuf messages in popular langua ## ProtobufSingle {#protobufsingle} -Same as [Protobuf](#protobuf) but for storing/parsing single Protobuf message without length delimiters. +Same as [Protobuf](#protobuf) but for storing/parsing a single Protobuf message without length delimiter. +As a result, only a single table row can be written/read. + +## ProtobufList {#protobuflist} + +Similar to Protobuf but rows are represented as a sequence of sub-messages contained in a message with fixed name "Envelope". + +Usage example: + +``` sql +SELECT * FROM test.table FORMAT ProtobufList SETTINGS format_schema = 'schemafile:MessageType' +``` + +``` bash +cat protobuflist_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT ProtobufList SETTINGS format_schema='schemafile:MessageType'" +``` + +where the file `schemafile.proto` looks like this: + +``` capnp +syntax = "proto3"; + +message Envelope { + message MessageType { + string name = 1; + string surname = 2; + uint32 birthDate = 3; + repeated string phoneNumbers = 4; + }; + MessageType row = 1; +}; +``` ## Avro {#data-format-avro} @@ -1364,7 +1396,8 @@ The table below shows supported data types and how they match ClickHouse [data t | `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `DATE64` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `TIMESTAMP` | [DateTime64](../sql-reference/data-types/datetime64.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | — | [FixedString](../sql-reference/data-types/fixedstring.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | @@ -1421,7 +1454,8 @@ The table below shows supported data types and how they match ClickHouse [data t | `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT32` | | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `FLOAT64` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `DATE64` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `TIMESTAMP` | [DateTime64](../sql-reference/data-types/datetime64.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | @@ -1483,7 +1517,8 @@ The table below shows supported data types and how they match ClickHouse [data t | `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `DATE32` | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `TIMESTAMP` | +| `DATE64` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `TIMESTAMP` | [DateTime64](../sql-reference/data-types/datetime64.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md index 7b73cec22a0..e747b93a1a6 100644 --- a/docs/en/interfaces/index.md +++ b/docs/en/interfaces/index.md @@ -12,7 +12,7 @@ ClickHouse provides three network interfaces (they can be optionally wrapped in - [Native TCP](../interfaces/tcp.md), which has less overhead. - [gRPC](grpc.md). -In most cases it is recommended to use appropriate tool or library instead of interacting with those directly. Officially supported by Yandex are the following: +In most cases it is recommended to use an appropriate tool or library instead of interacting with those directly. The following are officially supported by ClickHouse: - [Command-line client](../interfaces/cli.md) - [JDBC driver](../interfaces/jdbc.md) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index 393974c60c4..c0e270b7207 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -143,6 +143,10 @@ Features: - Backup and restore. - RBAC. +### Zeppelin-Interpreter-for-ClickHouse {#zeppelin-interpreter-for-clickhouse} + +[Zeppelin-Interpreter-for-ClickHouse](https://github.com/SiderZhang/Zeppelin-Interpreter-for-ClickHouse) is a [Zeppelin](https://zeppelin.apache.org) interpreter for ClickHouse. Compared with JDBC interpreter, it can provide better timeout control for long running queries. + ## Commercial {#commercial} ### DataGrip {#datagrip} diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md index 4a4eee770dc..3aac78f0878 100644 --- a/docs/en/interfaces/third-party/integrations.md +++ b/docs/en/interfaces/third-party/integrations.md @@ -6,7 +6,7 @@ toc_title: Integrations # Integration Libraries from Third-party Developers {#integration-libraries-from-third-party-developers} !!! warning "Disclaimer" - Yandex does **not** maintain the tools and libraries listed below and haven’t done any extensive testing to ensure their quality. + ClickHouse, Inc. does **not** maintain the tools and libraries listed below and haven’t done extensive testing to ensure their quality. ## Infrastructure Products {#infrastructure-products} diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 5efa1b971bc..ad199ce452e 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -67,6 +67,7 @@ toc_title: Adopters | Geniee | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | Genotek | Bioinformatics | Main product | — | — | [Video, August 2020](https://youtu.be/v3KyZbz9lEE) | | Gigapipe | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) | +| Gigasheet | Analytics | Main product | — | — | Direct Reference, February 2022| | Glaber | Monitoring | Main product | — | — | [Website](https://glaber.io/) | | GraphCDN | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) | | Grouparoo | Data Warehouse Integrations | Main product | — | — | [Official Website, November 2021](https://www.grouparoo.com/integrations) | @@ -194,5 +195,6 @@ toc_title: Adopters | ООО «МПЗ Богородский» | Agriculture | — | — | — | [Article in Russian, November 2020](https://cloud.yandex.ru/cases/okraina) | | ДомКлик | Real Estate | — | — | — | [Article in Russian, October 2021](https://habr.com/ru/company/domclick/blog/585936/) | | АС "Стрела" | Transportation | — | — | — | [Job posting, Jan 2022](https://vk.com/topic-111905078_35689124?post=3553) | +| Piwik PRO | Web Analytics | — | — | — | [Official website, Dec 2018](https://piwik.pro/blog/piwik-pro-clickhouse-faster-efficient-reports/) | [Original article](https://clickhouse.com/docs/en/introduction/adopters/) diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md index 6ae37086181..684b4ee4179 100644 --- a/docs/en/introduction/performance.md +++ b/docs/en/introduction/performance.md @@ -5,7 +5,7 @@ toc_title: Performance # Performance {#performance} -According to internal testing results at Yandex, ClickHouse shows the best performance (both the highest throughput for long queries and the lowest latency on short queries) for comparable operating scenarios among systems of its class that were available for testing. You can view the test results on a [separate page](https://clickhouse.com/benchmark/dbms/). +ClickHouse shows the best performance (both the highest throughput for long queries and the lowest latency on short queries) for comparable operating scenarios among systems of its class that were available for testing. You can view the test results on a [separate page](https://clickhouse.com/benchmark/dbms/). Numerous independent benchmarks came to similar conclusions. They are not difficult to find using an internet search, or you can see [our small collection of related links](https://clickhouse.com/#independent-benchmarks). diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md index 279204a8af1..9aa6419d89c 100644 --- a/docs/en/operations/caches.md +++ b/docs/en/operations/caches.md @@ -5,7 +5,7 @@ toc_title: Caches # Cache Types {#cache-types} -When performing queries, ClichHouse uses different caches. +When performing queries, ClickHouse uses different caches. Main cache types: diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md index 35ec5d858f5..a8ca2079070 100644 --- a/docs/en/operations/clickhouse-keeper.md +++ b/docs/en/operations/clickhouse-keeper.md @@ -55,7 +55,7 @@ Internal coordination settings are located in `..` section and contain servers description. @@ -121,7 +121,7 @@ clickhouse keeper --config /etc/your_path_to_config/config.xml ClickHouse Keeper also provides 4lw commands which are almost the same with Zookeeper. Each command is composed of four letters such as `mntr`, `stat` etc. There are some more interesting commands: `stat` gives some general information about the server and connected clients, while `srvr` and `cons` give extended details on server and connections respectively. -The 4lw commands has a white list configuration `four_letter_word_white_list` which has default value "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro". +The 4lw commands has a allow list configuration `four_letter_word_allow_list` which has default value "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro". You can issue the commands to ClickHouse Keeper via telnet or nc, at the client port. @@ -201,7 +201,7 @@ Server stats reset. ``` server_id=1 tcp_port=2181 -four_letter_word_white_list=* +four_letter_word_allow_list=* log_storage_path=./coordination/logs snapshot_storage_path=./coordination/snapshots max_requests_batch_size=100 diff --git a/docs/en/operations/external-authenticators/index.md b/docs/en/operations/external-authenticators/index.md index 13b07b8e51e..850b6594b71 100644 --- a/docs/en/operations/external-authenticators/index.md +++ b/docs/en/operations/external-authenticators/index.md @@ -12,5 +12,6 @@ The following external authenticators and directories are supported: - [LDAP](./ldap.md#external-authenticators-ldap) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory) - Kerberos [Authenticator](./kerberos.md#external-authenticators-kerberos) +- [SSL X.509 authentication](./ssl-x509.md#ssl-external-authentication) [Original article](https://clickhouse.com/docs/en/operations/external-authenticators/index/) diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md new file mode 100644 index 00000000000..dd4f35257bb --- /dev/null +++ b/docs/en/operations/external-authenticators/ssl-x509.md @@ -0,0 +1,24 @@ +# SSL X.509 certificate authentication {#ssl-external-authentication} + +[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` field of the certificate is used to identify connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. + +To enable SSL certificate authentication, a list of `Common Name`'s for each ClickHouse user must be sspecified in the settings file `config.xml `: + +**Example** +```xml + + + + + + host.domain.com:example_user + host.domain.com:example_user_dev + + + + + + +``` + +For the SSL [`chain of trust`](https://en.wikipedia.org/wiki/Chain_of_trust) to work correctly, it is also important to make sure that the [`caConfig`](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) parameter is configured properly. \ No newline at end of file diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md new file mode 100644 index 00000000000..dce7938f98b --- /dev/null +++ b/docs/en/operations/named-collections.md @@ -0,0 +1,229 @@ +--- +toc_priority: 69 +toc_title: "Named connections" +--- + +# Storing details for connecting to external sources in configuration files {#named-collections} + +Details for connecting to external sources (dictionaries, tables, table functions) can be saved +in configuration files and thus simplify the creation of objects and hide credentials +from users with only SQL access. + +Parameters can be set in XML `CSV` and overridden in SQL `, format = 'TSV'`. +The parameters in SQL can be overridden using format `key` = `value`: `compression_method = 'gzip'`. + +Named connections are stored in the `config.xml` file of the ClickHouse server in the `` section and are applied when ClickHouse starts. + +Example of configuration: +```xml +$ cat /etc/clickhouse-server/config.d/named_collections.xml + + + ... + + +``` + +## Named connections for accessing S3. + +The description of parameters see [s3 Table Function](../sql-reference/table-functions/s3.md). + +Example of configuration: +```xml + + + + AKIAIOSFODNN7EXAMPLE + wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + CSV + + + +``` + +### Example of using named connections with the s3 function + +```sql +INSERT INTO FUNCTION s3(s3_mydata, url = 'https://s3.us-east-1.amazonaws.com/yourbucket/mydata/test_file.tsv.gz', + format = 'TSV', structure = 'number UInt64', compression_method = 'gzip') +SELECT * FROM numbers(10000); + +SELECT count() +FROM s3(s3_mydata, url = 'https://s3.us-east-1.amazonaws.com/yourbucket/mydata/test_file.tsv.gz') + +┌─count()─┐ +│ 10000 │ +└─────────┘ +1 rows in set. Elapsed: 0.279 sec. Processed 10.00 thousand rows, 90.00 KB (35.78 thousand rows/s., 322.02 KB/s.) +``` + +### Example of using named connections with an S3 table + +```sql +CREATE TABLE s3_engine_table (number Int64) +ENGINE=S3(s3_mydata, url='https://s3.us-east-1.amazonaws.com/yourbucket/mydata/test_file.tsv.gz', format = 'TSV') +SETTINGS input_format_with_names_use_header = 0; + +SELECT * FROM s3_engine_table LIMIT 3; +┌─number─┐ +│ 0 │ +│ 1 │ +│ 2 │ +└────────┘ +``` + +## Named connections for accessing MySQL database + +The description of parameters see [mysql](../sql-reference/table-functions/mysql.md). + +Example of configuration: +```xml + + + + myuser + mypass + 127.0.0.1 + 3306 + test + 8 + 1 + 1 + + + +``` + +### Example of using named connections with the mysql function + +```sql +SELECT count() FROM mysql(mymysql, table = 'test'); + +┌─count()─┐ +│ 3 │ +└─────────┘ +``` + +### Example of using named connections with an MySQL table + +```sql +CREATE TABLE mytable(A Int64) ENGINE = MySQL(mymysql, table = 'test', connection_pool_size=3, replace_query=0); +SELECT count() FROM mytable; + +┌─count()─┐ +│ 3 │ +└─────────┘ +``` + +### Example of using named connections with database with engine MySQL + +```sql +CREATE DATABASE mydatabase ENGINE = MySQL(mymysql); + +SHOW TABLES FROM mydatabase; + +┌─name───┐ +│ source │ +│ test │ +└────────┘ +``` + +### Example of using named connections with an external dictionary with source MySQL + +```sql +CREATE DICTIONARY dict (A Int64, B String) +PRIMARY KEY A +SOURCE(MYSQL(NAME mymysql TABLE 'source')) +LIFETIME(MIN 1 MAX 2) +LAYOUT(HASHED()); + +SELECT dictGet('dict', 'B', 2); + +┌─dictGet('dict', 'B', 2)─┐ +│ two │ +└─────────────────────────┘ +``` + +## Named connections for accessing PostgreSQL database + +The description of parameters see [postgresql](../sql-reference/table-functions/postgresql.md). + +Example of configuration: +```xml + + + + pguser + jw8s0F4 + 127.0.0.1 + 5432 + test + test_schema + 8 + + + +``` + +### Example of using named connections with the postgresql function + +```sql +SELECT * FROM postgresql(mypg, table = 'test'); + +┌─a─┬─b───┐ +│ 2 │ two │ +│ 1 │ one │ +└───┴─────┘ + + +SELECT * FROM postgresql(mypg, table = 'test', schema = 'public'); + +┌─a─┐ +│ 1 │ +│ 2 │ +│ 3 │ +└───┘ +``` + + +### Example of using named connections with database with engine PostgreSQL + +```sql +CREATE TABLE mypgtable (a Int64) ENGINE = PostgreSQL(mypg, table = 'test', schema = 'public'); + +SELECT * FROM mypgtable; + +┌─a─┐ +│ 1 │ +│ 2 │ +│ 3 │ +└───┘ +``` + +### Example of using named connections with database with engine PostgreSQL + +```sql +CREATE DATABASE mydatabase ENGINE = PostgreSQL(mypg); + +SHOW TABLES FROM mydatabase + +┌─name─┐ +│ test │ +└──────┘ +``` + +### Example of using named connections with an external dictionary with source POSTGRESQL + +```sql +CREATE DICTIONARY dict (a Int64, b String) +PRIMARY KEY a +SOURCE(POSTGRESQL(NAME mypg TABLE test)) +LIFETIME(MIN 1 MAX 2) +LAYOUT(HASHED()); + +SELECT dictGet('dict', 'b', 2); + +┌─dictGet('dict', 'b', 2)─┐ +│ two │ +└─────────────────────────┘ +``` diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md index 6dac8736372..ec27ecfd6b2 100644 --- a/docs/en/operations/opentelemetry.md +++ b/docs/en/operations/opentelemetry.md @@ -14,7 +14,7 @@ toc_title: OpenTelemetry Support ClickHouse accepts trace context HTTP headers, as described by the [W3C recommendation](https://www.w3.org/TR/trace-context/). It also accepts trace context over a native protocol that is used for communication between ClickHouse servers or between the client and server. For manual testing, trace context headers conforming to the Trace Context recommendation can be supplied to `clickhouse-client` using `--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags. -If no parent trace context is supplied, ClickHouse can start a new trace, with probability controlled by the [opentelemetry_start_trace_probability](../operations/settings/settings.md#opentelemetry-start-trace-probability) setting. +If no parent trace context is supplied or the provided trace context does not comply with W3C standard above, ClickHouse can start a new trace, with probability controlled by the [opentelemetry_start_trace_probability](../operations/settings/settings.md#opentelemetry-start-trace-probability) setting. ## Propagating the Trace Context @@ -46,8 +46,8 @@ ENGINE = URL('http://127.0.0.1:9411/api/v2/spans', 'JSONEachRow') SETTINGS output_format_json_named_tuples_as_objects = 1, output_format_json_array_of_rows = 1 AS SELECT - lower(hex(reinterpretAsFixedString(trace_id))) AS traceId, - lower(hex(parent_span_id)) AS parentId, + lower(hex(trace_id)) AS traceId, + case when parent_span_id = 0 then '' else lower(hex(parent_span_id)) end AS parentId, lower(hex(span_id)) AS id, operation_name AS name, start_time_us AS timestamp, diff --git a/docs/en/operations/performance-test.md b/docs/en/operations/performance-test.md index a220575cb3c..e410b1b2dfd 100644 --- a/docs/en/operations/performance-test.md +++ b/docs/en/operations/performance-test.md @@ -38,6 +38,18 @@ Alternatively you can perform benchmark in the following steps. wget https://builds.clickhouse.com/master/amd64/clickhouse # For aarch64: wget https://builds.clickhouse.com/master/aarch64/clickhouse +# For powerpc64le: +wget https://builds.clickhouse.com/master/powerpc64le/clickhouse +# For freebsd: +wget https://builds.clickhouse.com/master/freebsd/clickhouse +# For freebsd-aarch64: +wget https://builds.clickhouse.com/master/freebsd-aarch64/clickhouse +# For freebsd-powerpc64le: +wget https://builds.clickhouse.com/master/freebsd-powerpc64le/clickhouse +# For macos: +wget https://builds.clickhouse.com/master/macos/clickhouse +# For macos-aarch64: +wget https://builds.clickhouse.com/master/macos-aarch64/clickhouse # Then do: chmod a+x clickhouse ``` @@ -47,7 +59,7 @@ wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/cl chmod a+x benchmark-new.sh wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/queries.sql ``` -3. Download test data according to the [Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md) instruction (“hits” table containing 100 million rows). +3. Download the [web analytics dataset](../getting-started/example-datasets/metrica.md) (“hits” table containing 100 million rows). ```bash wget https://datasets.clickhouse.com/hits/partitions/hits_100m_obfuscated_v1.tar.xz tar xvf hits_100m_obfuscated_v1.tar.xz -C . @@ -66,6 +78,6 @@ mv hits_100m_obfuscated_v1/* . ```bash ./benchmark-new.sh hits_100m_obfuscated ``` -7. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com +7. Send the numbers and the info about your hardware configuration to feedback@clickhouse.com All the results are published here: https://clickhouse.com/benchmark/hardware/ diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md index eec8961b595..6c6fbbf9cfb 100644 --- a/docs/en/operations/quotas.md +++ b/docs/en/operations/quotas.md @@ -101,7 +101,7 @@ Quotas can use the “quota key” feature to report on resources for multiple k diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 573e47ead54..c892b814957 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -220,7 +220,7 @@ Result: A fast, decent-quality non-cryptographic hash function for a string obtained from a URL using some type of normalization. `URLHash(s)` – Calculates a hash from a string without one of the trailing symbols `/`,`?` or `#` at the end, if present. `URLHash(s, N)` – Calculates a hash from a string up to the N level in the URL hierarchy, without one of the trailing symbols `/`,`?` or `#` at the end, if present. -Levels are the same as in URLHierarchy. This function is specific to Yandex.Metrica. +Levels are the same as in URLHierarchy. ## farmFingerprint64 {#farmfingerprint64} diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index ddc113d31f9..7cceec889bd 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -74,9 +74,10 @@ A function configuration contains the following settings: - `name` - a function name. - `command` - script name to execute or command if `execute_direct` is false. -- `argument` - argument description with the `type` of an argument. Each argument is described in a separate setting. +- `argument` - argument description with the `type`, and optional `name` of an argument. Each argument is described in a separate setting. Specifying name is necessary if argument names are part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Default argument name value is `c` + argument_number. - `format` - a [format](../../interfaces/formats.md) in which arguments are passed to the command. - `return_type` - the type of a returned value. +- `return_name` - name of retuned value. Specifying return name is necessary if return name is part of serialization for user defined function format like [Native](../../interfaces/formats.md#native) or [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Optional. Default value is `result`. - `type` - an executable type. If `type` is set to `executable` then single command is started. If it is set to `executable_pool` then a pool of commands is created. - `max_command_execution_time` - maximum execution time in seconds for processing block of data. This setting is valid for `executable_pool` commands only. Optional. Default value is `10`. - `command_termination_timeout` - time in seconds during which a command should finish after its pipe is closed. After that time `SIGTERM` is sent to the process executing the command. Optional. Default value is `10`. @@ -100,6 +101,7 @@ File test_function.xml. String UInt64 + value TabSeparated test_function.py @@ -144,9 +146,11 @@ File test_function.xml. UInt64 UInt64 + lhs UInt64 + rhs TabSeparated cd /; clickhouse-local --input-format TabSeparated --output-format TabSeparated --structure 'x UInt64, y UInt64' --query "SELECT x + y FROM table" @@ -169,6 +173,58 @@ Result: └─────────────────────────┘ ``` +Creating `test_function_sum_json` with named arguments and format [JSONEachRow](../../interfaces/formats.md#jsoneachrow) using XML configuration. +File test_function.xml. +```xml + + executable + test_function_sum_json + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + test_function_sum_json.py + +``` + +Script file inside `user_scripts` folder `test_function_sum_json.py`. + +```python +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['argument_1']) + second_arg = int(value['argument_2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() +``` + +Query: + +``` sql +SELECT test_function_sum_json(2, 2); +``` + +Result: + +``` text +┌─test_function_sum_json(2, 2)─┐ +│ 4 │ +└──────────────────────────────┘ +``` ## Error Handling {#error-handling} diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md index 469a66d460f..cf3f92580aa 100644 --- a/docs/en/sql-reference/functions/ip-address-functions.md +++ b/docs/en/sql-reference/functions/ip-address-functions.md @@ -13,10 +13,18 @@ Alias: `INET_NTOA`. ## IPv4StringToNum(s) {#ipv4stringtonums} -The reverse function of IPv4NumToString. If the IPv4 address has an invalid format, it returns 0. +The reverse function of IPv4NumToString. If the IPv4 address has an invalid format, it throws exception. Alias: `INET_ATON`. +## IPv4StringToNumOrDefault(s) {#ipv4stringtonums} + +Same as `IPv4StringToNum`, but if the IPv4 address has an invalid format, it returns 0. + +## IPv4StringToNumOrNull(s) {#ipv4stringtonums} + +Same as `IPv4StringToNum`, but if the IPv4 address has an invalid format, it returns null. + ## IPv4NumToStringClassC(num) {#ipv4numtostringclasscnum} Similar to IPv4NumToString, but using xxx instead of the last octet. @@ -123,7 +131,7 @@ LIMIT 10 ## IPv6StringToNum {#ipv6stringtonums} -The reverse function of [IPv6NumToString](#ipv6numtostringx). If the IPv6 address has an invalid format, it returns a string of null bytes. +The reverse function of [IPv6NumToString](#ipv6numtostringx). If the IPv6 address has an invalid format, it throws exception. If the input string contains a valid IPv4 address, returns its IPv6 equivalent. HEX can be uppercase or lowercase. @@ -168,6 +176,14 @@ Result: - [cutIPv6](#cutipv6x-bytestocutforipv6-bytestocutforipv4). +## IPv6StringToNumOrDefault(s) {#ipv6stringtonums} + +Same as `IPv6StringToNum`, but if the IPv6 address has an invalid format, it returns 0. + +## IPv6StringToNumOrNull(s) {#ipv6stringtonums} + +Same as `IPv6StringToNum`, but if the IPv6 address has an invalid format, it returns null. + ## IPv4ToIPv6(x) {#ipv4toipv6x} Takes a `UInt32` number. Interprets it as an IPv4 address in [big endian](https://en.wikipedia.org/wiki/Endianness). Returns a `FixedString(16)` value containing the IPv6 address in binary format. Examples: @@ -261,6 +277,14 @@ SELECT └───────────────────────────────────┴──────────────────────────┘ ``` +## toIPv4OrDefault(string) {#toipv4ordefaultstring} + +Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns 0. + +## toIPv4OrNull(string) {#toipv4ornullstring} + +Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns null. + ## toIPv6 {#toipv6string} Converts a string form of IPv6 address to [IPv6](../../sql-reference/data-types/domains/ipv6.md) type. If the IPv6 address has an invalid format, returns an empty value. @@ -317,6 +341,14 @@ Result: └─────────────────────┘ ``` +## IPv6StringToNumOrDefault(s) {#toipv6ordefaultstring} + +Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns 0. + +## IPv6StringToNumOrNull(s) {#toipv6ornullstring} + +Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null. + ## isIPv4String {#isipv4string} Determines whether the input string is an IPv4 address or not. If `string` is IPv6 address returns `0`. diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md index 498f0de7b6c..8270864de74 100644 --- a/docs/en/sql-reference/functions/json-functions.md +++ b/docs/en/sql-reference/functions/json-functions.md @@ -5,9 +5,7 @@ toc_title: JSON # Functions for Working with JSON {#functions-for-working-with-json} -In Yandex.Metrica, JSON is transmitted by users as session parameters. There are some special functions for working with this JSON. (Although in most of the cases, the JSONs are additionally pre-processed, and the resulting values are put in separate columns in their processed format.) All these functions are based on strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done. - -The following assumptions are made: +ClickHouse has special functions for working with this JSON. The `visitParam` functions make strong assumptions about what the JSON can be, but they try to do as little as possible to get the job done. The following assumptions are made: 1. The field name (function argument) must be a constant. 2. The field name is somehow canonically encoded in JSON. For example: `visitParamHas('{"abc":"def"}', 'abc') = 1`, but `visitParamHas('{"\\u0061\\u0062\\u0063":"def"}', 'abc') = 0` diff --git a/docs/en/sql-reference/functions/rounding-functions.md b/docs/en/sql-reference/functions/rounding-functions.md index ad92ba502e1..c9044c62ca4 100644 --- a/docs/en/sql-reference/functions/rounding-functions.md +++ b/docs/en/sql-reference/functions/rounding-functions.md @@ -189,11 +189,11 @@ Accepts a number. If the number is less than one, it returns 0. Otherwise, it ro ## roundDuration(num) {#rounddurationnum} -Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to numbers from the set: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. This function is specific to Yandex.Metrica and used for implementing the report on session length. +Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to numbers from the set: 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000. This function was specifically implemented for a web analytics use case for reporting on session lengths. ## roundAge(num) {#roundagenum} -Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rounds the number down to a number from the set: 18, 25, 35, 45, 55. This function is specific to Yandex.Metrica and used for implementing the report on user age. +Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rounds the number down to a number from the set: 18, 25, 35, 45, 55. ## roundDown(num, arr) {#rounddownnum-arr} diff --git a/docs/en/sql-reference/functions/statistics.md b/docs/en/sql-reference/functions/statistics.md new file mode 100644 index 00000000000..3f337b05cbc --- /dev/null +++ b/docs/en/sql-reference/functions/statistics.md @@ -0,0 +1,48 @@ +--- +toc_priority: 69 +toc_title: Statistics +--- + +# Functions for Working with Statistics {#functions-for-working-with-statistics} + +# proportionsZTest {#proportionsztest} + +Applies proportion z-test to samples from two populations (X and Y). The alternative is 'two-sided'. + +**Syntax** + +``` sql +proportionsZTest(successes_x, successes_y, trials_x, trials_y, significance_level, usevar) +``` + +**Arguments** + +- `successes_x` — The number of successes for X in trials. +- `successes_y` — The number of successes for X in trials. +- `trials_x` — The number of trials for X. +- `trials_y` — The number of trials for Y. +- `significance_level` +- `usevar` - It can be `'pooled'` or `'unpooled'`. + - `'pooled'` - The variance of the two populations are assumed to be equal. + - `'unpooled'` - The assumption of equal variances is dropped. + +**Returned value** + +- A tuple with the (z-statistic, p-value, confidence-interval-lower, confidence-interval-upper). + +Type: [Tuple](../../sql-reference/data-types/tuple.md). + +**Example** + +Query: + +``` sql +SELECT proportionsZTest(10, 11, 100, 101, 0.95, 'unpooled'); +``` + +Result: + +``` text +(-0.20656724435948853,0.8363478437079654,-0.09345975390115283,0.07563797172293502) +``` + diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 160e7be156e..18cc3d98561 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1012,7 +1012,7 @@ Result: Query: ``` sql -SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Europe/Moscow') +SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Asia/Istanbul') AS parseDateTimeBestEffort; ``` @@ -1206,7 +1206,7 @@ Result: Query: ``` sql -SELECT parseDateTimeBestEffortUSOrNull('02-10-2021 21:12:57 GMT', 'Europe/Moscow') AS parseDateTimeBestEffortUSOrNull; +SELECT parseDateTimeBestEffortUSOrNull('02-10-2021 21:12:57 GMT', 'Asia/Istanbul') AS parseDateTimeBestEffortUSOrNull; ``` Result: @@ -1292,7 +1292,7 @@ Result: Query: ``` sql -SELECT parseDateTimeBestEffortUSOrZero('02-10-2021 21:12:57 GMT', 'Europe/Moscow') AS parseDateTimeBestEffortUSOrZero; +SELECT parseDateTimeBestEffortUSOrZero('02-10-2021 21:12:57 GMT', 'Asia/Istanbul') AS parseDateTimeBestEffortUSOrZero; ``` Result: @@ -1362,7 +1362,7 @@ SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346') AS a, toTypeName(a UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t UNION ALL -SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t +SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Asia/Istanbul') AS a, toTypeName(a) AS t FORMAT PrettyCompactMonoBlock; ``` @@ -1373,7 +1373,7 @@ Result: │ 2021-01-01 01:01:00.123000 │ DateTime64(3) │ │ 2021-01-01 00:00:00.000000 │ DateTime64(3) │ │ 2021-01-01 01:01:00.123460 │ DateTime64(6) │ -│ 2020-12-31 22:01:00.123000 │ DateTime64(3, 'Europe/Moscow') │ +│ 2020-12-31 22:01:00.123000 │ DateTime64(3, 'Asia/Istanbul') │ └────────────────────────────┴────────────────────────────────┘ ``` diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 98c3135f2b4..5a305aa5033 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -34,7 +34,7 @@ The URL can be specified with or without a scheme. Examples: ``` text svn+ssh://some.svn-hosting.com:80/repo/trunk some.svn-hosting.com:80/repo/trunk -https://yandex.com/time/ +https://clickhouse.com/time/ ``` For these examples, the `domain` function returns the following results: @@ -42,7 +42,7 @@ For these examples, the `domain` function returns the following results: ``` text some.svn-hosting.com some.svn-hosting.com -yandex.com +clickhouse.com ``` **Returned values** @@ -85,7 +85,7 @@ The URL can be specified with or without a scheme. Examples: ``` text svn+ssh://some.svn-hosting.com:80/repo/trunk some.svn-hosting.com:80/repo/trunk -https://yandex.com/time/ +https://clickhouse.com/time/ ``` **Returned values** @@ -109,7 +109,7 @@ SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk'); ### firstSignificantSubdomain {#firstsignificantsubdomain} -Returns the “first significant subdomain”. This is a non-standard concept specific to Yandex.Metrica. The first significant subdomain is a second-level domain if it is ‘com’, ‘net’, ‘org’, or ‘co’. Otherwise, it is a third-level domain. For example, `firstSignificantSubdomain (‘https://news.yandex.ru/’) = ‘yandex’, firstSignificantSubdomain (‘https://news.yandex.com.tr/’) = ‘yandex’`. The list of “insignificant” second-level domains and other implementation details may change in the future. +Returns the “first significant subdomain”. The first significant subdomain is a second-level domain if it is ‘com’, ‘net’, ‘org’, or ‘co’. Otherwise, it is a third-level domain. For example, `firstSignificantSubdomain (‘https://news.clickhouse.com/’) = ‘clickhouse’, firstSignificantSubdomain (‘https://news.clickhouse.com.tr/’) = ‘clickhouse’`. The list of “insignificant” second-level domains and other implementation details may change in the future. ### cutToFirstSignificantSubdomain {#cuttofirstsignificantsubdomain} @@ -117,7 +117,7 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example: -- `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. +- `cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. - `cutToFirstSignificantSubdomain('www.tr') = 'tr'`. - `cutToFirstSignificantSubdomain('tr') = ''`. @@ -127,7 +127,7 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example: -- `cutToFirstSignificantSubdomain('https://news.yandex.com.tr/') = 'yandex.com.tr'`. +- `cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. - `cutToFirstSignificantSubdomain('www.tr') = 'www.tr'`. - `cutToFirstSignificantSubdomain('tr') = ''`. @@ -335,7 +335,7 @@ Returns an array containing the URL, truncated at the end by the symbols /,? in ### URLPathHierarchy(URL) {#urlpathhierarchyurl} -The same as above, but without the protocol and host in the result. The / element (root) is not included. Example: the function is used to implement tree reports the URL in Yandex. Metric. +The same as above, but without the protocol and host in the result. The / element (root) is not included. ``` text URLPathHierarchy('https://example.com/browse/CONV-6788') = @@ -345,6 +345,21 @@ URLPathHierarchy('https://example.com/browse/CONV-6788') = ] ``` +### encodeURLComponent(URL) {#encodeurlcomponenturl} + +Returns the encoded URL. +Example: + +``` sql +SELECT encodeURLComponent('http://127.0.0.1:8123/?query=SELECT 1;') AS EncodedURL; +``` + +``` text +┌─EncodedURL───────────────────────────────────────────────┐ +│ http%3A%2F%2F127.0.0.1%3A8123%2F%3Fquery%3DSELECT%201%3B │ +└──────────────────────────────────────────────────────────┘ +``` + ### decodeURLComponent(URL) {#decodeurlcomponenturl} Returns the decoded URL. @@ -360,6 +375,21 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod └────────────────────────────────────────┘ ``` +### encodeURLFormComponent(URL) {#encodeurlformcomponenturl} + +Returns the encoded URL. Follows rfc-1866, space(` `) is encoded as plus(`+`). +Example: + +``` sql +SELECT encodeURLFormComponent('http://127.0.0.1:8123/?query=SELECT 1 2+3') AS EncodedURL; +``` + +``` text +┌─EncodedURL────────────────────────────────────────────────┐ +│ http%3A%2F%2F127.0.0.1%3A8123%2F%3Fquery%3DSELECT+1+2%2B3 │ +└───────────────────────────────────────────────────────────┘ +``` + ### decodeURLFormComponent(URL) {#decodeurlformcomponenturl} Returns the decoded URL. Follows rfc-1866, plain plus(`+`) is decoded as space(` `). diff --git a/docs/en/sql-reference/functions/ym-dict-functions.md b/docs/en/sql-reference/functions/ym-dict-functions.md index f947c81c7a9..1e6c9cbd0b4 100644 --- a/docs/en/sql-reference/functions/ym-dict-functions.md +++ b/docs/en/sql-reference/functions/ym-dict-functions.md @@ -1,11 +1,11 @@ --- toc_priority: 59 -toc_title: Yandex.Metrica Dictionaries +toc_title: Embedded Dictionaries --- -# Functions for Working with Yandex.Metrica Dictionaries {#functions-for-working-with-yandex-metrica-dictionaries} +# Functions for Working with Embedded Dictionaries -In order for the functions below to work, the server config must specify the paths and addresses for getting all the Yandex.Metrica dictionaries. The dictionaries are loaded at the first call of any of these functions. If the reference lists can’t be loaded, an exception is thrown. +In order for the functions below to work, the server config must specify the paths and addresses for getting all the embedded dictionaries. The dictionaries are loaded at the first call of any of these functions. If the reference lists can’t be loaded, an exception is thrown. For information about creating reference lists, see the section “Dictionaries”. @@ -33,7 +33,7 @@ regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/g ### regionToCity(id\[, geobase\]) {#regiontocityid-geobase} -Accepts a UInt32 number – the region ID from the Yandex geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. +Accepts a UInt32 number – the region ID from the geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. ### regionToArea(id\[, geobase\]) {#regiontoareaid-geobase} @@ -117,7 +117,7 @@ regionToTopContinent(id[, geobase]) **Arguments** -- `id` — Region ID from the Yandex geobase. [UInt32](../../sql-reference/data-types/int-uint.md). +- `id` — Region ID from the geobase. [UInt32](../../sql-reference/data-types/int-uint.md). - `geobase` — Dictionary key. See [Multiple Geobases](#multiple-geobases). [String](../../sql-reference/data-types/string.md). Optional. **Returned value** @@ -132,7 +132,7 @@ Type: `UInt32`. Gets the population for a region. The population can be recorded in files with the geobase. See the section “External dictionaries”. If the population is not recorded for the region, it returns 0. -In the Yandex geobase, the population might be recorded for child regions, but not for parent regions. +In the geobase, the population might be recorded for child regions, but not for parent regions. ### regionIn(lhs, rhs\[, geobase\]) {#regioninlhs-rhs-geobase} @@ -141,12 +141,12 @@ The relationship is reflexive – any region also belongs to itself. ### regionHierarchy(id\[, geobase\]) {#regionhierarchyid-geobase} -Accepts a UInt32 number – the region ID from the Yandex geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. +Accepts a UInt32 number – the region ID from the geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. Example: `regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]`. ### regionToName(id\[, lang\]) {#regiontonameid-lang} -Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID does not exist, an empty string is returned. +Accepts a UInt32 number – the region ID from the geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID does not exist, an empty string is returned. `ua` and `uk` both mean Ukrainian. diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 1c59b25fc63..a64dcd70c6c 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -254,7 +254,7 @@ You can work with dates without using `INTERVAL`, just by adding or subtracting Examples: ``` sql -SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 60 * 24 AS time_plus_24_hours, time + toIntervalDay(1) AS time_plus_1_day; +SELECT toDateTime('2014-10-26 00:00:00', 'Asia/Istanbul') AS time, time + 60 * 60 * 24 AS time_plus_24_hours, time + toIntervalDay(1) AS time_plus_1_day; ``` ``` text diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 2e562e20467..6bb63ea06a6 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ## MATERIALIZE COLUMN {#materialize-column} -Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier. +Materializes or updates a column with an expression for a default value (`DEFAULT` or `MATERIALIZED`). +It is used if it is necessary to add or update a column with a complicated expression, because evaluating such an expression directly on `SELECT` executing turns out to be expensive. Syntax: ```sql -ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; +ALTER TABLE table MATERIALIZE COLUMN col; ``` **Example** @@ -211,20 +212,34 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; DROP TABLE IF EXISTS tmp; SET mutations_sync = 2; CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); -INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10; +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); ALTER TABLE tmp MATERIALIZE COLUMN s; +SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x); + +┌─groupArray(x)─┬─groupArray(s)─────────┐ +│ [0,1,2,3,4] │ ['0','1','2','3','4'] │ +└───────────────┴───────────────────────┘ + +ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x)); + +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5; + SELECT groupArray(x), groupArray(s) FROM tmp; -``` -**Result:** +┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │ +└───────────────────────┴────────────────────────────────────────────────┘ -```sql -┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐ -│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │ -└───────────────────────┴───────────────────────────────────────────┘ +ALTER TABLE tmp MATERIALIZE COLUMN s; + +SELECT groupArray(x), groupArray(s) FROM tmp; + +┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │ +└───────────────────────┴───────────────────────────────────────────────────────┘ ``` **See Also** diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 7bbbb6f32bd..409ec422ade 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -16,8 +16,8 @@ By default, tables are created only on the current server. Distributed DDL queri ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( - name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], - name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2], + name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr1] [compression_codec] [TTL expr1], + name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [compression_codec] [TTL expr2], ... ) ENGINE = engine ``` @@ -112,6 +112,13 @@ Materialized expression. Such a column can’t be specified for INSERT, because For an INSERT without a list of columns, these columns are not considered. In addition, this column is not substituted when using an asterisk in a SELECT query. This is to preserve the invariant that the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. +### EPHEMERAL {#ephemeral} + +`EPHEMERAL expr` + +Ephemeral column. Such a column isn't stored in the table and cannot be SELECTed, but can be referenced in the defaults of CREATE statement. +INSERT without list of columns will skip such column, so SELECT/INSERT invariant is preserved - the dump obtained using `SELECT *` can be inserted back into the table using INSERT without specifying the list of columns. + ### ALIAS {#alias} `ALIAS expr` diff --git a/docs/en/sql-reference/statements/select/array-join.md b/docs/en/sql-reference/statements/select/array-join.md index b4d99aaf6b2..f138bcc45c7 100644 --- a/docs/en/sql-reference/statements/select/array-join.md +++ b/docs/en/sql-reference/statements/select/array-join.md @@ -127,7 +127,7 @@ ARRAY JOIN [1, 2, 3] AS arr_external; └─────────────┴──────────────┘ ``` -Multiple arrays can be comma-separated in the `ARRAY JOIN` clause. In this case, `JOIN` is performed with them simultaneously (the direct sum, not the cartesian product). Note that all the arrays must have the same size. Example: +Multiple arrays can be comma-separated in the `ARRAY JOIN` clause. In this case, `JOIN` is performed with them simultaneously (the direct sum, not the cartesian product). Note that all the arrays must have the same size by default. Example: ``` sql SELECT s, arr, a, num, mapped @@ -162,6 +162,25 @@ ARRAY JOIN arr AS a, arrayEnumerate(arr) AS num; │ World │ [3,4,5] │ 5 │ 3 │ [1,2,3] │ └───────┴─────────┴───┴─────┴─────────────────────┘ ``` +Multiple arrays with different sizes can be joined by using: `SETTINGS enable_unaligned_array_join = 1`. Example: + +```sql +SELECT s, arr, a, b +FROM arrays_test ARRAY JOIN arr as a, [['a','b'],['c']] as b +SETTINGS enable_unaligned_array_join = 1; +``` + +```text +┌─s───────┬─arr─────┬─a─┬─b─────────┐ +│ Hello │ [1,2] │ 1 │ ['a','b'] │ +│ Hello │ [1,2] │ 2 │ ['c'] │ +│ World │ [3,4,5] │ 3 │ ['a','b'] │ +│ World │ [3,4,5] │ 4 │ ['c'] │ +│ World │ [3,4,5] │ 5 │ [] │ +│ Goodbye │ [] │ 0 │ ['a','b'] │ +│ Goodbye │ [] │ 0 │ ['c'] │ +└─────────┴─────────┴───┴───────────┘ +``` ## ARRAY JOIN with Nested Data Structure {#array-join-with-nested-data-structure} diff --git a/docs/en/sql-reference/statements/select/limit-by.md b/docs/en/sql-reference/statements/select/limit-by.md index e1ca58cdec8..68b459a46e8 100644 --- a/docs/en/sql-reference/statements/select/limit-by.md +++ b/docs/en/sql-reference/statements/select/limit-by.md @@ -11,7 +11,7 @@ ClickHouse supports the following syntax variants: - `LIMIT [offset_value, ]n BY expressions` - `LIMIT n OFFSET offset_value BY expressions` -During query processing, ClickHouse selects data ordered by sorting key. The sorting key is set explicitly using an [ORDER BY](../../../sql-reference/statements/select/order-by.md) clause or implicitly as a property of the table engine. Then ClickHouse applies `LIMIT n BY expressions` and returns the first `n` rows for each distinct combination of `expressions`. If `OFFSET` is specified, then for each data block that belongs to a distinct combination of `expressions`, ClickHouse skips `offset_value` number of rows from the beginning of the block and returns a maximum of `n` rows as a result. If `offset_value` is bigger than the number of rows in the data block, ClickHouse returns zero rows from the block. +During query processing, ClickHouse selects data ordered by sorting key. The sorting key is set explicitly using an [ORDER BY](order-by.md#select-order-by) clause or implicitly as a property of the table engine (row order is only guaranteed when using [ORDER BY](order-by.md#select-order-by), otherwise the row blocks will not be ordered due to multi-threading). Then ClickHouse applies `LIMIT n BY expressions` and returns the first `n` rows for each distinct combination of `expressions`. If `OFFSET` is specified, then for each data block that belongs to a distinct combination of `expressions`, ClickHouse skips `offset_value` number of rows from the beginning of the block and returns a maximum of `n` rows as a result. If `offset_value` is bigger than the number of rows in the data block, ClickHouse returns zero rows from the block. !!! note "Note" `LIMIT BY` is not related to [LIMIT](../../../sql-reference/statements/select/limit.md). They can both be used in the same query. diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 599213561f2..7dffd252dc9 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -26,11 +26,11 @@ A table with the specified structure for reading or writing data in the specifie **Examples** -Selecting the first two rows from the table from S3 file `https://storage.yandexcloud.net/my-test-bucket-768/data.csv`: +Selecting the first two rows from the table from S3 file `https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv`: ``` sql SELECT * -FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') +FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') LIMIT 2; ``` @@ -45,7 +45,7 @@ The similar but from file with `gzip` compression: ``` sql SELECT * -FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip') +FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip') LIMIT 2; ``` @@ -60,20 +60,20 @@ LIMIT 2; Suppose that we have several files with following URIs on S3: -- 'https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_3.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_4.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_1.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_2.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_3.csv' -- 'https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_4.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_1.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_2.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_3.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/some_prefix/some_file_4.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_1.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_2.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_3.csv' +- 'https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/another_prefix/some_file_4.csv' Count the amount of rows in files ending with numbers from 1 to 3: ``` sql SELECT count(*) -FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32') +FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32') ``` ``` text @@ -86,7 +86,7 @@ Count the total amount of rows in all files in these two directories: ``` sql SELECT count(*) -FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32') +FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32') ``` ``` text @@ -102,7 +102,7 @@ Count the total amount of rows in files named `file-000.csv`, `file-001.csv`, ``` sql SELECT count(*) -FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32'); +FROM s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32'); ``` ``` text @@ -114,14 +114,14 @@ FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000 Insert data into file `test-data.csv.gz`: ``` sql -INSERT INTO FUNCTION s3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +INSERT INTO FUNCTION s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') VALUES ('test-data', 1), ('test-data-2', 2); ``` Insert data into file `test-data.csv.gz` from existing table: ``` sql -INSERT INTO FUNCTION s3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +INSERT INTO FUNCTION s3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') SELECT name, value FROM existing_table; ``` diff --git a/docs/en/whats-new/changelog/2018.md b/docs/en/whats-new/changelog/2018.md index 3544c9a9b49..db09bcd8a03 100644 --- a/docs/en/whats-new/changelog/2018.md +++ b/docs/en/whats-new/changelog/2018.md @@ -926,7 +926,7 @@ toc_title: '2018' #### Backward Incompatible Changes: {#backward-incompatible-changes-10} - Removed the `distributed_ddl_allow_replicated_alter` option. This behavior is enabled by default. -- Removed the `strict_insert_defaults` setting. If you were using this functionality, write to `clickhouse-feedback@yandex-team.com`. +- Removed the `strict_insert_defaults` setting. If you were using this functionality, write to `feedback@clickhouse.com`. - Removed the `UnsortedMergeTree` engine. ### ClickHouse Release 1.1.54343, 2018-02-05 {#clickhouse-release-1-1-54343-2018-02-05} @@ -953,7 +953,7 @@ This release contains bug fixes for the previous release 1.1.54337: - Added support for storage of multi-dimensional arrays and tuples (`Tuple` data type) in tables. - Support for table functions for `DESCRIBE` and `INSERT` queries. Added support for subqueries in `DESCRIBE`. Examples: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. Support for `INSERT INTO TABLE` in addition to `INSERT INTO`. -- Improved support for time zones. The `DateTime` data type can be annotated with the timezone that is used for parsing and formatting in text formats. Example: `DateTime('Europe/Moscow')`. When timezones are specified in functions for `DateTime` arguments, the return type will track the timezone, and the value will be displayed as expected. +- Improved support for time zones. The `DateTime` data type can be annotated with the timezone that is used for parsing and formatting in text formats. Example: `DateTime('Asia/Istanbul')`. When timezones are specified in functions for `DateTime` arguments, the return type will track the timezone, and the value will be displayed as expected. - Added the functions `toTimeZone`, `timeDiff`, `toQuarter`, `toRelativeQuarterNum`. The `toRelativeHour`/`Minute`/`Second` functions can take a value of type `Date` as an argument. The `now` function name is case-sensitive. - Added the `toStartOfFifteenMinutes` function (Kirill Shvakov). - Added the `clickhouse format` tool for formatting queries. @@ -1049,7 +1049,7 @@ This release contains bug fixes for the previous release 1.1.54337: - The `runningIncome` function was renamed to `runningDifferenceStartingWithFirstvalue` to avoid confusion. - Removed the `FROM ARRAY JOIN arr` syntax when ARRAY JOIN is specified directly after FROM with no table (Amos Bird). - Removed the `BlockTabSeparated` format that was used solely for demonstration purposes. -- Changed the state format for aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. If you have stored states of these aggregate functions in tables (using the `AggregateFunction` data type or materialized views with corresponding states), please write to clickhouse-feedback@yandex-team.com. +- Changed the state format for aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. If you have stored states of these aggregate functions in tables (using the `AggregateFunction` data type or materialized views with corresponding states), please write to feedback@clickhouse.com. - In previous server versions there was an undocumented feature: if an aggregate function depends on parameters, you can still specify it without parameters in the AggregateFunction data type. Example: `AggregateFunction(quantiles, UInt64)` instead of `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. This feature was lost. Although it was undocumented, we plan to support it again in future releases. - Enum data types cannot be used in min/max aggregate functions. This ability will be returned in the next release. diff --git a/docs/en/whats-new/changelog/2019.md b/docs/en/whats-new/changelog/2019.md index bd86bf6ce8b..aa06f5cb1e3 100644 --- a/docs/en/whats-new/changelog/2019.md +++ b/docs/en/whats-new/changelog/2019.md @@ -1039,7 +1039,7 @@ toc_title: '2019' - Fix insert and select query to MySQL engine with MySQL style identifier quoting. [#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([Winter Zhang](https://github.com/zhang2014)) - Now `CHECK TABLE` query can work with MergeTree engine family. It returns check status and message if any for each part (or file in case of simplier engines). Also, fix bug in fetch of a broken part. [#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alesapin](https://github.com/alesapin)) - Fix SPLIT_SHARED_LIBRARIES runtime [#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) -- Fixed time zone initialization when `/etc/localtime` is a relative symlink like `../usr/share/zoneinfo/Europe/Moscow` [#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- Fixed time zone initialization when `/etc/localtime` is a relative symlink like `../usr/share/zoneinfo/Asia/Istanbul` [#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) - clickhouse-copier: Fix use-after free on shutdown [#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) - Updated `simdjson`. Fixed the issue that some invalid JSONs with zero bytes successfully parse. [#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([alexey-milovidov](https://github.com/alexey-milovidov)) - Fix shutdown of SystemLogs [#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([Anton Popov](https://github.com/CurtizJ)) diff --git a/docs/en/whats-new/changelog/2020.md b/docs/en/whats-new/changelog/2020.md index b54fbacd2b0..e0afe256777 100644 --- a/docs/en/whats-new/changelog/2020.md +++ b/docs/en/whats-new/changelog/2020.md @@ -2968,7 +2968,7 @@ No changes compared to v20.4.3.16-stable. * Updated checking for hung queries in clickhouse-test script [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) * Removed some useless files from repository. [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Changed type of math perftests from `once` to `loop`. [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html) +* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/ClickHouse/dbms/index.html) * Suppress some test failures under MSan. [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Speedup "exception while insert" test. This test often time out in debug-with-coverage build. [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Updated `libcxx` and `libcxxabi` to master. In preparation to [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) diff --git a/docs/en/whats-new/security-changelog.md b/docs/en/whats-new/security-changelog.md index bcfeaa06e24..685f1c6d21d 100644 --- a/docs/en/whats-new/security-changelog.md +++ b/docs/en/whats-new/security-changelog.md @@ -2,6 +2,49 @@ toc_priority: 76 toc_title: Security Changelog --- +## Fixed in ClickHouse 21.10.2.15, 2021-10-18 {#fixed-in-clickhouse-release-21-10-2-215-2021-10-18} + +### CVE-2021-43304 {#cve-2021-43304} + +Heap buffer overflow in Clickhouse's LZ4 compression codec when parsing a malicious query. There is no verification that the copy operations in the LZ4::decompressImpl loop and especially the arbitrary copy operation wildCopy(op, ip, copy_end), don’t exceed the destination buffer’s limits. + +Credits: JFrog Security Research Team + +### CVE-2021-43305 {#cve-2021-43305} + +Heap buffer overflow in Clickhouse's LZ4 compression codec when parsing a malicious query. There is no verification that the copy operations in the LZ4::decompressImpl loop and especially the arbitrary copy operation wildCopy(op, ip, copy_end), don’t exceed the destination buffer’s limits. This issue is very similar to CVE-2021-43304, but the vulnerable copy operation is in a different wildCopy call. + +Credits: JFrog Security Research Team + +### CVE-2021-42387 {#cve-2021-42387} + +Heap out-of-bounds read in Clickhouse's LZ4 compression codec when parsing a malicious query. As part of the LZ4::decompressImpl() loop, a 16-bit unsigned user-supplied value ('offset') is read from the compressed data. The offset is later used in the length of a copy operation, without checking the upper bounds of the source of the copy operation. + +Credits: JFrog Security Research Team + +### CVE-2021-42388 {#cve-2021-42388} + +Heap out-of-bounds read in Clickhouse's LZ4 compression codec when parsing a malicious query. As part of the LZ4::decompressImpl() loop, a 16-bit unsigned user-supplied value ('offset') is read from the compressed data. The offset is later used in the length of a copy operation, without checking the lower bounds of the source of the copy operation. + +Credits: JFrog Security Research Team + +### CVE-2021-42389 {#cve-2021-42389} + +Divide-by-zero in Clickhouse's Delta compression codec when parsing a malicious query. The first byte of the compressed buffer is used in a modulo operation without being checked for 0. + +Credits: JFrog Security Research Team + +### CVE-2021-42390 {#cve-2021-42390} + +Divide-by-zero in Clickhouse's DeltaDouble compression codec when parsing a malicious query. The first byte of the compressed buffer is used in a modulo operation without being checked for 0. + +Credits: JFrog Security Research Team + +### CVE-2021-42391 {#cve-2021-42391} + +Divide-by-zero in Clickhouse's Gorilla compression codec when parsing a malicious query. The first byte of the compressed buffer is used in a modulo operation without being checked for 0. + +Credits: JFrog Security Research Team ## Fixed in ClickHouse 21.4.3.21, 2021-04-12 {#fixed-in-clickhouse-release-21-4-3-21-2021-04-12} diff --git a/docs/ja/development/browse-code.md b/docs/ja/development/browse-code.md index 6539014eaf0..2f4cfb622c6 100644 --- a/docs/ja/development/browse-code.md +++ b/docs/ja/development/browse-code.md @@ -7,7 +7,7 @@ toc_title: "\u30BD\u30FC\u30B9\u30B3\u30FC\u30C9\u306E\u53C2\u7167" # ClickHouseのソースコードを参照 {#browse-clickhouse-source-code} -以下を使用できます **Woboq** オンラインのコードブラウザをご利用 [ここに](https://clickhouse.com/codebrowser/html_report/ClickHouse/src/index.html). このコードナビゲーションや意味のハイライト表示、検索インデックス. コードのスナップショットは随時更新中です。 +以下を使用できます **Woboq** オンラインのコードブラウザをご利用 [ここに](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). このコードナビゲーションや意味のハイライト表示、検索インデックス. コードのスナップショットは随時更新中です。 また、ソースを参照することもできます [GitHub](https://github.com/ClickHouse/ClickHouse) いつものように diff --git a/docs/ja/engines/table-engines/mergetree-family/replication.md b/docs/ja/engines/table-engines/mergetree-family/replication.md index 195c2453467..3819b8be068 100644 --- a/docs/ja/engines/table-engines/mergetree-family/replication.md +++ b/docs/ja/engines/table-engines/mergetree-family/replication.md @@ -14,7 +14,7 @@ toc_title: "\u30C7\u30FC\u30BF\u8907\u88FD" - レプリケートリプレースマージツリー - 複製された集合マージツリー - レプリケートコラプシングマージツリー -- ReplicatedVersionedCollapsingMergetree +- ReplicatedVersionedCollapsingMergeTree - レプリケートグラフィティマージツリー 複製の作品のレベルを個別のテーブルではなく、全体のサーバーです。 サーバーでの店舗も複製、非複製のテーブルでも同時に行います。 diff --git a/docs/ja/engines/table-engines/special/distributed.md b/docs/ja/engines/table-engines/special/distributed.md index 7d347b3eeb2..178940846b8 100644 --- a/docs/ja/engines/table-engines/special/distributed.md +++ b/docs/ja/engines/table-engines/special/distributed.md @@ -130,7 +130,7 @@ SELECT queries are sent to all the shards and work regardless of how data is dis 次の場合は、シャーディングスキームについて心配する必要があります: - 特定のキーによるデータの結合(INまたはJOIN)を必要とするクエリが使用されます。 このキーによってデータがシャードされる場合は、GLOBAL INまたはGLOBAL JOINの代わりにlocal INまたはJOINを使用できます。 -- 多数のサーバーが、多数の小さなクエリ（個々のクライアント-ウェブサイト、広告主、またはパートナーのクエリ）で使用されます（数百以上）。 小さなクエリがクラスタ全体に影響を与えないようにするには、単一のシャード上の単一のクライアントのデータを検索することが理にかなってい また、我々はYandexのでやったように。Metricaでは、biレベルのシャーディングを設定できます：クラスタ全体を次のように分割します “layers” ここで、レイヤーは複数のシャードで構成されます。 単一のクライアントのデータは単一のレイヤー上にありますが、必要に応じてシャードをレイヤーに追加することができ、データはランダムに分散されます。 分散テーブルはレイヤごとに作成され、グローバルクエリ用に単一の共有分散テーブルが作成されます。 +- 多数のサーバーが、多数の小さなクエリ（個々のクライアント-ウェブサイト、広告主、またはパートナーのクエリ）で使用されます（数百以上）。 小さなクエリがクラスタ全体に影響を与えないようにするには、単一のシャード上の単一のクライアントのデータを検索することが理にかなってい また レベルのシャーディングを設定できます：クラスタ全体を次のように分割します “layers” ここで、レイヤーは複数のシャードで構成されます。 単一のクライアントのデータは単一のレイヤー上にありますが、必要に応じてシャードをレイヤーに追加することができ、データはランダムに分散されます。 分散テーブルはレイヤごとに作成され、グローバルクエリ用に単一の共有分散テーブルが作成されます。 データは非同期に書き込まれます。 テーブルに挿入すると、データブロックはローカルファイルシステムに書き込まれます。 データはできるだけ早くバックグラウンドでリモートサーバーに送信されます。 データを送信するための期間は、 [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) と [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) 設定。 その `Distributed` エンジンは、挿入されたデータを含む各ファイルを別々に送信しますが、 [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) 設定。 この設定の改善にクラスターの性能をより一層の活用地域のサーバやネットワーク資源です。 を確認しておきましょうか否かのデータが正常に送信されるチェックリストファイル(データまたは間に-をはさんだ)はテーブルディレクトリ: `/var/lib/clickhouse/data/database/table/`. diff --git a/docs/ja/getting-started/example-datasets/nyc-taxi.md b/docs/ja/getting-started/example-datasets/nyc-taxi.md index e4864c74358..ddf4da6b58b 100644 --- a/docs/ja/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ja/getting-started/example-datasets/nyc-taxi.md @@ -21,7 +21,7 @@ toc_title: "ニューヨークタクシー" ファイルの中には、無効な行が含まれている場合があり、以下のように修正することができます。 ``` bash -sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-02.csv > data/yellow_tripdata_2010-02.csv_ +sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-02.csv > data/yellow_tripdata_2010-02.csv_Yand sed -E '/(.*,){18,}/d' data/yellow_tripdata_2010-03.csv > data/yellow_tripdata_2010-03.csv_ mv data/yellow_tripdata_2010-02.csv_ data/yellow_tripdata_2010-02.csv mv data/yellow_tripdata_2010-03.csv_ data/yellow_tripdata_2010-03.csv @@ -378,7 +378,6 @@ Q3:0.051秒 Q4:0.072秒 この場合、クエリの処理時間は、ネットワークのレイテンシによって決定されます。 -フィンランドのYandexデータセンターにあるクライアントをロシアのクラスター上に置いてクエリを実行したところ、約20ミリ秒のレイテンシが追加されました。 ## サマリ {#summary} diff --git a/docs/ja/getting-started/install.md b/docs/ja/getting-started/install.md index 7a2a822fe52..b348ae7a6ca 100644 --- a/docs/ja/getting-started/install.md +++ b/docs/ja/getting-started/install.md @@ -28,9 +28,17 @@ Debian や Ubuntu 用にコンパイル済みの公式パッケージ `deb` を {% include 'install/deb.sh' %} ``` +
+ +Deprecated Method for installing deb-packages +``` bash +{% include 'install/deb_repo.sh' %} +``` +
+ 最新版を使いたい場合は、`stable`を`testing`に置き換えてください。（テスト環境ではこれを推奨します） -同様に、[こちら](https://repo.clickhouse.com/deb/stable/main/)からパッケージをダウンロードして、手動でインストールすることもできます。 +同様に、[こちら](https://packages.clickhouse.com/deb/pool/stable)からパッケージをダウンロードして、手動でインストールすることもできます。 #### パッケージ {#packages} @@ -46,11 +54,17 @@ CentOS、RedHat、その他すべてのrpmベースのLinuxディストリビュ まず、公式リポジトリを追加する必要があります: ``` bash -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 +{% include 'install/rpm.sh' %} ``` +
+ +Deprecated Method for installing rpm-packages +``` bash +{% include 'install/rpm_repo.sh' %} +``` +
+ 最新版を使いたい場合は `stable` を `testing` に置き換えてください。(テスト環境ではこれが推奨されています)。`prestable` もしばしば同様に利用できます。 そして、以下のコマンドを実行してパッケージをインストールします: @@ -59,35 +73,26 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 sudo yum install clickhouse-server clickhouse-client ``` -同様に、[こちら](https://repo.clickhouse.com/rpm/stable/x86_64) からパッケージをダウンロードして、手動でインストールすることもできます。 +同様に、[こちら](https://packages.clickhouse.com/rpm/stable) からパッケージをダウンロードして、手動でインストールすることもできます。 ### Tgzアーカイブから {#from-tgz-archives} すべての Linux ディストリビューションで、`deb` や `rpm` パッケージがインストールできない場合は、公式のコンパイル済み `tgz` アーカイブを使用することをお勧めします。 -必要なバージョンは、リポジトリ https://repo.clickhouse.com/tgz/ から `curl` または `wget` でダウンロードできます。その後、ダウンロードしたアーカイブを解凍し、インストールスクリプトでインストールしてください。最新版の例は以下です: +必要なバージョンは、リポジトリ https://packages.clickhouse.com/tgz/ から `curl` または `wget` でダウンロードできます。その後、ダウンロードしたアーカイブを解凍し、インストールスクリプトでインストールしてください。最新版の例は以下です: ``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-client-$LATEST_VERSION.tgz - -tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz -sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz -sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-server-$LATEST_VERSION.tgz -sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh -sudo /etc/init.d/clickhouse-server start - -tar -xzvf clickhouse-client-$LATEST_VERSION.tgz -sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh +{% include 'install/tgz.sh' %} ``` +
+ +Deprecated Method for installing tgz archives +``` bash +{% include 'install/tgz_repo.sh' %} +``` +
+ 本番環境では、最新の `stable` バージョンを使うことをお勧めします。GitHub のページ https://github.com/ClickHouse/ClickHouse/tags で 接尾辞 `-stable` となっているバージョン番号として確認できます。 ### Dockerイメージから {#from-docker-image} @@ -186,6 +191,6 @@ SELECT 1 **おめでとうございます！システムが動きました！** -動作確認を続けるには、テストデータセットをダウンロードするか、[チュートリアル](https://clickhouse.com/tutorial.html)を参照してください。 +動作確認を続けるには、テストデータセットをダウンロードするか、[チュートリアル](./tutorial.md)を参照してください。 [元の記事](https://clickhouse.com/docs/en/getting_started/install/) diff --git a/docs/ja/getting-started/playground.md b/docs/ja/getting-started/playground.md index 905a26d6570..01d7dd5b69f 100644 --- a/docs/ja/getting-started/playground.md +++ b/docs/ja/getting-started/playground.md @@ -5,68 +5,39 @@ toc_title: Playground # ClickHouse Playground {#clickhouse-playground} -[ClickHouse Playground](https://play.clickhouse.com) では、サーバーやクラスタを設定することなく、即座にクエリを実行して ClickHouse を試すことができます。 -いくつかの例のデータセットは、Playground だけでなく、ClickHouse の機能を示すサンプルクエリとして利用可能です. また、 ClickHouse の LTS リリースで試すこともできます。 +[ClickHouse Playground](https://play.clickhouse.com/play?user=play) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. +Several example datasets are available in Playground. -ClickHouse Playground は、[Yandex.Cloud](https://cloud.yandex.com/)にホストされている m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) インスタンス(4 vCPU, 32 GB RAM) で提供されています。クラウドプロバイダの詳細情報については[こちら](../commercial/cloud.md)。 +You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). -任意の HTTP クライアントを使用してプレイグラウンドへのクエリを作成することができます。例えば[curl](https://curl.haxx.se)、[wget](https://www.gnu.org/software/wget/)、[JDBC](../interfaces/jdbc.md)または[ODBC](../interfaces/odbc.md)ドライバを使用して接続を設定します。 -ClickHouse をサポートするソフトウェア製品の詳細情報は[こちら](../interfaces/index.md)をご覧ください。 +## Credentials {#credentials} -## 資格情報 {#credentials} +| Parameter | Value | +|:--------------------|:-----------------------------------| +| HTTPS endpoint | `https://play.clickhouse.com:443/` | +| Native TCP endpoint | `play.clickhouse.com:9440` | +| User | `explorer` or `play` | +| Password | (empty) | -| パラメータ | 値 | -| :---------------------------- | :-------------------------------------- | -| HTTPS エンドポイント | `https://play-api.clickhouse.com:8443` | -| ネイティブ TCP エンドポイント | `play-api.clickhouse.com:9440` | -| ユーザ名 | `playgrounnd` | -| パスワード | `clickhouse` | +## Limitations {#limitations} +The queries are executed as a read-only user. It implies some limitations: -特定のClickHouseのリリースで試すために、追加のエンドポイントがあります。（ポートとユーザー/パスワードは上記と同じです）。 +- DDL queries are not allowed +- INSERT queries are not allowed -- 20.3 LTS: `play-api-v20-3.clickhouse.com` -- 19.14 LTS: `play-api-v19-14.clickhouse.com` +The service also have quotas on its usage. -!!! note "備考" -これらのエンドポイントはすべて、安全なTLS接続が必要です。 +## Examples {#examples} - -## 制限事項 {#limitations} - -クエリは読み取り専用のユーザとして実行されます。これにはいくつかの制限があります。 - -- DDL クエリは許可されていません。 -- INSERT クエリは許可されていません。 - -また、以下の設定がなされています。 - -- [max_result_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes) -- [max_result_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows) -- [result_overflow_mode=break](../operations/settings/query_complexity/#result-overflow-mode) -- [max_execution_time=60000](../operations/settings/query_complexity/#max-execution-time) - -## 例 {#examples} - -`curl` を用いて HTTPSエンドポイントへ接続する例: +HTTPS endpoint example with `curl`: ``` bash -curl "https://play-api.clickhouse.com:8443/?query=SELECT+'Play+ClickHouse\!';&user=playground&password=clickhouse&database=datasets" +curl "https://play.clickhouse.com/?user=explorer" --data-binary "SELECT 'Play ClickHouse'" ``` -[CLI](../interfaces/cli.md) で TCP エンドポイントへ接続する例: +TCP endpoint example with [CLI](../interfaces/cli.md): ``` bash -clickhouse client --secure -h play-api.clickhouse.com --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse\!'" +clickhouse client --secure --host play.clickhouse.com --user explorer ``` - -## 実装の詳細 {#implementation-details} - -ClickHouse PlaygroundのWebインタフェースは、ClickHouse [HTTP API](../interfaces/http.md)を介してリクエストを行います。 -Playgroundのバックエンドは、追加のサーバーサイドのアプリケーションを伴わない、ただのClickHouseクラスタです。 -上記のように, ClickHouse HTTPSとTCP/TLSのエンドポイントは Playground の一部としても公開されており、 -いずれも、上記の保護とよりよいグローバルな接続のためのレイヤを追加するために、[Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) を介してプロキシされています。 - -!!! warning "注意" - いかなる場合においても、インターネットにClickHouseサーバを公開することは **非推奨です**。 - プライベートネットワーク上でのみ接続を待機し、適切に設定されたファイアウォールによって保護されていることを確認してください。 diff --git a/docs/ja/getting-started/tutorial.md b/docs/ja/getting-started/tutorial.md index f80343cbad6..69ea68956e1 100644 --- a/docs/ja/getting-started/tutorial.md +++ b/docs/ja/getting-started/tutorial.md @@ -547,19 +547,19 @@ ClickHouseクラスタは均質なクラスタ(homogenous cluster)です。セ - example-perftest01j.yandex.ru + example-perftest01j 9000 - example-perftest02j.yandex.ru + example-perftest02j 9000 - example-perftest03j.yandex.ru + example-perftest03j 9000 @@ -607,15 +607,15 @@ INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; - example-perftest01j.yandex.ru + example-perftest01j 9000 - example-perftest02j.yandex.ru + example-perftest02j 9000 - example-perftest03j.yandex.ru + example-perftest03j 9000 @@ -637,15 +637,15 @@ ZooKeeperの場所は設定ファイルで指定します: ``` xml - zoo01.yandex.ru + zoo01 2181 - zoo02.yandex.ru + zoo02 2181 - zoo03.yandex.ru + zoo03 2181 diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md index 58ff96c728d..eaa77df681f 100644 --- a/docs/ja/interfaces/third-party/integrations.md +++ b/docs/ja/interfaces/third-party/integrations.md @@ -8,7 +8,7 @@ toc_title: "\u7D71\u5408" # サードパーティ開発者からの統合ライブラリ {#integration-libraries-from-third-party-developers} !!! warning "免責事項" - Yandexのは **ない** 以下のツールとライブラリを維持し、その品質を確保するための広範なテストを行っていません。 + ClickHouse, Inc.のは **ない** 以下のツールとライブラリを維持し、その品質を確保するための広範なテストを行っていません。 ## インフラ製品 {#infrastructure-products} diff --git a/docs/ja/operations/external-authenticators/ssl-x509.md b/docs/ja/operations/external-authenticators/ssl-x509.md new file mode 120000 index 00000000000..80b18e1b352 --- /dev/null +++ b/docs/ja/operations/external-authenticators/ssl-x509.md @@ -0,0 +1 @@ +../../../en/operations/external-authenticators/ssl-x509.md \ No newline at end of file diff --git a/docs/ja/operations/performance-test.md b/docs/ja/operations/performance-test.md index 068eb4fbc04..8c05acaf60b 100644 --- a/docs/ja/operations/performance-test.md +++ b/docs/ja/operations/performance-test.md @@ -20,9 +20,21 @@ toc_title: "\u30CF\u30FC\u30C9\u30A6\u30A7\u30A2\u8A66\u9A13" # For amd64: - wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578163263_binary/clickhouse + wget https://builds.clickhouse.com/master/amd64/clickhouse # For aarch64: - wget https://clickhouse-builds.s3.yandex.net/0/00ba767f5d2a929394ea3be193b1f79074a1c4bc/1578161264_binary/clickhouse + wget https://builds.clickhouse.com/master/aarch64/clickhouse + # For powerpc64le: + wget https://builds.clickhouse.com/master/powerpc64le/clickhouse + # For freebsd: + wget https://builds.clickhouse.com/master/freebsd/clickhouse + # For freebsd-aarch64: + wget https://builds.clickhouse.com/master/freebsd-aarch64/clickhouse + # For freebsd-powerpc64le: + wget https://builds.clickhouse.com/master/freebsd-powerpc64le/clickhouse + # For macos: + wget https://builds.clickhouse.com/master/macos/clickhouse + # For macos-aarch64: + wget https://builds.clickhouse.com/master/macos-aarch64/clickhouse # Then do: chmod a+x clickhouse diff --git a/docs/ja/operations/server-configuration-parameters/settings.md b/docs/ja/operations/server-configuration-parameters/settings.md index 7e0d5ebcc22..3e5a643eb6a 100644 --- a/docs/ja/operations/server-configuration-parameters/settings.md +++ b/docs/ja/operations/server-configuration-parameters/settings.md @@ -694,7 +694,7 @@ UTCタイムゾーンまたは地理的位置(たとえば、Africa/Abidjan)のI **例** ``` xml -Europe/Moscow +Asia/Istanbul ``` ## tcp_port {#server_configuration_parameters-tcp_port} diff --git a/docs/ja/sql-reference/data-types/date.md b/docs/ja/sql-reference/data-types/date.md index 47b105627c2..20ce8d524ea 100644 --- a/docs/ja/sql-reference/data-types/date.md +++ b/docs/ja/sql-reference/data-types/date.md @@ -7,6 +7,8 @@ toc_title: "\u65E5\u4ED8" 日付型です。 1970-01-01 からの日数が2バイトの符号なし整数として格納されます。 UNIX時間の開始直後から、変換段階で定数として定義される上限しきい値までの値を格納できます（現在は2106年までですが、一年分を完全にサポートしているのは2105年までです）。 +サポートされる値の範囲: \[1970-01-01, 2149-06-06\]. + 日付値は、タイムゾーンなしで格納されます。 [元の記事](https://clickhouse.com/docs/en/data_types/date/) diff --git a/docs/ja/sql-reference/data-types/datetime.md b/docs/ja/sql-reference/data-types/datetime.md index 50f06cee21c..01f5e76762c 100644 --- a/docs/ja/sql-reference/data-types/datetime.md +++ b/docs/ja/sql-reference/data-types/datetime.md @@ -15,7 +15,7 @@ toc_title: DateTime DateTime([timezone]) ``` -サポートされる値の範囲: \[1970-01-01 00:00:00, 2105-12-31 23:59:59\]. +サポートされる値の範囲: \[1970-01-01 00:00:00, 2106-02-07 06:28:15\]. 解像度:1秒. @@ -40,7 +40,7 @@ ClickHouseにデータを挿入するときは、データの値に応じて、 ``` sql CREATE TABLE dt ( - `timestamp` DateTime('Europe/Moscow'), + `timestamp` DateTime('Asia/Istanbul'), `event_id` UInt8 ) ENGINE = TinyLog; @@ -61,13 +61,13 @@ SELECT * FROM dt; └─────────────────────┴──────────┘ ``` -- Datetimeを整数として挿入する場合は、Unix Timestamp(UTC)として扱われます。 `1546300800` を表す `'2019-01-01 00:00:00'` UTC しかし、 `timestamp` 列は `Europe/Moscow` (UTC+3)タイムゾーンが指定されている場合、文字列として出力すると、値は次のように表示されます `'2019-01-01 03:00:00'` -- 文字列値をdatetimeとして挿入すると、列タイムゾーンにあるものとして扱われます。 `'2019-01-01 00:00:00'` であるとして扱われます `Europe/Moscow` タイムゾーンとして保存 `1546290000`. +- Datetimeを整数として挿入する場合は、Unix Timestamp(UTC)として扱われます。 `1546300800` を表す `'2019-01-01 00:00:00'` UTC しかし、 `timestamp` 列は `Asia/Istanbul` (UTC+3)タイムゾーンが指定されている場合、文字列として出力すると、値は次のように表示されます `'2019-01-01 03:00:00'` +- 文字列値をdatetimeとして挿入すると、列タイムゾーンにあるものとして扱われます。 `'2019-01-01 00:00:00'` であるとして扱われます `Asia/Istanbul` タイムゾーンとして保存 `1546290000`. **2.** フィルタリング `DateTime` 値 ``` sql -SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Europe/Moscow') +SELECT * FROM dt WHERE timestamp = toDateTime('2019-01-01 00:00:00', 'Asia/Istanbul') ``` ``` text @@ -91,12 +91,12 @@ SELECT * FROM dt WHERE timestamp = '2019-01-01 00:00:00' **3.** Aのタイムゾーンの取得 `DateTime`-タイプ列: ``` sql -SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x +SELECT toDateTime(now(), 'Asia/Istanbul') AS column, toTypeName(column) AS x ``` ``` text ┌──────────────column─┬─x─────────────────────────┐ -│ 2019-10-16 04:12:04 │ DateTime('Europe/Moscow') │ +│ 2019-10-16 04:12:04 │ DateTime('Asia/Istanbul') │ └─────────────────────┴───────────────────────────┘ ``` @@ -105,7 +105,7 @@ SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x ``` sql SELECT toDateTime(timestamp, 'Europe/London') as lon_time, -toDateTime(timestamp, 'Europe/Moscow') as mos_time +toDateTime(timestamp, 'Asia/Istanbul') as mos_time FROM dt ``` diff --git a/docs/ja/sql-reference/data-types/datetime64.md b/docs/ja/sql-reference/data-types/datetime64.md index 12a31595b70..ff575c3d493 100644 --- a/docs/ja/sql-reference/data-types/datetime64.md +++ b/docs/ja/sql-reference/data-types/datetime64.md @@ -19,6 +19,8 @@ DateTime64(precision, [timezone]) 内部的には、データを ‘ticks’ エポック開始（1970-01-01 00:00:00UTC）以来、Int64として。 目盛りの解像度は、精度パラメータによって決定されます。 さらに、 `DateTime64` 型は、列全体で同じタイムゾーンを格納することができます。 `DateTime64` 型の値はテキスト形式で表示され、文字列として指定された値がどのように解析されるか (‘2020-01-01 05:00:01.000’). タイムゾーンは、テーブルの行(またはresultset)には格納されませんが、列メタデータに格納されます。 詳細はを参照。 [DateTime](datetime.md). +サポートされる値の範囲: \[1925-01-01 00:00:00, 2283-11-11 23:59:59.99999999\] (注）最大値の精度は、8). + ## 例 {#examples} **1.** テーブルの作成 `DateTime64`-列を入力し、そこにデータを挿入する: @@ -26,7 +28,7 @@ DateTime64(precision, [timezone]) ``` sql CREATE TABLE dt ( - `timestamp` DateTime64(3, 'Europe/Moscow'), + `timestamp` DateTime64(3, 'Asia/Istanbul'), `event_id` UInt8 ) ENGINE = TinyLog @@ -47,13 +49,13 @@ SELECT * FROM dt └─────────────────────────┴──────────┘ ``` -- Datetimeを整数として挿入する場合、適切にスケーリングされたUnixタイムスタンプ(UTC)として扱われます。 `1546300800000` （精度3で）を表します `'2019-01-01 00:00:00'` UTC しかし、 `timestamp` 列は `Europe/Moscow` (UTC+3)タイムゾーンが指定されている場合、文字列として出力すると、値は次のように表示されます `'2019-01-01 03:00:00'` -- 文字列値をdatetimeとして挿入すると、列タイムゾーンにあるものとして扱われます。 `'2019-01-01 00:00:00'` であるとして扱われます `Europe/Moscow` タイムゾーンとして保存 `1546290000000`. +- Datetimeを整数として挿入する場合、適切にスケーリングされたUnixタイムスタンプ(UTC)として扱われます。 `1546300800000` （精度3で）を表します `'2019-01-01 00:00:00'` UTC しかし、 `timestamp` 列は `Asia/Istanbul` (UTC+3)タイムゾーンが指定されている場合、文字列として出力すると、値は次のように表示されます `'2019-01-01 03:00:00'` +- 文字列値をdatetimeとして挿入すると、列タイムゾーンにあるものとして扱われます。 `'2019-01-01 00:00:00'` であるとして扱われます `Asia/Istanbul` タイムゾーンとして保存 `1546290000000`. **2.** フィルタリング `DateTime64` 値 ``` sql -SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow') +SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') ``` ``` text @@ -67,12 +69,12 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europ **3.** Aのタイムゾーンの取得 `DateTime64`-タイプ値: ``` sql -SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x +SELECT toDateTime64(now(), 3, 'Asia/Istanbul') AS column, toTypeName(column) AS x ``` ``` text ┌──────────────────column─┬─x──────────────────────────────┐ -│ 2019-10-16 04:12:04.000 │ DateTime64(3, 'Europe/Moscow') │ +│ 2019-10-16 04:12:04.000 │ DateTime64(3, 'Asia/Istanbul') │ └─────────────────────────┴────────────────────────────────┘ ``` @@ -81,7 +83,7 @@ SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS ``` sql SELECT toDateTime64(timestamp, 3, 'Europe/London') as lon_time, -toDateTime64(timestamp, 3, 'Europe/Moscow') as mos_time +toDateTime64(timestamp, 3, 'Asia/Istanbul') as mos_time FROM dt ``` diff --git a/docs/ja/sql-reference/functions/type-conversion-functions.md b/docs/ja/sql-reference/functions/type-conversion-functions.md index a16bca0c1f9..4462f0fa25d 100644 --- a/docs/ja/sql-reference/functions/type-conversion-functions.md +++ b/docs/ja/sql-reference/functions/type-conversion-functions.md @@ -460,7 +460,7 @@ AS parseDateTimeBestEffort; クエリ: ``` sql -SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Europe/Moscow') +SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Asia/Istanbul') AS parseDateTimeBestEffort ``` diff --git a/docs/ko/images/column-oriented.gif b/docs/ko/images/column-oriented.gif deleted file mode 100644 index d5ac7c82848..00000000000 Binary files a/docs/ko/images/column-oriented.gif and /dev/null differ diff --git a/docs/ko/images/logo.svg b/docs/ko/images/logo.svg deleted file mode 100644 index b5ab923ff65..00000000000 --- a/docs/ko/images/logo.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/ko/images/play.png b/docs/ko/images/play.png deleted file mode 100644 index b75aebe4089..00000000000 Binary files a/docs/ko/images/play.png and /dev/null differ diff --git a/docs/ko/images/row-oriented.gif b/docs/ko/images/row-oriented.gif deleted file mode 100644 index 41395b5693e..00000000000 Binary files a/docs/ko/images/row-oriented.gif and /dev/null differ diff --git a/docs/ko/index.md b/docs/ko/index.md deleted file mode 100644 index f2a6396c069..00000000000 --- a/docs/ko/index.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -toc_priority: 0 -toc_title: 목차 ---- - -# ClickHouse란? {#what-is-clickhouse} - -ClickHouse® 는 query의 온라인 분석 처리(OLAP)를 위한 열 지향(column-oriented) 데이터베이스 관리 시스템(DBMS)입니다. - -"보통의" 행 지향(row-oriented) DMBS에서는 데이터가 다음과 같은 순서로 저장됩니다. - -| row | WatchID | JavaEnable | Title | GoodEvent | EventTime | -|-----|-------------|------------|--------------------|-----------|---------------------| -| #0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | -| #1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | -| #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | -| #N | … | … | … | … | … | - -즉, 행과 관련된 모든 값들은 물리적으로 나란히 저장됩니다. - -행 지향(row-oriented) DMBS의 예시로는 MySQL, Postgres, 그리고 MS SQL 서버 등이 있습니다. - -열 지향 (column-oriented) DBMS에서는 데이터가 아래와 같은 방식으로 저장됩니다: - -| Row: | #0 | #1 | #2 | #N | -|-------------|---------------------|---------------------|---------------------|-----| -| WatchID: | 89354350662 | 90329509958 | 89953706054 | … | -| JavaEnable: | 1 | 0 | 1 | … | -| Title: | Investor Relations | Contact us | Mission | … | -| GoodEvent: | 1 | 1 | 1 | … | -| EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | … | - -이 예에서는 데이터가 정렬된 순서만을 보여줍니다. 다른 열의 값들은 서로 분리되어 저장되고, 같은 열의 정보들은 함께 저장됩니다. - -열 지향(column-oriented) DBMS 의 종류는 Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, 그리고 kdb+ 등이 있습니다. - -데이터를 저장하기 위한 서로 다른 순서는 다른 시나리오에 더 적합합니다. 데이터 접근 시나리오는 쿼리가 수행되는 빈도, 비율 및 비율을 나타내거나, 각 쿼리 유형(행, 열 및 바이트)에 대해 읽은 데이터의 양 데이터 읽기와 업데이트 사이의 관계, 데이터의 작업 크기 및 로컬에서 사용되는 방법 트랜잭션이 사용되는지 여부, 트랜잭션이 얼마나 격리되어 있는지, 데이터 복제 및 논리적 무결성에 대한 요구 사항, 각 쿼리 유형에 대한 대기 시간 및 처리량 요구 사항 등이 있습니다. - -시스템의 부하가 높을수록 사용 시나리오의 요구 사항에 맞게 시스템 설정을 사용자 지정하는 것이 더 중요하며 이 사용자 지정은 더욱 세분화됩니다. 상당히 다른 시나리오에 똑같이 적합한 시스템은 없습니다. 만약 높은 부하에서 시스템이 넓은 시나리오 집합에 대해 적응한다면 시스템은 모든 시나리오를 모두 제대로 처리하지 못하거나 가능한 시나리오 중 하나 또는 몇 개에 대해서만 잘 작동할 것입니다. - -## OLAP 시나리오의 중요 속성들 {#key-properties-of-olap-scenario} - -- 요청(request)의 대부분은 읽기 접근에 관한 것입니다. -- 데이터는 단일 행이 아니라 상당히 큰 일괄 처리(\> 1000개 행)로 업데이트됩니다. 또는 전혀 업데이트되지 않습니다. -- 데이터는 DB에 추가되지만 수정되지는 않습니다. -- 읽기의 경우 DB에서 상당히 많은 수의 행이 추출되지만 열은 일부만 추출됩니다. -- 테이블은 "넓습니다". 이는 열의 수가 많다는 것을 의미합니다. -- 쿼리는 상대적으로 드뭅니다(일반적으로 서버당 수백 또는 초당 쿼리 미만). -- 간단한 쿼리의 경우 약 50ms의 대기 시간이 허용됩니다. -- 열 값은 숫자와 짧은 문자열(예: URL당 60바이트)과 같이 상당히 작습니다 -- 단일 쿼리를 처리할 때 높은 처리량이 필요합니다(서버당 초당 최대 수십억 행). -- 트랜잭션이 필요하지 않습니다. -- 데이터 일관성에 대한 요구 사항이 낮습니다. -- 쿼리당 하나의 큰 테이블이 존재하고 하나를 제외한 모든 테이블은 작습니다. -- 쿼리 결과가 원본 데이터보다 훨씬 작습니다. 즉, 데이터가 필터링되거나 집계되므로 결과가 단일 서버의 RAM에 꼭 들어맞습니다. - -OLAP 시나리오가 다른 일반적인 시나리오(OLTP 또는 키-값 액세스와 같은)와 매우 다르다는 것을 쉽게 알 수 있습니다. 따라서 적절한 성능을 얻으려면 분석 쿼리를 처리하기 위해 OLTP 또는 키-값 DB를 사용하는 것은 의미가 없습니다. 예를 들어 분석에 MongoDB나 Redis를 사용하려고 하면 OLAP 데이터베이스에 비해 성능이 매우 저하됩니다. - -## 왜 열 지향 데이터베이스가 OLAP 시나리오에 적합한가{#why-column-oriented-databases-work-better-in-the-olap-scenario} - -열 지향(column-oriented) 데이터베이스는 OLAP 시나리오에 더 적합합니다. 대부분의 쿼리를 처리하는 데 있어서 행 지향(row-oriented) 데이터베이스보다 100배 이상 빠릅니다. 그 이유는 아래에 자세히 설명되어 있지만 사실은 시각적으로 더 쉽게 설명할 수 있습니다. - -**행 지향 DBMS** - -![Row-oriented](images/row-oriented.gif#) - -**열 지향 DBMS** - -![Column-oriented](images/column-oriented.gif#) - -차이가 보이시나요? - -### 입출력 {#inputoutput} - -1. 분석 쿼리의 경우 적은 수의 테이블 열만 읽어야 합니다. 열 지향 데이터베이스에서는 필요한 데이터만 읽을 수 있습니다. 예를 들어 100개 중 5개의 열이 필요한 경우 I/O가 20배 감소할 것으로 예상할 수 있습니다. -2. 데이터는 패킷으로 읽히므로 압축하기가 더 쉽습니다. 열의 데이터도 압축하기 쉽습니다. 이것은 I/O의 볼륨을 더욱 감소시킵니다. -3. 감소된 I/O로 인해 시스템 캐시에 더 많은 데이터가 들어갑니다. - -예를 들어, "각 광고 플랫폼에 대한 레코드 수 계산" 쿼리는 압축되지 않은 1바이트를 차지하는 하나의 "광고 플랫폼 ID" 열을 읽어야 합니다. 트래픽의 대부분이 광고 플랫폼에서 발생하지 않은 경우 이 열의 최소 10배 압축을 기대할 수 있습니다. 빠른 압축 알고리즘을 사용하면 초당 최소 몇 기가바이트의 압축되지 않은 데이터의 속도로 데이터 압축 해제가 가능합니다. 즉, 이 쿼리는 단일 서버에서 초당 약 수십억 행의 속도로 처리될 수 있습니다. 이 속도는 정말 실제로 달성됩니다. - -### CPU {#cpu} - -쿼리를 수행하려면 많은 행을 처리해야 하므로 별도의 행이 아닌 전체 벡터에 대한 모든 연산을 디스패치하거나 쿼리 엔진을 구현하여 디스패치 비용이 거의 들지 않습니다. 반쯤 괜찮은 디스크 하위 시스템에서 이렇게 하지 않으면 쿼리 인터프리터가 불가피하게 CPU를 정지시킵니다. 데이터를 열에 저장하고 가능한 경우 열별로 처리하는 것이 좋습니다. - -이를 수행하기위한 두가지 방법이 있습니다. - -1. 벡터 엔진. 모든 연산은 별도의 값 대신 벡터에 대해 작성됩니다. 즉, 작업을 자주 호출할 필요가 없으며 파견 비용도 무시할 수 있습니다. 작업 코드에는 최적화된 내부 주기가 포함되어 있습니다. -2. 코드 생성. 쿼리에 대해 생성된 코드에는 모든 간접 호출이 있습니다. - -이것은 단순한 쿼리를 실행할 때 의미가 없기 때문에 "일반" 데이터베이스에서는 수행되지 않습니다. 그러나 예외가 있습니다. 예를 들어 MemSQL은 코드 생성을 사용하여 SQL 쿼리를 처리할 때 대기 시간을 줄입니다. (비교되게, 분석 DBMS는 대기 시간이 아닌 처리량 최적화가 필요합니다.) - -CPU 효율성을 위해 쿼리 언어는 선언적(SQL 또는 MDX)이거나 최소한 벡터(J, K)여야 합니다. 쿼리는 최적화를 허용하는 암시적 루프만 포함해야 합니다. - -{## [원문](https://clickhouse.com/docs/en/) ##} diff --git a/docs/redirects.txt b/docs/redirects.txt index d0d4d4d6c2c..949b9d48ca8 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -6,6 +6,7 @@ changelog/2017.md whats-new/changelog/2017.md changelog/2018.md whats-new/changelog/2018.md changelog/2019.md whats-new/changelog/2019.md changelog/index.md whats-new/changelog/index.md +commercial/cloud.md https://clickhouse.com/cloud/ data_types/array.md sql-reference/data-types/array.md data_types/boolean.md sql-reference/data-types/boolean.md data_types/date.md sql-reference/data-types/date.md diff --git a/docs/ru/development/browse-code.md b/docs/ru/development/browse-code.md index 26b3f491599..730e97aed27 100644 --- a/docs/ru/development/browse-code.md +++ b/docs/ru/development/browse-code.md @@ -6,7 +6,7 @@ toc_title: "Навигация по коду ClickHouse" # Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse} -Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/html_report///ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 8508ba18d9e..af55b7cf419 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -53,15 +53,15 @@ WHERE table = 'visits' ``` ``` text -┌─partition─┬─name───────────┬─active─┐ -│ 201901 │ 201901_1_3_1 │ 0 │ -│ 201901 │ 201901_1_9_2 │ 1 │ -│ 201901 │ 201901_8_8_0 │ 0 │ -│ 201901 │ 201901_9_9_0 │ 0 │ -│ 201902 │ 201902_4_6_1 │ 1 │ -│ 201902 │ 201902_10_10_0 │ 1 │ -│ 201902 │ 201902_11_11_0 │ 1 │ -└───────────┴────────────────┴────────┘ +┌─partition─┬─name──────────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2_11 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1_11 │ 1 │ +│ 201902 │ 201902_10_10_0_11 │ 1 │ +│ 201902 │ 201902_11_11_0_11 │ 1 │ +└───────────┴───────────────────┴────────┘ ``` Столбец `partition` содержит имена всех партиций таблицы. Таблица `visits` из нашего примера содержит две партиции: `201901` и `201902`. Используйте значения из этого столбца в запросах [ALTER … PARTITION](../../../sql-reference/statements/alter/partition.md). @@ -70,12 +70,13 @@ WHERE table = 'visits' Столбец `active` отображает состояние куска. `1` означает, что кусок активен; `0` – неактивен. К неактивным можно отнести куски, оставшиеся после слияния данных. Поврежденные куски также отображаются как неактивные. Неактивные куски удаляются приблизительно через 10 минут после того, как было выполнено слияние. -Рассмотрим детальнее имя первого куска `201901_1_3_1`: +Рассмотрим детальнее имя куска `201901_1_9_2_11`: - `201901` имя партиции; - `1` – минимальный номер блока данных; -- `3` – максимальный номер блока данных; -- `1` – уровень куска (глубина дерева слияний, которыми этот кусок образован). +- `9` – максимальный номер блока данных; +- `2` – уровень куска (глубина дерева слияний, которыми этот кусок образован). +- `11` - версия мутации (если парт мутировал) !!! info "Info" Названия кусков для таблиц старого типа образуются следующим образом: `20190117_20190123_2_2_0` (минимальная дата _ максимальная дата _ номер минимального блока _ номер максимального блока _ уровень). @@ -89,16 +90,16 @@ OPTIMIZE TABLE visits PARTITION 201902; ``` ``` text -┌─partition─┬─name───────────┬─active─┐ -│ 201901 │ 201901_1_3_1 │ 0 │ -│ 201901 │ 201901_1_9_2 │ 1 │ -│ 201901 │ 201901_8_8_0 │ 0 │ -│ 201901 │ 201901_9_9_0 │ 0 │ -│ 201902 │ 201902_4_6_1 │ 0 │ -│ 201902 │ 201902_4_11_2 │ 1 │ -│ 201902 │ 201902_10_10_0 │ 0 │ -│ 201902 │ 201902_11_11_0 │ 0 │ -└───────────┴────────────────┴────────┘ +┌─partition─┬─name─────────────┬─active─┐ +│ 201901 │ 201901_1_3_1 │ 0 │ +│ 201901 │ 201901_1_9_2_11 │ 1 │ +│ 201901 │ 201901_8_8_0 │ 0 │ +│ 201901 │ 201901_9_9_0 │ 0 │ +│ 201902 │ 201902_4_6_1 │ 0 │ +│ 201902 │ 201902_4_11_2_11 │ 1 │ +│ 201902 │ 201902_10_10_0 │ 0 │ +│ 201902 │ 201902_11_11_0 │ 0 │ +└───────────┴──────────────────┴────────┘ ``` Неактивные куски будут удалены примерно через 10 минут после слияния. @@ -109,12 +110,12 @@ OPTIMIZE TABLE visits PARTITION 201902; /var/lib/clickhouse/data/default/visits$ ls -l total 40 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 201901_1_3_1 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201901_1_9_2_11 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_8_8_0 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 15:52 201901_9_9_0 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_10_10_0 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:17 201902_11_11_0 -drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:19 201902_4_11_2 +drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 16:19 201902_4_11_2_11 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 5 12:09 201902_4_6_1 drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached ``` diff --git a/docs/ru/engines/table-engines/special/distributed.md b/docs/ru/engines/table-engines/special/distributed.md index 3d7b8cf32d3..b82703e6991 100644 --- a/docs/ru/engines/table-engines/special/distributed.md +++ b/docs/ru/engines/table-engines/special/distributed.md @@ -128,7 +128,7 @@ logs - имя кластера в конфигурационном файле с Беспокоиться о схеме шардирования имеет смысл в следующих случаях: - используются запросы, требующие соединение данных (IN, JOIN) по определённому ключу - тогда если данные шардированы по этому ключу, то можно использовать локальные IN, JOIN вместо GLOBAL IN, GLOBAL JOIN, что кардинально более эффективно. -- используется большое количество серверов (сотни и больше) и большое количество маленьких запросов (запросы отдельных клиентов - сайтов, рекламодателей, партнёров) - тогда, для того, чтобы маленькие запросы не затрагивали весь кластер, имеет смысл располагать данные одного клиента на одном шарде, или (вариант, который используется в Яндекс.Метрике) сделать двухуровневое шардирование: разбить весь кластер на «слои», где слой может состоять из нескольких шардов; данные для одного клиента располагаются на одном слое, но в один слой можно по мере необходимости добавлять шарды, в рамках которых данные распределены произвольным образом; создаются распределённые таблицы на каждый слой и одна общая распределённая таблица для глобальных запросов. +- используется большое количество серверов (сотни и больше) и большое количество маленьких запросов (запросы отдельных клиентов - сайтов, рекламодателей, партнёров) - тогда, для того, чтобы маленькие запросы не затрагивали весь кластер, имеет смысл располагать данные одного клиента на одном шарде, или сделать двухуровневое шардирование: разбить весь кластер на «слои», где слой может состоять из нескольких шардов; данные для одного клиента располагаются на одном слое, но в один слой можно по мере необходимости добавлять шарды, в рамках которых данные распределены произвольным образом; создаются распределённые таблицы на каждый слой и одна общая распределённая таблица для глобальных запросов. Запись данных осуществляется полностью асинхронно. При вставке в таблицу, блок данных сначала записывается в файловую систему. Затем, в фоновом режиме отправляются на удалённые серверы при первой возможности. Период отправки регулируется настройками [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) и [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms). Движок таблиц `Distributed` отправляет каждый файл со вставленными данными отдельно, но можно включить пакетную отправку данных настройкой [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts). Эта настройка улучшает производительность кластера за счет более оптимального использования ресурсов сервера-отправителя и сети. Необходимо проверять, что данные отправлены успешно, для этого проверьте список файлов (данных, ожидающих отправки) в каталоге таблицы `/var/lib/clickhouse/data/database/table/`. Количество потоков для выполнения фоновых задач можно задать с помощью настройки [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size). diff --git a/docs/ru/faq/general/mapreduce.md b/docs/ru/faq/general/mapreduce.md index 2e7520aef1a..764ad9045f0 100644 --- a/docs/ru/faq/general/mapreduce.md +++ b/docs/ru/faq/general/mapreduce.md @@ -6,7 +6,7 @@ toc_priority: 110 # Почему бы не использовать системы типа MapReduce? {#why-not-use-something-like-mapreduce} -Системами типа MapReduce будем называть системы распределённых вычислений, в которых операция свёртки реализована на основе распределённой сортировки. Наиболее распространённое решение с открытым кодом в данном классе — [Apache Hadoop](http://hadoop.apache.org). Яндекс пользуется собственным решением — YT. +Системами типа MapReduce будем называть системы распределённых вычислений, в которых операция свёртки реализована на основе распределённой сортировки. Наиболее распространённое решение с открытым кодом в данном классе — [Apache Hadoop](http://hadoop.apache.org). В крупных IT компаниях вроде Google или Яндекс часто используются собственные закрытые решения. Такие системы не подходят для онлайн запросов в силу слишком большой задержки. То есть не могут быть использованы в качестве бэкенда для веб-интерфейса. Также эти системы не подходят для обновления данных в реальном времени. Распределённая сортировка является не оптимальным способом для выполнения операции свёртки в случае запросов, выполняющихся в режиме онлайн, потому что результат выполнения операции и все промежуточные результаты (если такие есть) помещаются в оперативную память на одном сервере. В таком случае оптимальным способом выполнения операции свёртки является хеш-таблица. Частым способом оптимизации "map-reduce" задач является предагрегация (частичная свёртка) с использованием хеш-таблицы в оперативной памяти. Пользователь делает эту оптимизацию в ручном режиме. Распределённая сортировка — основная причина тормозов при выполнении несложных задач типа "map-reduce". diff --git a/docs/ru/getting-started/example-datasets/nyc-taxi.md b/docs/ru/getting-started/example-datasets/nyc-taxi.md index 9c3ef6b0652..8ff91d0e6a8 100644 --- a/docs/ru/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ru/getting-started/example-datasets/nyc-taxi.md @@ -380,7 +380,7 @@ Q3: 0.051 sec. Q4: 0.072 sec. В этом случае, время выполнения запросов определяется в первую очередь сетевыми задержками. -Мы выполняли запросы с помощью клиента, расположенного в дата-центре Яндекса в Мянтсяля (Финляндия), на кластер в России, что добавляет порядка 20 мс задержки. +Мы выполняли запросы с помощью клиента, расположенного в другом дата-центре, не там где кластер, что добавляет порядка 20 мс задержки. ## Резюме {#reziume} diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index a12773a75b0..8b35b8a836d 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -27,11 +27,17 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su {% include 'install/deb.sh' %} ``` -Также эти пакеты можно скачать и установить вручную отсюда: https://repo.clickhouse.com/deb/stable/main/. +
+ +Устаревший способ установки deb-пакетов +``` bash +{% include 'install/deb_repo.sh' %} +``` +
Чтобы использовать различные [версии ClickHouse](../faq/operations/production.md) в зависимости от ваших потребностей, вы можете заменить `stable` на `lts` или `testing`. -Также вы можете вручную скачать и установить пакеты из [репозитория](https://repo.clickhouse.com/deb/stable/main/). +Также вы можете вручную скачать и установить пакеты из [репозитория](https://packages.clickhouse.com/deb/pool/stable). #### Пакеты {#packages} @@ -51,11 +57,17 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su Сначала нужно подключить официальный репозиторий: ``` bash -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 +{% include 'install/rpm.sh' %} ``` +
+ +Устаревший способ установки rpm-пакетов +``` bash +{% include 'install/rpm_repo.sh' %} +``` +
+ Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). Также иногда доступен `prestable`. Для, собственно, установки пакетов необходимо выполнить следующие команды: @@ -64,36 +76,27 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 sudo yum install clickhouse-server clickhouse-client ``` -Также есть возможность установить пакеты вручную, скачав отсюда: https://repo.clickhouse.com/rpm/stable/x86_64. +Также есть возможность установить пакеты вручную, скачав отсюда: https://packages.clickhouse.com/rpm/stable. ### Из Tgz архивов {#from-tgz-archives} Команда ClickHouse в Яндексе рекомендует использовать предкомпилированные бинарники из `tgz` архивов для всех дистрибутивов, где невозможна установка `deb` и `rpm` пакетов. -Интересующую версию архивов можно скачать вручную с помощью `curl` или `wget` из репозитория https://repo.clickhouse.com/tgz/. +Интересующую версию архивов можно скачать вручную с помощью `curl` или `wget` из репозитория https://packages.clickhouse.com/tgz/. После этого архивы нужно распаковать и воспользоваться скриптами установки. Пример установки самой свежей версии: ``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-client-$LATEST_VERSION.tgz - -tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz -sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz -sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-server-$LATEST_VERSION.tgz -sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh -sudo /etc/init.d/clickhouse-server start - -tar -xzvf clickhouse-client-$LATEST_VERSION.tgz -sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh +{% include 'install/tgz.sh' %} ``` +
+ +Устаревший способ установки из архивов tgz +``` bash +{% include 'install/tgz_repo.sh' %} +``` +
+ Для production окружений рекомендуется использовать последнюю `stable`-версию. Её номер также можно найти на github с на вкладке https://github.com/ClickHouse/ClickHouse/tags c постфиксом `-stable`. ### Из Docker образа {#from-docker-image} @@ -195,4 +198,4 @@ SELECT 1 **Поздравляем, система работает!** -Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.com/tutorial.html). +Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](./tutorial.md). diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md index 029fa706576..01d7dd5b69f 100644 --- a/docs/ru/getting-started/playground.md +++ b/docs/ru/getting-started/playground.md @@ -5,60 +5,39 @@ toc_title: Playground # ClickHouse Playground {#clickhouse-playground} -[ClickHouse Playground](https://play.clickhouse.com) позволяет пользователям экспериментировать с ClickHouse, мгновенно выполняя запросы без настройки своего сервера или кластера. -В Playground доступны несколько тестовых массивов данных, а также примеры запросов, которые показывают возможности ClickHouse. Кроме того, вы можете выбрать LTS релиз ClickHouse, который хотите протестировать. +[ClickHouse Playground](https://play.clickhouse.com/play?user=play) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. +Several example datasets are available in Playground. -ClickHouse Playground дает возможность поработать с [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) в конфигурации m2.small (4 vCPU, 32 ГБ ОЗУ), которую предосталяет [Яндекс.Облако](https://cloud.yandex.com/). Дополнительную информацию об облачных провайдерах читайте в разделе [Поставщики облачных услуг ClickHouse](../commercial/cloud.md). +You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). -Вы можете отправлять запросы к Playground с помощью любого HTTP-клиента, например [curl](https://curl.haxx.se) или [wget](https://www.gnu.org/software/wget/), также можно установить соединение с помощью драйверов [JDBC](../interfaces/jdbc.md) или [ODBC](../interfaces/odbc.md). Более подробная информация о программных продуктах, поддерживающих ClickHouse, доступна [здесь](../interfaces/index.md). +## Credentials {#credentials} -## Параметры доступа {#credentials} +| Parameter | Value | +|:--------------------|:-----------------------------------| +| HTTPS endpoint | `https://play.clickhouse.com:443/` | +| Native TCP endpoint | `play.clickhouse.com:9440` | +| User | `explorer` or `play` | +| Password | (empty) | -| Параметр | Значение | -|:--------------------|:----------------------------------------| -| Конечная точка HTTPS| `https://play-api.clickhouse.com:8443` | -| Конечная точка TCP | `play-api.clickhouse.com:9440` | -| Пользователь | `playground` | -| Пароль | `clickhouse` | +## Limitations {#limitations} -Также можно подключаться к ClickHouse определённых релизов, чтобы протестировать их различия (порты и пользователь / пароль остаются неизменными): +The queries are executed as a read-only user. It implies some limitations: -- 20.3 LTS: `play-api-v20-3.clickhouse.com` -- 19.14 LTS: `play-api-v19-14.clickhouse.com` +- DDL queries are not allowed +- INSERT queries are not allowed -!!! note "Примечание" - Для всех этих конечных точек требуется безопасное соединение TLS. +The service also have quotas on its usage. -## Ограничения {#limitations} +## Examples {#examples} -Запросы выполняются под пользователем с правами `readonly`, для которого есть следующие ограничения: -- запрещены DDL запросы -- запрещены INSERT запросы - -Также установлены следующие опции: -- [max_result_bytes=10485760](../operations/settings/query-complexity.md#max-result-bytes) -- [max_result_rows=2000](../operations/settings/query-complexity.md#setting-max_result_rows) -- [result_overflow_mode=break](../operations/settings/query-complexity.md#result-overflow-mode) -- [max_execution_time=60000](../operations/settings/query-complexity.md#max-execution-time) - -## Примеры {#examples} - -Пример конечной точки HTTPS с `curl`: +HTTPS endpoint example with `curl`: ``` bash -curl "https://play-api.clickhouse.com:8443/?query=SELECT+'Play+ClickHouse\!';&user=playground&password=clickhouse&database=datasets" +curl "https://play.clickhouse.com/?user=explorer" --data-binary "SELECT 'Play ClickHouse'" ``` -Пример конечной точки TCP с [CLI](../interfaces/cli.md): +TCP endpoint example with [CLI](../interfaces/cli.md): ``` bash -clickhouse client --secure -h play-api.clickhouse.com --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse\!'" +clickhouse client --secure --host play.clickhouse.com --user explorer ``` - -## Детали реализации {#implementation-details} - -Веб-интерфейс ClickHouse Playground выполняет запросы через ClickHouse [HTTP API](../interfaces/http.md). -Бэкэнд Playground - это кластер ClickHouse без дополнительных серверных приложений. Как упоминалось выше, способы подключения по HTTPS и TCP/TLS общедоступны как часть Playground. Они проксируются через [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) для добавления дополнительного уровня защиты и улучшенного глобального подключения. - -!!! warning "Предупреждение" - Открывать сервер ClickHouse для публичного доступа в любой другой ситуации **настоятельно не рекомендуется**. Убедитесь, что он настроен только на частную сеть и защищен брандмауэром. diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 62557edff53..e948e77cb24 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -6,7 +6,7 @@ toc_title: "Библиотеки для интеграции от сторонн # Библиотеки для интеграции от сторонних разработчиков {#biblioteki-dlia-integratsii-ot-storonnikh-razrabotchikov} !!! warning "Disclaimer" - Яндекс не занимается поддержкой перечисленных ниже инструментов и библиотек и не проводит тщательного тестирования для проверки их качества. + ClickHouse, Inc. не занимается поддержкой перечисленных ниже инструментов и библиотек и не проводит тщательного тестирования для проверки их качества. ## Инфраструктурные продукты {#infrastrukturnye-produkty} diff --git a/docs/ru/operations/clickhouse-keeper.md b/docs/ru/operations/clickhouse-keeper.md index 2f3f3c0f63c..fe0f7d12893 100644 --- a/docs/ru/operations/clickhouse-keeper.md +++ b/docs/ru/operations/clickhouse-keeper.md @@ -54,7 +54,7 @@ ClickHouse Keeper может использоваться как равноце - `auto_forwarding` — разрешить пересылку запросов на запись от последователей лидеру (по умолчанию: true). - `shutdown_timeout` — время ожидания завершения внутренних подключений и выключения, в миллисекундах (по умолчанию: 5000). - `startup_timeout` — время отключения сервера, если он не подключается к другим участникам кворума, в миллисекундах (по умолчанию: 30000). -- `four_letter_word_white_list` — список разрешенных 4-х буквенных команд (по умолчанию: "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro"). +- `four_letter_word_allow_list` — список разрешенных 4-х буквенных команд (по умолчанию: "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro"). Конфигурация кворума находится в `.` и содержит описание серверов. @@ -114,7 +114,7 @@ clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon ClickHouse Keeper также поддерживает 4-х буквенные команды, почти такие же, как у Zookeeper. Каждая команда состоит из 4-х символов, например, `mntr`, `stat` и т. д. Несколько интересных команд: `stat` предоставляет общую информацию о сервере и подключенных клиентах, а `srvr` и `cons` предоставляют расширенные сведения о сервере и подключениях соответственно. -У 4-х буквенных команд есть параметр для настройки разрешенного списка `four_letter_word_white_list`, который имеет значение по умолчанию "conf,cons,crst,envi,ruok,srst,srvr,stat, wchc,wchs,dirs,mntr,isro". +У 4-х буквенных команд есть параметр для настройки разрешенного списка `four_letter_word_allow_list`, который имеет значение по умолчанию "conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro". Вы можете отправлять команды в ClickHouse Keeper через telnet или nc на порт для клиента. @@ -194,7 +194,7 @@ Server stats reset. ``` server_id=1 tcp_port=2181 -four_letter_word_white_list=* +four_letter_word_allow_list=* log_storage_path=./coordination/logs snapshot_storage_path=./coordination/snapshots max_requests_batch_size=100 diff --git a/docs/ru/operations/external-authenticators/index.md b/docs/ru/operations/external-authenticators/index.md index 8465e57d792..9fb07450d7c 100644 --- a/docs/ru/operations/external-authenticators/index.md +++ b/docs/ru/operations/external-authenticators/index.md @@ -12,5 +12,6 @@ ClickHouse поддерживает аутентификацию и управл - [LDAP](./ldap.md#external-authenticators-ldap) [аутентификатор](./ldap.md#ldap-external-authenticator) и [каталог](./ldap.md#ldap-external-user-directory) - Kerberos [аутентификатор](./kerberos.md#external-authenticators-kerberos) +- [SSL X.509 аутентификация](./ssl-x509.md#ssl-external-authentication) [Оригинальная статья](https://clickhouse.com/docs/ru/operations/external-authenticators/index/) diff --git a/docs/ru/operations/external-authenticators/ssl-x509.md b/docs/ru/operations/external-authenticators/ssl-x509.md new file mode 100644 index 00000000000..12ae7e4eec3 --- /dev/null +++ b/docs/ru/operations/external-authenticators/ssl-x509.md @@ -0,0 +1,24 @@ +# Аутентификация по сертификату SSL X.509 {#ssl-external-authentication} + +[Опция 'strict'](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) включает обязательную проверку сертификатов входящих соединений в библиотеке `SSL`. В этом случае могут быть установлены только соединения, представившие действительный сертификат. Соединения с недоверенными сертификатами будут отвергнуты. Таким образом, проверка сертификата позволяет однозначно аутентифицировать входящее соединение. Идентификация пользователя осуществляется по полю `Common Name` сертификата. Это позволяет ассоциировать несколько сертификатов с одним и тем же пользователем. Дополнительно, перевыпуск и отзыв сертификата не требуют изменения конфигурации ClickHouse. + +Для включения аутентификации по SSL сертификату, необходимо указать список `Common Name` для каждого пользователя ClickHouse в файле настройки `config.xml`: + +**Example** +```xml + + + + + + host.domain.com:example_user + host.domain.com:example_user_dev + + + + + + +``` + +Для правильной работы SSL [`chain of trust`](https://en.wikipedia.org/wiki/Chain_of_trust) важно также убедиться в правильной настройке параметра [`caConfig`](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) diff --git a/docs/ru/operations/named-collections.md b/docs/ru/operations/named-collections.md new file mode 100644 index 00000000000..d39177daa12 --- /dev/null +++ b/docs/ru/operations/named-collections.md @@ -0,0 +1,228 @@ +--- +toc_priority: 69 +toc_title: "Именованные соединения" +--- + +# Хранение реквизитов для подключения к внешним источникам в конфигурационных файлах {#named-collections} + +Реквизиты для подключения к внешним источникам (словарям, таблицам, табличным функциям) можно сохранить +в конфигурационных файлах и таким образом упростить создание объектов и скрыть реквизиты (пароли) +от пользователей, имеющих только SQL доступ. + +Параметры можно задать в XML `CSV` и переопределить в SQL `, format = 'TSV'`. +При использовании именованных соединений, параметры в SQL задаются в формате `ключ` = `значение`: `compression_method = 'gzip'`. + +Именованные соединения хранятся в файле `config.xml` сервера ClickHouse в секции `` и применяются при старте ClickHouse. + +Пример конфигурации: +```xml +$ cat /etc/clickhouse-server/config.d/named_collections.xml + + + ... + + +``` + +## Именованные соединения для доступа к S3 + +Описание параметров смотри [Табличная Функция S3](../sql-reference/table-functions/s3.md). + +Пример конфигурации: +```xml + + + + AKIAIOSFODNN7EXAMPLE + wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + CSV + + + +``` + +### Пример использования именованных соединений с функцией s3 + +```sql +INSERT INTO FUNCTION s3(s3_mydata, url = 'https://s3.us-east-1.amazonaws.com/yourbucket/mydata/test_file.tsv.gz', + format = 'TSV', structure = 'number UInt64', compression_method = 'gzip') +SELECT * FROM numbers(10000); + +SELECT count() +FROM s3(s3_mydata, url = 'https://s3.us-east-1.amazonaws.com/yourbucket/mydata/test_file.tsv.gz') + +┌─count()─┐ +│ 10000 │ +└─────────┘ +1 rows in set. Elapsed: 0.279 sec. Processed 10.00 thousand rows, 90.00 KB (35.78 thousand rows/s., 322.02 KB/s.) +``` + +### Пример использования именованных соединений с таблицей S3 + +```sql +CREATE TABLE s3_engine_table (number Int64) +ENGINE=S3(s3_mydata, url='https://s3.us-east-1.amazonaws.com/yourbucket/mydata/test_file.tsv.gz', format = 'TSV') +SETTINGS input_format_with_names_use_header = 0; + +SELECT * FROM s3_engine_table LIMIT 3; +┌─number─┐ +│ 0 │ +│ 1 │ +│ 2 │ +└────────┘ +``` + +## Пример использования именованных соединений с базой данных MySQL + +Описание параметров смотри [mysql](../sql-reference/table-functions/mysql.md). + +Пример конфигурации: +```xml + + + + myuser + mypass + 127.0.0.1 + 3306 + test + 8 + 1 + 1 + + + +``` + +### Пример использования именованных соединений с табличной функцией mysql + +```sql +SELECT count() FROM mysql(mymysql, table = 'test'); + +┌─count()─┐ +│ 3 │ +└─────────┘ +``` + +### Пример использования именованных соединений таблицей с движком mysql + +```sql +CREATE TABLE mytable(A Int64) ENGINE = MySQL(mymysql, table = 'test', connection_pool_size=3, replace_query=0); +SELECT count() FROM mytable; + +┌─count()─┐ +│ 3 │ +└─────────┘ +``` + +### Пример использования именованных соединений базой данных с движком MySQL + +```sql +CREATE DATABASE mydatabase ENGINE = MySQL(mymysql); + +SHOW TABLES FROM mydatabase; + +┌─name───┐ +│ source │ +│ test │ +└────────┘ +``` + +### Пример использования именованных соединений с внешним словарем с источником mysql + +```sql +CREATE DICTIONARY dict (A Int64, B String) +PRIMARY KEY A +SOURCE(MYSQL(NAME mymysql TABLE 'source')) +LIFETIME(MIN 1 MAX 2) +LAYOUT(HASHED()); + +SELECT dictGet('dict', 'B', 2); + +┌─dictGet('dict', 'B', 2)─┐ +│ two │ +└─────────────────────────┘ +``` + +## Пример использования именованных соединений с базой данных PostgreSQL + +Описание параметров смотри [postgresql](../sql-reference/table-functions/postgresql.md). + +Пример конфигурации: +```xml + + + + pguser + jw8s0F4 + 127.0.0.1 + 5432 + test + test_schema + 8 + + + +``` + +### Пример использования именованных соединений с табличной функцией postgresql + +```sql +SELECT * FROM postgresql(mypg, table = 'test'); + +┌─a─┬─b───┐ +│ 2 │ two │ +│ 1 │ one │ +└───┴─────┘ + + +SELECT * FROM postgresql(mypg, table = 'test', schema = 'public'); + +┌─a─┐ +│ 1 │ +│ 2 │ +│ 3 │ +└───┘ +``` + +### Пример использования именованных соединений таблицей с движком PostgreSQL + +```sql +CREATE TABLE mypgtable (a Int64) ENGINE = PostgreSQL(mypg, table = 'test', schema = 'public'); + +SELECT * FROM mypgtable; + +┌─a─┐ +│ 1 │ +│ 2 │ +│ 3 │ +└───┘ +``` + +### Пример использования именованных соединений базой данных с движком PostgreSQL + +```sql +CREATE DATABASE mydatabase ENGINE = PostgreSQL(mypg); + +SHOW TABLES FROM mydatabase + +┌─name─┐ +│ test │ +└──────┘ +``` + +### Пример использования именованных соединений с внешним словарем с источником POSTGRESQL + +```sql +CREATE DICTIONARY dict (a Int64, b String) +PRIMARY KEY a +SOURCE(POSTGRESQL(NAME mypg TABLE test)) +LIFETIME(MIN 1 MAX 2) +LAYOUT(HASHED()); + +SELECT dictGet('dict', 'b', 2); + +┌─dictGet('dict', 'b', 2)─┐ +│ two │ +└─────────────────────────┘ +``` diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index ba5fc63331a..5e4b7c6bcb7 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1736,6 +1736,48 @@ ClickHouse генерирует исключение: Т.е. если `INSERT` в основную таблицу д.б. пропущен (сдедуплицирован), то автоматически не будет вставки и в материализованные представления. Это имплементировано для того, чтобы работали материализованные представления, которые сильно группируют данные основных `INSERT`, до такой степени что блоки вставляемые в материализованные представления получаются одинаковыми для разных `INSERT` в основную таблицу. Одновременно это «ломает» идемпотентность вставки в материализованные представления. Т.е. если `INSERT` был успешен в основную таблицу и неуспешен в таблицу материализованного представления (напр. из-за сетевого сбоя при коммуникации с Zookeeper), клиент получит ошибку и попытается повторить `INSERT`. Но вставки в материализованные представления произведено не будет, потому что дедупликация сработает на основной таблице. Настройка `deduplicate_blocks_in_dependent_materialized_views` позволяет это изменить. Т.е. при повторном `INSERT` будет произведена дедупликация на таблице материализованного представления, и повторный инсерт вставит данные в таблицу материализованного представления, которые не удалось вставить из-за сбоя первого `INSERT`. +## insert_deduplication_token {#insert_deduplication_token} + +Этот параметр позволяет пользователю указать собственную семантику дедупликации в MergeTree/ReplicatedMergeTree. +Например, предоставляя уникальное значение параметра в каждом операторе INSERT, +пользователь может избежать дедупликации одних и тех же вставленных данных. + +Возможные значения: + +- Любая строка + +Значение по умолчанию: пустая строка (выключено). + +`insert_deduplication_token` используется для дедупликации _только_ когда значение не пустое + +Example: + +```sql +CREATE TABLE test_table +( A Int64 ) +ENGINE = MergeTree +ORDER BY A +SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test' (1); + +-- следующая вставка не будет дедуплицирована, потому что insert_deduplication_token отличается +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test1' (1); + +-- следующая вставка будет дедуплицирована, потому что insert_deduplication_token +-- тот же самый, что и один из предыдущих +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test' (2); + +SELECT * FROM test_table + +┌─A─┐ +│ 1 │ +└───┘ +┌─A─┐ +│ 1 │ +└───┘ +``` + ## count_distinct_implementation {#settings-count_distinct_implementation} Задаёт, какая из функций `uniq*` используется при выполнении конструкции [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference/count.md#agg_function-count). diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index abc16751a79..e37bdf1af77 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -5,7 +5,7 @@ toc_title: Array(T) # Array(T) {#data-type-array} -Массив из элементов типа `T`. `T` может любым, в том числе массивом. Таким образом поддерживаются многомерные массивы. +Массив из элементов типа `T`. `T` может любым, в том числе массивом. Таким образом поддерживаются многомерные массивы. Первый элемент массива имеет индекс 1. ## Создание массива {#creating-an-array} diff --git a/docs/ru/sql-reference/data-types/date.md b/docs/ru/sql-reference/data-types/date.md index 17b4ec99d9a..7157f0dc4c7 100644 --- a/docs/ru/sql-reference/data-types/date.md +++ b/docs/ru/sql-reference/data-types/date.md @@ -7,6 +7,8 @@ toc_title: Date Дата. Хранится в двух байтах в виде (беззнакового) числа дней, прошедших от 1970-01-01. Позволяет хранить значения от чуть больше, чем начала unix-эпохи до верхнего порога, определяющегося константой на этапе компиляции (сейчас - до 2106 года, последний полностью поддерживаемый год - 2105). +Диапазон значений: \[1970-01-01, 2149-06-06\]. + Дата хранится без учёта часового пояса. **Пример** diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index c9804f57c33..804d590e65d 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -13,7 +13,7 @@ toc_title: DateTime DateTime([timezone]) ``` -Диапазон значений: \[1970-01-01 00:00:00, 2105-12-31 23:59:59\]. +Диапазон значений: \[1970-01-01 00:00:00, 2106-02-07 06:28:15\]. Точность: 1 секунда. diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md index 869543dbbaf..01ee26d3496 100644 --- a/docs/ru/sql-reference/data-types/datetime64.md +++ b/docs/ru/sql-reference/data-types/datetime64.md @@ -18,7 +18,7 @@ DateTime64(precision, [timezone]) Данные хранятся в виде количества ‘тиков’, прошедших с момента начала эпохи (1970-01-01 00:00:00 UTC), в Int64. Размер тика определяется параметром precision. Дополнительно, тип `DateTime64` позволяет хранить часовой пояс, единый для всей колонки, который влияет на то, как будут отображаться значения типа `DateTime64` в текстовом виде и как будут парситься значения заданные в виде строк (‘2020-01-01 05:00:01.000’). Часовой пояс не хранится в строках таблицы (выборки), а хранится в метаданных колонки. Подробнее см. [DateTime](datetime.md). -Поддерживаются значения от 1 января 1925 г. и до 11 ноября 2283 г. +Диапазон значений: \[1925-01-01 00:00:00, 2283-11-11 23:59:59.99999999\] (Примечание: Точность максимального значения составляет 8). ## Примеры {#examples} diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index a63c76d8833..30b9f18ec58 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -72,37 +72,89 @@ ClickHouse может вызывать внешнюю программу или Конфигурация функции содержит следующие настройки: - `name` - имя функции. -- `command` - исполняемая команда или скрипт. -- `argument` - описание аргумента, содержащее его тип во вложенной настройке `type`. Каждый аргумент описывается отдельно. +- `command` - имя скрипта для выполнения или команды, если `execute_direct` равно false. +- `argument` - описание аргумента, содержащее его тип во вложенной настройке `type`, и опционально его имя во вложенной настройке `name`. Каждый аргумент описывается отдельно. Указание имени для аргумента необходимо, если имена аргументов являются частью сериализации для пользовательского формата функции, например [Native](../../interfaces/formats.md#native) или [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Значение имени аргумента по умолчанию `c` + номер аргумента. - `format` - [формат](../../interfaces/formats.md) передачи аргументов. - `return_type` - тип возвращаемого значения. +- `return_name` - имя возвращаемого значения. Указание имени возвращаемого значения необходимо, если имя возвращаемого значения является частью сериализации для пользовательского формата функции, например [Native](../../interfaces/formats.md#native) или [JSONEachRow](../../interfaces/formats.md#jsoneachrow). Необязательный. Значение по умолчанию — `result`. - `type` - вариант запуска команды. Если задан вариант `executable`, то запускается одна команда. При указании `executable_pool` создается пул команд. - `max_command_execution_time` - максимальное время в секундах, которое отводится на обработку блока данных. Эта настройка применима только для команд с вариантом запуска `executable_pool`. Необязательная настройка. Значение по умолчанию `10`. - `command_termination_timeout` - максимальное время завершения команды в секундах после закрытия конвейера. Если команда не завершается, то процессу отправляется сигнал `SIGTERM`. Эта настройка применима только для команд с вариантом запуска `executable_pool`. Необязательная настройка. Значение по умолчанию `10`. +- `command_read_timeout` - время ожидания чтения данных из команды stdout в миллисекундах. Значение по умолчанию 10000. Необязательная настройка. +- `command_write_timeout` - время ожидания записи данных в команду stdin в миллисекундах. Значение по умолчанию 10000. Необязательная настройка. - `pool_size` - размер пула команд. Необязательная настройка. Значение по умолчанию `16`. -- `lifetime` - интервал перезагрузки функций в секундах. Если задан `0`, то функция не перезагружается. - `send_chunk_header` - управляет отправкой количества строк перед отправкой блока данных для обработки. Необязательная настройка. Значение по умолчанию `false`. +- `execute_direct` - Если `execute_direct` = `1`, то будет произведен поиск `command` в папке user_scripts. Дополнительные аргументы скрипта можно указать с помощью разделителя пробелов. Пример: `script_name arg1 arg2`. Если `execute_direct` = `0`, `command` передается как аргумент для `bin/sh -c`. Значение по умолчанию `1`. Необязательный параметр. +- `lifetime` - интервал перезагрузки функций в секундах. Если задан `0`, то функция не перезагружается. Команда должна читать аргументы из `STDIN` и выводить результат в `STDOUT`. Обработка должна выполняться в цикле. То есть после обработки группы аргументов команда должна ожидать следующую группу. **Пример** -XML конфигурация, описывающая функцию `test_function`: -``` +Создание `test_function` с использованием конфигурации XML. +Файл test_function.xml. +```xml executable - test_function + test_function_python + String + + UInt64 + value + + TabSeparated + test_function.py + + +``` + +Файл скрипта внутри папки `user_scripts` `test_function.py`. + +```python +#!/usr/bin/python3 + +import sys + +if __name__ == '__main__': + for line in sys.stdin: + print("Value " + line, end='') + sys.stdout.flush() +``` + +Запрос: + +``` sql +SELECT test_function_python(toUInt64(2)); +``` + +Результат: + +``` text +┌─test_function_python(2)─┐ +│ Value 2 │ +└─────────────────────────┘ +``` + +Создание `test_function_sum`, указав для `execute_direct` значение `0`, используя конфигурацию XML. +File test_function.xml. +```xml + + + executable + test_function_sum UInt64 UInt64 + lhs UInt64 + rhs TabSeparated cd /; clickhouse-local --input-format TabSeparated --output-format TabSeparated --structure 'x UInt64, y UInt64' --query "SELECT x + y FROM table" - 0 + 0 ``` @@ -110,15 +162,68 @@ XML конфигурация, описывающая функцию `test_functi Запрос: ``` sql -SELECT test_function(toUInt64(2), toUInt64(2)); +SELECT test_function_sum(2, 2); ``` Результат: ``` text -┌─test_function(toUInt64(2), toUInt64(2))─┐ -│ 4 │ -└─────────────────────────────────────────┘ +┌─test_function_sum(2, 2)─┐ +│ 4 │ +└─────────────────────────┘ +``` + +Создание `test_function_sum_json` с именноваными аргументами и форматом [JSONEachRow](../../interfaces/formats.md#jsoneachrow) с использованием конфигурации XML. +Файл test_function.xml. +```xml + + executable + test_function_sum_json + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + test_function_sum_json.py + +``` + +Файл скрипта внутри папки `user_scripts` `test_function_sum_json.py`. + +```python +#!/usr/bin/python3 + +import sys +import json + +if __name__ == '__main__': + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value['argument_1']) + second_arg = int(value['argument_2']) + result = {'result_name': first_arg + second_arg} + print(json.dumps(result), end='\n') + sys.stdout.flush() +``` + +Запрос: + +``` sql +SELECT test_function_sum_json(2, 2); +``` + +Результат: + +``` text +┌─test_function_sum_json(2, 2)─┐ +│ 4 │ +└──────────────────────────────┘ ``` ## Обработка ошибок {#obrabotka-oshibok} diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 4de2d067cce..fea4c00ac05 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ## MATERIALIZE COLUMN {#materialize-column} -Материализует столбец таблицы в кусках, в которых отсутствуют значения. Используется, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`), потому как вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`. +Материализует или обновляет столбец таблицы с выражением для значения по умолчанию (`DEFAULT` или `MATERIALIZED`). +Используется, если необходимо добавить или обновить столбец со сложным выражением, потому как вычисление такого выражения прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Синтаксис: ```sql -ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; +ALTER TABLE table MATERIALIZE COLUMN col; ``` **Пример** @@ -211,21 +212,39 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; DROP TABLE IF EXISTS tmp; SET mutations_sync = 2; CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); -INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10; +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); ALTER TABLE tmp MATERIALIZE COLUMN s; +SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x); + +┌─groupArray(x)─┬─groupArray(s)─────────┐ +│ [0,1,2,3,4] │ ['0','1','2','3','4'] │ +└───────────────┴───────────────────────┘ + +ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x)); + +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5; + SELECT groupArray(x), groupArray(s) FROM tmp; + +┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │ +└───────────────────────┴────────────────────────────────────────────────┘ + +ALTER TABLE tmp MATERIALIZE COLUMN s; + +SELECT groupArray(x), groupArray(s) FROM tmp; + +┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │ +└───────────────────────┴───────────────────────────────────────────────────────┘ ``` -**Результат:** +**Смотрите также** -```sql -┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐ -│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │ -└───────────────────────┴───────────────────────────────────────────┘ -``` +- [MATERIALIZED](../../statements/create/table.md#materialized). ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 8567a0ff2db..b9c2a4f0f0b 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -14,8 +14,8 @@ toc_title: "Таблица" ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( - name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1], - name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr2] [compression_codec] [TTL expr2], + name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr1] [compression_codec] [TTL expr1], + name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [compression_codec] [TTL expr2], ... ) ENGINE = engine ``` @@ -108,6 +108,13 @@ SELECT x, toTypeName(x) FROM t1; При INSERT без указания списка столбцов, такие столбцы не рассматриваются. Также этот столбец не подставляется при использовании звёздочки в запросе SELECT. Это необходимо, чтобы сохранить инвариант, что дамп, полученный путём `SELECT *`, можно вставить обратно в таблицу INSERT-ом без указания списка столбцов. +### EPHEMERAL {#ephemeral} + +`EPHEMERAL expr` + +Эфемерное выражение. Такой столбец не хранится в таблице и не может быть получен в запросе SELECT, но на него можно ссылаться в выражениях по умолчанию запроса CREATE. +INSERT без списка столбцов игнорирует этот столбец, таким образом сохраняется инвариант - т.е. дамп, полученный путём `SELECT *`, можно вставить обратно в таблицу INSERT-ом без указания списка столбцов. + ### ALIAS {#alias} `ALIAS expr` diff --git a/docs/ru/sql-reference/statements/select/limit-by.md b/docs/ru/sql-reference/statements/select/limit-by.md index 861d88dcafb..5da001addf4 100644 --- a/docs/ru/sql-reference/statements/select/limit-by.md +++ b/docs/ru/sql-reference/statements/select/limit-by.md @@ -11,7 +11,7 @@ ClickHouse поддерживает следующий синтаксис: - `LIMIT [offset_value, ]n BY expressions` - `LIMIT n OFFSET offset_value BY expressions` -Во время обработки запроса, ClickHouse выбирает данные, упорядоченные по ключу сортировки. Ключ сортировки задаётся явно в секции [ORDER BY](order-by.md#select-order-by) или неявно в свойствах движка таблицы. Затем ClickHouse применяет `LIMIT n BY expressions` и возвращает первые `n` для каждой отличной комбинации `expressions`. Если указан `OFFSET`, то для каждого блока данных, который принадлежит отдельной комбинации `expressions`, ClickHouse отступает `offset_value` строк от начала блока и возвращает не более `n`. Если `offset_value` больше, чем количество строк в блоке данных, ClickHouse не возвращает ни одной строки. +Во время обработки запроса, ClickHouse выбирает данные, упорядоченные по ключу сортировки. Ключ сортировки задаётся явно в секции [ORDER BY](order-by.md#select-order-by) или неявно в свойствах движка таблицы (порядок строк гарантирован только при использовании [ORDER BY](order-by.md#select-order-by), в ином случае блоки строк не будут упорядочены из-за многопоточной обработки). Затем ClickHouse применяет `LIMIT n BY expressions` и возвращает первые `n` для каждой отличной комбинации `expressions`. Если указан `OFFSET`, то для каждого блока данных, который принадлежит отдельной комбинации `expressions`, ClickHouse отступает `offset_value` строк от начала блока и возвращает не более `n`. Если `offset_value` больше, чем количество строк в блоке данных, ClickHouse не возвращает ни одной строки. `LIMIT BY` не связана с секцией `LIMIT`. Их можно использовать в одном запросе. diff --git a/docs/tools/amp.py b/docs/tools/amp.py index 22417407946..584a40c4bba 100644 --- a/docs/tools/amp.py +++ b/docs/tools/amp.py @@ -15,24 +15,24 @@ import website def prepare_amp_html(lang, args, root, site_temp, main_site_dir): src_path = root - src_index = os.path.join(src_path, 'index.html') + src_index = os.path.join(src_path, "index.html") rel_path = os.path.relpath(src_path, site_temp) - dst_path = os.path.join(main_site_dir, rel_path, 'amp') - dst_index = os.path.join(dst_path, 'index.html') + dst_path = os.path.join(main_site_dir, rel_path, "amp") + dst_index = os.path.join(dst_path, "index.html") - logging.debug(f'Generating AMP version for {rel_path} ({lang})') + logging.debug(f"Generating AMP version for {rel_path} ({lang})") os.makedirs(dst_path) - with open(src_index, 'r') as f: + with open(src_index, "r") as f: content = f.read() - css_in = ' '.join(website.get_css_in(args)) + css_in = " ".join(website.get_css_in(args)) command = f"purifycss --min {css_in} '{src_index}'" logging.debug(command) - inline_css = subprocess.check_output(command, shell=True).decode('utf-8') - inline_css = inline_css.replace('!important', '').replace('/*!', '/*') + inline_css = subprocess.check_output(command, shell=True).decode("utf-8") + inline_css = inline_css.replace("!important", "").replace("/*!", "/*") inline_css = cssmin.cssmin(inline_css) - content = content.replace('CUSTOM_CSS_PLACEHOLDER', inline_css) + content = content.replace("CUSTOM_CSS_PLACEHOLDER", inline_css) - with open(dst_index, 'w') as f: + with open(dst_index, "w") as f: f.write(content) return dst_index @@ -40,15 +40,12 @@ def prepare_amp_html(lang, args, root, site_temp, main_site_dir): def build_amp(lang, args, cfg): # AMP docs: https://amp.dev/documentation/ - logging.info(f'Building AMP version for {lang}') + logging.info(f"Building AMP version for {lang}") with util.temp_dir() as site_temp: - extra = cfg.data['extra'] - main_site_dir = cfg.data['site_dir'] - extra['is_amp'] = True - cfg.load_dict({ - 'site_dir': site_temp, - 'extra': extra - }) + extra = cfg.data["extra"] + main_site_dir = cfg.data["site_dir"] + extra["is_amp"] = True + cfg.load_dict({"site_dir": site_temp, "extra": extra}) try: mkdocs.commands.build.build(cfg) @@ -60,50 +57,49 @@ def build_amp(lang, args, cfg): paths = [] for root, _, filenames in os.walk(site_temp): - if 'index.html' in filenames: - paths.append(prepare_amp_html(lang, args, root, site_temp, main_site_dir)) - logging.info(f'Finished building AMP version for {lang}') + if "index.html" in filenames: + paths.append( + prepare_amp_html(lang, args, root, site_temp, main_site_dir) + ) + logging.info(f"Finished building AMP version for {lang}") def html_to_amp(content): - soup = bs4.BeautifulSoup( - content, - features='html.parser' - ) + soup = bs4.BeautifulSoup(content, features="html.parser") for tag in soup.find_all(): - if tag.attrs.get('id') == 'tostring': - tag.attrs['id'] = '_tostring' - if tag.name == 'img': - tag.name = 'amp-img' - tag.attrs['layout'] = 'responsive' - src = tag.attrs['src'] - if not (src.startswith('/') or src.startswith('http')): - tag.attrs['src'] = f'../{src}' - if not tag.attrs.get('width'): - tag.attrs['width'] = '640' - if not tag.attrs.get('height'): - tag.attrs['height'] = '320' - if tag.name == 'iframe': - tag.name = 'amp-iframe' - tag.attrs['layout'] = 'responsive' - del tag.attrs['alt'] - del tag.attrs['allowfullscreen'] - if not tag.attrs.get('width'): - tag.attrs['width'] = '640' - if not tag.attrs.get('height'): - tag.attrs['height'] = '320' - elif tag.name == 'a': - href = tag.attrs.get('href') + if tag.attrs.get("id") == "tostring": + tag.attrs["id"] = "_tostring" + if tag.name == "img": + tag.name = "amp-img" + tag.attrs["layout"] = "responsive" + src = tag.attrs["src"] + if not (src.startswith("/") or src.startswith("http")): + tag.attrs["src"] = f"../{src}" + if not tag.attrs.get("width"): + tag.attrs["width"] = "640" + if not tag.attrs.get("height"): + tag.attrs["height"] = "320" + if tag.name == "iframe": + tag.name = "amp-iframe" + tag.attrs["layout"] = "responsive" + del tag.attrs["alt"] + del tag.attrs["allowfullscreen"] + if not tag.attrs.get("width"): + tag.attrs["width"] = "640" + if not tag.attrs.get("height"): + tag.attrs["height"] = "320" + elif tag.name == "a": + href = tag.attrs.get("href") if href: - if not (href.startswith('/') or href.startswith('http')): - if '#' in href: - href, anchor = href.split('#') + if not (href.startswith("/") or href.startswith("http")): + if "#" in href: + href, anchor = href.split("#") else: anchor = None - href = f'../{href}amp/' + href = f"../{href}amp/" if anchor: - href = f'{href}#{anchor}' - tag.attrs['href'] = href + href = f"{href}#{anchor}" + tag.attrs["href"] = href content = str(soup) return website.minify_html(content) diff --git a/docs/tools/blog.py b/docs/tools/blog.py index e4fb6f77865..d1fc540d8bf 100644 --- a/docs/tools/blog.py +++ b/docs/tools/blog.py @@ -17,56 +17,52 @@ import util def build_for_lang(lang, args): - logging.info(f'Building {lang} blog') + logging.info(f"Building {lang} blog") try: theme_cfg = { - 'name': None, - 'custom_dir': os.path.join(os.path.dirname(__file__), '..', args.theme_dir), - 'language': lang, - 'direction': 'ltr', - 'static_templates': ['404.html'], - 'extra': { - 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching - } + "name": None, + "custom_dir": os.path.join(os.path.dirname(__file__), "..", args.theme_dir), + "language": lang, + "direction": "ltr", + "static_templates": ["404.html"], + "extra": { + "now": int( + time.mktime(datetime.datetime.now().timetuple()) + ) # TODO better way to avoid caching + }, } # the following list of languages is sorted according to # https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers - languages = { - 'en': 'English', - 'ru': 'Русский' - } + languages = {"en": "English"} - site_names = { - 'en': 'ClickHouse Blog', - 'ru': 'Блог ClickHouse' - } + site_names = {"en": "ClickHouse Blog"} assert len(site_names) == len(languages) site_dir = os.path.join(args.blog_output_dir, lang) - plugins = ['macros'] + plugins = ["macros"] if args.htmlproofer: - plugins.append('htmlproofer') + plugins.append("htmlproofer") - website_url = 'https://clickhouse.com' - site_name = site_names.get(lang, site_names['en']) + website_url = "https://clickhouse.com" + site_name = site_names.get(lang, site_names["en"]) blog_nav, post_meta = nav.build_blog_nav(lang, args) raw_config = dict( site_name=site_name, - site_url=f'{website_url}/blog/{lang}/', + site_url=f"{website_url}/blog/{lang}/", docs_dir=os.path.join(args.blog_dir, lang), site_dir=site_dir, strict=True, theme=theme_cfg, nav=blog_nav, - copyright='©2016–2022 ClickHouse, Inc.', + copyright="©2016–2022 ClickHouse, Inc.", use_directory_urls=True, - repo_name='ClickHouse/ClickHouse', - repo_url='https://github.com/ClickHouse/ClickHouse/', - edit_uri=f'edit/master/website/blog/{lang}', + repo_name="ClickHouse/ClickHouse", + repo_url="https://github.com/ClickHouse/ClickHouse/", + edit_uri=f"edit/master/website/blog/{lang}", markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS, plugins=plugins, extra=dict( @@ -77,12 +73,12 @@ def build_for_lang(lang, args): website_url=website_url, events=args.events, languages=languages, - includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'), + includes_dir=os.path.join(os.path.dirname(__file__), "..", "_includes"), is_amp=False, is_blog=True, post_meta=post_meta, - today=datetime.date.today().isoformat() - ) + today=datetime.date.today().isoformat(), + ), ) cfg = config.load_config(**raw_config) @@ -91,21 +87,28 @@ def build_for_lang(lang, args): redirects.build_blog_redirects(args) env = util.init_jinja2_env(args) - with open(os.path.join(args.website_dir, 'templates', 'blog', 'rss.xml'), 'rb') as f: - rss_template_string = f.read().decode('utf-8').strip() + with open( + os.path.join(args.website_dir, "templates", "blog", "rss.xml"), "rb" + ) as f: + rss_template_string = f.read().decode("utf-8").strip() rss_template = env.from_string(rss_template_string) - with open(os.path.join(args.blog_output_dir, lang, 'rss.xml'), 'w') as f: - f.write(rss_template.render({'config': raw_config})) + with open(os.path.join(args.blog_output_dir, lang, "rss.xml"), "w") as f: + f.write(rss_template.render({"config": raw_config})) - logging.info(f'Finished building {lang} blog') + logging.info(f"Finished building {lang} blog") except exceptions.ConfigurationError as e: - raise SystemExit('\n' + str(e)) + raise SystemExit("\n" + str(e)) def build_blog(args): tasks = [] - for lang in args.blog_lang.split(','): + for lang in args.blog_lang.split(","): if lang: - tasks.append((lang, args,)) + tasks.append( + ( + lang, + args, + ) + ) util.run_function_in_parallel(build_for_lang, tasks, threads=False) diff --git a/docs/tools/build.py b/docs/tools/build.py index 75278075996..612be0229d3 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -30,76 +30,76 @@ import website from cmake_in_clickhouse_generator import generate_cmake_flags_files + class ClickHouseMarkdown(markdown.extensions.Extension): class ClickHousePreprocessor(markdown.util.Processor): def run(self, lines): for line in lines: - if '' not in line: + if "" not in line: yield line def extendMarkdown(self, md): - md.preprocessors.register(self.ClickHousePreprocessor(), 'clickhouse_preprocessor', 31) + md.preprocessors.register( + self.ClickHousePreprocessor(), "clickhouse_preprocessor", 31 + ) markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown def build_for_lang(lang, args): - logging.info(f'Building {lang} docs') - os.environ['SINGLE_PAGE'] = '0' + logging.info(f"Building {lang} docs") + os.environ["SINGLE_PAGE"] = "0" try: theme_cfg = { - 'name': None, - 'custom_dir': os.path.join(os.path.dirname(__file__), '..', args.theme_dir), - 'language': lang, - 'direction': 'rtl' if lang == 'fa' else 'ltr', - 'static_templates': ['404.html'], - 'extra': { - 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching - } + "name": None, + "custom_dir": os.path.join(os.path.dirname(__file__), "..", args.theme_dir), + "language": lang, + "direction": "rtl" if lang == "fa" else "ltr", + "static_templates": ["404.html"], + "extra": { + "now": int( + time.mktime(datetime.datetime.now().timetuple()) + ) # TODO better way to avoid caching + }, } # the following list of languages is sorted according to # https://en.wikipedia.org/wiki/List_of_languages_by_total_number_of_speakers - languages = { - 'en': 'English', - 'zh': '中文', - 'ru': 'Русский', - 'ja': '日本語' - } + languages = {"en": "English", "zh": "中文", "ru": "Русский", "ja": "日本語"} site_names = { - 'en': 'ClickHouse %s Documentation', - 'zh': 'ClickHouse文档 %s', - 'ru': 'Документация ClickHouse %s', - 'ja': 'ClickHouseドキュメント %s' + "en": "ClickHouse %s Documentation", + "zh": "ClickHouse文档 %s", + "ru": "Документация ClickHouse %s", + "ja": "ClickHouseドキュメント %s", } assert len(site_names) == len(languages) site_dir = os.path.join(args.docs_output_dir, lang) - plugins = ['macros'] + plugins = ["macros"] if args.htmlproofer: - plugins.append('htmlproofer') + plugins.append("htmlproofer") - website_url = 'https://clickhouse.com' - site_name = site_names.get(lang, site_names['en']) % '' - site_name = site_name.replace(' ', ' ') + website_url = "https://clickhouse.com" + site_name = site_names.get(lang, site_names["en"]) % "" + site_name = site_name.replace(" ", " ") raw_config = dict( site_name=site_name, - site_url=f'{website_url}/docs/{lang}/', + site_url=f"{website_url}/docs/{lang}/", docs_dir=os.path.join(args.docs_dir, lang), site_dir=site_dir, strict=True, theme=theme_cfg, - copyright='©2016–2022 ClickHouse, Inc.', + copyright="©2016–2022 ClickHouse, Inc.", use_directory_urls=True, - repo_name='ClickHouse/ClickHouse', - repo_url='https://github.com/ClickHouse/ClickHouse/', - edit_uri=f'edit/master/docs/{lang}', + repo_name="ClickHouse/ClickHouse", + repo_url="https://github.com/ClickHouse/ClickHouse/", + edit_uri=f"edit/master/docs/{lang}", markdown_extensions=mdx_clickhouse.MARKDOWN_EXTENSIONS, plugins=plugins, extra=dict( @@ -111,16 +111,16 @@ def build_for_lang(lang, args): website_url=website_url, events=args.events, languages=languages, - includes_dir=os.path.join(os.path.dirname(__file__), '..', '_includes'), + includes_dir=os.path.join(os.path.dirname(__file__), "..", "_includes"), is_amp=False, - is_blog=False - ) + is_blog=False, + ), ) # Clean to be safe if last build finished abnormally single_page.remove_temporary_files(lang, args) - raw_config['nav'] = nav.build_docs_nav(lang, args) + raw_config["nav"] = nav.build_docs_nav(lang, args) cfg = config.load_config(**raw_config) @@ -131,21 +131,28 @@ def build_for_lang(lang, args): amp.build_amp(lang, args, cfg) if not args.skip_single_page: - single_page.build_single_page_version(lang, args, raw_config.get('nav'), cfg) + single_page.build_single_page_version( + lang, args, raw_config.get("nav"), cfg + ) mdx_clickhouse.PatchedMacrosPlugin.disabled = False - logging.info(f'Finished building {lang} docs') + logging.info(f"Finished building {lang} docs") except exceptions.ConfigurationError as e: - raise SystemExit('\n' + str(e)) + raise SystemExit("\n" + str(e)) def build_docs(args): tasks = [] - for lang in args.lang.split(','): + for lang in args.lang.split(","): if lang: - tasks.append((lang, args,)) + tasks.append( + ( + lang, + args, + ) + ) util.run_function_in_parallel(build_for_lang, tasks, threads=False) redirects.build_docs_redirects(args) @@ -171,56 +178,64 @@ def build(args): redirects.build_static_redirects(args) -if __name__ == '__main__': - os.chdir(os.path.join(os.path.dirname(__file__), '..')) +if __name__ == "__main__": + os.chdir(os.path.join(os.path.dirname(__file__), "..")) # A root path to ClickHouse source code. - src_dir = '..' + src_dir = ".." - website_dir = os.path.join(src_dir, 'website') + website_dir = os.path.join(src_dir, "website") arg_parser = argparse.ArgumentParser() - arg_parser.add_argument('--lang', default='en,ru,zh,ja') - arg_parser.add_argument('--blog-lang', default='en,ru') - arg_parser.add_argument('--docs-dir', default='.') - arg_parser.add_argument('--theme-dir', default=website_dir) - arg_parser.add_argument('--website-dir', default=website_dir) - arg_parser.add_argument('--src-dir', default=src_dir) - arg_parser.add_argument('--blog-dir', default=os.path.join(website_dir, 'blog')) - arg_parser.add_argument('--output-dir', default='build') - arg_parser.add_argument('--nav-limit', type=int, default='0') - arg_parser.add_argument('--skip-multi-page', action='store_true') - arg_parser.add_argument('--skip-single-page', action='store_true') - arg_parser.add_argument('--skip-amp', action='store_true') - arg_parser.add_argument('--skip-website', action='store_true') - arg_parser.add_argument('--skip-blog', action='store_true') - arg_parser.add_argument('--skip-git-log', action='store_true') - arg_parser.add_argument('--skip-docs', action='store_true') - arg_parser.add_argument('--test-only', action='store_true') - arg_parser.add_argument('--minify', action='store_true') - arg_parser.add_argument('--htmlproofer', action='store_true') - arg_parser.add_argument('--no-docs-macros', action='store_true') - arg_parser.add_argument('--save-raw-single-page', type=str) - arg_parser.add_argument('--livereload', type=int, default='0') - arg_parser.add_argument('--verbose', action='store_true') + arg_parser.add_argument("--lang", default="en,ru,zh,ja") + arg_parser.add_argument("--blog-lang", default="en") + arg_parser.add_argument("--docs-dir", default=".") + arg_parser.add_argument("--theme-dir", default=website_dir) + arg_parser.add_argument("--website-dir", default=website_dir) + arg_parser.add_argument("--src-dir", default=src_dir) + arg_parser.add_argument("--blog-dir", default=os.path.join(website_dir, "blog")) + arg_parser.add_argument("--output-dir", default="build") + arg_parser.add_argument("--nav-limit", type=int, default="0") + arg_parser.add_argument("--skip-multi-page", action="store_true") + arg_parser.add_argument("--skip-single-page", action="store_true") + arg_parser.add_argument("--skip-amp", action="store_true") + arg_parser.add_argument("--skip-website", action="store_true") + arg_parser.add_argument("--skip-blog", action="store_true") + arg_parser.add_argument("--skip-git-log", action="store_true") + arg_parser.add_argument("--skip-docs", action="store_true") + arg_parser.add_argument("--test-only", action="store_true") + arg_parser.add_argument("--minify", action="store_true") + arg_parser.add_argument("--htmlproofer", action="store_true") + arg_parser.add_argument("--no-docs-macros", action="store_true") + arg_parser.add_argument("--save-raw-single-page", type=str) + arg_parser.add_argument("--livereload", type=int, default="0") + arg_parser.add_argument("--verbose", action="store_true") args = arg_parser.parse_args() args.minify = False # TODO remove logging.basicConfig( - level=logging.DEBUG if args.verbose else logging.INFO, - stream=sys.stderr + level=logging.DEBUG if args.verbose else logging.INFO, stream=sys.stderr ) - logging.getLogger('MARKDOWN').setLevel(logging.INFO) + logging.getLogger("MARKDOWN").setLevel(logging.INFO) - args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), 'docs') - args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), 'blog') + args.docs_output_dir = os.path.join(os.path.abspath(args.output_dir), "docs") + args.blog_output_dir = os.path.join(os.path.abspath(args.output_dir), "blog") from github import get_events - args.rev = subprocess.check_output('git rev-parse HEAD', shell=True).decode('utf-8').strip() - args.rev_short = subprocess.check_output('git rev-parse --short HEAD', shell=True).decode('utf-8').strip() - args.rev_url = f'https://github.com/ClickHouse/ClickHouse/commit/{args.rev}' + + args.rev = ( + subprocess.check_output("git rev-parse HEAD", shell=True) + .decode("utf-8") + .strip() + ) + args.rev_short = ( + subprocess.check_output("git rev-parse --short HEAD", shell=True) + .decode("utf-8") + .strip() + ) + args.rev_url = f"https://github.com/ClickHouse/ClickHouse/commit/{args.rev}" args.events = get_events(args) if args.test_only: @@ -233,18 +248,20 @@ if __name__ == '__main__': mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True from build import build + build(args) if args.livereload: - new_args = [arg for arg in sys.argv if not arg.startswith('--livereload')] - new_args = sys.executable + ' ' + ' '.join(new_args) + new_args = [arg for arg in sys.argv if not arg.startswith("--livereload")] + new_args = sys.executable + " " + " ".join(new_args) server = livereload.Server() - server.watch(args.docs_dir + '**/*', livereload.shell(new_args, cwd='tools', shell=True)) - server.watch(args.website_dir + '**/*', livereload.shell(new_args, cwd='tools', shell=True)) - server.serve( - root=args.output_dir, - host='0.0.0.0', - port=args.livereload + server.watch( + args.docs_dir + "**/*", livereload.shell(new_args, cwd="tools", shell=True) ) + server.watch( + args.website_dir + "**/*", + livereload.shell(new_args, cwd="tools", shell=True), + ) + server.serve(root=args.output_dir, host="0.0.0.0", port=args.livereload) sys.exit(0) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index aa4cbbddd18..9bbc94fd206 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -6,11 +6,13 @@ from typing import TextIO, List, Tuple, Optional, Dict Entity = Tuple[str, str, str] # https://regex101.com/r/R6iogw/12 -cmake_option_regex: str = r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" +cmake_option_regex: str = ( + r"^\s*option\s*\(([A-Z_0-9${}]+)\s*(?:\"((?:.|\n)*?)\")?\s*(.*)?\).*$" +) ch_master_url: str = "https://github.com/clickhouse/clickhouse/blob/master/" -name_str: str = "[`{name}`](" + ch_master_url + "{path}#L{line})" +name_str: str = '[`{name}`](' + ch_master_url + "{path}#L{line})" default_anchor_str: str = "[`{name}`](#{anchor})" comment_var_regex: str = r"\${(.+)}" @@ -27,11 +29,15 @@ entities: Dict[str, Tuple[str, str]] = {} def make_anchor(t: str) -> str: - return "".join(["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"]) + return "".join( + ["-" if i == "_" else i.lower() for i in t if i.isalpha() or i == "_"] + ) + def process_comment(comment: str) -> str: return re.sub(comment_var_regex, comment_var_replace, comment, flags=re.MULTILINE) + def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> None: (line, comment) = line_comment (name, description, default) = entity @@ -47,22 +53,22 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No formatted_default: str = "`" + default + "`" formatted_name: str = name_str.format( - anchor=make_anchor(name), - name=name, - path=path, - line=line) + anchor=make_anchor(name), name=name, path=path, line=line + ) formatted_description: str = "".join(description.split("\n")) formatted_comment: str = process_comment(comment) formatted_entity: str = "| {} | {} | {} | {} |".format( - formatted_name, formatted_default, formatted_description, formatted_comment) + formatted_name, formatted_default, formatted_description, formatted_comment + ) entities[name] = path, formatted_entity + def process_file(root_path: str, file_path: str, file_name: str) -> None: - with open(os.path.join(file_path, file_name), 'r') as cmake_file: + with open(os.path.join(file_path, file_name), "r") as cmake_file: contents: str = cmake_file.read() def get_line_and_comment(target: str) -> Tuple[int, str]: @@ -70,10 +76,10 @@ def process_file(root_path: str, file_path: str, file_name: str) -> None: comment: str = "" for n, line in enumerate(contents_list): - if 'option' not in line.lower() or target not in line: + if "option" not in line.lower() or target not in line: continue - for maybe_comment_line in contents_list[n - 1::-1]: + for maybe_comment_line in contents_list[n - 1 :: -1]: if not re.match("\s*#\s*", maybe_comment_line): break @@ -82,16 +88,22 @@ def process_file(root_path: str, file_path: str, file_name: str) -> None: # line numbering starts with 1 return n + 1, comment - matches: Optional[List[Entity]] = re.findall(cmake_option_regex, contents, re.MULTILINE) + matches: Optional[List[Entity]] = re.findall( + cmake_option_regex, contents, re.MULTILINE + ) - - file_rel_path_with_name: str = os.path.join(file_path[len(root_path):], file_name) - if file_rel_path_with_name.startswith('/'): + file_rel_path_with_name: str = os.path.join( + file_path[len(root_path) :], file_name + ) + if file_rel_path_with_name.startswith("/"): file_rel_path_with_name = file_rel_path_with_name[1:] if matches: for entity in matches: - build_entity(file_rel_path_with_name, entity, get_line_and_comment(entity[0])) + build_entity( + file_rel_path_with_name, entity, get_line_and_comment(entity[0]) + ) + def process_folder(root_path: str, name: str) -> None: for root, _, files in os.walk(os.path.join(root_path, name)): @@ -99,12 +111,19 @@ def process_folder(root_path: str, name: str) -> None: if f == "CMakeLists.txt" or ".cmake" in f: process_file(root_path, root, f) -def generate_cmake_flags_files() -> None: - root_path: str = os.path.join(os.path.dirname(__file__), '..', '..') - output_file_name: str = os.path.join(root_path, "docs/en/development/cmake-in-clickhouse.md") - header_file_name: str = os.path.join(root_path, "docs/_includes/cmake_in_clickhouse_header.md") - footer_file_name: str = os.path.join(root_path, "docs/_includes/cmake_in_clickhouse_footer.md") +def generate_cmake_flags_files() -> None: + root_path: str = os.path.join(os.path.dirname(__file__), "..", "..") + + output_file_name: str = os.path.join( + root_path, "docs/en/development/cmake-in-clickhouse.md" + ) + header_file_name: str = os.path.join( + root_path, "docs/_includes/cmake_in_clickhouse_header.md" + ) + footer_file_name: str = os.path.join( + root_path, "docs/_includes/cmake_in_clickhouse_footer.md" + ) process_file(root_path, root_path, "CMakeLists.txt") process_file(root_path, os.path.join(root_path, "programs"), "CMakeLists.txt") @@ -127,8 +146,10 @@ def generate_cmake_flags_files() -> None: f.write(entities[k][1] + "\n") ignored_keys.append(k) - f.write("\n### External libraries\nNote that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.\n" + - table_header) + f.write( + "\n### External libraries\nNote that ClickHouse uses forks of these libraries, see https://github.com/ClickHouse-Extras.\n" + + table_header + ) for k in sorted_keys: if k.startswith("ENABLE_") and ".cmake" in entities[k][0]: @@ -143,15 +164,18 @@ def generate_cmake_flags_files() -> None: with open(footer_file_name, "r") as footer: f.write(footer.read()) - other_languages = ["docs/ja/development/cmake-in-clickhouse.md", - "docs/zh/development/cmake-in-clickhouse.md", - "docs/ru/development/cmake-in-clickhouse.md"] + other_languages = [ + "docs/ja/development/cmake-in-clickhouse.md", + "docs/zh/development/cmake-in-clickhouse.md", + "docs/ru/development/cmake-in-clickhouse.md", + ] for lang in other_languages: other_file_name = os.path.join(root_path, lang) if os.path.exists(other_file_name): - os.unlink(other_file_name) + os.unlink(other_file_name) os.symlink(output_file_name, other_file_name) -if __name__ == '__main__': + +if __name__ == "__main__": generate_cmake_flags_files() diff --git a/docs/tools/easy_diff.py b/docs/tools/easy_diff.py index 22d305d3da3..14e3ca91776 100755 --- a/docs/tools/easy_diff.py +++ b/docs/tools/easy_diff.py @@ -8,7 +8,7 @@ import contextlib from git import cmd from tempfile import NamedTemporaryFile -SCRIPT_DESCRIPTION = ''' +SCRIPT_DESCRIPTION = """ usage: ./easy_diff.py language/document path Show the difference between a language document and an English document. @@ -53,16 +53,16 @@ SCRIPT_DESCRIPTION = ''' OPTIONS: -h, --help show this help message and exit --no-pager use stdout as difference result output -''' +""" SCRIPT_PATH = os.path.abspath(__file__) -CLICKHOUSE_REPO_HOME = os.path.join(os.path.dirname(SCRIPT_PATH), '..', '..') +CLICKHOUSE_REPO_HOME = os.path.join(os.path.dirname(SCRIPT_PATH), "..", "..") SCRIPT_COMMAND_EXECUTOR = cmd.Git(CLICKHOUSE_REPO_HOME) SCRIPT_COMMAND_PARSER = argparse.ArgumentParser(add_help=False) -SCRIPT_COMMAND_PARSER.add_argument('path', type=bytes, nargs='?', default=None) -SCRIPT_COMMAND_PARSER.add_argument('--no-pager', action='store_true', default=False) -SCRIPT_COMMAND_PARSER.add_argument('-h', '--help', action='store_true', default=False) +SCRIPT_COMMAND_PARSER.add_argument("path", type=bytes, nargs="?", default=None) +SCRIPT_COMMAND_PARSER.add_argument("--no-pager", action="store_true", default=False) +SCRIPT_COMMAND_PARSER.add_argument("-h", "--help", action="store_true", default=False) def execute(commands): @@ -70,19 +70,41 @@ def execute(commands): def get_hash(file_name): - return execute(['git', 'log', '-n', '1', '--pretty=format:"%H"', file_name]) + return execute(["git", "log", "-n", "1", '--pretty=format:"%H"', file_name]) def diff_file(reference_file, working_file, out): if not os.path.exists(reference_file): - raise RuntimeError('reference file [' + os.path.abspath(reference_file) + '] is not exists.') + raise RuntimeError( + "reference file [" + os.path.abspath(reference_file) + "] is not exists." + ) if os.path.islink(working_file): out.writelines(["Need translate document:" + os.path.abspath(reference_file)]) elif not os.path.exists(working_file): - out.writelines(['Need link document ' + os.path.abspath(reference_file) + ' to ' + os.path.abspath(working_file)]) + out.writelines( + [ + "Need link document " + + os.path.abspath(reference_file) + + " to " + + os.path.abspath(working_file) + ] + ) elif get_hash(working_file) != get_hash(reference_file): - out.writelines([(execute(['git', 'diff', get_hash(working_file).strip('"'), reference_file]).encode('utf-8'))]) + out.writelines( + [ + ( + execute( + [ + "git", + "diff", + get_hash(working_file).strip('"'), + reference_file, + ] + ).encode("utf-8") + ) + ] + ) return 0 @@ -94,20 +116,30 @@ def diff_directory(reference_directory, working_directory, out): for list_item in os.listdir(reference_directory): working_item = os.path.join(working_directory, list_item) reference_item = os.path.join(reference_directory, list_item) - if diff_file(reference_item, working_item, out) if os.path.isfile(reference_item) else diff_directory(reference_item, working_item, out) != 0: + if ( + diff_file(reference_item, working_item, out) + if os.path.isfile(reference_item) + else diff_directory(reference_item, working_item, out) != 0 + ): return 1 return 0 -def find_language_doc(custom_document, other_language='en', children=[]): +def find_language_doc(custom_document, other_language="en", children=[]): if len(custom_document) == 0: - raise RuntimeError('The ' + os.path.join(custom_document, *children) + " is not in docs directory.") + raise RuntimeError( + "The " + + os.path.join(custom_document, *children) + + " is not in docs directory." + ) - if os.path.samefile(os.path.join(CLICKHOUSE_REPO_HOME, 'docs'), custom_document): - return os.path.join(CLICKHOUSE_REPO_HOME, 'docs', other_language, *children[1:]) + if os.path.samefile(os.path.join(CLICKHOUSE_REPO_HOME, "docs"), custom_document): + return os.path.join(CLICKHOUSE_REPO_HOME, "docs", other_language, *children[1:]) children.insert(0, os.path.split(custom_document)[1]) - return find_language_doc(os.path.split(custom_document)[0], other_language, children) + return find_language_doc( + os.path.split(custom_document)[0], other_language, children + ) class ToPager: @@ -119,7 +151,7 @@ class ToPager: def close(self): self.temp_named_file.flush() - git_pager = execute(['git', 'var', 'GIT_PAGER']) + git_pager = execute(["git", "var", "GIT_PAGER"]) subprocess.check_call([git_pager, self.temp_named_file.name]) self.temp_named_file.close() @@ -135,12 +167,20 @@ class ToStdOut: self.system_stdout_stream = system_stdout_stream -if __name__ == '__main__': +if __name__ == "__main__": arguments = SCRIPT_COMMAND_PARSER.parse_args() if arguments.help or not arguments.path: sys.stdout.write(SCRIPT_DESCRIPTION) sys.exit(0) - working_language = os.path.join(CLICKHOUSE_REPO_HOME, 'docs', arguments.path) - with contextlib.closing(ToStdOut(sys.stdout) if arguments.no_pager else ToPager(NamedTemporaryFile('r+'))) as writer: - exit(diff_directory(find_language_doc(working_language), working_language, writer)) + working_language = os.path.join(CLICKHOUSE_REPO_HOME, "docs", arguments.path) + with contextlib.closing( + ToStdOut(sys.stdout) + if arguments.no_pager + else ToPager(NamedTemporaryFile("r+")) + ) as writer: + exit( + diff_directory( + find_language_doc(working_language), working_language, writer + ) + ) diff --git a/docs/tools/github.py b/docs/tools/github.py index 465695d1512..3a6f155e25d 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -16,27 +16,26 @@ import util def get_events(args): events = [] skip = True - with open(os.path.join(args.docs_dir, '..', 'README.md')) as f: + with open(os.path.join(args.docs_dir, "..", "README.md")) as f: for line in f: if skip: - if 'Upcoming Events' in line: + if "Upcoming Events" in line: skip = False else: if not line: continue - line = line.strip().split('](') + line = line.strip().split("](") if len(line) == 2: - tail = line[1].split(') ') - events.append({ - 'signup_link': tail[0], - 'event_name': line[0].replace('* [', ''), - 'event_date': tail[1].replace('on ', '').replace('.', '') - }) + tail = line[1].split(") ") + events.append( + { + "signup_link": tail[0], + "event_name": line[0].replace("* [", ""), + "event_date": tail[1].replace("on ", "").replace(".", ""), + } + ) return events -if __name__ == '__main__': - logging.basicConfig( - level=logging.DEBUG, - stream=sys.stderr - ) +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG, stream=sys.stderr) diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index 18ecc890b6e..6b5a5bb5813 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -16,74 +16,79 @@ import slugify as slugify_impl def slugify(value, separator): - return slugify_impl.slugify(value, separator=separator, word_boundary=True, save_order=True) + return slugify_impl.slugify( + value, separator=separator, word_boundary=True, save_order=True + ) MARKDOWN_EXTENSIONS = [ - 'mdx_clickhouse', - 'admonition', - 'attr_list', - 'def_list', - 'codehilite', - 'nl2br', - 'sane_lists', - 'pymdownx.details', - 'pymdownx.magiclink', - 'pymdownx.superfences', - 'extra', - { - 'toc': { - 'permalink': True, - 'slugify': slugify - } - } + "mdx_clickhouse", + "admonition", + "attr_list", + "def_list", + "codehilite", + "nl2br", + "sane_lists", + "pymdownx.details", + "pymdownx.magiclink", + "pymdownx.superfences", + "extra", + {"toc": {"permalink": True, "slugify": slugify}}, ] class ClickHouseLinkMixin(object): - def handleMatch(self, m, data): - single_page = (os.environ.get('SINGLE_PAGE') == '1') + single_page = os.environ.get("SINGLE_PAGE") == "1" try: el, start, end = super(ClickHouseLinkMixin, self).handleMatch(m, data) except IndexError: return if el is not None: - href = el.get('href') or '' - is_external = href.startswith('http:') or href.startswith('https:') + href = el.get("href") or "" + is_external = href.startswith("http:") or href.startswith("https:") if is_external: - if not href.startswith('https://clickhouse.com'): - el.set('rel', 'external nofollow noreferrer') + if not href.startswith("https://clickhouse.com"): + el.set("rel", "external nofollow noreferrer") elif single_page: - if '#' in href: - el.set('href', '#' + href.split('#', 1)[1]) + if "#" in href: + el.set("href", "#" + href.split("#", 1)[1]) else: - el.set('href', '#' + href.replace('/index.md', '/').replace('.md', '/')) + el.set( + "href", "#" + href.replace("/index.md", "/").replace(".md", "/") + ) return el, start, end -class ClickHouseAutolinkPattern(ClickHouseLinkMixin, markdown.inlinepatterns.AutolinkInlineProcessor): +class ClickHouseAutolinkPattern( + ClickHouseLinkMixin, markdown.inlinepatterns.AutolinkInlineProcessor +): pass -class ClickHouseLinkPattern(ClickHouseLinkMixin, markdown.inlinepatterns.LinkInlineProcessor): +class ClickHouseLinkPattern( + ClickHouseLinkMixin, markdown.inlinepatterns.LinkInlineProcessor +): pass class ClickHousePreprocessor(markdown.util.Processor): def run(self, lines): for line in lines: - if '' not in line: + if "" not in line: yield line class ClickHouseMarkdown(markdown.extensions.Extension): - def extendMarkdown(self, md, md_globals): - md.preprocessors['clickhouse'] = ClickHousePreprocessor() - md.inlinePatterns['link'] = ClickHouseLinkPattern(markdown.inlinepatterns.LINK_RE, md) - md.inlinePatterns['autolink'] = ClickHouseAutolinkPattern(markdown.inlinepatterns.AUTOLINK_RE, md) + md.preprocessors["clickhouse"] = ClickHousePreprocessor() + md.inlinePatterns["link"] = ClickHouseLinkPattern( + markdown.inlinepatterns.LINK_RE, md + ) + md.inlinePatterns["autolink"] = ClickHouseAutolinkPattern( + markdown.inlinepatterns.AUTOLINK_RE, md + ) def makeExtension(**kwargs): @@ -92,10 +97,8 @@ def makeExtension(**kwargs): def get_translations(dirname, lang): import babel.support - return babel.support.Translations.load( - dirname=dirname, - locales=[lang, 'en'] - ) + + return babel.support.Translations.load(dirname=dirname, locales=[lang, "en"]) class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): @@ -104,22 +107,22 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): def on_config(self, config): super(PatchedMacrosPlugin, self).on_config(config) - self.env.comment_start_string = '{##' - self.env.comment_end_string = '##}' - self.env.loader = jinja2.FileSystemLoader([ - os.path.join(config.data['site_dir']), - os.path.join(config.data['extra']['includes_dir']) - ]) + self.env.comment_start_string = "{##" + self.env.comment_end_string = "##}" + self.env.loader = jinja2.FileSystemLoader( + [ + os.path.join(config.data["site_dir"]), + os.path.join(config.data["extra"]["includes_dir"]), + ] + ) def on_env(self, env, config, files): import util - env.add_extension('jinja2.ext.i18n') - dirname = os.path.join(config.data['theme'].dirs[0], 'locale') - lang = config.data['theme']['language'] - env.install_gettext_translations( - get_translations(dirname, lang), - newstyle=True - ) + + env.add_extension("jinja2.ext.i18n") + dirname = os.path.join(config.data["theme"].dirs[0], "locale") + lang = config.data["theme"]["language"] + env.install_gettext_translations(get_translations(dirname, lang), newstyle=True) util.init_jinja2_filters(env) return env @@ -130,13 +133,17 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin): return markdown def on_page_markdown(self, markdown, page, config, files): - markdown = super(PatchedMacrosPlugin, self).on_page_markdown(markdown, page, config, files) + markdown = super(PatchedMacrosPlugin, self).on_page_markdown( + markdown, page, config, files + ) if os.path.islink(page.file.abs_src_path): - lang = config.data['theme']['language'] - page.canonical_url = page.canonical_url.replace(f'/{lang}/', '/en/', 1) + lang = config.data["theme"]["language"] + page.canonical_url = page.canonical_url.replace(f"/{lang}/", "/en/", 1) - if config.data['extra'].get('version_prefix') or config.data['extra'].get('single_page'): + if config.data["extra"].get("version_prefix") or config.data["extra"].get( + "single_page" + ): return markdown if self.skip_git_log: return markdown diff --git a/docs/tools/nav.py b/docs/tools/nav.py index db64d1ba404..e3df85bbe4e 100644 --- a/docs/tools/nav.py +++ b/docs/tools/nav.py @@ -10,57 +10,59 @@ import util def find_first_header(content): - for line in content.split('\n'): - if line.startswith('#'): - no_hash = line.lstrip('#') - return no_hash.split('{', 1)[0].strip() + for line in content.split("\n"): + if line.startswith("#"): + no_hash = line.lstrip("#") + return no_hash.split("{", 1)[0].strip() def build_nav_entry(root, args): - if root.endswith('images'): + if root.endswith("images"): return None, None, None result_items = [] - index_meta, index_content = util.read_md_file(os.path.join(root, 'index.md')) - current_title = index_meta.get('toc_folder_title', index_meta.get('toc_title')) - current_title = current_title or index_meta.get('title', find_first_header(index_content)) + index_meta, index_content = util.read_md_file(os.path.join(root, "index.md")) + current_title = index_meta.get("toc_folder_title", index_meta.get("toc_title")) + current_title = current_title or index_meta.get( + "title", find_first_header(index_content) + ) for filename in os.listdir(root): path = os.path.join(root, filename) if os.path.isdir(path): prio, title, payload = build_nav_entry(path, args) if title and payload: result_items.append((prio, title, payload)) - elif filename.endswith('.md'): + elif filename.endswith(".md"): path = os.path.join(root, filename) - meta = '' - content = '' + meta = "" + content = "" try: meta, content = util.read_md_file(path) except: - print('Error in file: {}'.format(path)) + print("Error in file: {}".format(path)) raise - path = path.split('/', 2)[-1] - title = meta.get('toc_title', find_first_header(content)) + path = path.split("/", 2)[-1] + title = meta.get("toc_title", find_first_header(content)) if title: - title = title.strip().rstrip('.') + title = title.strip().rstrip(".") else: - title = meta.get('toc_folder_title', 'hidden') - prio = meta.get('toc_priority', 9999) - logging.debug(f'Nav entry: {prio}, {title}, {path}') - if meta.get('toc_hidden') or not content.strip(): - title = 'hidden' - if title == 'hidden': - title = 'hidden-' + hashlib.sha1(content.encode('utf-8')).hexdigest() + title = meta.get("toc_folder_title", "hidden") + prio = meta.get("toc_priority", 9999) + logging.debug(f"Nav entry: {prio}, {title}, {path}") + if meta.get("toc_hidden") or not content.strip(): + title = "hidden" + if title == "hidden": + title = "hidden-" + hashlib.sha1(content.encode("utf-8")).hexdigest() if args.nav_limit and len(result_items) >= args.nav_limit: break result_items.append((prio, title, path)) result_items = sorted(result_items, key=lambda x: (x[0], x[1])) result = collections.OrderedDict([(item[1], item[2]) for item in result_items]) - if index_meta.get('toc_hidden_folder'): - current_title += '|hidden-folder' - return index_meta.get('toc_priority', 10000), current_title, result + if index_meta.get("toc_hidden_folder"): + current_title += "|hidden-folder" + return index_meta.get("toc_priority", 10000), current_title, result def build_docs_nav(lang, args): @@ -70,7 +72,7 @@ def build_docs_nav(lang, args): index_key = None for key, value in list(nav.items()): if key and value: - if value == 'index.md': + if value == "index.md": index_key = key continue result.append({key: value}) @@ -78,7 +80,7 @@ def build_docs_nav(lang, args): break if index_key: key = list(result[0].keys())[0] - result[0][key][index_key] = 'index.md' + result[0][key][index_key] = "index.md" result[0][key].move_to_end(index_key, last=False) return result @@ -86,7 +88,7 @@ def build_docs_nav(lang, args): def build_blog_nav(lang, args): blog_dir = os.path.join(args.blog_dir, lang) years = sorted(os.listdir(blog_dir), reverse=True) - result_nav = [{'hidden': 'index.md'}] + result_nav = [{"hidden": "index.md"}] post_meta = collections.OrderedDict() for year in years: year_dir = os.path.join(blog_dir, year) @@ -97,38 +99,53 @@ def build_blog_nav(lang, args): post_meta_items = [] for post in os.listdir(year_dir): post_path = os.path.join(year_dir, post) - if not post.endswith('.md'): - raise RuntimeError(f'Unexpected non-md file in posts folder: {post_path}') + if not post.endswith(".md"): + raise RuntimeError( + f"Unexpected non-md file in posts folder: {post_path}" + ) meta, _ = util.read_md_file(post_path) - post_date = meta['date'] - post_title = meta['title'] + post_date = meta["date"] + post_title = meta["title"] if datetime.date.fromisoformat(post_date) > datetime.date.today(): continue posts.append( - (post_date, post_title, os.path.join(year, post),) + ( + post_date, + post_title, + os.path.join(year, post), + ) ) if post_title in post_meta: - raise RuntimeError(f'Duplicate post title: {post_title}') - if not post_date.startswith(f'{year}-'): - raise RuntimeError(f'Post date {post_date} doesn\'t match the folder year {year}: {post_title}') - post_url_part = post.replace('.md', '') - post_meta_items.append((post_date, { - 'date': post_date, - 'title': post_title, - 'image': meta.get('image'), - 'url': f'/blog/{lang}/{year}/{post_url_part}/' - },)) + raise RuntimeError(f"Duplicate post title: {post_title}") + if not post_date.startswith(f"{year}-"): + raise RuntimeError( + f"Post date {post_date} doesn't match the folder year {year}: {post_title}" + ) + post_url_part = post.replace(".md", "") + post_meta_items.append( + ( + post_date, + { + "date": post_date, + "title": post_title, + "image": meta.get("image"), + "url": f"/blog/{lang}/{year}/{post_url_part}/", + }, + ) + ) for _, title, path in sorted(posts, reverse=True): result_nav[-1][year][title] = path - for _, post_meta_item in sorted(post_meta_items, - reverse=True, - key=lambda item: item[0]): - post_meta[post_meta_item['title']] = post_meta_item + for _, post_meta_item in sorted( + post_meta_items, reverse=True, key=lambda item: item[0] + ): + post_meta[post_meta_item["title"]] = post_meta_item return result_nav, post_meta def _custom_get_navigation(files, config): - nav_config = config['nav'] or mkdocs.structure.nav.nest_paths(f.src_path for f in files.documentation_pages()) + nav_config = config["nav"] or mkdocs.structure.nav.nest_paths( + f.src_path for f in files.documentation_pages() + ) items = mkdocs.structure.nav._data_to_navigation(nav_config, files, config) if not isinstance(items, list): items = [items] @@ -138,19 +155,25 @@ def _custom_get_navigation(files, config): mkdocs.structure.nav._add_previous_and_next_links(pages) mkdocs.structure.nav._add_parent_links(items) - missing_from_config = [file for file in files.documentation_pages() if file.page is None] + missing_from_config = [ + file for file in files.documentation_pages() if file.page is None + ] if missing_from_config: - files._files = [file for file in files._files if file not in missing_from_config] + files._files = [ + file for file in files._files if file not in missing_from_config + ] links = mkdocs.structure.nav._get_by_type(items, mkdocs.structure.nav.Link) for link in links: - scheme, netloc, path, params, query, fragment = mkdocs.structure.nav.urlparse(link.url) + scheme, netloc, path, params, query, fragment = mkdocs.structure.nav.urlparse( + link.url + ) if scheme or netloc: mkdocs.structure.nav.log.debug( "An external link to '{}' is included in " "the 'nav' configuration.".format(link.url) ) - elif link.url.startswith('/'): + elif link.url.startswith("/"): mkdocs.structure.nav.log.debug( "An absolute path to '{}' is included in the 'nav' configuration, " "which presumably points to an external resource.".format(link.url) diff --git a/docs/tools/redirects.py b/docs/tools/redirects.py index 20e3ec7aa6f..5d222376683 100644 --- a/docs/tools/redirects.py +++ b/docs/tools/redirects.py @@ -7,8 +7,9 @@ def write_redirect_html(out_path, to_url): os.makedirs(out_dir) except OSError: pass - with open(out_path, 'w') as f: - f.write(f''' + with open(out_path, "w") as f: + f.write( + f""" @@ -22,48 +23,70 @@ def write_redirect_html(out_path, to_url): If you are not redirected automatically, follow this link. -''') +""" + ) def build_redirect_html(args, base_prefix, lang, output_dir, from_path, to_path): out_path = os.path.join( - output_dir, lang, - from_path.replace('/index.md', '/index.html').replace('.md', '/index.html') + output_dir, + lang, + from_path.replace("/index.md", "/index.html").replace(".md", "/index.html"), ) - target_path = to_path.replace('/index.md', '/').replace('.md', '/') - to_url = f'/{base_prefix}/{lang}/{target_path}' + target_path = to_path.replace("/index.md", "/").replace(".md", "/") + + if target_path[0:7] != "http://" and target_path[0:8] != "https://": + to_url = f"/{base_prefix}/{lang}/{target_path}" + else: + to_url = target_path + to_url = to_url.strip() write_redirect_html(out_path, to_url) def build_docs_redirects(args): - with open(os.path.join(args.docs_dir, 'redirects.txt'), 'r') as f: + with open(os.path.join(args.docs_dir, "redirects.txt"), "r") as f: for line in f: - for lang in args.lang.split(','): - from_path, to_path = line.split(' ', 1) - build_redirect_html(args, 'docs', lang, args.docs_output_dir, from_path, to_path) + for lang in args.lang.split(","): + from_path, to_path = line.split(" ", 1) + build_redirect_html( + args, "docs", lang, args.docs_output_dir, from_path, to_path + ) def build_blog_redirects(args): - for lang in args.blog_lang.split(','): - redirects_path = os.path.join(args.blog_dir, lang, 'redirects.txt') + for lang in args.blog_lang.split(","): + redirects_path = os.path.join(args.blog_dir, lang, "redirects.txt") if os.path.exists(redirects_path): - with open(redirects_path, 'r') as f: + with open(redirects_path, "r") as f: for line in f: - from_path, to_path = line.split(' ', 1) - build_redirect_html(args, 'blog', lang, args.blog_output_dir, from_path, to_path) + from_path, to_path = line.split(" ", 1) + build_redirect_html( + args, "blog", lang, args.blog_output_dir, from_path, to_path + ) def build_static_redirects(args): for static_redirect in [ - ('benchmark.html', '/benchmark/dbms/'), - ('benchmark_hardware.html', '/benchmark/hardware/'), - ('tutorial.html', '/docs/en/getting_started/tutorial/',), - ('reference_en.html', '/docs/en/single/', ), - ('reference_ru.html', '/docs/ru/single/',), - ('docs/index.html', '/docs/en/',), + ("benchmark.html", "/benchmark/dbms/"), + ("benchmark_hardware.html", "/benchmark/hardware/"), + ( + "tutorial.html", + "/docs/en/getting_started/tutorial/", + ), + ( + "reference_en.html", + "/docs/en/single/", + ), + ( + "reference_ru.html", + "/docs/ru/single/", + ), + ( + "docs/index.html", + "/docs/en/", + ), ]: write_redirect_html( - os.path.join(args.output_dir, static_redirect[0]), - static_redirect[1] + os.path.join(args.output_dir, static_redirect[0]), static_redirect[1] ) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 4e0789b5d24..8bf1a5f477c 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -1,4 +1,4 @@ -Babel==2.8.0 +Babel==2.9.1 backports-abc==0.5 backports.functools-lru-cache==1.6.1 beautifulsoup4==4.9.1 @@ -10,22 +10,22 @@ cssmin==0.2.0 future==0.18.2 htmlmin==0.1.12 idna==2.10 -Jinja2>=2.11.3 +Jinja2>=3.0.3 jinja2-highlight==0.6.1 jsmin==3.0.0 -livereload==2.6.2 +livereload==2.6.3 Markdown==3.3.2 -MarkupSafe==1.1.1 +MarkupSafe==2.1.0 mkdocs==1.1.2 mkdocs-htmlproofer-plugin==0.0.3 mkdocs-macros-plugin==0.4.20 -nltk==3.5 +nltk==3.7 nose==1.3.7 protobuf==3.14.0 numpy==1.21.2 pymdown-extensions==8.0 python-slugify==4.0.1 -PyYAML==5.4.1 +PyYAML==6.0 repackage==0.7.3 requests==2.25.1 singledispatch==3.4.0.3 @@ -34,5 +34,6 @@ soupsieve==2.0.1 termcolor==1.1.0 tornado==6.1 Unidecode==1.1.1 -urllib3>=1.26.5 -Pygments>=2.7.4 +urllib3>=1.26.8 +Pygments>=2.11.2 + diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index 3d32ba30a21..ed285fce9f8 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -12,7 +12,8 @@ import test import util import website -TEMPORARY_FILE_NAME = 'single.md' +TEMPORARY_FILE_NAME = "single.md" + def recursive_values(item): if isinstance(item, dict): @@ -25,11 +26,14 @@ def recursive_values(item): yield item -anchor_not_allowed_chars = re.compile(r'[^\w\-]') -def generate_anchor_from_path(path): - return re.sub(anchor_not_allowed_chars, '-', path) +anchor_not_allowed_chars = re.compile(r"[^\w\-]") -absolute_link = re.compile(r'^https?://') + +def generate_anchor_from_path(path): + return re.sub(anchor_not_allowed_chars, "-", path) + + +absolute_link = re.compile(r"^https?://") def replace_link(match, path): @@ -40,46 +44,55 @@ def replace_link(match, path): if re.search(absolute_link, link): return match.group(0) - if link.endswith('/'): - link = link[0:-1] + '.md' + if link.endswith("/"): + link = link[0:-1] + ".md" - return '{}(#{})'.format(title, generate_anchor_from_path(os.path.normpath(os.path.join(os.path.dirname(path), link)))) + return "{}(#{})".format( + title, + generate_anchor_from_path( + os.path.normpath(os.path.join(os.path.dirname(path), link)) + ), + ) # Concatenates Markdown files to a single file. def concatenate(lang, docs_path, single_page_file, nav): lang_path = os.path.join(docs_path, lang) - proj_config = f'{docs_path}/toc_{lang}.yml' + proj_config = f"{docs_path}/toc_{lang}.yml" if os.path.exists(proj_config): with open(proj_config) as cfg_file: - nav = yaml.full_load(cfg_file.read())['nav'] + nav = yaml.full_load(cfg_file.read())["nav"] files_to_concatenate = list(recursive_values(nav)) files_count = len(files_to_concatenate) - logging.info(f'{files_count} files will be concatenated into single md-file for {lang}.') - logging.debug('Concatenating: ' + ', '.join(files_to_concatenate)) - assert files_count > 0, f'Empty single-page for {lang}' + logging.info( + f"{files_count} files will be concatenated into single md-file for {lang}." + ) + logging.debug("Concatenating: " + ", ".join(files_to_concatenate)) + assert files_count > 0, f"Empty single-page for {lang}" - link_regexp = re.compile(r'(\[[^\]]+\])\(([^)#]+)(?:#[^\)]+)?\)') + link_regexp = re.compile(r"(\[[^\]]+\])\(([^)#]+)(?:#[^\)]+)?\)") for path in files_to_concatenate: try: with open(os.path.join(lang_path, path)) as f: # Insert a horizontal ruler. Then insert an anchor that we will link to. Its name will be a path to the .md file. - single_page_file.write('\n______\n\n' % generate_anchor_from_path(path)) + single_page_file.write( + '\n______\n\n' % generate_anchor_from_path(path) + ) in_metadata = False for line in f: # Skip YAML metadata. - if line == '---\n': + if line == "---\n": in_metadata = not in_metadata continue if not in_metadata: # Increase the level of headers. - if line.startswith('#'): - line = '#' + line + if line.startswith("#"): + line = "#" + line # Replace links within the docs. @@ -87,14 +100,19 @@ def concatenate(lang, docs_path, single_page_file, nav): line = re.sub( link_regexp, lambda match: replace_link(match, path), - line) + line, + ) # If failed to replace the relative link, print to log # But with some exceptions: # - "../src/" -- for cmake-in-clickhouse.md (link to sources) # - "../usr/share" -- changelog entry that has "../usr/share/zoneinfo" - if '../' in line and (not '../usr/share' in line) and (not '../src/' in line): - logging.info('Failed to resolve relative link:') + if ( + "../" in line + and (not "../usr/share" in line) + and (not "../src/" in line) + ): + logging.info("Failed to resolve relative link:") logging.info(path) logging.info(line) @@ -105,9 +123,11 @@ def concatenate(lang, docs_path, single_page_file, nav): single_page_file.flush() + def get_temporary_file_name(lang, args): return os.path.join(args.docs_dir, lang, TEMPORARY_FILE_NAME) + def remove_temporary_files(lang, args): single_md_path = get_temporary_file_name(lang, args) if os.path.exists(single_md_path): @@ -115,14 +135,14 @@ def remove_temporary_files(lang, args): def build_single_page_version(lang, args, nav, cfg): - logging.info(f'Building single page version for {lang}') - os.environ['SINGLE_PAGE'] = '1' - extra = cfg.data['extra'] - extra['single_page'] = True - extra['is_amp'] = False + logging.info(f"Building single page version for {lang}") + os.environ["SINGLE_PAGE"] = "1" + extra = cfg.data["extra"] + extra["single_page"] = True + extra["is_amp"] = False single_md_path = get_temporary_file_name(lang, args) - with open(single_md_path, 'w') as single_md: + with open(single_md_path, "w") as single_md: concatenate(lang, args.docs_dir, single_md, nav) with util.temp_dir() as site_temp: @@ -132,72 +152,83 @@ def build_single_page_version(lang, args, nav, cfg): shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: - if filename != 'single.md' and filename.endswith('.md'): + if filename != "single.md" and filename.endswith(".md"): os.unlink(os.path.join(root, filename)) - cfg.load_dict({ - 'docs_dir': docs_temp_lang, - 'site_dir': site_temp, - 'extra': extra, - 'nav': [ - {cfg.data.get('site_name'): 'single.md'} - ] - }) + cfg.load_dict( + { + "docs_dir": docs_temp_lang, + "site_dir": site_temp, + "extra": extra, + "nav": [{cfg.data.get("site_name"): "single.md"}], + } + ) if not args.test_only: mkdocs.commands.build.build(cfg) - single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single') + single_page_output_path = os.path.join( + args.docs_dir, args.docs_output_dir, lang, "single" + ) if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree( - os.path.join(site_temp, 'single'), - single_page_output_path + os.path.join(site_temp, "single"), single_page_output_path ) - single_page_index_html = os.path.join(single_page_output_path, 'index.html') - single_page_content_js = os.path.join(single_page_output_path, 'content.js') + single_page_index_html = os.path.join( + single_page_output_path, "index.html" + ) + single_page_content_js = os.path.join( + single_page_output_path, "content.js" + ) - with open(single_page_index_html, 'r') as f: - sp_prefix, sp_js, sp_suffix = f.read().split('') + with open(single_page_index_html, "r") as f: + sp_prefix, sp_js, sp_suffix = f.read().split("") - with open(single_page_index_html, 'w') as f: + with open(single_page_index_html, "w") as f: f.write(sp_prefix) f.write(sp_suffix) - with open(single_page_content_js, 'w') as f: + with open(single_page_content_js, "w") as f: if args.minify: import jsmin + sp_js = jsmin.jsmin(sp_js) f.write(sp_js) - logging.info(f'Re-building single page for {lang} pdf/test') + logging.info(f"Re-building single page for {lang} pdf/test") with util.temp_dir() as test_dir: - extra['single_page'] = False - cfg.load_dict({ - 'docs_dir': docs_temp_lang, - 'site_dir': test_dir, - 'extra': extra, - 'nav': [ - {cfg.data.get('site_name'): 'single.md'} - ] - }) + extra["single_page"] = False + cfg.load_dict( + { + "docs_dir": docs_temp_lang, + "site_dir": test_dir, + "extra": extra, + "nav": [{cfg.data.get("site_name"): "single.md"}], + } + ) mkdocs.commands.build.build(cfg) - css_in = ' '.join(website.get_css_in(args)) - js_in = ' '.join(website.get_js_in(args)) - subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) - subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) + css_in = " ".join(website.get_css_in(args)) + js_in = " ".join(website.get_js_in(args)) + subprocess.check_call( + f"cat {css_in} > {test_dir}/css/base.css", shell=True + ) + subprocess.check_call( + f"cat {js_in} > {test_dir}/js/base.js", shell=True + ) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) - logging.info(f'Running tests for {lang}') + logging.info(f"Running tests for {lang}") test.test_single_page( - os.path.join(test_dir, 'single', 'index.html'), lang) + os.path.join(test_dir, "single", "index.html"), lang + ) - logging.info(f'Finished building single page version for {lang}') + logging.info(f"Finished building single page version for {lang}") remove_temporary_files(lang, args) diff --git a/docs/tools/test.py b/docs/tools/test.py index 53ed9505acd..d0469d042ee 100755 --- a/docs/tools/test.py +++ b/docs/tools/test.py @@ -8,14 +8,11 @@ import subprocess def test_single_page(input_path, lang): - if not (lang == 'en' or lang == 'ru'): + if not (lang == "en"): return with open(input_path) as f: - soup = bs4.BeautifulSoup( - f, - features='html.parser' - ) + soup = bs4.BeautifulSoup(f, features="html.parser") anchor_points = set() @@ -23,30 +20,27 @@ def test_single_page(input_path, lang): links_to_nowhere = 0 for tag in soup.find_all(): - for anchor_point in [tag.attrs.get('name'), tag.attrs.get('id')]: + for anchor_point in [tag.attrs.get("name"), tag.attrs.get("id")]: if anchor_point: anchor_points.add(anchor_point) for tag in soup.find_all(): - href = tag.attrs.get('href') - if href and href.startswith('#') and href != '#': + href = tag.attrs.get("href") + if href and href.startswith("#") and href != "#": if href[1:] not in anchor_points: links_to_nowhere += 1 logging.info("Tag %s", tag) - logging.info('Link to nowhere: %s' % href) + logging.info("Link to nowhere: %s" % href) if links_to_nowhere: - logging.error(f'Found {links_to_nowhere} links to nowhere in {lang}') + logging.error(f"Found {links_to_nowhere} links to nowhere in {lang}") sys.exit(1) if len(anchor_points) <= 10: - logging.error('Html parsing is probably broken') + logging.error("Html parsing is probably broken") sys.exit(1) -if __name__ == '__main__': - logging.basicConfig( - level=logging.DEBUG, - stream=sys.stderr - ) +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG, stream=sys.stderr) test_single_page(sys.argv[1], sys.argv[2]) diff --git a/docs/tools/util.py b/docs/tools/util.py index 25961561f99..fb2f135c85e 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -15,7 +15,7 @@ import yaml @contextlib.contextmanager def temp_dir(): - path = tempfile.mkdtemp(dir=os.environ.get('TEMP')) + path = tempfile.mkdtemp(dir=os.environ.get("TEMP")) try: yield path finally: @@ -34,7 +34,7 @@ def cd(new_cwd): def get_free_port(): with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.bind(('', 0)) + s.bind(("", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) return s.getsockname()[1] @@ -61,12 +61,12 @@ def read_md_file(path): meta_text = [] content = [] if os.path.exists(path): - with open(path, 'r') as f: + with open(path, "r") as f: for line in f: - if line.startswith('---'): + if line.startswith("---"): if in_meta: in_meta = False - meta = yaml.full_load(''.join(meta_text)) + meta = yaml.full_load("".join(meta_text)) else: in_meta = True else: @@ -74,7 +74,7 @@ def read_md_file(path): meta_text.append(line) else: content.append(line) - return meta, ''.join(content) + return meta, "".join(content) def write_md_file(path, meta, content): @@ -82,13 +82,13 @@ def write_md_file(path, meta, content): if not os.path.exists(dirname): os.makedirs(dirname) - with open(path, 'w') as f: + with open(path, "w") as f: if meta: - print('---', file=f) + print("---", file=f) yaml.dump(meta, f) - print('---', file=f) - if not content.startswith('\n'): - print('', file=f) + print("---", file=f) + if not content.startswith("\n"): + print("", file=f) f.write(content) @@ -100,7 +100,7 @@ def represent_ordereddict(dumper, data): value.append((node_key, node_value)) - return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) + return yaml.nodes.MappingNode("tag:yaml.org,2002:map", value) yaml.add_representer(collections.OrderedDict, represent_ordereddict) @@ -109,30 +109,31 @@ yaml.add_representer(collections.OrderedDict, represent_ordereddict) def init_jinja2_filters(env): import amp import website + chunk_size = 10240 - env.filters['chunks'] = lambda line: [line[i:i + chunk_size] for i in range(0, len(line), chunk_size)] - env.filters['html_to_amp'] = amp.html_to_amp - env.filters['adjust_markdown_html'] = website.adjust_markdown_html - env.filters['to_rfc882'] = lambda d: datetime.datetime.strptime(d, '%Y-%m-%d').strftime('%a, %d %b %Y %H:%M:%S GMT') + env.filters["chunks"] = lambda line: [ + line[i : i + chunk_size] for i in range(0, len(line), chunk_size) + ] + env.filters["html_to_amp"] = amp.html_to_amp + env.filters["adjust_markdown_html"] = website.adjust_markdown_html + env.filters["to_rfc882"] = lambda d: datetime.datetime.strptime( + d, "%Y-%m-%d" + ).strftime("%a, %d %b %Y %H:%M:%S GMT") def init_jinja2_env(args): import mdx_clickhouse + env = jinja2.Environment( - loader=jinja2.FileSystemLoader([ - args.website_dir, - os.path.join(args.docs_dir, '_includes') - ]), - extensions=[ - 'jinja2.ext.i18n', - 'jinja2_highlight.HighlightExtension' - ] + loader=jinja2.FileSystemLoader( + [args.website_dir, os.path.join(args.docs_dir, "_includes")] + ), + extensions=["jinja2.ext.i18n", "jinja2_highlight.HighlightExtension"], ) - env.extend(jinja2_highlight_cssclass='syntax p-3 my-3') - translations_dir = os.path.join(args.website_dir, 'locale') + env.extend(jinja2_highlight_cssclass="syntax p-3 my-3") + translations_dir = os.path.join(args.website_dir, "locale") env.install_gettext_translations( - mdx_clickhouse.get_translations(translations_dir, 'en'), - newstyle=True + mdx_clickhouse.get_translations(translations_dir, "en"), newstyle=True ) init_jinja2_filters(env) return env diff --git a/docs/tools/webpack.config.js b/docs/tools/webpack.config.js index fcb3e7bf32d..e0dea964101 100644 --- a/docs/tools/webpack.config.js +++ b/docs/tools/webpack.config.js @@ -14,7 +14,6 @@ module.exports = { entry: [ path.resolve(scssPath, 'bootstrap.scss'), - path.resolve(scssPath, 'greenhouse.scss'), path.resolve(scssPath, 'main.scss'), path.resolve(jsPath, 'main.js'), ], diff --git a/docs/tools/website.py b/docs/tools/website.py index 11772fe7a73..2c748d96414 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -17,108 +17,112 @@ import util def handle_iframe(iframe, soup): - allowed_domains = ['https://www.youtube.com/', 'https://datalens.yandex/'] + allowed_domains = ["https://www.youtube.com/", "https://datalens.yandex/"] illegal_domain = True - iframe_src = iframe.attrs['src'] + iframe_src = iframe.attrs["src"] for domain in allowed_domains: if iframe_src.startswith(domain): illegal_domain = False break if illegal_domain: - raise RuntimeError(f'iframe from illegal domain: {iframe_src}') - wrapper = soup.new_tag('div') - wrapper.attrs['class'] = ['embed-responsive', 'embed-responsive-16by9'] + raise RuntimeError(f"iframe from illegal domain: {iframe_src}") + wrapper = soup.new_tag("div") + wrapper.attrs["class"] = ["embed-responsive", "embed-responsive-16by9"] iframe.insert_before(wrapper) iframe.extract() wrapper.insert(0, iframe) - if 'width' in iframe.attrs: - del iframe.attrs['width'] - if 'height' in iframe.attrs: - del iframe.attrs['height'] - iframe.attrs['allow'] = 'accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture' - iframe.attrs['class'] = 'embed-responsive-item' - iframe.attrs['frameborder'] = '0' - iframe.attrs['allowfullscreen'] = '1' + if "width" in iframe.attrs: + del iframe.attrs["width"] + if "height" in iframe.attrs: + del iframe.attrs["height"] + iframe.attrs[ + "allow" + ] = "accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" + iframe.attrs["class"] = "embed-responsive-item" + iframe.attrs["frameborder"] = "0" + iframe.attrs["allowfullscreen"] = "1" def adjust_markdown_html(content): - soup = bs4.BeautifulSoup( - content, - features='html.parser' - ) + soup = bs4.BeautifulSoup(content, features="html.parser") - for a in soup.find_all('a'): - a_class = a.attrs.get('class') - a_href = a.attrs.get('href') - if a_class and 'headerlink' in a_class: - a.string = '\xa0' - if a_href and a_href.startswith('http'): - a.attrs['target'] = '_blank' + for a in soup.find_all("a"): + a_class = a.attrs.get("class") + a_href = a.attrs.get("href") + if a_class and "headerlink" in a_class: + a.string = "\xa0" + if a_href and a_href.startswith("http"): + a.attrs["target"] = "_blank" - for code in soup.find_all('code'): - code_class = code.attrs.get('class') + for code in soup.find_all("code"): + code_class = code.attrs.get("class") if code_class: - code.attrs['class'] = code_class + ['syntax'] + code.attrs["class"] = code_class + ["syntax"] else: - code.attrs['class'] = 'syntax' + code.attrs["class"] = "syntax" - for iframe in soup.find_all('iframe'): + for iframe in soup.find_all("iframe"): handle_iframe(iframe, soup) - for img in soup.find_all('img'): - if img.attrs.get('alt') == 'iframe': - img.name = 'iframe' - img.string = '' + for img in soup.find_all("img"): + if img.attrs.get("alt") == "iframe": + img.name = "iframe" + img.string = "" handle_iframe(img, soup) continue - img_class = img.attrs.get('class') + img_class = img.attrs.get("class") if img_class: - img.attrs['class'] = img_class + ['img-fluid'] + img.attrs["class"] = img_class + ["img-fluid"] else: - img.attrs['class'] = 'img-fluid' + img.attrs["class"] = "img-fluid" - for details in soup.find_all('details'): - for summary in details.find_all('summary'): + for details in soup.find_all("details"): + for summary in details.find_all("summary"): if summary.parent != details: summary.extract() details.insert(0, summary) - for dd in soup.find_all('dd'): - dd_class = dd.attrs.get('class') + for dd in soup.find_all("dd"): + dd_class = dd.attrs.get("class") if dd_class: - dd.attrs['class'] = dd_class + ['pl-3'] + dd.attrs["class"] = dd_class + ["pl-3"] else: - dd.attrs['class'] = 'pl-3' + dd.attrs["class"] = "pl-3" - for div in soup.find_all('div'): - div_class = div.attrs.get('class') - is_admonition = div_class and 'admonition' in div.attrs.get('class') + for div in soup.find_all("div"): + div_class = div.attrs.get("class") + is_admonition = div_class and "admonition" in div.attrs.get("class") if is_admonition: - for a in div.find_all('a'): - a_class = a.attrs.get('class') + for a in div.find_all("a"): + a_class = a.attrs.get("class") if a_class: - a.attrs['class'] = a_class + ['alert-link'] + a.attrs["class"] = a_class + ["alert-link"] else: - a.attrs['class'] = 'alert-link' + a.attrs["class"] = "alert-link" - for p in div.find_all('p'): - p_class = p.attrs.get('class') - if is_admonition and p_class and ('admonition-title' in p_class): - p.attrs['class'] = p_class + ['alert-heading', 'display-4', 'text-reset', 'mb-2'] + for p in div.find_all("p"): + p_class = p.attrs.get("class") + if is_admonition and p_class and ("admonition-title" in p_class): + p.attrs["class"] = p_class + [ + "alert-heading", + "display-4", + "text-reset", + "mb-2", + ] if is_admonition: - div.attrs['role'] = 'alert' - if ('info' in div_class) or ('note' in div_class): - mode = 'alert-primary' - elif ('attention' in div_class) or ('warning' in div_class): - mode = 'alert-warning' - elif 'important' in div_class: - mode = 'alert-danger' - elif 'tip' in div_class: - mode = 'alert-info' + div.attrs["role"] = "alert" + if ("info" in div_class) or ("note" in div_class): + mode = "alert-primary" + elif ("attention" in div_class) or ("warning" in div_class): + mode = "alert-warning" + elif "important" in div_class: + mode = "alert-danger" + elif "tip" in div_class: + mode = "alert-info" else: - mode = 'alert-secondary' - div.attrs['class'] = div_class + ['alert', 'pb-0', 'mb-4', mode] + mode = "alert-secondary" + div.attrs["class"] = div_class + ["alert", "pb-0", "mb-4", mode] return str(soup) @@ -128,56 +132,63 @@ def minify_html(content): def build_website(args): - logging.info('Building website') + logging.info("Building website") env = util.init_jinja2_env(args) shutil.copytree( args.website_dir, args.output_dir, ignore=shutil.ignore_patterns( - '*.md', - '*.sh', - '*.css', - '*.json', - 'js/*.js', - 'build', - 'docs', - 'public', - 'node_modules', - 'src', - 'templates', - 'locale', - '.gitkeep' - ) + "*.md", + "*.sh", + "*.css", + "*.json", + "js/*.js", + "build", + "docs", + "public", + "node_modules", + "src", + "templates", + "locale", + ".gitkeep", + ), + ) + + shutil.copytree( + os.path.join(args.website_dir, "images"), + os.path.join(args.output_dir, "docs", "images"), ) # This file can be requested to check for available ClickHouse releases. shutil.copy2( - os.path.join(args.src_dir, 'utils', 'list-versions', 'version_date.tsv'), - os.path.join(args.output_dir, 'data', 'version_date.tsv')) + os.path.join(args.src_dir, "utils", "list-versions", "version_date.tsv"), + os.path.join(args.output_dir, "data", "version_date.tsv"), + ) # This file can be requested to install ClickHouse. shutil.copy2( - os.path.join(args.src_dir, 'docs', '_includes', 'install', 'universal.sh'), - os.path.join(args.output_dir, 'data', 'install.sh')) + os.path.join(args.src_dir, "docs", "_includes", "install", "universal.sh"), + os.path.join(args.output_dir, "data", "install.sh"), + ) for root, _, filenames in os.walk(args.output_dir): for filename in filenames: - if filename == 'main.html': + if filename == "main.html": continue path = os.path.join(root, filename) - if not filename.endswith('.html'): + if not filename.endswith(".html"): continue - logging.info('Processing %s', path) - with open(path, 'rb') as f: - content = f.read().decode('utf-8') + logging.info("Processing %s", path) + with open(path, "rb") as f: + content = f.read().decode("utf-8") template = env.from_string(content) content = template.render(args.__dict__) - with open(path, 'wb') as f: - f.write(content.encode('utf-8')) + with open(path, "wb") as f: + f.write(content.encode("utf-8")) def get_css_in(args): @@ -188,7 +199,7 @@ def get_css_in(args): f"'{args.website_dir}/css/blog.css'", f"'{args.website_dir}/css/docs.css'", f"'{args.website_dir}/css/highlight.css'", - f"'{args.website_dir}/css/main.css'" + f"'{args.website_dir}/css/main.css'", ] @@ -202,92 +213,103 @@ def get_js_in(args): f"'{args.website_dir}/js/index.js'", f"'{args.website_dir}/js/docsearch.js'", f"'{args.website_dir}/js/docs.js'", - f"'{args.website_dir}/js/main.js'" + f"'{args.website_dir}/js/main.js'", ] def minify_file(path, css_digest, js_digest): - if not ( - path.endswith('.html') or - path.endswith('.css') - ): + if not (path.endswith(".html") or path.endswith(".css")): return - logging.info('Minifying %s', path) - with open(path, 'rb') as f: - content = f.read().decode('utf-8') - if path.endswith('.html'): + logging.info("Minifying %s", path) + with open(path, "rb") as f: + content = f.read().decode("utf-8") + if path.endswith(".html"): content = minify_html(content) - content = content.replace('base.css?css_digest', f'base.css?{css_digest}') - content = content.replace('base.js?js_digest', f'base.js?{js_digest}') -# TODO: restore cssmin -# elif path.endswith('.css'): -# content = cssmin.cssmin(content) -# TODO: restore jsmin -# elif path.endswith('.js'): -# content = jsmin.jsmin(content) - with open(path, 'wb') as f: - f.write(content.encode('utf-8')) + content = content.replace("base.css?css_digest", f"base.css?{css_digest}") + content = content.replace("base.js?js_digest", f"base.js?{js_digest}") + # TODO: restore cssmin + # elif path.endswith('.css'): + # content = cssmin.cssmin(content) + # TODO: restore jsmin + # elif path.endswith('.js'): + # content = jsmin.jsmin(content) + with open(path, "wb") as f: + f.write(content.encode("utf-8")) def minify_website(args): - # Output greenhouse css separately from main bundle to be included via the greenhouse iframe - command = f"cat '{args.website_dir}/css/greenhouse.css' > '{args.output_dir}/css/greenhouse.css'" - logging.info(command) - output = subprocess.check_output(command, shell=True) - logging.debug(output) + css_in = " ".join(get_css_in(args)) + css_out = f"{args.output_dir}/docs/css/base.css" + os.makedirs(f"{args.output_dir}/docs/css") - css_in = ' '.join(get_css_in(args)) - css_out = f'{args.output_dir}/css/base.css' - if args.minify: - command = f"purifycss -w '*algolia*' --min {css_in} '{args.output_dir}/*.html' " \ - f"'{args.output_dir}/docs/en/**/*.html' '{args.website_dir}/js/**/*.js' > {css_out}" - else: - command = f'cat {css_in} > {css_out}' - - logging.info(command) - output = subprocess.check_output(command, shell=True) - logging.debug(output) - with open(css_out, 'rb') as f: - css_digest = hashlib.sha3_224(f.read()).hexdigest()[0:8] - - js_in = get_js_in(args) - js_out = f'{args.output_dir}/js/base.js' if args.minify and False: # TODO: return closure - js_in = [js[1:-1] for js in js_in] - closure_args = [ - '--js', *js_in, '--js_output_file', js_out, - '--compilation_level', 'SIMPLE', - '--dependency_mode', 'NONE', - '--third_party', '--use_types_for_optimization', - '--isolation_mode', 'IIFE' - ] - logging.info(closure_args) - if closure.run(*closure_args): - raise RuntimeError('failed to run closure compiler') - with open(js_out, 'r') as f: - js_content = jsmin.jsmin(f.read()) - with open(js_out, 'w') as f: - f.write(js_content) - - else: - js_in = ' '.join(js_in) - command = f'cat {js_in} > {js_out}' + command = ( + f"purifycss -w '*algolia*' --min {css_in} '{args.output_dir}/*.html' " + f"'{args.output_dir}/docs/en/**/*.html' '{args.website_dir}/js/**/*.js' > {css_out}" + ) + logging.info(css_in) logging.info(command) output = subprocess.check_output(command, shell=True) logging.debug(output) - with open(js_out, 'rb') as f: + + else: + command = f"cat {css_in}" + output = subprocess.check_output(command, shell=True) + with open(css_out, "wb+") as f: + f.write(output) + + with open(css_out, "rb") as f: + css_digest = hashlib.sha3_224(f.read()).hexdigest()[0:8] + + js_in = " ".join(get_js_in(args)) + js_out = f"{args.output_dir}/docs/js/base.js" + os.makedirs(f"{args.output_dir}/docs/js") + + if args.minify and False: # TODO: return closure + js_in = [js[1:-1] for js in js_in] + closure_args = [ + "--js", + *js_in, + "--js_output_file", + js_out, + "--compilation_level", + "SIMPLE", + "--dependency_mode", + "NONE", + "--third_party", + "--use_types_for_optimization", + "--isolation_mode", + "IIFE", + ] + logging.info(closure_args) + if closure.run(*closure_args): + raise RuntimeError("failed to run closure compiler") + with open(js_out, "r") as f: + js_content = jsmin.jsmin(f.read()) + with open(js_out, "w") as f: + f.write(js_content) + + else: + command = f"cat {js_in}" + output = subprocess.check_output(command, shell=True) + with open(js_out, "wb+") as f: + f.write(output) + + with open(js_out, "rb") as f: js_digest = hashlib.sha3_224(f.read()).hexdigest()[0:8] logging.info(js_digest) if args.minify: - logging.info('Minifying website') + logging.info("Minifying website") with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] for root, _, filenames in os.walk(args.output_dir): for filename in filenames: path = os.path.join(root, filename) - futures.append(executor.submit(minify_file, path, css_digest, js_digest)) + futures.append( + executor.submit(minify_file, path, css_digest, js_digest) + ) for future in futures: exc = future.exception() if exc: @@ -296,24 +318,28 @@ def minify_website(args): def process_benchmark_results(args): - benchmark_root = os.path.join(args.website_dir, 'benchmark') + benchmark_root = os.path.join(args.website_dir, "benchmark") required_keys = { - 'dbms': ['result'], - 'hardware': ['result', 'system', 'system_full', 'kind'] + "dbms": ["result"], + "hardware": ["result", "system", "system_full", "kind"], } - for benchmark_kind in ['dbms', 'hardware']: + for benchmark_kind in ["dbms", "hardware"]: results = [] - results_root = os.path.join(benchmark_root, benchmark_kind, 'results') + results_root = os.path.join(benchmark_root, benchmark_kind, "results") for result in sorted(os.listdir(results_root)): result_file = os.path.join(results_root, result) - logging.debug(f'Reading benchmark result from {result_file}') - with open(result_file, 'r') as f: + logging.debug(f"Reading benchmark result from {result_file}") + with open(result_file, "r") as f: result = json.loads(f.read()) for item in result: for required_key in required_keys[benchmark_kind]: - assert required_key in item, f'No "{required_key}" in {result_file}' + assert ( + required_key in item + ), f'No "{required_key}" in {result_file}' results += result - results_js = os.path.join(args.output_dir, 'benchmark', benchmark_kind, 'results.js') - with open(results_js, 'w') as f: + results_js = os.path.join( + args.output_dir, "benchmark", benchmark_kind, "results.js" + ) + with open(results_js, "w") as f: data = json.dumps(results) - f.write(f'var results = {data};') + f.write(f"var results = {data};") diff --git a/docs/zh/changelog/index.md b/docs/zh/changelog/index.md index d36a676134e..306c72103fb 100644 --- a/docs/zh/changelog/index.md +++ b/docs/zh/changelog/index.md @@ -247,7 +247,7 @@ toc_title: "\u53D8\u66F4\u65E5\u5FD7" - 更新了clickhouse-test脚本中挂起查询的检查 [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([亚历山大\*卡扎科夫](https://github.com/Akazz)) - 从存储库中删除了一些无用的文件。 [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 更改类型的数学perftests从 `once` 到 `loop`. [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([尼古拉\*科切托夫](https://github.com/KochetovNicolai)) -- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse.com/codebrowser/html_report///ClickHouse/dbms/index.html) +- 添加码头镜像，它允许为我们的代码库构建交互式代码浏览器HTML报告。 [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([阿利沙平](https://github.com/alesapin)）见 [Woboq代码浏览器](https://clickhouse.com/codebrowser/ClickHouse/dbms/index.html) - 抑制MSan下的一些测试失败。 [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) - 加速 “exception while insert” 测试 此测试通常在具有复盖率的调试版本中超时。 [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) - 更新 `libcxx` 和 `libcxxabi` 为了主人 在准备 [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov)) diff --git a/docs/zh/development/browse-code.md b/docs/zh/development/browse-code.md index 9cee0a37444..f0ad6fd0984 100644 --- a/docs/zh/development/browse-code.md +++ b/docs/zh/development/browse-code.md @@ -5,7 +5,7 @@ toc_title: "\u6D4F\u89C8\u6E90\u4EE3\u7801" # 浏览ClickHouse源代码 {#browse-clickhouse-source-code} -您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/html_report/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。 +您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。 此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 6cff83067de..5bebb3aec2a 100644 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -42,18 +42,12 @@ git push 使用`utils/check-style/check-style`二进制文件执行一些简单的基于正则表达式的代码样式检查(注意, 它可以在本地运行). 如果失败, 按照[代码样式指南](./style.md)修复样式错误. +使用 [black](https://github.com/psf/black/) 檢查 python 代碼. + ### 报告详情 {#report-details} - [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) - `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt) -### PVS 检查 {#pvs-check} -使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告以查看确切的错误.如果可以则修复它们, 如果不行, 可以向ClickHouse的维护人员寻求帮忙. - -### 报告详情 {#report-details} -- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) -- `test_run.txt.out.log`包含构建和分析日志文件.它只包含解析或未找到的错误. -- `HTML report`包含分析结果.有关说明请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) - ## 快速测试 {#fast-test} 通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 查看报告以了解哪些测试失败, 然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败. @@ -144,12 +138,3 @@ git push ## 性能测试 {#performance-tests} 测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison#how-to-read-the-report)有详细描述. - -## 质量保证 {#qa} -什么是状态页面上的任务(专用网络)项目? - -它是 Yandex 内部工作系统的链接. Yandex 员工可以看到检查的开始时间及其更详细的状态. - -运行测试的地方 - -Yandex 内部基础设施的某个地方. diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md index 9389e8871c4..43b37fffa9f 100644 --- a/docs/zh/development/tests.md +++ b/docs/zh/development/tests.md @@ -254,7 +254,7 @@ Yandex安全团队的人员从安全的角度对ClickHouse的功能做了一些 ## 静态分析仪 {#static-analyzers} -我们在每次提交的基础上运行 `clang-tidy` 和 `PVS-Studio`. `clang-static-analyzer` 检查也被启用. `clang-tidy` 也用于一些样式检查. +我们在每次提交的基础上运行 `clang-tidy`. `clang-static-analyzer` 检查也被启用. `clang-tidy` 也用于一些样式检查. 我们已经评估了 `clang-tidy`、`Coverity`、`cppcheck`、`PVS-Studio`、`tscancode`、`CodeQL`. 您将在 `tests/instructions/` 目录中找到使用说明. 你也可以阅读[俄文文章](https://habr.com/company/yandex/blog/342018/). diff --git a/docs/zh/engines/table-engines/integrations/embedded-rocksdb.md b/docs/zh/engines/table-engines/integrations/embedded-rocksdb.md index 79ca8f0cd10..abb2af6332d 100644 --- a/docs/zh/engines/table-engines/integrations/embedded-rocksdb.md +++ b/docs/zh/engines/table-engines/integrations/embedded-rocksdb.md @@ -38,5 +38,46 @@ CREATE TABLE test ENGINE = EmbeddedRocksDB PRIMARY KEY key ``` +## 指标 + +还有一个`system.rocksdb` 表, 公开rocksdb的统计信息: + +```sql +SELECT + name, + value +FROM system.rocksdb + +┌─name──────────────────────┬─value─┐ +│ no.file.opens │ 1 │ +│ number.block.decompressed │ 1 │ +└───────────────────────────┴───────┘ +``` + +## 配置 + +你能修改任何[rocksdb options](https://github.com/facebook/rocksdb/wiki/Option-String-and-Option-Map) 配置，使用配置文件: + +```xml + + + 8 + + + 2 + + + + TABLE + + 8 + + + 2 + +
+ + +``` [原始文章](https://clickhouse.com/docs/en/engines/table-engines/integrations/embedded-rocksdb/) diff --git a/docs/zh/engines/table-engines/integrations/kafka.md b/docs/zh/engines/table-engines/integrations/kafka.md index ee6bbbe67fc..0a8fbdbf5ae 100644 --- a/docs/zh/engines/table-engines/integrations/kafka.md +++ b/docs/zh/engines/table-engines/integrations/kafka.md @@ -153,8 +153,9 @@ clickhouse也支持自己使用keyfile的方式来维护kerbros的凭证。配 - `_topic` – Kafka 主题。 - `_key` – 信息的键。 - `_offset` – 消息的偏移量。 -- `_timestamp ` – 消息的时间戳。 -- `_partition ` – Kafka 主题的分区。 +- `_timestamp` – 消息的时间戳。 +- `_timestamp_ms` – 消息的时间戳(毫秒)。 +- `_partition` – Kafka 主题的分区。 **另请参阅** diff --git a/docs/zh/engines/table-engines/integrations/sqlite.md b/docs/zh/engines/table-engines/integrations/sqlite.md deleted file mode 120000 index 56d0a490f52..00000000000 --- a/docs/zh/engines/table-engines/integrations/sqlite.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/engines/table-engines/integrations/sqlite.md \ No newline at end of file diff --git a/docs/zh/engines/table-engines/integrations/sqlite.md b/docs/zh/engines/table-engines/integrations/sqlite.md new file mode 100644 index 00000000000..f492148ed10 --- /dev/null +++ b/docs/zh/engines/table-engines/integrations/sqlite.md @@ -0,0 +1,59 @@ +--- +toc_priority: 7 +toc_title: SQLite +--- + +# SQLite {#sqlite} + +该引擎允许导入和导出数据到SQLite，并支持查询SQLite表直接从ClickHouse。 + +## 创建数据表 {#creating-a-table} + +``` sql + CREATE TABLE [IF NOT EXISTS] [db.]table_name + ( + name1 [type1], + name2 [type2], ... + ) ENGINE = SQLite('db_path', 'table') +``` + +**引擎参数** + +- `db_path` — SQLite数据库文件的具体路径地址。 +- `table` — SQLite数据库中的表名。 + +## 使用示例 {#usage-example} + +显示创建表的查询语句： + +```sql +SHOW CREATE TABLE sqlite_db.table2; +``` + +``` text +CREATE TABLE SQLite.table2 +( + `col1` Nullable(Int32), + `col2` Nullable(String) +) +ENGINE = SQLite('sqlite.db','table2'); +``` + +从数据表查询数据： + +``` sql +SELECT * FROM sqlite_db.table2 ORDER BY col1; +``` + +```text +┌─col1─┬─col2──┐ +│ 1 │ text1 │ +│ 2 │ text2 │ +│ 3 │ text3 │ +└──────┴───────┘ +``` + +**详见** + +- [SQLite](../../../engines/database-engines/sqlite.md) 引擎 +- [sqlite](../../../sql-reference/table-functions/sqlite.md) 表方法函数 \ No newline at end of file diff --git a/docs/zh/engines/table-engines/mergetree-family/mergetree.md b/docs/zh/engines/table-engines/mergetree-family/mergetree.md index b2e240f4f19..72db60deed1 100644 --- a/docs/zh/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/mergetree.md @@ -1,812 +1,821 @@ -# MergeTree {#table_engines-mergetree} - -Clickhouse 中最强大的表引擎当属 `MergeTree` （合并树）引擎及该系列（`*MergeTree`）中的其他引擎。 - -`MergeTree` 系列的引擎被设计用于插入极大量的数据到一张表当中。数据可以以数据片段的形式一个接着一个的快速写入，数据片段在后台按照一定的规则进行合并。相比在插入时不断修改（重写）已存储的数据，这种策略会高效很多。 - -主要特点: - -- 存储的数据按主键排序。 - - 这使得您能够创建一个小型的稀疏索引来加快数据检索。 - -- 如果指定了 [分区键](custom-partitioning-key.md) 的话，可以使用分区。 - - 在相同数据集和相同结果集的情况下 ClickHouse 中某些带分区的操作会比普通操作更快。查询中指定了分区键时 ClickHouse 会自动截取分区数据。这也有效增加了查询性能。 - -- 支持数据副本。 - - `ReplicatedMergeTree` 系列的表提供了数据副本功能。更多信息，请参阅 [数据副本](replication.md) 一节。 - -- 支持数据采样。 - - 需要的话，您可以给表设置一个采样方法。 - -!!! note "注意" - [合并](../special/merge.md#merge) 引擎并不属于 `*MergeTree` 系列。 - -## 建表 {#table_engine-mergetree-creating-a-table} - -``` sql -CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], - ... - INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1, - INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2 -) ENGINE = MergeTree() -ORDER BY expr -[PARTITION BY expr] -[PRIMARY KEY expr] -[SAMPLE BY expr] -[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...] -[SETTINGS name=value, ...] -``` - -对于以上参数的描述，可参考 [CREATE 语句 的描述](../../../engines/table-engines/mergetree-family/mergetree.md) 。 - - - -**子句** - -- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`. `MergeTree` 引擎没有参数。 - -- `ORDER BY` — 排序键。 - - 可以是一组列的元组或任意的表达式。 例如: `ORDER BY (CounterID, EventDate)` 。 - - 如果没有使用 `PRIMARY KEY` 显式指定的主键，ClickHouse 会使用排序键作为主键。 - - 如果不需要排序，可以使用 `ORDER BY tuple()`. 参考 [选择主键](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#selecting-the-primary-key) - -- `PARTITION BY` — [分区键](custom-partitioning-key.md) ，可选项。 - - 要按月分区，可以使用表达式 `toYYYYMM(date_column)` ，这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"` 。 - -- `PRIMARY KEY` - 如果要 [选择与排序键不同的主键](#choosing-a-primary-key-that-differs-from-the-sorting-key)，在这里指定，可选项。 - - 默认情况下主键跟排序键（由 `ORDER BY` 子句指定）相同。 - 因此，大部分情况下不需要再专门指定一个 `PRIMARY KEY` 子句。 - -- `SAMPLE BY` - 用于抽样的表达式，可选项。 - - 如果要用抽样表达式，主键中必须包含这个表达式。例如： - `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))` 。 - -- `TTL` - 指定行存储的持续时间并定义数据片段在硬盘和卷上的移动逻辑的规则列表，可选项。 - - 表达式中必须存在至少一个 `Date` 或 `DateTime` 类型的列，比如： - - `TTL date + INTERVAl 1 DAY` - - 规则的类型 `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'`指定了当满足条件（到达指定时间）时所要执行的动作：移除过期的行，还是将数据片段（如果数据片段中的所有行都满足表达式的话）移动到指定的磁盘（`TO DISK 'xxx'`) 或 卷（`TO VOLUME 'xxx'`）。默认的规则是移除（`DELETE`）。可以在列表中指定多个规则，但最多只能有一个`DELETE`的规则。 - - 更多细节，请查看 [表和列的 TTL](#table_engine-mergetree-ttl) - -- `SETTINGS` — 控制 `MergeTree` 行为的额外参数，可选项： - - - `index_granularity` — 索引粒度。索引中相邻的『标记』间的数据行数。默认值8192 。参考[数据存储](#mergetree-data-storage)。 - - `index_granularity_bytes` — 索引粒度，以字节为单位，默认值: 10Mb。如果想要仅按数据行数限制索引粒度, 请设置为0(不建议)。 - - `min_index_granularity_bytes` - 允许的最小数据粒度，默认值：1024b。该选项用于防止误操作，添加了一个非常低索引粒度的表。参考[数据存储](#mergetree-data-storage) - - `enable_mixed_granularity_parts` — 是否启用通过 `index_granularity_bytes` 控制索引粒度的大小。在19.11版本之前, 只有 `index_granularity` 配置能够用于限制索引粒度的大小。当从具有很大的行（几十上百兆字节）的表中查询数据时候，`index_granularity_bytes` 配置能够提升ClickHouse的性能。如果您的表里有很大的行，可以开启这项配置来提升`SELECT` 查询的性能。 - - `use_minimalistic_part_header_in_zookeeper` — ZooKeeper中数据片段存储方式 。如果`use_minimalistic_part_header_in_zookeeper=1` ，ZooKeeper 会存储更少的数据。更多信息参考[服务配置参数]([Server Settings | ClickHouse Documentation](https://clickhouse.com/docs/zh/operations/server-configuration-parameters/settings/))这章中的 [设置描述](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) 。 - - `min_merge_bytes_to_use_direct_io` — 使用直接 I/O 来操作磁盘的合并操作时要求的最小数据量。合并数据片段时，ClickHouse 会计算要被合并的所有数据的总存储空间。如果大小超过了 `min_merge_bytes_to_use_direct_io` 设置的字节数，则 ClickHouse 将使用直接 I/O 接口（`O_DIRECT` 选项）对磁盘读写。如果设置 `min_merge_bytes_to_use_direct_io = 0` ，则会禁用直接 I/O。默认值：`10 * 1024 * 1024 * 1024` 字节。 - - - `merge_with_ttl_timeout` — TTL合并频率的最小间隔时间，单位：秒。默认值: 86400 (1 天)。 - - `write_final_mark` — 是否启用在数据片段尾部写入最终索引标记。默认值: 1（不要关闭）。 - - `merge_max_block_size` — 在块中进行合并操作时的最大行数限制。默认值：8192 - - `storage_policy` — 存储策略。 参见 [使用具有多个块的设备进行数据存储](#table_engine-mergetree-multiple-volumes). - - `min_bytes_for_wide_part`,`min_rows_for_wide_part` 在数据片段中可以使用`Wide`格式进行存储的最小字节数/行数。您可以不设置、只设置一个，或全都设置。参考：[数据存储](#mergetree-data-storage) - - `max_parts_in_total` - 所有分区中最大块的数量(意义不明) - - `max_compress_block_size` - 在数据压缩写入表前，未压缩数据块的最大大小。您可以在全局设置中设置该值(参见[max_compress_block_size](https://clickhouse.com/docs/zh/operations/settings/settings/#max-compress-block-size))。建表时指定该值会覆盖全局设置。 - - `min_compress_block_size` - 在数据压缩写入表前，未压缩数据块的最小大小。您可以在全局设置中设置该值(参见[min_compress_block_size](https://clickhouse.com/docs/zh/operations/settings/settings/#min-compress-block-size))。建表时指定该值会覆盖全局设置。 - - `max_partitions_to_read` - 一次查询中可访问的分区最大数。您可以在全局设置中设置该值(参见[max_partitions_to_read](https://clickhouse.com/docs/zh/operations/settings/settings/#max_partitions_to_read))。 - -**示例配置** - -``` sql -ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192 -``` - -在这个例子中，我们设置了按月进行分区。 - -同时我们设置了一个按用户 ID 哈希的抽样表达式。这使得您可以对该表中每个 `CounterID` 和 `EventDate` 的数据伪随机分布。如果您在查询时指定了 [SAMPLE](../../../engines/table-engines/mergetree-family/mergetree.md#select-sample-clause) 子句。 ClickHouse会返回对于用户子集的一个均匀的伪随机数据采样。 - -`index_granularity` 可省略因为 8192 是默认设置 。 - -
-已弃用的建表方法 - -!!! attention "注意" - 不要在新版项目中使用该方法，可能的话，请将旧项目切换到上述方法。 - - CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - ( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], - ... - ) ENGINE [=] MergeTree(date-column [, sampling_expression], (primary, key), index_granularity) - -**MergeTree() 参数** - -- `date-column` — 类型为 [日期](../../../engines/table-engines/mergetree-family/mergetree.md) 的列名。ClickHouse 会自动依据这个列按月创建分区。分区名格式为 `"YYYYMM"` 。 -- `sampling_expression` — 采样表达式。 -- `(primary, key)` — 主键。类型 — [元组()](../../../engines/table-engines/mergetree-family/mergetree.md) -- `index_granularity` — 索引粒度。即索引中相邻『标记』间的数据行数。设为 8192 可以适用大部分场景。 - -**示例** - - MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) - -对于主要的配置方法，这里 `MergeTree` 引擎跟前面的例子一样，可以以同样的方式配置。 -
- -## 数据存储 {#mergetree-data-storage} - -表由按主键排序的数据片段（DATA PART）组成。 - -当数据被插入到表中时，会创建多个数据片段并按主键的字典序排序。例如，主键是 `(CounterID, Date)` 时，片段中数据首先按 `CounterID` 排序，具有相同 `CounterID` 的部分按 `Date` 排序。 - -不同分区的数据会被分成不同的片段，ClickHouse 在后台合并数据片段以便更高效存储。不同分区的数据片段不会进行合并。合并机制并不保证具有相同主键的行全都合并到同一个数据片段中。 - -数据片段可以以 `Wide` 或 `Compact` 格式存储。在 `Wide` 格式下，每一列都会在文件系统中存储为单独的文件，在 `Compact` 格式下所有列都存储在一个文件中。`Compact` 格式可以提高插入量少插入频率频繁时的性能。 - -数据存储格式由 `min_bytes_for_wide_part` 和 `min_rows_for_wide_part` 表引擎参数控制。如果数据片段中的字节数或行数少于相应的设置值，数据片段会以 `Compact` 格式存储，否则会以 `Wide` 格式存储。 - -每个数据片段被逻辑的分割成颗粒（granules）。颗粒是 ClickHouse 中进行数据查询时的最小不可分割数据集。ClickHouse 不会对行或值进行拆分，所以每个颗粒总是包含整数个行。每个颗粒的第一行通过该行的主键值进行标记， -ClickHouse 会为每个数据片段创建一个索引文件来存储这些标记。对于每列，无论它是否包含在主键当中，ClickHouse 都会存储类似标记。这些标记让您可以在列文件中直接找到数据。 - -颗粒的大小通过表引擎参数 `index_granularity` 和 `index_granularity_bytes` 控制。颗粒的行数的在 `[1, index_granularity]` 范围中，这取决于行的大小。如果单行的大小超过了 `index_granularity_bytes` 设置的值，那么一个颗粒的大小会超过 `index_granularity_bytes`。在这种情况下，颗粒的大小等于该行的大小。 - -## 主键和索引在查询中的表现 {#primary-keys-and-indexes-in-queries} - -我们以 `(CounterID, Date)` 以主键。排序好的索引的图示会是下面这样： - -``` text - 全部数据 : [-------------------------------------------------------------------------] - CounterID: [aaaaaaaaaaaaaaaaaabbbbcdeeeeeeeeeeeeefgggggggghhhhhhhhhiiiiiiiiikllllllll] - Date: [1111111222222233331233211111222222333211111112122222223111112223311122333] - 标记: | | | | | | | | | | | - a,1 a,2 a,3 b,3 e,2 e,3 g,1 h,2 i,1 i,3 l,3 - 标记号: 0 1 2 3 4 5 6 7 8 9 10 -``` - -如果指定查询如下： - -- `CounterID in ('a', 'h')`，服务器会读取标记号在 `[0, 3)` 和 `[6, 8)` 区间中的数据。 -- `CounterID IN ('a', 'h') AND Date = 3`，服务器会读取标记号在 `[1, 3)` 和 `[7, 8)` 区间中的数据。 -- `Date = 3`，服务器会读取标记号在 `[1, 10]` 区间中的数据。 - -上面例子可以看出使用索引通常会比全表描述要高效。 - -稀疏索引会引起额外的数据读取。当读取主键单个区间范围的数据时，每个数据块中最多会多读 `index_granularity * 2` 行额外的数据。 - -稀疏索引使得您可以处理极大量的行，因为大多数情况下，这些索引常驻于内存。 - -ClickHouse 不要求主键唯一，所以您可以插入多条具有相同主键的行。 - -您可以在`PRIMARY KEY`与`ORDER BY`条件中使用`可为空的`类型的表达式，但强烈建议不要这么做。为了启用这项功能，请打开[allow_nullable_key](https://clickhouse.com/docs/zh/operations/settings/settings/#allow-nullable-key)，[NULLS_LAST](https://clickhouse.com/docs/zh/sql-reference/statements/select/order-by/#sorting-of-special-values)规则也适用于`ORDER BY`条件中有NULL值的情况下。 - -### 主键的选择 {#zhu-jian-de-xuan-ze} - -主键中列的数量并没有明确的限制。依据数据结构，您可以在主键包含多些或少些列。这样可以： - -- 改善索引的性能。 - - 如果当前主键是 `(a, b)` ，在下列情况下添加另一个 `c` 列会提升性能： - - - 查询会使用 `c` 列作为条件 - - 很长的数据范围（ `index_granularity` 的数倍）里 `(a, b)` 都是相同的值，并且这样的情况很普遍。换言之，就是加入另一列后，可以让您的查询略过很长的数据范围。 - -- 改善数据压缩。 - - ClickHouse 以主键排序片段数据，所以，数据的一致性越高，压缩越好。 - -- 在[CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) 和 [SummingMergeTree](summingmergetree.md) 引擎里进行数据合并时会提供额外的处理逻辑。 - - 在这种情况下，指定与主键不同的 *排序键* 也是有意义的。 - -长的主键会对插入性能和内存消耗有负面影响，但主键中额外的列并不影响 `SELECT` 查询的性能。 - -可以使用 `ORDER BY tuple()` 语法创建没有主键的表。在这种情况下 ClickHouse 根据数据插入的顺序存储。如果在使用 `INSERT ... SELECT` 时希望保持数据的排序，请设置 [max_insert_threads = 1](../../../operations/settings/settings.md#settings-max-insert-threads)。 - -想要根据初始顺序进行数据查询，使用 [单线程查询](../../../operations/settings/settings.md#settings-max_threads) - -### 选择与排序键不同的主键 {#choosing-a-primary-key-that-differs-from-the-sorting-key} - -Clickhouse可以做到指定一个跟排序键不一样的主键，此时排序键用于在数据片段中进行排序，主键用于在索引文件中进行标记的写入。这种情况下，主键表达式元组必须是排序键表达式元组的前缀(即主键为(a,b)，排序列必须为(a,b,******))。 - -当使用 [SummingMergeTree](summingmergetree.md) 和 [AggregatingMergeTree](aggregatingmergetree.md) 引擎时，这个特性非常有用。通常在使用这类引擎时，表里的列分两种：*维度* 和 *度量* 。典型的查询会通过任意的 `GROUP BY` 对度量列进行聚合并通过维度列进行过滤。由于 SummingMergeTree 和 AggregatingMergeTree 会对排序键相同的行进行聚合，所以把所有的维度放进排序键是很自然的做法。但这将导致排序键中包含大量的列，并且排序键会伴随着新添加的维度不断的更新。 - -在这种情况下合理的做法是，只保留少量的列在主键当中用于提升扫描效率，将维度列添加到排序键中。 - -对排序键进行 [ALTER](../../../sql-reference/statements/alter.md) 是轻量级的操作，因为当一个新列同时被加入到表里和排序键里时，已存在的数据片段并不需要修改。由于旧的排序键是新排序键的前缀，并且新添加的列中没有数据，因此在表修改时的数据对于新旧的排序键来说都是有序的。 - -### 索引和分区在查询中的应用 {#use-of-indexes-and-partitions-in-queries} - -对于 `SELECT` 查询，ClickHouse 分析是否可以使用索引。如果 `WHERE/PREWHERE` 子句具有下面这些表达式（作为完整WHERE条件的一部分或全部）则可以使用索引：进行相等/不相等的比较；对主键列或分区列进行`IN`运算、有固定前缀的`LIKE`运算(如name like 'test%')、函数运算(部分函数适用)，还有对上述表达式进行逻辑运算。 - - - - - - -因此，在索引键的一个或多个区间上快速地执行查询是可能的。下面例子中，指定标签；指定标签和日期范围；指定标签和日期；指定多个标签和日期范围等执行查询，都会非常快。 - -当引擎配置如下时： - -``` sql - ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate) SETTINGS index_granularity=8192 -``` - -这种情况下，这些查询： - -``` sql -SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 -SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) -SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) -``` - -ClickHouse 会依据主键索引剪掉不符合的数据，依据按月分区的分区键剪掉那些不包含符合数据的分区。 - -上文的查询显示，即使索引用于复杂表达式，因为读表操作经过优化，所以使用索引不会比完整扫描慢。 - -下面这个例子中，不会使用索引。 - -``` sql -SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' -``` - -要检查 ClickHouse 执行一个查询时能否使用索引，可设置 [force_index_by_date](../../../operations/settings/settings.md#settings-force_index_by_date) 和 [force_primary_key](../../../operations/settings/settings.md) 。 - -使用按月分区的分区列允许只读取包含适当日期区间的数据块，这种情况下，数据块会包含很多天（最多整月）的数据。在块中，数据按主键排序，主键第一列可能不包含日期。因此，仅使用日期而没有用主键字段作为条件的查询将会导致需要读取超过这个指定日期以外的数据。 - -### 部分单调主键的使用 - -考虑这样的场景，比如一个月中的天数。它们在一个月的范围内形成一个[单调序列](https://zh.wikipedia.org/wiki/单调函数) ，但如果扩展到更大的时间范围它们就不再单调了。这就是一个部分单调序列。如果用户使用部分单调的主键创建表，ClickHouse同样会创建一个稀疏索引。当用户从这类表中查询数据时，ClickHouse 会对查询条件进行分析。如果用户希望获取两个索引标记之间的数据并且这两个标记在一个月以内，ClickHouse 可以在这种特殊情况下使用到索引，因为它可以计算出查询参数与索引标记之间的距离。 - -如果查询参数范围内的主键不是单调序列，那么 ClickHouse 无法使用索引。在这种情况下，ClickHouse 会进行全表扫描。 - -ClickHouse 在任何主键代表一个部分单调序列的情况下都会使用这个逻辑。 - -### 跳数索引 {#tiao-shu-suo-yin-fen-duan-hui-zong-suo-yin-shi-yan-xing-de} - -此索引在 `CREATE` 语句的列部分里定义。 - -``` sql -INDEX index_name expr TYPE type(...) GRANULARITY granularity_value -``` - -`*MergeTree` 系列的表可以指定跳数索引。 -跳数索引是指数据片段按照粒度(建表时指定的`index_granularity`)分割成小块后，将上述SQL的granularity_value数量的小块组合成一个大的块，对这些大块写入索引信息，这样有助于使用`where`筛选时跳过大量不必要的数据，减少`SELECT`需要读取的数据量。 - -**示例** - -``` sql -CREATE TABLE table_name -( - u64 UInt64, - i32 Int32, - s String, - ... - INDEX a (u64 * i32, s) TYPE minmax GRANULARITY 3, - INDEX b (u64 * length(s)) TYPE set(1000) GRANULARITY 4 -) ENGINE = MergeTree() -... -``` - -上例中的索引能让 ClickHouse 执行下面这些查询时减少读取数据量。 - -``` sql -SELECT count() FROM table WHERE s < 'z' -SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 -``` - -#### 可用的索引类型 {#table_engine-mergetree-data_skipping-indexes} - -- `minmax` - 存储指定表达式的极值（如果表达式是 `tuple` ，则存储 `tuple` 中每个元素的极值），这些信息用于跳过数据块，类似主键。 - -- `set(max_rows)` - 存储指定表达式的不重复值（不超过 `max_rows` 个，`max_rows=0` 则表示『无限制』）。这些信息可用于检查数据块是否满足 `WHERE` 条件。 - -- `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - 存储一个包含数据块中所有 n元短语（ngram） 的 [布隆过滤器](https://en.wikipedia.org/wiki/Bloom_filter) 。只可用在字符串上。 - 可用于优化 `equals` ， `like` 和 `in` 表达式的性能。 - - `n` – 短语长度。 - - `size_of_bloom_filter_in_bytes` – 布隆过滤器大小，字节为单位。（因为压缩得好，可以指定比较大的值，如 256 或 512）。 - - `number_of_hash_functions` – 布隆过滤器中使用的哈希函数的个数。 - - `random_seed` – 哈希函数的随机种子。 - -- `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` - 跟 `ngrambf_v1` 类似，但是存储的是token而不是ngrams。Token是由非字母数字的符号分割的序列。 - -- `bloom_filter(bloom_filter([false_positive])` – 为指定的列存储布隆过滤器 - - 可选参数`false_positive`用来指定从布隆过滤器收到错误响应的几率。取值范围是 (0,1)，默认值：0.025 - - 支持的数据类型：`Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`。 - - 以下函数会用到这个索引： [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [has](../../../sql-reference/functions/array-functions.md) - -``` sql -INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 -INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARITY 4 -INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 -``` - -#### 函数支持 {#functions-support} - -WHERE 子句中的条件可以包含对某列数据进行运算的函数表达式，如果列是索引的一部分，ClickHouse会在执行函数时尝试使用索引。不同的函数对索引的支持是不同的。 - -`set` 索引会对所有函数生效，其他索引对函数的生效情况见下表 - -| 函数 (操作符) / 索引 | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | -| ------------------------------------------------------------ | ----------- | ------ | ---------- | ---------- | ------------ | -| [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [like](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | -| [endsWith](../../../sql-reference/functions/string-functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | -| [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | -| [in](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [empty](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | -| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | - -常量参数小于 ngram 大小的函数不能使用 `ngrambf_v1` 进行查询优化。 - -!!! note "注意" -布隆过滤器可能会包含不符合条件的匹配，所以 `ngrambf_v1`, `tokenbf_v1` 和 `bloom_filter` 索引不能用于结果返回为假的函数，例如： - -- 可以用来优化的场景 - - `s LIKE '%test%'` - - `NOT s NOT LIKE '%test%'` - - `s = 1` - - `NOT s != 1` - - `startsWith(s, 'test')` -- 不能用来优化的场景 - - `NOT s LIKE '%test%'` - - `s NOT LIKE '%test%'` - - `NOT s = 1` - - `s != 1` - - `NOT startsWith(s, 'test')` - -## 并发数据访问 {#concurrent-data-access} - -对于表的并发访问，我们使用多版本机制。换言之，当一张表同时被读和更新时，数据从当前查询到的一组片段中读取。没有冗长的的锁。插入不会阻碍读取。 - -对表的读操作是自动并行的。 - -## 列和表的 TTL {#table_engine-mergetree-ttl} - -TTL用于设置值的生命周期，它既可以为整张表设置，也可以为每个列字段单独设置。表级别的 TTL 还会指定数据在磁盘和卷上自动转移的逻辑。 - -TTL 表达式的计算结果必须是 [日期](../../../engines/table-engines/mergetree-family/mergetree.md) 或 [日期时间](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的字段。 - -示例： - -``` sql -TTL time_column -TTL time_column + interval -``` - -要定义`interval`, 需要使用 [时间间隔](../../../engines/table-engines/mergetree-family/mergetree.md#operators-datetime) 操作符。 - -``` sql -TTL date_time + INTERVAL 1 MONTH -TTL date_time + INTERVAL 15 HOUR -``` - -### 列 TTL {#mergetree-column-ttl} - -当列中的值过期时, ClickHouse会将它们替换成该列数据类型的默认值。如果数据片段中列的所有值均已过期，则ClickHouse 会从文件系统中的数据片段中删除此列。 - -`TTL`子句不能被用于主键字段。 - -**示例:** - -创建表时指定 `TTL` - -``` sql -CREATE TABLE example_table -( - d DateTime, - a Int TTL d + INTERVAL 1 MONTH, - b Int TTL d + INTERVAL 1 MONTH, - c String -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(d) -ORDER BY d; -``` - -为表中已存在的列字段添加 `TTL` - -``` sql -ALTER TABLE example_table - MODIFY COLUMN - c String TTL d + INTERVAL 1 DAY; -``` - -修改列字段的 `TTL` - -``` sql -ALTER TABLE example_table - MODIFY COLUMN - c String TTL d + INTERVAL 1 MONTH; -``` - -### 表 TTL {#mergetree-table-ttl} - -表可以设置一个用于移除过期行的表达式，以及多个用于在磁盘或卷上自动转移数据片段的表达式。当表中的行过期时，ClickHouse 会删除所有对应的行。对于数据片段的转移特性，必须所有的行都满足转移条件。 - -``` sql -TTL expr - [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'][, DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'] ... - [WHERE conditions] - [GROUP BY key_expr [SET v1 = aggr_func(v1) [, v2 = aggr_func(v2) ...]] ] - -``` - -TTL 规则的类型紧跟在每个 TTL 表达式后面，它会影响满足表达式时（到达指定时间时）应当执行的操作： - -- `DELETE` - 删除过期的行（默认操作）; -- `TO DISK 'aaa'` - 将数据片段移动到磁盘 `aaa`; -- `TO VOLUME 'bbb'` - 将数据片段移动到卷 `bbb`. -- `GROUP BY` - 聚合过期的行 - -使用`WHERE`从句，您可以指定哪些过期的行会被删除或聚合(不适用于移动)。`GROUP BY`表达式必须是表主键的前缀。如果某列不是`GROUP BY`表达式的一部分，也没有在SET从句显示引用，结果行中相应列的值是随机的(就好像使用了`any`函数)。 - -**示例**: - -创建时指定 TTL - -``` sql -CREATE TABLE example_table -( - d DateTime, - a Int -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(d) -ORDER BY d -TTL d + INTERVAL 1 MONTH [DELETE], - d + INTERVAL 1 WEEK TO VOLUME 'aaa', - d + INTERVAL 2 WEEK TO DISK 'bbb'; -``` - -修改表的 `TTL` - -``` sql -ALTER TABLE example_table - MODIFY TTL d + INTERVAL 1 DAY; -``` - -创建一张表，设置一个月后数据过期，这些过期的行中日期为星期一的删除： - -``` sql -CREATE TABLE table_with_where -( - d DateTime, - a Int -) -ENGINE = MergeTree -PARTITION BY toYYYYMM(d) -ORDER BY d -TTL d + INTERVAL 1 MONTH DELETE WHERE toDayOfWeek(d) = 1; -``` - -创建一张表，设置过期的列会被聚合。列`x`包含每组行中的最大值，`y`为最小值，`d`为可能任意值。 - -``` sql -CREATE TABLE table_for_aggregation -( - d DateTime, - k1 Int, - k2 Int, - x Int, - y Int -) -ENGINE = MergeTree -ORDER BY (k1, k2) -TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); -``` - -**删除数据** - -ClickHouse 在数据片段合并时会删除掉过期的数据。 - -当ClickHouse发现数据过期时, 它将会执行一个计划外的合并。要控制这类合并的频率, 您可以设置 `merge_with_ttl_timeout`。如果该值被设置的太低, 它将引发大量计划外的合并，这可能会消耗大量资源。 - -如果在合并的过程中执行 `SELECT` 查询, 则可能会得到过期的数据。为了避免这种情况，可以在 `SELECT` 之前使用 [OPTIMIZE](../../../engines/table-engines/mergetree-family/mergetree.md#misc_operations-optimize) 。 - -## 使用多个块设备进行数据存储 {#table_engine-mergetree-multiple-volumes} - -### 介绍 {#introduction} - -MergeTree 系列表引擎可以将数据存储在多个块设备上。这对某些可以潜在被划分为“冷”“热”的表来说是很有用的。最新数据被定期的查询但只需要很小的空间。相反，详尽的历史数据很少被用到。如果有多块磁盘可用，那么“热”的数据可以放置在快速的磁盘上（比如 NVMe 固态硬盘或内存），“冷”的数据可以放在相对较慢的磁盘上（比如机械硬盘）。 - -数据片段是 `MergeTree` 引擎表的最小可移动单元。属于同一个数据片段的数据被存储在同一块磁盘上。数据片段会在后台自动的在磁盘间移动，也可以通过 [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) 查询来移动。 - -### 术语 {#terms} - -- 磁盘 — 挂载到文件系统的块设备 -- 默认磁盘 — 在服务器设置中通过 [path](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-path) 参数指定的数据存储 -- 卷 — 相同磁盘的顺序列表 （类似于 [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)） -- 存储策略 — 卷的集合及他们之间的数据移动规则 - - 以上名称的信息在Clickhouse中系统表[system.storage_policies](https://clickhouse.com/docs/zh/operations/system-tables/storage_policies/#system_tables-storage_policies)和[system.disks](https://clickhouse.com/docs/zh/operations/system-tables/disks/#system_tables-disks)体现。为了应用存储策略，可以在建表时使用`storage_policy`设置。 - -### 配置 {#table_engine-mergetree-multiple-volumes_configure} - -磁盘、卷和存储策略应当在主配置文件 `config.xml` 或 `config.d` 目录中的独立文件中的 `` 标签内定义。 - -配置结构： - -``` xml - - - - /mnt/fast_ssd/clickhouse/ - - - /mnt/hdd1/clickhouse/ - 10485760 - - - /mnt/hdd2/clickhouse/ - 10485760 - - - ... - - - ... - -``` - -标签： - -- `` — 磁盘名，名称必须与其他磁盘不同. -- `path` — 服务器将用来存储数据 (`data` 和 `shadow` 目录) 的路径, 应当以 ‘/’ 结尾. -- `keep_free_space_bytes` — 需要保留的剩余磁盘空间. - -磁盘定义的顺序无关紧要。 - -存储策略配置： - -``` xml - - ... - - - - - disk_name_from_disks_configuration - 1073741824 - - - - - - - 0.2 - - - - - - - - ... - -``` - -标签： - -- `policy_name_N` — 策略名称，不能重复。 -- `volume_name_N` — 卷名称，不能重复。 -- `disk` — 卷中的磁盘。 -- `max_data_part_size_bytes` — 卷中的磁盘可以存储的数据片段的最大大小。 -- `move_factor` — 当可用空间少于这个因子时，数据将自动的向下一个卷（如果有的话）移动 (默认值为 0.1)。 -- `prefer_not_to_merge` - 禁止在这个卷中进行数据合并。该选项启用时，对该卷的数据不能进行合并。这个选项主要用于慢速磁盘。 - -配置示例： - -``` xml - - ... - - - - - disk1 - disk2 - - - - - - - - fast_ssd - 1073741824 - - - disk1 - - - 0.2 - - - - -
- jbod1 -
- - external - true - - - - - ... - -``` - -在给出的例子中， `hdd_in_order` 策略实现了 [循环制](https://zh.wikipedia.org/wiki/循环制) 方法。因此这个策略只定义了一个卷（`single`），数据片段会以循环的顺序全部存储到它的磁盘上。当有多个类似的磁盘挂载到系统上，但没有配置 RAID 时，这种策略非常有用。请注意一个每个独立的磁盘驱动都并不可靠，您可能需要用3份或更多的复制份数来补偿它。 - -如果在系统中有不同类型的磁盘可用，可以使用 `moving_from_ssd_to_hdd`。`hot` 卷由 SSD 磁盘（`fast_ssd`）组成，这个卷上可以存储的数据片段的最大大小为 1GB。所有大于 1GB 的数据片段都会被直接存储到 `cold` 卷上，`cold` 卷包含一个名为 `disk1` 的 HDD 磁盘。 -同样，一旦 `fast_ssd` 被填充超过 80%，数据会通过后台进程向 `disk1` 进行转移。 - -存储策略中卷的枚举顺序是很重要的。因为当一个卷被充满时，数据会向下一个卷转移。磁盘的枚举顺序同样重要，因为数据是依次存储在磁盘上的。 - -在创建表时，可以应用存储策略： - -``` sql -CREATE TABLE table_with_non_default_policy ( - EventDate Date, - OrderID UInt64, - BannerID UInt64, - SearchPhrase String -) ENGINE = MergeTree -ORDER BY (OrderID, BannerID) -PARTITION BY toYYYYMM(EventDate) -SETTINGS storage_policy = 'moving_from_ssd_to_hdd' -``` - -`default` 存储策略意味着只使用一个卷，这个卷只包含一个在 `` 中定义的磁盘。您可以使用[ALTER TABLE ... MODIFY SETTING]来修改存储策略，新的存储策略应该包含所有以前的磁盘和卷，并使用相同的名称。 - -可以通过 [background_move_pool_size](../../../operations/settings/settings.md#background_move_pool_size) 设置调整执行后台任务的线程数。 - -### 详细说明 {#details} - -对于 `MergeTree` 表，数据通过以下不同的方式写入到磁盘当中： - -- 插入（`INSERT`查询） -- 后台合并和[数据变异](../../../sql-reference/statements/alter.md#alter-mutations) -- 从另一个副本下载 -- [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition) 冻结分区 - -除了数据变异和冻结分区以外的情况下，数据按照以下逻辑存储到卷或磁盘上： - -1. 首个卷（按定义顺序）拥有足够的磁盘空间存储数据片段（`unreserved_space > current_part_size`）并且允许存储给定数据片段的大小（`max_data_part_size_bytes > current_part_size`） -2. 在这个数据卷内，紧挨着先前存储数据的那块磁盘之后的磁盘，拥有比数据片段大的剩余空间。（`unreserved_space - keep_free_space_bytes > current_part_size`） - -更进一步，数据变异和分区冻结使用的是 [硬链接](https://en.wikipedia.org/wiki/Hard_link)。不同磁盘之间的硬链接是不支持的，所以在这种情况下数据片段都会被存储到原来的那一块磁盘上。 - -在后台，数据片段基于剩余空间（`move_factor`参数）根据卷在配置文件中定义的顺序进行转移。数据永远不会从最后一个移出也不会从第一个移入。可以通过系统表 [system.part_log](../../../operations/system-tables/part_log.md#system_tables-part-log) (字段 `type = MOVE_PART`) 和 [system.parts](../../../operations/system-tables/parts.md#system_tables-parts) (字段 `path` 和 `disk`) 来监控后台的移动情况。具体细节可以通过服务器日志查看。 - -用户可以通过 [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition) 强制移动一个数据片段或分区到另外一个卷，所有后台移动的限制都会被考虑在内。这个查询会自行启动，无需等待后台操作完成。如果没有足够的可用空间或任何必须条件没有被满足，用户会收到报错信息。 - -数据移动不会妨碍到数据复制。也就是说，同一张表的不同副本可以指定不同的存储策略。 - -在后台合并和数据变异之后，旧的数据片段会在一定时间后被移除 (`old_parts_lifetime`)。在这期间，他们不能被移动到其他的卷或磁盘。也就是说，直到数据片段被完全移除，它们仍然会被磁盘占用空间计算在内。 - -## 使用S3进行数据存储 {#using-s3-data-storage} - -`MergeTree`系列表引擎允许使用[S3](https://aws.amazon.com/s3/)存储数据，需要修改磁盘类型为`S3`。 - -示例配置： - -``` xml - - ... - - - s3 - https://storage.yandexcloud.net/my-bucket/root-path/ - your_access_key_id - your_secret_access_key - - your_base64_encoded_customer_key - - http://proxy1 - http://proxy2 - - 10000 - 5000 - 10 - 4 - 1000 - /var/lib/clickhouse/disks/s3/ - true - /var/lib/clickhouse/disks/s3/cache/ - false - - - ... - -``` - -必须的参数： - -- `endpoint` - S3的结点URL，以`path`或`virtual hosted`[格式](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html)书写。 -- `access_key_id` - S3的Access Key ID。 -- `secret_access_key` - S3的Secret Access Key。 - -可选参数： - -- `region` - S3的区域名称 -- `use_environment_credentials` - 从环境变量AWS_ACCESS_KEY_ID、AWS_SECRET_ACCESS_KEY和AWS_SESSION_TOKEN中读取认证参数。默认值为`false`。 -- `use_insecure_imds_request` - 如果设置为`true`，S3客户端在认证时会使用不安全的IMDS请求。默认值为`false`。 -- `proxy` - 访问S3结点URL时代理设置。每一个`uri`项的值都应该是合法的代理URL。 -- `connect_timeout_ms` - Socket连接超时时间，默认值为`10000`，即10秒。 -- `request_timeout_ms` - 请求超时时间，默认值为`5000`，即5秒。 -- `retry_attempts` - 请求失败后的重试次数，默认值为10。 -- `single_read_retries` - 读过程中连接丢失后重试次数，默认值为4。 -- `min_bytes_for_seek` - 使用查找操作，而不是顺序读操作的最小字节数，默认值为1000。 -- `metadata_path` - 本地存放S3元数据文件的路径，默认值为`/var/lib/clickhouse/disks//` -- `cache_enabled` - 是否允许缓存标记和索引文件。默认值为`true`。 -- `cache_path` - 本地缓存标记和索引文件的路径。默认值为`/var/lib/clickhouse/disks//cache/`。 -- `skip_access_check` - 如果为`true`，Clickhouse启动时不检查磁盘是否可用。默认为`false`。 -- `server_side_encryption_customer_key_base64` - 如果指定该项的值，请求时会加上为了访问SSE-C加密数据而必须的头信息。 - -S3磁盘也可以设置冷热存储： -```xml - - ... - - - s3 - https://storage.yandexcloud.net/my-bucket/root-path/ - your_access_key_id - your_secret_access_key - - - - - -
- s3 -
- - - - -
- default -
- - s3 - - - 0.2 - - - ... - -``` - -指定了`cold`选项后，本地磁盘剩余空间如果小于`move_factor * disk_size`，或有TTL设置时，数据就会定时迁移至S3了。 - -[原始文章](https://clickhouse.com/docs/en/operations/table_engines/mergetree/) +# MergeTree {#table_engines-mergetree} + +Clickhouse 中最强大的表引擎当属 `MergeTree` （合并树）引擎及该系列（`*MergeTree`）中的其他引擎。 + +`MergeTree` 系列的引擎被设计用于插入极大量的数据到一张表当中。数据可以以数据片段的形式一个接着一个的快速写入，数据片段在后台按照一定的规则进行合并。相比在插入时不断修改（重写）已存储的数据，这种策略会高效很多。 + +主要特点: + +- 存储的数据按主键排序。 + + 这使得您能够创建一个小型的稀疏索引来加快数据检索。 + +- 如果指定了 [分区键](custom-partitioning-key.md) 的话，可以使用分区。 + + 在相同数据集和相同结果集的情况下 ClickHouse 中某些带分区的操作会比普通操作更快。查询中指定了分区键时 ClickHouse 会自动截取分区数据。这也有效增加了查询性能。 + +- 支持数据副本。 + + `ReplicatedMergeTree` 系列的表提供了数据副本功能。更多信息，请参阅 [数据副本](replication.md) 一节。 + +- 支持数据采样。 + + 需要的话，您可以给表设置一个采样方法。 + +!!! note "注意" + [合并](../special/merge.md#merge) 引擎并不属于 `*MergeTree` 系列。 + +## 建表 {#table_engine-mergetree-creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], + ... + INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1, + INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2 +) ENGINE = MergeTree() +ORDER BY expr +[PARTITION BY expr] +[PRIMARY KEY expr] +[SAMPLE BY expr] +[TTL expr [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'], ...] +[SETTINGS name=value, ...] +``` + +对于以上参数的描述，可参考 [CREATE 语句 的描述](../../../engines/table-engines/mergetree-family/mergetree.md) 。 + + + +**子句** + +- `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`. `MergeTree` 引擎没有参数。 + +- `ORDER BY` — 排序键。 + + 可以是一组列的元组或任意的表达式。 例如: `ORDER BY (CounterID, EventDate)` 。 + + 如果没有使用 `PRIMARY KEY` 显式指定的主键，ClickHouse 会使用排序键作为主键。 + + 如果不需要排序，可以使用 `ORDER BY tuple()`. 参考 [选择主键](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#selecting-the-primary-key) + +- `PARTITION BY` — [分区键](custom-partitioning-key.md) ，可选项。 + + 要按月分区，可以使用表达式 `toYYYYMM(date_column)` ，这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"` 。 + +- `PRIMARY KEY` - 如果要 [选择与排序键不同的主键](#choosing-a-primary-key-that-differs-from-the-sorting-key)，在这里指定，可选项。 + + 默认情况下主键跟排序键（由 `ORDER BY` 子句指定）相同。 + 因此，大部分情况下不需要再专门指定一个 `PRIMARY KEY` 子句。 + +- `SAMPLE BY` - 用于抽样的表达式，可选项。 + + 如果要用抽样表达式，主键中必须包含这个表达式。例如： + `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))` 。 + +- `TTL` - 指定行存储的持续时间并定义数据片段在硬盘和卷上的移动逻辑的规则列表，可选项。 + + 表达式中必须存在至少一个 `Date` 或 `DateTime` 类型的列，比如： + + `TTL date + INTERVAl 1 DAY` + + 规则的类型 `DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'`指定了当满足条件（到达指定时间）时所要执行的动作：移除过期的行，还是将数据片段（如果数据片段中的所有行都满足表达式的话）移动到指定的磁盘（`TO DISK 'xxx'`) 或 卷（`TO VOLUME 'xxx'`）。默认的规则是移除（`DELETE`）。可以在列表中指定多个规则，但最多只能有一个`DELETE`的规则。 + + 更多细节，请查看 [表和列的 TTL](#table_engine-mergetree-ttl) + +- `SETTINGS` — 控制 `MergeTree` 行为的额外参数，可选项： + + - `index_granularity` — 索引粒度。索引中相邻的『标记』间的数据行数。默认值8192 。参考[数据存储](#mergetree-data-storage)。 + - `index_granularity_bytes` — 索引粒度，以字节为单位，默认值: 10Mb。如果想要仅按数据行数限制索引粒度, 请设置为0(不建议)。 + - `min_index_granularity_bytes` - 允许的最小数据粒度，默认值：1024b。该选项用于防止误操作，添加了一个非常低索引粒度的表。参考[数据存储](#mergetree-data-storage) + - `enable_mixed_granularity_parts` — 是否启用通过 `index_granularity_bytes` 控制索引粒度的大小。在19.11版本之前, 只有 `index_granularity` 配置能够用于限制索引粒度的大小。当从具有很大的行（几十上百兆字节）的表中查询数据时候，`index_granularity_bytes` 配置能够提升ClickHouse的性能。如果您的表里有很大的行，可以开启这项配置来提升`SELECT` 查询的性能。 + - `use_minimalistic_part_header_in_zookeeper` — ZooKeeper中数据片段存储方式 。如果`use_minimalistic_part_header_in_zookeeper=1` ，ZooKeeper 会存储更少的数据。更多信息参考[服务配置参数]([Server Settings | ClickHouse Documentation](https://clickhouse.com/docs/zh/operations/server-configuration-parameters/settings/))这章中的 [设置描述](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) 。 + - `min_merge_bytes_to_use_direct_io` — 使用直接 I/O 来操作磁盘的合并操作时要求的最小数据量。合并数据片段时，ClickHouse 会计算要被合并的所有数据的总存储空间。如果大小超过了 `min_merge_bytes_to_use_direct_io` 设置的字节数，则 ClickHouse 将使用直接 I/O 接口（`O_DIRECT` 选项）对磁盘读写。如果设置 `min_merge_bytes_to_use_direct_io = 0` ，则会禁用直接 I/O。默认值：`10 * 1024 * 1024 * 1024` 字节。 + + - `merge_with_ttl_timeout` — TTL合并频率的最小间隔时间，单位：秒。默认值: 86400 (1 天)。 + - `write_final_mark` — 是否启用在数据片段尾部写入最终索引标记。默认值: 1（不要关闭）。 + - `merge_max_block_size` — 在块中进行合并操作时的最大行数限制。默认值：8192 + - `storage_policy` — 存储策略。 参见 [使用具有多个块的设备进行数据存储](#table_engine-mergetree-multiple-volumes). + - `min_bytes_for_wide_part`,`min_rows_for_wide_part` 在数据片段中可以使用`Wide`格式进行存储的最小字节数/行数。您可以不设置、只设置一个，或全都设置。参考：[数据存储](#mergetree-data-storage) + - `max_parts_in_total` - 所有分区中最大块的数量(意义不明) + - `max_compress_block_size` - 在数据压缩写入表前，未压缩数据块的最大大小。您可以在全局设置中设置该值(参见[max_compress_block_size](https://clickhouse.com/docs/zh/operations/settings/settings/#max-compress-block-size))。建表时指定该值会覆盖全局设置。 + - `min_compress_block_size` - 在数据压缩写入表前，未压缩数据块的最小大小。您可以在全局设置中设置该值(参见[min_compress_block_size](https://clickhouse.com/docs/zh/operations/settings/settings/#min-compress-block-size))。建表时指定该值会覆盖全局设置。 + - `max_partitions_to_read` - 一次查询中可访问的分区最大数。您可以在全局设置中设置该值(参见[max_partitions_to_read](https://clickhouse.com/docs/zh/operations/settings/settings/#max_partitions_to_read))。 + +**示例配置** + +``` sql +ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity=8192 +``` + +在这个例子中，我们设置了按月进行分区。 + +同时我们设置了一个按用户 ID 哈希的抽样表达式。这使得您可以对该表中每个 `CounterID` 和 `EventDate` 的数据伪随机分布。如果您在查询时指定了 [SAMPLE](../../../engines/table-engines/mergetree-family/mergetree.md#select-sample-clause) 子句。 ClickHouse会返回对于用户子集的一个均匀的伪随机数据采样。 + +`index_granularity` 可省略因为 8192 是默认设置 。 + +
+已弃用的建表方法 + +!!! attention "注意" + 不要在新版项目中使用该方法，可能的话，请将旧项目切换到上述方法。 + + CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] + ( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... + ) ENGINE [=] MergeTree(date-column [, sampling_expression], (primary, key), index_granularity) + +**MergeTree() 参数** + +- `date-column` — 类型为 [日期](../../../engines/table-engines/mergetree-family/mergetree.md) 的列名。ClickHouse 会自动依据这个列按月创建分区。分区名格式为 `"YYYYMM"` 。 +- `sampling_expression` — 采样表达式。 +- `(primary, key)` — 主键。类型 — [元组()](../../../engines/table-engines/mergetree-family/mergetree.md) +- `index_granularity` — 索引粒度。即索引中相邻『标记』间的数据行数。设为 8192 可以适用大部分场景。 + +**示例** + + MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID)), 8192) + +对于主要的配置方法，这里 `MergeTree` 引擎跟前面的例子一样，可以以同样的方式配置。 +
+ +## 数据存储 {#mergetree-data-storage} + +表由按主键排序的数据片段（DATA PART）组成。 + +当数据被插入到表中时，会创建多个数据片段并按主键的字典序排序。例如，主键是 `(CounterID, Date)` 时，片段中数据首先按 `CounterID` 排序，具有相同 `CounterID` 的部分按 `Date` 排序。 + +不同分区的数据会被分成不同的片段，ClickHouse 在后台合并数据片段以便更高效存储。不同分区的数据片段不会进行合并。合并机制并不保证具有相同主键的行全都合并到同一个数据片段中。 + +数据片段可以以 `Wide` 或 `Compact` 格式存储。在 `Wide` 格式下，每一列都会在文件系统中存储为单独的文件，在 `Compact` 格式下所有列都存储在一个文件中。`Compact` 格式可以提高插入量少插入频率频繁时的性能。 + +数据存储格式由 `min_bytes_for_wide_part` 和 `min_rows_for_wide_part` 表引擎参数控制。如果数据片段中的字节数或行数少于相应的设置值，数据片段会以 `Compact` 格式存储，否则会以 `Wide` 格式存储。 + +每个数据片段被逻辑的分割成颗粒（granules）。颗粒是 ClickHouse 中进行数据查询时的最小不可分割数据集。ClickHouse 不会对行或值进行拆分，所以每个颗粒总是包含整数个行。每个颗粒的第一行通过该行的主键值进行标记， +ClickHouse 会为每个数据片段创建一个索引文件来存储这些标记。对于每列，无论它是否包含在主键当中，ClickHouse 都会存储类似标记。这些标记让您可以在列文件中直接找到数据。 + +颗粒的大小通过表引擎参数 `index_granularity` 和 `index_granularity_bytes` 控制。颗粒的行数的在 `[1, index_granularity]` 范围中，这取决于行的大小。如果单行的大小超过了 `index_granularity_bytes` 设置的值，那么一个颗粒的大小会超过 `index_granularity_bytes`。在这种情况下，颗粒的大小等于该行的大小。 + +## 主键和索引在查询中的表现 {#primary-keys-and-indexes-in-queries} + +我们以 `(CounterID, Date)` 以主键。排序好的索引的图示会是下面这样： + +``` text + 全部数据 : [-------------------------------------------------------------------------] + CounterID: [aaaaaaaaaaaaaaaaaabbbbcdeeeeeeeeeeeeefgggggggghhhhhhhhhiiiiiiiiikllllllll] + Date: [1111111222222233331233211111222222333211111112122222223111112223311122333] + 标记: | | | | | | | | | | | + a,1 a,2 a,3 b,3 e,2 e,3 g,1 h,2 i,1 i,3 l,3 + 标记号: 0 1 2 3 4 5 6 7 8 9 10 +``` + +如果指定查询如下： + +- `CounterID in ('a', 'h')`，服务器会读取标记号在 `[0, 3)` 和 `[6, 8)` 区间中的数据。 +- `CounterID IN ('a', 'h') AND Date = 3`，服务器会读取标记号在 `[1, 3)` 和 `[7, 8)` 区间中的数据。 +- `Date = 3`，服务器会读取标记号在 `[1, 10]` 区间中的数据。 + +上面例子可以看出使用索引通常会比全表描述要高效。 + +稀疏索引会引起额外的数据读取。当读取主键单个区间范围的数据时，每个数据块中最多会多读 `index_granularity * 2` 行额外的数据。 + +稀疏索引使得您可以处理极大量的行，因为大多数情况下，这些索引常驻于内存。 + +ClickHouse 不要求主键唯一，所以您可以插入多条具有相同主键的行。 + +您可以在`PRIMARY KEY`与`ORDER BY`条件中使用`可为空的`类型的表达式，但强烈建议不要这么做。为了启用这项功能，请打开[allow_nullable_key](https://clickhouse.com/docs/zh/operations/settings/settings/#allow-nullable-key)，[NULLS_LAST](https://clickhouse.com/docs/zh/sql-reference/statements/select/order-by/#sorting-of-special-values)规则也适用于`ORDER BY`条件中有NULL值的情况下。 + +### 主键的选择 {#zhu-jian-de-xuan-ze} + +主键中列的数量并没有明确的限制。依据数据结构，您可以在主键包含多些或少些列。这样可以： + +- 改善索引的性能。 + + 如果当前主键是 `(a, b)` ，在下列情况下添加另一个 `c` 列会提升性能： + + - 查询会使用 `c` 列作为条件 + - 很长的数据范围（ `index_granularity` 的数倍）里 `(a, b)` 都是相同的值，并且这样的情况很普遍。换言之，就是加入另一列后，可以让您的查询略过很长的数据范围。 + +- 改善数据压缩。 + + ClickHouse 以主键排序片段数据，所以，数据的一致性越高，压缩越好。 + +- 在[CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) 和 [SummingMergeTree](summingmergetree.md) 引擎里进行数据合并时会提供额外的处理逻辑。 + + 在这种情况下，指定与主键不同的 *排序键* 也是有意义的。 + +长的主键会对插入性能和内存消耗有负面影响，但主键中额外的列并不影响 `SELECT` 查询的性能。 + +可以使用 `ORDER BY tuple()` 语法创建没有主键的表。在这种情况下 ClickHouse 根据数据插入的顺序存储。如果在使用 `INSERT ... SELECT` 时希望保持数据的排序，请设置 [max_insert_threads = 1](../../../operations/settings/settings.md#settings-max-insert-threads)。 + +想要根据初始顺序进行数据查询，使用 [单线程查询](../../../operations/settings/settings.md#settings-max_threads) + +### 选择与排序键不同的主键 {#choosing-a-primary-key-that-differs-from-the-sorting-key} + +Clickhouse可以做到指定一个跟排序键不一样的主键，此时排序键用于在数据片段中进行排序，主键用于在索引文件中进行标记的写入。这种情况下，主键表达式元组必须是排序键表达式元组的前缀(即主键为(a,b)，排序列必须为(a,b,******))。 + +当使用 [SummingMergeTree](summingmergetree.md) 和 [AggregatingMergeTree](aggregatingmergetree.md) 引擎时，这个特性非常有用。通常在使用这类引擎时，表里的列分两种：*维度* 和 *度量* 。典型的查询会通过任意的 `GROUP BY` 对度量列进行聚合并通过维度列进行过滤。由于 SummingMergeTree 和 AggregatingMergeTree 会对排序键相同的行进行聚合，所以把所有的维度放进排序键是很自然的做法。但这将导致排序键中包含大量的列，并且排序键会伴随着新添加的维度不断的更新。 + +在这种情况下合理的做法是，只保留少量的列在主键当中用于提升扫描效率，将维度列添加到排序键中。 + +对排序键进行 [ALTER](../../../sql-reference/statements/alter.md) 是轻量级的操作，因为当一个新列同时被加入到表里和排序键里时，已存在的数据片段并不需要修改。由于旧的排序键是新排序键的前缀，并且新添加的列中没有数据，因此在表修改时的数据对于新旧的排序键来说都是有序的。 + +### 索引和分区在查询中的应用 {#use-of-indexes-and-partitions-in-queries} + +对于 `SELECT` 查询，ClickHouse 分析是否可以使用索引。如果 `WHERE/PREWHERE` 子句具有下面这些表达式（作为完整WHERE条件的一部分或全部）则可以使用索引：进行相等/不相等的比较；对主键列或分区列进行`IN`运算、有固定前缀的`LIKE`运算(如name like 'test%')、函数运算(部分函数适用)，还有对上述表达式进行逻辑运算。 + + + + + + +因此，在索引键的一个或多个区间上快速地执行查询是可能的。下面例子中，指定标签；指定标签和日期范围；指定标签和日期；指定多个标签和日期范围等执行查询，都会非常快。 + +当引擎配置如下时： + +``` sql + ENGINE MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate) SETTINGS index_granularity=8192 +``` + +这种情况下，这些查询： + +``` sql +SELECT count() FROM table WHERE EventDate = toDate(now()) AND CounterID = 34 +SELECT count() FROM table WHERE EventDate = toDate(now()) AND (CounterID = 34 OR CounterID = 42) +SELECT count() FROM table WHERE ((EventDate >= toDate('2014-01-01') AND EventDate <= toDate('2014-01-31')) OR EventDate = toDate('2014-05-01')) AND CounterID IN (101500, 731962, 160656) AND (CounterID = 101500 OR EventDate != toDate('2014-05-01')) +``` + +ClickHouse 会依据主键索引剪掉不符合的数据，依据按月分区的分区键剪掉那些不包含符合数据的分区。 + +上文的查询显示，即使索引用于复杂表达式，因为读表操作经过优化，所以使用索引不会比完整扫描慢。 + +下面这个例子中，不会使用索引。 + +``` sql +SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' +``` + +要检查 ClickHouse 执行一个查询时能否使用索引，可设置 [force_index_by_date](../../../operations/settings/settings.md#settings-force_index_by_date) 和 [force_primary_key](../../../operations/settings/settings.md) 。 + +使用按月分区的分区列允许只读取包含适当日期区间的数据块，这种情况下，数据块会包含很多天（最多整月）的数据。在块中，数据按主键排序，主键第一列可能不包含日期。因此，仅使用日期而没有用主键字段作为条件的查询将会导致需要读取超过这个指定日期以外的数据。 + +### 部分单调主键的使用 + +考虑这样的场景，比如一个月中的天数。它们在一个月的范围内形成一个[单调序列](https://zh.wikipedia.org/wiki/单调函数) ，但如果扩展到更大的时间范围它们就不再单调了。这就是一个部分单调序列。如果用户使用部分单调的主键创建表，ClickHouse同样会创建一个稀疏索引。当用户从这类表中查询数据时，ClickHouse 会对查询条件进行分析。如果用户希望获取两个索引标记之间的数据并且这两个标记在一个月以内，ClickHouse 可以在这种特殊情况下使用到索引，因为它可以计算出查询参数与索引标记之间的距离。 + +如果查询参数范围内的主键不是单调序列，那么 ClickHouse 无法使用索引。在这种情况下，ClickHouse 会进行全表扫描。 + +ClickHouse 在任何主键代表一个部分单调序列的情况下都会使用这个逻辑。 + +### 跳数索引 {#tiao-shu-suo-yin-fen-duan-hui-zong-suo-yin-shi-yan-xing-de} + +此索引在 `CREATE` 语句的列部分里定义。 + +``` sql +INDEX index_name expr TYPE type(...) GRANULARITY granularity_value +``` + +`*MergeTree` 系列的表可以指定跳数索引。 +跳数索引是指数据片段按照粒度(建表时指定的`index_granularity`)分割成小块后，将上述SQL的granularity_value数量的小块组合成一个大的块，对这些大块写入索引信息，这样有助于使用`where`筛选时跳过大量不必要的数据，减少`SELECT`需要读取的数据量。 + +**示例** + +``` sql +CREATE TABLE table_name +( + u64 UInt64, + i32 Int32, + s String, + ... + INDEX a (u64 * i32, s) TYPE minmax GRANULARITY 3, + INDEX b (u64 * length(s)) TYPE set(1000) GRANULARITY 4 +) ENGINE = MergeTree() +... +``` + +上例中的索引能让 ClickHouse 执行下面这些查询时减少读取数据量。 + +``` sql +SELECT count() FROM table WHERE s < 'z' +SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 +``` + +#### 可用的索引类型 {#table_engine-mergetree-data_skipping-indexes} + +- `minmax` + 存储指定表达式的极值（如果表达式是 `tuple` ，则存储 `tuple` 中每个元素的极值），这些信息用于跳过数据块，类似主键。 + +- `set(max_rows)` + 存储指定表达式的不重复值（不超过 `max_rows` 个，`max_rows=0` 则表示『无限制』）。这些信息可用于检查数据块是否满足 `WHERE` 条件。 + +- `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` + 存储一个包含数据块中所有 n元短语（ngram） 的 [布隆过滤器](https://en.wikipedia.org/wiki/Bloom_filter) 。只可用在字符串上。 + 可用于优化 `equals` ， `like` 和 `in` 表达式的性能。 + - `n` – 短语长度。 + - `size_of_bloom_filter_in_bytes` – 布隆过滤器大小，字节为单位。（因为压缩得好，可以指定比较大的值，如 256 或 512）。 + - `number_of_hash_functions` – 布隆过滤器中使用的哈希函数的个数。 + - `random_seed` – 哈希函数的随机种子。 + +- `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` + 跟 `ngrambf_v1` 类似，但是存储的是token而不是ngrams。Token是由非字母数字的符号分割的序列。 + +- `bloom_filter(bloom_filter([false_positive])` – 为指定的列存储布隆过滤器 + + 可选参数`false_positive`用来指定从布隆过滤器收到错误响应的几率。取值范围是 (0,1)，默认值：0.025 + + 支持的数据类型：`Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`, `Array`, `LowCardinality`, `Nullable`。 + + 以下函数会用到这个索引： [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [has](../../../sql-reference/functions/array-functions.md) + +``` sql +INDEX sample_index (u64 * length(s)) TYPE minmax GRANULARITY 4 +INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARITY 4 +INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 +``` + +#### 函数支持 {#functions-support} + +WHERE 子句中的条件可以包含对某列数据进行运算的函数表达式，如果列是索引的一部分，ClickHouse会在执行函数时尝试使用索引。不同的函数对索引的支持是不同的。 + +`set` 索引会对所有函数生效，其他索引对函数的生效情况见下表 + +| 函数 (操作符) / 索引 | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | +| ------------------------------------------------------------ | ----------- | ------ | ---------- | ---------- | ------------ | +| [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [like](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | +| [endsWith](../../../sql-reference/functions/string-functions.md#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | +| [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | +| [in](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [less (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [empty](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | +| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | + +常量参数小于 ngram 大小的函数不能使用 `ngrambf_v1` 进行查询优化。 + +!!! note "注意" +布隆过滤器可能会包含不符合条件的匹配，所以 `ngrambf_v1`, `tokenbf_v1` 和 `bloom_filter` 索引不能用于结果返回为假的函数，例如： + +- 可以用来优化的场景 + - `s LIKE '%test%'` + - `NOT s NOT LIKE '%test%'` + - `s = 1` + - `NOT s != 1` + - `startsWith(s, 'test')` +- 不能用来优化的场景 + - `NOT s LIKE '%test%'` + - `s NOT LIKE '%test%'` + - `NOT s = 1` + - `s != 1` + - `NOT startsWith(s, 'test')` + +## 并发数据访问 {#concurrent-data-access} + +对于表的并发访问，我们使用多版本机制。换言之，当一张表同时被读和更新时，数据从当前查询到的一组片段中读取。没有冗长的的锁。插入不会阻碍读取。 + +对表的读操作是自动并行的。 + +## 列和表的 TTL {#table_engine-mergetree-ttl} + +TTL用于设置值的生命周期，它既可以为整张表设置，也可以为每个列字段单独设置。表级别的 TTL 还会指定数据在磁盘和卷上自动转移的逻辑。 + +TTL 表达式的计算结果必须是 [日期](../../../engines/table-engines/mergetree-family/mergetree.md) 或 [日期时间](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的字段。 + +示例： + +``` sql +TTL time_column +TTL time_column + interval +``` + +要定义`interval`, 需要使用 [时间间隔](../../../engines/table-engines/mergetree-family/mergetree.md#operators-datetime) 操作符。 + +``` sql +TTL date_time + INTERVAL 1 MONTH +TTL date_time + INTERVAL 15 HOUR +``` + +### 列 TTL {#mergetree-column-ttl} + +当列中的值过期时, ClickHouse会将它们替换成该列数据类型的默认值。如果数据片段中列的所有值均已过期，则ClickHouse 会从文件系统中的数据片段中删除此列。 + +`TTL`子句不能被用于主键字段。 + +**示例:** + +创建表时指定 `TTL` + +``` sql +CREATE TABLE example_table +( + d DateTime, + a Int TTL d + INTERVAL 1 MONTH, + b Int TTL d + INTERVAL 1 MONTH, + c String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d; +``` + +为表中已存在的列字段添加 `TTL` + +``` sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 DAY; +``` + +修改列字段的 `TTL` + +``` sql +ALTER TABLE example_table + MODIFY COLUMN + c String TTL d + INTERVAL 1 MONTH; +``` + +### 表 TTL {#mergetree-table-ttl} + +表可以设置一个用于移除过期行的表达式，以及多个用于在磁盘或卷上自动转移数据片段的表达式。当表中的行过期时，ClickHouse 会删除所有对应的行。对于数据片段的转移特性，必须所有的行都满足转移条件。 + +``` sql +TTL expr + [DELETE|TO DISK 'xxx'|TO VOLUME 'xxx'][, DELETE|TO DISK 'aaa'|TO VOLUME 'bbb'] ... + [WHERE conditions] + [GROUP BY key_expr [SET v1 = aggr_func(v1) [, v2 = aggr_func(v2) ...]] ] + +``` + +TTL 规则的类型紧跟在每个 TTL 表达式后面，它会影响满足表达式时（到达指定时间时）应当执行的操作： + +- `DELETE` - 删除过期的行（默认操作）; +- `TO DISK 'aaa'` - 将数据片段移动到磁盘 `aaa`; +- `TO VOLUME 'bbb'` - 将数据片段移动到卷 `bbb`. +- `GROUP BY` - 聚合过期的行 + +使用`WHERE`从句，您可以指定哪些过期的行会被删除或聚合(不适用于移动)。`GROUP BY`表达式必须是表主键的前缀。如果某列不是`GROUP BY`表达式的一部分，也没有在SET从句显示引用，结果行中相应列的值是随机的(就好像使用了`any`函数)。 + +**示例**: + +创建时指定 TTL + +``` sql +CREATE TABLE example_table +( + d DateTime, + a Int +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d +TTL d + INTERVAL 1 MONTH [DELETE], + d + INTERVAL 1 WEEK TO VOLUME 'aaa', + d + INTERVAL 2 WEEK TO DISK 'bbb'; +``` + +修改表的 `TTL` + +``` sql +ALTER TABLE example_table + MODIFY TTL d + INTERVAL 1 DAY; +``` + +创建一张表，设置一个月后数据过期，这些过期的行中日期为星期一的删除： + +``` sql +CREATE TABLE table_with_where +( + d DateTime, + a Int +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d +TTL d + INTERVAL 1 MONTH DELETE WHERE toDayOfWeek(d) = 1; +``` + +创建一张表，设置过期的列会被聚合。列`x`包含每组行中的最大值，`y`为最小值，`d`为可能任意值。 + +``` sql +CREATE TABLE table_for_aggregation +( + d DateTime, + k1 Int, + k2 Int, + x Int, + y Int +) +ENGINE = MergeTree +ORDER BY (k1, k2) +TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); +``` + +**删除数据** + +ClickHouse 在数据片段合并时会删除掉过期的数据。 + +当ClickHouse发现数据过期时, 它将会执行一个计划外的合并。要控制这类合并的频率, 您可以设置 `merge_with_ttl_timeout`。如果该值被设置的太低, 它将引发大量计划外的合并，这可能会消耗大量资源。 + +如果在合并的过程中执行 `SELECT` 查询, 则可能会得到过期的数据。为了避免这种情况，可以在 `SELECT` 之前使用 [OPTIMIZE](../../../engines/table-engines/mergetree-family/mergetree.md#misc_operations-optimize) 。 + +## 使用多个块设备进行数据存储 {#table_engine-mergetree-multiple-volumes} + +### 介绍 {#introduction} + +MergeTree 系列表引擎可以将数据存储在多个块设备上。这对某些可以潜在被划分为“冷”“热”的表来说是很有用的。最新数据被定期的查询但只需要很小的空间。相反，详尽的历史数据很少被用到。如果有多块磁盘可用，那么“热”的数据可以放置在快速的磁盘上（比如 NVMe 固态硬盘或内存），“冷”的数据可以放在相对较慢的磁盘上（比如机械硬盘）。 + +数据片段是 `MergeTree` 引擎表的最小可移动单元。属于同一个数据片段的数据被存储在同一块磁盘上。数据片段会在后台自动的在磁盘间移动，也可以通过 [ALTER](../../../sql-reference/statements/alter.md#alter_move-partition) 查询来移动。 + +### 术语 {#terms} + +- 磁盘 — 挂载到文件系统的块设备 +- 默认磁盘 — 在服务器设置中通过 [path](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-path) 参数指定的数据存储 +- 卷 — 相同磁盘的顺序列表 （类似于 [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures)） +- 存储策略 — 卷的集合及他们之间的数据移动规则 + + 以上名称的信息在Clickhouse中系统表[system.storage_policies](https://clickhouse.com/docs/zh/operations/system-tables/storage_policies/#system_tables-storage_policies)和[system.disks](https://clickhouse.com/docs/zh/operations/system-tables/disks/#system_tables-disks)体现。为了应用存储策略，可以在建表时使用`storage_policy`设置。 + +### 配置 {#table_engine-mergetree-multiple-volumes_configure} + +磁盘、卷和存储策略应当在主配置文件 `config.xml` 或 `config.d` 目录中的独立文件中的 `` 标签内定义。 + +配置结构： + +``` xml + + + + /mnt/fast_ssd/clickhouse/ + + + /mnt/hdd1/clickhouse/ + 10485760 + + + /mnt/hdd2/clickhouse/ + 10485760 + + + ... + + + ... + +``` + +标签： + +- `` — 磁盘名，名称必须与其他磁盘不同. +- `path` — 服务器将用来存储数据 (`data` 和 `shadow` 目录) 的路径, 应当以 ‘/’ 结尾. +- `keep_free_space_bytes` — 需要保留的剩余磁盘空间. + +磁盘定义的顺序无关紧要。 + +存储策略配置： + +``` xml + + ... + + + + + disk_name_from_disks_configuration + 1073741824 + + + + + + + 0.2 + + + + + + + + ... + +``` + +标签： + +- `policy_name_N` — 策略名称，不能重复。 +- `volume_name_N` — 卷名称，不能重复。 +- `disk` — 卷中的磁盘。 +- `max_data_part_size_bytes` — 卷中的磁盘可以存储的数据片段的最大大小。 +- `move_factor` — 当可用空间少于这个因子时，数据将自动的向下一个卷（如果有的话）移动 (默认值为 0.1)。 +- `prefer_not_to_merge` - 禁止在这个卷中进行数据合并。该选项启用时，对该卷的数据不能进行合并。这个选项主要用于慢速磁盘。 + +配置示例： + +``` xml + + ... + + + + + disk1 + disk2 + + + + + + + + fast_ssd + 1073741824 + + + disk1 + + + 0.2 + + + + +
+ jbod1 +
+ + external + true + + + + + ... + +``` + +在给出的例子中， `hdd_in_order` 策略实现了 [循环制](https://zh.wikipedia.org/wiki/循环制) 方法。因此这个策略只定义了一个卷（`single`），数据片段会以循环的顺序全部存储到它的磁盘上。当有多个类似的磁盘挂载到系统上，但没有配置 RAID 时，这种策略非常有用。请注意一个每个独立的磁盘驱动都并不可靠，您可能需要用3份或更多的复制份数来补偿它。 + +如果在系统中有不同类型的磁盘可用，可以使用 `moving_from_ssd_to_hdd`。`hot` 卷由 SSD 磁盘（`fast_ssd`）组成，这个卷上可以存储的数据片段的最大大小为 1GB。所有大于 1GB 的数据片段都会被直接存储到 `cold` 卷上，`cold` 卷包含一个名为 `disk1` 的 HDD 磁盘。 +同样，一旦 `fast_ssd` 被填充超过 80%，数据会通过后台进程向 `disk1` 进行转移。 + +存储策略中卷的枚举顺序是很重要的。因为当一个卷被充满时，数据会向下一个卷转移。磁盘的枚举顺序同样重要，因为数据是依次存储在磁盘上的。 + +在创建表时，可以应用存储策略： + +``` sql +CREATE TABLE table_with_non_default_policy ( + EventDate Date, + OrderID UInt64, + BannerID UInt64, + SearchPhrase String +) ENGINE = MergeTree +ORDER BY (OrderID, BannerID) +PARTITION BY toYYYYMM(EventDate) +SETTINGS storage_policy = 'moving_from_ssd_to_hdd' +``` + +`default` 存储策略意味着只使用一个卷，这个卷只包含一个在 `` 中定义的磁盘。您可以使用[ALTER TABLE ... MODIFY SETTING]来修改存储策略，新的存储策略应该包含所有以前的磁盘和卷，并使用相同的名称。 + +可以通过 [background_move_pool_size](../../../operations/settings/settings.md#background_move_pool_size) 设置调整执行后台任务的线程数。 + +### 详细说明 {#details} + +对于 `MergeTree` 表，数据通过以下不同的方式写入到磁盘当中： + +- 插入（`INSERT`查询） +- 后台合并和[数据变异](../../../sql-reference/statements/alter.md#alter-mutations) +- 从另一个副本下载 +- [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter.md#alter_freeze-partition) 冻结分区 + +除了数据变异和冻结分区以外的情况下，数据按照以下逻辑存储到卷或磁盘上： + +1. 首个卷（按定义顺序）拥有足够的磁盘空间存储数据片段（`unreserved_space > current_part_size`）并且允许存储给定数据片段的大小（`max_data_part_size_bytes > current_part_size`） +2. 在这个数据卷内，紧挨着先前存储数据的那块磁盘之后的磁盘，拥有比数据片段大的剩余空间。（`unreserved_space - keep_free_space_bytes > current_part_size`） + +更进一步，数据变异和分区冻结使用的是 [硬链接](https://en.wikipedia.org/wiki/Hard_link)。不同磁盘之间的硬链接是不支持的，所以在这种情况下数据片段都会被存储到原来的那一块磁盘上。 + +在后台，数据片段基于剩余空间（`move_factor`参数）根据卷在配置文件中定义的顺序进行转移。数据永远不会从最后一个移出也不会从第一个移入。可以通过系统表 [system.part_log](../../../operations/system-tables/part_log.md#system_tables-part-log) (字段 `type = MOVE_PART`) 和 [system.parts](../../../operations/system-tables/parts.md#system_tables-parts) (字段 `path` 和 `disk`) 来监控后台的移动情况。具体细节可以通过服务器日志查看。 + +用户可以通过 [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter.md#alter_move-partition) 强制移动一个数据片段或分区到另外一个卷，所有后台移动的限制都会被考虑在内。这个查询会自行启动，无需等待后台操作完成。如果没有足够的可用空间或任何必须条件没有被满足，用户会收到报错信息。 + +数据移动不会妨碍到数据复制。也就是说，同一张表的不同副本可以指定不同的存储策略。 + +在后台合并和数据变异之后，旧的数据片段会在一定时间后被移除 (`old_parts_lifetime`)。在这期间，他们不能被移动到其他的卷或磁盘。也就是说，直到数据片段被完全移除，它们仍然会被磁盘占用空间计算在内。 + +## 使用S3进行数据存储 {#using-s3-data-storage} + +`MergeTree`系列表引擎允许使用[S3](https://aws.amazon.com/s3/)存储数据，需要修改磁盘类型为`S3`。 + +示例配置： + +``` xml + + ... + + + s3 + https://storage.yandexcloud.net/my-bucket/root-path/ + your_access_key_id + your_secret_access_key + + your_base64_encoded_customer_key + + http://proxy1 + http://proxy2 + + 10000 + 5000 + 10 + 4 + 1000 + /var/lib/clickhouse/disks/s3/ + true + /var/lib/clickhouse/disks/s3/cache/ + false + + + ... + +``` + +必须的参数： + +- `endpoint` - S3的结点URL，以`path`或`virtual hosted`[格式](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html)书写。 +- `access_key_id` - S3的Access Key ID。 +- `secret_access_key` - S3的Secret Access Key。 + +可选参数： + +- `region` - S3的区域名称 +- `use_environment_credentials` - 从环境变量AWS_ACCESS_KEY_ID、AWS_SECRET_ACCESS_KEY和AWS_SESSION_TOKEN中读取认证参数。默认值为`false`。 +- `use_insecure_imds_request` - 如果设置为`true`，S3客户端在认证时会使用不安全的IMDS请求。默认值为`false`。 +- `proxy` - 访问S3结点URL时代理设置。每一个`uri`项的值都应该是合法的代理URL。 +- `connect_timeout_ms` - Socket连接超时时间，默认值为`10000`，即10秒。 +- `request_timeout_ms` - 请求超时时间，默认值为`5000`，即5秒。 +- `retry_attempts` - 请求失败后的重试次数，默认值为10。 +- `single_read_retries` - 读过程中连接丢失后重试次数，默认值为4。 +- `min_bytes_for_seek` - 使用查找操作，而不是顺序读操作的最小字节数，默认值为1000。 +- `metadata_path` - 本地存放S3元数据文件的路径，默认值为`/var/lib/clickhouse/disks//` +- `cache_enabled` - 是否允许缓存标记和索引文件。默认值为`true`。 +- `cache_path` - 本地缓存标记和索引文件的路径。默认值为`/var/lib/clickhouse/disks//cache/`。 +- `skip_access_check` - 如果为`true`，Clickhouse启动时不检查磁盘是否可用。默认为`false`。 +- `server_side_encryption_customer_key_base64` - 如果指定该项的值，请求时会加上为了访问SSE-C加密数据而必须的头信息。 + +S3磁盘也可以设置冷热存储： +```xml + + ... + + + s3 + https://storage.yandexcloud.net/my-bucket/root-path/ + your_access_key_id + your_secret_access_key + + + + + +
+ s3 +
+ + + + +
+ default +
+ + s3 + + + 0.2 + + + ... + +``` + +指定了`cold`选项后，本地磁盘剩余空间如果小于`move_factor * disk_size`，或有TTL设置时，数据就会定时迁移至S3了。 + +## 虚拟列 {#virtual-columns} + +- `_part` - 分区名称。 +- `_part_index` - 作为请求的结果，按顺序排列的分区数。 +- `_partition_id` — 分区名称。 +- `_part_uuid` - 唯一部分标识符（如果 MergeTree 设置`assign_part_uuids` 已启用）。 +- `_partition_value` — `partition by` 表达式的值（元组）。 +- `_sample_factor` - 采样因子（来自请求）。 + +[原始文章](https://clickhouse.com/docs/en/operations/table_engines/mergetree/) diff --git a/docs/zh/engines/table-engines/mergetree-family/replication.md b/docs/zh/engines/table-engines/mergetree-family/replication.md index 2e6391c01dd..c3be3a382cb 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replication.md +++ b/docs/zh/engines/table-engines/mergetree-family/replication.md @@ -7,7 +7,7 @@ - ReplicatedReplacingMergeTree - ReplicatedAggregatingMergeTree - ReplicatedCollapsingMergeTree -- ReplicatedVersionedCollapsingMergetree +- ReplicatedVersionedCollapsingMergeTree - ReplicatedGraphiteMergeTree 副本是表级别的，不是整个服务器级的。所以，服务器里可以同时有复制表和非复制表。 diff --git a/docs/zh/engines/table-engines/special/distributed.md b/docs/zh/engines/table-engines/special/distributed.md index f24d509f151..edc4c1f4854 100644 --- a/docs/zh/engines/table-engines/special/distributed.md +++ b/docs/zh/engines/table-engines/special/distributed.md @@ -1,31 +1,125 @@ -# 分布 {#distributed} +--- +toc_priority: 33 +toc_title: 分布式引擎 +--- + +# 分布式引擎 {#distributed} **分布式引擎本身不存储数据**, 但可以在多个服务器上进行分布式查询。 读是自动并行的。读取时，远程服务器表的索引（如果有的话）会被使用。 -分布式引擎参数：服务器配置文件中的集群名，远程数据库名，远程表名，数据分片键（可选）。 -示例： - Distributed(logs, default, hits[, sharding_key]) +## 创建数据表 {#distributed-creating-a-table} -将会从位于«logs»集群中 default.hits 表所有服务器上读取数据。 -远程服务器不仅用于读取数据，还会对尽可能数据做部分处理。 -例如，对于使用 GROUP BY 的查询，数据首先在远程服务器聚合，之后返回聚合函数的中间状态给查询请求的服务器。再在请求的服务器上进一步汇总数据。 +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + ... +) ENGINE = Distributed(cluster, database, table[, sharding_key[, policy_name]]) +[SETTINGS name=value, ...] +``` -数据库名参数除了用数据库名之外，也可用返回字符串的常量表达式。例如：currentDatabase()。 +## 已有数据表 {#distributed-from-a-table} +当 `Distributed` 表指向当前服务器上的一个表时，你可以采用以下语句: -logs – 服务器配置文件中的集群名称。 -集群示例配置如下： +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] AS [db2.]name2 ENGINE = Distributed(cluster, database, table[, sharding_key[, policy_name]]) [SETTINGS name=value, ...] +``` + +**分布式引擎参数** + +- `cluster` - 服务为配置中的集群名 + +- `database` - 远程数据库名 + +- `table` - 远程数据表名 + +- `sharding_key` - (可选) 分片key + +- `policy_name` - (可选) 规则名，它会被用作存储临时文件以便异步发送数据 + +**详见** + + - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) 设置 + - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) 查看示例 + + **分布式设置** + +- `fsync_after_insert` - 对异步插入到分布式的文件数据执行`fsync`。确保操作系统将所有插入的数据刷新到启动节点**磁盘上的一个文件**中。 + +- `fsync_directories` - 对目录执行`fsync`。保证操作系统在分布式表上进行异步插入相关操作(插入后，发送数据到分片等)后刷新目录元数据. + +- `bytes_to_throw_insert` - 如果超过这个数量的压缩字节将等待异步INSERT，将抛出一个异常。0 - 不抛出。默认值0. + +- `bytes_to_delay_insert` - 如果超过这个数量的压缩字节将等待异步INSERT，查询将被延迟。0 - 不要延迟。默认值0. + +- `max_delay_to_insert` - 最大延迟多少秒插入数据到分布式表，如果有很多挂起字节异步发送。默认值60。 + +- `monitor_batch_inserts` - 等同于 [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) + +- `monitor_split_batch_on_failure` - 等同于[distributed_directory_monitor_split_batch_on_failure](../../../operations/settings/settings.md#distributed_directory_monitor_split_batch_on_failure) + +- `monitor_sleep_time_ms` - 等同于 [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) + +- `monitor_max_sleep_time_ms` - 等同于 [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) + +!!! note "备注" + + **稳定性设置** (`fsync_...`): + + - 只影响异步插入(例如:`insert_distributed_sync=false`), 当数据首先存储在启动节点磁盘上，然后再异步发送到shard。 + — 可能会显著降低`insert`的性能 + - 影响将存储在分布式表文件夹中的数据写入 **接受您插入的节点** 。如果你需要保证写入数据到底层的MergeTree表中，请参阅 `system.merge_tree_settings` 中的持久性设置(`...fsync...`) + + **插入限制设置** (`..._insert`) 请见: + + - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) 设置 + - [prefer_localhost_replica](../../../operations/settings/settings.md#settings-prefer-localhost-replica) 设置 + - `bytes_to_throw_insert` 在 `bytes_to_delay_insert` 之前处理，所以你不应该设置它的值小于 `bytes_to_delay_insert` +**示例** + +``` sql +CREATE TABLE hits_all AS hits +ENGINE = Distributed(logs, default, hits[, sharding_key[, policy_name]]) +SETTINGS + fsync_after_insert=0, + fsync_directories=0; +``` + +数据将从`logs`集群中的所有服务器中，从位于集群中的每个服务器上的`default.hits`表读取。。 +数据不仅在远程服务器上读取，而且在远程服务器上进行部分处理(在可能的范围内)。 +例如，对于带有 `GROUP BY`的查询，数据将在远程服务器上聚合，聚合函数的中间状态将被发送到请求者服务器。然后将进一步聚合数据。 + +您可以使用一个返回字符串的常量表达式来代替数据库名称。例如: `currentDatabase()`。 + +## 集群 {#distributed-clusters} + + +集群是通过[服务器配置文件](../../../operations/configuration-files.md)来配置的 ``` xml + + - + 1 - + false + + 1 example01-01-1 9000 @@ -58,6 +152,7 @@ logs – 服务器配置文件中的集群名称。 集群名称不能包含点号。 每个服务器需要指定 `host`，`port`，和可选的 `user`，`password`，`secure`，`compression` 的参数： + - `host` – 远程服务器地址。可以域名、IPv4或IPv6。如果指定域名，则服务在启动时发起一个 DNS 请求，并且请求结果会在服务器运行期间一直被记录。如果 DNS 请求失败，则服务不会启动。如果你修改了 DNS 记录，则需要重启服务。 - `port` – 消息传递的 TCP 端口（「tcp_port」配置通常设为 9000）。不要跟 http_port 混淆。 - `user` – 用于连接远程服务器的用户名。默认值：default。该用户必须有权限访问该远程服务器。访问权限配置在 users.xml 文件中。更多信息，请查看«访问权限»部分。 @@ -78,9 +173,10 @@ logs – 服务器配置文件中的集群名称。 通过分布式引擎可以像使用本地服务器一样使用集群。但是，集群不是自动扩展的：你必须编写集群配置到服务器配置文件中（最好，给所有集群的服务器写上完整配置）。 不支持用分布式表查询别的分布式表（除非该表只有一个分片）。或者说，要用分布表查查询«最终»的数据表。 - 分布式引擎需要将集群信息写入配置文件。配置文件中的集群信息会即时更新，无需重启服务器。如果你每次是要向不确定的一组分片和副本发送查询，则不适合创建分布式表 - 而应该使用«远程»表函数。 请参阅«表函数»部分。 +## 写入数据 + 向集群写数据的方法有两种： 一，自已指定要将哪些数据写入哪些服务器，并直接在每个分片上执行写入。换句话说，在分布式表上«查询»，在数据表上 INSERT。 @@ -109,12 +205,32 @@ SELECT 查询会被发送到所有分片，并且无论数据在分片中如何 下面的情况，你需要关注分片方案： - 使用需要特定键连接数据（ IN 或 JOIN ）的查询。如果数据是用该键进行分片，则应使用本地 IN 或 JOIN 而不是 GLOBAL IN 或 GLOBAL JOIN，这样效率更高。 -- 使用大量服务器（上百或更多），但有大量小查询（个别客户的查询 - 网站，广告商或合作伙伴）。为了使小查询不影响整个集群，让单个客户的数据处于单个分片上是有意义的。或者，正如我们在 Yandex.Metrica 中所做的那样，你可以配置两级分片：将整个集群划分为«层»，一个层可以包含多个分片。单个客户的数据位于单个层上，根据需要将分片添加到层中，层中的数据随机分布。然后给每层创建分布式表，再创建一个全局的分布式表用于全局的查询。 +- 使用大量服务器（上百或更多），但有大量小查询（个别客户的查询 - 网站，广告商或合作伙伴）。为了使小查询不影响整个集群，让单个客户的数据处于单个分片上是有意义的。或者 你可以配置两级分片：将整个集群划分为«层»，一个层可以包含多个分片。单个客户的数据位于单个层上，根据需要将分片添加到层中，层中的数据随机分布。然后给每层创建分布式表，再创建一个全局的分布式表用于全局的查询。 -数据是异步写入的。对于分布式表的 INSERT，数据块只写本地文件系统。之后会尽快地在后台发送到远程服务器。你可以通过查看表目录中的文件列表（等待发送的数据）来检查数据是否成功发送：/var/lib/clickhouse/data/database/table/ 。 +数据是异步写入的。对于分布式表的 INSERT，数据块只写本地文件系统。之后会尽快地在后台发送到远程服务器。发送数据的周期性是由[distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms)和[distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms)设置。分布式引擎会分别发送每个插入数据的文件，但是你可以使用[distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts)设置启用批量发送文件。该设置通过更好地利用本地服务器和网络资源来提高集群性能。你应该检查表目录`/var/lib/clickhouse/data/database/table/`中的文件列表(等待发送的数据)来检查数据是否发送成功。执行后台任务的线程数可以通过[background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size)设置。 如果在 INSERT 到分布式表时服务器节点丢失或重启（如，设备故障），则插入的数据可能会丢失。如果在表目录中检测到损坏的数据分片，则会将其转移到«broken»子目录，并不再使用。 -启用 max_parallel_replicas 选项后，会在分表的所有副本上并行查询处理。更多信息，请参阅«设置，max_parallel_replicas»部分。 + +## 读取数据 {#distributed-reading-data} + +当查询一个`Distributed`表时，`SELECT`查询被发送到所有的分片，不管数据是如何分布在分片上的(它们可以完全随机分布)。当您添加一个新分片时，您不必将旧数据传输到它。相反，您可以使用更重的权重向其写入新数据——数据的分布会稍微不均匀，但查询将正确有效地工作。 + +当启用`max_parallel_replicas`选项时，查询处理将在单个分片中的所有副本之间并行化。更多信息，请参见[max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas)。 + +要了解更多关于分布式`in`和`global in`查询是如何处理的，请参考[这里](../../../sql-reference/operators/in.md#select-distributed-subqueries)文档。 + +## 虚拟列 {#virtual-columns} + +- `_shard_num` — 表`system.clusters` 中的 `shard_num` 值 . 数据类型: [UInt32](../../../sql-reference/data-types/int-uint.md). + +!!! note "备注" + 因为 [remote](../../../sql-reference/table-functions/remote.md) 和 [cluster](../../../sql-reference/table-functions/cluster.md) 表方法内部创建了分布式表， `_shard_num` 对他们都有效. + +**详见** +- [虚拟列](../../../engines/table-engines/index.md#table_engines-virtual_columns) 描述 +- [background_distributed_schedule_pool_size](../../../operations/settings/settings.md#background_distributed_schedule_pool_size) 设置 +- [shardNum()](../../../sql-reference/functions/other-functions.md#shard-num) 和 [shardCount()](../../../sql-reference/functions/other-functions.md#shard-count) 方法 + [原始文章](https://clickhouse.com/docs/en/operations/table_engines/distributed/) diff --git a/docs/zh/engines/table-engines/special/memory.md b/docs/zh/engines/table-engines/special/memory.md index 2e880934652..f906d64d4b3 100644 --- a/docs/zh/engines/table-engines/special/memory.md +++ b/docs/zh/engines/table-engines/special/memory.md @@ -1,7 +1,12 @@ +--- +toc_priority: 44 +toc_title: Memory +--- + # 内存表 {#memory} Memory 引擎以未压缩的形式将数据存储在 RAM 中。数据完全以读取时获得的形式存储。换句话说，从这张表中读取是很轻松的。并发数据访问是同步的。锁范围小：读写操作不会相互阻塞。不支持索引。查询是并行化的。在简单查询上达到最大速率（超过10 GB /秒），因为没有磁盘读取，不需要解压缩或反序列化数据。（值得注意的是，在许多情况下，与 MergeTree 引擎的性能几乎一样高）。重新启动服务器时，表中的数据消失，表将变为空。通常，使用此表引擎是不合理的。但是，它可用于测试，以及在相对较少的行（最多约100,000,000）上需要最高性能的查询。 Memory 引擎是由系统用于临时表进行外部数据的查询（请参阅 «外部数据用于请求处理» 部分），以及用于实现 `GLOBAL IN`（请参见 «IN 运算符» 部分）。 -[原始文章](https://clickhouse.com/docs/zh/operations/table_engines/memory/) +[原始文章](https://clickhouse.com/docs/en/operations/table_engines/memory/) diff --git a/docs/zh/faq/general/mapreduce.md b/docs/zh/faq/general/mapreduce.md index f70ca8a2583..99cb6c031ae 100644 --- a/docs/zh/faq/general/mapreduce.md +++ b/docs/zh/faq/general/mapreduce.md @@ -6,7 +6,7 @@ toc_priority: 110 # 为何不使用 MapReduce等技术? {#why-not-use-something-like-mapreduce} -我们可以将MapReduce这样的系统称为分布式计算系统，其中的reduce操作是基于分布式排序的。这个领域中最常见的开源解决方案是[Apache Hadoop](http://hadoop.apache.org)。Yandex使用其内部解决方案YT。 +我们可以将MapReduce这样的系统称为分布式计算系统，其中的reduce操作是基于分布式排序的。这个领域中最常见的开源解决方案是[Apache Hadoop](http://hadoop.apache.org)。 这些系统不适合用于在线查询，因为它们的延迟很大。换句话说，它们不能被用作网页界面的后端。这些类型的系统对于实时数据更新并不是很有用。如果操作的结果和所有中间结果(如果有的话)都位于单个服务器的内存中，那么分布式排序就不是执行reduce操作的最佳方式，这通常是在线查询的情况。在这种情况下，哈希表是执行reduce操作的最佳方式。优化map-reduce任务的一种常见方法是使用内存中的哈希表进行预聚合(部分reduce)。用户手动执行此优化。在运行简单的map-reduce任务时，分布式排序是导致性能下降的主要原因之一。 diff --git a/docs/zh/faq/general/who-is-using-clickhouse.md b/docs/zh/faq/general/who-is-using-clickhouse.md deleted file mode 120000 index b4e9782df7e..00000000000 --- a/docs/zh/faq/general/who-is-using-clickhouse.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/general/who-is-using-clickhouse.md \ No newline at end of file diff --git a/docs/zh/faq/general/who-is-using-clickhouse.md b/docs/zh/faq/general/who-is-using-clickhouse.md new file mode 100644 index 00000000000..3bfd94719d8 --- /dev/null +++ b/docs/zh/faq/general/who-is-using-clickhouse.md @@ -0,0 +1,19 @@ +--- +title: 谁在使用 ClickHouse? +toc_hidden: true +toc_priority: 9 +--- + +# 谁在使用 ClickHouse? {#who-is-using-clickhouse} + +作为一个开源产品，这个问题的答案并不那么简单。如果你想开始使用ClickHouse，你不需要告诉任何人，你只需要获取源代码或预编译包。不需要签署任何合同，[Apache 2.0许可证](https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE)允许不受约束的软件分发。 + +此外，技术堆栈通常处于保密协议所涵盖的灰色地带。一些公司认为他们使用的技术是一种竞争优势，即使这些技术是开源的，并且不允许员工公开分享任何细节。一些公司看到了一些公关风险，只允许员工在获得公关部门批准后分享实施细节。 + +那么，如何辨别谁在使用ClickHouse呢? + +一种方法是询问周围的人。如果不是书面形式，人们更愿意分享他们公司使用的技术、用例、使用的硬件类型、数据量等。我们定期在[ClickHouse meetup](https://www.youtube.com/channel/UChtmrD-dsdpspr42P_PyRAw/playlists)上与世界各地的用户进行交流，并听到了大约1000多家使用ClickHouse的公司的故事。不幸的是，这是不可复制的，我们试图把这些故事当作是在保密协议下被告知的，以避免任何潜在的麻烦。但你可以参加我们未来的任何聚会，并与其他用户单独交谈。有多种方式宣布聚会，例如，你可以订阅[我们的Twitter](http://twitter.com/ClickHouseDB/)。 + +第二种方法是寻找**公开表示**使用ClickHouse的公司。因为通常会有一些确凿的证据，如博客文章、谈话视频录音、幻灯片等。我们在我们的[**Adopters**](../../introduction/adopters.md)页面上收集指向此类证据的链接。你可以随意提供你雇主的故事，或者只是一些你偶然发现的链接(但尽量不要在这个过程中违反保密协议)。 + +你可以在采用者名单中找到一些非常大的公司，比如彭博社、思科、中国电信、腾讯或优步，但通过第一种方法，我们发现还有更多。例如，如果你看看《福布斯》[(2020年)列出的最大IT公司名单](https://www.forbes.com/sites/hanktucker/2020/05/13/worlds-largest-technology-companies-2020-apple-stays-on-top-zoom-and-uber-debut/)，超过一半的公司都在以某种方式使用ClickHouse。此外，不提[Yandex](../../introduction/history.md)是不公平的，该公司最初于2016年开放ClickHouse，碰巧是欧洲最大的it公司之一。 \ No newline at end of file diff --git a/docs/zh/faq/integration/file-export.md b/docs/zh/faq/integration/file-export.md deleted file mode 120000 index 19a5c67148b..00000000000 --- a/docs/zh/faq/integration/file-export.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/integration/file-export.md \ No newline at end of file diff --git a/docs/zh/faq/integration/file-export.md b/docs/zh/faq/integration/file-export.md new file mode 100644 index 00000000000..3582bfb1008 --- /dev/null +++ b/docs/zh/faq/integration/file-export.md @@ -0,0 +1,37 @@ +--- +title: 如何从 ClickHouse 导出数据到一个文件? +toc_hidden: true +toc_priority: 10 +--- + +# 如何从 ClickHouse 导出数据到一个文件? {#how-to-export-to-file} + +## 使用 INTO OUTFILE 语法 {#using-into-outfile-clause} + +加一个 [INTO OUTFILE](../../sql-reference/statements/select/into-outfile.md#into-outfile-clause) 语法到你的查询语句中. + +例如: + +``` sql +SELECT * FROM table INTO OUTFILE 'file' +``` + +ClickHouse 默认使用[TabSeparated](../../interfaces/formats.md#tabseparated) 格式写入数据. 修改[数据格式](../../interfaces/formats.md), 请用 [FORMAT 语法](../../sql-reference/statements/select/format.md#format-clause). + +例如: + +``` sql +SELECT * FROM table INTO OUTFILE 'file' FORMAT CSV +``` + +## 使用一个文件引擎表 {#using-a-file-engine-table} + +查看 [File](../../engines/table-engines/special/file.md) 表引擎. + +## 使用命令行重定向 {#using-command-line-redirection} + +``` bash +$ clickhouse-client --query "SELECT * from table" --format FormatName > result.txt +``` + +查看 [clickhouse-client](../../interfaces/cli.md). diff --git a/docs/zh/getting-started/example-datasets/github-events.md b/docs/zh/getting-started/example-datasets/github-events.md deleted file mode 120000 index c9649c0a61f..00000000000 --- a/docs/zh/getting-started/example-datasets/github-events.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/getting-started/example-datasets/github-events.md \ No newline at end of file diff --git a/docs/zh/getting-started/example-datasets/github-events.md b/docs/zh/getting-started/example-datasets/github-events.md new file mode 100644 index 00000000000..eabc3c1bf1f --- /dev/null +++ b/docs/zh/getting-started/example-datasets/github-events.md @@ -0,0 +1,10 @@ +--- +toc_priority: 11 +toc_title: GitHub 事件数据集 +--- + +# GitHub 事件数据集 + +数据集包含了GitHub上从2011年到2020年12月6日的所有事件，大小为31亿条记录。下载大小为75 GB，如果存储在使用lz4压缩的表中，则需要多达200 GB的磁盘空间。 + +完整的数据集描述，见解，下载说明和交互式查询请参考[这里](https://ghe.clickhouse.tech/)。 \ No newline at end of file diff --git a/docs/zh/getting-started/example-datasets/nyc-taxi.md b/docs/zh/getting-started/example-datasets/nyc-taxi.md index b10fe931c20..bf948528d89 100644 --- a/docs/zh/getting-started/example-datasets/nyc-taxi.md +++ b/docs/zh/getting-started/example-datasets/nyc-taxi.md @@ -375,7 +375,6 @@ Q3：0.051秒。 Q4：0.072秒。 在这种情况下，查询处理时间首先由网络延迟确定。 -我们使用位于芬兰Yandex数据中心的客户机在俄罗斯的一个集群上运行查询，这增加了大约20毫秒的延迟。 ## 总结 {#zong-jie} diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index 1bf8ecd9826..e74a05a9913 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -27,9 +27,17 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not {% include 'install/deb.sh' %} ``` +
+ +Deprecated Method for installing deb-packages +``` bash +{% include 'install/deb_repo.sh' %} +``` +
+ 如果您想使用最新的版本，请用`testing`替代`stable`(我们只推荐您用于测试环境)。 -你也可以从这里手动下载安装包：[下载](https://repo.clickhouse.com/deb/stable/main/)。 +你也可以从这里手动下载安装包：[下载](https://packages.clickhouse.com/deb/pool/stable)。 安装包列表： @@ -45,11 +53,17 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not 首先，您需要添加官方存储库： ``` bash -sudo yum install yum-utils -sudo rpm --import https://repo.clickhouse.com/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 +{% include 'install/rpm.sh' %} ``` +
+ +Deprecated Method for installing rpm-packages +``` bash +{% include 'install/rpm_repo.sh' %} +``` +
+ 如果您想使用最新的版本，请用`testing`替代`stable`(我们只推荐您用于测试环境)。`prestable`有时也可用。 然后运行命令安装： @@ -58,37 +72,28 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/stable/x86_64 sudo yum install clickhouse-server clickhouse-client ``` -你也可以从这里手动下载安装包：[下载](https://repo.clickhouse.com/rpm/stable/x86_64)。 +你也可以从这里手动下载安装包：[下载](https://packages.clickhouse.com/rpm/stable)。 ### `Tgz`安装包 {#from-tgz-archives} 如果您的操作系统不支持安装`deb`或`rpm`包，建议使用官方预编译的`tgz`软件包。 -所需的版本可以通过`curl`或`wget`从存储库`https://repo.clickhouse.com/tgz/`下载。 +所需的版本可以通过`curl`或`wget`从存储库`https://packages.clickhouse.com/tgz/`下载。 -下载后解压缩下载资源文件并使用安装脚本进行安装。以下是一个最新版本的安装示例: +下载后解压缩下载资源文件并使用安装脚本进行安装。以下是一个最新稳定版本的安装示例: ``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-client-$LATEST_VERSION.tgz - -tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz -sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-common-static-dbg-$LATEST_VERSION.tgz -sudo clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh - -tar -xzvf clickhouse-server-$LATEST_VERSION.tgz -sudo clickhouse-server-$LATEST_VERSION/install/doinst.sh -sudo /etc/init.d/clickhouse-server start - -tar -xzvf clickhouse-client-$LATEST_VERSION.tgz -sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh +{% include 'install/tgz.sh' %} ``` +
+ +Deprecated Method for installing tgz archives +``` bash +{% include 'install/tgz_repo.sh' %} +``` +
+ 对于生产环境，建议使用最新的`stable`版本。你可以在GitHub页面https://github.com/ClickHouse/ClickHouse/tags找到它，它以后缀`-stable`标志。 ### `Docker`安装包 {#from-docker-image} @@ -183,6 +188,6 @@ SELECT 1 **恭喜，系统已经工作了!** -为了继续进行实验，你可以尝试下载测试数据集或查看[教程](https://clickhouse.com/tutorial.html)。 +为了继续进行实验，你可以尝试下载测试数据集或查看[教程](./tutorial.md)。 [原始文章](https://clickhouse.com/docs/en/getting_started/install/) diff --git a/docs/zh/getting-started/playground.md b/docs/zh/getting-started/playground.md index 8fee8966a4f..f8f611d9d8d 100644 --- a/docs/zh/getting-started/playground.md +++ b/docs/zh/getting-started/playground.md @@ -3,69 +3,41 @@ toc_priority: 14 toc_title: 体验平台 --- -# ClickHouse体验平台 {#clickhouse-playground} +# ClickHouse Playground {#clickhouse-playground} -[ClickHouse体验平台](https://play.clickhouse.com?file=welcome) 允许人们通过即时运行查询来尝试ClickHouse，而无需设置他们的服务器或集群。 +[ClickHouse Playground](https://play.clickhouse.com/play?user=play) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. +Several example datasets are available in Playground. -体验平台中提供几个示例数据集以及显示ClickHouse特性的示例查询。还有一些ClickHouse LTS版本可供尝试。 - -ClickHouse体验平台提供了小型集群[Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse)实例配置(4 vCPU, 32 GB RAM)它们托管在[Yandex.Cloud](https://cloud.yandex.com/). 更多信息查询[cloud providers](../commercial/cloud.md). - -您可以使用任何HTTP客户端对ClickHouse体验平台进行查询，例如[curl](https://curl.haxx.se)或者[wget](https://www.gnu.org/software/wget/),或使用[JDBC](../interfaces/jdbc.md)或者[ODBC](../interfaces/odbc.md)驱动连接。关于支持ClickHouse的软件产品的更多信息详见[here](../interfaces/index.md). +You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). ## Credentials {#credentials} -| 参数 | 值 | -|:--------------------|:----------------------------------------| -| HTTPS端点 | `https://play-api.clickhouse.com:8443` | -| TCP端点 | `play-api.clickhouse.com:9440` | -| 用户 | `playground` | -| 密码 | `clickhouse` | +| Parameter | Value | +|:--------------------|:-----------------------------------| +| HTTPS endpoint | `https://play.clickhouse.com:443/` | +| Native TCP endpoint | `play.clickhouse.com:9440` | +| User | `explorer` or `play` | +| Password | (empty) | -还有一些带有特定ClickHouse版本的附加信息来试验它们之间的差异(端口和用户/密码与上面相同): +## Limitations {#limitations} -- 20.3 LTS: `play-api-v20-3.clickhouse.com` -- 19.14 LTS: `play-api-v19-14.clickhouse.com` +The queries are executed as a read-only user. It implies some limitations: -!!! note "注意" - 所有这些端点都需要安全的TLS连接。 +- DDL queries are not allowed +- INSERT queries are not allowed -## 查询限制 {#limitations} +The service also have quotas on its usage. -查询以只读用户身份执行。 这意味着一些局限性: +## Examples {#examples} -- 不允许DDL查询 -- 不允许插入查询 - -还强制执行以下设置: -- [max_result_bytes=10485760](../operations/settings/query-complexity/#max-result-bytes) -- [max_result_rows=2000](../operations/settings/query-complexity/#setting-max_result_rows) -- [result_overflow_mode=break](../operations/settings/query-complexity/#result-overflow-mode) -- [max_execution_time=60000](../operations/settings/query-complexity/#max-execution-time) - -ClickHouse体验还有如下： -[ClickHouse管理服务](https://cloud.yandex.com/services/managed-clickhouse) -实例托管 [Yandex云](https://cloud.yandex.com/)。 -更多信息 [云提供商](../commercial/cloud.md)。 - -## 示例 {#examples} - -使用`curl`连接Https服务： +HTTPS endpoint example with `curl`: ``` bash -curl "https://play-api.clickhouse.com:8443/?query=SELECT+'Play+ClickHouse\!';&user=playground&password=clickhouse&database=datasets" +curl "https://play.clickhouse.com/?user=explorer" --data-binary "SELECT 'Play ClickHouse'" ``` -TCP连接示例[CLI](../interfaces/cli.md): +TCP endpoint example with [CLI](../interfaces/cli.md): ``` bash -clickhouse client --secure -h play-api.clickhouse.com --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse\!'" +clickhouse client --secure --host play.clickhouse.com --user explorer ``` - -## Implementation Details {#implementation-details} - -ClickHouse体验平台界面实际上是通过ClickHouse [HTTP API](../interfaces/http.md)接口实现的。 -ClickHouse体验平台是一个ClickHouse集群，没有任何附加的服务器端应用程序。如上所述，ClickHouse的HTTPS和TCP/TLS端点也可以作为体验平台的一部分公开使用, 代理通过[Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/)增加一层额外的保护和改善连接。 - -!!! warning "注意" - **强烈不推荐**在任何其他情况下将ClickHouse服务器暴露给公共互联网。确保它只在私有网络上侦听，并由正确配置的防火墙监控。 diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index b579d57c634..40e9bfe7ff1 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -1240,7 +1240,8 @@ SELECT * FROM topic1_stream; | `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `DATE64` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `TIMESTAMP` | [DateTime64](../sql-reference/data-types/datetime64.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `STRING` | | — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | @@ -1295,7 +1296,8 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ | `FLOAT`, `HALF_FLOAT` | [Float32](../sql-reference/data-types/float.md) | `FLOAT` | | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `DATE32` | -| `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `TIMESTAMP` | +| `DATE64` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | +| `TIMESTAMP` | [DateTime64](../sql-reference/data-types/datetime64.md) | `TIMESTAMP` | | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `-` | [Array](../sql-reference/data-types/array.md) | `LIST` | @@ -1439,7 +1441,7 @@ f9725a22f9191e064120d718e26862a9 - 如果您通过[Client](../interfaces/cli.md) 在 [交互模式](https://clickhouse.com/docs/zh/interfaces/cli/#cli_usage)下输入或输出数据，格式架构中指定的文件名可以使用绝对路径或客户端当前目录的相对路径。 如果在[批处理模式](https://clickhouse.com/docs/zh/interfaces/cli/#cli_usage)下使用客户端，则由于安全原因，架构的路径必须使用相对路径。 -如果您通过 HTTP接口](../interfaces/http.md)输入或输出数据，格式架构中指定的文件名应该位于服务器设置的[format_schema_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path)指定的目录中。 +如果您通过 [HTTP接口](../interfaces/http.md)输入或输出数据，格式架构中指定的文件名应该位于服务器设置的[format_schema_path](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-format_schema_path)指定的目录中。 ## 跳过错误 {#skippingerrors} diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md index 075e1ca0870..291e6d907fb 100644 --- a/docs/zh/interfaces/third-party/integrations.md +++ b/docs/zh/interfaces/third-party/integrations.md @@ -6,7 +6,7 @@ toc_title: 第三方集成库 # 第三方集成库 {#integration-libraries-from-third-party-developers} !!! warning "声明" -Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。 + ClickHouse, Inc.**没有**维护下面列出的库，也没有做过任何广泛的测试来确保它们的质量。 ## 基础设施 {#infrastructure-products} diff --git a/docs/zh/operations/external-authenticators/ssl-x509.md b/docs/zh/operations/external-authenticators/ssl-x509.md new file mode 120000 index 00000000000..80b18e1b352 --- /dev/null +++ b/docs/zh/operations/external-authenticators/ssl-x509.md @@ -0,0 +1 @@ +../../../en/operations/external-authenticators/ssl-x509.md \ No newline at end of file diff --git a/docs/zh/operations/performance-test.md b/docs/zh/operations/performance-test.md index d3643969c2e..9761d516ddd 100644 --- a/docs/zh/operations/performance-test.md +++ b/docs/zh/operations/performance-test.md @@ -36,6 +36,18 @@ chmod a+x ./hardware.sh wget https://builds.clickhouse.com/master/amd64/clickhouse # For aarch64: wget https://builds.clickhouse.com/master/aarch64/clickhouse +# For powerpc64le: +wget https://builds.clickhouse.com/master/powerpc64le/clickhouse +# For freebsd: +wget https://builds.clickhouse.com/master/freebsd/clickhouse +# For freebsd-aarch64: +wget https://builds.clickhouse.com/master/freebsd-aarch64/clickhouse +# For freebsd-powerpc64le: +wget https://builds.clickhouse.com/master/freebsd-powerpc64le/clickhouse +# For macos: +wget https://builds.clickhouse.com/master/macos/clickhouse +# For macos-aarch64: +wget https://builds.clickhouse.com/master/macos-aarch64/clickhouse # Then do: chmod a+x clickhouse ``` diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md index d31dd5da805..8c1be5429d1 100644 --- a/docs/zh/operations/server-configuration-parameters/settings.md +++ b/docs/zh/operations/server-configuration-parameters/settings.md @@ -668,7 +668,7 @@ SSL客户端/服务器配置。 **示例** ``` xml -Europe/Moscow +Asia/Istanbul ``` ## tcp_port {#server_configuration_parameters-tcp_port} diff --git a/docs/zh/operations/system-tables/mutations.md b/docs/zh/operations/system-tables/mutations.md index 17313274bd5..8620436b8e3 100644 --- a/docs/zh/operations/system-tables/mutations.md +++ b/docs/zh/operations/system-tables/mutations.md @@ -1,30 +1,50 @@ ---- -machine_translated: true -machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ---- +# system.mutations {#system_tables-mutations} -# 系统。突变 {#system_tables-mutations} +该表包含关于MergeTree表的[mutation](../../sql-reference/statements/alter.md#alter-mutations)及其进度信息 。每条mutation命令都用一行来表示。 -该表包含以下信息 [突变](../../sql-reference/statements/alter.md#alter-mutations) MergeTree表及其进展。 每个突变命令由一行表示。 该表具有以下列: +该表具有以下列属性: -**数据库**, **表** -应用突变的数据库和表的名称。 +- `database` ([String](../../sql-reference/data-types/string.md)) — 应用mutation的数据库名称。 -**mutation_id** -变异的ID 对于复制的表，这些Id对应于znode中的名称 `/mutations/` 动物园管理员的目录。 对于未复制的表，Id对应于表的数据目录中的文件名。 +- `table` ([String](../../sql-reference/data-types/string.md)) — 应用mutation的表名称。 -**命令** -Mutation命令字符串（查询后的部分 `ALTER TABLE [db.]table`). +- `mutation_id` ([String](../../sql-reference/data-types/string.md)) — mutation的ID。对于复制表，这些ID对应于ZooKeeper中/mutations/目录下的znode名称。对于非复制表，ID对应表的数据目录中的文件名。 -**create_time** -当这个突变命令被提交执行。 +- `command` ([String](../../sql-reference/data-types/string.md)) — mutation命令字符串（`ALTER TABLE [db.]table`语句之后的部分)。 -**block_numbers.partition_id**, **block_numbers.编号** -嵌套列。 对于复制表的突变，它包含每个分区的一条记录：分区ID和通过突变获取的块编号（在每个分区中，只有包含编号小于该分区中突变获取的块编号的块的 在非复制表中，所有分区中的块编号形成一个序列。 这意味着对于非复制表的突变，该列将包含一条记录，其中包含由突变获取的单个块编号。 +- `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — mutation命令提交执行的日期和时间。 -**parts_to_do** -为了完成突变，需要突变的数据部分的数量。 +- `block_numbers.partition_id` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — 对于复制表的mutation，该数组包含分区的ID（每个分区都有一条记录）。对于非复制表的mutation，该数组为空。 -**is_done** -变异完成了?？ 请注意，即使 `parts_to_do = 0` 由于长时间运行的INSERT将创建需要突变的新数据部分，因此可能尚未完成复制表的突变。 +- `block_numbers.number` ([Array](../../sql-reference/data-types/array.md)([Int64](../../sql-reference/data-types/int-uint.md))) — 对于复制表的mutation，该数组包含每个分区的一条记录，以及通过mutation获取的块号。只有包含块号小于该数字的块的part才会在分区中应用mutation。 + + 在非复制表中，所有分区中的块号组成一个序列。这意味着对于非复制表的mutation，该列将包含一条记录，该记录具有通过mutation获得的单个块号。 + +- `parts_to_do_names` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — 由需要应用mutation的part名称构成的数组。 -如果在改变某些部分时出现问题，以下列将包含其他信息: +- `parts_to_do` ([Int64](../../sql-reference/data-types/int-uint.md)) — 需要应用mutation的part的数量。 -**latest_failed_part** -不能变异的最新部分的名称。 +- `is_done` ([UInt8](../../sql-reference/data-types/int-uint.md)) — mutation是否完成的标志。其中： + - 1，表示mutation已经完成。 + - 0，表示mutation仍在进行中。 -**latest_fail_time** -最近的部分突变失败的时间。 -**latest_fail_reason** -导致最近部件变异失败的异常消息。 +!!! info "注意" + 即使 parts_to_do = 0，由于长时间运行的`INSERT`查询将创建需要mutate的新part，也可能导致复制表mutation尚未完成。 + +如果某些parts在mutation时出现问题，以下列将包含附加信息： + +- `latest_failed_part`([String](../../sql-reference/data-types/string.md)) — 最近不能mutation的part的名称。 + +- `latest_fail_time`([Datetime](../../sql-reference/data-types/datetime.md)) — 最近的一个mutation失败的时间。 + +- `latest_fail_reason`([String](../../sql-reference/data-types/string.md)) — 导致最近part的mutation失败的异常消息。 + + +**另请参阅** + +- Mutations +- [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 表引擎 +- [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) 族 + +[Original article](https://clickhouse.com/docs/en/operations/system_tables/mutations) \ No newline at end of file diff --git a/docs/zh/operations/system-tables/numbers_mt.md b/docs/zh/operations/system-tables/numbers_mt.md index 185bee95171..cf1c96acaab 100644 --- a/docs/zh/operations/system-tables/numbers_mt.md +++ b/docs/zh/operations/system-tables/numbers_mt.md @@ -1,10 +1,5 @@ ---- -machine_translated: true -machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ---- +# system.numbers_mt {#system-numbers-mt} -# 系统。numbers_mt {#system-numbers-mt} - -一样的 [系统。数字](../../operations/system-tables/numbers.md) 但读取是并行的。 这些数字可以以任何顺序返回。 +与[system.numbers](../../operations/system-tables/numbers.md)相似，但读取是并行的。 这些数字可以以任何顺序返回。 用于测试。 diff --git a/docs/zh/sql-reference/data-types/date.md b/docs/zh/sql-reference/data-types/date.md index 8f1e0752179..ab5d3acae1b 100644 --- a/docs/zh/sql-reference/data-types/date.md +++ b/docs/zh/sql-reference/data-types/date.md @@ -2,4 +2,6 @@ 日期类型，用两个字节存储，表示从 1970-01-01 (无符号) 到当前的日期值。允许存储从 Unix 纪元开始到编译阶段定义的上限阈值常量（目前上限是2106年，但最终完全支持的年份为2105）。最小值输出为1970-01-01。 +值的范围: \[1970-01-01, 2149-06-06\]。 + 日期中没有存储时区信息。 diff --git a/docs/zh/sql-reference/data-types/datetime.md b/docs/zh/sql-reference/data-types/datetime.md index 0b3a7524f63..b6c8c3d2d35 100644 --- a/docs/zh/sql-reference/data-types/datetime.md +++ b/docs/zh/sql-reference/data-types/datetime.md @@ -2,6 +2,8 @@ 时间戳类型。用四个字节（无符号的）存储 Unix 时间戳）。允许存储与日期类型相同的范围内的值。最小值为 1970-01-01 00:00:00。时间戳类型值精确到秒（没有闰秒）。 +值的范围: \[1970-01-01 00:00:00, 2106-02-07 06:28:15\]。 + ## 时区 {#shi-qu} 使用启动客户端或服务器时的系统时区，时间戳是从文本（分解为组件）转换为二进制并返回。在文本格式中，有关夏令时的信息会丢失。 diff --git a/docs/zh/sql-reference/data-types/datetime64.md b/docs/zh/sql-reference/data-types/datetime64.md index 46e8e9a5fa4..6361b77d245 100644 --- a/docs/zh/sql-reference/data-types/datetime64.md +++ b/docs/zh/sql-reference/data-types/datetime64.md @@ -19,6 +19,8 @@ DateTime64(precision, [timezone]) 在内部，此类型以Int64类型将数据存储为自Linux纪元开始(1970-01-01 00:00:00UTC)的时间刻度数（ticks）。时间刻度的分辨率由precision参数确定。此外，`DateTime64` 类型可以像存储其他数据列一样存储时区信息，时区会影响 `DateTime64` 类型的值如何以文本格式显示，以及如何解析以字符串形式指定的时间数据 (‘2020-01-01 05:00:01.000’)。时区不存储在表的行中（也不在resultset中），而是存储在列的元数据中。详细信息请参考 [DateTime](datetime.md) 数据类型. +值的范围: \[1925-01-01 00:00:00, 2283-11-11 23:59:59.99999999\] (注意: 最大值的精度是8)。 + ## 示例 {#examples} **1.** 创建一个具有 `DateTime64` 类型列的表，并向其中插入数据: @@ -26,7 +28,7 @@ DateTime64(precision, [timezone]) ``` sql CREATE TABLE dt ( - `timestamp` DateTime64(3, 'Europe/Moscow'), + `timestamp` DateTime64(3, 'Asia/Istanbul'), `event_id` UInt8 ) ENGINE = TinyLog @@ -47,13 +49,13 @@ SELECT * FROM dt └─────────────────────────┴──────────┘ ``` -- 将日期时间作为integer类型插入时，它会被视为适当缩放的Unix时间戳(UTC)。`1546300800000` （精度为3）表示 `'2019-01-01 00:00:00'` UTC. 不过，因为 `timestamp` 列指定了 `Europe/Moscow` （UTC+3）的时区，当作为字符串输出时，它将显示为 `'2019-01-01 03:00:00'` -- 当把字符串作为日期时间插入时，它会被赋予时区信息。 `'2019-01-01 00:00:00'` 将被认为处于 `Europe/Moscow` 时区并被存储为 `1546290000000`. +- 将日期时间作为integer类型插入时，它会被视为适当缩放的Unix时间戳(UTC)。`1546300800000` （精度为3）表示 `'2019-01-01 00:00:00'` UTC. 不过，因为 `timestamp` 列指定了 `Asia/Istanbul` （UTC+3）的时区，当作为字符串输出时，它将显示为 `'2019-01-01 03:00:00'` +- 当把字符串作为日期时间插入时，它会被赋予时区信息。 `'2019-01-01 00:00:00'` 将被认为处于 `Asia/Istanbul` 时区并被存储为 `1546290000000`. **2.** 过滤 `DateTime64` 类型的值 ``` sql -SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow') +SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') ``` ``` text @@ -67,12 +69,12 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europ **3.** 获取 `DateTime64` 类型值的时区信息: ``` sql -SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x +SELECT toDateTime64(now(), 3, 'Asia/Istanbul') AS column, toTypeName(column) AS x ``` ``` text ┌──────────────────column─┬─x──────────────────────────────┐ -│ 2019-10-16 04:12:04.000 │ DateTime64(3, 'Europe/Moscow') │ +│ 2019-10-16 04:12:04.000 │ DateTime64(3, 'Asia/Istanbul') │ └─────────────────────────┴────────────────────────────────┘ ``` @@ -81,7 +83,7 @@ SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS ``` sql SELECT toDateTime64(timestamp, 3, 'Europe/London') as lon_time, -toDateTime64(timestamp, 3, 'Europe/Moscow') as mos_time +toDateTime64(timestamp, 3, 'Asia/Istanbul') as mos_time FROM dt ``` diff --git a/docs/zh/sql-reference/functions/date-time-functions.md b/docs/zh/sql-reference/functions/date-time-functions.md index 1225cf33699..969f71011fd 100644 --- a/docs/zh/sql-reference/functions/date-time-functions.md +++ b/docs/zh/sql-reference/functions/date-time-functions.md @@ -212,13 +212,13 @@ SELECT toStartOfSecond(dt64); ``` sql WITH toDateTime64('2020-01-01 10:20:30.999', 3) AS dt64 -SELECT toStartOfSecond(dt64, 'Europe/Moscow'); +SELECT toStartOfSecond(dt64, 'Asia/Istanbul'); ``` 结果: ``` text -┌─toStartOfSecond(dt64, 'Europe/Moscow')─┐ +┌─toStartOfSecond(dt64, 'Asia/Istanbul')─┐ │ 2020-01-01 13:20:30.000 │ └────────────────────────────────────────┘ ``` @@ -414,13 +414,13 @@ SELECT now(), date_trunc('hour', now()); 指定时区查询: ```sql -SELECT now(), date_trunc('hour', now(), 'Europe/Moscow'); +SELECT now(), date_trunc('hour', now(), 'Asia/Istanbul'); ``` 结果: ```text -┌───────────────now()─┬─date_trunc('hour', now(), 'Europe/Moscow')─┐ +┌───────────────now()─┬─date_trunc('hour', now(), 'Asia/Istanbul')─┐ │ 2020-09-28 10:46:26 │ 2020-09-28 13:00:00 │ └─────────────────────┴────────────────────────────────────────────┘ ``` @@ -468,13 +468,13 @@ SELECT now(); 指定时区查询: ``` sql -SELECT now('Europe/Moscow'); +SELECT now('Asia/Istanbul'); ``` 结果: ``` text -┌─now('Europe/Moscow')─┐ +┌─now('Asia/Istanbul')─┐ │ 2020-10-17 10:42:23 │ └──────────────────────┘ ``` diff --git a/docs/zh/sql-reference/functions/ext-dict-functions.md b/docs/zh/sql-reference/functions/ext-dict-functions.md index 12b9499cb64..87e19dc0119 100644 --- a/docs/zh/sql-reference/functions/ext-dict-functions.md +++ b/docs/zh/sql-reference/functions/ext-dict-functions.md @@ -31,7 +31,7 @@ - 对于’dict_name’分层字典，查找’child_id’键是否位于’ancestor_id’内（或匹配’ancestor_id’）。返回UInt8。 -## 独裁主义 {#dictgethierarchy} +## dictGetHierarchy {#dictgethierarchy} `dictGetHierarchy('dict_name', id)` diff --git a/docs/zh/sql-reference/functions/type-conversion-functions.md b/docs/zh/sql-reference/functions/type-conversion-functions.md index c1d1e66664e..09fe30a4400 100644 --- a/docs/zh/sql-reference/functions/type-conversion-functions.md +++ b/docs/zh/sql-reference/functions/type-conversion-functions.md @@ -439,7 +439,7 @@ AS parseDateTimeBestEffort; 查询: ``` sql -SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Europe/Moscow') +SELECT parseDateTimeBestEffort('Sat, 18 Aug 2018 07:22:16 GMT', 'Asia/Istanbul') AS parseDateTimeBestEffort ``` diff --git a/docs/zh/sql-reference/statements/alter/constraint.md b/docs/zh/sql-reference/statements/alter/constraint.md deleted file mode 120000 index e3b245408d1..00000000000 --- a/docs/zh/sql-reference/statements/alter/constraint.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/constraint.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/constraint.md b/docs/zh/sql-reference/statements/alter/constraint.md new file mode 100644 index 00000000000..9a71f0b055c --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/constraint.md @@ -0,0 +1,22 @@ +--- +toc_priority: 43 +toc_title: 约束 +--- + +# 操作约束 {#manipulations-with-constraints} + +约束可以使用以下语法添加或删除: + +``` sql +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +查看[constraints](../../../sql-reference/statements/create/table.md#constraints)。 + +查询将从表中添加或删除关于约束的元数据，因此它们将被立即处理。 + +!!! warning "警告" + 如果已有数据被添加，约束检查**将不会被执行**。 + +复制表上的所有更改都会被广播到ZooKeeper，并应用到其他副本上。 \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/delete.md b/docs/zh/sql-reference/statements/alter/delete.md deleted file mode 120000 index b9d6a233197..00000000000 --- a/docs/zh/sql-reference/statements/alter/delete.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/delete.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/delete.md b/docs/zh/sql-reference/statements/alter/delete.md new file mode 100644 index 00000000000..390c7becb14 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/delete.md @@ -0,0 +1,27 @@ +--- +toc_priority: 39 +toc_title: DELETE +--- + +# ALTER TABLE … DELETE 语句 {#alter-mutations} + +``` sql +ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr +``` + +删除匹配指定过滤表达式的数据。实现为[突变](../../../sql-reference/statements/alter/index.md#mutations). + +!!! note "备注" + `ALTER TABLE`前缀使得这个语法不同于大多数其他支持SQL的系统。它的目的是表示，与OLTP数据库中的类似查询不同，这是一个不为经常使用而设计的繁重操作。 + +`filter_expr` 的类型必须是`UInt8`。该查询删除表中该表达式接受非零值的行。 + +一个查询可以包含多个用逗号分隔的命令。 + +查询处理的同步性由[mutations_sync](../../../operations/settings/settings.md#mutations_sync)设置定义。缺省情况下，是异步的。 + +**详见** + +- [突变](../../../sql-reference/statements/alter/index.md#mutations) +- [ALTER查询的同步性](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) +- [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting diff --git a/docs/zh/sql-reference/statements/alter/order-by.md b/docs/zh/sql-reference/statements/alter/order-by.md deleted file mode 120000 index ef5a22dc165..00000000000 --- a/docs/zh/sql-reference/statements/alter/order-by.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/order-by.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/order-by.md b/docs/zh/sql-reference/statements/alter/order-by.md new file mode 100644 index 00000000000..112571b55f6 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/order-by.md @@ -0,0 +1,17 @@ +--- +toc_priority: 41 +toc_title: ORDER BY +--- + +# 操作排序键表达式 {#manipulations-with-key-expressions} + +```sql +ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY ORDER BY new_expression +``` +该命令将表的[排序键](../../../engines/table-engines/mergetree-family/mergetree.md)更改为 `new_expression`(表达式或表达式元组)。主键保持不变。 + +从某种意义上说，该命令是轻量级的，它只更改元数据。要保持数据部分行按排序键表达式排序的属性，您不能向排序键添加包含现有列的表达式(仅在相同的`ALTER`查询中由`ADD COLUMN`命令添加的列，没有默认的列值)。 + + +!!! note "备注" + 它只适用于[`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md)表族(包括[replicated](../../../engines/table-engines/mergetree-family/replication.md)表)。 diff --git a/docs/zh/sql-reference/statements/alter/role.md b/docs/zh/sql-reference/statements/alter/role.md deleted file mode 120000 index ce1f0a94eb3..00000000000 --- a/docs/zh/sql-reference/statements/alter/role.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/role.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/role.md b/docs/zh/sql-reference/statements/alter/role.md new file mode 100644 index 00000000000..3f5c5daf7b8 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/role.md @@ -0,0 +1,16 @@ +--- +toc_priority: 46 +toc_title: 角色 +--- + +## 操作角色 {#alter-role-statement} + +修改角色. + +语法示例: + +``` sql +ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] + [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` diff --git a/docs/zh/sql-reference/statements/alter/row-policy.md b/docs/zh/sql-reference/statements/alter/row-policy.md deleted file mode 120000 index 09ad2d301f3..00000000000 --- a/docs/zh/sql-reference/statements/alter/row-policy.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/row-policy.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/row-policy.md b/docs/zh/sql-reference/statements/alter/row-policy.md new file mode 100644 index 00000000000..0cdba239b84 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/row-policy.md @@ -0,0 +1,19 @@ +--- +toc_priority: 47 +toc_title: 行策略 +--- + +# 操作行策略 {#alter-row-policy-statement} + +修改行策略. + +语法: + +``` sql +ALTER [ROW] POLICY [IF EXISTS] name1 [ON CLUSTER cluster_name1] ON [database1.]table1 [RENAME TO new_name1] + [, name2 [ON CLUSTER cluster_name2] ON [database2.]table2 [RENAME TO new_name2] ...] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` diff --git a/docs/zh/sql-reference/statements/alter/settings-profile.md b/docs/zh/sql-reference/statements/alter/settings-profile.md deleted file mode 120000 index 0e71ac4e831..00000000000 --- a/docs/zh/sql-reference/statements/alter/settings-profile.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/settings-profile.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/settings-profile.md b/docs/zh/sql-reference/statements/alter/settings-profile.md new file mode 100644 index 00000000000..045b2461e8c --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/settings-profile.md @@ -0,0 +1,16 @@ +--- +toc_priority: 48 +toc_title: 配置文件设置 +--- + +## 更改配置文件设置 {#alter-settings-profile-statement} + +更改配置文件设置。 + +语法: + +``` sql +ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] + [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` diff --git a/docs/zh/sql-reference/statements/alter/view.md b/docs/zh/sql-reference/statements/alter/view.md deleted file mode 120000 index 89bdcee5912..00000000000 --- a/docs/zh/sql-reference/statements/alter/view.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/view.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/view.md b/docs/zh/sql-reference/statements/alter/view.md new file mode 100644 index 00000000000..b8319ea35a6 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/view.md @@ -0,0 +1,44 @@ +--- +toc_priority: 50 +toc_title: VIEW +--- + +# ALTER TABLE … MODIFY QUERY 语句 {#alter-modify-query} + +当使用`ALTER TABLE … MODIFY QUERY`语句创建一个[物化视图](../create/view.md#materialized)时，可以修改`SELECT`查询。当物化视图在没有 `TO [db.]name` 的情况下创建时使用它。必须启用 `allow_experimental_alter_materialized_view_structure`设置。 + +如果一个物化视图使用`TO [db.]name`，你必须先 [DETACH](../detach.md) 视图。用[ALTER TABLE](index.md)修改目标表，然后 [ATTACH](../attach.md)之前分离的(`DETACH`)视图。 + +**示例** + +```sql +CREATE TABLE src_table (`a` UInt32) ENGINE = MergeTree ORDER BY a; +CREATE MATERIALIZED VIEW mv (`a` UInt32) ENGINE = MergeTree ORDER BY a AS SELECT a FROM src_table; +INSERT INTO src_table (a) VALUES (1), (2); +SELECT * FROM mv; +``` +```text +┌─a─┐ +│ 1 │ +│ 2 │ +└───┘ +``` +```sql +ALTER TABLE mv MODIFY QUERY SELECT a * 2 as a FROM src_table; +INSERT INTO src_table (a) VALUES (3), (4); +SELECT * FROM mv; +``` +```text +┌─a─┐ +│ 6 │ +│ 8 │ +└───┘ +┌─a─┐ +│ 1 │ +│ 2 │ +└───┘ +``` + +## ALTER LIVE VIEW 语句 {#alter-live-view} + +`ALTER LIVE VIEW ... REFRESH` 语句刷新一个 [实时视图](../create/view.md#live-view). 参见 [强制实时视图刷新](../create/view.md#live-view-alter-refresh). diff --git a/docs/zh/sql-reference/statements/select/limit-by.md b/docs/zh/sql-reference/statements/select/limit-by.md index f5ed5b1bf98..9b93bb9cf21 100644 --- a/docs/zh/sql-reference/statements/select/limit-by.md +++ b/docs/zh/sql-reference/statements/select/limit-by.md @@ -11,7 +11,7 @@ ClickHouse支持以下语法变体: - `LIMIT [offset_value, ]n BY expressions` - `LIMIT n OFFSET offset_value BY expressions` -在查询处理过程中，ClickHouse会选择按排序键排序的数据。 排序键使用以下命令显式设置 [ORDER BY](../../../sql-reference/statements/select/order-by.md) 子句或隐式作为表引擎的属性。 然后ClickHouse应用 `LIMIT n BY expressions` 并返回第一 `n` 每个不同组合的行 `expressions`. 如果 `OFFSET` 被指定，则对于每个数据块属于一个不同的组合 `expressions`，ClickHouse跳过 `offset_value` 从块开始的行数，并返回最大值 `n` 行的结果。 如果 `offset_value` 如果数据块中的行数大于数据块中的行数，ClickHouse将从该块返回零行。 +在进行查询处理时，ClickHouse选择按排序键排序的数据。排序键设置显式地使用一个[ORDER BY](order-by.md#select-order-by)条款或隐式属性表的引擎(行顺序只是保证在使用[ORDER BY](order-by.md#select-order-by),否则不会命令行块由于多线程)。然后ClickHouse应用`LIMIT n BY 表达式`，并为每个不同的`表达式`组合返回前n行。如果指定了`OFFSET`，那么对于每个属于不同`表达式`组合的数据块，ClickHouse将跳过`offset_value`从块开始的行数，并最终返回最多`n`行的结果。如果`offset_value`大于数据块中的行数，则ClickHouse从数据块中返回零行。 !!! note "注" `LIMIT BY` 是不相关的 [LIMIT](../../../sql-reference/statements/select/limit.md). 它们都可以在同一个查询中使用。 diff --git a/docs/zh/whats-new/changelog/2018.md b/docs/zh/whats-new/changelog/2018.md index c461a3e7d0a..c87df7966f4 100644 --- a/docs/zh/whats-new/changelog/2018.md +++ b/docs/zh/whats-new/changelog/2018.md @@ -948,7 +948,7 @@ - 添加了对表中多维数组和元组 (`Tuple` 数据类型) 的存储的支持. - 支持用于 `DESCRIBE` 和 `INSERT` 查询的表函数. 在 `DESCRIBE` 中添加了对子查询的支持. 示例：`DESC TABLE remote('host', default.hits)`; `DESC 表（选择 1）`; `插入表功能远程（'host'，default.hits）`. 除了 `INSERT INTO` , 还支持 `INSERT INTO TABLE`. -- 改进了对时区的支持. `DateTime` 数据类型可以使用用于解析和格式化文本格式的时区进行注释. 示例: `DateTime('Europe/Moscow')`. 当在函数中为 DateTime 参数指定时区时, 返回类型将跟踪时区, 并且值将按预期显示. +- 改进了对时区的支持. `DateTime` 数据类型可以使用用于解析和格式化文本格式的时区进行注释. 示例: `DateTime('Asia/Istanbul')`. 当在函数中为 DateTime 参数指定时区时, 返回类型将跟踪时区, 并且值将按预期显示. - 添加了函数`toTimeZone`、`timeDiff`、`toQuarter`、`toRelativeQuarterNum`. `toRelativeHour`/`Minute`/`Second` 函数可以将 `Date` 类型的值作为参数. `now` 函数名区分大小写. - 添加了 `toStartOfFifteenMinutes` 函数 (Kirill Shvakov). - 添加了用于格式化查询的 `clickhouse format` 工具. diff --git a/docs/zh/whats-new/changelog/2019.md b/docs/zh/whats-new/changelog/2019.md index aa7dc777f9c..5eeaf9226c4 100644 --- a/docs/zh/whats-new/changelog/2019.md +++ b/docs/zh/whats-new/changelog/2019.md @@ -858,7 +858,7 @@ - 使用 MySQL 样式标识符引用修复对 MySQL 引擎的插入和选择查询. [#5704](https://github.com/ClickHouse/ClickHouse/pull/5704) ([Winter Zhang](https://github.com/zhang2014)) - 现在 `CHECK TABLE` 查询可以与 MergeTree 引擎系列一起使用. 如果每个部分 (或在更简单的引擎情况下的文件) 有任何检查状态和消息, 它会返回检查状态和消息. 此外, 修复了获取损坏部分的错误. [#5865](https://github.com/ClickHouse/ClickHouse/pull/5865) ([alesapin](https://github.com/alesapin)) - 修复 SPLIT_SHARED_LIBRARIES 运行时. [#5793](https://github.com/ClickHouse/ClickHouse/pull/5793) ([Danila Kutenin](https://github.com/danlark1)) -- 当 `/etc/localtime` 是一个像 `../usr/share/zoneinfo/Europe/Moscow` 这样的相对符号链接时, 修复了时区初始化 [#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) +- 当 `/etc/localtime` 是一个像 `../usr/share/zoneinfo/Asia/Istanbul` 这样的相对符号链接时, 修复了时区初始化 [#5922](https://github.com/ClickHouse/ClickHouse/pull/5922) ([alexey-milovidov](https://github.com/alexey-milovidov)) - clickhouse-copier: 修复关机后免费使用. [#5752](https://github.com/ClickHouse/ClickHouse/pull/5752) ([proller](https://github.com/proller)) - 更新了 `simdjson` . 修复部分无效的零字节JSON解析成功的问题. [#5938](https://github.com/ClickHouse/ClickHouse/pull/5938) ([alexey-milovidov](https://github.com/alexey-milovidov)) - 修复关闭系统日志 [#5802](https://github.com/ClickHouse/ClickHouse/pull/5802) ([Anton Popov](https://github.com/CurtizJ)) diff --git a/docs/zh/whats-new/changelog/2020.md b/docs/zh/whats-new/changelog/2020.md index 19e9125224c..6890f0f551e 100644 --- a/docs/zh/whats-new/changelog/2020.md +++ b/docs/zh/whats-new/changelog/2020.md @@ -2962,7 +2962,7 @@ * 更新了对 clickhouse-test 脚本中挂起查询的检查. [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz)) * 从存储库中删除了一些无用的文件. [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov)) * 将数学性能测试的类型从 `once` 更改为 `loop` . [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* 添加 docker 图像，它允许为我们的代码库构建交互式代码浏览器 HTML 报告. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html) +* 添加 docker 图像，它允许为我们的代码库构建交互式代码浏览器 HTML 报告. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/ClickHouse/dbms/index.html) * 抑制 MSan 下的一些测试失败. [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm)) * 加速 `exception while insert` 测试. 此测试经常在 debug-with-coverage 构建中超时. [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov)) * 将 `libcxx` 和 `libcxxabi` 更新为 master. 准备 [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov)) diff --git a/packages/.gitignore b/packages/.gitignore new file mode 100644 index 00000000000..355164c1265 --- /dev/null +++ b/packages/.gitignore @@ -0,0 +1 @@ +*/ diff --git a/packages/build b/packages/build new file mode 100755 index 00000000000..53a7538f80e --- /dev/null +++ b/packages/build @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -e + +# Avoid dependency on locale +LC_ALL=C + +# Normalize output directory +if [ -n "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(realpath -m "$OUTPUT_DIR") +fi + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$CUR_DIR" + +ROOT_DIR=$(readlink -f "$(git rev-parse --show-cdup)") + +PKG_ROOT='root' + +DEB_ARCH=${DEB_ARCH:-amd64} +OUTPUT_DIR=${OUTPUT_DIR:-$ROOT_DIR} +[ -d "${OUTPUT_DIR}" ] || mkdir -p "${OUTPUT_DIR}" +SANITIZER=${SANITIZER:-""} +SOURCE=${SOURCE:-$PKG_ROOT} + +HELP="${0} [--test] [--rpm] [-h|--help] + --test - adds '+test' prefix to version + --apk - build APK packages + --rpm - build RPM packages + --tgz - build tarball package + --help - show this help and exit + +Used envs: + DEB_ARCH='${DEB_ARCH}' + OUTPUT_DIR='${OUTPUT_DIR}' - where the artifact will be placed + SANITIZER='${SANITIZER}' - if any sanitizer is used, affects version string + SOURCE='${SOURCE}' - directory with sources tree + VERSION_STRING='${VERSION_STRING}' - the package version to overwrite +" + +if [ -z "${VERSION_STRING}" ]; then + # Get CLICKHOUSE_VERSION_STRING from the current git repo + eval "$("$ROOT_DIR/tests/ci/version_helper.py" -e)" +else + CLICKHOUSE_VERSION_STRING=${VERSION_STRING} +fi +export CLICKHOUSE_VERSION_STRING + + + +while [[ $1 == --* ]] +do + case "$1" in + --test ) + VERSION_POSTFIX+='+test' + shift ;; + --apk ) + MAKE_APK=1 + shift ;; + --rpm ) + MAKE_RPM=1 + shift ;; + --tgz ) + MAKE_TGZ=1 + shift ;; + --help ) + echo "$HELP" + exit ;; + * ) + echo "Unknown option $1" + exit 2 ;; + esac +done + +function deb2tgz { + local FILE PKG_NAME PKG_DIR PKG_PATH TARBALL + FILE=$1 + PKG_NAME=${FILE##*/}; PKG_NAME=${PKG_NAME%%_*} + PKG_DIR="$PKG_NAME-$CLICKHOUSE_VERSION_STRING" + PKG_PATH="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING" + TARBALL="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING-$DEB_ARCH.tgz" + rm -rf "$PKG_PATH" + dpkg-deb -R "$FILE" "$PKG_PATH" + mkdir -p "$PKG_PATH/install" + cat > "$PKG_PATH/install/doinst.sh" << 'EOF' +#!/bin/sh +set -e + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +for filepath in `find $SCRIPTPATH/.. -type f -or -type l | grep -v "\.\./install/"`; do + destpath=${filepath##$SCRIPTPATH/..} + mkdir -p $(dirname "$destpath") + cp -r "$filepath" "$destpath" +done +EOF + chmod +x "$PKG_PATH/install/doinst.sh" + if [ -f "$PKG_PATH/DEBIAN/postinst" ]; then + tail +2 "$PKG_PATH/DEBIAN/postinst" > "$PKG_PATH/install/doinst.sh" + fi + rm -rf "$PKG_PATH/DEBIAN" + if [ -f "/usr/bin/pigz" ]; then + tar --use-compress-program=pigz -cf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR" + else + tar -czf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR" + fi + + rm -r "$PKG_PATH" +} + +# Build options +if [ -n "$SANITIZER" ]; then + if [[ "$SANITIZER" == "address" ]]; then VERSION_POSTFIX+="+asan" + elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan" + elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan" + elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan" + else + echo "Unknown value of SANITIZER variable: $SANITIZER" + exit 3 + fi +elif [[ $BUILD_TYPE == 'debug' ]]; then + VERSION_POSTFIX+="+debug" +fi + +if [[ "$PKG_ROOT" != "$SOURCE" ]]; then + # packages are built only from PKG_SOURCE + rm -rf "./$PKG_ROOT" + ln -sf "$SOURCE" "$PKG_SOURCE" +fi + +CLICKHOUSE_VERSION_STRING+=$VERSION_POSTFIX +echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING" + +for config in clickhouse*.yaml; do + echo "Building deb package for $config" + + # Preserve package path + exec 9>&1 + PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/fd/9) + PKG_PATH=${PKG_PATH##*created package: } + exec 9>&- + + if [ -n "$MAKE_APK" ]; then + echo "Building apk package for $config" + nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk + fi + if [ -n "$MAKE_RPM" ]; then + echo "Building rpm package for $config" + nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm + fi + if [ -n "$MAKE_TGZ" ]; then + echo "Building tarball for $config" + deb2tgz "$PKG_PATH" + fi +done + +# vim: ts=4: sw=4: sts=4: expandtab diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml new file mode 100644 index 00000000000..2a1389b6625 --- /dev/null +++ b/packages/clickhouse-client.yaml @@ -0,0 +1,57 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-client" +arch: "all" +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +replaces: +- clickhouse-compressor +conflicts: +- clickhouse-compressor + +maintainer: "ClickHouse Dev Team " +description: | + Client binary for ClickHouse + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + This package provides clickhouse-client , clickhouse-local and clickhouse-benchmark + +overrides: + deb: + depends: + - clickhouse-common-static (= ${CLICKHOUSE_VERSION_STRING}) + rpm: + depends: + - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} + +contents: +- src: root/etc/clickhouse-client/config.xml + dst: /etc/clickhouse-client/config.xml + type: config +- src: root/usr/bin/clickhouse-benchmark + dst: /usr/bin/clickhouse-benchmark +- src: root/usr/bin/clickhouse-compressor + dst: /usr/bin/clickhouse-compressor +- src: root/usr/bin/clickhouse-format + dst: /usr/bin/clickhouse-format +- src: root/usr/bin/clickhouse-client + dst: /usr/bin/clickhouse-client +- src: root/usr/bin/clickhouse-local + dst: /usr/bin/clickhouse-local +- src: root/usr/bin/clickhouse-obfuscator + dst: /usr/bin/clickhouse-obfuscator +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-client/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-client/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-client/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-client/README.md diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml new file mode 100644 index 00000000000..1213f4215c8 --- /dev/null +++ b/packages/clickhouse-common-static-dbg.yaml @@ -0,0 +1,34 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-common-static-dbg" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +replaces: +- clickhouse-common-dbg +conflicts: +- clickhouse-common-dbg + +maintainer: "ClickHouse Dev Team " +description: | + debugging symbols for clickhouse-common-static + This package contains the debugging symbols for clickhouse-common. + +contents: +- src: root/usr/lib/debug + dst: /usr/lib/debug +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-common-static-dbg/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-common-static-dbg/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-common-static-dbg/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-common-static-dbg/README.md diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml new file mode 100644 index 00000000000..269d4318e5e --- /dev/null +++ b/packages/clickhouse-common-static.yaml @@ -0,0 +1,48 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-common-static" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +replaces: +- clickhouse-common +- clickhouse-server-base +provides: +- clickhouse-common +- clickhouse-server-base +suggests: +- clickhouse-common-static-dbg + +maintainer: "ClickHouse Dev Team " +description: | + Common files for ClickHouse + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + This package provides common files for both clickhouse server and client + +contents: +- src: root/usr/bin/clickhouse + dst: /usr/bin/clickhouse +- src: root/usr/bin/clickhouse-odbc-bridge + dst: /usr/bin/clickhouse-odbc-bridge +- src: root/usr/bin/clickhouse-library-bridge + dst: /usr/bin/clickhouse-library-bridge +- src: root/usr/bin/clickhouse-extract-from-config + dst: /usr/bin/clickhouse-extract-from-config +- src: root/usr/share/bash-completion/completions + dst: /usr/share/bash-completion/completions +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-common-static/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-common-static/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-common-static/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-common-static/README.md diff --git a/packages/clickhouse-rpm.repo b/packages/clickhouse-rpm.repo new file mode 100644 index 00000000000..27321123dc1 --- /dev/null +++ b/packages/clickhouse-rpm.repo @@ -0,0 +1,31 @@ +[clickhouse-stable] +name=ClickHouse - Stable Repository +baseurl=https://packages.clickhouse.com/rpm/stable/ +gpgkey=https://packages.clickhouse.com/rpm/stable/repodata/repomd.xml.key +gpgcheck=0 +repo_gpgcheck=1 +enabled=0 + +[clickhouse-lts] +name=ClickHouse - LTS Repository +baseurl=https://packages.clickhouse.com/rpm/lts/ +gpgkey=https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key +gpgcheck=0 +repo_gpgcheck=1 +enabled=0 + +[clickhouse-prestable] +name=ClickHouse - Pre-stable Repository +baseurl=https://packages.clickhouse.com/rpm/prestable/ +gpgkey=https://packages.clickhouse.com/rpm/prestable/repodata/repomd.xml.key +gpgcheck=0 +repo_gpgcheck=1 +enabled=0 + +[clickhouse-testing] +name=ClickHouse - Testing Repository +baseurl=https://packages.clickhouse.com/rpm/testing/ +gpgkey=https://packages.clickhouse.com/rpm/testing/repodata/repomd.xml.key +gpgcheck=0 +repo_gpgcheck=1 +enabled=1 diff --git a/packages/clickhouse-server.init b/packages/clickhouse-server.init new file mode 100755 index 00000000000..1695f6286b8 --- /dev/null +++ b/packages/clickhouse-server.init @@ -0,0 +1,227 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: clickhouse-server +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Should-Start: $time $network +# Should-Stop: $network +# Short-Description: clickhouse-server daemon +### END INIT INFO +# +# NOTES: +# - Should-* -- script can start if the listed facilities are missing, unlike Required-* +# +# For the documentation [1]: +# +# [1]: https://wiki.debian.org/LSBInitScripts + +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_GROUP=${CLICKHOUSE_USER} +SHELL=/bin/bash +PROGRAM=clickhouse-server +CLICKHOUSE_GENERIC_PROGRAM=clickhouse +CLICKHOUSE_PROGRAM_ENV="" +EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config +CLICKHOUSE_CONFDIR=/etc/$PROGRAM +CLICKHOUSE_LOGDIR=/var/log/clickhouse-server +CLICKHOUSE_LOGDIR_USER=root +CLICKHOUSE_DATADIR=/var/lib/clickhouse +if [ -d "/var/lock" ]; then + LOCALSTATEDIR=/var/lock +else + LOCALSTATEDIR=/run/lock +fi + +if [ ! -d "$LOCALSTATEDIR" ]; then + mkdir -p "$LOCALSTATEDIR" +fi + +CLICKHOUSE_BINDIR=/usr/bin +CLICKHOUSE_CRONFILE=/etc/cron.d/clickhouse-server +CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml +LOCKFILE=$LOCALSTATEDIR/$PROGRAM +CLICKHOUSE_PIDDIR=/var/run/$PROGRAM +CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid" +# CLICKHOUSE_STOP_TIMEOUT=60 # Disabled by default. Place to /etc/default/clickhouse if you need. + +# Some systems lack "flock" +command -v flock >/dev/null && FLOCK=flock + +# Override defaults from optional config file +test -f /etc/default/clickhouse && . /etc/default/clickhouse + + +die() +{ + echo $1 >&2 + exit 1 +} + + +# Check that configuration file is Ok. +check_config() +{ + if [ -x "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG" ]; then + su -s $SHELL ${CLICKHOUSE_USER} -c "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path" >/dev/null || die "Configuration file ${CLICKHOUSE_CONFIG} doesn't parse successfully. Won't restart server. You may use forcerestart if you are sure."; + fi +} + + +initdb() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" +} + + +start() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" +} + + +stop() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}" +} + + +restart() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" +} + + +forcestop() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} stop --force --pid-path "${CLICKHOUSE_PIDDIR}" +} + + +service_or_func() +{ + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + systemctl $1 $PROGRAM + else + $1 + fi +} + +forcerestart() +{ + forcestop + # Should not use 'start' function if systemd active + service_or_func start +} + +use_cron() +{ + # 1. running systemd + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + return 1 + fi + # 2. disabled by config + if [ -z "$CLICKHOUSE_CRONFILE" ]; then + return 2 + fi + return 0 +} +# returns false if cron disabled (with systemd) +enable_cron() +{ + use_cron && sed -i 's/^#*//' "$CLICKHOUSE_CRONFILE" +} +# returns false if cron disabled (with systemd) +disable_cron() +{ + use_cron && sed -i 's/^#*/#/' "$CLICKHOUSE_CRONFILE" +} + + +is_cron_disabled() +{ + use_cron || return 0 + + # Assumes that either no lines are commented or all lines are commented. + # Also please note, that currently cron file for ClickHouse has only one line (but some time ago there was more). + grep -q -E '^#' "$CLICKHOUSE_CRONFILE"; +} + + +main() +{ + # See how we were called. + EXIT_STATUS=0 + case "$1" in + start) + service_or_func start && enable_cron + ;; + stop) + disable_cron + service_or_func stop + ;; + restart) + service_or_func restart && enable_cron + ;; + forcestop) + disable_cron + forcestop + ;; + forcerestart) + forcerestart && enable_cron + ;; + reload) + service_or_func restart + ;; + condstart) + service_or_func start + ;; + condstop) + service_or_func stop + ;; + condrestart) + service_or_func restart + ;; + condreload) + service_or_func restart + ;; + initdb) + initdb + ;; + enable_cron) + enable_cron + ;; + disable_cron) + disable_cron + ;; + *) + echo "Usage: $0 {start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload|initdb}" + exit 2 + ;; + esac + + exit $EXIT_STATUS +} + + +status() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} status --pid-path "${CLICKHOUSE_PIDDIR}" +} + + +# Running commands without need of locking +case "$1" in +status) + status + exit 0 + ;; +esac + + +( + if $FLOCK -n 9; then + main "$@" + else + echo "Init script is already running" && exit 1 + fi +) 9> $LOCKFILE diff --git a/packages/clickhouse-server.postinstall b/packages/clickhouse-server.postinstall new file mode 100644 index 00000000000..419c13e3daf --- /dev/null +++ b/packages/clickhouse-server.postinstall @@ -0,0 +1,47 @@ +#!/bin/sh +set -e +# set -x + +PROGRAM=clickhouse-server +CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse} +CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}} +# Please note that we don't support paths with whitespaces. This is rather ignorant. +CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:=/etc/clickhouse-server} +CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:=/var/lib/clickhouse} +CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:=/var/log/clickhouse-server} +CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin} +CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse} +EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config +CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml +CLICKHOUSE_PIDDIR=/var/run/$PROGRAM + +[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule +[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse + +if [ ! -f "/etc/debian_version" ]; then + not_deb_os=1 +fi + +if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then + + ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}" + + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + # if old rc.d service present - remove it + if [ -x "/etc/init.d/clickhouse-server" ] && [ -x "/usr/sbin/update-rc.d" ]; then + /usr/sbin/update-rc.d clickhouse-server remove + fi + + /bin/systemctl daemon-reload + /bin/systemctl enable clickhouse-server + else + # If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server + if [ -x "/etc/init.d/clickhouse-server" ]; then + if [ -x "/usr/sbin/update-rc.d" ]; then + /usr/sbin/update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $? + else + echo # Other OS + fi + fi + fi +fi diff --git a/packages/clickhouse-server.service b/packages/clickhouse-server.service new file mode 100644 index 00000000000..a9400b24270 --- /dev/null +++ b/packages/clickhouse-server.service @@ -0,0 +1,27 @@ +[Unit] +Description=ClickHouse Server (analytic DBMS for big data) +Requires=network-online.target +# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure +# that the time was adjusted already, if you use systemd-timesyncd you are +# safe, but if you use ntp or some other daemon, you should configure it +# additionaly. +After=time-sync.target network-online.target +Wants=time-sync.target + +[Service] +Type=simple +User=clickhouse +Group=clickhouse +Restart=always +RestartSec=30 +RuntimeDirectory=clickhouse-server +ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid +# Minus means that this file is optional. +EnvironmentFile=-/etc/default/clickhouse +LimitCORE=infinity +LimitNOFILE=500000 +CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE + +[Install] +# ClickHouse should not start from the rescue shell (rescue.target). +WantedBy=multi-user.target diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml new file mode 100644 index 00000000000..ed56eb27e54 --- /dev/null +++ b/packages/clickhouse-server.yaml @@ -0,0 +1,68 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-server" +arch: "all" +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +conflicts: +- clickhouse-keeper +depends: +- adduser +replaces: +- clickhouse-server-common +- clickhouse-server-base +provides: +- clickhouse-server-common +recommends: +- libcap2-bin + +maintainer: "ClickHouse Dev Team " +description: | + Server binary for ClickHouse + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + This package provides clickhouse common configuration files + +overrides: + deb: + depends: + - clickhouse-common-static (= ${CLICKHOUSE_VERSION_STRING}) + rpm: + depends: + - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} + +contents: +- src: root/etc/clickhouse-server + dst: /etc/clickhouse-server + type: config +- src: clickhouse-server.init + dst: /etc/init.d/clickhouse-server +- src: clickhouse-server.service + dst: /lib/systemd/system/clickhouse-server.service +- src: root/usr/bin/clickhouse-copier + dst: /usr/bin/clickhouse-copier +- src: clickhouse + dst: /usr/bin/clickhouse-keeper + type: symlink +- src: root/usr/bin/clickhouse-report + dst: /usr/bin/clickhouse-report +- src: root/usr/bin/clickhouse-server + dst: /usr/bin/clickhouse-server +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-server/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-server/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-server/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-server/README.md + +scripts: + postinstall: ./clickhouse-server.postinstall diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 8906d186bfc..1e2420021b6 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -2,6 +2,8 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () +include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) + # The `clickhouse` binary is a multi purpose tool that contains multiple execution modes (client, server, etc.), # each of them may be built and linked as a separate library. # If you do not know what modes you need, turn this option OFF and enable SERVER and CLIENT only. @@ -445,8 +447,11 @@ else () list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader) endif () if (ENABLE_CLICKHOUSE_KEEPER) - add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + if (NOT BUILD_STANDALONE_KEEPER) + add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + endif() + list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper) endif () if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) @@ -455,8 +460,6 @@ else () list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter) endif () - install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) if (USE_GDB_ADD_INDEX) @@ -466,7 +469,21 @@ else () if (USE_BINARY_HASH) add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .note.ClickHouse.hash=hash clickhouse COMMENT "Adding .note.ClickHouse.hash to clickhouse" VERBATIM) endif() -endif () + + if (INSTALL_STRIPPED_BINARIES) + clickhouse_strip_binary(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${STRIPPED_BINARIES_OUTPUT} BINARY_PATH clickhouse) + else() + install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + endif() +endif() + +if (NOT INSTALL_STRIPPED_BINARIES) + # Install dunny debug directory + # TODO: move logic to every place where clickhouse_strip_binary is used + add_custom_command(TARGET clickhouse POST_BUILD COMMAND echo > .empty ) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/.empty" DESTINATION ${CMAKE_INSTALL_LIBDIR}/debug/.empty) +endif() + if (ENABLE_TESTS) set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_dbms) diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 35ffb97b8e2..60e5ca92f77 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -435,6 +435,8 @@ private: Progress progress; executor.setProgressCallback([&progress](const Progress & value) { progress.incrementPiecewiseAtomically(value); }); + executor.sendQuery(ClientInfo::QueryKind::INITIAL_QUERY); + ProfileInfo info; while (Block block = executor.read()) info.update(block); diff --git a/programs/client/CMakeLists.txt b/programs/client/CMakeLists.txt index 97616e9b69f..d212da59908 100644 --- a/programs/client/CMakeLists.txt +++ b/programs/client/CMakeLists.txt @@ -1,6 +1,5 @@ set (CLICKHOUSE_CLIENT_SOURCES Client.cpp - TestTags.cpp ) set (CLICKHOUSE_CLIENT_LINK diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 86bf4a007a8..c2094b3b00d 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -21,7 +21,6 @@ #include #include -#include #include #include @@ -43,7 +42,6 @@ #include #include #include -#include "TestTags.h" #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") @@ -102,212 +100,6 @@ void Client::processError(const String & query) const } -bool Client::executeMultiQuery(const String & all_queries_text) -{ - // It makes sense not to base any control flow on this, so that it is - // the same in tests and in normal usage. The only difference is that in - // normal mode we ignore the test hints. - const bool test_mode = config().has("testmode"); - if (test_mode) - { - /// disable logs if expects errors - TestHint test_hint(test_mode, all_queries_text); - if (test_hint.clientError() || test_hint.serverError()) - processTextAsSingleQuery("SET send_logs_level = 'fatal'"); - } - - bool echo_query = echo_queries; - - /// Test tags are started with "--" so they are interpreted as comments anyway. - /// But if the echo is enabled we have to remove the test tags from `all_queries_text` - /// because we don't want test tags to be echoed. - size_t test_tags_length = test_mode ? getTestTagsLength(all_queries_text) : 0; - - /// Several queries separated by ';'. - /// INSERT data is ended by the end of line, not ';'. - /// An exception is VALUES format where we also support semicolon in - /// addition to end of line. - const char * this_query_begin = all_queries_text.data() + test_tags_length; - const char * this_query_end; - const char * all_queries_end = all_queries_text.data() + all_queries_text.size(); - - String full_query; // full_query is the query + inline INSERT data + trailing comments (the latter is our best guess for now). - String query_to_execute; - ASTPtr parsed_query; - std::optional current_exception; - - while (true) - { - auto stage = analyzeMultiQueryText(this_query_begin, this_query_end, all_queries_end, - query_to_execute, parsed_query, all_queries_text, current_exception); - switch (stage) - { - case MultiQueryProcessingStage::QUERIES_END: - case MultiQueryProcessingStage::PARSING_FAILED: - { - return true; - } - case MultiQueryProcessingStage::CONTINUE_PARSING: - { - continue; - } - case MultiQueryProcessingStage::PARSING_EXCEPTION: - { - this_query_end = find_first_symbols<'\n'>(this_query_end, all_queries_end); - - // Try to find test hint for syntax error. We don't know where - // the query ends because we failed to parse it, so we consume - // the entire line. - TestHint hint(test_mode, String(this_query_begin, this_query_end - this_query_begin)); - if (hint.serverError()) - { - // Syntax errors are considered as client errors - current_exception->addMessage("\nExpected server error '{}'.", hint.serverError()); - current_exception->rethrow(); - } - - if (hint.clientError() != current_exception->code()) - { - if (hint.clientError()) - current_exception->addMessage("\nExpected client error: " + std::to_string(hint.clientError())); - current_exception->rethrow(); - } - - /// It's expected syntax error, skip the line - this_query_begin = this_query_end; - current_exception.reset(); - - continue; - } - case MultiQueryProcessingStage::EXECUTE_QUERY: - { - full_query = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin); - if (query_fuzzer_runs) - { - if (!processWithFuzzing(full_query)) - return false; - this_query_begin = this_query_end; - continue; - } - - // Now we know for sure where the query ends. - // Look for the hint in the text of query + insert data + trailing - // comments, - // e.g. insert into t format CSV 'a' -- { serverError 123 }. - // Use the updated query boundaries we just calculated. - TestHint test_hint(test_mode, full_query); - // Echo all queries if asked; makes for a more readable reference - // file. - echo_query = test_hint.echoQueries().value_or(echo_query); - try - { - processParsedSingleQuery(full_query, query_to_execute, parsed_query, echo_query, false); - } - catch (...) - { - // Surprisingly, this is a client error. A server error would - // have been reported w/o throwing (see onReceiveSeverException()). - client_exception = std::make_unique(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); - have_error = true; - } - // Check whether the error (or its absence) matches the test hints - // (or their absence). - bool error_matches_hint = true; - if (have_error) - { - if (test_hint.serverError()) - { - if (!server_exception) - { - error_matches_hint = false; - fmt::print(stderr, "Expected server error code '{}' but got no server error (query: {}).\n", - test_hint.serverError(), full_query); - } - else if (server_exception->code() != test_hint.serverError()) - { - error_matches_hint = false; - fmt::print(stderr, "Expected server error code: {} but got: {} (query: {}).\n", - test_hint.serverError(), server_exception->code(), full_query); - } - } - if (test_hint.clientError()) - { - if (!client_exception) - { - error_matches_hint = false; - fmt::print(stderr, "Expected client error code '{}' but got no client error (query: {}).\n", - test_hint.clientError(), full_query); - } - else if (client_exception->code() != test_hint.clientError()) - { - error_matches_hint = false; - fmt::print(stderr, "Expected client error code '{}' but got '{}' (query: {}).\n", - test_hint.clientError(), client_exception->code(), full_query); - } - } - if (!test_hint.clientError() && !test_hint.serverError()) - { - // No error was expected but it still occurred. This is the - // default case w/o test hint, doesn't need additional - // diagnostics. - error_matches_hint = false; - } - } - else - { - if (test_hint.clientError()) - { - fmt::print(stderr, "The query succeeded but the client error '{}' was expected (query: {}).\n", - test_hint.clientError(), full_query); - error_matches_hint = false; - } - if (test_hint.serverError()) - { - fmt::print(stderr, "The query succeeded but the server error '{}' was expected (query: {}).\n", - test_hint.serverError(), full_query); - error_matches_hint = false; - } - } - // If the error is expected, force reconnect and ignore it. - if (have_error && error_matches_hint) - { - client_exception.reset(); - server_exception.reset(); - have_error = false; - - if (!connection->checkConnected()) - connect(); - } - - // For INSERTs with inline data: use the end of inline data as - // reported by the format parser (it is saved in sendData()). - // This allows us to handle queries like: - // insert into t values (1); select 1 - // , where the inline data is delimited by semicolon and not by a - // newline. - auto * insert_ast = parsed_query->as(); - if (insert_ast && isSyncInsertWithData(*insert_ast, global_context)) - { - this_query_end = insert_ast->end; - adjustQueryEnd(this_query_end, all_queries_end, global_context->getSettingsRef().max_parser_depth); - } - - // Report error. - if (have_error) - processError(full_query); - - // Stop processing queries if needed. - if (have_error && !ignore_error) - return is_interactive; - - this_query_begin = this_query_end; - break; - } - } - } -} - - /// Make query to get all server warnings std::vector Client::loadWarningMessages() { @@ -371,6 +163,13 @@ void Client::initialize(Poco::Util::Application & self) configReadClient(config(), home_path); + const char * env_user = getenv("CLICKHOUSE_USER"); + const char * env_password = getenv("CLICKHOUSE_PASSWORD"); + if (env_user) + config().setString("user", env_user); + if (env_password) + config().setString("password", env_password); + // global_context->setApplicationType(Context::ApplicationType::CLIENT); global_context->setQueryParameters(query_parameters); @@ -481,29 +280,32 @@ catch (...) void Client::connect() { - UInt16 default_port = ConnectionParameters::getPortFromConfig(config()); - connection_parameters = ConnectionParameters(config(), hosts_ports[0].host, - hosts_ports[0].port.value_or(default_port)); - String server_name; UInt64 server_version_major = 0; UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; - for (size_t attempted_address_index = 0; attempted_address_index < hosts_ports.size(); ++attempted_address_index) + if (hosts_and_ports.empty()) { - connection_parameters.host = hosts_ports[attempted_address_index].host; - connection_parameters.port = hosts_ports[attempted_address_index].port.value_or(default_port); - - if (is_interactive) - std::cout << "Connecting to " - << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " - : "") - << connection_parameters.host << ":" << connection_parameters.port - << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl; + String host = config().getString("host", "localhost"); + UInt16 port = ConnectionParameters::getPortFromConfig(config()); + hosts_and_ports.emplace_back(HostAndPort{host, port}); + } + for (size_t attempted_address_index = 0; attempted_address_index < hosts_and_ports.size(); ++attempted_address_index) + { try { + connection_parameters = ConnectionParameters( + config(), hosts_and_ports[attempted_address_index].host, hosts_and_ports[attempted_address_index].port); + + if (is_interactive) + std::cout << "Connecting to " + << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " + : "") + << connection_parameters.host << ":" << connection_parameters.port + << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl; + connection = Connection::createConnection(connection_parameters, global_context); if (max_client_network_bandwidth) @@ -535,7 +337,7 @@ void Client::connect() } else { - if (attempted_address_index == hosts_ports.size() - 1) + if (attempted_address_index == hosts_and_ports.size() - 1) throw; if (is_interactive) @@ -985,6 +787,7 @@ void Client::printHelpMessage(const OptionsDescription & options_description) { std::cout << options_description.main_description.value() << "\n"; std::cout << options_description.external_description.value() << "\n"; + std::cout << options_description.hosts_and_ports_description.value() << "\n"; std::cout << "In addition, --param_name=value can be specified for substitution of parameters for parametrized queries.\n"; } @@ -994,11 +797,6 @@ void Client::addOptions(OptionsDescription & options_description) /// Main commandline options related to client functionality and all parameters from Settings. options_description.main_description->add_options() ("config,c", po::value(), "config-file path (another shorthand)") - ("host,h", po::value>()->multitoken()->default_value({{"localhost"}}, "localhost"), - "list of server hosts with optionally assigned port to connect. List elements are separated by a space." - "Every list element looks like '[:]'. If port isn't assigned, connection is made by port from '--port' param" - "Example of usage: '-h host1:1 host2 host3:3'") - ("port", po::value()->default_value(9000), "server port, which is default port for every host from '--host' param") ("secure,s", "Use TLS connection") ("user,u", po::value()->default_value("default"), "user") /** If "--password [value]" is used but the value is omitted, the bad argument exception will be thrown. @@ -1010,7 +808,6 @@ void Client::addOptions(OptionsDescription & options_description) ("password", po::value()->implicit_value("\n", ""), "password") ("ask-password", "ask-password") ("quota_key", po::value(), "A string to differentiate quotas when the user have keyed quotas configured on server") - ("testmode,T", "enable test hints in comments") ("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") ("compression", po::value(), "enable or disable compression") @@ -1023,6 +820,7 @@ void Client::addOptions(OptionsDescription & options_description) ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ("no-warnings", "disable warnings when client connects to server") + ("fake-drop", "Ignore all DROP queries, should be used only for testing") ; /// Commandline options related to external tables. @@ -1044,12 +842,24 @@ void Client::addOptions(OptionsDescription & options_description) ( "types", po::value(), "types" ); + + /// Commandline options related to hosts and ports. + options_description.hosts_and_ports_description.emplace(createOptionsDescription("Hosts and ports options", terminal_width)); + options_description.hosts_and_ports_description->add_options() + ("host,h", po::value()->default_value("localhost"), + "Server hostname. Multiple hosts can be passed via multiple arguments" + "Example of usage: '--host host1 --host host2 --port port2 --host host3 ...'" + "Each '--port port' will be attached to the last seen host that doesn't have a port yet," + "if there is no such host, the port will be attached to the next first host or to default host.") + ("port", po::value(), "server ports") + ; } void Client::processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector & external_tables_arguments) + const std::vector & external_tables_arguments, + const std::vector & hosts_and_ports_arguments) { namespace po = boost::program_options; @@ -1081,6 +891,21 @@ void Client::processOptions(const OptionsDescription & options_description, exit(exit_code); } } + + for (const auto & hosts_and_ports_argument : hosts_and_ports_arguments) + { + /// Parse commandline options related to external tables. + po::parsed_options parsed_hosts_and_ports + = po::command_line_parser(hosts_and_ports_argument).options(options_description.hosts_and_ports_description.value()).run(); + po::variables_map host_and_port_options; + po::store(parsed_hosts_and_ports, host_and_port_options); + std::string host = host_and_port_options["host"].as(); + std::optional port = !host_and_port_options["port"].empty() + ? std::make_optional(host_and_port_options["port"].as()) + : std::nullopt; + hosts_and_ports.emplace_back(HostAndPort{host, port}); + } + send_external_tables = true; shared_context = Context::createShared(); @@ -1097,7 +922,12 @@ void Client::processOptions(const OptionsDescription & options_description, { const auto & name = setting.getName(); if (options.count(name)) - config().setString(name, options[name].as()); + { + if (allow_repeated_settings) + config().setString(name, options[name].as().back()); + else + config().setString(name, options[name].as()); + } } if (options.count("config-file") && options.count("config")) @@ -1105,12 +935,8 @@ void Client::processOptions(const OptionsDescription & options_description, if (options.count("config")) config().setString("config-file", options["config"].as()); - if (options.count("host")) - hosts_ports = options["host"].as>(); if (options.count("interleave-queries-file")) interleave_queries_files = options["interleave-queries-file"].as>(); - if (options.count("port") && !options["port"].defaulted()) - config().setInt("port", options["port"].as()); if (options.count("secure")) config().setBool("secure", true); if (options.count("user") && !options["user"].defaulted()) @@ -1121,14 +947,14 @@ void Client::processOptions(const OptionsDescription & options_description, config().setBool("ask-password", true); if (options.count("quota_key")) config().setString("quota_key", options["quota_key"].as()); - if (options.count("testmode")) - config().setBool("testmode", true); if (options.count("max_client_network_bandwidth")) max_client_network_bandwidth = options["max_client_network_bandwidth"].as(); if (options.count("compression")) config().setBool("compression", options["compression"].as()); if (options.count("no-warnings")) config().setBool("no-warnings", true); + if (options.count("fake-drop")) + fake_drop = true; if ((query_fuzzer_runs = options["query-fuzzer-runs"].as())) { diff --git a/programs/client/Client.h b/programs/client/Client.h index 2def74ef3fc..d235da45139 100644 --- a/programs/client/Client.h +++ b/programs/client/Client.h @@ -16,17 +16,24 @@ public: int main(const std::vector & /*args*/) override; protected: - bool executeMultiQuery(const String & all_queries_text) override; bool processWithFuzzing(const String & full_query) override; void connect() override; + void processError(const String & query) const override; + String getName() const override { return "client"; } void printHelpMessage(const OptionsDescription & options_description) override; + void addOptions(OptionsDescription & options_description) override; - void processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector & external_tables_arguments) override; + + void processOptions( + const OptionsDescription & options_description, + const CommandLineOptions & options, + const std::vector & external_tables_arguments, + const std::vector & hosts_and_ports_arguments) override; + void processConfig() override; private: diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index f8df823ecb7..5dec09ea901 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -792,9 +792,9 @@ int mainEntryClickHouseInstall(int argc, char ** argv) fmt::print("Setting capabilities for clickhouse binary. This is optional.\n"); std::string command = fmt::format("command -v setcap >/dev/null" " && command -v capsh >/dev/null" - " && capsh --has-p=cap_net_admin,cap_ipc_lock,cap_sys_nice+ep >/dev/null 2>&1" - " && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice+ep' {0}" - " || echo \"Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary." + " && capsh --has-p=cap_net_admin,cap_ipc_lock,cap_sys_nice,cap_net_bind_service+ep >/dev/null 2>&1" + " && setcap 'cap_net_admin,cap_ipc_lock,cap_sys_nice,cap_net_bind_service+ep' {0}" + " || echo \"Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' or 'net_bind_service' capability for clickhouse binary." " This is optional. Taskstats accounting will be disabled." " To enable taskstats accounting you may add the required capability later manually.\"", fs::canonical(main_bin_path).string()); diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 5a50a7074d3..92bb5dc45a3 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -1,13 +1,21 @@ include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake) -set(CLICKHOUSE_KEEPER_SOURCES - Keeper.cpp -) - if (OS_LINUX) set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") + # for some reason INTERFACE linkage doesn't work for standalone binary + set (LINK_RESOURCE_LIB_STANDALONE_KEEPER "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") endif () +clickhouse_embed_binaries( + TARGET clickhouse_keeper_configs + RESOURCES keeper_config.xml keeper_embedded.xml +) + +set(CLICKHOUSE_KEEPER_SOURCES + Keeper.cpp + TinyContext.cpp +) + set (CLICKHOUSE_KEEPER_LINK PRIVATE clickhouse_common_config @@ -21,10 +29,113 @@ set (CLICKHOUSE_KEEPER_LINK clickhouse_program_add(keeper) -install (FILES keeper_config.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper) - -clickhouse_embed_binaries( - TARGET clickhouse_keeper_configs - RESOURCES keeper_config.xml keeper_embedded.xml -) +install(FILES keeper_config.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper) add_dependencies(clickhouse-keeper-lib clickhouse_keeper_configs) + +if (BUILD_STANDALONE_KEEPER) + # Sraight list of all required sources + set(CLICKHOUSE_KEEPER_STANDALONE_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/ACLMap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/Changelog.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/CoordinationSettings.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/FourLetterCommand.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/InMemoryLogStore.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperConnectionStats.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperDispatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperLogStore.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperServer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperSnapshotManager.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStateMachine.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStateManager.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStorage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/pathUtils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/SessionExpiryQueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/SummingStateMachine.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/WriteBufferFromNuraftBuffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/ZooKeeperDataReader.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/SettingsFields.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/BaseSettings.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/Field.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/SettingsEnums.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/ServerUUID.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/UUID.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/KeeperTCPHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/TCPServer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/ProtocolServerAdapter.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CachedCompressedReadBuffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CheckingCompressedReadBuffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBufferBase.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBuffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBufferFromFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedWriteBuffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecDelta.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecDoubleDelta.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecEncrypted.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecGorilla.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecLZ4.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecMultiple.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecNone.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecT64.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecZSTD.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionFactory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/getCompressionCodecForFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/ICompressionCodec.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/LZ4_decompress_faster.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/IKeeper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/TestKeeper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperCommon.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperConstants.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperImpl.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperIO.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperLock.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperNodeCache.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/../../base/daemon/BaseDaemon.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../base/daemon/SentryWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../base/daemon/GraphiteWriter.cpp + + Keeper.cpp + TinyContext.cpp + clickhouse-keeper.cpp + + ) + + add_executable(clickhouse-keeper ${CLICKHOUSE_KEEPER_STANDALONE_SOURCES}) + + # Remove some redundant dependencies + target_compile_definitions (clickhouse-keeper PRIVATE -DKEEPER_STANDALONE_BUILD) + + target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../src") # uses includes from src directory + target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/../../src/Core/include") # uses some includes from core + target_include_directories(clickhouse-keeper PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/../../src") # uses some includes from common + + target_link_libraries(clickhouse-keeper + PRIVATE + ch_contrib::abseil_swiss_tables + ch_contrib::nuraft + ch_contrib::lz4 + ch_contrib::zstd + ch_contrib::cityhash + common ch_contrib::double_conversion + ch_contrib::dragonbox_to_chars + pcg_random + ch_contrib::pdqsort + ch_contrib::miniselect + clickhouse_common_config_no_zookeeper_log + loggers_no_text_log + clickhouse_common_io + clickhouse_parsers # Otherwise compression will not built. FIXME. + + ${LINK_RESOURCE_LIB_STANDALONE_KEEPER} + ) + + add_dependencies(clickhouse-keeper clickhouse_keeper_configs) + set_target_properties(clickhouse-keeper PROPERTIES RUNTIME_OUTPUT_DIRECTORY ../) + + install(TARGETS clickhouse-keeper RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) +endif() diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 88df4d5b3e7..1d9bbef58a5 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -34,11 +35,6 @@ #include #include -#if defined(OS_LINUX) -# include -# include -#endif - int mainEntryClickHouseKeeper(int argc, char ** argv) { @@ -127,18 +123,6 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port return socket_address; } -[[noreturn]] void forceShutdown() -{ -#if defined(THREAD_SANITIZER) && defined(OS_LINUX) - /// Thread sanitizer tries to do something on exit that we don't need if we want to exit immediately, - /// while connection handling threads are still run. - (void)syscall(SYS_exit_group, 0); - __builtin_unreachable(); -#else - _exit(0); -#endif -} - std::string getUserName(uid_t user_id) { /// Try to convert user id into user name. @@ -286,16 +270,9 @@ int Keeper::main(const std::vector & /*args*/) LOG_WARNING(log, "Keeper was built with sanitizer. It will work slowly."); #endif - auto shared_context = Context::createShared(); - global_context = Context::createGlobal(shared_context.get()); - - global_context->makeGlobalContext(); - global_context->setApplicationType(Context::ApplicationType::KEEPER); - if (!config().has("keeper_server")) throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Keeper configuration ( section) not found in config"); - std::string path; if (config().has("keeper_server.storage_path")) @@ -364,8 +341,13 @@ int Keeper::main(const std::vector & /*args*/) auto servers = std::make_shared>(); /// Initialize keeper RAFT. Do nothing if no keeper_server in config. - global_context->initializeKeeperDispatcher(/* start_async = */false); - FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher()); + tiny_context.initializeKeeperDispatcher(/* start_async = */false); + FourLetterCommandFactory::registerCommands(*tiny_context.getKeeperDispatcher()); + + auto config_getter = [this] () -> const Poco::Util::AbstractConfiguration & + { + return tiny_context.getConfigRef(); + }; for (const auto & listen_host : listen_hosts) { @@ -382,7 +364,10 @@ int Keeper::main(const std::vector & /*args*/) port_name, "Keeper (tcp): " + address.toString(), std::make_unique( - new KeeperTCPHandlerFactory(*this, false), server_pool, socket)); + new KeeperTCPHandlerFactory( + config_getter, tiny_context.getKeeperDispatcher(), + config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), + config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC), false), server_pool, socket)); }); const char * secure_port_name = "keeper_server.tcp_port_secure"; @@ -398,7 +383,10 @@ int Keeper::main(const std::vector & /*args*/) secure_port_name, "Keeper with secure protocol (tcp_secure): " + address.toString(), std::make_unique( - new KeeperTCPHandlerFactory(*this, true), server_pool, socket)); + new KeeperTCPHandlerFactory( + config_getter, tiny_context.getKeeperDispatcher(), + config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), + config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC), true), server_pool, socket)); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -425,18 +413,14 @@ int Keeper::main(const std::vector & /*args*/) [&](ConfigurationPtr config, bool /* initial_loading */) { if (config->has("keeper_server")) - global_context->updateKeeperConfiguration(*config); + tiny_context.updateKeeperConfiguration(*config); }, /* already_loaded = */ false); /// Reload it right now (initial loading) SCOPE_EXIT({ LOG_INFO(log, "Shutting down."); - /// Stop reloading of the main config. This must be done before `global_context->shutdown()` because - /// otherwise the reloading may pass a changed config to some destroyed parts of ContextSharedPart. main_config_reloader.reset(); - global_context->shutdown(); - LOG_DEBUG(log, "Waiting for current connections to Keeper to finish."); int current_connections = 0; for (auto & server : *servers) @@ -458,23 +442,17 @@ int Keeper::main(const std::vector & /*args*/) else LOG_INFO(log, "Closed connections to Keeper."); - global_context->shutdownKeeperDispatcher(); + tiny_context.shutdownKeeperDispatcher(); /// Wait server pool to avoid use-after-free of destroyed context in the handlers server_pool.joinAll(); - /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. - * At this moment, no one could own shared part of Context. - */ - global_context.reset(); - shared_context.reset(); - LOG_DEBUG(log, "Destroyed global context."); if (current_connections) { LOG_INFO(log, "Will shutdown forcefully."); - forceShutdown(); + safeExit(0); } }); diff --git a/programs/keeper/Keeper.h b/programs/keeper/Keeper.h index f5b97dacf7d..5b8fbadd0a2 100644 --- a/programs/keeper/Keeper.h +++ b/programs/keeper/Keeper.h @@ -2,6 +2,7 @@ #include #include +#include "TinyContext.h" namespace Poco { @@ -17,27 +18,22 @@ namespace DB /// standalone clickhouse-keeper server (replacement for ZooKeeper). Uses the same /// config as clickhouse-server. Serves requests on TCP ports with or without /// SSL using ZooKeeper protocol. -class Keeper : public BaseDaemon, public IServer +class Keeper : public BaseDaemon { public: using ServerApplication::run; - Poco::Util::LayeredConfiguration & config() const override + Poco::Util::LayeredConfiguration & config() const { return BaseDaemon::config(); } - Poco::Logger & logger() const override + Poco::Logger & logger() const { return BaseDaemon::logger(); } - ContextMutablePtr context() const override - { - return global_context; - } - - bool isCancelled() const override + bool isCancelled() const { return BaseDaemon::isCancelled(); } @@ -58,7 +54,7 @@ protected: std::string getDefaultConfigFileName() const override; private: - ContextMutablePtr global_context; + TinyContext tiny_context; Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; diff --git a/programs/keeper/TinyContext.cpp b/programs/keeper/TinyContext.cpp new file mode 100644 index 00000000000..386fb1e0c1d --- /dev/null +++ b/programs/keeper/TinyContext.cpp @@ -0,0 +1,71 @@ +#include "TinyContext.h" + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +void TinyContext::setConfig(const ConfigurationPtr & config_) +{ + std::lock_guard lock(keeper_dispatcher_mutex); + config = config_; +} + +const Poco::Util::AbstractConfiguration & TinyContext::getConfigRef() const +{ + std::lock_guard lock(keeper_dispatcher_mutex); + return config ? *config : Poco::Util::Application::instance().config(); +} + + +void TinyContext::initializeKeeperDispatcher([[maybe_unused]] bool start_async) const +{ + const auto & config_ref = getConfigRef(); + + std::lock_guard lock(keeper_dispatcher_mutex); + + if (keeper_dispatcher) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to initialize Keeper multiple times"); + + if (config_ref.has("keeper_server")) + { + keeper_dispatcher = std::make_shared(); + keeper_dispatcher->initialize(config_ref, true, start_async); + } +} + +std::shared_ptr TinyContext::getKeeperDispatcher() const +{ + std::lock_guard lock(keeper_dispatcher_mutex); + if (!keeper_dispatcher) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Keeper must be initialized before requests"); + + return keeper_dispatcher; +} + +void TinyContext::shutdownKeeperDispatcher() const +{ + std::lock_guard lock(keeper_dispatcher_mutex); + if (keeper_dispatcher) + { + keeper_dispatcher->shutdown(); + keeper_dispatcher.reset(); + } +} + +void TinyContext::updateKeeperConfiguration([[maybe_unused]] const Poco::Util::AbstractConfiguration & config_) +{ + std::lock_guard lock(keeper_dispatcher_mutex); + if (!keeper_dispatcher) + return; + + keeper_dispatcher->updateConfiguration(config_); +} + +} diff --git a/programs/keeper/TinyContext.h b/programs/keeper/TinyContext.h new file mode 100644 index 00000000000..a53a6d0377d --- /dev/null +++ b/programs/keeper/TinyContext.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + +#include + +namespace DB +{ + +class KeeperDispatcher; + +class TinyContext: public std::enable_shared_from_this +{ +public: + std::shared_ptr getKeeperDispatcher() const; + void initializeKeeperDispatcher(bool start_async) const; + void shutdownKeeperDispatcher() const; + void updateKeeperConfiguration(const Poco::Util::AbstractConfiguration & config); + + using ConfigurationPtr = Poco::AutoPtr; + + void setConfig(const ConfigurationPtr & config); + const Poco::Util::AbstractConfiguration & getConfigRef() const; + +private: + mutable std::mutex keeper_dispatcher_mutex; + mutable std::shared_ptr keeper_dispatcher; + + ConfigurationPtr config; +}; + +} diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 0913c6e4a9a..aded9664b35 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -1,3 +1,5 @@ +include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) + set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES library-bridge.cpp LibraryInterface.cpp @@ -22,4 +24,8 @@ target_link_libraries(clickhouse-library-bridge PRIVATE set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) +if (INSTALL_STRIPPED_BINARIES) + clickhouse_strip_binary(TARGET clickhouse-library-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-library-bridge) +else() + install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) +endif() diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 70363c62cac..26d42a11315 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -91,92 +92,6 @@ void LocalServer::processError(const String &) const } -bool LocalServer::executeMultiQuery(const String & all_queries_text) -{ - bool echo_query = echo_queries; - - /// Several queries separated by ';'. - /// INSERT data is ended by the end of line, not ';'. - /// An exception is VALUES format where we also support semicolon in - /// addition to end of line. - const char * this_query_begin = all_queries_text.data(); - const char * this_query_end; - const char * all_queries_end = all_queries_text.data() + all_queries_text.size(); - - String full_query; // full_query is the query + inline INSERT data + trailing comments (the latter is our best guess for now). - String query_to_execute; - ASTPtr parsed_query; - std::optional current_exception; - - while (true) - { - auto stage = analyzeMultiQueryText(this_query_begin, this_query_end, all_queries_end, - query_to_execute, parsed_query, all_queries_text, current_exception); - switch (stage) - { - case MultiQueryProcessingStage::QUERIES_END: - case MultiQueryProcessingStage::PARSING_FAILED: - { - return true; - } - case MultiQueryProcessingStage::CONTINUE_PARSING: - { - continue; - } - case MultiQueryProcessingStage::PARSING_EXCEPTION: - { - if (current_exception) - current_exception->rethrow(); - return true; - } - case MultiQueryProcessingStage::EXECUTE_QUERY: - { - full_query = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin); - - try - { - processParsedSingleQuery(full_query, query_to_execute, parsed_query, echo_query, false); - } - catch (...) - { - if (!is_interactive && !ignore_error) - throw; - - // Surprisingly, this is a client error. A server error would - // have been reported w/o throwing (see onReceiveSeverException()). - client_exception = std::make_unique(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); - have_error = true; - } - - // For INSERTs with inline data: use the end of inline data as - // reported by the format parser (it is saved in sendData()). - // This allows us to handle queries like: - // insert into t values (1); select 1 - // , where the inline data is delimited by semicolon and not by a - // newline. - auto * insert_ast = parsed_query->as(); - if (insert_ast && insert_ast->data) - { - this_query_end = insert_ast->end; - adjustQueryEnd(this_query_end, all_queries_end, global_context->getSettingsRef().max_parser_depth); - } - - // Report error. - if (have_error) - processError(full_query); - - // Stop processing queries if needed. - if (have_error && !ignore_error) - return is_interactive; - - this_query_begin = this_query_end; - break; - } - } - } -} - - void LocalServer::initialize(Poco::Util::Application & self) { Poco::Util::Application::initialize(self); @@ -313,9 +228,15 @@ void LocalServer::cleanup() } +static bool checkIfStdinIsRegularFile() +{ + struct stat file_stat; + return fstat(STDIN_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); +} + std::string LocalServer::getInitialCreateTableQuery() { - if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format")) + if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!checkIfStdinIsRegularFile() || !config().has("query"))) return {}; auto table_name = backQuoteIfNeed(config().getString("table-name", "table")); @@ -337,8 +258,9 @@ std::string LocalServer::getInitialCreateTableQuery() format_from_file_name = FormatFactory::instance().getFormatFromFileName(file_name, false); } - auto data_format - = backQuoteIfNeed(config().getString("table-data-format", format_from_file_name.empty() ? "TSV" : format_from_file_name)); + auto data_format = backQuoteIfNeed( + config().getString("table-data-format", config().getString("format", format_from_file_name.empty() ? "TSV" : format_from_file_name))); + if (table_structure == "auto") table_structure = ""; @@ -381,7 +303,9 @@ void LocalServer::setupUsers() ""; ConfigurationPtr users_config; - + auto & access_control = global_context->getAccessControl(); + access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true)); + access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true)); if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml")) { const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); @@ -390,10 +314,7 @@ void LocalServer::setupUsers() users_config = loaded_config.configuration; } else - { users_config = getConfigurationFromXMLString(minimal_default_user_xml); - } - if (users_config) global_context->setUsersConfig(users_config); else @@ -404,7 +325,8 @@ void LocalServer::setupUsers() void LocalServer::connect() { connection_parameters = ConnectionParameters(config()); - connection = LocalConnection::createConnection(connection_parameters, global_context, need_render_progress); + connection = LocalConnection::createConnection( + connection_parameters, global_context, need_render_progress, need_render_profile_events, server_display_name); } @@ -518,22 +440,17 @@ void LocalServer::processConfig() if (config().has("multiquery")) is_multiquery = true; - - load_suggestions = true; } else { - if (delayed_interactive) - { - load_suggestions = true; - } - need_render_progress = config().getBool("progress", false); echo_queries = config().hasOption("echo") || config().hasOption("verbose"); ignore_error = config().getBool("ignore-error", false); is_multiquery = true; } + print_stack_trace = config().getBool("stacktrace", false); + load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false); auto logging = (config().has("logger.console") || config().has("logger.level") @@ -773,7 +690,7 @@ void LocalServer::applyCmdOptions(ContextMutablePtr context) } -void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector &) +void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector &, const std::vector &) { if (options.count("table")) config().setString("table-name", options["table"].as()); @@ -799,7 +716,6 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp } - #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wmissing-declarations" diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 06e3746eacd..969af7f1b77 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -31,17 +31,19 @@ public: int main(const std::vector & /*args*/) override; protected: - bool executeMultiQuery(const String & all_queries_text) override; - void connect() override; + void processError(const String & query) const override; + String getName() const override { return "local"; } void printHelpMessage(const OptionsDescription & options_description) override; void addOptions(OptionsDescription & options_description) override; + void processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector &) override; + const std::vector &, const std::vector &) override; + void processConfig() override; private: diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 947e7ab1768..1ffb0b437a6 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -909,7 +909,7 @@ public: ColumnPtr new_nested_column = nested_model->generate(nested_column); - return ColumnArray::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(std::move(column_array.getOffsetsPtr()))); + return ColumnArray::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(column_array.getOffsetsPtr())); } void updateSeed() override @@ -947,7 +947,7 @@ public: ColumnPtr new_nested_column = nested_model->generate(nested_column); - return ColumnNullable::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(std::move(column_nullable.getNullMapColumnPtr()))); + return ColumnNullable::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(column_nullable.getNullMapColumnPtr())); } void updateSeed() override diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 54f47204259..50a8bb629c8 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -1,3 +1,5 @@ +include(${ClickHouse_SOURCE_DIR}/cmake/strip_binary.cmake) + set (CLICKHOUSE_ODBC_BRIDGE_SOURCES ColumnInfoHandler.cpp getIdentifierQuote.cpp @@ -37,7 +39,11 @@ if (USE_GDB_ADD_INDEX) add_custom_command(TARGET clickhouse-odbc-bridge POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} ../clickhouse-odbc-bridge COMMENT "Adding .gdb-index to clickhouse-odbc-bridge" VERBATIM) endif() -install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) +if (INSTALL_STRIPPED_BINARIES) + clickhouse_strip_binary(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${STRIPPED_BINARIES_OUTPUT} BINARY_PATH ../clickhouse-odbc-bridge) +else() + install(TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) +endif() if(ENABLE_TESTS) add_subdirectory(tests) diff --git a/programs/odbc-bridge/ODBCBlockInputStream.cpp b/programs/odbc-bridge/ODBCBlockInputStream.cpp index 3cf10171a94..45ab4e51d8f 100644 --- a/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -149,8 +149,6 @@ void ODBCSource::insertValue( DateTime64 time = 0; const auto * datetime_type = assert_cast(data_type.get()); readDateTime64Text(time, datetime_type->getScale(), in, datetime_type->getTimeZone()); - if (time < 0) - time = 0; assert_cast(column).insertValue(time); break; } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 82522068d85..eaa03c0d08a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -23,6 +23,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -31,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -96,8 +99,6 @@ # include # include # include -# include -# include #endif #if USE_SSL @@ -659,6 +660,10 @@ int Server::main(const std::vector & /*args*/) config().getUInt("thread_pool_queue_size", 10000) ); + IOThreadPool::initialize( + config().getUInt("max_io_thread_pool_size", 100), + config().getUInt("max_io_thread_pool_free_size", 0), + config().getUInt("io_thread_pool_queue_size", 10000)); /// Initialize global local cache for remote filesystem. if (config().has("local_cache_for_remote_fs")) @@ -1049,6 +1054,14 @@ int Server::main(const std::vector & /*args*/) total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); + if (config->has("global_memory_usage_overcommit_max_wait_microseconds")) + { + UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 0); + global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time); + } + total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); + // FIXME logging-related things need synchronization -- see the 'Logger * log' saved // in a lot of places. For now, disable updating log configuration without server restart. //setTextLog(global_context->getTextLog()); @@ -1061,6 +1074,8 @@ int Server::main(const std::vector & /*args*/) global_context->loadOrReloadModels(*config); global_context->loadOrReloadUserDefinedExecutableFunctions(*config); + global_context->setRemoteHostFilter(*config); + /// Setup protection to avoid accidental DROP for big tables (that are greater than 50 GB by default) if (config->has("max_table_size_to_drop")) global_context->setMaxTableSizeToDrop(config->getUInt64("max_table_size_to_drop")); @@ -1124,6 +1139,11 @@ int Server::main(const std::vector & /*args*/) global_context->initializeKeeperDispatcher(can_initialize_keeper_async); FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher()); + auto config_getter = [this] () -> const Poco::Util::AbstractConfiguration & + { + return global_context->getConfigRef(); + }; + for (const auto & listen_host : listen_hosts) { /// TCP Keeper @@ -1142,7 +1162,11 @@ int Server::main(const std::vector & /*args*/) port_name, "Keeper (tcp): " + address.toString(), std::make_unique( - new KeeperTCPHandlerFactory(*this, false), server_pool, socket)); + new KeeperTCPHandlerFactory( + config_getter, global_context->getKeeperDispatcher(), + global_context->getSettingsRef().receive_timeout.totalSeconds(), + global_context->getSettingsRef().send_timeout.totalSeconds(), + false), server_pool, socket)); }); const char * secure_port_name = "keeper_server.tcp_port_secure"; @@ -1161,7 +1185,10 @@ int Server::main(const std::vector & /*args*/) secure_port_name, "Keeper with secure protocol (tcp_secure): " + address.toString(), std::make_unique( - new KeeperTCPHandlerFactory(*this, true), server_pool, socket)); + new KeeperTCPHandlerFactory( + config_getter, global_context->getKeeperDispatcher(), + global_context->getSettingsRef().receive_timeout.totalSeconds(), + global_context->getSettingsRef().send_timeout.totalSeconds(), true), server_pool, socket)); #else UNUSED(port); throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", @@ -1185,6 +1212,9 @@ int Server::main(const std::vector & /*args*/) if (config().has("custom_settings_prefixes")) access_control.setCustomSettingsPrefixes(config().getString("custom_settings_prefixes")); + access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true)); + access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true)); + /// Initialize access storages. try { @@ -1432,7 +1462,7 @@ int Server::main(const std::vector & /*args*/) #endif #if !defined(__x86_64__) - LOG_INFO(log, "Query Profiler is only tested on x86_64. It also known to not work under qemu-user."); + LOG_INFO(log, "Query Profiler and TraceCollector is only tested on x86_64. It also known to not work under qemu-user."); #endif if (!hasPHDRCache()) @@ -1642,7 +1672,7 @@ int Server::main(const std::vector & /*args*/) /// Dump coverage here, because std::atexit callback would not be called. dumpCoverageReportIfPossible(); LOG_INFO(log, "Will shutdown forcefully."); - forceShutdown(); + safeExit(0); } }); diff --git a/programs/server/config.xml b/programs/server/config.xml index def64607caf..6ca64dc30c5 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -243,7 +243,7 @@ openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 Only file format with BEGIN DH PARAMETERS is supported. --> - + none true true @@ -367,6 +367,10 @@ /var/lib/clickhouse/tmp/ + + + + ` - + int needs explicit cast /// 2. customized types needs explicit cast template - enable_if_not_field_or_bool_or_stringlike_t & + enable_if_not_field_or_bool_or_stringlike_t & /// NOLINT operator=(T && rhs); Field & operator= (bool rhs) @@ -409,7 +425,7 @@ public: template const auto & get() const { - auto mutable_this = const_cast *>(this); + auto * mutable_this = const_cast *>(this); return mutable_this->get(); } @@ -422,7 +438,7 @@ public: template const T & reinterpret() const { - auto mutable_this = const_cast *>(this); + auto * mutable_this = const_cast *>(this); return mutable_this->reinterpret(); } @@ -472,6 +488,7 @@ public: case Types::Array: return get() < rhs.get(); case Types::Tuple: return get() < rhs.get(); case Types::Map: return get() < rhs.get(); + case Types::Object: return get() < rhs.get(); case Types::Decimal32: return get>() < rhs.get>(); case Types::Decimal64: return get>() < rhs.get>(); case Types::Decimal128: return get>() < rhs.get>(); @@ -510,6 +527,7 @@ public: case Types::Array: return get() <= rhs.get(); case Types::Tuple: return get() <= rhs.get(); case Types::Map: return get() <= rhs.get(); + case Types::Object: return get() <= rhs.get(); case Types::Decimal32: return get>() <= rhs.get>(); case Types::Decimal64: return get>() <= rhs.get>(); case Types::Decimal128: return get>() <= rhs.get>(); @@ -548,6 +566,7 @@ public: case Types::Array: return get() == rhs.get(); case Types::Tuple: return get() == rhs.get(); case Types::Map: return get() == rhs.get(); + case Types::Object: return get() == rhs.get(); case Types::UInt128: return get() == rhs.get(); case Types::UInt256: return get() == rhs.get(); case Types::Int128: return get() == rhs.get(); @@ -597,6 +616,7 @@ public: bool value = bool(field.template get()); return f(value); } + case Types::Object: return f(field.template get()); case Types::Decimal32: return f(field.template get>()); case Types::Decimal64: return f(field.template get>()); case Types::Decimal128: return f(field.template get>()); @@ -650,7 +670,8 @@ private: } template - std::enable_if_t assignString(const CharT * data, size_t size) + requires (sizeof(CharT) == 1) + void assignString(const CharT * data, size_t size) { assert(which == Types::String); String * ptr = reinterpret_cast(&storage); @@ -685,7 +706,8 @@ private: } template - std::enable_if_t create(const CharT * data, size_t size) + requires (sizeof(CharT) == 1) + void create(const CharT * data, size_t size) { new (&storage) String(reinterpret_cast(data), size); which = Types::String; @@ -713,6 +735,9 @@ private: case Types::Map: destroy(); break; + case Types::Object: + destroy(); + break; case Types::AggregateFunctionState: destroy(); break; @@ -737,26 +762,27 @@ private: using Row = std::vector; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Null; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt64; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt128; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt256; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Int64; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Int128; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Int256; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::UUID; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Float64; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::String; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Array; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Tuple; }; -template <> struct Field::TypeToEnum { static const Types::Which value = Types::Map; }; -template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal32; }; -template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal64; }; -template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal128; }; -template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal256; }; -template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal64; }; -template <> struct Field::TypeToEnum{ static const Types::Which value = Types::AggregateFunctionState; }; -template <> struct Field::TypeToEnum{ static const Types::Which value = Types::Bool; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Null; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::UInt64; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::UInt128; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::UInt256; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Int64; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Int128; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Int256; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::UUID; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Float64; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::String; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Array; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Tuple; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Map; }; +template <> struct Field::TypeToEnum { static constexpr Types::Which value = Types::Object; }; +template <> struct Field::TypeToEnum>{ static constexpr Types::Which value = Types::Decimal32; }; +template <> struct Field::TypeToEnum>{ static constexpr Types::Which value = Types::Decimal64; }; +template <> struct Field::TypeToEnum>{ static constexpr Types::Which value = Types::Decimal128; }; +template <> struct Field::TypeToEnum>{ static constexpr Types::Which value = Types::Decimal256; }; +template <> struct Field::TypeToEnum>{ static constexpr Types::Which value = Types::Decimal64; }; +template <> struct Field::TypeToEnum{ static constexpr Types::Which value = Types::AggregateFunctionState; }; +template <> struct Field::TypeToEnum{ static constexpr Types::Which value = Types::Bool; }; template <> struct Field::EnumToType { using Type = Null; }; template <> struct Field::EnumToType { using Type = UInt64; }; @@ -771,6 +797,7 @@ template <> struct Field::EnumToType { using Type = Strin template <> struct Field::EnumToType { using Type = Array; }; template <> struct Field::EnumToType { using Type = Tuple; }; template <> struct Field::EnumToType { using Type = Map; }; +template <> struct Field::EnumToType { using Type = Object; }; template <> struct Field::EnumToType { using Type = DecimalField; }; template <> struct Field::EnumToType { using Type = DecimalField; }; template <> struct Field::EnumToType { using Type = DecimalField; }; @@ -887,7 +914,7 @@ Field::Field(T && rhs, enable_if_not_field_or_bool_or_stringlike_t) //-V730 } template -Field::enable_if_not_field_or_bool_or_stringlike_t & +Field::enable_if_not_field_or_bool_or_stringlike_t & /// NOLINT Field::operator=(T && rhs) { auto && val = castToNearestFieldType(std::forward(rhs)); @@ -931,34 +958,39 @@ class WriteBuffer; /// It is assumed that all elements of the array have the same type. void readBinary(Array & x, ReadBuffer & buf); - [[noreturn]] inline void readText(Array &, ReadBuffer &) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); } [[noreturn]] inline void readQuoted(Array &, ReadBuffer &) { throw Exception("Cannot read Array.", ErrorCodes::NOT_IMPLEMENTED); } /// It is assumed that all elements of the array have the same type. /// Also write size and type into buf. UInt64 and Int64 is written in variadic size form void writeBinary(const Array & x, WriteBuffer & buf); - void writeText(const Array & x, WriteBuffer & buf); - [[noreturn]] inline void writeQuoted(const Array &, WriteBuffer &) { throw Exception("Cannot write Array quoted.", ErrorCodes::NOT_IMPLEMENTED); } void readBinary(Tuple & x, ReadBuffer & buf); - [[noreturn]] inline void readText(Tuple &, ReadBuffer &) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); } [[noreturn]] inline void readQuoted(Tuple &, ReadBuffer &) { throw Exception("Cannot read Tuple.", ErrorCodes::NOT_IMPLEMENTED); } void writeBinary(const Tuple & x, WriteBuffer & buf); - void writeText(const Tuple & x, WriteBuffer & buf); +[[noreturn]] inline void writeQuoted(const Tuple &, WriteBuffer &) { throw Exception("Cannot write Tuple quoted.", ErrorCodes::NOT_IMPLEMENTED); } void readBinary(Map & x, ReadBuffer & buf); [[noreturn]] inline void readText(Map &, ReadBuffer &) { throw Exception("Cannot read Map.", ErrorCodes::NOT_IMPLEMENTED); } [[noreturn]] inline void readQuoted(Map &, ReadBuffer &) { throw Exception("Cannot read Map.", ErrorCodes::NOT_IMPLEMENTED); } + void writeBinary(const Map & x, WriteBuffer & buf); void writeText(const Map & x, WriteBuffer & buf); [[noreturn]] inline void writeQuoted(const Map &, WriteBuffer &) { throw Exception("Cannot write Map quoted.", ErrorCodes::NOT_IMPLEMENTED); } +void readBinary(Object & x, ReadBuffer & buf); +[[noreturn]] inline void readText(Object &, ReadBuffer &) { throw Exception("Cannot read Object.", ErrorCodes::NOT_IMPLEMENTED); } +[[noreturn]] inline void readQuoted(Object &, ReadBuffer &) { throw Exception("Cannot read Object.", ErrorCodes::NOT_IMPLEMENTED); } + +void writeBinary(const Object & x, WriteBuffer & buf); +void writeText(const Object & x, WriteBuffer & buf); +[[noreturn]] inline void writeQuoted(const Object &, WriteBuffer &) { throw Exception("Cannot write Object quoted.", ErrorCodes::NOT_IMPLEMENTED); } + __attribute__ ((noreturn)) inline void writeText(const AggregateFunctionStateData &, WriteBuffer &) { // This probably doesn't make any sense, but we have to have it for @@ -977,8 +1009,6 @@ void readQuoted(DecimalField & x, ReadBuffer & buf); void writeFieldText(const Field & x, WriteBuffer & buf); -[[noreturn]] inline void writeQuoted(const Tuple &, WriteBuffer &) { throw Exception("Cannot write Tuple quoted.", ErrorCodes::NOT_IMPLEMENTED); } - String toString(const Field & x); } @@ -986,10 +1016,10 @@ String toString(const Field & x); template <> struct fmt::formatter { - constexpr auto parse(format_parse_context & ctx) + static constexpr auto parse(format_parse_context & ctx) { - auto it = ctx.begin(); - auto end = ctx.end(); + const auto * it = ctx.begin(); + const auto * end = ctx.end(); /// Only support {}. if (it != end && *it != '}') diff --git a/src/Core/MultiEnum.h b/src/Core/MultiEnum.h index 40cf166ccf7..32aae93c6d5 100644 --- a/src/Core/MultiEnum.h +++ b/src/Core/MultiEnum.h @@ -17,7 +17,8 @@ struct MultiEnum : MultiEnum((toBitFlag(v) | ... | 0u)) {} - template >> + template + requires std::is_convertible_v constexpr explicit MultiEnum(ValueType v) : bitset(v) { @@ -53,7 +54,8 @@ struct MultiEnum return bitset; } - template >> + template + requires std::is_convertible_v void setValue(ValueType new_value) { // Can't set value from any enum avoid confusion @@ -66,7 +68,8 @@ struct MultiEnum return bitset == other.bitset; } - template >> + template + requires std::is_convertible_v bool operator==(ValueType other) const { // Shouldn't be comparable with any enum to avoid confusion @@ -80,13 +83,15 @@ struct MultiEnum return !(*this == other); } - template >> + template + requires std::is_convertible_v friend bool operator==(ValueType left, MultiEnum right) { return right.operator==(left); } - template >::type> + template + requires (!std::is_same_v) friend bool operator!=(L left, MultiEnum right) { return !(right.operator==(left)); diff --git a/src/Core/MySQL/MySQLClient.cpp b/src/Core/MySQL/MySQLClient.cpp index 26535f05be7..98797b3d284 100644 --- a/src/Core/MySQL/MySQLClient.cpp +++ b/src/Core/MySQL/MySQLClient.cpp @@ -24,12 +24,12 @@ namespace ErrorCodes } MySQLClient::MySQLClient(const String & host_, UInt16 port_, const String & user_, const String & password_) - : host(host_), port(port_), user(user_), password(std::move(password_)), + : host(host_), port(port_), user(user_), password(password_), client_capabilities(CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION) { } -MySQLClient::MySQLClient(MySQLClient && other) +MySQLClient::MySQLClient(MySQLClient && other) noexcept : host(std::move(other.host)), port(other.port), user(std::move(other.user)), password(std::move(other.password)) , client_capabilities(other.client_capabilities) { @@ -142,7 +142,7 @@ void MySQLClient::setBinlogChecksum(const String & binlog_checksum) replication.setChecksumSignatureLength(Poco::toUpper(binlog_checksum) == "NONE" ? 0 : 4); } -void MySQLClient::startBinlogDumpGTID(UInt32 slave_id, String replicate_db, String gtid_str, const String & binlog_checksum) +void MySQLClient::startBinlogDumpGTID(UInt32 slave_id, String replicate_db, std::unordered_set replicate_tables, String gtid_str, const String & binlog_checksum) { /// Maybe CRC32 or NONE. mysqlbinlog.cc use NONE, see its below comments: /// Make a notice to the server that this client is checksum-aware. @@ -165,6 +165,7 @@ void MySQLClient::startBinlogDumpGTID(UInt32 slave_id, String replicate_db, Stri /// Set Filter rule to replication. replication.setReplicateDatabase(replicate_db); + replication.setReplicateTables(replicate_tables); BinlogDumpGTID binlog_dump(slave_id, gtid_sets.toPayload()); packet_endpoint->sendPacket(binlog_dump, true); diff --git a/src/Core/MySQL/MySQLClient.h b/src/Core/MySQL/MySQLClient.h index 5b33a8f852b..9fa3ace6baa 100644 --- a/src/Core/MySQL/MySQLClient.h +++ b/src/Core/MySQL/MySQLClient.h @@ -22,7 +22,7 @@ class MySQLClient { public: MySQLClient(const String & host_, UInt16 port_, const String & user_, const String & password_); - MySQLClient(MySQLClient && other); + MySQLClient(MySQLClient && other) noexcept; void connect(); void disconnect(); @@ -33,7 +33,7 @@ public: /// Start replication stream by GTID. /// replicate_db: replication database schema, events from other databases will be ignored. /// gtid: executed gtid sets format like 'hhhhhhhh-hhhh-hhhh-hhhh-hhhhhhhhhhhh:x-y'. - void startBinlogDumpGTID(UInt32 slave_id, String replicate_db, String gtid, const String & binlog_checksum); + void startBinlogDumpGTID(UInt32 slave_id, String replicate_db, std::unordered_set replicate_tables, String gtid, const String & binlog_checksum); BinlogEventPtr readOneBinlogEvent(UInt64 milliseconds = 0); Position getPosition() const { return replication.getPosition(); } diff --git a/src/Core/MySQL/MySQLGtid.cpp b/src/Core/MySQL/MySQLGtid.cpp index bfd0bd02b45..43fa90b6160 100644 --- a/src/Core/MySQL/MySQLGtid.cpp +++ b/src/Core/MySQL/MySQLGtid.cpp @@ -21,7 +21,7 @@ void GTIDSet::tryMerge(size_t i) intervals.erase(intervals.begin() + i + 1, intervals.begin() + i + 1 + 1); } -void GTIDSets::parse(const String gtid_format) +void GTIDSets::parse(String gtid_format) { if (gtid_format.empty()) { diff --git a/src/Core/MySQL/MySQLGtid.h b/src/Core/MySQL/MySQLGtid.h index c8a571d2569..45eeaf02fa2 100644 --- a/src/Core/MySQL/MySQLGtid.h +++ b/src/Core/MySQL/MySQLGtid.h @@ -35,7 +35,7 @@ class GTIDSets public: std::vector sets; - void parse(const String gtid_format_); + void parse(String gtid_format_); void update(const GTID & other); String toString() const; diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 50f6be23f83..1c1f6535550 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -142,8 +142,7 @@ namespace MySQLReplication out << "XID: " << this->xid << '\n'; } - /// https://dev.mysql.com/doc/internals/en/table-map-event.html - void TableMapEvent::parseImpl(ReadBuffer & payload) + void TableMapEventHeader::parse(ReadBuffer & payload) { payload.readStrict(reinterpret_cast(&table_id), 6); payload.readStrict(reinterpret_cast(&flags), 2); @@ -157,7 +156,11 @@ namespace MySQLReplication table.resize(table_len); payload.readStrict(reinterpret_cast(table.data()), table_len); payload.ignore(1); + } + /// https://dev.mysql.com/doc/internals/en/table-map-event.html + void TableMapEvent::parseImpl(ReadBuffer & payload) + { column_count = readLengthEncodedNumber(payload); for (auto i = 0U; i < column_count; ++i) { @@ -165,7 +168,6 @@ namespace MySQLReplication payload.readStrict(reinterpret_cast(&v), 1); column_type.emplace_back(v); } - String meta; readLengthEncodedString(meta, payload); parseMeta(meta); @@ -429,7 +431,7 @@ namespace MySQLReplication UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); - const DayNum date_day_number(DateLUT::instance().makeDayNum( + const ExtendedDayNum date_day_number(DateLUT::instance().makeDayNum( static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()); row.push_back(Field(date_day_number.toUnderType())); @@ -957,10 +959,20 @@ namespace MySQLReplication } case TABLE_MAP_EVENT: { - event = std::make_shared(std::move(event_header)); - event->parseEvent(event_payload); - auto table_map = std::static_pointer_cast(event); - table_maps[table_map->table_id] = table_map; + TableMapEventHeader map_event_header; + map_event_header.parse(event_payload); + if (doReplicate(map_event_header.schema, map_event_header.table)) + { + event = std::make_shared(std::move(event_header), map_event_header); + event->parseEvent(event_payload); + auto table_map = std::static_pointer_cast(event); + table_maps[table_map->table_id] = table_map; + } + else + { + event = std::make_shared(std::move(event_header)); + event->parseEvent(event_payload); + } break; } case WRITE_ROWS_EVENT_V1: @@ -1030,8 +1042,21 @@ namespace MySQLReplication // Special "dummy event" return false; } - auto table_map = table_maps.at(table_id); - return table_map->schema == replicate_do_db; + if (table_maps.contains(table_id)) + { + auto table_map = table_maps.at(table_id); + return (table_map->schema == replicate_do_db) && (replicate_tables.empty() || replicate_tables.contains(table_map->table)); + } + return false; + } + + bool MySQLFlavor::doReplicate(const String & db, const String & table_name) + { + if (replicate_do_db.empty()) + return false; + if (replicate_do_db != db) + return false; + return replicate_tables.empty() || table_name.empty() || replicate_tables.contains(table_name); } } diff --git a/src/Core/MySQL/MySQLReplication.h b/src/Core/MySQL/MySQLReplication.h index cb67ce73de9..8900eee0102 100644 --- a/src/Core/MySQL/MySQLReplication.h +++ b/src/Core/MySQL/MySQLReplication.h @@ -409,6 +409,20 @@ namespace MySQLReplication void parseImpl(ReadBuffer & payload) override; }; + class TableMapEventHeader + { + public: + UInt64 table_id; + UInt16 flags; + UInt8 schema_len; + String schema; + UInt8 table_len; + String table; + + TableMapEventHeader(): table_id(0), flags(0), schema_len(0), table_len(0) {} + void parse(ReadBuffer & payload); + }; + class TableMapEvent : public EventBase { public: @@ -423,7 +437,15 @@ namespace MySQLReplication std::vector column_meta; Bitmap null_bitmap; - TableMapEvent(EventHeader && header_) : EventBase(std::move(header_)), table_id(0), flags(0), schema_len(0), table_len(0), column_count(0) {} + TableMapEvent(EventHeader && header_, const TableMapEventHeader & map_event_header) : EventBase(std::move(header_)), column_count(0) + { + table_id = map_event_header.table_id; + flags = map_event_header.flags; + schema_len = map_event_header.schema_len; + schema = map_event_header.schema; + table_len = map_event_header.table_len; + table = map_event_header.table; + } void dump(WriteBuffer & out) const override; protected: @@ -563,6 +585,7 @@ namespace MySQLReplication Position getPosition() const override { return position; } BinlogEventPtr readOneEvent() override { return event; } void setReplicateDatabase(String db) override { replicate_do_db = std::move(db); } + void setReplicateTables(std::unordered_set tables) { replicate_tables = std::move(tables); } void setGTIDSets(GTIDSets sets) override { position.gtid_sets = std::move(sets); } void setChecksumSignatureLength(size_t checksum_signature_length_) override { checksum_signature_length = checksum_signature_length_; } @@ -570,10 +593,13 @@ namespace MySQLReplication Position position; BinlogEventPtr event; String replicate_do_db; + // only for filter data(Row Event), not include DDL Event + std::unordered_set replicate_tables; std::map > table_maps; size_t checksum_signature_length = 4; bool doReplicate(UInt64 table_id); + bool doReplicate(const String & db, const String & table_name); }; } diff --git a/src/Core/MySQL/PacketsConnection.cpp b/src/Core/MySQL/PacketsConnection.cpp index 32a8a9cf8ab..a2eaa0ba7ba 100644 --- a/src/Core/MySQL/PacketsConnection.cpp +++ b/src/Core/MySQL/PacketsConnection.cpp @@ -99,8 +99,8 @@ HandshakeResponse::HandshakeResponse() : capability_flags(0x00), max_packet_size HandshakeResponse::HandshakeResponse( UInt32 capability_flags_, UInt32 max_packet_size_, UInt8 character_set_, const String & username_, const String & database_, const String & auth_response_, const String & auth_plugin_name_) - : capability_flags(capability_flags_), max_packet_size(max_packet_size_), character_set(character_set_), username(std::move(username_)), - database(std::move(database_)), auth_response(std::move(auth_response_)), auth_plugin_name(std::move(auth_plugin_name_)) + : capability_flags(capability_flags_), max_packet_size(max_packet_size_), character_set(character_set_), username(username_), + database(database_), auth_response(auth_response_), auth_plugin_name(auth_plugin_name_) { } diff --git a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp index f4d47049554..b51bbafef54 100644 --- a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp +++ b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp @@ -108,8 +108,6 @@ void insertPostgreSQLValue( ReadBufferFromString in(value); DateTime64 time = 0; readDateTime64Text(time, 6, in, assert_cast(data_type.get())->getTimeZone()); - if (time < 0) - time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index dd26bf41b4a..6ccdcb4d524 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -152,7 +152,7 @@ private: WriteBuffer * out; public: - MessageTransport(WriteBuffer * out_) : in(nullptr), out(out_) {} + explicit MessageTransport(WriteBuffer * out_) : in(nullptr), out(out_) {} MessageTransport(ReadBuffer * in_, WriteBuffer * out_): in(in_), out(out_) {} @@ -257,7 +257,7 @@ public: Int32 payload_size; FirstMessage() = delete; - FirstMessage(int payload_size_) : payload_size(payload_size_) {} + explicit FirstMessage(int payload_size_) : payload_size(payload_size_) {} }; class CancelRequest : public FirstMessage @@ -266,7 +266,7 @@ public: Int32 process_id = 0; Int32 secret_key = 0; - CancelRequest(int payload_size_) : FirstMessage(payload_size_) {} + explicit CancelRequest(int payload_size_) : FirstMessage(payload_size_) {} void deserialize(ReadBuffer & in) override { @@ -391,7 +391,7 @@ public: // includes username, may also include database and other runtime parameters std::unordered_map parameters; - StartupMessage(Int32 payload_size_) : FirstMessage(payload_size_) {} + explicit StartupMessage(Int32 payload_size_) : FirstMessage(payload_size_) {} void deserialize(ReadBuffer & in) override { @@ -643,7 +643,7 @@ private: const std::vector & fields_descr; public: - RowDescription(const std::vector & fields_descr_) : fields_descr(fields_descr_) {} + explicit RowDescription(const std::vector & fields_descr_) : fields_descr(fields_descr_) {} void serialize(WriteBuffer & out) const override { @@ -673,7 +673,7 @@ class StringField : public ISerializable private: String str; public: - StringField(String str_) : str(str_) {} + explicit StringField(String str_) : str(str_) {} void serialize(WriteBuffer & out) const override { @@ -703,7 +703,7 @@ private: const std::vector> & row; public: - DataRow(const std::vector> & row_) : row(row_) {} + explicit DataRow(const std::vector> & row_) : row(row_) {} void serialize(WriteBuffer & out) const override { @@ -886,7 +886,7 @@ private: std::unordered_map> type_to_method = {}; public: - AuthenticationManager(const std::vector> & auth_methods) + explicit AuthenticationManager(const std::vector> & auth_methods) { for (const std::shared_ptr & method : auth_methods) { diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 93f44b02ce3..6ee491f3ab5 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -8,7 +8,6 @@ #define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372 #define DBMS_MIN_REVISION_WITH_VERSION_PATCH 54401 #define DBMS_MIN_REVISION_WITH_SERVER_LOGS 54406 -#define DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA 54415 /// Minimum revision with exactly the same set of aggregation methods and rules to select them. /// Two-level (bucketed) aggregation is incompatible if servers are inconsistent in these rules /// (keys will be placed in different buckets and result will not be fully aggregated). diff --git a/src/Core/QualifiedTableName.h b/src/Core/QualifiedTableName.h index 4642465f461..3310130629d 100644 --- a/src/Core/QualifiedTableName.h +++ b/src/Core/QualifiedTableName.h @@ -72,7 +72,7 @@ struct QualifiedTableName QualifiedTableName name; if (pos == std::string::npos) { - name.table = std::move(maybe_qualified_name); + name.table = maybe_qualified_name; } else if (maybe_qualified_name.find('.', pos + 1) != std::string::npos) { @@ -119,7 +119,7 @@ namespace fmt template <> struct formatter { - constexpr auto parse(format_parse_context & ctx) + static constexpr auto parse(format_parse_context & ctx) { return ctx.begin(); } diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 87d7eee0daa..411e73bdf1a 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -89,6 +89,14 @@ void Settings::addProgramOptions(boost::program_options::options_description & o } } +void Settings::addProgramOptionsAsMultitokens(boost::program_options::options_description & options) +{ + for (const auto & field : all()) + { + addProgramOptionAsMultitoken(options, field); + } +} + void Settings::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field) { const std::string_view name = field.getName(); @@ -97,6 +105,14 @@ void Settings::addProgramOption(boost::program_options::options_description & op name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); } +void Settings::addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field) +{ + const std::string_view name = field.getName(); + auto on_program_option = boost::function1([this, name](const Strings & values) { set(name, values.back()); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); +} + void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path) { if (config.getBool("skip_check_for_incorrect_settings", false)) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ad7a64783d7..ca2e9f12e66 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -44,8 +44,11 @@ class IColumn; M(UInt64, min_insert_block_size_bytes_for_materialized_views, 0, "Like min_insert_block_size_bytes, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_bytes)", 0) \ M(UInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algorithm supports it). 0 means unlimited.", 0) \ M(UInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. Values 0 or 1 means that INSERT SELECT is not run in parallel. Higher values will lead to higher memory usage. Parallel INSERT SELECT has effect only if the SELECT part is run on parallel, see 'max_threads' setting.", 0) \ + M(UInt64, max_insert_delayed_streams_for_parallel_write, 0, "The maximum number of streams (columns) to delay final part flush. Default - auto (1000 in case of underlying storage supports parallel write, for example S3 and disabled otherwise)", 0) \ M(UInt64, max_final_threads, 16, "The maximum number of threads to read from table with FINAL.", 0) \ M(MaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ + M(MaxThreads, max_download_threads, 4, "The maximum number of threads to download data (e.g. for URL engine).", 0) \ + M(UInt64, max_download_buffer_size, 10*1024*1024, "The maximal size of buffer for parallel downloading (e.g. for URL engine) per each thread.", 0) \ M(UInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ M(UInt64, max_distributed_connections, 1024, "The maximum number of connections for distributed processing of one query (should be greater than max_threads).", 0) \ M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "Which part of the query can be read into RAM for parsing (the remaining data for INSERT, if any, is read later)", 0) \ @@ -136,7 +139,7 @@ class IColumn; \ M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ \ - M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ + M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard; if set to 1 - SELECT is executed on each shard; if set to 2 - SELECT and INSERT are executed on each shard", 0) \ M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ M(UInt64, distributed_push_down_limit, 1, "If 1, LIMIT will be applied on each shard separatelly. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.", 0) \ M(Bool, optimize_distributed_group_by_sharding_key, true, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ @@ -358,11 +361,15 @@ class IColumn; M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ \ M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ + M(UInt64, max_guaranteed_memory_usage, 0, "Maximum guaranteed memory usage for processing of single query. It represents soft limit. Zero means unlimited.", 0) \ M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ + M(UInt64, max_guaranteed_memory_usage_for_user, 0, "Maximum guaranteed memory usage for processing all concurrently running queries for the user. It represents soft limit. Zero means unlimited.", 0) \ M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \ M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ \ + M(UInt64, memory_usage_overcommit_max_wait_microseconds, 0, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown", 0) \ + \ M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ M(UInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.", 0)\ @@ -467,11 +474,12 @@ class IColumn; M(Bool, allow_experimental_geo_types, false, "Allow geo data types such as Point, Ring, Polygon, MultiPolygon", 0) \ M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \ + M(Bool, cast_ipv4_ipv6_default_on_conversion_error, false, "CAST operator into IPv4, CAST operator into IPV6 type, toIPv4, toIPv6 functions will return default value instead of throwing exception on conversion error.", 0) \ M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \ M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \ M(Bool, system_events_show_zero_values, false, "Include all metrics, even with zero values", 0) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal', 'datetime64', 'date2Date32' or 'date2String'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ M(Bool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \ @@ -486,6 +494,7 @@ class IColumn; M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \ M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \ + M(Bool, describe_extend_object_types, false, "Deduce concrete type of columns of type Object in DESCRIBE query", 0) \ M(Bool, describe_include_subcolumns, false, "If true, subcolumns of all table columns will be included into result of DESCRIBE query", 0) \ \ M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ @@ -494,7 +503,6 @@ class IColumn; /** Experimental feature for moving data between shards. */ \ \ M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \ - M(Bool, experimental_query_deduplication_send_all_part_uuids, false, "If false only part UUIDs for currently moving parts are sent. If true all read part UUIDs are sent (useful only for testing).", 0) \ \ M(Bool, engine_file_empty_if_not_exists, false, "Allows to select data from a file engine table without file", 0) \ M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ @@ -503,6 +511,7 @@ class IColumn; M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \ M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \ + M(Bool, database_replicated_allow_only_replicated_engine, false, "Allow to create only Replicated tables in database with engine Replicated", 0) \ M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result", 0) \ M(UInt64, distributed_ddl_entry_format_version, 1, "Version of DDL entry to write into ZooKeeper", 0) \ \ @@ -534,7 +543,7 @@ class IColumn; M(Int64, read_priority, 0, "Priority to read data from local filesystem. Only supported for 'pread_threadpool' method.", 0) \ M(UInt64, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \ M(UInt64, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \ - M(UInt64, remote_read_min_bytes_for_seek, DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead for read with ignore.", 0) \ + M(UInt64, remote_read_min_bytes_for_seek, 4 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead for read with ignore.", 0) \ \ M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \ M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background. Makes sense only for inserts via HTTP protocol. If wait_for_async_insert is false, INSERT query is processed almost instantly, otherwise client will wait until data will be flushed to table", 0) \ @@ -544,10 +553,12 @@ class IColumn; M(Milliseconds, async_insert_busy_timeout_ms, 200, "Maximum time to wait before dumping collected data per query since the first data appeared", 0) \ M(Milliseconds, async_insert_stale_timeout_ms, 0, "Maximum time to wait before dumping collected data per query since the last data appeared. Zero means no timeout at all", 0) \ \ - M(Int64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ - M(Int64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ + M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ + M(UInt64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ + M(Bool, remote_fs_enable_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must me done via disk config), but allows to bypass cache for some queries if intended", 0) \ + M(UInt64, remote_fs_cache_max_wait_sec, 5, "Allow to wait at most this number of seconds for download of current remote_fs_buffer_size bytes, and skip cache if exceeded", 0) \ \ - M(UInt64, http_max_tries, 1, "Max attempts to read via http.", 0) \ + M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \ M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \ \ @@ -555,9 +566,12 @@ class IColumn; M(Bool, check_table_dependencies, true, "Check that DDL query (such as DROP TABLE or RENAME) will not break dependencies", 0) \ M(Bool, use_local_cache_for_remote_storage, true, "Use local cache for remote storage like HDFS or S3, it's used for remote table engine only", 0) \ \ + M(Bool, allow_unrestricted_reads_from_keeper, false, "Allow unrestricted (w/o condition on path) reads from system.zookeeper table, can be handy, but is not safe for zookeeper", 0) \ + \ /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \ + M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \ M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. @@ -602,6 +616,7 @@ class IColumn; M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices \\N", 0) \ M(Bool, input_format_null_as_default, true, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ + M(Bool, input_format_use_lowercase_column_name, false, "Use lowercase column name while reading input formats", 0) \ M(Bool, input_format_arrow_import_nested, false, "Allow to insert array of structs into Nested table in Arrow input format.", 0) \ M(Bool, input_format_orc_import_nested, false, "Allow to insert array of structs into Nested table in ORC input format.", 0) \ M(Int64, input_format_orc_row_batch_size, 100'000, "Batch size when reading ORC stripes.", 0) \ @@ -617,7 +632,7 @@ class IColumn; M(MsgPackUUIDRepresentation, output_format_msgpack_uuid_representation, FormatSettings::MsgPackUUIDRepresentation::EXT, "The way how to output UUID in MsgPack format.", 0) \ M(UInt64, input_format_max_rows_to_read_for_schema_inference, 100, "The maximum rows of data to read for automatic schema inference", 0) \ \ - M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \ + M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ \ M(String, bool_true_representation, "true", "Text to represent bool value in TSV/CSV formats.", 0) \ @@ -717,6 +732,11 @@ struct Settings : public BaseSettings, public IHints<2, Settings /// (Don't forget to call notify() on the `variables_map` after parsing it!) void addProgramOptions(boost::program_options::options_description & options); + /// Adds program options as to set the settings from a command line. + /// Allows to set one setting multiple times, the last value will be used. + /// (Don't forget to call notify() on the `variables_map` after parsing it!) + void addProgramOptionsAsMultitokens(boost::program_options::options_description & options); + /// Check that there is no user-level settings at the top level in config. /// This is a common source of mistake (user don't know where to write user-level setting). static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path); @@ -724,6 +744,8 @@ struct Settings : public BaseSettings, public IHints<2, Settings std::vector getAllRegisteredNames() const override; void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field); + + void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field); }; /* diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 17d24946cd8..ddd1c29785c 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -64,7 +64,8 @@ IMPLEMENT_SETTING_ENUM(DistributedProductMode, ErrorCodes::UNKNOWN_DISTRIBUTED_P IMPLEMENT_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, ErrorCodes::BAD_ARGUMENTS, {{"basic", FormatSettings::DateTimeInputFormat::Basic}, - {"best_effort", FormatSettings::DateTimeInputFormat::BestEffort}}) + {"best_effort", FormatSettings::DateTimeInputFormat::BestEffort}, + {"best_effort_us", FormatSettings::DateTimeInputFormat::BestEffortUS}}) IMPLEMENT_SETTING_ENUM_WITH_RENAME(DateTimeOutputFormat, ErrorCodes::BAD_ARGUMENTS, @@ -105,7 +106,9 @@ IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultTableEngine, ErrorCodes::BAD_ARGUMENTS IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL, {{"decimal", MySQLDataTypesSupport::DECIMAL}, - {"datetime64", MySQLDataTypesSupport::DATETIME64}}) + {"datetime64", MySQLDataTypesSupport::DATETIME64}, + {"date2Date32", MySQLDataTypesSupport::DATE2DATE32}, + {"date2String", MySQLDataTypesSupport::DATE2STRING}}) IMPLEMENT_SETTING_ENUM(UnionMode, ErrorCodes::UNKNOWN_UNION, {{"", UnionMode::Unspecified}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 27994529a0b..47bd4b9a928 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -138,7 +138,8 @@ enum class MySQLDataTypesSupport { DECIMAL, // convert MySQL's decimal and number to ClickHouse Decimal when applicable DATETIME64, // convert MySQL's DATETIME and TIMESTAMP and ClickHouse DateTime64 if precision is > 0 or range is greater that for DateTime. - // ENUM + DATE2DATE32, // convert MySQL's date type to ClickHouse Date32 + DATE2STRING // convert MySQL's date type to ClickHouse String(This is usually used when your mysql date is less than 1925) }; DECLARE_SETTING_MULTI_ENUM(MySQLDataTypesSupport) diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index b27763ad0d6..474786eb963 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -43,7 +43,7 @@ struct SettingFieldNumber SettingFieldNumber & operator=(Type x) { value = x; changed = true; return *this; } SettingFieldNumber & operator=(const Field & f); - operator Type() const { return value; } + operator Type() const { return value; } /// NOLINT explicit operator Field() const { return value; } String toString() const; @@ -75,7 +75,7 @@ struct SettingFieldMaxThreads SettingFieldMaxThreads & operator=(UInt64 x) { is_auto = !x; value = is_auto ? getAuto() : x; changed = true; return *this; } SettingFieldMaxThreads & operator=(const Field & f); - operator UInt64() const { return value; } + operator UInt64() const { return value; } /// NOLINT explicit operator Field() const { return value; } /// Writes "auto()" instead of simple "" if `is_auto==true`. @@ -118,10 +118,10 @@ struct SettingFieldTimespan SettingFieldTimespan & operator =(UInt64 x) { *this = Poco::Timespan{static_cast(x * microseconds_per_unit)}; return *this; } SettingFieldTimespan & operator =(const Field & f); - operator Poco::Timespan() const { return value; } + operator Poco::Timespan() const { return value; } /// NOLINT template > - operator std::chrono::duration() const { return std::chrono::duration_cast>(std::chrono::microseconds(value.totalMicroseconds())); } + operator std::chrono::duration() const { return std::chrono::duration_cast>(std::chrono::microseconds(value.totalMicroseconds())); } /// NOLINT explicit operator UInt64() const { return value.totalMicroseconds() / microseconds_per_unit; } explicit operator Field() const { return operator UInt64(); } @@ -158,7 +158,7 @@ struct SettingFieldString SettingFieldString & operator =(const char * str) { *this = std::string_view{str}; return *this; } SettingFieldString & operator =(const Field & f) { *this = f.safeGet(); return *this; } - operator const String &() const { return value; } + operator const String &() const { return value; } /// NOLINT explicit operator Field() const { return value; } const String & toString() const { return value; } @@ -181,7 +181,7 @@ public: SettingFieldChar & operator =(char c) { value = c; changed = true; return *this; } SettingFieldChar & operator =(const Field & f); - operator char() const { return value; } + operator char() const { return value; } /// NOLINT explicit operator Field() const { return toString(); } String toString() const { return String(&value, 1); } @@ -207,7 +207,7 @@ struct SettingFieldURI SettingFieldURI & operator =(const char * str) { *this = Poco::URI{str}; return *this; } SettingFieldURI & operator =(const Field & f) { *this = f.safeGet(); return *this; } - operator const Poco::URI &() const { return value; } + operator const Poco::URI &() const { return value; } /// NOLINT explicit operator String() const { return toString(); } explicit operator Field() const { return toString(); } @@ -244,7 +244,7 @@ struct SettingFieldEnum SettingFieldEnum & operator =(EnumType x) { value = x; changed = true; return *this; } SettingFieldEnum & operator =(const Field & f) { *this = Traits::fromString(f.safeGet()); return *this; } - operator EnumType() const { return value; } + operator EnumType() const { return value; } /// NOLINT explicit operator Field() const { return toString(); } String toString() const { return Traits::toString(value); } @@ -272,12 +272,15 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) *this = Traits::fromString(SettingFieldEnumHelpers::readBinary(in)); } +/// NOLINTNEXTLINE #define DECLARE_SETTING_ENUM(ENUM_TYPE) \ DECLARE_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE) +/// NOLINTNEXTLINE #define IMPLEMENT_SETTING_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ IMPLEMENT_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__) +/// NOLINTNEXTLINE #define DECLARE_SETTING_ENUM_WITH_RENAME(NEW_NAME, ENUM_TYPE) \ struct SettingField##NEW_NAME##Traits \ { \ @@ -288,6 +291,7 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) \ using SettingField##NEW_NAME = SettingFieldEnum; +/// NOLINTNEXTLINE #define IMPLEMENT_SETTING_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ const String & SettingField##NEW_NAME##Traits::toString(typename SettingField##NEW_NAME::EnumType value) \ { \ @@ -346,7 +350,7 @@ struct SettingFieldMultiEnum explicit SettingFieldMultiEnum(StorageType s) : value(s) {} explicit SettingFieldMultiEnum(const Field & f) : value(parseValueFromString(f.safeGet())) {} - operator ValueType() const { return value; } + operator ValueType() const { return value; } /// NOLINT explicit operator StorageType() const { return value.getValue(); } explicit operator Field() const { return toString(); } @@ -368,7 +372,7 @@ struct SettingFieldMultiEnum } } - if (result.size() > 0) + if (!result.empty()) result.erase(result.size() - separator.size()); return result; @@ -415,9 +419,11 @@ void SettingFieldMultiEnum::readBinary(ReadBuffer & in) parseFromString(SettingFieldEnumHelpers::readBinary(in)); } +/// NOLINTNEXTLINE #define DECLARE_SETTING_MULTI_ENUM(ENUM_TYPE) \ DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE) +/// NOLINTNEXTLINE #define DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, NEW_NAME) \ struct SettingField##NEW_NAME##Traits \ { \ @@ -429,9 +435,11 @@ void SettingFieldMultiEnum::readBinary(ReadBuffer & in) \ using SettingField##NEW_NAME = SettingFieldMultiEnum; +/// NOLINTNEXTLINE #define IMPLEMENT_SETTING_MULTI_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__) +/// NOLINTNEXTLINE #define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \ IMPLEMENT_SETTING_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\ size_t SettingField##NEW_NAME##Traits::getEnumSize() {\ diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index dd804bd4675..a5daba9fbee 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -53,7 +53,7 @@ struct SortCursorImpl */ IColumn::Permutation * permutation = nullptr; - SortCursorImpl() {} + SortCursorImpl() = default; SortCursorImpl(const Block & block, const SortDescription & desc_, size_t order_ = 0, IColumn::Permutation * perm = nullptr) : desc(desc_), sort_columns_size(desc.size()), order(order_), need_collation(desc.size()) @@ -140,7 +140,7 @@ struct SortCursorHelper const Derived & derived() const { return static_cast(*this); } - SortCursorHelper(SortCursorImpl * impl_) : impl(impl_) {} + explicit SortCursorHelper(SortCursorImpl * impl_) : impl(impl_) {} SortCursorImpl * operator-> () { return impl; } const SortCursorImpl * operator-> () const { return impl; } @@ -245,7 +245,7 @@ public: SortingHeap() = default; template - SortingHeap(Cursors & cursors) + explicit SortingHeap(Cursors & cursors) { size_t size = cursors.size(); queue.reserve(size); diff --git a/src/Core/Types.h b/src/Core/Types.h index 9d3ff15d29c..92546d7d07a 100644 --- a/src/Core/Types.h +++ b/src/Core/Types.h @@ -87,6 +87,7 @@ enum class TypeIndex AggregateFunction, LowCardinality, Map, + Object, }; #if !defined(__clang__) #pragma GCC diagnostic pop diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 5d37f8cf361..3fc2503aaa5 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -15,6 +15,8 @@ #cmakedefine01 USE_NURAFT #cmakedefine01 USE_NLP #cmakedefine01 USE_KRB5 +#cmakedefine01 USE_SIMDJSON +#cmakedefine01 USE_RAPIDJSON #cmakedefine01 USE_FILELOG #cmakedefine01 USE_ODBC #cmakedefine01 USE_REPLXX diff --git a/src/Core/examples/coro.cpp b/src/Core/examples/coro.cpp index 0f152d8090a..ecff0e23d11 100644 --- a/src/Core/examples/coro.cpp +++ b/src/Core/examples/coro.cpp @@ -84,7 +84,7 @@ struct Task std::cout << " Task " << tag << std::endl; } Task(Task &) = delete; - Task(Task &&rhs) : my(rhs.my), tag(rhs.tag) + Task(Task &&rhs) noexcept : my(rhs.my), tag(rhs.tag) { rhs.my = {}; std::cout << " Task&& " << tag << std::endl; diff --git a/src/Core/examples/mysql_protocol.cpp b/src/Core/examples/mysql_protocol.cpp index 1b81d856c9a..396bc6f7e9b 100644 --- a/src/Core/examples/mysql_protocol.cpp +++ b/src/Core/examples/mysql_protocol.cpp @@ -330,7 +330,7 @@ int main(int argc, char ** argv) /// Connect to the master. slave.connect(); - slave.startBinlogDumpGTID(slave_id, replicate_db, gtid_sets, binlog_checksum); + slave.startBinlogDumpGTID(slave_id, replicate_db, {}, gtid_sets, binlog_checksum); WriteBufferFromOStream cerr(std::cerr); diff --git a/src/Core/iostream_debug_helpers.h b/src/Core/iostream_debug_helpers.h index 8aafe0b6c9c..e40bf74583e 100644 --- a/src/Core/iostream_debug_helpers.h +++ b/src/Core/iostream_debug_helpers.h @@ -7,7 +7,8 @@ namespace DB // Use template to disable implicit casting for certain overloaded types such as Field, which leads // to overload resolution ambiguity. class Field; -template >> +template +requires std::is_same_v std::ostream & operator<<(std::ostream & stream, const T & what); struct NameAndTypePair; diff --git a/src/Core/tests/gtest_settings.cpp b/src/Core/tests/gtest_settings.cpp index 8833d86c397..46d8f9665dc 100644 --- a/src/Core/tests/gtest_settings.cpp +++ b/src/Core/tests/gtest_settings.cpp @@ -53,6 +53,29 @@ GTEST_TEST(SettingMySQLDataTypesSupport, WithDECIMAL) ASSERT_EQ(Field("decimal"), setting); } +GTEST_TEST(SettingMySQLDataTypesSupport, WithDATE) +{ + SettingMySQLDataTypesSupport setting; + setting = String("date2Date32"); + ASSERT_EQ(4, setting.value.getValue()); + + ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DATE2DATE32)); + ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DECIMAL)); + ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATETIME64)); + + ASSERT_EQ("date2Date32", setting.toString()); + ASSERT_EQ(Field("date2Date32"), setting); + + setting = String("date2String"); + ASSERT_EQ(8, setting.value.getValue()); + + ASSERT_TRUE(setting.value.isSet(MySQLDataTypesSupport::DATE2STRING)); + ASSERT_FALSE(setting.value.isSet(MySQLDataTypesSupport::DATE2DATE32)); + + ASSERT_EQ("date2String", setting.toString()); + ASSERT_EQ(Field("date2String"), setting); +} + GTEST_TEST(SettingMySQLDataTypesSupport, With1) { // Setting can be initialized with int value corresponding to DECIMAL diff --git a/src/DataTypes/CMakeLists.txt b/src/DataTypes/CMakeLists.txt index a6176efc7f3..4a60d6c54cf 100644 --- a/src/DataTypes/CMakeLists.txt +++ b/src/DataTypes/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory (Serializations) + if (ENABLE_EXAMPLES) - add_subdirectory(examples) + add_subdirectory (examples) endif () diff --git a/src/DataTypes/DataTypeArray.h b/src/DataTypes/DataTypeArray.h index 564dbba8503..122ac8e03a3 100644 --- a/src/DataTypes/DataTypeArray.h +++ b/src/DataTypes/DataTypeArray.h @@ -17,7 +17,7 @@ private: public: static constexpr bool is_parametric = true; - DataTypeArray(const DataTypePtr & nested_); + explicit DataTypeArray(const DataTypePtr & nested_); TypeIndex getTypeId() const override { return TypeIndex::Array; } diff --git a/src/DataTypes/DataTypeCustom.h b/src/DataTypes/DataTypeCustom.h index 55796e3cc7a..e8e4160af07 100644 --- a/src/DataTypes/DataTypeCustom.h +++ b/src/DataTypes/DataTypeCustom.h @@ -19,7 +19,7 @@ class IColumn; class IDataTypeCustomName { public: - virtual ~IDataTypeCustomName() {} + virtual ~IDataTypeCustomName() = default; virtual String getName() const = 0; }; @@ -33,7 +33,7 @@ struct DataTypeCustomDesc DataTypeCustomNamePtr name; SerializationPtr serialization; - DataTypeCustomDesc( + explicit DataTypeCustomDesc( DataTypeCustomNamePtr name_, SerializationPtr serialization_ = nullptr) : name(std::move(name_)) @@ -49,7 +49,7 @@ class DataTypeCustomFixedName : public IDataTypeCustomName private: String name; public: - DataTypeCustomFixedName(String name_) : name(name_) {} + explicit DataTypeCustomFixedName(String name_) : name(name_) {} String getName() const override { return name; } }; diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h index dc054144e14..926dfd9cc82 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.h @@ -34,7 +34,7 @@ public: DataTypeCustomSimpleAggregateFunction(const AggregateFunctionPtr & function_, const DataTypes & argument_types_, const Array & parameters_) : function(function_), argument_types(argument_types_), parameters(parameters_) {} - const AggregateFunctionPtr getFunction() const { return function; } + AggregateFunctionPtr getFunction() const { return function; } String getName() const override; static void checkSupportedFunctions(const AggregateFunctionPtr & function); }; diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h index 57052144216..91a09ff7cb9 100644 --- a/src/DataTypes/DataTypeDateTime.h +++ b/src/DataTypes/DataTypeDateTime.h @@ -16,7 +16,7 @@ namespace DB * * To cast from/to text format, time zone may be specified explicitly or implicit time zone may be used. * - * Time zone may be specified explicitly as type parameter, example: DateTime('Europe/Moscow'). + * Time zone may be specified explicitly as type parameter, example: DateTime('Pacific/Pitcairn'). * As it does not affect the internal representation of values, * all types with different time zones are equivalent and may be used interchangingly. * Time zone only affects parsing and displaying in text formats. @@ -48,4 +48,3 @@ public: }; } - diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/DataTypeDecimalBase.h index bdb39978825..9e37de8a35b 100644 --- a/src/DataTypes/DataTypeDecimalBase.h +++ b/src/DataTypes/DataTypeDecimalBase.h @@ -172,14 +172,14 @@ inline auto decimalResultType(const DecimalType & tx, const DecimalType & } template typename DecimalType> -inline const DecimalType decimalResultType(const DecimalType & tx, const DataTypeNumber & ty) +inline DecimalType decimalResultType(const DecimalType & tx, const DataTypeNumber & ty) { const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); return DecimalType(result_trait.precision, result_trait.scale); } template typename DecimalType> -inline const DecimalType decimalResultType(const DataTypeNumber & tx, const DecimalType & ty) +inline DecimalType decimalResultType(const DataTypeNumber & tx, const DecimalType & ty) { const auto result_trait = DecimalUtils::binaryOpResult(tx, ty); return DecimalType(result_trait.precision, result_trait.scale); diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp index 582b42accd9..e1567d3a1b0 100644 --- a/src/DataTypes/DataTypeFactory.cpp +++ b/src/DataTypes/DataTypeFactory.cpp @@ -33,7 +33,7 @@ DataTypePtr DataTypeFactory::get(const String & full_name) const /// Value 315 is known to cause stack overflow in some test configurations (debug build, sanitizers) /// let's make the threshold significantly lower. /// It is impractical for user to have complex data types with this depth. - static constexpr size_t data_type_max_parse_depth = 200; + static constexpr size_t data_type_max_parse_depth = 150; ParserDataType parser; ASTPtr ast = parseQuery(parser, full_name.data(), full_name.data() + full_name.size(), "data type", 0, data_type_max_parse_depth); @@ -213,6 +213,7 @@ DataTypeFactory::DataTypeFactory() registerDataTypeDomainSimpleAggregateFunction(*this); registerDataTypeDomainGeo(*this); registerDataTypeMap(*this); + registerDataTypeObject(*this); } DataTypeFactory & DataTypeFactory::instance() diff --git a/src/DataTypes/DataTypeFactory.h b/src/DataTypes/DataTypeFactory.h index 81d7d991bdc..704d8926bf0 100644 --- a/src/DataTypes/DataTypeFactory.h +++ b/src/DataTypes/DataTypeFactory.h @@ -51,7 +51,6 @@ public: private: const Value & findCreatorByName(const String & family_name) const; -private: DataTypesDictionary data_types; /// Case insensitive data types will be additionally added here with lowercased name. @@ -88,5 +87,6 @@ void registerDataTypeDomainIPv4AndIPv6(DataTypeFactory & factory); void registerDataTypeDomainBool(DataTypeFactory & factory); void registerDataTypeDomainSimpleAggregateFunction(DataTypeFactory & factory); void registerDataTypeDomainGeo(DataTypeFactory & factory); +void registerDataTypeObject(DataTypeFactory & factory); } diff --git a/src/DataTypes/DataTypeFixedString.h b/src/DataTypes/DataTypeFixedString.h index a53fde42b29..7c089866b23 100644 --- a/src/DataTypes/DataTypeFixedString.h +++ b/src/DataTypes/DataTypeFixedString.h @@ -29,7 +29,7 @@ public: static constexpr bool is_parametric = true; static constexpr auto type_id = TypeIndex::FixedString; - DataTypeFixedString(size_t n_) : n(n_) + explicit DataTypeFixedString(size_t n_) : n(n_) { if (n == 0) throw Exception("FixedString size must be positive", ErrorCodes::ARGUMENT_OUT_OF_BOUND); diff --git a/src/DataTypes/DataTypeFunction.h b/src/DataTypes/DataTypeFunction.h index 489ed4545f4..888bcb6a775 100644 --- a/src/DataTypes/DataTypeFunction.h +++ b/src/DataTypes/DataTypeFunction.h @@ -19,7 +19,7 @@ public: bool isParametric() const override { return true; } /// Some types could be still unknown. - DataTypeFunction(const DataTypes & argument_types_ = DataTypes(), const DataTypePtr & return_type_ = nullptr) + explicit DataTypeFunction(const DataTypes & argument_types_ = DataTypes(), const DataTypePtr & return_type_ = nullptr) : argument_types(argument_types_), return_type(return_type_) {} std::string doGetName() const override; diff --git a/src/DataTypes/DataTypeInterval.cpp b/src/DataTypes/DataTypeInterval.cpp index 57d071a8666..9faf0cec2d8 100644 --- a/src/DataTypes/DataTypeInterval.cpp +++ b/src/DataTypes/DataTypeInterval.cpp @@ -13,6 +13,9 @@ bool DataTypeInterval::equals(const IDataType & rhs) const void registerDataTypeInterval(DataTypeFactory & factory) { + factory.registerSimpleDataType("IntervalNanosecond", [] { return DataTypePtr(std::make_shared(IntervalKind::Nanosecond)); }); + factory.registerSimpleDataType("IntervalMicrosecond", [] { return DataTypePtr(std::make_shared(IntervalKind::Microsecond)); }); + factory.registerSimpleDataType("IntervalMillisecond", [] { return DataTypePtr(std::make_shared(IntervalKind::Millisecond)); }); factory.registerSimpleDataType("IntervalSecond", [] { return DataTypePtr(std::make_shared(IntervalKind::Second)); }); factory.registerSimpleDataType("IntervalMinute", [] { return DataTypePtr(std::make_shared(IntervalKind::Minute)); }); factory.registerSimpleDataType("IntervalHour", [] { return DataTypePtr(std::make_shared(IntervalKind::Hour)); }); diff --git a/src/DataTypes/DataTypeInterval.h b/src/DataTypes/DataTypeInterval.h index 9ef6237ec41..83d89a73460 100644 --- a/src/DataTypes/DataTypeInterval.h +++ b/src/DataTypes/DataTypeInterval.h @@ -25,7 +25,7 @@ public: IntervalKind getKind() const { return kind; } - DataTypeInterval(IntervalKind kind_) : kind(kind_) {} + explicit DataTypeInterval(IntervalKind kind_) : kind(kind_) {} std::string doGetName() const override { return fmt::format("Interval{}", kind.toString()); } const char * getFamilyName() const override { return "Interval"; } diff --git a/src/DataTypes/DataTypeMap.h b/src/DataTypes/DataTypeMap.h index 04377f85cfb..65bdd93ca4d 100644 --- a/src/DataTypes/DataTypeMap.h +++ b/src/DataTypes/DataTypeMap.h @@ -23,7 +23,7 @@ private: public: static constexpr bool is_parametric = true; - DataTypeMap(const DataTypes & elems); + explicit DataTypeMap(const DataTypes & elems); DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & value_type_); TypeIndex getTypeId() const override { return TypeIndex::Map; } diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp new file mode 100644 index 00000000000..659f69b6c68 --- /dev/null +++ b/src/DataTypes/DataTypeObject.cpp @@ -0,0 +1,82 @@ +#include +#include +#include + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int UNEXPECTED_AST_STRUCTURE; +} + +DataTypeObject::DataTypeObject(const String & schema_format_, bool is_nullable_) + : schema_format(Poco::toLower(schema_format_)) + , is_nullable(is_nullable_) +{ +} + +bool DataTypeObject::equals(const IDataType & rhs) const +{ + if (const auto * object = typeid_cast(&rhs)) + return schema_format == object->schema_format && is_nullable == object->is_nullable; + return false; +} + +SerializationPtr DataTypeObject::doGetDefaultSerialization() const +{ + return getObjectSerialization(schema_format); +} + +String DataTypeObject::doGetName() const +{ + WriteBufferFromOwnString out; + if (is_nullable) + out << "Object(Nullable(" << quote << schema_format << "))"; + else + out << "Object(" << quote << schema_format << ")"; + return out.str(); +} + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (!arguments || arguments->children.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Object data type family must have one argument - name of schema format"); + + ASTPtr schema_argument = arguments->children[0]; + bool is_nullable = false; + + if (const auto * func = schema_argument->as()) + { + if (func->name != "Nullable" || func->arguments->children.size() != 1) + throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, + "Expected 'Nullable()' as parameter for type Object", func->name); + + schema_argument = func->arguments->children[0]; + is_nullable = true; + } + + const auto * literal = schema_argument->as(); + if (!literal || literal->value.getType() != Field::Types::String) + throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, + "Object data type family must have a const string as its schema name parameter"); + + return std::make_shared(literal->value.get(), is_nullable); +} + +void registerDataTypeObject(DataTypeFactory & factory) +{ + factory.registerDataType("Object", create); + factory.registerSimpleDataType("JSON", + [] { return std::make_shared("JSON", false); }, + DataTypeFactory::CaseInsensitive); +} + +} diff --git a/src/DataTypes/DataTypeObject.h b/src/DataTypes/DataTypeObject.h new file mode 100644 index 00000000000..503947c3738 --- /dev/null +++ b/src/DataTypes/DataTypeObject.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +class DataTypeObject : public IDataType +{ +private: + String schema_format; + bool is_nullable; + +public: + DataTypeObject(const String & schema_format_, bool is_nullable_); + + const char * getFamilyName() const override { return "Object"; } + String doGetName() const override; + TypeIndex getTypeId() const override { return TypeIndex::Object; } + + MutableColumnPtr createColumn() const override { return ColumnObject::create(is_nullable); } + + Field getDefault() const override + { + throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + + bool haveSubtypes() const override { return false; } + bool equals(const IDataType & rhs) const override; + bool isParametric() const override { return true; } + + SerializationPtr doGetDefaultSerialization() const override; + + bool hasNullableSubcolumns() const { return is_nullable; } +}; + +} diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/DataTypeTuple.h index c56e87ca22d..db122aae5df 100644 --- a/src/DataTypes/DataTypeTuple.h +++ b/src/DataTypes/DataTypeTuple.h @@ -26,7 +26,7 @@ private: public: static constexpr bool is_parametric = true; - DataTypeTuple(const DataTypes & elems); + explicit DataTypeTuple(const DataTypes & elems); DataTypeTuple(const DataTypes & elems, const Strings & names, bool serialize_names_ = true); static bool canBeCreatedWithNames(const Strings & names); diff --git a/src/DataTypes/DataTypesDecimal.h b/src/DataTypes/DataTypesDecimal.h index fb590dd1d4b..0ec29e3c5f4 100644 --- a/src/DataTypes/DataTypesDecimal.h +++ b/src/DataTypes/DataTypesDecimal.h @@ -60,26 +60,26 @@ inline const DataTypeDecimal * checkDecimal(const IDataType & data_type) inline UInt32 getDecimalScale(const IDataType & data_type, UInt32 default_value = std::numeric_limits::max()) { - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getScale(); - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getScale(); - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getScale(); - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getScale(); return default_value; } inline UInt32 getDecimalPrecision(const IDataType & data_type) { - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getPrecision(); - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getPrecision(); - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getPrecision(); - if (auto * decimal_type = checkDecimal(data_type)) + if (const auto * decimal_type = checkDecimal(data_type)) return decimal_type->getPrecision(); return 0; } diff --git a/src/DataTypes/EnumValues.h b/src/DataTypes/EnumValues.h index 17c292c5551..0747cd4aed8 100644 --- a/src/DataTypes/EnumValues.h +++ b/src/DataTypes/EnumValues.h @@ -29,7 +29,7 @@ private: void fillMaps(); public: - EnumValues(const Values & values_); + explicit EnumValues(const Values & values_); const Values & getValues() const { return values; } diff --git a/src/DataTypes/FieldToDataType.cpp b/src/DataTypes/FieldToDataType.cpp index 8ca5ffac7c5..283d1b1e41a 100644 --- a/src/DataTypes/FieldToDataType.cpp +++ b/src/DataTypes/FieldToDataType.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -108,12 +109,11 @@ DataTypePtr FieldToDataType::operator() (const Array & x) const element_types.reserve(x.size()); for (const Field & elem : x) - element_types.emplace_back(applyVisitor(FieldToDataType(), elem)); + element_types.emplace_back(applyVisitor(FieldToDataType(allow_convertion_to_string), elem)); - return std::make_shared(getLeastSupertype(element_types)); + return std::make_shared(getLeastSupertype(element_types, allow_convertion_to_string)); } - DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const { if (tuple.empty()) @@ -123,7 +123,7 @@ DataTypePtr FieldToDataType::operator() (const Tuple & tuple) const element_types.reserve(tuple.size()); for (const auto & element : tuple) - element_types.push_back(applyVisitor(FieldToDataType(), element)); + element_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), element)); return std::make_shared(element_types); } @@ -139,11 +139,19 @@ DataTypePtr FieldToDataType::operator() (const Map & map) const { const auto & tuple = elem.safeGet(); assert(tuple.size() == 2); - key_types.push_back(applyVisitor(FieldToDataType(), tuple[0])); - value_types.push_back(applyVisitor(FieldToDataType(), tuple[1])); + key_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), tuple[0])); + value_types.push_back(applyVisitor(FieldToDataType(allow_convertion_to_string), tuple[1])); } - return std::make_shared(getLeastSupertype(key_types), getLeastSupertype(value_types)); + return std::make_shared( + getLeastSupertype(key_types, allow_convertion_to_string), + getLeastSupertype(value_types, allow_convertion_to_string)); +} + +DataTypePtr FieldToDataType::operator() (const Object &) const +{ + /// TODO: Do we need different parameters for type Object? + return std::make_shared("json", false); } DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) const diff --git a/src/DataTypes/FieldToDataType.h b/src/DataTypes/FieldToDataType.h index 72575c070f5..1922ac8b746 100644 --- a/src/DataTypes/FieldToDataType.h +++ b/src/DataTypes/FieldToDataType.h @@ -20,26 +20,34 @@ using DataTypePtr = std::shared_ptr; class FieldToDataType : public StaticVisitor { public: + FieldToDataType(bool allow_convertion_to_string_ = false) + : allow_convertion_to_string(allow_convertion_to_string_) + { + } + DataTypePtr operator() (const Null & x) const; DataTypePtr operator() (const UInt64 & x) const; DataTypePtr operator() (const UInt128 & x) const; - DataTypePtr operator() (const UInt256 & x) const; DataTypePtr operator() (const Int64 & x) const; DataTypePtr operator() (const Int128 & x) const; - DataTypePtr operator() (const Int256 & x) const; DataTypePtr operator() (const UUID & x) const; DataTypePtr operator() (const Float64 & x) const; DataTypePtr operator() (const String & x) const; DataTypePtr operator() (const Array & x) const; DataTypePtr operator() (const Tuple & tuple) const; DataTypePtr operator() (const Map & map) const; + DataTypePtr operator() (const Object & map) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const AggregateFunctionStateData & x) const; + DataTypePtr operator() (const UInt256 & x) const; + DataTypePtr operator() (const Int256 & x) const; DataTypePtr operator() (const bool & x) const; + +private: + bool allow_convertion_to_string; }; } - diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index edc9e4159f4..0976233c031 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -126,19 +126,25 @@ DataTypePtr IDataType::tryGetSubcolumnType(const String & subcolumn_name) const DataTypePtr IDataType::getSubcolumnType(const String & subcolumn_name) const { SubstreamData data = { getDefaultSerialization(), getPtr(), nullptr, nullptr }; - return getForSubcolumn(subcolumn_name, data, &SubstreamData::type); + return getForSubcolumn(subcolumn_name, data, &SubstreamData::type, true); } -SerializationPtr IDataType::getSubcolumnSerialization(const String & subcolumn_name, const SerializationPtr & serialization) const +ColumnPtr IDataType::tryGetSubcolumn(const String & subcolumn_name, const ColumnPtr & column) const { - SubstreamData data = { serialization, nullptr, nullptr, nullptr }; - return getForSubcolumn(subcolumn_name, data, &SubstreamData::serialization); + SubstreamData data = { getDefaultSerialization(), nullptr, column, nullptr }; + return getForSubcolumn(subcolumn_name, data, &SubstreamData::column, false); } ColumnPtr IDataType::getSubcolumn(const String & subcolumn_name, const ColumnPtr & column) const { SubstreamData data = { getDefaultSerialization(), nullptr, column, nullptr }; - return getForSubcolumn(subcolumn_name, data, &SubstreamData::column); + return getForSubcolumn(subcolumn_name, data, &SubstreamData::column, true); +} + +SerializationPtr IDataType::getSubcolumnSerialization(const String & subcolumn_name, const SerializationPtr & serialization) const +{ + SubstreamData data = { serialization, nullptr, nullptr, nullptr }; + return getForSubcolumn(subcolumn_name, data, &SubstreamData::serialization, true); } Names IDataType::getSubcolumnNames() const diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 5bc089e085f..fc9e50dc55b 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -82,9 +82,11 @@ public: DataTypePtr tryGetSubcolumnType(const String & subcolumn_name) const; DataTypePtr getSubcolumnType(const String & subcolumn_name) const; - SerializationPtr getSubcolumnSerialization(const String & subcolumn_name, const SerializationPtr & serialization) const; + ColumnPtr tryGetSubcolumn(const String & subcolumn_name, const ColumnPtr & column) const; ColumnPtr getSubcolumn(const String & subcolumn_name, const ColumnPtr & column) const; + SerializationPtr getSubcolumnSerialization(const String & subcolumn_name, const SerializationPtr & serialization) const; + using SubstreamData = ISerialization::SubstreamData; using SubstreamPath = ISerialization::SubstreamPath; @@ -309,7 +311,7 @@ private: const String & subcolumn_name, const SubstreamData & data, Ptr SubstreamData::*member, - bool throw_if_null = true) const; + bool throw_if_null) const; }; @@ -318,12 +320,12 @@ struct WhichDataType { TypeIndex idx; - constexpr WhichDataType(TypeIndex idx_ = TypeIndex::Nothing) : idx(idx_) {} - constexpr WhichDataType(const IDataType & data_type) : idx(data_type.getTypeId()) {} - constexpr WhichDataType(const IDataType * data_type) : idx(data_type->getTypeId()) {} + constexpr WhichDataType(TypeIndex idx_ = TypeIndex::Nothing) : idx(idx_) {} /// NOLINT + constexpr WhichDataType(const IDataType & data_type) : idx(data_type.getTypeId()) {} /// NOLINT + constexpr WhichDataType(const IDataType * data_type) : idx(data_type->getTypeId()) {} /// NOLINT // shared ptr -> is non-constexpr in gcc - WhichDataType(const DataTypePtr & data_type) : idx(data_type->getTypeId()) {} + WhichDataType(const DataTypePtr & data_type) : idx(data_type->getTypeId()) {} /// NOLINT constexpr bool isUInt8() const { return idx == TypeIndex::UInt8; } constexpr bool isUInt16() const { return idx == TypeIndex::UInt16; } @@ -373,11 +375,13 @@ struct WhichDataType constexpr bool isMap() const {return idx == TypeIndex::Map; } constexpr bool isSet() const { return idx == TypeIndex::Set; } constexpr bool isInterval() const { return idx == TypeIndex::Interval; } + constexpr bool isObject() const { return idx == TypeIndex::Object; } constexpr bool isNothing() const { return idx == TypeIndex::Nothing; } constexpr bool isNullable() const { return idx == TypeIndex::Nullable; } constexpr bool isFunction() const { return idx == TypeIndex::Function; } constexpr bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; } + constexpr bool isSimple() const { return isInt() || isUInt() || isFloat() || isString(); } constexpr bool isLowCarnality() const { return idx == TypeIndex::LowCardinality; } }; @@ -399,10 +403,16 @@ inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_ty inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); } inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); } inline bool isArray(const DataTypePtr & data_type) { return WhichDataType(data_type).isArray(); } -inline bool isMap(const DataTypePtr & data_type) { return WhichDataType(data_type).isMap(); } +inline bool isMap(const DataTypePtr & data_type) {return WhichDataType(data_type).isMap(); } inline bool isNothing(const DataTypePtr & data_type) { return WhichDataType(data_type).isNothing(); } inline bool isUUID(const DataTypePtr & data_type) { return WhichDataType(data_type).isUUID(); } +template +inline bool isObject(const T & data_type) +{ + return WhichDataType(data_type).isObject(); +} + template inline bool isUInt8(const T & data_type) { diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index b72e479cb1d..3a635d2e240 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -201,7 +201,7 @@ static inline llvm::Value * nativeCast(llvm::IRBuilder<> & b, const DataTypePtr return nativeCast(b, from, value, n_to); } -static inline std::pair nativeCastToCommon(llvm::IRBuilder<> & b, const DataTypePtr & lhs_type, llvm::Value * lhs, const DataTypePtr & rhs_type, llvm::Value * rhs) +static inline std::pair nativeCastToCommon(llvm::IRBuilder<> & b, const DataTypePtr & lhs_type, llvm::Value * lhs, const DataTypePtr & rhs_type, llvm::Value * rhs) /// NOLINT { llvm::Type * common; diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index b35a0713519..df504bc34a8 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -30,6 +30,12 @@ namespace Nested std::string concatenateName(const std::string & nested_table_name, const std::string & nested_field_name) { + if (nested_table_name.empty()) + return nested_field_name; + + if (nested_field_name.empty()) + return nested_table_name; + return nested_table_name + "." + nested_field_name; } diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp new file mode 100644 index 00000000000..9004a5296e0 --- /dev/null +++ b/src/DataTypes/ObjectUtils.cpp @@ -0,0 +1,703 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TYPE_MISMATCH; + extern const int LOGICAL_ERROR; + extern const int DUPLICATE_COLUMN; +} + +size_t getNumberOfDimensions(const IDataType & type) +{ + if (const auto * type_array = typeid_cast(&type)) + return type_array->getNumberOfDimensions(); + return 0; +} + +size_t getNumberOfDimensions(const IColumn & column) +{ + if (const auto * column_array = checkAndGetColumn(column)) + return column_array->getNumberOfDimensions(); + return 0; +} + +DataTypePtr getBaseTypeOfArray(const DataTypePtr & type) +{ + /// Get raw pointers to avoid extra copying of type pointers. + const DataTypeArray * last_array = nullptr; + const auto * current_type = type.get(); + while (const auto * type_array = typeid_cast(current_type)) + { + current_type = type_array->getNestedType().get(); + last_array = type_array; + } + + return last_array ? last_array->getNestedType() : type; +} + +ColumnPtr getBaseColumnOfArray(const ColumnPtr & column) +{ + /// Get raw pointers to avoid extra copying of column pointers. + const ColumnArray * last_array = nullptr; + const auto * current_column = column.get(); + while (const auto * column_array = checkAndGetColumn(current_column)) + { + current_column = &column_array->getData(); + last_array = column_array; + } + + return last_array ? last_array->getDataPtr() : column; +} + +DataTypePtr createArrayOfType(DataTypePtr type, size_t num_dimensions) +{ + for (size_t i = 0; i < num_dimensions; ++i) + type = std::make_shared(std::move(type)); + return type; +} + +ColumnPtr createArrayOfColumn(ColumnPtr column, size_t num_dimensions) +{ + for (size_t i = 0; i < num_dimensions; ++i) + column = ColumnArray::create(column); + return column; +} + +Array createEmptyArrayField(size_t num_dimensions) +{ + if (num_dimensions == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create array field with 0 dimensions"); + + Array array; + Array * current_array = &array; + for (size_t i = 1; i < num_dimensions; ++i) + { + current_array->push_back(Array()); + current_array = ¤t_array->back().get(); + } + + return array; +} + +DataTypePtr getDataTypeByColumn(const IColumn & column) +{ + auto idx = column.getDataType(); + if (WhichDataType(idx).isSimple()) + return DataTypeFactory::instance().get(String(magic_enum::enum_name(idx))); + + if (const auto * column_array = checkAndGetColumn(&column)) + return std::make_shared(getDataTypeByColumn(column_array->getData())); + + if (const auto * column_nullable = checkAndGetColumn(&column)) + return makeNullable(getDataTypeByColumn(column_nullable->getNestedColumn())); + + /// TODO: add more types. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get data type of column {}", column.getFamilyName()); +} + +template +static auto extractVector(const std::vector & vec) +{ + static_assert(I < std::tuple_size_v); + std::vector> res; + res.reserve(vec.size()); + for (const auto & elem : vec) + res.emplace_back(std::get(elem)); + return res; +} + +void convertObjectsToTuples(NamesAndTypesList & columns_list, Block & block, const NamesAndTypesList & extended_storage_columns) +{ + std::unordered_map storage_columns_map; + for (const auto & [name, type] : extended_storage_columns) + storage_columns_map[name] = type; + + for (auto & name_type : columns_list) + { + if (!isObject(name_type.type)) + continue; + + auto & column = block.getByName(name_type.name); + if (!isObject(column.type)) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Type for column '{}' mismatch in columns list and in block. In list: {}, in block: {}", + name_type.name, name_type.type->getName(), column.type->getName()); + + const auto & column_object = assert_cast(*column.column); + const auto & subcolumns = column_object.getSubcolumns(); + + if (!column_object.isFinalized()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot convert to tuple column '{}' from type {}. Column should be finalized first", + name_type.name, name_type.type->getName()); + + PathsInData tuple_paths; + DataTypes tuple_types; + Columns tuple_columns; + + for (const auto & entry : subcolumns) + { + tuple_paths.emplace_back(entry->path); + tuple_types.emplace_back(entry->data.getLeastCommonType()); + tuple_columns.emplace_back(entry->data.getFinalizedColumnPtr()); + } + + auto it = storage_columns_map.find(name_type.name); + if (it == storage_columns_map.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column '{}' not found in storage", name_type.name); + + std::tie(column.column, column.type) = unflattenTuple(tuple_paths, tuple_types, tuple_columns); + name_type.type = column.type; + + /// Check that constructed Tuple type and type in storage are compatible. + getLeastCommonTypeForObject({column.type, it->second}, true); + } +} + +static bool isPrefix(const PathInData::Parts & prefix, const PathInData::Parts & parts) +{ + if (prefix.size() > parts.size()) + return false; + + for (size_t i = 0; i < prefix.size(); ++i) + if (prefix[i].key != parts[i].key) + return false; + return true; +} + +void checkObjectHasNoAmbiguosPaths(const PathsInData & paths) +{ + size_t size = paths.size(); + for (size_t i = 0; i < size; ++i) + { + for (size_t j = 0; j < i; ++j) + { + if (isPrefix(paths[i].getParts(), paths[j].getParts()) + || isPrefix(paths[j].getParts(), paths[i].getParts())) + throw Exception(ErrorCodes::DUPLICATE_COLUMN, + "Data in Object has ambiguous paths: '{}' and '{}'", + paths[i].getPath(), paths[j].getPath()); + } + } +} + +DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths) +{ + if (types.empty()) + return nullptr; + + bool all_equal = true; + for (size_t i = 1; i < types.size(); ++i) + { + if (!types[i]->equals(*types[0])) + { + all_equal = false; + break; + } + } + + if (all_equal) + return types[0]; + + /// Types of subcolumns by path from all tuples. + std::unordered_map subcolumns_types; + + /// First we flatten tuples, then get common type for paths + /// and finally unflatten paths and create new tuple type. + for (const auto & type : types) + { + const auto * type_tuple = typeid_cast(type.get()); + if (!type_tuple) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Least common type for object can be deduced only from tuples, but {} given", type->getName()); + + auto [tuple_paths, tuple_types] = flattenTuple(type); + assert(tuple_paths.size() == tuple_types.size()); + + for (size_t i = 0; i < tuple_paths.size(); ++i) + subcolumns_types[tuple_paths[i]].push_back(tuple_types[i]); + } + + PathsInData tuple_paths; + DataTypes tuple_types; + + /// Get the least common type for all paths. + for (const auto & [key, subtypes] : subcolumns_types) + { + assert(!subtypes.empty()); + if (key.getPath() == ColumnObject::COLUMN_NAME_DUMMY) + continue; + + size_t first_dim = getNumberOfDimensions(*subtypes[0]); + for (size_t i = 1; i < subtypes.size(); ++i) + if (first_dim != getNumberOfDimensions(*subtypes[i])) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Uncompatible types of subcolumn '{}': {} and {}", + key.getPath(), subtypes[0]->getName(), subtypes[i]->getName()); + + tuple_paths.emplace_back(key); + tuple_types.emplace_back(getLeastSupertype(subtypes, /*allow_conversion_to_string=*/ true)); + } + + if (tuple_paths.empty()) + { + tuple_paths.emplace_back(ColumnObject::COLUMN_NAME_DUMMY); + tuple_types.emplace_back(std::make_shared()); + } + + if (check_ambiguos_paths) + checkObjectHasNoAmbiguosPaths(tuple_paths); + + return unflattenTuple(tuple_paths, tuple_types); +} + +NameSet getNamesOfObjectColumns(const NamesAndTypesList & columns_list) +{ + NameSet res; + for (const auto & [name, type] : columns_list) + if (isObject(type)) + res.insert(name); + + return res; +} + +bool hasObjectColumns(const ColumnsDescription & columns) +{ + return std::any_of(columns.begin(), columns.end(), [](const auto & column) { return isObject(column.type); }); +} + +void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns) +{ + NamesAndTypesList subcolumns_list; + for (auto & column : columns_list) + { + auto object_column = object_columns.tryGetColumn(GetColumnsOptions::All, column.name); + if (object_column) + { + column.type = object_column->type; + + if (with_subcolumns) + subcolumns_list.splice(subcolumns_list.end(), object_columns.getSubcolumns(column.name)); + } + } + + columns_list.splice(columns_list.end(), std::move(subcolumns_list)); +} + +void updateObjectColumns(ColumnsDescription & object_columns, const NamesAndTypesList & new_columns) +{ + for (const auto & new_column : new_columns) + { + auto object_column = object_columns.tryGetColumn(GetColumnsOptions::All, new_column.name); + if (object_column && !object_column->type->equals(*new_column.type)) + { + object_columns.modify(new_column.name, [&](auto & column) + { + column.type = getLeastCommonTypeForObject({object_column->type, new_column.type}); + }); + } + } +} + +namespace +{ + +void flattenTupleImpl( + PathInDataBuilder & builder, + DataTypePtr type, + std::vector & new_paths, + DataTypes & new_types) +{ + if (const auto * type_tuple = typeid_cast(type.get())) + { + const auto & tuple_names = type_tuple->getElementNames(); + const auto & tuple_types = type_tuple->getElements(); + + for (size_t i = 0; i < tuple_names.size(); ++i) + { + builder.append(tuple_names[i], false); + flattenTupleImpl(builder, tuple_types[i], new_paths, new_types); + builder.popBack(); + } + } + else if (const auto * type_array = typeid_cast(type.get())) + { + PathInDataBuilder element_builder; + std::vector element_paths; + DataTypes element_types; + + flattenTupleImpl(element_builder, type_array->getNestedType(), element_paths, element_types); + assert(element_paths.size() == element_types.size()); + + for (size_t i = 0; i < element_paths.size(); ++i) + { + builder.append(element_paths[i], true); + new_paths.emplace_back(builder.getParts()); + new_types.emplace_back(std::make_shared(element_types[i])); + builder.popBack(element_paths[i].size()); + } + } + else + { + new_paths.emplace_back(builder.getParts()); + new_types.emplace_back(type); + } +} + +/// @offsets_columns are used as stack of array offsets and allows to recreate Array columns. +void flattenTupleImpl(const ColumnPtr & column, Columns & new_columns, Columns & offsets_columns) +{ + if (const auto * column_tuple = checkAndGetColumn(column.get())) + { + const auto & subcolumns = column_tuple->getColumns(); + for (const auto & subcolumn : subcolumns) + flattenTupleImpl(subcolumn, new_columns, offsets_columns); + } + else if (const auto * column_array = checkAndGetColumn(column.get())) + { + offsets_columns.push_back(column_array->getOffsetsPtr()); + flattenTupleImpl(column_array->getDataPtr(), new_columns, offsets_columns); + offsets_columns.pop_back(); + } + else + { + if (!offsets_columns.empty()) + { + auto new_column = ColumnArray::create(column, offsets_columns.back()); + for (auto it = offsets_columns.rbegin() + 1; it != offsets_columns.rend(); ++it) + new_column = ColumnArray::create(new_column, *it); + + new_columns.push_back(std::move(new_column)); + } + else + { + new_columns.push_back(column); + } + } +} + +DataTypePtr reduceNumberOfDimensions(DataTypePtr type, size_t dimensions_to_reduce) +{ + while (dimensions_to_reduce--) + { + const auto * type_array = typeid_cast(type.get()); + if (!type_array) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not enough dimensions to reduce"); + + type = type_array->getNestedType(); + } + + return type; +} + +ColumnPtr reduceNumberOfDimensions(ColumnPtr column, size_t dimensions_to_reduce) +{ + while (dimensions_to_reduce--) + { + const auto * column_array = typeid_cast(column.get()); + if (!column_array) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not enough dimensions to reduce"); + + column = column_array->getDataPtr(); + } + + return column; +} + +/// We save intermediate column, type and number of array +/// dimensions for each intermediate node in path in subcolumns tree. +struct ColumnWithTypeAndDimensions +{ + ColumnPtr column; + DataTypePtr type; + size_t array_dimensions; +}; + +using SubcolumnsTreeWithColumns = SubcolumnsTree; +using Node = SubcolumnsTreeWithColumns::Node; + +/// Creates data type and column from tree of subcolumns. +ColumnWithTypeAndDimensions createTypeFromNode(const Node * node) +{ + auto collect_tuple_elemets = [](const auto & children) + { + std::vector> tuple_elements; + tuple_elements.reserve(children.size()); + for (const auto & [name, child] : children) + { + auto column = createTypeFromNode(child.get()); + tuple_elements.emplace_back(name, std::move(column)); + } + + /// Sort to always create the same type for the same set of subcolumns. + std::sort(tuple_elements.begin(), tuple_elements.end(), + [](const auto & lhs, const auto & rhs) { return std::get<0>(lhs) < std::get<0>(rhs); }); + + auto tuple_names = extractVector<0>(tuple_elements); + auto tuple_columns = extractVector<1>(tuple_elements); + + return std::make_tuple(std::move(tuple_names), std::move(tuple_columns)); + }; + + if (node->kind == Node::SCALAR) + { + return node->data; + } + else if (node->kind == Node::NESTED) + { + auto [tuple_names, tuple_columns] = collect_tuple_elemets(node->children); + + Columns offsets_columns; + offsets_columns.reserve(tuple_columns[0].array_dimensions + 1); + + /// If we have a Nested node and child node with anonymous array levels + /// we need to push a Nested type through all array levels. + /// Example: { "k1": [[{"k2": 1, "k3": 2}] } should be parsed as + /// `k1 Array(Nested(k2 Int, k3 Int))` and k1 is marked as Nested + /// and `k2` and `k3` has anonymous_array_level = 1 in that case. + + const auto & current_array = assert_cast(*node->data.column); + offsets_columns.push_back(current_array.getOffsetsPtr()); + + auto first_column = tuple_columns[0].column; + for (size_t i = 0; i < tuple_columns[0].array_dimensions; ++i) + { + const auto & column_array = assert_cast(*first_column); + offsets_columns.push_back(column_array.getOffsetsPtr()); + first_column = column_array.getDataPtr(); + } + + size_t num_elements = tuple_columns.size(); + Columns tuple_elements_columns(num_elements); + DataTypes tuple_elements_types(num_elements); + + /// Reduce extra array dimensions to get columns and types of Nested elements. + for (size_t i = 0; i < num_elements; ++i) + { + assert(tuple_columns[i].array_dimensions == tuple_columns[0].array_dimensions); + tuple_elements_columns[i] = reduceNumberOfDimensions(tuple_columns[i].column, tuple_columns[i].array_dimensions); + tuple_elements_types[i] = reduceNumberOfDimensions(tuple_columns[i].type, tuple_columns[i].array_dimensions); + } + + auto result_column = ColumnArray::create(ColumnTuple::create(tuple_elements_columns), offsets_columns.back()); + auto result_type = createNested(tuple_elements_types, tuple_names); + + /// Recreate result Array type and Array column. + for (auto it = offsets_columns.rbegin() + 1; it != offsets_columns.rend(); ++it) + { + result_column = ColumnArray::create(result_column, *it); + result_type = std::make_shared(result_type); + } + + return {result_column, result_type, tuple_columns[0].array_dimensions}; + } + else + { + auto [tuple_names, tuple_columns] = collect_tuple_elemets(node->children); + + size_t num_elements = tuple_columns.size(); + Columns tuple_elements_columns(num_elements); + DataTypes tuple_elements_types(num_elements); + + for (size_t i = 0; i < tuple_columns.size(); ++i) + { + assert(tuple_columns[i].array_dimensions == tuple_columns[0].array_dimensions); + tuple_elements_columns[i] = tuple_columns[i].column; + tuple_elements_types[i] = tuple_columns[i].type; + } + + auto result_column = ColumnTuple::create(tuple_elements_columns); + auto result_type = std::make_shared(tuple_elements_types, tuple_names); + + return {result_column, result_type, tuple_columns[0].array_dimensions}; + } +} + +} + +std::pair flattenTuple(const DataTypePtr & type) +{ + std::vector new_path_parts; + DataTypes new_types; + PathInDataBuilder builder; + + flattenTupleImpl(builder, type, new_path_parts, new_types); + + PathsInData new_paths(new_path_parts.begin(), new_path_parts.end()); + return {new_paths, new_types}; +} + +ColumnPtr flattenTuple(const ColumnPtr & column) +{ + Columns new_columns; + Columns offsets_columns; + + flattenTupleImpl(column, new_columns, offsets_columns); + return ColumnTuple::create(new_columns); +} + +DataTypePtr unflattenTuple(const PathsInData & paths, const DataTypes & tuple_types) +{ + assert(paths.size() == tuple_types.size()); + Columns tuple_columns; + tuple_columns.reserve(tuple_types.size()); + for (const auto & type : tuple_types) + tuple_columns.emplace_back(type->createColumn()); + + return unflattenTuple(paths, tuple_types, tuple_columns).second; +} + +std::pair unflattenTuple( + const PathsInData & paths, + const DataTypes & tuple_types, + const Columns & tuple_columns) +{ + assert(paths.size() == tuple_types.size()); + assert(paths.size() == tuple_columns.size()); + + /// We add all paths to the subcolumn tree and then create a type from it. + /// The tree stores column, type and number of array dimensions + /// for each intermediate node. + SubcolumnsTreeWithColumns tree; + + for (size_t i = 0; i < paths.size(); ++i) + { + auto column = tuple_columns[i]; + auto type = tuple_types[i]; + + const auto & parts = paths[i].getParts(); + size_t num_parts = parts.size(); + + size_t pos = 0; + tree.add(paths[i], [&](Node::Kind kind, bool exists) -> std::shared_ptr + { + if (pos >= num_parts) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Not enough name parts for path {}. Expected at least {}, got {}", + paths[i].getPath(), pos + 1, num_parts); + + size_t array_dimensions = kind == Node::NESTED ? 1 : parts[pos].anonymous_array_level; + ColumnWithTypeAndDimensions current_column{column, type, array_dimensions}; + + /// Get type and column for next node. + if (array_dimensions) + { + type = reduceNumberOfDimensions(type, array_dimensions); + column = reduceNumberOfDimensions(column, array_dimensions); + } + + ++pos; + if (exists) + return nullptr; + + return kind == Node::SCALAR + ? std::make_shared(kind, current_column, paths[i]) + : std::make_shared(kind, current_column); + }); + } + + auto [column, type, _] = createTypeFromNode(tree.getRoot()); + return std::make_pair(std::move(column), std::move(type)); +} + +static void addConstantToWithClause(const ASTPtr & query, const String & column_name, const DataTypePtr & data_type) +{ + auto & select = query->as(); + if (!select.with()) + select.setExpression(ASTSelectQuery::Expression::WITH, std::make_shared()); + + /// TODO: avoid materialize + auto node = makeASTFunction("materialize", + makeASTFunction("CAST", + std::make_shared(data_type->getDefault()), + std::make_shared(data_type->getName()))); + + node->alias = column_name; + node->prefer_alias_to_column_name = true; + select.with()->children.push_back(std::move(node)); +} + +/// @expected_columns and @available_columns contain descriptions +/// of extended Object columns. +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + ASTPtr query) +{ + NamesAndTypes missed_names_types; + + /// Find all subcolumns that are in @expected_columns, but not in @available_columns. + for (const auto & column : available_columns) + { + auto expected_column = expected_columns.getColumn(GetColumnsOptions::All, column.name); + + /// Extract all paths from both descriptions to easily check existence of subcolumns. + auto [available_paths, available_types] = flattenTuple(column.type); + auto [expected_paths, expected_types] = flattenTuple(expected_column.type); + + auto extract_names_and_types = [&column](const auto & paths, const auto & types) + { + NamesAndTypes res; + res.reserve(paths.size()); + for (size_t i = 0; i < paths.size(); ++i) + { + auto full_name = Nested::concatenateName(column.name, paths[i].getPath()); + res.emplace_back(full_name, types[i]); + } + + std::sort(res.begin(), res.end()); + return res; + }; + + auto available_names_types = extract_names_and_types(available_paths, available_types); + auto expected_names_types = extract_names_and_types(expected_paths, expected_types); + + std::set_difference( + expected_names_types.begin(), expected_names_types.end(), + available_names_types.begin(), available_names_types.end(), + std::back_inserter(missed_names_types), + [](const auto & lhs, const auto & rhs) { return lhs.name < rhs.name; }); + } + + if (missed_names_types.empty()) + return; + + IdentifierNameSet identifiers; + query->collectIdentifierNames(identifiers); + + /// Replace missed subcolumns to default literals of theirs type. + for (const auto & [name, type] : missed_names_types) + if (identifiers.count(name)) + addConstantToWithClause(query, name, type); +} + +void finalizeObjectColumns(MutableColumns & columns) +{ + for (auto & column : columns) + if (auto * column_object = typeid_cast(column.get())) + column_object->finalize(); +} + +} diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h new file mode 100644 index 00000000000..199a048c8cd --- /dev/null +++ b/src/DataTypes/ObjectUtils.h @@ -0,0 +1,140 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Returns number of dimensions in Array type. 0 if type is not array. +size_t getNumberOfDimensions(const IDataType & type); + +/// Returns number of dimensions in Array column. 0 if column is not array. +size_t getNumberOfDimensions(const IColumn & column); + +/// Returns type of scalars of Array of arbitrary dimensions. +DataTypePtr getBaseTypeOfArray(const DataTypePtr & type); + +/// Returns Array type with requested scalar type and number of dimensions. +DataTypePtr createArrayOfType(DataTypePtr type, size_t num_dimensions); + +/// Returns column of scalars of Array of arbitrary dimensions. +ColumnPtr getBaseColumnOfArray(const ColumnPtr & column); + +/// Returns empty Array column with requested scalar column and number of dimensions. +ColumnPtr createArrayOfColumn(const ColumnPtr & column, size_t num_dimensions); + +/// Returns Array with requested number of dimensions and no scalars. +Array createEmptyArrayField(size_t num_dimensions); + +/// Tries to get data type by column. Only limited subset of types is supported +DataTypePtr getDataTypeByColumn(const IColumn & column); + +/// Converts Object types and columns to Tuples in @columns_list and @block +/// and checks that types are consistent with types in @extended_storage_columns. +void convertObjectsToTuples(NamesAndTypesList & columns_list, Block & block, const NamesAndTypesList & extended_storage_columns); + +/// Checks that each path is not the prefix of any other path. +void checkObjectHasNoAmbiguosPaths(const PathsInData & paths); + +/// Receives several Tuple types and deduces the least common type among them. +DataTypePtr getLeastCommonTypeForObject(const DataTypes & types, bool check_ambiguos_paths = false); + +/// Converts types of object columns to tuples in @columns_list +/// according to @object_columns and adds all tuple's subcolumns if needed. +void extendObjectColumns(NamesAndTypesList & columns_list, const ColumnsDescription & object_columns, bool with_subcolumns); + +NameSet getNamesOfObjectColumns(const NamesAndTypesList & columns_list); +bool hasObjectColumns(const ColumnsDescription & columns); +void finalizeObjectColumns(MutableColumns & columns); + +/// Updates types of objects in @object_columns inplace +/// according to types in new_columns. +void updateObjectColumns(ColumnsDescription & object_columns, const NamesAndTypesList & new_columns); + +using DataTypeTuplePtr = std::shared_ptr; + +/// Flattens nested Tuple to plain Tuple. I.e extracts all paths and types from tuple. +/// E.g. Tuple(t Tuple(c1 UInt32, c2 String), c3 UInt64) -> Tuple(t.c1 UInt32, t.c2 String, c3 UInt32) +std::pair flattenTuple(const DataTypePtr & type); + +/// Flattens nested Tuple column to plain Tuple column. +ColumnPtr flattenTuple(const ColumnPtr & column); + +/// The reverse operation to 'flattenTuple'. +/// Creates nested Tuple from all paths and types. +/// E.g. Tuple(t.c1 UInt32, t.c2 String, c3 UInt32) -> Tuple(t Tuple(c1 UInt32, c2 String), c3 UInt64) +DataTypePtr unflattenTuple( + const PathsInData & paths, + const DataTypes & tuple_types); + +std::pair unflattenTuple( + const PathsInData & paths, + const DataTypes & tuple_types, + const Columns & tuple_columns); + +/// For all columns which exist in @expected_columns and +/// don't exist in @available_columns adds to WITH clause +/// an alias with column name to literal of default value of column type. +void replaceMissedSubcolumnsByConstants( + const ColumnsDescription & expected_columns, + const ColumnsDescription & available_columns, + ASTPtr query); + +/// Receives range of objects, which contains collections +/// of columns-like objects (e.g. ColumnsDescription or NamesAndTypesList) +/// and deduces the common types of object columns for all entries. +/// @entry_columns_getter should extract reference to collection of +/// columns-like objects from entry to which Iterator points. +/// columns-like object should have fields "name" and "type". +template +ColumnsDescription getObjectColumns( + Iterator begin, Iterator end, + const ColumnsDescription & storage_columns, + EntryColumnsGetter && entry_columns_getter) +{ + ColumnsDescription res; + + if (begin == end) + { + for (const auto & column : storage_columns) + { + if (isObject(column.type)) + { + auto tuple_type = std::make_shared( + DataTypes{std::make_shared()}, + Names{ColumnObject::COLUMN_NAME_DUMMY}); + + res.add({column.name, std::move(tuple_type)}); + } + } + + return res; + } + + std::unordered_map types_in_entries; + + for (auto it = begin; it != end; ++it) + { + const auto & entry_columns = entry_columns_getter(*it); + for (const auto & column : entry_columns) + { + auto storage_column = storage_columns.tryGetPhysical(column.name); + if (storage_column && isObject(storage_column->type)) + types_in_entries[column.name].push_back(column.type); + } + } + + for (const auto & [name, types] : types_in_entries) + res.add({name, getLeastCommonTypeForObject(types)}); + + return res; +} + +} diff --git a/src/DataTypes/Serializations/CMakeLists.txt b/src/DataTypes/Serializations/CMakeLists.txt new file mode 100644 index 00000000000..65172356645 --- /dev/null +++ b/src/DataTypes/Serializations/CMakeLists.txt @@ -0,0 +1,3 @@ +if (ENABLE_TESTS) + add_subdirectory (tests) +endif () diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 7df4a956c1a..512653ecb13 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -172,6 +172,10 @@ String getNameForSubstreamPath( else stream_name += "." + it->tuple_element_name; } + else if (it->type == Substream::ObjectElement) + { + stream_name += escapeForFileName(".") + escapeForFileName(it->object_key_name); + } } return stream_name; diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index b1fd4d0a9da..6c6b64f2416 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -125,6 +125,9 @@ public: SparseElements, SparseOffsets, + ObjectStructure, + ObjectElement, + Regular, }; @@ -133,6 +136,9 @@ public: /// Index of tuple element, starting at 1 or name. String tuple_element_name; + /// Name of subcolumn of object column. + String object_key_name; + /// Do we need to escape a dot in filenames for tuple elements. bool escape_tuple_delimiter = true; @@ -145,7 +151,7 @@ public: /// Flag, that may help to traverse substream paths. mutable bool visited = false; - Substream(Type type_) : type(type_) {} + Substream(Type type_) : type(type_) {} /// NOLINT String toString() const; }; diff --git a/src/DataTypes/Serializations/JSONDataParser.h b/src/DataTypes/Serializations/JSONDataParser.h new file mode 100644 index 00000000000..36abc9278d1 --- /dev/null +++ b/src/DataTypes/Serializations/JSONDataParser.h @@ -0,0 +1,183 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +class ReadBuffer; +class WriteBuffer; + +template +static Field getValueAsField(const Element & element) +{ + if (element.isBool()) return element.getBool(); + if (element.isInt64()) return element.getInt64(); + if (element.isUInt64()) return element.getUInt64(); + if (element.isDouble()) return element.getDouble(); + if (element.isString()) return element.getString(); + if (element.isNull()) return Field(); + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsupported type of JSON field"); +} + +template +class JSONDataParser +{ +public: + using Element = typename ParserImpl::Element; + + static void readJSON(String & s, ReadBuffer & buf) + { + readJSONObjectPossiblyInvalid(s, buf); + } + + std::optional parse(const char * begin, size_t length) + { + std::string_view json{begin, length}; + Element document; + if (!parser.parse(json, document)) + return {}; + + ParseResult result; + PathInDataBuilder builder; + std::vector paths; + + traverse(document, builder, paths, result.values); + + result.paths.reserve(paths.size()); + for (auto && path : paths) + result.paths.emplace_back(std::move(path)); + + return result; + } + +private: + void traverse( + const Element & element, + PathInDataBuilder & builder, + std::vector & paths, + std::vector & values) + { + checkStackSize(); + + if (element.isObject()) + { + auto object = element.getObject(); + + paths.reserve(paths.size() + object.size()); + values.reserve(values.size() + object.size()); + + for (auto it = object.begin(); it != object.end(); ++it) + { + const auto & [key, value] = *it; + traverse(value, builder.append(key, false), paths, values); + builder.popBack(); + } + } + else if (element.isArray()) + { + auto array = element.getArray(); + + using PathPartsWithArray = std::pair; + using PathToArray = HashMapWithStackMemory; + + /// Traverse elements of array and collect an array + /// of fields by each path. + + PathToArray arrays_by_path; + Arena strings_pool; + + size_t current_size = 0; + for (auto it = array.begin(); it != array.end(); ++it) + { + std::vector element_paths; + std::vector element_values; + PathInDataBuilder element_builder; + + traverse(*it, element_builder, element_paths, element_values); + size_t size = element_paths.size(); + size_t keys_to_update = arrays_by_path.size(); + + for (size_t i = 0; i < size; ++i) + { + UInt128 hash = PathInData::getPartsHash(element_paths[i]); + if (auto * found = arrays_by_path.find(hash)) + { + auto & path_array = found->getMapped().second; + + assert(path_array.size() == current_size); + path_array.push_back(std::move(element_values[i])); + --keys_to_update; + } + else + { + /// We found a new key. Add and empty array with current size. + Array path_array; + path_array.reserve(array.size()); + path_array.resize(current_size); + path_array.push_back(std::move(element_values[i])); + + auto & elem = arrays_by_path[hash]; + elem.first = std::move(element_paths[i]); + elem.second = std::move(path_array); + } + } + + /// If some of the keys are missed in current element, + /// add default values for them. + if (keys_to_update) + { + for (auto & [_, value] : arrays_by_path) + { + auto & path_array = value.second; + assert(path_array.size() == current_size || path_array.size() == current_size + 1); + if (path_array.size() == current_size) + path_array.push_back(Field()); + } + } + + ++current_size; + } + + if (arrays_by_path.empty()) + { + paths.push_back(builder.getParts()); + values.push_back(Array()); + } + else + { + paths.reserve(paths.size() + arrays_by_path.size()); + values.reserve(values.size() + arrays_by_path.size()); + + for (auto && [_, value] : arrays_by_path) + { + auto && [path, path_array] = value; + + /// Merge prefix path and path of array element. + paths.push_back(builder.append(path, true).getParts()); + values.push_back(std::move(path_array)); + + builder.popBack(path.size()); + } + } + } + else + { + paths.push_back(builder.getParts()); + values.push_back(getValueAsField(element)); + } + } + + ParserImpl parser; +}; + +} diff --git a/src/DataTypes/Serializations/PathInData.cpp b/src/DataTypes/Serializations/PathInData.cpp new file mode 100644 index 00000000000..574c34f1c27 --- /dev/null +++ b/src/DataTypes/Serializations/PathInData.cpp @@ -0,0 +1,198 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace DB +{ + +PathInData::PathInData(std::string_view path_) + : path(path_) +{ + const char * begin = path.data(); + const char * end = path.data() + path.size(); + + for (const char * it = path.data(); it != end; ++it) + { + if (*it == '.') + { + size_t size = static_cast(it - begin); + parts.emplace_back(std::string_view{begin, size}, false, 0); + begin = it + 1; + } + } + + size_t size = static_cast(end - begin); + parts.emplace_back(std::string_view{begin, size}, false, 0.); +} + +PathInData::PathInData(const Parts & parts_) +{ + buildPath(parts_); + buildParts(parts_); +} + +PathInData::PathInData(const PathInData & other) + : path(other.path) +{ + buildParts(other.getParts()); +} + +PathInData & PathInData::operator=(const PathInData & other) +{ + if (this != &other) + { + path = other.path; + buildParts(other.parts); + } + return *this; +} + +UInt128 PathInData::getPartsHash(const Parts & parts_) +{ + SipHash hash; + hash.update(parts_.size()); + for (const auto & part : parts_) + { + hash.update(part.key.data(), part.key.length()); + hash.update(part.is_nested); + hash.update(part.anonymous_array_level); + } + + UInt128 res; + hash.get128(res); + return res; +} + +void PathInData::writeBinary(WriteBuffer & out) const +{ + writeVarUInt(parts.size(), out); + for (const auto & part : parts) + { + writeStringBinary(part.key, out); + writeIntBinary(part.is_nested, out); + writeIntBinary(part.anonymous_array_level, out); + } +} + +void PathInData::readBinary(ReadBuffer & in) +{ + size_t num_parts; + readVarUInt(num_parts, in); + + Arena arena; + Parts temp_parts; + temp_parts.reserve(num_parts); + + for (size_t i = 0; i < num_parts; ++i) + { + bool is_nested; + UInt8 anonymous_array_level; + + auto ref = readStringBinaryInto(arena, in); + readIntBinary(is_nested, in); + readIntBinary(anonymous_array_level, in); + + temp_parts.emplace_back(static_cast(ref), is_nested, anonymous_array_level); + } + + /// Recreate path and parts. + buildPath(temp_parts); + buildParts(temp_parts); +} + +void PathInData::buildPath(const Parts & other_parts) +{ + if (other_parts.empty()) + return; + + path.clear(); + auto it = other_parts.begin(); + path += it->key; + ++it; + for (; it != other_parts.end(); ++it) + { + path += "."; + path += it->key; + } +} + +void PathInData::buildParts(const Parts & other_parts) +{ + if (other_parts.empty()) + return; + + parts.clear(); + parts.reserve(other_parts.size()); + const char * begin = path.data(); + for (const auto & part : other_parts) + { + has_nested |= part.is_nested; + parts.emplace_back(std::string_view{begin, part.key.length()}, part.is_nested, part.anonymous_array_level); + begin += part.key.length() + 1; + } +} + +size_t PathInData::Hash::operator()(const PathInData & value) const +{ + auto hash = getPartsHash(value.parts); + return hash.items[0] ^ hash.items[1]; +} + +PathInDataBuilder & PathInDataBuilder::append(std::string_view key, bool is_array) +{ + if (parts.empty()) + current_anonymous_array_level += is_array; + + if (!key.empty()) + { + if (!parts.empty()) + parts.back().is_nested = is_array; + + parts.emplace_back(key, false, current_anonymous_array_level); + current_anonymous_array_level = 0; + } + + return *this; +} + +PathInDataBuilder & PathInDataBuilder::append(const PathInData::Parts & path, bool is_array) +{ + if (parts.empty()) + current_anonymous_array_level += is_array; + + if (!path.empty()) + { + if (!parts.empty()) + parts.back().is_nested = is_array; + + auto it = parts.insert(parts.end(), path.begin(), path.end()); + for (; it != parts.end(); ++it) + it->anonymous_array_level += current_anonymous_array_level; + current_anonymous_array_level = 0; + } + + return *this; +} + +void PathInDataBuilder::popBack() +{ + parts.pop_back(); +} + +void PathInDataBuilder::popBack(size_t n) +{ + assert(n <= parts.size()); + parts.resize(parts.size() - n); +} + +} diff --git a/src/DataTypes/Serializations/PathInData.h b/src/DataTypes/Serializations/PathInData.h new file mode 100644 index 00000000000..323bc37d99b --- /dev/null +++ b/src/DataTypes/Serializations/PathInData.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class ReadBuffer; +class WriteBuffer; + +/// Class that represents path in document, e.g. JSON. +class PathInData +{ +public: + struct Part + { + Part() = default; + Part(std::string_view key_, bool is_nested_, UInt8 anonymous_array_level_) + : key(key_), is_nested(is_nested_), anonymous_array_level(anonymous_array_level_) + { + } + + /// Name of part of path. + std::string_view key; + + /// If this part is Nested, i.e. element + /// related to this key is the array of objects. + bool is_nested = false; + + /// Number of array levels between current key and previous key. + /// E.g. in JSON {"k1": [[[{"k2": 1, "k3": 2}]]]} + /// "k1" is nested and has anonymous_array_level = 0. + /// "k2" and "k3" are not nested and have anonymous_array_level = 2. + UInt8 anonymous_array_level = 0; + + bool operator==(const Part & other) const = default; + }; + + using Parts = std::vector; + + PathInData() = default; + explicit PathInData(std::string_view path_); + explicit PathInData(const Parts & parts_); + + PathInData(const PathInData & other); + PathInData & operator=(const PathInData & other); + + static UInt128 getPartsHash(const Parts & parts_); + + bool empty() const { return parts.empty(); } + + const String & getPath() const { return path; } + const Parts & getParts() const { return parts; } + + bool isNested(size_t i) const { return parts[i].is_nested; } + bool hasNested() const { return has_nested; } + + void writeBinary(WriteBuffer & out) const; + void readBinary(ReadBuffer & in); + + bool operator==(const PathInData & other) const { return parts == other.parts; } + struct Hash { size_t operator()(const PathInData & value) const; }; + +private: + /// Creates full path from parts. + void buildPath(const Parts & other_parts); + + /// Creates new parts full from full path with correct string pointers. + void buildParts(const Parts & other_parts); + + /// The full path. Parts are separated by dots. + String path; + + /// Parts of the path. All string_view-s in parts must point to the @path. + Parts parts; + + /// True if at least one part is nested. + /// Cached to avoid linear complexity at 'hasNested'. + bool has_nested = false; +}; + +class PathInDataBuilder +{ +public: + const PathInData::Parts & getParts() const { return parts; } + + PathInDataBuilder & append(std::string_view key, bool is_array); + PathInDataBuilder & append(const PathInData::Parts & path, bool is_array); + + void popBack(); + void popBack(size_t n); + +private: + PathInData::Parts parts; + + /// Number of array levels without key to which + /// next non-empty key will be nested. + /// Example: for JSON { "k1": [[{"k2": 1, "k3": 2}] } + // `k2` and `k3` has anonymous_array_level = 1 in that case. + size_t current_anonymous_array_level = 0; +}; + +using PathsInData = std::vector; + +/// Result of parsing of a document. +/// Contains all paths extracted from document +/// and values which are related to them. +struct ParseResult +{ + std::vector paths; + std::vector values; +}; + +} diff --git a/src/DataTypes/Serializations/SerializationArray.h b/src/DataTypes/Serializations/SerializationArray.h index cd8cac54881..3769f8a4513 100644 --- a/src/DataTypes/Serializations/SerializationArray.h +++ b/src/DataTypes/Serializations/SerializationArray.h @@ -11,7 +11,7 @@ private: SerializationPtr nested; public: - SerializationArray(const SerializationPtr & nested_) : nested(nested_) {} + explicit SerializationArray(const SerializationPtr & nested_) : nested(nested_) {} void serializeBinary(const Field & field, WriteBuffer & ostr) const override; void deserializeBinary(Field & field, ReadBuffer & istr) const override; @@ -71,7 +71,7 @@ private: { const ColumnPtr offsets; - SubcolumnCreator(const ColumnPtr & offsets_) : offsets(offsets_) {} + explicit SubcolumnCreator(const ColumnPtr & offsets_) : offsets(offsets_) {} DataTypePtr create(const DataTypePtr & prev) const override; SerializationPtr create(const SerializationPtr & prev) const override; diff --git a/src/DataTypes/Serializations/SerializationBool.h b/src/DataTypes/Serializations/SerializationBool.h index a9f4c6404b3..a5aa0ca80a2 100644 --- a/src/DataTypes/Serializations/SerializationBool.h +++ b/src/DataTypes/Serializations/SerializationBool.h @@ -10,7 +10,7 @@ namespace DB class SerializationBool final : public SerializationWrapper { public: - SerializationBool(const SerializationPtr & nested_); + explicit SerializationBool(const SerializationPtr & nested_); void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; diff --git a/src/DataTypes/Serializations/SerializationCustomSimpleText.h b/src/DataTypes/Serializations/SerializationCustomSimpleText.h index ba7c712f86c..21d6f8af650 100644 --- a/src/DataTypes/Serializations/SerializationCustomSimpleText.h +++ b/src/DataTypes/Serializations/SerializationCustomSimpleText.h @@ -15,7 +15,7 @@ class IColumn; class SerializationCustomSimpleText : public SerializationWrapper { public: - SerializationCustomSimpleText(const SerializationPtr & nested_); + explicit SerializationCustomSimpleText(const SerializationPtr & nested_); // Methods that subclasses must override in order to get full serialization/deserialization support. virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override = 0; diff --git a/src/DataTypes/Serializations/SerializationDateTime.cpp b/src/DataTypes/Serializations/SerializationDateTime.cpp index b4269fb0f8c..fd56c1baebd 100644 --- a/src/DataTypes/Serializations/SerializationDateTime.cpp +++ b/src/DataTypes/Serializations/SerializationDateTime.cpp @@ -27,6 +27,9 @@ inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & setti case FormatSettings::DateTimeInputFormat::BestEffort: parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone); return; + case FormatSettings::DateTimeInputFormat::BestEffortUS: + parseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone); + return; } } diff --git a/src/DataTypes/Serializations/SerializationDateTime.h b/src/DataTypes/Serializations/SerializationDateTime.h index 75334592422..f4a142483e5 100644 --- a/src/DataTypes/Serializations/SerializationDateTime.h +++ b/src/DataTypes/Serializations/SerializationDateTime.h @@ -11,7 +11,7 @@ namespace DB class SerializationDateTime final : public SerializationNumber, public TimezoneMixin { public: - SerializationDateTime(const TimezoneMixin & time_zone_); + explicit SerializationDateTime(const TimezoneMixin & time_zone_); void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/Serializations/SerializationDateTime64.cpp b/src/DataTypes/Serializations/SerializationDateTime64.cpp index b9ed5bd4a02..78c7ea56529 100644 --- a/src/DataTypes/Serializations/SerializationDateTime64.cpp +++ b/src/DataTypes/Serializations/SerializationDateTime64.cpp @@ -69,6 +69,9 @@ static inline void readText(DateTime64 & x, UInt32 scale, ReadBuffer & istr, con case FormatSettings::DateTimeInputFormat::BestEffort: parseDateTime64BestEffort(x, scale, istr, time_zone, utc_time_zone); return; + case FormatSettings::DateTimeInputFormat::BestEffortUS: + parseDateTime64BestEffortUS(x, scale, istr, time_zone, utc_time_zone); + return; } } diff --git a/src/DataTypes/Serializations/SerializationEnum.h b/src/DataTypes/Serializations/SerializationEnum.h index dfa9e74c7a1..bdd769b59c5 100644 --- a/src/DataTypes/Serializations/SerializationEnum.h +++ b/src/DataTypes/Serializations/SerializationEnum.h @@ -14,7 +14,7 @@ public: using typename SerializationNumber::ColumnType; using typename EnumValues::Values; - SerializationEnum(const Values & values_) : EnumValues(values_) {} + explicit SerializationEnum(const Values & values_) : EnumValues(values_) {} void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; diff --git a/src/DataTypes/Serializations/SerializationFixedString.h b/src/DataTypes/Serializations/SerializationFixedString.h index 82559d10800..c3c08b20419 100644 --- a/src/DataTypes/Serializations/SerializationFixedString.h +++ b/src/DataTypes/Serializations/SerializationFixedString.h @@ -12,7 +12,7 @@ private: size_t n; public: - SerializationFixedString(size_t n_) : n(n_) {} + explicit SerializationFixedString(size_t n_) : n(n_) {} size_t getN() const { return n; } void serializeBinary(const Field & field, WriteBuffer & ostr) const override; diff --git a/src/DataTypes/Serializations/SerializationIP.h b/src/DataTypes/Serializations/SerializationIP.h index a7bf1aeb2c6..282105b6b1e 100644 --- a/src/DataTypes/Serializations/SerializationIP.h +++ b/src/DataTypes/Serializations/SerializationIP.h @@ -8,7 +8,7 @@ namespace DB class SerializationIPv4 final : public SerializationCustomSimpleText { public: - SerializationIPv4(const SerializationPtr & nested_); + explicit SerializationIPv4(const SerializationPtr & nested_); void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override; @@ -17,7 +17,7 @@ public: class SerializationIPv6 : public SerializationCustomSimpleText { public: - SerializationIPv6(const SerializationPtr & nested_); + explicit SerializationIPv6(const SerializationPtr & nested_); void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override; diff --git a/src/DataTypes/Serializations/SerializationInfo.cpp b/src/DataTypes/Serializations/SerializationInfo.cpp index 22df95fc8f7..a0dc20b6479 100644 --- a/src/DataTypes/Serializations/SerializationInfo.cpp +++ b/src/DataTypes/Serializations/SerializationInfo.cpp @@ -181,10 +181,10 @@ void SerializationInfoByName::writeJSON(WriteBuffer & out) const { auto info_json = info->toJSON(); info_json.set(KEY_NAME, name); - column_infos.add(std::move(info_json)); + column_infos.add(std::move(info_json)); /// NOLINT } - object.set(KEY_COLUMNS, std::move(column_infos)); + object.set(KEY_COLUMNS, std::move(column_infos)); /// NOLINT std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); diff --git a/src/DataTypes/Serializations/SerializationInfoTuple.cpp b/src/DataTypes/Serializations/SerializationInfoTuple.cpp index 378bed2af53..803302f9642 100644 --- a/src/DataTypes/Serializations/SerializationInfoTuple.cpp +++ b/src/DataTypes/Serializations/SerializationInfoTuple.cpp @@ -89,7 +89,7 @@ Poco::JSON::Object SerializationInfoTuple::toJSON() const for (const auto & elem : elems) subcolumns.add(elem->toJSON()); - object.set("subcolumns", std::move(subcolumns)); + object.set("subcolumns", subcolumns); return object; } diff --git a/src/DataTypes/Serializations/SerializationLowCardinality.h b/src/DataTypes/Serializations/SerializationLowCardinality.h index 5f8a2a95a25..0a3597e86c7 100644 --- a/src/DataTypes/Serializations/SerializationLowCardinality.h +++ b/src/DataTypes/Serializations/SerializationLowCardinality.h @@ -15,7 +15,7 @@ private: SerializationPtr dict_inner_serialization; public: - SerializationLowCardinality(const DataTypePtr & dictionary_type); + explicit SerializationLowCardinality(const DataTypePtr & dictionary_type); void enumerateStreams( SubstreamPath & path, diff --git a/src/DataTypes/Serializations/SerializationNothing.h b/src/DataTypes/Serializations/SerializationNothing.h index 4a062931ac2..2de93a29763 100644 --- a/src/DataTypes/Serializations/SerializationNothing.h +++ b/src/DataTypes/Serializations/SerializationNothing.h @@ -14,7 +14,7 @@ namespace ErrorCodes class SerializationNothing : public SimpleTextSerialization { private: - [[noreturn]] void throwNoSerialization() const + [[noreturn]] static void throwNoSerialization() { throw Exception("Serialization is not implemented", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/DataTypes/Serializations/SerializationNullable.h b/src/DataTypes/Serializations/SerializationNullable.h index eb3e9bfb430..c22f2f57786 100644 --- a/src/DataTypes/Serializations/SerializationNullable.h +++ b/src/DataTypes/Serializations/SerializationNullable.h @@ -11,7 +11,7 @@ private: SerializationPtr nested; public: - SerializationNullable(const SerializationPtr & nested_) : nested(nested_) {} + explicit SerializationNullable(const SerializationPtr & nested_) : nested(nested_) {} void enumerateStreams( SubstreamPath & path, @@ -96,7 +96,7 @@ private: { const ColumnPtr null_map; - SubcolumnCreator(const ColumnPtr & null_map_) : null_map(null_map_) {} + explicit SubcolumnCreator(const ColumnPtr & null_map_) : null_map(null_map_) {} DataTypePtr create(const DataTypePtr & prev) const override; SerializationPtr create(const SerializationPtr & prev) const override; diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp new file mode 100644 index 00000000000..60b196092ed --- /dev/null +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -0,0 +1,465 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; + extern const int LOGICAL_ERROR; +} + +namespace +{ + +/// Visitor that keeps @num_dimensions_to_keep dimensions in arrays +/// and replaces all scalars or nested arrays to @replacement at that level. +class FieldVisitorReplaceScalars : public StaticVisitor +{ +public: + FieldVisitorReplaceScalars(const Field & replacement_, size_t num_dimensions_to_keep_) + : replacement(replacement_), num_dimensions_to_keep(num_dimensions_to_keep_) + { + } + + template + Field operator()(const T & x) const + { + if constexpr (std::is_same_v) + { + if (num_dimensions_to_keep == 0) + return replacement; + + const size_t size = x.size(); + Array res(size); + for (size_t i = 0; i < size; ++i) + res[i] = applyVisitor(FieldVisitorReplaceScalars(replacement, num_dimensions_to_keep - 1), x[i]); + return res; + } + else + return replacement; + } + +private: + const Field & replacement; + size_t num_dimensions_to_keep; +}; + +using Node = typename ColumnObject::SubcolumnsTree::Node; + +/// Finds a subcolumn from the same Nested type as @entry and inserts +/// an array with default values with consistent sizes as in Nested type. +bool tryInsertDefaultFromNested( + const std::shared_ptr & entry, const ColumnObject::SubcolumnsTree & subcolumns) +{ + if (!entry->path.hasNested()) + return false; + + const Node * current_node = subcolumns.findLeaf(entry->path); + const Node * leaf = nullptr; + size_t num_skipped_nested = 0; + + while (current_node) + { + /// Try to find the first Nested up to the current node. + const auto * node_nested = subcolumns.findParent(current_node, + [](const auto & candidate) { return candidate.isNested(); }); + + if (!node_nested) + break; + + /// If there are no leaves, skip current node and find + /// the next node up to the current. + leaf = subcolumns.findLeaf(node_nested, + [&](const auto & candidate) + { + return candidate.data.size() == entry->data.size() + 1; + }); + + if (leaf) + break; + + current_node = node_nested->parent; + ++num_skipped_nested; + } + + if (!leaf) + return false; + + auto last_field = leaf->data.getLastField(); + if (last_field.isNull()) + return false; + + const auto & least_common_type = entry->data.getLeastCommonType(); + size_t num_dimensions = getNumberOfDimensions(*least_common_type); + assert(num_skipped_nested < num_dimensions); + + /// Replace scalars to default values with consistent array sizes. + size_t num_dimensions_to_keep = num_dimensions - num_skipped_nested; + auto default_scalar = num_skipped_nested + ? createEmptyArrayField(num_skipped_nested) + : getBaseTypeOfArray(least_common_type)->getDefault(); + + auto default_field = applyVisitor(FieldVisitorReplaceScalars(default_scalar, num_dimensions_to_keep), last_field); + entry->data.insert(std::move(default_field)); + + return true; +} + +} + +template +template +void SerializationObject::deserializeTextImpl(IColumn & column, Reader && reader) const +{ + auto & column_object = assert_cast(column); + + String buf; + reader(buf); + std::optional result; + + { + auto parser = parsers_pool.get([] { return new Parser; }); + result = parser->parse(buf.data(), buf.size()); + } + + if (!result) + throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot parse object"); + + auto & [paths, values] = *result; + assert(paths.size() == values.size()); + + HashSet paths_set; + size_t column_size = column_object.size(); + + for (size_t i = 0; i < paths.size(); ++i) + { + auto field_info = getFieldInfo(values[i]); + if (isNothing(field_info.scalar_type)) + continue; + + if (!paths_set.insert(paths[i].getPath()).second) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Object has ambiguous path: {}", paths[i].getPath()); + + if (!column_object.hasSubcolumn(paths[i])) + { + if (paths[i].hasNested()) + column_object.addNestedSubcolumn(paths[i], field_info, column_size); + else + column_object.addSubcolumn(paths[i], column_size); + } + + auto & subcolumn = column_object.getSubcolumn(paths[i]); + assert(subcolumn.size() == column_size); + + subcolumn.insert(std::move(values[i]), std::move(field_info)); + } + + /// Insert default values to missed subcolumns. + const auto & subcolumns = column_object.getSubcolumns(); + for (const auto & entry : subcolumns) + { + if (!paths_set.has(entry->path.getPath())) + { + bool inserted = tryInsertDefaultFromNested(entry, subcolumns); + if (!inserted) + entry->data.insertDefault(); + } + } + + column_object.incrementNumRows(); +} + +template +void SerializationObject::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +{ + deserializeTextImpl(column, [&](String & s) { readStringInto(s, istr); }); +} + +template +void SerializationObject::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +{ + deserializeTextImpl(column, [&](String & s) { readEscapedStringInto(s, istr); }); +} + +template +void SerializationObject::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +{ + deserializeTextImpl(column, [&](String & s) { readQuotedStringInto(s, istr); }); +} + +template +void SerializationObject::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +{ + deserializeTextImpl(column, [&](String & s) { Parser::readJSON(s, istr); }); +} + +template +void SerializationObject::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + deserializeTextImpl(column, [&](String & s) { readCSVStringInto(s, istr, settings.csv); }); +} + +template +template +void SerializationObject::checkSerializationIsSupported(const TSettings & settings, const TStatePtr & state) const +{ + if (settings.position_independent_encoding) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "DataTypeObject doesn't support serialization with position independent encoding"); + + if (state) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "DataTypeObject doesn't support serialization with non-trivial state"); +} + +template +void SerializationObject::serializeBinaryBulkStatePrefix( + SerializeBinaryBulkSettings & settings, + SerializeBinaryBulkStatePtr & state) const +{ + checkSerializationIsSupported(settings, state); +} + +template +void SerializationObject::serializeBinaryBulkStateSuffix( + SerializeBinaryBulkSettings & settings, + SerializeBinaryBulkStatePtr & state) const +{ + checkSerializationIsSupported(settings, state); +} + +template +void SerializationObject::deserializeBinaryBulkStatePrefix( + DeserializeBinaryBulkSettings & settings, + DeserializeBinaryBulkStatePtr & state) const +{ + checkSerializationIsSupported(settings, state); +} + +template +void SerializationObject::serializeBinaryBulkWithMultipleStreams( + const IColumn & column, + size_t offset, + size_t limit, + SerializeBinaryBulkSettings & settings, + SerializeBinaryBulkStatePtr & state) const +{ + checkSerializationIsSupported(settings, state); + const auto & column_object = assert_cast(column); + + if (!column_object.isFinalized()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot write non-finalized ColumnObject"); + + settings.path.push_back(Substream::ObjectStructure); + if (auto * stream = settings.getter(settings.path)) + writeVarUInt(column_object.getSubcolumns().size(), *stream); + + const auto & subcolumns = column_object.getSubcolumns(); + for (const auto & entry : subcolumns) + { + settings.path.back() = Substream::ObjectStructure; + settings.path.back().object_key_name = entry->path.getPath(); + + const auto & type = entry->data.getLeastCommonType(); + if (auto * stream = settings.getter(settings.path)) + { + entry->path.writeBinary(*stream); + writeStringBinary(type->getName(), *stream); + } + + settings.path.back() = Substream::ObjectElement; + if (auto * stream = settings.getter(settings.path)) + { + auto serialization = type->getDefaultSerialization(); + serialization->serializeBinaryBulkWithMultipleStreams( + entry->data.getFinalizedColumn(), offset, limit, settings, state); + } + } + + settings.path.pop_back(); +} + +template +void SerializationObject::deserializeBinaryBulkWithMultipleStreams( + ColumnPtr & column, + size_t limit, + DeserializeBinaryBulkSettings & settings, + DeserializeBinaryBulkStatePtr & state, + SubstreamsCache * cache) const +{ + checkSerializationIsSupported(settings, state); + if (!column->empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "DataTypeObject cannot be deserialized to non-empty column"); + + auto mutable_column = column->assumeMutable(); + auto & column_object = typeid_cast(*mutable_column); + + size_t num_subcolumns = 0; + settings.path.push_back(Substream::ObjectStructure); + if (auto * stream = settings.getter(settings.path)) + readVarUInt(num_subcolumns, *stream); + + settings.path.back() = Substream::ObjectElement; + for (size_t i = 0; i < num_subcolumns; ++i) + { + PathInData key; + String type_name; + + settings.path.back() = Substream::ObjectStructure; + if (auto * stream = settings.getter(settings.path)) + { + key.readBinary(*stream); + readStringBinary(type_name, *stream); + } + else + { + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read structure of DataTypeObject, because its stream is missing"); + } + + settings.path.back() = Substream::ObjectElement; + settings.path.back().object_key_name = key.getPath(); + + if (auto * stream = settings.getter(settings.path)) + { + auto type = DataTypeFactory::instance().get(type_name); + auto serialization = type->getDefaultSerialization(); + ColumnPtr subcolumn_data = type->createColumn(); + serialization->deserializeBinaryBulkWithMultipleStreams(subcolumn_data, limit, settings, state, cache); + column_object.addSubcolumn(key, subcolumn_data->assumeMutable()); + } + else + { + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read subcolumn '{}' of DataTypeObject, because its stream is missing", key.getPath()); + } + } + + settings.path.pop_back(); + column_object.checkConsistency(); + column_object.finalize(); + column = std::move(mutable_column); +} + +template +void SerializationObject::serializeBinary(const Field &, WriteBuffer &) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented for SerializationObject"); +} + +template +void SerializationObject::deserializeBinary(Field &, ReadBuffer &) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented for SerializationObject"); +} + +template +void SerializationObject::serializeBinary(const IColumn &, size_t, WriteBuffer &) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented for SerializationObject"); +} + +template +void SerializationObject::deserializeBinary(IColumn &, ReadBuffer &) const +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented for SerializationObject"); +} + +/// TODO: use format different of JSON in serializations. + +template +void SerializationObject::serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + const auto & column_object = assert_cast(column); + const auto & subcolumns = column_object.getSubcolumns(); + + writeChar('{', ostr); + for (auto it = subcolumns.begin(); it != subcolumns.end(); ++it) + { + if (it != subcolumns.begin()) + writeCString(",", ostr); + + writeDoubleQuoted((*it)->path.getPath(), ostr); + writeChar(':', ostr); + + auto serialization = (*it)->data.getLeastCommonType()->getDefaultSerialization(); + serialization->serializeTextJSON((*it)->data.getFinalizedColumn(), row_num, ostr, settings); + } + writeChar('}', ostr); +} + +template +void SerializationObject::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + serializeTextImpl(column, row_num, ostr, settings); +} + +template +void SerializationObject::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + WriteBufferFromOwnString ostr_str; + serializeTextImpl(column, row_num, ostr_str, settings); + writeEscapedString(ostr_str.str(), ostr); +} + +template +void SerializationObject::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + WriteBufferFromOwnString ostr_str; + serializeTextImpl(column, row_num, ostr_str, settings); + writeQuotedString(ostr_str.str(), ostr); +} + +template +void SerializationObject::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + serializeTextImpl(column, row_num, ostr, settings); +} + +template +void SerializationObject::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + WriteBufferFromOwnString ostr_str; + serializeTextImpl(column, row_num, ostr_str, settings); + writeCSVString(ostr_str.str(), ostr); +} + +SerializationPtr getObjectSerialization(const String & schema_format) +{ + if (schema_format == "json") + { +#if USE_SIMDJSON + return std::make_shared>>(); +#elif USE_RAPIDJSON + return std::make_shared>>(); +#else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "To use data type Object with JSON format ClickHouse should be built with Simdjson or Rapidjson"); +#endif + } + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown schema format '{}'", schema_format); +} + +} diff --git a/src/DataTypes/Serializations/SerializationObject.h b/src/DataTypes/Serializations/SerializationObject.h new file mode 100644 index 00000000000..549c8735aee --- /dev/null +++ b/src/DataTypes/Serializations/SerializationObject.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include + +namespace DB +{ + +/// Serialization for data type Object. +/// Supported only test serialization/deserialization. +/// and binary bulk serialization/deserialization without position independent +/// encoding, i.e. serialization/deserialization into Native format. +template +class SerializationObject : public ISerialization +{ +public: + void serializeBinaryBulkStatePrefix( + SerializeBinaryBulkSettings & settings, + SerializeBinaryBulkStatePtr & state) const override; + + void serializeBinaryBulkStateSuffix( + SerializeBinaryBulkSettings & settings, + SerializeBinaryBulkStatePtr & state) const override; + + void deserializeBinaryBulkStatePrefix( + DeserializeBinaryBulkSettings & settings, + DeserializeBinaryBulkStatePtr & state) const override; + + void serializeBinaryBulkWithMultipleStreams( + const IColumn & column, + size_t offset, + size_t limit, + SerializeBinaryBulkSettings & settings, + SerializeBinaryBulkStatePtr & state) const override; + + void deserializeBinaryBulkWithMultipleStreams( + ColumnPtr & column, + size_t limit, + DeserializeBinaryBulkSettings & settings, + DeserializeBinaryBulkStatePtr & state, + SubstreamsCache * cache) const override; + + void serializeBinary(const Field & field, WriteBuffer & ostr) const override; + void deserializeBinary(Field & field, ReadBuffer & istr) const override; + void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void deserializeBinary(IColumn & column, ReadBuffer & istr) const override; + + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + + void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + +private: + template + void checkSerializationIsSupported(const TSettings & settings, const TStatePtr & state) const; + + template + void deserializeTextImpl(IColumn & column, Reader && reader) const; + + void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const; + + /// Pool of parser objects to make SerializationObject thread safe. + mutable SimpleObjectPool parsers_pool; +}; + +SerializationPtr getObjectSerialization(const String & schema_format); + +} diff --git a/src/DataTypes/Serializations/SerializationSparse.h b/src/DataTypes/Serializations/SerializationSparse.h index 51d9df2cb5d..54ab4853360 100644 --- a/src/DataTypes/Serializations/SerializationSparse.h +++ b/src/DataTypes/Serializations/SerializationSparse.h @@ -23,7 +23,7 @@ namespace DB class SerializationSparse final : public ISerialization { public: - SerializationSparse(const SerializationPtr & nested_); + explicit SerializationSparse(const SerializationPtr & nested_); Kind getKind() const override { return Kind::SPARSE; } diff --git a/src/DataTypes/Serializations/SerializationWrapper.h b/src/DataTypes/Serializations/SerializationWrapper.h index 4cdcffc21a8..43fc7e9914a 100644 --- a/src/DataTypes/Serializations/SerializationWrapper.h +++ b/src/DataTypes/Serializations/SerializationWrapper.h @@ -14,7 +14,7 @@ protected: SerializationPtr nested_serialization; public: - SerializationWrapper(const SerializationPtr & nested_serialization_) : nested_serialization(nested_serialization_) {} + explicit SerializationWrapper(const SerializationPtr & nested_serialization_) : nested_serialization(nested_serialization_) {} const SerializationPtr & getNested() const { return nested_serialization; } diff --git a/src/DataTypes/Serializations/SubcolumnsTree.h b/src/DataTypes/Serializations/SubcolumnsTree.h new file mode 100644 index 00000000000..f66f557bc8f --- /dev/null +++ b/src/DataTypes/Serializations/SubcolumnsTree.h @@ -0,0 +1,211 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +/// Tree that represents paths in document +/// with additional data in nodes. +template +class SubcolumnsTree +{ +public: + struct Node + { + enum Kind + { + TUPLE, + NESTED, + SCALAR, + }; + + explicit Node(Kind kind_) : kind(kind_) {} + Node(Kind kind_, const NodeData & data_) : kind(kind_), data(data_) {} + Node(Kind kind_, const NodeData & data_, const PathInData & path_) + : kind(kind_), data(data_), path(path_) {} + + Kind kind = TUPLE; + const Node * parent = nullptr; + + Arena strings_pool; + HashMapWithStackMemory, StringRefHash, 4> children; + + NodeData data; + PathInData path; + + bool isNested() const { return kind == NESTED; } + bool isScalar() const { return kind == SCALAR; } + + void addChild(std::string_view key, std::shared_ptr next_node) + { + next_node->parent = this; + StringRef key_ref{strings_pool.insert(key.data(), key.length()), key.length()}; + children[key_ref] = std::move(next_node); + } + }; + + using NodeKind = typename Node::Kind; + using NodePtr = std::shared_ptr; + + /// Add a leaf without any data in other nodes. + bool add(const PathInData & path, const NodeData & leaf_data) + { + return add(path, [&](NodeKind kind, bool exists) -> NodePtr + { + if (exists) + return nullptr; + + if (kind == Node::SCALAR) + return std::make_shared(kind, leaf_data, path); + + return std::make_shared(kind); + }); + } + + /// Callback for creation of node. Receives kind of node and + /// flag, which is true if node already exists. + using NodeCreator = std::function; + + bool add(const PathInData & path, const NodeCreator & node_creator) + { + const auto & parts = path.getParts(); + + if (parts.empty()) + return false; + + if (!root) + root = std::make_shared(Node::TUPLE); + + Node * current_node = root.get(); + for (size_t i = 0; i < parts.size() - 1; ++i) + { + assert(current_node->kind != Node::SCALAR); + + auto it = current_node->children.find(StringRef{parts[i].key}); + if (it != current_node->children.end()) + { + current_node = it->getMapped().get(); + node_creator(current_node->kind, true); + + if (current_node->isNested() != parts[i].is_nested) + return false; + } + else + { + auto next_kind = parts[i].is_nested ? Node::NESTED : Node::TUPLE; + auto next_node = node_creator(next_kind, false); + current_node->addChild(String(parts[i].key), next_node); + current_node = next_node.get(); + } + } + + auto it = current_node->children.find(StringRef{parts.back().key}); + if (it != current_node->children.end()) + return false; + + auto next_node = node_creator(Node::SCALAR, false); + current_node->addChild(String(parts.back().key), next_node); + leaves.push_back(std::move(next_node)); + + return true; + } + + /// Find node that matches the path the best. + const Node * findBestMatch(const PathInData & path) const + { + return findImpl(path, false); + } + + /// Find node that matches the path exactly. + const Node * findExact(const PathInData & path) const + { + return findImpl(path, true); + } + + /// Find leaf by path. + const Node * findLeaf(const PathInData & path) const + { + const auto * candidate = findExact(path); + if (!candidate || !candidate->isScalar()) + return nullptr; + return candidate; + } + + using NodePredicate = std::function; + + /// Finds leaf that satisfies the predicate. + const Node * findLeaf(const NodePredicate & predicate) + { + return findLeaf(root.get(), predicate); + } + + static const Node * findLeaf(const Node * node, const NodePredicate & predicate) + { + if (!node) + return nullptr; + + if (node->isScalar()) + return predicate(*node) ? node : nullptr; + + for (const auto & [_, child] : node->children) + if (const auto * leaf = findLeaf(child.get(), predicate)) + return leaf; + + return nullptr; + } + + /// Find first parent node that satisfies the predicate. + static const Node * findParent(const Node * node, const NodePredicate & predicate) + { + while (node && !predicate(*node)) + node = node->parent; + return node; + } + + bool empty() const { return root == nullptr; } + size_t size() const { return leaves.size(); } + + using Nodes = std::vector; + + const Nodes & getLeaves() const { return leaves; } + const Node * getRoot() const { return root.get(); } + + using iterator = typename Nodes::iterator; + using const_iterator = typename Nodes::const_iterator; + + iterator begin() { return leaves.begin(); } + iterator end() { return leaves.end(); } + + const_iterator begin() const { return leaves.begin(); } + const_iterator end() const { return leaves.end(); } + +private: + const Node * findImpl(const PathInData & path, bool find_exact) const + { + if (!root) + return nullptr; + + const auto & parts = path.getParts(); + const Node * current_node = root.get(); + + for (const auto & part : parts) + { + auto it = current_node->children.find(StringRef{part.key}); + if (it == current_node->children.end()) + return find_exact ? nullptr : current_node; + + current_node = it->getMapped().get(); + } + + return current_node; + } + + NodePtr root; + Nodes leaves; +}; + +} diff --git a/tests/integration/test_query_deduplication/__init__.py b/src/DataTypes/Serializations/tests/CMakeLists.txt similarity index 100% rename from tests/integration/test_query_deduplication/__init__.py rename to src/DataTypes/Serializations/tests/CMakeLists.txt diff --git a/src/DataTypes/Serializations/tests/gtest_json_parser.cpp b/src/DataTypes/Serializations/tests/gtest_json_parser.cpp new file mode 100644 index 00000000000..4dddb3cd03d --- /dev/null +++ b/src/DataTypes/Serializations/tests/gtest_json_parser.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include + +#include +#include + +#if USE_SIMDJSON + +using namespace DB; + +const String json1 = R"({"k1" : 1, "k2" : {"k3" : "aa", "k4" : 2}})"; + +/// Nested(k2 String, k3 Nested(k4 String)) +const String json2 = +R"({"k1" : [ + { + "k2" : "aaa", + "k3" : [{ "k4" : "bbb" }, { "k4" : "ccc" }] + }, + { + "k2" : "ddd", + "k3" : [{ "k4" : "eee" }, { "k4" : "fff" }] + } + ] +})"; + +TEST(JSONDataParser, ReadJSON) +{ + { + String json_bad = json1 + "aaaaaaa"; + + JSONDataParser parser; + ReadBufferFromString buf(json_bad); + String res; + parser.readJSON(res, buf); + ASSERT_EQ(json1, res); + } + + { + String json_bad = json2 + "aaaaaaa"; + + JSONDataParser parser; + ReadBufferFromString buf(json_bad); + String res; + parser.readJSON(res, buf); + ASSERT_EQ(json2, res); + } +} + +struct JSONPathAndValue +{ + PathInData path; + Field value; + + JSONPathAndValue(const PathInData & path_, const Field & value_) + : path(path_), value(value_) + { + } + + bool operator==(const JSONPathAndValue & other) const = default; + bool operator<(const JSONPathAndValue & other) const { return path.getPath() < other.path.getPath(); } +}; + +static std::ostream & operator<<(std::ostream & ostr, const JSONPathAndValue & path_and_value) +{ + ostr << "{ PathInData{"; + bool first = true; + for (const auto & part : path_and_value.path.getParts()) + { + ostr << (first ? "{" : ", {") << part.key << ", " << part.is_nested << ", " << part.anonymous_array_level << "}"; + first = false; + } + + ostr << "}, Field{" << applyVisitor(FieldVisitorToString(), path_and_value.value) << "} }"; + return ostr; +} + +using JSONValues = std::vector; + +static void check( + const String & json_str, + const String & tag, + JSONValues expected_values) +{ + JSONDataParser parser; + auto res = parser.parse(json_str.data(), json_str.size()); + ASSERT_TRUE(res.has_value()) << tag; + + const auto & [paths, values] = *res; + + ASSERT_EQ(paths.size(), expected_values.size()) << tag; + ASSERT_EQ(values.size(), expected_values.size()) << tag; + + JSONValues result_values; + for (size_t i = 0; i < paths.size(); ++i) + result_values.emplace_back(paths[i], values[i]); + + std::sort(expected_values.begin(), expected_values.end()); + std::sort(result_values.begin(), result_values.end()); + + ASSERT_EQ(result_values, expected_values) << tag; +} + +TEST(JSONDataParser, Parse) +{ + { + check(json1, "json1", + { + { PathInData{{{"k1", false, 0}}}, 1 }, + { PathInData{{{"k2", false, 0}, {"k3", false, 0}}}, "aa" }, + { PathInData{{{"k2", false, 0}, {"k4", false, 0}}}, 2 }, + }); + } + + { + check(json2, "json2", + { + { PathInData{{{"k1", true, 0}, {"k2", false, 0}}}, Array{"aaa", "ddd"} }, + { PathInData{{{"k1", true, 0}, {"k3", true, 0}, {"k4", false, 0}}}, Array{Array{"bbb", "ccc"}, Array{"eee", "fff"}} }, + }); + } + + { + /// Nested(k2 Tuple(k3 Array(Int), k4 Array(Int)), k5 String) + const String json3 = + R"({"k1": [ + { + "k2": { + "k3": [1, 2], + "k4": [3, 4] + }, + "k5": "foo" + }, + { + "k2": { + "k3": [5, 6], + "k4": [7, 8] + }, + "k5": "bar" + } + ]})"; + + check(json3, "json3", + { + { PathInData{{{"k1", true, 0}, {"k5", false, 0}}}, Array{"foo", "bar"} }, + { PathInData{{{"k1", true, 0}, {"k2", false, 0}, {"k3", false, 0}}}, Array{Array{1, 2}, Array{5, 6}} }, + { PathInData{{{"k1", true, 0}, {"k2", false, 0}, {"k4", false, 0}}}, Array{Array{3, 4}, Array{7, 8}} }, + }); + } + + { + /// Nested(k2 Nested(k3 Int, k4 Int), k5 String) + const String json4 = + R"({"k1": [ + { + "k2": [{"k3": 1, "k4": 3}, {"k3": 2, "k4": 4}], + "k5": "foo" + }, + { + "k2": [{"k3": 5, "k4": 7}, {"k3": 6, "k4": 8}], + "k5": "bar" + } + ]})"; + + check(json4, "json4", + { + { PathInData{{{"k1", true, 0}, {"k5", false, 0}}}, Array{"foo", "bar"} }, + { PathInData{{{"k1", true, 0}, {"k2", true, 0}, {"k3", false, 0}}}, Array{Array{1, 2}, Array{5, 6}} }, + { PathInData{{{"k1", true, 0}, {"k2", true, 0}, {"k4", false, 0}}}, Array{Array{3, 4}, Array{7, 8}} }, + }); + } + + { + const String json5 = R"({"k1": [[1, 2, 3], [4, 5], [6]]})"; + check(json5, "json5", + { + { PathInData{{{"k1", false, 0}}}, Array{Array{1, 2, 3}, Array{4, 5}, Array{6}} } + }); + } + + { + /// Array(Nested(k2 Int, k3 Int)) + const String json6 = R"({ + "k1": [ + [{"k2": 1, "k3": 2}, {"k2": 3, "k3": 4}], + [{"k2": 5, "k3": 6}] + ] + })"; + + check(json6, "json6", + { + { PathInData{{{"k1", true, 0}, {"k2", false, 1}}}, Array{Array{1, 3}, Array{5}} }, + { PathInData{{{"k1", true, 0}, {"k3", false, 1}}}, Array{Array{2, 4}, Array{6}} }, + }); + } + + { + /// Nested(k2 Array(Int), k3 Array(Int)) + const String json7 = R"({ + "k1": [ + {"k2": [1, 3], "k3": [2, 4]}, + {"k2": [5], "k3": [6]} + ] + })"; + + check(json7, "json7", + { + { PathInData{{{"k1", true, 0}, {"k2", false, 0}}}, Array{Array{1, 3}, Array{5}} }, + { PathInData{{{"k1", true, 0}, {"k3", false, 0}}}, Array{Array{2, 4}, Array{6}} }, + }); + } +} + +#endif diff --git a/src/DataTypes/convertMySQLDataType.cpp b/src/DataTypes/convertMySQLDataType.cpp index ee897de9597..7e2f2e7c6b9 100644 --- a/src/DataTypes/convertMySQLDataType.cpp +++ b/src/DataTypes/convertMySQLDataType.cpp @@ -7,6 +7,7 @@ #include #include #include "DataTypeDate.h" +#include "DataTypeDate32.h" #include "DataTypeDateTime.h" #include "DataTypeDateTime64.h" #include "DataTypeEnum.h" @@ -73,7 +74,14 @@ DataTypePtr convertMySQLDataType(MultiEnum type_support, else if (type_name == "double") res = std::make_shared(); else if (type_name == "date") - res = std::make_shared(); + { + if (type_support.isSet(MySQLDataTypesSupport::DATE2DATE32)) + res = std::make_shared(); + else if (type_support.isSet(MySQLDataTypesSupport::DATE2STRING)) + res = std::make_shared(); + else + res = std::make_shared(); + } else if (type_name == "binary") res = std::make_shared(length); else if (type_name == "datetime" || type_name == "timestamp") diff --git a/src/DataTypes/getLeastSupertype.cpp b/src/DataTypes/getLeastSupertype.cpp index 22f6a077504..3fcb3fef25b 100644 --- a/src/DataTypes/getLeastSupertype.cpp +++ b/src/DataTypes/getLeastSupertype.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include namespace DB @@ -30,28 +32,181 @@ namespace ErrorCodes namespace { - String getExceptionMessagePrefix(const DataTypes & types) + +String typeToString(const DataTypePtr & type) { return type->getName(); } +String typeToString(const TypeIndex & type) { return String(magic_enum::enum_name(type)); } + +template +String getExceptionMessagePrefix(const DataTypes & types) +{ + WriteBufferFromOwnString res; + res << "There is no supertype for types "; + + bool first = true; + for (const auto & type : types) { - WriteBufferFromOwnString res; - res << "There is no supertype for types "; + if (!first) + res << ", "; + first = false; - bool first = true; - for (const auto & type : types) - { - if (!first) - res << ", "; - first = false; - - res << type->getName(); - } - - return res.str(); + res << typeToString(type); } + + return res.str(); } - -DataTypePtr getLeastSupertype(const DataTypes & types) +DataTypePtr getNumericType(const TypeIndexSet & types, bool allow_conversion_to_string) { + auto throw_or_return = [&](std::string_view message, int error_code) + { + if (allow_conversion_to_string) + return std::make_shared(); + + throw Exception(String(message), error_code); + }; + + bool all_numbers = true; + + size_t max_bits_of_signed_integer = 0; + size_t max_bits_of_unsigned_integer = 0; + size_t max_mantissa_bits_of_floating = 0; + + auto maximize = [](size_t & what, size_t value) + { + if (value > what) + what = value; + }; + + for (const auto & type : types) + { + if (type == TypeIndex::UInt8) + maximize(max_bits_of_unsigned_integer, 8); + else if (type == TypeIndex::UInt16) + maximize(max_bits_of_unsigned_integer, 16); + else if (type == TypeIndex::UInt32) + maximize(max_bits_of_unsigned_integer, 32); + else if (type == TypeIndex::UInt64) + maximize(max_bits_of_unsigned_integer, 64); + else if (type == TypeIndex::UInt128) + maximize(max_bits_of_unsigned_integer, 128); + else if (type == TypeIndex::UInt256) + maximize(max_bits_of_unsigned_integer, 256); + else if (type == TypeIndex::Int8 || type == TypeIndex::Enum8) + maximize(max_bits_of_signed_integer, 8); + else if (type == TypeIndex::Int16 || type == TypeIndex::Enum16) + maximize(max_bits_of_signed_integer, 16); + else if (type == TypeIndex::Int32) + maximize(max_bits_of_signed_integer, 32); + else if (type == TypeIndex::Int64) + maximize(max_bits_of_signed_integer, 64); + else if (type == TypeIndex::Int128) + maximize(max_bits_of_signed_integer, 128); + else if (type == TypeIndex::Int256) + maximize(max_bits_of_signed_integer, 256); + else if (type == TypeIndex::Float32) + maximize(max_mantissa_bits_of_floating, 24); + else if (type == TypeIndex::Float64) + maximize(max_mantissa_bits_of_floating, 53); + else + all_numbers = false; + } + + if (max_bits_of_signed_integer || max_bits_of_unsigned_integer || max_mantissa_bits_of_floating) + { + if (!all_numbers) + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are numbers and some of them are not", ErrorCodes::NO_COMMON_TYPE); + + /// If there are signed and unsigned types of same bit-width, the result must be signed number with at least one more bit. + /// Example, common of Int32, UInt32 = Int64. + + size_t min_bit_width_of_integer = std::max(max_bits_of_signed_integer, max_bits_of_unsigned_integer); + + /// If unsigned is not covered by signed. + if (max_bits_of_signed_integer && max_bits_of_unsigned_integer >= max_bits_of_signed_integer) //-V1051 + { + // Because 128 and 256 bit integers are significantly slower, we should not promote to them. + // But if we already have wide numbers, promotion is necessary. + if (min_bit_width_of_integer != 64) + ++min_bit_width_of_integer; + else + return throw_or_return( + getExceptionMessagePrefix(types) + + " because some of them are signed integers and some are unsigned integers," + " but there is no signed integer type, that can exactly represent all required unsigned integer values", + ErrorCodes::NO_COMMON_TYPE); + } + + /// If the result must be floating. + if (max_mantissa_bits_of_floating) + { + size_t min_mantissa_bits = std::max(min_bit_width_of_integer, max_mantissa_bits_of_floating); + if (min_mantissa_bits <= 24) + return std::make_shared(); + else if (min_mantissa_bits <= 53) + return std::make_shared(); + else + return throw_or_return(getExceptionMessagePrefix(types) + + " because some of them are integers and some are floating point," + " but there is no floating point type, that can exactly represent all required integers", ErrorCodes::NO_COMMON_TYPE); + } + + /// If the result must be signed integer. + if (max_bits_of_signed_integer) + { + if (min_bit_width_of_integer <= 8) + return std::make_shared(); + else if (min_bit_width_of_integer <= 16) + return std::make_shared(); + else if (min_bit_width_of_integer <= 32) + return std::make_shared(); + else if (min_bit_width_of_integer <= 64) + return std::make_shared(); + else if (min_bit_width_of_integer <= 128) + return std::make_shared(); + else if (min_bit_width_of_integer <= 256) + return std::make_shared(); + else + return throw_or_return(getExceptionMessagePrefix(types) + + " because some of them are signed integers and some are unsigned integers," + " but there is no signed integer type, that can exactly represent all required unsigned integer values", ErrorCodes::NO_COMMON_TYPE); + } + + /// All unsigned. + { + if (min_bit_width_of_integer <= 8) + return std::make_shared(); + else if (min_bit_width_of_integer <= 16) + return std::make_shared(); + else if (min_bit_width_of_integer <= 32) + return std::make_shared(); + else if (min_bit_width_of_integer <= 64) + return std::make_shared(); + else if (min_bit_width_of_integer <= 128) + return std::make_shared(); + else if (min_bit_width_of_integer <= 256) + return std::make_shared(); + else + return throw_or_return("Logical error: " + getExceptionMessagePrefix(types) + + " but as all data types are unsigned integers, we must have found maximum unsigned integer type", ErrorCodes::NO_COMMON_TYPE); + + } + } + + return {}; +} + +} + +DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_string) +{ + auto throw_or_return = [&](std::string_view message, int error_code) + { + if (allow_conversion_to_string) + return std::make_shared(); + + throw Exception(String(message), error_code); + }; + /// Trivial cases if (types.empty()) @@ -88,7 +243,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) non_nothing_types.emplace_back(type); if (non_nothing_types.size() < types.size()) - return getLeastSupertype(non_nothing_types); + return getLeastSupertype(non_nothing_types, allow_conversion_to_string); } /// For Arrays @@ -113,9 +268,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types) if (have_array) { if (!all_arrays) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are Array and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Array and some of them are not", ErrorCodes::NO_COMMON_TYPE); - return std::make_shared(getLeastSupertype(nested_types)); + return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); } } @@ -139,7 +294,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) nested_types[elem_idx].reserve(types.size()); } else if (tuple_size != type_tuple->getElements().size()) - throw Exception(getExceptionMessagePrefix(types) + " because Tuples have different sizes", ErrorCodes::NO_COMMON_TYPE); + return throw_or_return(getExceptionMessagePrefix(types) + " because Tuples have different sizes", ErrorCodes::NO_COMMON_TYPE); have_tuple = true; @@ -153,11 +308,11 @@ DataTypePtr getLeastSupertype(const DataTypes & types) if (have_tuple) { if (!all_tuples) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are Tuple and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Tuple and some of them are not", ErrorCodes::NO_COMMON_TYPE); DataTypes common_tuple_types(tuple_size); for (size_t elem_idx = 0; elem_idx < tuple_size; ++elem_idx) - common_tuple_types[elem_idx] = getLeastSupertype(nested_types[elem_idx]); + common_tuple_types[elem_idx] = getLeastSupertype(nested_types[elem_idx], allow_conversion_to_string); return std::make_shared(common_tuple_types); } @@ -187,9 +342,11 @@ DataTypePtr getLeastSupertype(const DataTypes & types) if (have_maps) { if (!all_maps) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are Maps and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are Maps and some of them are not", ErrorCodes::NO_COMMON_TYPE); - return std::make_shared(getLeastSupertype(key_types), getLeastSupertype(value_types)); + return std::make_shared( + getLeastSupertype(key_types, allow_conversion_to_string), + getLeastSupertype(value_types, allow_conversion_to_string)); } } @@ -220,9 +377,9 @@ DataTypePtr getLeastSupertype(const DataTypes & types) if (have_low_cardinality) { if (have_not_low_cardinality) - return getLeastSupertype(nested_types); + return getLeastSupertype(nested_types, allow_conversion_to_string); else - return std::make_shared(getLeastSupertype(nested_types)); + return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); } } @@ -248,13 +405,13 @@ DataTypePtr getLeastSupertype(const DataTypes & types) if (have_nullable) { - return std::make_shared(getLeastSupertype(nested_types)); + return std::make_shared(getLeastSupertype(nested_types, allow_conversion_to_string)); } } /// Non-recursive rules - std::unordered_set type_ids; + TypeIndexSet type_ids; for (const auto & type : types) type_ids.insert(type->getTypeId()); @@ -268,7 +425,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) { bool all_strings = type_ids.size() == (have_string + have_fixed_string); if (!all_strings) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are String/FixedString and some of them are not", ErrorCodes::NO_COMMON_TYPE); + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are String/FixedString and some of them are not", ErrorCodes::NO_COMMON_TYPE); return std::make_shared(); } @@ -285,7 +442,8 @@ DataTypePtr getLeastSupertype(const DataTypes & types) { bool all_date_or_datetime = type_ids.size() == (have_date + have_date32 + have_datetime + have_datetime64); if (!all_date_or_datetime) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are Date/Date32/DateTime/DateTime64 and some of them are not", + return throw_or_return(getExceptionMessagePrefix(types) + + " because some of them are Date/Date32/DateTime/DateTime64 and some of them are not", ErrorCodes::NO_COMMON_TYPE); if (have_datetime64 == 0 && have_date32 == 0) @@ -362,7 +520,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) } if (num_supported != type_ids.size()) - throw Exception(getExceptionMessagePrefix(types) + " because some of them have no lossless conversion to Decimal", + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them have no lossless conversion to Decimal", ErrorCodes::NO_COMMON_TYPE); UInt32 max_scale = 0; @@ -385,7 +543,7 @@ DataTypePtr getLeastSupertype(const DataTypes & types) } if (min_precision > DataTypeDecimal::maxPrecision()) - throw Exception(getExceptionMessagePrefix(types) + " because the least supertype is Decimal(" + return throw_or_return(getExceptionMessagePrefix(types) + " because the least supertype is Decimal(" + toString(min_precision) + ',' + toString(max_scale) + ')', ErrorCodes::NO_COMMON_TYPE); @@ -399,135 +557,56 @@ DataTypePtr getLeastSupertype(const DataTypes & types) /// For numeric types, the most complicated part. { - bool all_numbers = true; - - size_t max_bits_of_signed_integer = 0; - size_t max_bits_of_unsigned_integer = 0; - size_t max_mantissa_bits_of_floating = 0; - - auto maximize = [](size_t & what, size_t value) - { - if (value > what) - what = value; - }; - - for (const auto & type : types) - { - if (typeid_cast(type.get())) - maximize(max_bits_of_unsigned_integer, 8); - else if (typeid_cast(type.get())) - maximize(max_bits_of_unsigned_integer, 16); - else if (typeid_cast(type.get())) - maximize(max_bits_of_unsigned_integer, 32); - else if (typeid_cast(type.get())) - maximize(max_bits_of_unsigned_integer, 64); - else if (typeid_cast(type.get())) - maximize(max_bits_of_unsigned_integer, 128); - else if (typeid_cast(type.get())) - maximize(max_bits_of_unsigned_integer, 256); - else if (typeid_cast(type.get()) || typeid_cast(type.get())) - maximize(max_bits_of_signed_integer, 8); - else if (typeid_cast(type.get()) || typeid_cast(type.get())) - maximize(max_bits_of_signed_integer, 16); - else if (typeid_cast(type.get())) - maximize(max_bits_of_signed_integer, 32); - else if (typeid_cast(type.get())) - maximize(max_bits_of_signed_integer, 64); - else if (typeid_cast(type.get())) - maximize(max_bits_of_signed_integer, 128); - else if (typeid_cast(type.get())) - maximize(max_bits_of_signed_integer, 256); - else if (typeid_cast(type.get())) - maximize(max_mantissa_bits_of_floating, 24); - else if (typeid_cast(type.get())) - maximize(max_mantissa_bits_of_floating, 53); - else - all_numbers = false; - } - - if (max_bits_of_signed_integer || max_bits_of_unsigned_integer || max_mantissa_bits_of_floating) - { - if (!all_numbers) - throw Exception(getExceptionMessagePrefix(types) + " because some of them are numbers and some of them are not", ErrorCodes::NO_COMMON_TYPE); - - /// If there are signed and unsigned types of same bit-width, the result must be signed number with at least one more bit. - /// Example, common of Int32, UInt32 = Int64. - - size_t min_bit_width_of_integer = std::max(max_bits_of_signed_integer, max_bits_of_unsigned_integer); - - /// If unsigned is not covered by signed. - if (max_bits_of_signed_integer && max_bits_of_unsigned_integer >= max_bits_of_signed_integer) //-V1051 - { - // Because 128 and 256 bit integers are significantly slower, we should not promote to them. - // But if we already have wide numbers, promotion is necessary. - if (min_bit_width_of_integer != 64) - ++min_bit_width_of_integer; - else - throw Exception( - getExceptionMessagePrefix(types) - + " because some of them are signed integers and some are unsigned integers," - " but there is no signed integer type, that can exactly represent all required unsigned integer values", - ErrorCodes::NO_COMMON_TYPE); - } - - /// If the result must be floating. - if (max_mantissa_bits_of_floating) - { - size_t min_mantissa_bits = std::max(min_bit_width_of_integer, max_mantissa_bits_of_floating); - if (min_mantissa_bits <= 24) - return std::make_shared(); - else if (min_mantissa_bits <= 53) - return std::make_shared(); - else - throw Exception(getExceptionMessagePrefix(types) - + " because some of them are integers and some are floating point," - " but there is no floating point type, that can exactly represent all required integers", ErrorCodes::NO_COMMON_TYPE); - } - - /// If the result must be signed integer. - if (max_bits_of_signed_integer) - { - if (min_bit_width_of_integer <= 8) - return std::make_shared(); - else if (min_bit_width_of_integer <= 16) - return std::make_shared(); - else if (min_bit_width_of_integer <= 32) - return std::make_shared(); - else if (min_bit_width_of_integer <= 64) - return std::make_shared(); - else if (min_bit_width_of_integer <= 128) - return std::make_shared(); - else if (min_bit_width_of_integer <= 256) - return std::make_shared(); - else - throw Exception(getExceptionMessagePrefix(types) - + " because some of them are signed integers and some are unsigned integers," - " but there is no signed integer type, that can exactly represent all required unsigned integer values", ErrorCodes::NO_COMMON_TYPE); - } - - /// All unsigned. - { - if (min_bit_width_of_integer <= 8) - return std::make_shared(); - else if (min_bit_width_of_integer <= 16) - return std::make_shared(); - else if (min_bit_width_of_integer <= 32) - return std::make_shared(); - else if (min_bit_width_of_integer <= 64) - return std::make_shared(); - else if (min_bit_width_of_integer <= 128) - return std::make_shared(); - else if (min_bit_width_of_integer <= 256) - return std::make_shared(); - else - throw Exception("Logical error: " + getExceptionMessagePrefix(types) - + " but as all data types are unsigned integers, we must have found maximum unsigned integer type", ErrorCodes::NO_COMMON_TYPE); - } - } + auto numeric_type = getNumericType(type_ids, allow_conversion_to_string); + if (numeric_type) + return numeric_type; } /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). - throw Exception(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); + return throw_or_return(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); +} + +DataTypePtr getLeastSupertype(const TypeIndexSet & types, bool allow_conversion_to_string) +{ + auto throw_or_return = [&](std::string_view message, int error_code) + { + if (allow_conversion_to_string) + return std::make_shared(); + + throw Exception(String(message), error_code); + }; + + TypeIndexSet types_set; + for (const auto & type : types) + { + if (WhichDataType(type).isNothing()) + continue; + + if (!WhichDataType(type).isSimple()) + throw Exception(ErrorCodes::NO_COMMON_TYPE, + "Cannot get common type by type ids with parametric type {}", typeToString(type)); + + types_set.insert(type); + } + + if (types_set.empty()) + return std::make_shared(); + + if (types.count(TypeIndex::String)) + { + if (types.size() != 1) + return throw_or_return(getExceptionMessagePrefix(types) + " because some of them are String and some of them are not", ErrorCodes::NO_COMMON_TYPE); + + return std::make_shared(); + } + + /// For numeric types, the most complicated part. + auto numeric_type = getNumericType(types, allow_conversion_to_string); + if (numeric_type) + return numeric_type; + + /// All other data types (UUID, AggregateFunction, Enum...) are compatible only if they are the same (checked in trivial cases). + return throw_or_return(getExceptionMessagePrefix(types), ErrorCodes::NO_COMMON_TYPE); } DataTypePtr tryGetLeastSupertype(const DataTypes & types) diff --git a/src/DataTypes/getLeastSupertype.h b/src/DataTypes/getLeastSupertype.h index c35ec7d722c..5444bb34d06 100644 --- a/src/DataTypes/getLeastSupertype.h +++ b/src/DataTypes/getLeastSupertype.h @@ -7,12 +7,16 @@ namespace DB { /** Get data type that covers all possible values of passed data types. - * If there is no such data type, throws an exception. + * If there is no such data type, throws an exception + * or if 'allow_conversion_to_string' is true returns String as common type. * * Examples: least common supertype for UInt8, Int8 - Int16. * Examples: there is no least common supertype for Array(UInt8), Int8. */ -DataTypePtr getLeastSupertype(const DataTypes & types); +DataTypePtr getLeastSupertype(const DataTypes & types, bool allow_conversion_to_string = false); + +using TypeIndexSet = std::unordered_set; +DataTypePtr getLeastSupertype(const TypeIndexSet & types, bool allow_conversion_to_string = false); /// Same as above but return nullptr instead of throwing exception. DataTypePtr tryGetLeastSupertype(const DataTypes & types); diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 721bf79199b..adfcd83f5a7 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -37,7 +37,7 @@ public: }; DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, const String & logger_name, ContextPtr context_) - : DatabaseOrdinary(name_, std::move(metadata_path_), "store/", logger_name, context_) + : DatabaseOrdinary(name_, metadata_path_, "store/", logger_name, context_) , path_to_table_symlinks(fs::path(getContext()->getPath()) / "data" / escapeForFileName(name_) / "") , path_to_metadata_symlink(fs::path(getContext()->getPath()) / "metadata" / escapeForFileName(name_)) , db_uuid(uuid) @@ -141,9 +141,6 @@ void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_na if (table->storesDataOnDisk()) tryRemoveSymlink(table_name); - if (table->dropTableImmediately()) - table->drop(); - /// Notify DatabaseCatalog that table was dropped. It will remove table data in background. /// Cleanup is performed outside of database to allow easily DROP DATABASE without waiting for cleanup to complete. DatabaseCatalog::instance().enqueueDroppedTableCleanup(table->getStorageID(), table, table_metadata_path_drop, no_delay); diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index b854d9be1f3..87fae115b59 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -50,6 +50,9 @@ public: void alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) override; + /// This database can contain tables to backup. + bool hasTablesToBackup() const override { return true; } + private: String data_path; using NameToASTCreate = std::unordered_map; diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index 9bc8fa3bcef..64122ae66e5 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -69,7 +69,7 @@ public: static ASTPtr parseQueryFromMetadata(Poco::Logger * log, ContextPtr context, const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false); /// will throw when the table we want to attach already exists (in active / detached / detached permanently form) - void checkMetadataFilenameAvailability(const String & to_table_name) const; + void checkMetadataFilenameAvailability(const String & to_table_name) const override; void checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name, std::unique_lock &) const; void modifySettingsMetadata(const SettingsChanges & settings_changes, ContextPtr query_context); diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 982be2024ce..2144f874b03 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -36,6 +36,9 @@ public: const StorageID & table_id, const StorageInMemoryMetadata & metadata) override; + /// This database can contain tables to backup. + bool hasTablesToBackup() const override { return true; } + protected: virtual void commitAlterTable( const StorageID & table_id, diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h index 773612e403c..6b957e567ff 100644 --- a/src/Databases/DatabaseReplicatedWorker.h +++ b/src/Databases/DatabaseReplicatedWorker.h @@ -30,7 +30,7 @@ public: void shutdown() override; static String enqueueQueryImpl(const ZooKeeperPtr & zookeeper, DDLLogEntry & entry, - DatabaseReplicated * const database, bool committed = false); + DatabaseReplicated * const database, bool committed = false); /// NOLINT private: bool initializeMainThread() override; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index b1aa4eb1aae..51d4b8bb6b1 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -51,8 +51,8 @@ public: /// - it maintains a list of tables but tables are loaded lazily). virtual const StoragePtr & table() const = 0; - IDatabaseTablesIterator(const String & database_name_) : database_name(database_name_) { } - IDatabaseTablesIterator(String && database_name_) : database_name(std::move(database_name_)) { } + explicit IDatabaseTablesIterator(const String & database_name_) : database_name(database_name_) { } + explicit IDatabaseTablesIterator(String && database_name_) : database_name(std::move(database_name_)) { } virtual ~IDatabaseTablesIterator() = default; @@ -61,7 +61,7 @@ public: const String & databaseName() const { assert(!database_name.empty()); return database_name; } protected: - const String database_name; + String database_name; }; /// Copies list of tables and iterates through such snapshot. @@ -72,7 +72,7 @@ private: Tables::iterator it; protected: - DatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && other) + DatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && other) noexcept : IDatabaseTablesIterator(std::move(other.database_name)) { size_t idx = std::distance(other.tables.begin(), other.it); @@ -118,7 +118,7 @@ class IDatabase : public std::enable_shared_from_this { public: IDatabase() = delete; - IDatabase(String database_name_) : database_name(std::move(database_name_)) {} + explicit IDatabase(String database_name_) : database_name(std::move(database_name_)) {} /// Get name of database engine. virtual String getEngineName() const = 0; @@ -129,7 +129,7 @@ public: /// Load a set of existing tables. /// You can call only once, right after the object is created. - virtual void loadStoredObjects( + virtual void loadStoredObjects( /// NOLINT ContextMutablePtr /*context*/, bool /*force_restore*/, bool /*force_attach*/ = false, @@ -158,9 +158,14 @@ public: virtual void startupTables(ThreadPool & /*thread_pool*/, bool /*force_restore*/, bool /*force_attach*/) {} - /// Check the existence of the table. + /// Check the existence of the table in memory (attached). virtual bool isTableExist(const String & name, ContextPtr context) const = 0; + /// Check the existence of the table in any state (in active / detached / detached permanently state). + /// Throws exception when table exists. + virtual void checkMetadataFilenameAvailability(const String & /*table_name*/) const {} + + /// Get the table for work. Return nullptr if there is no table. virtual StoragePtr tryGetTable(const String & name, ContextPtr context) const = 0; @@ -170,7 +175,7 @@ public: /// Get an iterator that allows you to pass through all the tables. /// It is possible to have "hidden" tables that are not visible when passing through, but are visible if you get them by name using the functions above. - virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}) const = 0; + virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}) const = 0; /// NOLINT /// Is the database empty. virtual bool empty() const = 0; @@ -186,7 +191,7 @@ public: } /// Delete the table from the database, drop table and delete the metadata. - virtual void dropTable( + virtual void dropTable( /// NOLINT ContextPtr /*context*/, const String & /*name*/, [[maybe_unused]] bool no_delay = false) @@ -197,7 +202,7 @@ public: /// Add a table to the database, but do not add it to the metadata. The database may not support this method. /// /// Note: ATTACH TABLE statement actually uses createTable method. - virtual void attachTable(ContextPtr /* context */, const String & /*name*/, const StoragePtr & /*table*/, [[maybe_unused]] const String & relative_table_path = {}) + virtual void attachTable(ContextPtr /* context */, const String & /*name*/, const StoragePtr & /*table*/, [[maybe_unused]] const String & relative_table_path = {}) /// NOLINT { throw Exception("There is no ATTACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED); } @@ -284,12 +289,6 @@ public: throw Exception(getEngineName() + ": RENAME DATABASE is not supported", ErrorCodes::NOT_IMPLEMENTED); } - /// Whether the contained tables should be written to a backup. - virtual DatabaseTablesIteratorPtr getTablesIteratorForBackup(ContextPtr context) const - { - return getTablesIterator(context); /// By default we backup each table. - } - /// Returns path for persistent data storage if the database supports it, empty string otherwise virtual String getDataPath() const { return {}; } @@ -330,6 +329,10 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not run a replication thread!", getEngineName()); } + /// Returns true if the backup of the database is hollow, which means it doesn't contain + /// any tables which can be stored to a backup. + virtual bool hasTablesToBackup() const { return false; } + virtual ~IDatabase() = default; protected: diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index cc6d808a564..5f4027a26b3 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -61,7 +61,7 @@ DatabaseMySQL::DatabaseMySQL( , database_engine_define(database_engine_define_->clone()) , database_name_in_mysql(database_name_in_mysql_) , database_settings(std::move(settings_)) - , mysql_pool(std::move(pool)) + , mysql_pool(std::move(pool)) /// NOLINT { try { diff --git a/src/Databases/MySQL/MaterializeMetadata.cpp b/src/Databases/MySQL/MaterializeMetadata.cpp index 0facdfc20be..580eb41b449 100644 --- a/src/Databases/MySQL/MaterializeMetadata.cpp +++ b/src/Databases/MySQL/MaterializeMetadata.cpp @@ -30,11 +30,15 @@ namespace ErrorCodes static std::unordered_map fetchTablesCreateQuery( const mysqlxx::PoolWithFailover::Entry & connection, const String & database_name, - const std::vector & fetch_tables, const Settings & global_settings) + const std::vector & fetch_tables, std::unordered_set & materialized_tables_list, + const Settings & global_settings) { std::unordered_map tables_create_query; for (const auto & fetch_table_name : fetch_tables) { + if (!materialized_tables_list.empty() && !materialized_tables_list.contains(fetch_table_name)) + continue; + Block show_create_table_header{ {std::make_shared(), "Table"}, {std::make_shared(), "Create Table"}, @@ -253,7 +257,7 @@ void MaterializeMetadata::transaction(const MySQLReplication::Position & positio out.close(); } - commitMetadata(std::move(fun), persistent_tmp_path, persistent_path); + commitMetadata(fun, persistent_tmp_path, persistent_path); } MaterializeMetadata::MaterializeMetadata(const String & path_, const Settings & settings_) : persistent_path(path_), settings(settings_) @@ -276,7 +280,8 @@ MaterializeMetadata::MaterializeMetadata(const String & path_, const Settings & void MaterializeMetadata::startReplication( mysqlxx::PoolWithFailover::Entry & connection, const String & database, - bool & opened_transaction, std::unordered_map & need_dumping_tables) + bool & opened_transaction, std::unordered_map & need_dumping_tables, + std::unordered_set & materialized_tables_list) { checkSyncUserPriv(connection, settings); @@ -297,7 +302,7 @@ void MaterializeMetadata::startReplication( connection->query("START TRANSACTION /*!40100 WITH CONSISTENT SNAPSHOT */;").execute(); opened_transaction = true; - need_dumping_tables = fetchTablesCreateQuery(connection, database, fetchTablesInDB(connection, database, settings), settings); + need_dumping_tables = fetchTablesCreateQuery(connection, database, fetchTablesInDB(connection, database, settings), materialized_tables_list, settings); connection->query("UNLOCK TABLES;").execute(); } catch (...) diff --git a/src/Databases/MySQL/MaterializeMetadata.h b/src/Databases/MySQL/MaterializeMetadata.h index bcb0465b61e..b828c901fbb 100644 --- a/src/Databases/MySQL/MaterializeMetadata.h +++ b/src/Databases/MySQL/MaterializeMetadata.h @@ -48,7 +48,8 @@ struct MaterializeMetadata mysqlxx::PoolWithFailover::Entry & connection, const String & database, bool & opened_transaction, - std::unordered_map & need_dumping_tables); + std::unordered_map & need_dumping_tables, + std::unordered_set & materialized_tables_list); MaterializeMetadata(const String & path_, const Settings & settings_); }; diff --git a/src/Databases/MySQL/MaterializedMySQLSettings.h b/src/Databases/MySQL/MaterializedMySQLSettings.h index d5acdc81602..43235d502c3 100644 --- a/src/Databases/MySQL/MaterializedMySQLSettings.h +++ b/src/Databases/MySQL/MaterializedMySQLSettings.h @@ -16,6 +16,7 @@ class ASTStorage; M(UInt64, max_flush_data_time, 1000, "Max milliseconds that data is allowed to cache in memory(for database and the cache data unable to query). when this time is exceeded, the data will be materialized", 0) \ M(Int64, max_wait_time_when_mysql_unavailable, 1000, "Retry interval when MySQL is not available (milliseconds). Negative value disable retry.", 0) \ M(Bool, allows_query_when_mysql_lost, false, "Allow query materialized table when mysql is lost.", 0) \ + M(String, materialized_mysql_tables_list, "", "a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated.", 0) \ DECLARE_SETTINGS_TRAITS(MaterializedMySQLSettingsTraits, LIST_OF_MATERIALIZE_MODE_SETTINGS) diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 8033d65c549..230b158b231 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -25,6 +25,10 @@ #include #include #include +#include +#include +#include +#include namespace DB { @@ -148,6 +152,61 @@ static void checkMySQLVariables(const mysqlxx::Pool::Entry & connection, const S } } +static std::tuple tryExtractTableNameFromDDL(const String & ddl) +{ + String table_name; + String database_name; + if (ddl.empty()) return std::make_tuple(database_name, table_name); + + bool parse_failed = false; + Tokens tokens(ddl.data(), ddl.data() + ddl.size()); + IParser::Pos pos(tokens, 0); + Expected expected; + ASTPtr res; + ASTPtr table; + if (ParserKeyword("CREATE TEMPORARY TABLE").ignore(pos, expected) || ParserKeyword("CREATE TABLE").ignore(pos, expected)) + { + ParserKeyword("IF NOT EXISTS").ignore(pos, expected); + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) + parse_failed = true; + } + else if (ParserKeyword("ALTER TABLE").ignore(pos, expected)) + { + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) + parse_failed = true; + } + else if (ParserKeyword("DROP TABLE").ignore(pos, expected) || ParserKeyword("DROP TEMPORARY TABLE").ignore(pos, expected)) + { + ParserKeyword("IF EXISTS").ignore(pos, expected); + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) + parse_failed = true; + } + else if (ParserKeyword("TRUNCATE").ignore(pos, expected)) + { + ParserKeyword("TABLE").ignore(pos, expected); + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) + parse_failed = true; + } + else if (ParserKeyword("RENAME TABLE").ignore(pos, expected)) + { + if (!ParserCompoundIdentifier(true).parse(pos, table, expected)) + parse_failed = true; + } + else + { + parse_failed = true; + } + if (!parse_failed) + { + if (auto table_id = table->as()->getTableId()) + { + database_name = table_id.database_name; + table_name = table_id.table_name; + } + } + return std::make_tuple(database_name, table_name); +} + MaterializedMySQLSyncThread::MaterializedMySQLSyncThread( ContextPtr context_, const String & database_name_, @@ -159,11 +218,22 @@ MaterializedMySQLSyncThread::MaterializedMySQLSyncThread( , log(&Poco::Logger::get("MaterializedMySQLSyncThread")) , database_name(database_name_) , mysql_database_name(mysql_database_name_) - , pool(std::move(pool_)) + , pool(std::move(pool_)) /// NOLINT , client(std::move(client_)) , settings(settings_) { query_prefix = "EXTERNAL DDL FROM MySQL(" + backQuoteIfNeed(database_name) + ", " + backQuoteIfNeed(mysql_database_name) + ") "; + + if (!settings->materialized_mysql_tables_list.value.empty()) + { + Names tables_list; + boost::split(tables_list, settings->materialized_mysql_tables_list.value, [](char c){ return c == ','; }); + for (String & table_name: tables_list) + { + boost::trim(table_name); + materialized_tables_list.insert(table_name); + } + } } void MaterializedMySQLSyncThread::synchronization() @@ -434,7 +504,7 @@ bool MaterializedMySQLSyncThread::prepareSynchronized(MaterializeMetadata & meta checkMySQLVariables(connection, getContext()->getSettingsRef()); std::unordered_map need_dumping_tables; - metadata.startReplication(connection, mysql_database_name, opened_transaction, need_dumping_tables); + metadata.startReplication(connection, mysql_database_name, opened_transaction, need_dumping_tables, materialized_tables_list); if (!need_dumping_tables.empty()) { @@ -464,7 +534,7 @@ bool MaterializedMySQLSyncThread::prepareSynchronized(MaterializeMetadata & meta connection->query("COMMIT").execute(); client.connect(); - client.startBinlogDumpGTID(randomNumber(), mysql_database_name, metadata.executed_gtid_set, metadata.binlog_checksum); + client.startBinlogDumpGTID(randomNumber(), mysql_database_name, materialized_tables_list, metadata.executed_gtid_set, metadata.binlog_checksum); setSynchronizationThreadException(nullptr); return true; @@ -792,9 +862,24 @@ void MaterializedMySQLSyncThread::executeDDLAtomic(const QueryEvent & query_even auto query_context = createQueryContext(getContext()); CurrentThread::QueryScope query_scope(query_context); + String query = query_event.query; + if (!materialized_tables_list.empty()) + { + auto [ddl_database_name, ddl_table_name] = tryExtractTableNameFromDDL(query_event.query); + + if (!ddl_table_name.empty()) + { + ddl_database_name = ddl_database_name.empty() ? query_event.schema: ddl_database_name; + if (ddl_database_name != mysql_database_name || !materialized_tables_list.contains(ddl_table_name)) + { + LOG_DEBUG(log, "Skip MySQL DDL: \n {}", query_event.query); + return; + } + } + } String comment = "Materialize MySQL step 2: execute MySQL DDL for sync data"; String event_database = query_event.schema == mysql_database_name ? database_name : ""; - tryToExecuteQuery(query_prefix + query_event.query, query_context, event_database, comment); + tryToExecuteQuery(query_prefix + query, query_context, event_database, comment); } catch (Exception & exception) { diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.h b/src/Databases/MySQL/MaterializedMySQLSyncThread.h index ba5022137bf..163a3732fb9 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.h +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.h @@ -63,15 +63,16 @@ private: mutable MySQLClient client; MaterializedMySQLSettings * settings; String query_prefix; + NameSet materialized_tables_list; // USE MySQL ERROR CODE: // https://dev.mysql.com/doc/mysql-errors/5.7/en/server-error-reference.html - const int ER_ACCESS_DENIED_ERROR = 1045; - const int ER_DBACCESS_DENIED_ERROR = 1044; - const int ER_BAD_DB_ERROR = 1049; + const int ER_ACCESS_DENIED_ERROR = 1045; /// NOLINT + const int ER_DBACCESS_DENIED_ERROR = 1044; /// NOLINT + const int ER_BAD_DB_ERROR = 1049; /// NOLINT // https://dev.mysql.com/doc/mysql-errors/8.0/en/client-error-reference.html - const int CR_SERVER_LOST = 2013; + const int CR_SERVER_LOST = 2013; /// NOLINT struct Buffers { @@ -87,7 +88,7 @@ private: using BufferAndSortingColumnsPtr = std::shared_ptr; std::unordered_map data; - Buffers(const String & database_) : database(database_) {} + explicit Buffers(const String & database_) : database(database_) {} void commit(ContextPtr context); diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index dba8bf64798..dd125294615 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -34,6 +34,7 @@ namespace ErrorCodes extern const int QUERY_NOT_ALLOWED; extern const int UNKNOWN_TABLE; extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; } DatabaseMaterializedPostgreSQL::DatabaseMaterializedPostgreSQL( @@ -309,8 +310,12 @@ void DatabaseMaterializedPostgreSQL::attachTable(ContextPtr context_, const Stri } } +StoragePtr DatabaseMaterializedPostgreSQL::detachTable(ContextPtr, const String &) +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DETACH TABLE not allowed, use DETACH PERMANENTLY"); +} -StoragePtr DatabaseMaterializedPostgreSQL::detachTable(ContextPtr context_, const String & table_name) +void DatabaseMaterializedPostgreSQL::detachTablePermanently(ContextPtr, const String & table_name) { /// If there is query context then we need to detach materialized storage. /// If there is no query context then we need to detach internal storage from atomic database. @@ -360,11 +365,6 @@ StoragePtr DatabaseMaterializedPostgreSQL::detachTable(ContextPtr context_, cons } materialized_tables.erase(table_name); - return nullptr; - } - else - { - return DatabaseAtomic::detachTable(context_, table_name); } } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h index 40ff0d9262d..08420f4ba5e 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h @@ -51,6 +51,8 @@ public: void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override; + void detachTablePermanently(ContextPtr context, const String & table_name) override; + StoragePtr detachTable(ContextPtr context, const String & table_name) override; void dropTable(ContextPtr local_context, const String & name, bool no_delay) override; diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index d43bde0b886..a6b4a978c7b 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -174,7 +174,7 @@ StoragePtr DatabasePostgreSQL::tryGetTable(const String & table_name, ContextPtr } -StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, ContextPtr, const bool table_checked) const +StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, ContextPtr, bool table_checked) const { if (!cache_tables || !cached_tables.count(table_name)) { @@ -194,7 +194,7 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, ContextPtr, if (cache_tables) cached_tables[table_name] = storage; - return std::move(storage); + return storage; } if (table_checked || checkPostgresTable(table_name)) @@ -406,15 +406,26 @@ ASTPtr DatabasePostgreSQL::getCreateTableQueryImpl(const String & table_name, Co ASTs storage_children = ast_storage->children; auto storage_engine_arguments = ast_storage->engine->arguments; - /// Remove extra engine argument (`schema` and `use_table_cache`) - if (storage_engine_arguments->children.size() >= 5) - storage_engine_arguments->children.resize(4); + if (storage_engine_arguments->children.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected number of arguments: {}", storage_engine_arguments->children.size()); - /// Add table_name to engine arguments - assert(storage_engine_arguments->children.size() >= 2); - storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, std::make_shared(table_id.table_name)); + /// Check for named collection. + if (typeid_cast(storage_engine_arguments->children[0].get())) + { + storage_engine_arguments->children.push_back(makeASTFunction("equals", std::make_shared("table"), std::make_shared(table_id.table_name))); + } + else + { + /// Remove extra engine argument (`schema` and `use_table_cache`) + if (storage_engine_arguments->children.size() >= 5) + storage_engine_arguments->children.resize(4); - return std::move(create_table_query); + /// Add table_name to engine arguments. + if (storage_engine_arguments->children.size() >= 2) + storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 2, std::make_shared(table_id.table_name)); + } + + return create_table_query; } diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index d41dbff1f54..3397dcc8076 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -81,7 +81,7 @@ private: bool checkPostgresTable(const String & table_name) const; - StoragePtr fetchTable(const String & table_name, ContextPtr context, const bool table_checked) const; + StoragePtr fetchTable(const String & table_name, ContextPtr context, bool table_checked) const; void removeOutdatedTables(); diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 67d328db00b..9f136efa1ff 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,8 @@ static DataTypePtr convertPostgreSQLDataType(String & type, Fn auto && r res = std::make_shared(6); else if (type == "date") res = std::make_shared(); + else if (type == "uuid") + res = std::make_shared(); else if (type.starts_with("numeric")) { /// Numeric and decimal will both end up here as numeric. If it has type and precision, diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index 5b38caeabee..88e6d0ab6fd 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -23,35 +23,36 @@ void processSQLiteError(const String & message, bool throw_on_error) LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), fmt::runtime(message)); } - String validateSQLiteDatabasePath(const String & path, const String & user_files_path, bool throw_on_error) { - String canonical_user_files_path = fs::canonical(user_files_path); - - String canonical_path; - std::error_code err; - if (fs::path(path).is_relative()) - canonical_path = fs::canonical(fs::path(user_files_path) / path, err); - else - canonical_path = fs::canonical(path, err); + return fs::absolute(fs::path(user_files_path) / path).lexically_normal(); - if (err) - processSQLiteError(fmt::format("SQLite database path '{}' is invalid. Error: {}", path, err.message()), throw_on_error); + String absolute_path = fs::absolute(path).lexically_normal(); + String absolute_user_files_path = fs::absolute(user_files_path).lexically_normal(); - if (!canonical_path.starts_with(canonical_user_files_path)) + if (!absolute_path.starts_with(absolute_user_files_path)) + { processSQLiteError(fmt::format("SQLite database file path '{}' must be inside 'user_files' directory", path), throw_on_error); - - return canonical_path; + return ""; + } + return absolute_path; } - -SQLitePtr openSQLiteDB(const String & database_path, ContextPtr context, bool throw_on_error) +SQLitePtr openSQLiteDB(const String & path, ContextPtr context, bool throw_on_error) { - auto validated_path = validateSQLiteDatabasePath(database_path, context->getUserFilesPath(), throw_on_error); + auto user_files_path = context->getUserFilesPath(); + auto database_path = validateSQLiteDatabasePath(path, user_files_path, throw_on_error); + + /// For attach database there is no throw mode. + if (database_path.empty()) + return nullptr; + + if (!fs::exists(database_path)) + LOG_DEBUG(&Poco::Logger::get("SQLite"), "SQLite database path {} does not exist, will create an empty SQLite database", database_path); sqlite3 * tmp_sqlite_db = nullptr; - int status = sqlite3_open(validated_path.c_str(), &tmp_sqlite_db); + int status = sqlite3_open(database_path.c_str(), &tmp_sqlite_db); if (status != SQLITE_OK) { diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index cad3e3b8799..8b8d0a57cc7 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -494,7 +494,7 @@ Pipe CacheDictionary::read(const Names & column_names, size { auto keys = cache_storage_ptr->getCachedSimpleKeys(); auto keys_column = getColumnFromPODArray(std::move(keys)); - key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; + key_columns = {ColumnWithTypeAndName(keys_column, std::make_shared(), dict_struct.id->name)}; } else { diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.h b/src/Dictionaries/CacheDictionaryUpdateQueue.h index 7725ce7588f..d6a195ca7b8 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.h +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.h @@ -75,7 +75,7 @@ private: friend class CacheDictionaryUpdateQueue; std::atomic is_done{false}; - std::exception_ptr current_exception{nullptr}; + std::exception_ptr current_exception{nullptr}; /// NOLINT /// While UpdateUnit is alive, it is accounted in update_queue size. CurrentMetrics::Increment alive_batch{CurrentMetrics::CacheDictionaryUpdateQueueBatches}; diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index 2ff349043ff..c76f2578579 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -1,6 +1,7 @@ #include "CassandraDictionarySource.h" #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" +#include namespace DB { @@ -17,13 +18,17 @@ void registerDictionarySourceCassandra(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - ContextPtr /* global_context */, + [[maybe_unused]] ContextPtr global_context, const std::string & /* default_database */, bool /*created_from_ddl*/) -> DictionarySourcePtr { #if USE_CASSANDRA setupCassandraDriverLibraryLogging(CASS_LOG_INFO); - return std::make_unique(dict_struct, config, config_prefix + ".cassandra", sample_block); + + auto source_config_prefix = config_prefix + ".cassandra"; + global_context->getRemoteHostFilter().checkHostAndPort(config.getString(source_config_prefix + ".host"), toString(config.getUInt(source_config_prefix + ".port", 0))); + + return std::make_unique(dict_struct, config, source_config_prefix, sample_block); #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `cassandra` is disabled because ClickHouse was built without cassandra support."); diff --git a/src/Dictionaries/CassandraHelpers.h b/src/Dictionaries/CassandraHelpers.h index 30111e11686..3b90d46acdf 100644 --- a/src/Dictionaries/CassandraHelpers.h +++ b/src/Dictionaries/CassandraHelpers.h @@ -23,8 +23,8 @@ class ObjectHolder CassT * ptr = nullptr; public: template - ObjectHolder(Args &&... args) : ptr(Ctor(std::forward(args)...)) {} - ObjectHolder(CassT * ptr_) : ptr(ptr_) {} + ObjectHolder(Args &&... args) : ptr(Ctor(std::forward(args)...)) {} /// NOLINT + ObjectHolder(CassT * ptr_) : ptr(ptr_) {} /// NOLINT ObjectHolder(const ObjectHolder &) = delete; ObjectHolder & operator = (const ObjectHolder &) = delete; @@ -46,8 +46,8 @@ public: } /// For implicit conversion when passing object to driver library functions - operator CassT * () { return ptr; } - operator const CassT * () const { return ptr; } + operator CassT * () { return ptr; } /// NOLINT + operator const CassT * () const { return ptr; } /// NOLINT }; } diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index bd9a1f7776e..5a18dcffb22 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -30,7 +30,7 @@ namespace ErrorCodes static const std::unordered_set dictionary_allowed_keys = { "host", "port", "user", "password", "db", "database", "table", - "update_field", "update_tag", "invalidate_query", "query", "where", "name", "secure"}; + "update_field", "update_lag", "invalidate_query", "query", "where", "name", "secure"}; namespace { @@ -277,7 +277,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) { /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). Session session(global_context, ClientInfo::Interface::LOCAL); - session.authenticate(configuration.user, configuration.password, {}); + session.authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress{}); context = session.makeQueryContext(); } else diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index f2d7febfa8e..80b15eb2569 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -187,7 +187,7 @@ private: DataTypes dictionary_attributes_types; }; -static inline void insertDefaultValuesIntoColumns( +static inline void insertDefaultValuesIntoColumns( /// NOLINT MutableColumns & columns, const DictionaryStorageFetchRequest & fetch_request, size_t row_index) @@ -206,7 +206,7 @@ static inline void insertDefaultValuesIntoColumns( /// Deserialize column value and insert it in columns. /// Skip unnecessary columns that were not requested from deserialization. -static inline void deserializeAndInsertIntoColumns( +static inline void deserializeAndInsertIntoColumns( /// NOLINT MutableColumns & columns, const DictionaryStorageFetchRequest & fetch_request, const char * place_for_serialized_columns) diff --git a/src/Dictionaries/DictionarySourceHelpers.cpp b/src/Dictionaries/DictionarySourceHelpers.cpp index cd87cf831a2..fcad8398c0b 100644 --- a/src/Dictionaries/DictionarySourceHelpers.cpp +++ b/src/Dictionaries/DictionarySourceHelpers.cpp @@ -52,7 +52,7 @@ Block blockForKeys( auto filtered_column = source_column->filter(filter, requested_rows.size()); - block.insert({std::move(filtered_column), (*dict_struct.key)[i].type, (*dict_struct.key)[i].name}); + block.insert({filtered_column, (*dict_struct.key)[i].type, (*dict_struct.key)[i].name}); } return block; diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 3e29f3efe76..012750bde60 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -33,8 +33,8 @@ namespace DictionaryTypedSpecialAttribute makeDictionaryTypedSpecialAttribute( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const std::string & default_type) { - const auto name = config.getString(config_prefix + ".name", ""); - const auto expression = config.getString(config_prefix + ".expression", ""); + auto name = config.getString(config_prefix + ".name", ""); + auto expression = config.getString(config_prefix + ".expression", ""); if (name.empty() && !expression.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Element {}.name is empty"); diff --git a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h index 198f13e0f32..c2e36f59e1e 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h +++ b/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h @@ -14,7 +14,7 @@ private: FileUpdatesTracker updates_tracker; public: - RegionsHierarchyDataSource(const std::string & path_) : path(path_), updates_tracker(path_) {} + explicit RegionsHierarchyDataSource(const std::string & path_) : path(path_), updates_tracker(path_) {} bool isModified() const override; @@ -40,7 +40,7 @@ public: * For example, if /opt/geo/regions_hierarchy.txt is specified, * then the /opt/geo/regions_hierarchy_ua.txt file will also be loaded, if any, it will be accessible by the `ua` key. */ - RegionsHierarchiesDataProvider(const std::string & path_); + explicit RegionsHierarchiesDataProvider(const std::string & path_); std::vector listCustomHierarchies() const override; diff --git a/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h b/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h index 85dd8ce58b7..64f393ada62 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h +++ b/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h @@ -11,7 +11,7 @@ private: DB::ReadBufferPtr input; public: - RegionsHierarchyFormatReader(DB::ReadBufferPtr input_) : input(std::move(input_)) {} + explicit RegionsHierarchyFormatReader(DB::ReadBufferPtr input_) : input(std::move(input_)) {} bool readNext(RegionEntry & entry) override; }; diff --git a/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h b/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h index 0606896c951..f7d51135440 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h +++ b/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h @@ -27,7 +27,7 @@ public: virtual IRegionsHierarchyReaderPtr createReader() = 0; - virtual ~IRegionsHierarchyDataSource() {} + virtual ~IRegionsHierarchyDataSource() = default; }; using IRegionsHierarchyDataSourcePtr = std::shared_ptr; @@ -42,7 +42,7 @@ public: virtual IRegionsHierarchyDataSourcePtr getDefaultHierarchySource() const = 0; virtual IRegionsHierarchyDataSourcePtr getHierarchySource(const std::string & name) const = 0; - virtual ~IRegionsHierarchiesDataProvider() {} + virtual ~IRegionsHierarchiesDataProvider() = default; }; using IRegionsHierarchiesDataProviderPtr = std::shared_ptr; diff --git a/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h b/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h index 26de5d9116b..679c14d546b 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h +++ b/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h @@ -10,7 +10,7 @@ class ILanguageRegionsNamesReader public: virtual bool readNext(RegionNameEntry & entry) = 0; - virtual ~ILanguageRegionsNamesReader() {} + virtual ~ILanguageRegionsNamesReader() = default; }; using ILanguageRegionsNamesReaderPtr = std::unique_ptr; @@ -32,7 +32,7 @@ public: virtual std::string getSourceName() const = 0; - virtual ~ILanguageRegionsNamesDataSource() {} + virtual ~ILanguageRegionsNamesDataSource() = default; }; using ILanguageRegionsNamesDataSourcePtr = std::unique_ptr; @@ -45,7 +45,7 @@ public: /// Returns nullptr if the language data does not exist. virtual ILanguageRegionsNamesDataSourcePtr getLanguageRegionsNamesSource(const std::string & language) const = 0; - virtual ~IRegionsNamesDataProvider() {} + virtual ~IRegionsNamesDataProvider() = default; }; using IRegionsNamesDataProviderPtr = std::unique_ptr; diff --git a/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h b/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h index 573569ab115..49d324d434e 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h +++ b/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h @@ -11,7 +11,7 @@ private: DB::ReadBufferPtr input; public: - LanguageRegionsNamesFormatReader(DB::ReadBufferPtr input_) : input(std::move(input_)) {} + explicit LanguageRegionsNamesFormatReader(DB::ReadBufferPtr input_) : input(std::move(input_)) {} bool readNext(RegionNameEntry & entry) override; }; diff --git a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h index c380fcb7d1d..2d49cceab86 100644 --- a/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h +++ b/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h @@ -39,7 +39,7 @@ private: std::string directory; public: - RegionsNamesDataProvider(const std::string & directory_); + explicit RegionsNamesDataProvider(const std::string & directory_); ILanguageRegionsNamesDataSourcePtr getLanguageRegionsNamesSource(const std::string & language) const override; diff --git a/src/Dictionaries/Embedded/RegionsHierarchies.h b/src/Dictionaries/Embedded/RegionsHierarchies.h index 67cd7c2a658..925b7b490ff 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchies.h +++ b/src/Dictionaries/Embedded/RegionsHierarchies.h @@ -8,7 +8,7 @@ /** Contains several hierarchies of regions. * Used to support several different perspectives on the ownership of regions by countries. - * First of all, for the Crimea (Russian and Ukrainian points of view). + * First of all, for the Falklands/Malvinas (UK and Argentina points of view). */ class RegionsHierarchies { @@ -17,7 +17,7 @@ private: Container data; public: - RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_provider); + explicit RegionsHierarchies(IRegionsHierarchiesDataProviderPtr data_provider); /** Reloads, if necessary, all hierarchies of regions. */ @@ -27,7 +27,6 @@ public: elem.second.reload(); } - const RegionsHierarchy & get(const std::string & key) const { auto it = data.find(key); diff --git a/src/Dictionaries/Embedded/RegionsHierarchy.h b/src/Dictionaries/Embedded/RegionsHierarchy.h index 45d6c5246ca..508bca0d1e1 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchy.h +++ b/src/Dictionaries/Embedded/RegionsHierarchy.h @@ -49,7 +49,7 @@ private: IRegionsHierarchyDataSourcePtr data_source; public: - RegionsHierarchy(IRegionsHierarchyDataSourcePtr data_source_); + explicit RegionsHierarchy(IRegionsHierarchyDataSourcePtr data_source_); /// Reloads, if necessary, the hierarchy of regions. Not threadsafe. void reload(); diff --git a/src/Dictionaries/Embedded/RegionsNames.h b/src/Dictionaries/Embedded/RegionsNames.h index ff60c274401..ec06a0b1a33 100644 --- a/src/Dictionaries/Embedded/RegionsNames.h +++ b/src/Dictionaries/Embedded/RegionsNames.h @@ -40,7 +40,7 @@ class RegionsNames public: enum class Language : size_t { - #define M(NAME, FALLBACK, NUM) NAME = NUM, + #define M(NAME, FALLBACK, NUM) NAME = (NUM), FOR_EACH_LANGUAGE(M) #undef M }; @@ -78,7 +78,7 @@ private: static std::string dumpSupportedLanguagesNames(); public: - RegionsNames(IRegionsNamesDataProviderPtr data_provider); + explicit RegionsNames(IRegionsNamesDataProviderPtr data_provider); StringRef getRegionName(RegionID region_id, Language language) const { @@ -104,7 +104,7 @@ public: #define M(NAME, FALLBACK, NUM) \ if (0 == language.compare(#NAME)) \ return Language::NAME; - FOR_EACH_LANGUAGE(M) + FOR_EACH_LANGUAGE(M) /// NOLINT #undef M throw Poco::Exception("Unsupported language for region name. Supported languages are: " + dumpSupportedLanguagesNames() + "."); } diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 0c82da7b73b..cb2419633bf 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -32,13 +32,11 @@ FlatDictionary::FlatDictionary( const StorageID & dict_id_, const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, - const DictionaryLifetime dict_lifetime_, Configuration configuration_, BlockPtr update_field_loaded_block_) : IDictionary(dict_id_) , dict_struct(dict_struct_) , source_ptr{std::move(source_ptr_)} - , dict_lifetime(dict_lifetime_) , configuration(configuration_) , loaded_keys(configuration.initial_array_size, false) , update_field_loaded_block(std::move(update_field_loaded_block_)) @@ -147,7 +145,7 @@ ColumnPtr FlatDictionary::getColumn( callOnDictionaryAttributeType(attribute.type, type_call); if (attribute.is_nullable_set) - result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + result = ColumnNullable::create(result, std::move(col_null_map_to)); return result; } @@ -572,7 +570,7 @@ Pipe FlatDictionary::read(const Names & column_names, size_t max_block_size, siz keys.push_back(key_index); auto keys_column = getColumnFromPODArray(std::move(keys)); - ColumnsWithTypeAndName key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; + ColumnsWithTypeAndName key_columns = {ColumnWithTypeAndName(keys_column, std::make_shared(), dict_struct.id->name)}; std::shared_ptr dictionary = shared_from_this(); auto coordinator = DictionarySourceCoordinator::create(dictionary, column_names, std::move(key_columns), max_block_size); @@ -604,18 +602,19 @@ void registerDictionaryFlat(DictionaryFactory & factory) static constexpr size_t default_max_array_size = 500000; String dictionary_layout_prefix = config_prefix + ".layout" + ".flat"; + const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; FlatDictionary::Configuration configuration { .initial_array_size = config.getUInt64(dictionary_layout_prefix + ".initial_array_size", default_initial_array_size), .max_array_size = config.getUInt64(dictionary_layout_prefix + ".max_array_size", default_max_array_size), - .require_nonempty = config.getBool(config_prefix + ".require_nonempty", false) + .require_nonempty = config.getBool(config_prefix + ".require_nonempty", false), + .dict_lifetime = dict_lifetime }; const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix); - const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; - return std::make_unique(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, std::move(configuration)); + return std::make_unique(dict_id, dict_struct, std::move(source_ptr), std::move(configuration)); }; factory.registerLayout("flat", create_layout, false); diff --git a/src/Dictionaries/FlatDictionary.h b/src/Dictionaries/FlatDictionary.h index 2578fef3ecb..f342c38802d 100644 --- a/src/Dictionaries/FlatDictionary.h +++ b/src/Dictionaries/FlatDictionary.h @@ -26,13 +26,13 @@ public: size_t initial_array_size; size_t max_array_size; bool require_nonempty; + DictionaryLifetime dict_lifetime; }; FlatDictionary( const StorageID & dict_id_, const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, - const DictionaryLifetime dict_lifetime_, Configuration configuration_, BlockPtr update_field_loaded_block_ = nullptr); @@ -58,12 +58,12 @@ public: std::shared_ptr clone() const override { - return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), dict_lifetime, configuration, update_field_loaded_block); + return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), configuration, update_field_loaded_block); } DictionarySourcePtr getSource() const override { return source_ptr; } - const DictionaryLifetime & getLifetime() const override { return dict_lifetime; } + const DictionaryLifetime & getLifetime() const override { return configuration.dict_lifetime; } const DictionaryStructure & getStructure() const override { return dict_struct; } @@ -159,7 +159,6 @@ private: const DictionaryStructure dict_struct; const DictionarySourcePtr source_ptr; - const DictionaryLifetime dict_lifetime; const Configuration configuration; std::vector attributes; diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 308570644d1..ad9ddc71b19 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -104,7 +104,7 @@ Pipe HTTPDictionarySource::loadAll() context->getReadSettings(), configuration.header_entries, ReadWriteBufferFromHTTP::Range{}, - RemoteHostFilter{}, false); + nullptr, false); return createWrappedBuffer(std::move(in_ptr)); } @@ -125,7 +125,7 @@ Pipe HTTPDictionarySource::loadUpdatedAll() context->getReadSettings(), configuration.header_entries, ReadWriteBufferFromHTTP::Range{}, - RemoteHostFilter{}, false); + nullptr, false); return createWrappedBuffer(std::move(in_ptr)); } @@ -155,7 +155,7 @@ Pipe HTTPDictionarySource::loadIds(const std::vector & ids) context->getReadSettings(), configuration.header_entries, ReadWriteBufferFromHTTP::Range{}, - RemoteHostFilter{}, false); + nullptr, false); return createWrappedBuffer(std::move(in_ptr)); } @@ -185,7 +185,7 @@ Pipe HTTPDictionarySource::loadKeys(const Columns & key_columns, const std::vect context->getReadSettings(), configuration.header_entries, ReadWriteBufferFromHTTP::Range{}, - RemoteHostFilter{}, false); + nullptr, false); return createWrappedBuffer(std::move(in_ptr)); } diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index ea041c63d73..65d9b3e7d42 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -578,7 +578,7 @@ ColumnPtr HashedArrayDictionary::getAttributeColumn( callOnDictionaryAttributeType(attribute.type, type_call); if (is_attribute_nullable) - result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + result = ColumnNullable::create(result, std::move(col_null_map_to)); return result; } diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index b70f018df6b..178631d9c53 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -159,7 +159,7 @@ ColumnPtr HashedDictionary::getColumn( callOnDictionaryAttributeType(attribute.type, type_call); if (is_attribute_nullable) - result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + result = ColumnNullable::create(result, std::move(col_null_map_to)); return result; } diff --git a/src/Dictionaries/ICacheDictionaryStorage.h b/src/Dictionaries/ICacheDictionaryStorage.h index b094d76a9a7..a4990528a4e 100644 --- a/src/Dictionaries/ICacheDictionaryStorage.h +++ b/src/Dictionaries/ICacheDictionaryStorage.h @@ -22,7 +22,7 @@ struct KeyState , fetched_column_index(fetched_column_index_) {} - KeyState(State state_) + KeyState(State state_) /// NOLINT : state(state_) {} diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index 042153f0971..c18dbcfbea7 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -150,7 +150,7 @@ public: auto & key_column_to_cast = key_columns[key_attribute_type_index]; ColumnWithTypeAndName column_to_cast = {key_column_to_cast, key_type, ""}; - auto casted_column = castColumnAccurate(std::move(column_to_cast), key_attribute_type); + auto casted_column = castColumnAccurate(column_to_cast, key_attribute_type); key_column_to_cast = std::move(casted_column); key_type = key_attribute_type; } diff --git a/src/Dictionaries/IPAddressDictionary.h b/src/Dictionaries/IPAddressDictionary.h index 8dddc988caa..894af5ceb71 100644 --- a/src/Dictionaries/IPAddressDictionary.h +++ b/src/Dictionaries/IPAddressDictionary.h @@ -26,7 +26,7 @@ public: const StorageID & dict_id_, const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, - const DictionaryLifetime dict_lifetime_, + const DictionaryLifetime dict_lifetime_, /// NOLINT bool require_nonempty_); std::string getKeyDescription() const { return key_description; } @@ -160,7 +160,7 @@ private: template static void createAttributeImpl(Attribute & attribute, const Field & null_value); - static Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value); + static Attribute createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value); /// NOLINT template void getItemsByTwoKeyColumnsImpl( @@ -177,7 +177,7 @@ private: DefaultValueExtractor & default_value_extractor) const; template - void setAttributeValueImpl(Attribute & attribute, const T value); + void setAttributeValueImpl(Attribute & attribute, const T value); /// NOLINT void setAttributeValue(Attribute & attribute, const Field & value); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 29d70f3a7c4..6578f91aa73 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -34,7 +34,7 @@ static const std::unordered_set dictionary_allowed_keys = { "host", "port", "user", "password", "db", "database", "table", "schema", "update_field", "invalidate_query", "priority", - "update_tag", "dont_check_update_time", + "update_lag", "dont_check_update_time", "query", "where", "name" /* name_collection */, "socket", "share_connection", "fail_on_connection_loss", "close_connection", "ssl_ca", "ssl_cert", "ssl_key", diff --git a/src/Dictionaries/PolygonDictionary.cpp b/src/Dictionaries/PolygonDictionary.cpp index deec1e6a588..1a4e01d4aa3 100644 --- a/src/Dictionaries/PolygonDictionary.cpp +++ b/src/Dictionaries/PolygonDictionary.cpp @@ -61,7 +61,7 @@ void IPolygonDictionary::convertKeyColumns(Columns & key_columns, DataTypes & ke auto & key_column_to_cast = key_columns[key_type_index]; ColumnWithTypeAndName column_to_cast = {key_column_to_cast, key_type, ""}; - auto casted_column = castColumnAccurate(std::move(column_to_cast), float_64_type); + auto casted_column = castColumnAccurate(column_to_cast, float_64_type); key_column_to_cast = std::move(casted_column); key_type = float_64_type; } diff --git a/src/Dictionaries/PolygonDictionaryUtils.h b/src/Dictionaries/PolygonDictionaryUtils.h index 0aca7cd8af0..9d6d6ae0501 100644 --- a/src/Dictionaries/PolygonDictionaryUtils.h +++ b/src/Dictionaries/PolygonDictionaryUtils.h @@ -38,7 +38,7 @@ public: SlabsPolygonIndex() = default; /** Builds an index by splitting all edges with all points x coordinates. */ - SlabsPolygonIndex(const std::vector & polygons); + explicit SlabsPolygonIndex(const std::vector & polygons); /** Finds polygon id the same way as IPolygonIndex. */ bool find(const Point & point, size_t & id) const; @@ -179,7 +179,7 @@ class GridRoot : public ICell { public: GridRoot(size_t min_intersections_, size_t max_depth_, const std::vector & polygons_): - kMinIntersections(min_intersections_), kMaxDepth(max_depth_), polygons(polygons_) + k_min_intersections(min_intersections_), k_max_depth(max_depth_), polygons(polygons_) { setBoundingBox(); std::vector order(polygons.size()); @@ -209,8 +209,8 @@ private: std::unique_ptr> root = nullptr; Coord min_x = 0, min_y = 0; Coord max_x = 0, max_y = 0; - const size_t kMinIntersections; - const size_t kMaxDepth; + const size_t k_min_intersections; + const size_t k_max_depth; const std::vector & polygons; @@ -236,7 +236,7 @@ private: } #endif size_t intersections = possible_ids.size() - covered; - if (intersections <= kMinIntersections || depth++ == kMaxDepth) + if (intersections <= k_min_intersections || depth++ == k_max_depth) return std::make_unique(possible_ids, polygons, current_box, covered); auto x_shift = (current_max_x - current_min_x) / DividedCell::kSplit; auto y_shift = (current_max_y - current_min_y) / DividedCell::kSplit; diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index 6fdf486fdbf..511d6a7288e 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -30,7 +30,7 @@ static const UInt64 max_block_size = 8192; static const std::unordered_set dictionary_allowed_keys = { "host", "port", "user", "password", "db", "database", "table", "schema", - "update_field", "update_tag", "invalidate_query", "query", "where", "name", "priority"}; + "update_field", "update_lag", "invalidate_query", "query", "where", "name", "priority"}; namespace { diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 5330bc684c3..e82fcd580e2 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -198,7 +198,7 @@ ColumnPtr RangeHashedDictionary::getColumn( callOnDictionaryAttributeType(attribute.type, type_call); if (is_attribute_nullable) - result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + result = ColumnNullable::create(result, std::move(col_null_map_to)); return result; } @@ -298,7 +298,7 @@ ColumnPtr RangeHashedDictionary::getColumnInternal( callOnDictionaryAttributeType(attribute.type, type_call); if (is_attribute_nullable) - result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + result = ColumnNullable::create(result, std::move(col_null_map_to)); return result; } diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index a1b406b3424..85a11e9a33d 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -40,15 +41,20 @@ namespace DB const Poco::Util::AbstractConfiguration & config, const String & config_prefix, Block & sample_block, - ContextPtr /* global_context */, + ContextPtr global_context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { auto redis_config_prefix = config_prefix + ".redis"; + + auto host = config.getString(redis_config_prefix + ".host"); + auto port = config.getUInt(redis_config_prefix + ".port"); + global_context->getRemoteHostFilter().checkHostAndPort(host, toString(port)); + RedisDictionarySource::Configuration configuration = { - .host = config.getString(redis_config_prefix + ".host"), - .port = static_cast(config.getUInt(redis_config_prefix + ".port")), + .host = host, + .port = static_cast(port), .db_index = config.getUInt(redis_config_prefix + ".db_index", 0), .password = config.getString(redis_config_prefix + ".password", ""), .storage_type = parseStorageType(config.getString(redis_config_prefix + ".storage_type", "")), @@ -136,9 +142,9 @@ namespace DB RedisArray keys; auto key_type = storageTypeToKeyType(configuration.storage_type); - for (const auto & key : all_keys) + for (auto && key : all_keys) if (key_type == connection->client->execute(RedisCommand("TYPE").addRedisType(key))) - keys.addRedisType(std::move(key)); + keys.addRedisType(key); if (configuration.storage_type == RedisStorageType::HASH_MAP) { @@ -165,10 +171,10 @@ namespace DB } if (primary_with_secondary.size() > 1) - hkeys.add(std::move(primary_with_secondary)); + hkeys.add(primary_with_secondary); } - keys = std::move(hkeys); + keys = hkeys; } return Pipe(std::make_shared( diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index adbe4084d81..9b1a4ed1e6d 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -761,9 +761,9 @@ private: FileDescriptor() = default; - FileDescriptor(FileDescriptor && rhs) : fd(rhs.fd) { rhs.fd = -1; } + FileDescriptor(FileDescriptor && rhs) noexcept : fd(rhs.fd) { rhs.fd = -1; } - FileDescriptor & operator=(FileDescriptor && rhs) + FileDescriptor & operator=(FileDescriptor && rhs) noexcept { if (this == &rhs) return *this; diff --git a/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp b/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp index 31e85442c6a..fb07d8c356b 100644 --- a/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp +++ b/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp @@ -53,7 +53,7 @@ DiskAzureBlobStorage::DiskAzureBlobStorage( std::shared_ptr blob_container_client_, SettingsPtr settings_, GetDiskSettings settings_getter_) : - IDiskRemote(name_, "", metadata_disk_, "DiskAzureBlobStorage", settings_->thread_pool_size), + IDiskRemote(name_, "", metadata_disk_, nullptr, "DiskAzureBlobStorage", settings_->thread_pool_size), blob_container_client(blob_container_client_), current_settings(std::move(settings_)), settings_getter(settings_getter_) {} @@ -66,17 +66,15 @@ std::unique_ptr DiskAzureBlobStorage::readFile( std::optional) const { auto settings = current_settings.get(); - auto metadata = readMeta(path); + auto metadata = readMetadata(path); LOG_TEST(log, "Read from file by path: {}", backQuote(metadata_disk->getPath() + path)); - bool threadpool_read = read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; - auto reader_impl = std::make_unique( path, blob_container_client, metadata, settings->max_single_read_retries, - settings->max_single_download_retries, read_settings, threadpool_read); + settings->max_single_download_retries, read_settings); - if (threadpool_read) + if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { auto reader = getThreadPoolReader(); return std::make_unique(reader, read_settings, std::move(reader_impl)); @@ -94,7 +92,6 @@ std::unique_ptr DiskAzureBlobStorage::writeFile( size_t buf_size, WriteMode mode) { - auto metadata = readOrCreateMetaForWriting(path, mode); auto blob_path = path + "_" + getRandomASCIIString(8); /// NOTE: path contains the tmp_* prefix in the blob name LOG_TRACE(log, "{} to file by path: {}. AzureBlob Storage path: {}", @@ -106,7 +103,12 @@ std::unique_ptr DiskAzureBlobStorage::writeFile( current_settings.get()->max_single_part_upload_size, buf_size); - return std::make_unique>(std::move(buffer), std::move(metadata), blob_path); + auto create_metadata_callback = [this, path, mode, blob_path] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_path, count] (Metadata & metadata) { metadata.addObject(blob_path, count); return true; }); + }; + + return std::make_unique(std::move(buffer), std::move(create_metadata_callback), path); } diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index da27eff0b54..3519b1212a4 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -23,7 +23,7 @@ public: { } - virtual ~WritingToCacheWriteBuffer() override + ~WritingToCacheWriteBuffer() override { try { @@ -144,6 +144,14 @@ DiskCacheWrapper::readFile( } } + auto current_read_settings = settings; + /// Do not use RemoteFSReadMethod::threadpool for index and mark files. + /// Here it does not make sense since the files are small. + /// Note: enabling `threadpool` read requires to call setReadUntilEnd(). + current_read_settings.remote_fs_method = RemoteFSReadMethod::read; + /// Disable data cache. + current_read_settings.remote_fs_enable_cache = false; + if (metadata->status == DOWNLOADING) { FileDownloadStatus result_status = DOWNLOADED; @@ -158,7 +166,7 @@ DiskCacheWrapper::readFile( auto tmp_path = path + ".tmp"; { - auto src_buffer = DiskDecorator::readFile(path, settings, read_hint, file_size); + auto src_buffer = DiskDecorator::readFile(path, current_read_settings, read_hint, file_size); auto dst_buffer = cache_disk->writeFile(tmp_path, settings.local_fs_buffer_size, WriteMode::Rewrite); copyData(*src_buffer, *dst_buffer); } @@ -184,7 +192,7 @@ DiskCacheWrapper::readFile( if (metadata->status == DOWNLOADED) return cache_disk->readFile(path, settings, read_hint, file_size); - return DiskDecorator::readFile(path, settings, read_hint, file_size); + return DiskDecorator::readFile(path, current_read_settings, read_hint, file_size); } std::unique_ptr @@ -274,6 +282,7 @@ void DiskCacheWrapper::removeDirectory(const String & path) { if (cache_disk->exists(path)) cache_disk->removeDirectory(path); + DiskDecorator::removeDirectory(path); } @@ -298,6 +307,18 @@ void DiskCacheWrapper::removeSharedRecursive(const String & path, bool keep_s3) DiskDecorator::removeSharedRecursive(path, keep_s3); } + +void DiskCacheWrapper::removeSharedFiles(const RemoveBatchRequest & files, bool keep_s3) +{ + for (const auto & file : files) + { + if (cache_disk->exists(file.path)) + cache_disk->removeSharedFile(file.path, keep_s3); + } + + DiskDecorator::removeSharedFiles(files, keep_s3); +} + void DiskCacheWrapper::createHardLink(const String & src_path, const String & dst_path) { /// Don't create hardlinks for cache files to shadow directory as it just waste cache disk space. diff --git a/src/Disks/DiskCacheWrapper.h b/src/Disks/DiskCacheWrapper.h index 6eb79114a54..dc66333758f 100644 --- a/src/Disks/DiskCacheWrapper.h +++ b/src/Disks/DiskCacheWrapper.h @@ -48,6 +48,7 @@ public: void removeRecursive(const String & path) override; void removeSharedFile(const String & path, bool keep_s3) override; void removeSharedRecursive(const String & path, bool keep_s3) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_s3) override; void createHardLink(const String & src_path, const String & dst_path) override; ReservationPtr reserve(UInt64 bytes) override; diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 0bdfffa8f01..bace54ff22a 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -72,17 +72,9 @@ public: void startup() override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; - std::unique_ptr readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const override { return delegate->readMetaFile(path, settings, size); } + DiskPtr getMetadataDiskIfExistsOrSelf() override { return delegate->getMetadataDiskIfExistsOrSelf(); } - std::unique_ptr writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) override { return delegate->writeMetaFile(path, buf_size, mode); } - - void removeMetaFileIfExists(const String & path) override { delegate->removeMetaFileIfExists(path); } + std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override { return delegate->getSerializedMetadata(file_paths); } UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); } diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index cbdf6d6440b..44fdbb77323 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -6,7 +6,8 @@ #include #include #include -#include +#include +#include #include #include @@ -325,7 +326,7 @@ DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) void DiskLocal::moveFile(const String & from_path, const String & to_path) { - fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); + renameNoReplace(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path); } void DiskLocal::replaceFile(const String & from_path, const String & to_path) @@ -539,14 +540,14 @@ catch (...) struct DiskWriteCheckData { - constexpr static size_t PAGE_SIZE = 4096; - char data[PAGE_SIZE]{}; + constexpr static size_t PAGE_SIZE_IN_BYTES = 4096; + char data[PAGE_SIZE_IN_BYTES]{}; DiskWriteCheckData() { static const char * magic_string = "ClickHouse disk local write check"; static size_t magic_string_len = strlen(magic_string); memcpy(data, magic_string, magic_string_len); - memcpy(data + PAGE_SIZE - magic_string_len, magic_string, magic_string_len); + memcpy(data + PAGE_SIZE_IN_BYTES - magic_string_len, magic_string, magic_string_len); } }; @@ -557,7 +558,7 @@ try String tmp_template = fs::path(disk_path) / ""; { auto buf = WriteBufferFromTemporaryFile::create(tmp_template); - buf->write(data.data, data.PAGE_SIZE); + buf->write(data.data, data.PAGE_SIZE_IN_BYTES); buf->sync(); } return true; diff --git a/src/Disks/DiskMemory.h b/src/Disks/DiskMemory.h index eef7b78502d..fe108f53c68 100644 --- a/src/Disks/DiskMemory.h +++ b/src/Disks/DiskMemory.h @@ -22,7 +22,7 @@ class WriteBufferFromFileBase; class DiskMemory : public IDisk { public: - DiskMemory(const String & name_) : name(name_), disk_path("memory://" + name_ + '/') {} + explicit DiskMemory(const String & name_) : name(name_), disk_path("memory://" + name_ + '/') {} const String & getName() const override { return name; } @@ -97,7 +97,6 @@ private: void createDirectoriesImpl(const String & path); void replaceFileImpl(const String & from_path, const String & to_path); -private: friend class WriteIndirectBuffer; enum class FileType @@ -112,7 +111,7 @@ private: String data; FileData(FileType type_, String data_) : type(type_), data(std::move(data_)) {} - explicit FileData(FileType type_) : type(type_), data("") {} + explicit FileData(FileType type_) : type(type_) {} }; using Files = std::unordered_map; /// file path -> file data diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index fe9dd8421b1..43011a4cf72 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -20,11 +20,28 @@ public: RestartAwareReadBuffer(const DiskRestartProxy & disk, std::unique_ptr impl_) : ReadBufferFromFileDecorator(std::move(impl_)), lock(disk.mutex) { } - void prefetch() override { impl->prefetch(); } + void prefetch() override + { + swap(*impl); + impl->prefetch(); + swap(*impl); + } - void setReadUntilPosition(size_t position) override { impl->setReadUntilPosition(position); } + void setReadUntilPosition(size_t position) override + { + swap(*impl); + impl->setReadUntilPosition(position); + swap(*impl); + } - void setReadUntilEnd() override { impl->setReadUntilEnd(); } + void setReadUntilEnd() override + { + swap(*impl); + impl->setReadUntilEnd(); + swap(*impl); + } + + String getInfoForLog() override { return impl->getInfoForLog(); } private: ReadLock lock; diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 0cd1267c6ef..a2fce4b14d1 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -19,7 +19,7 @@ class DiskSelector { public: DiskSelector(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context); - DiskSelector(const DiskSelector & from) : disks(from.disks) { } + DiskSelector(const DiskSelector & from) = default; DiskSelectorPtr updateFromConfig( const Poco::Util::AbstractConfiguration & config, diff --git a/src/Disks/DiskWebServer.cpp b/src/Disks/DiskWebServer.cpp index 7c94a5b98b1..f3039d9af2e 100644 --- a/src/Disks/DiskWebServer.cpp +++ b/src/Disks/DiskWebServer.cpp @@ -168,11 +168,9 @@ std::unique_ptr DiskWebServer::readFile(const String & p RemoteMetadata meta(path, remote_path); meta.remote_fs_objects.emplace_back(std::make_pair(remote_path, iter->second.size)); - bool threadpool_read = read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; + auto web_impl = std::make_unique(path, url, meta, getContext(), read_settings); - auto web_impl = std::make_unique(path, url, meta, getContext(), threadpool_read, read_settings); - - if (threadpool_read) + if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { auto reader = IDiskRemote::getThreadPoolReader(); return std::make_unique(reader, read_settings, std::move(web_impl), min_bytes_for_seek); diff --git a/src/Disks/DiskWebServer.h b/src/Disks/DiskWebServer.h index bda8c8adaad..e2da0b2a1e1 100644 --- a/src/Disks/DiskWebServer.h +++ b/src/Disks/DiskWebServer.h @@ -38,7 +38,7 @@ namespace ErrorCodes * * To get files for upload run: * clickhouse static-files-disk-uploader --metadata-path --output-dir - * (--metadata-path can be found in query: `select data_paths from system.tables where name='';`) + * (--metadata-path can be found in query: `select data_paths from system.tables where name='';`) /// NOLINT * * When loading files by they must be loaded into /store/ path, but config must conrain only . * diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index 572c908768b..7f60b219a4b 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -1,4 +1,7 @@ #include + +#if USE_HDFS + #include #include @@ -62,7 +65,7 @@ DiskHDFS::DiskHDFS( SettingsPtr settings_, DiskPtr metadata_disk_, const Poco::Util::AbstractConfiguration & config_) - : IDiskRemote(disk_name_, hdfs_root_path_, metadata_disk_, "DiskHDFS", settings_->thread_pool_size) + : IDiskRemote(disk_name_, hdfs_root_path_, metadata_disk_, nullptr, "DiskHDFS", settings_->thread_pool_size) , config(config_) , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config)) , hdfs_fs(createHDFSFS(hdfs_builder.get())) @@ -73,13 +76,13 @@ DiskHDFS::DiskHDFS( std::unique_ptr DiskHDFS::readFile(const String & path, const ReadSettings & read_settings, std::optional, std::optional) const { - auto metadata = readMeta(path); + auto metadata = readMetadata(path); LOG_TEST(log, "Read from file by path: {}. Existing HDFS objects: {}", backQuote(metadata_disk->getPath() + path), metadata.remote_fs_objects.size()); - auto hdfs_impl = std::make_unique(path, config, remote_fs_root_path, metadata, read_settings.remote_fs_buffer_size); + auto hdfs_impl = std::make_unique(path, config, remote_fs_root_path, metadata, read_settings); auto buf = std::make_unique(std::move(hdfs_impl)); return std::make_unique(std::move(buf), settings->min_bytes_for_seek); } @@ -87,8 +90,6 @@ std::unique_ptr DiskHDFS::readFile(const String & path, std::unique_ptr DiskHDFS::writeFile(const String & path, size_t buf_size, WriteMode mode) { - auto metadata = readOrCreateMetaForWriting(path, mode); - /// Path to store new HDFS object. auto file_name = getRandomName(); auto hdfs_path = remote_fs_root_path + file_name; @@ -100,10 +101,12 @@ std::unique_ptr DiskHDFS::writeFile(const String & path auto hdfs_buffer = std::make_unique(hdfs_path, config, settings->replication, buf_size, mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND); + auto create_metadata_callback = [this, path, mode, file_name] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [file_name, count] (Metadata & metadata) { metadata.addObject(file_name, count); return true; }); + }; - return std::make_unique>(std::move(hdfs_buffer), - std::move(metadata), - file_name); + return std::make_unique(std::move(hdfs_buffer), std::move(create_metadata_callback), path); } @@ -179,3 +182,4 @@ void registerDiskHDFS(DiskFactory & factory) } } +#endif diff --git a/src/Disks/HDFS/DiskHDFS.h b/src/Disks/HDFS/DiskHDFS.h index de373d8d6ee..23a108507b4 100644 --- a/src/Disks/HDFS/DiskHDFS.h +++ b/src/Disks/HDFS/DiskHDFS.h @@ -1,5 +1,9 @@ #pragma once +#include + +#if USE_HDFS + #include #include #include @@ -79,3 +83,4 @@ private: }; } +#endif diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index b1d7b33fec3..42d5f5fce10 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -86,28 +86,4 @@ SyncGuardPtr IDisk::getDirectorySyncGuard(const String & /* path */) const return nullptr; } -std::unique_ptr IDisk::readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const -{ - LOG_TRACE(&Poco::Logger::get("IDisk"), "Read local metafile: {}", path); - return readFile(path, settings, size); -} - -std::unique_ptr IDisk::writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) -{ - LOG_TRACE(&Poco::Logger::get("IDisk"), "Write local metafile: {}", path); - return writeFile(path, buf_size, mode); -} - -void IDisk::removeMetaFileIfExists(const String & path) -{ - LOG_TRACE(&Poco::Logger::get("IDisk"), "Remove local metafile: {}", path); - removeFileIfExists(path); -} - } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 223d2d48e30..4fa73b8eba8 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -158,14 +158,14 @@ public: virtual void listFiles(const String & path, std::vector & file_names) = 0; /// Open the file for read and return ReadBufferFromFileBase object. - virtual std::unique_ptr readFile( + virtual std::unique_ptr readFile( /// NOLINT const String & path, const ReadSettings & settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const = 0; /// Open the file for write and return WriteBufferFromFileBase object. - virtual std::unique_ptr writeFile( + virtual std::unique_ptr writeFile( /// NOLINT const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) = 0; @@ -248,6 +248,10 @@ public: /// Overrode in remote fs disks. virtual bool supportZeroCopyReplication() const = 0; + /// Whether this disk support parallel write + /// Overrode in remote fs disks. + virtual bool supportParallelWrite() const { return false; } + virtual bool isReadOnly() const { return false; } /// Check if disk is broken. Broken disks will have 0 space and not be used. @@ -277,28 +281,34 @@ public: /// Applies new settings for disk in runtime. virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr, const String &, const DisksMap &) {} - /// Open the local file for read and return ReadBufferFromFileBase object. - /// Overridden in IDiskRemote. - /// Used for work with custom metadata. - virtual std::unique_ptr readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const; + /// Quite leaky abstraction. Some disks can use additional disk to store + /// some parts of metadata. In general case we have only one disk itself and + /// return pointer to it. + /// + /// Actually it's a part of IDiskRemote implementation but we have so + /// complex hierarchy of disks (with decorators), so we cannot even + /// dynamic_cast some pointer to IDisk to pointer to IDiskRemote. + virtual std::shared_ptr getMetadataDiskIfExistsOrSelf() { return std::static_pointer_cast(shared_from_this()); } - /// Open the local file for write and return WriteBufferFromFileBase object. - /// Overridden in IDiskRemote. - /// Used for work with custom metadata. - virtual std::unique_ptr writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode); - - virtual void removeMetaFileIfExists(const String & path); + /// Very similar case as for getMetadataDiskIfExistsOrSelf(). If disk has "metadata" + /// it will return mapping for each required path: path -> metadata as string. + /// Only for IDiskRemote. + virtual std::unordered_map getSerializedMetadata(const std::vector & /* paths */) const { return {}; } /// Return reference count for remote FS. - /// Overridden in IDiskRemote. + /// You can ask -- why we have zero and what does it mean? For some unknown reason + /// the decision was made to take 0 as "no references exist", but only file itself left. + /// With normal file system we will get 1 in this case: + /// $ stat clickhouse + /// File: clickhouse + /// Size: 3014014920 Blocks: 5886760 IO Block: 4096 regular file + /// Device: 10301h/66305d Inode: 3109907 Links: 1 + /// Why we have always zero by default? Because normal filesystem + /// manages hardlinks by itself. So you can always remove hardlink and all + /// other alive harlinks will not be removed. virtual UInt32 getRefCount(const String &) const { return 0; } + protected: friend class DiskDecorator; @@ -348,7 +358,7 @@ public: virtual UInt64 getSize() const = 0; /// Get i-th disk where reservation take place. - virtual DiskPtr getDisk(size_t i = 0) const = 0; + virtual DiskPtr getDisk(size_t i = 0) const = 0; /// NOLINT /// Get all disks, used in reservation virtual Disks getDisks() const = 0; diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index 05aa4d3350b..fa4189abc53 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -13,6 +12,7 @@ #include #include #include +#include namespace DB @@ -24,23 +24,64 @@ namespace ErrorCodes extern const int UNKNOWN_FORMAT; extern const int FILE_ALREADY_EXISTS; extern const int PATH_ACCESS_DENIED;; - extern const int CANNOT_DELETE_DIRECTORY; + extern const int FILE_DOESNT_EXIST; + extern const int BAD_FILE_TYPE; + extern const int MEMORY_LIMIT_EXCEEDED; } -/// Load metadata by path or create empty if `create` flag is set. -IDiskRemote::Metadata::Metadata( - const String & remote_fs_root_path_, - DiskPtr metadata_disk_, - const String & metadata_file_path_, - bool create) - : RemoteMetadata(remote_fs_root_path_, metadata_file_path_) - , metadata_disk(metadata_disk_) - , total_size(0), ref_count(0) +IDiskRemote::Metadata IDiskRemote::Metadata::readMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_) { - if (create) - return; + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + result.load(); + return result; +} + +IDiskRemote::Metadata IDiskRemote::Metadata::createAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync) +{ + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + result.save(sync); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::readUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, IDiskRemote::MetadataUpdater updater) +{ + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + result.load(); + if (updater(result)) + result.save(sync); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::createUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, IDiskRemote::MetadataUpdater updater) +{ + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + updater(result); + result.save(sync); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::createAndStoreMetadataIfNotExists(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, bool overwrite) +{ + if (overwrite || !metadata_disk_->exists(metadata_file_path_)) + { + return createAndStoreMetadata(remote_fs_root_path_, metadata_disk_, metadata_file_path_, sync); + } + else + { + auto result = readMetadata(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + if (result.read_only) + throw Exception("File is read-only: " + metadata_file_path_, ErrorCodes::PATH_ACCESS_DENIED); + return result; + } +} + +void IDiskRemote::Metadata::load() +{ try { const ReadSettings read_settings; @@ -98,107 +139,174 @@ IDiskRemote::Metadata::Metadata( if (e.code() == ErrorCodes::UNKNOWN_FORMAT) throw; + if (e.code() == ErrorCodes::MEMORY_LIMIT_EXCEEDED) + throw; + throw Exception("Failed to read metadata file", e, ErrorCodes::UNKNOWN_FORMAT); } } +/// Load metadata by path or create empty if `create` flag is set. +IDiskRemote::Metadata::Metadata( + const String & remote_fs_root_path_, + DiskPtr metadata_disk_, + const String & metadata_file_path_) + : RemoteMetadata(remote_fs_root_path_, metadata_file_path_) + , metadata_disk(metadata_disk_) + , total_size(0), ref_count(0) +{ +} + void IDiskRemote::Metadata::addObject(const String & path, size_t size) { total_size += size; remote_fs_objects.emplace_back(path, size); } + +void IDiskRemote::Metadata::saveToBuffer(WriteBuffer & buf, bool sync) +{ + writeIntText(VERSION_RELATIVE_PATHS, buf); + writeChar('\n', buf); + + writeIntText(remote_fs_objects.size(), buf); + writeChar('\t', buf); + writeIntText(total_size, buf); + writeChar('\n', buf); + + for (const auto & [remote_fs_object_path, remote_fs_object_size] : remote_fs_objects) + { + writeIntText(remote_fs_object_size, buf); + writeChar('\t', buf); + writeEscapedString(remote_fs_object_path, buf); + writeChar('\n', buf); + } + + writeIntText(ref_count, buf); + writeChar('\n', buf); + + writeBoolText(read_only, buf); + writeChar('\n', buf); + + buf.finalize(); + if (sync) + buf.sync(); + +} + /// Fsync metadata file if 'sync' flag is set. void IDiskRemote::Metadata::save(bool sync) { auto buf = metadata_disk->writeFile(metadata_file_path, 1024); + saveToBuffer(*buf, sync); +} - writeIntText(VERSION_RELATIVE_PATHS, *buf); - writeChar('\n', *buf); +std::string IDiskRemote::Metadata::serializeToString() +{ + WriteBufferFromOwnString write_buf; + saveToBuffer(write_buf, false); + return write_buf.str(); +} - writeIntText(remote_fs_objects.size(), *buf); - writeChar('\t', *buf); - writeIntText(total_size, *buf); - writeChar('\n', *buf); +IDiskRemote::Metadata IDiskRemote::readMetadataUnlocked(const String & path, std::shared_lock &) const +{ + return Metadata::readMetadata(remote_fs_root_path, metadata_disk, path); +} - for (const auto & [remote_fs_object_path, remote_fs_object_size] : remote_fs_objects) + +IDiskRemote::Metadata IDiskRemote::readMetadata(const String & path) const +{ + std::shared_lock lock(metadata_mutex); + return readMetadataUnlocked(path, lock); +} + +IDiskRemote::Metadata IDiskRemote::readUpdateAndStoreMetadata(const String & path, bool sync, IDiskRemote::MetadataUpdater updater) +{ + std::unique_lock lock(metadata_mutex); + return Metadata::readUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); +} + + +IDiskRemote::Metadata IDiskRemote::readOrCreateUpdateAndStoreMetadata(const String & path, WriteMode mode, bool sync, IDiskRemote::MetadataUpdater updater) +{ + if (mode == WriteMode::Rewrite || !metadata_disk->exists(path)) { - writeIntText(remote_fs_object_size, *buf); - writeChar('\t', *buf); - writeEscapedString(remote_fs_object_path, *buf); - writeChar('\n', *buf); + std::unique_lock lock(metadata_mutex); + return Metadata::createUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); + } + else + { + return Metadata::readUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); + } +} + +IDiskRemote::Metadata IDiskRemote::createAndStoreMetadata(const String & path, bool sync) +{ + return Metadata::createAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync); +} + +IDiskRemote::Metadata IDiskRemote::createUpdateAndStoreMetadata(const String & path, bool sync, IDiskRemote::MetadataUpdater updater) +{ + return Metadata::createUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); +} + + +std::unordered_map IDiskRemote::getSerializedMetadata(const std::vector & file_paths) const +{ + std::unordered_map metadatas; + + std::shared_lock lock(metadata_mutex); + + for (const auto & path : file_paths) + { + IDiskRemote::Metadata metadata = readMetadataUnlocked(path, lock); + metadata.ref_count = 0; + metadatas[path] = metadata.serializeToString(); } - writeIntText(ref_count, *buf); - writeChar('\n', *buf); - - writeBoolText(read_only, *buf); - writeChar('\n', *buf); - - buf->finalize(); - if (sync) - buf->sync(); + return metadatas; } -IDiskRemote::Metadata IDiskRemote::readOrCreateMetaForWriting(const String & path, WriteMode mode) -{ - bool exist = exists(path); - if (exist) - { - auto metadata = readMeta(path); - if (metadata.read_only) - throw Exception("File is read-only: " + path, ErrorCodes::PATH_ACCESS_DENIED); - - if (mode == WriteMode::Rewrite) - removeFile(path); /// Remove for re-write. - else - return metadata; - } - - auto metadata = createMeta(path); - /// Save empty metadata to disk to have ability to get file size while buffer is not finalized. - metadata.save(); - - return metadata; -} - - -IDiskRemote::Metadata IDiskRemote::readMeta(const String & path) const -{ - return Metadata(remote_fs_root_path, metadata_disk, path); -} - - -IDiskRemote::Metadata IDiskRemote::createMeta(const String & path) const -{ - return Metadata(remote_fs_root_path, metadata_disk, path, true); -} - - -void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) +void IDiskRemote::removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) { LOG_TRACE(log, "Remove file by path: {}", backQuote(metadata_disk->getPath() + path)); + if (!metadata_disk->exists(path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Metadata path '{}' doesn't exist", path); + if (!metadata_disk->isFile(path)) - throw Exception(ErrorCodes::CANNOT_DELETE_DIRECTORY, "Path '{}' is a directory", path); + throw Exception(ErrorCodes::BAD_FILE_TYPE, "Path '{}' is not a regular file", path); try { - auto metadata = readMeta(path); + auto metadata_updater = [fs_paths_keeper, this] (Metadata & metadata) + { + if (metadata.ref_count == 0) + { + for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) + { + fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); + if (cache) + { + auto key = cache->hash(remote_fs_object_path); + cache->remove(key); + } + } + + return false; + } + else /// In other case decrement number of references, save metadata and delete hardlink. + { + --metadata.ref_count; + } + + return true; + }; + + readUpdateAndStoreMetadata(path, false, metadata_updater); + metadata_disk->removeFile(path); /// If there is no references - delete content from remote FS. - if (metadata.ref_count == 0) - { - metadata_disk->removeFile(path); - for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) - fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); - } - else /// In other case decrement number of references, save metadata and delete file. - { - --metadata.ref_count; - metadata.save(); - metadata_disk->removeFile(path); - } } catch (const Exception & e) { @@ -216,18 +324,19 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths } -void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) +void IDiskRemote::removeMetadataRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. if (metadata_disk->isFile(path)) { - removeMeta(path, fs_paths_keeper); + removeMetadata(path, fs_paths_keeper); } else { - for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) - removeMetaRecursive(it->path(), fs_paths_keeper); + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + removeMetadataRecursive(it->path(), fs_paths_keeper); + metadata_disk->removeDirectory(path); } } @@ -280,6 +389,7 @@ IDiskRemote::IDiskRemote( const String & name_, const String & remote_fs_root_path_, DiskPtr metadata_disk_, + FileCachePtr cache_, const String & log_name_, size_t thread_pool_size) : IDisk(std::make_unique(log_name_, thread_pool_size)) @@ -287,6 +397,7 @@ IDiskRemote::IDiskRemote( , name(name_) , remote_fs_root_path(remote_fs_root_path_) , metadata_disk(metadata_disk_) + , cache(cache_) { } @@ -305,16 +416,13 @@ bool IDiskRemote::isFile(const String & path) const void IDiskRemote::createFile(const String & path) { - /// Create empty metadata file. - auto metadata = createMeta(path); - metadata.save(); + createAndStoreMetadata(path, false); } size_t IDiskRemote::getFileSize(const String & path) const { - auto metadata = readMeta(path); - return metadata.total_size; + return readMetadata(path).total_size; } @@ -341,45 +449,48 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path) } -void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs) +void IDiskRemote::removeSharedFile(const String & path, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - removeMeta(path, fs_paths_keeper); - if (!keep_in_remote_fs) + removeMetadata(path, fs_paths_keeper); + + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } -void IDiskRemote::removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) +void IDiskRemote::removeSharedFileIfExists(const String & path, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); + if (metadata_disk->exists(path)) { - removeMeta(path, fs_paths_keeper); - if (!keep_in_remote_fs) + removeMetadata(path, fs_paths_keeper); + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } } -void IDiskRemote::removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) +void IDiskRemote::removeSharedFiles(const RemoveBatchRequest & files, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); for (const auto & file : files) { bool skip = file.if_exists && !metadata_disk->exists(file.path); if (!skip) - removeMeta(file.path, fs_paths_keeper); + removeMetadata(file.path, fs_paths_keeper); } - if (!keep_in_remote_fs) + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } -void IDiskRemote::removeSharedRecursive(const String & path, bool keep_in_remote_fs) +void IDiskRemote::removeSharedRecursive(const String & path, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - removeMetaRecursive(path, fs_paths_keeper); - if (!keep_in_remote_fs) + removeMetadataRecursive(path, fs_paths_keeper); + + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } @@ -388,9 +499,7 @@ void IDiskRemote::setReadOnly(const String & path) { /// We should store read only flag inside metadata file (instead of using FS flag), /// because we modify metadata file when create hard-links from it. - auto metadata = readMeta(path); - metadata.read_only = true; - metadata.save(); + readUpdateAndStoreMetadata(path, false, [] (Metadata & metadata) { metadata.read_only = true; return true; }); } @@ -414,7 +523,7 @@ void IDiskRemote::createDirectories(const String & path) void IDiskRemote::clearDirectory(const String & path) { - for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) + for (auto it = iterateDirectory(path); it->isValid(); it->next()) if (isFile(it->path())) removeFile(it->path()); } @@ -453,10 +562,7 @@ Poco::Timestamp IDiskRemote::getLastModified(const String & path) void IDiskRemote::createHardLink(const String & src_path, const String & dst_path) { - /// Increment number of references. - auto src = readMeta(src_path); - ++src.ref_count; - src.save(); + readUpdateAndStoreMetadata(src_path, false, [] (Metadata & metadata) { metadata.ref_count++; return true; }); /// Create FS hardlink to metadata file. metadata_disk->createHardLink(src_path, dst_path); @@ -498,7 +604,7 @@ bool IDiskRemote::tryReserve(UInt64 bytes) String IDiskRemote::getUniqueId(const String & path) const { LOG_TRACE(log, "Remote path: {}, Path: {}", remote_fs_root_path, path); - Metadata metadata(remote_fs_root_path, metadata_disk, path); + auto metadata = readMetadata(path); String id; if (!metadata.remote_fs_objects.empty()) id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first; @@ -514,34 +620,9 @@ AsynchronousReaderPtr IDiskRemote::getThreadPoolReader() return reader; } -std::unique_ptr IDiskRemote::readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const -{ - LOG_TRACE(log, "Read metafile: {}", path); - return metadata_disk->readFile(path, settings, size); -} - -std::unique_ptr IDiskRemote::writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) -{ - LOG_TRACE(log, "Write metafile: {}", path); - return metadata_disk->writeFile(path, buf_size, mode); -} - -void IDiskRemote::removeMetaFileIfExists(const String & path) -{ - LOG_TRACE(log, "Remove metafile: {}", path); - return metadata_disk->removeFileIfExists(path); -} - UInt32 IDiskRemote::getRefCount(const String & path) const { - auto meta = readMeta(path); - return meta.ref_count; + return readMetadata(path).ref_count; } ThreadPool & IDiskRemote::getThreadPoolWriter() diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index 8bb93cc345d..82e76b8f68d 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -3,14 +3,16 @@ #include #include +#include #include #include #include +#include +#include #include #include #include -namespace fs = std::filesystem; namespace CurrentMetrics { @@ -25,7 +27,7 @@ namespace DB class RemoteFSPathKeeper { public: - RemoteFSPathKeeper(size_t chunk_limit_) : chunk_limit(chunk_limit_) {} + explicit RemoteFSPathKeeper(size_t chunk_limit_) : chunk_limit(chunk_limit_) {} virtual ~RemoteFSPathKeeper() = default; @@ -53,20 +55,28 @@ public: const String & name_, const String & remote_fs_root_path_, DiskPtr metadata_disk_, + FileCachePtr cache_, const String & log_name_, size_t thread_pool_size); struct Metadata; + using MetadataUpdater = std::function; const String & getName() const final override { return name; } const String & getPath() const final override { return metadata_disk->getPath(); } - Metadata readMeta(const String & path) const; + /// Methods for working with metadata. For some operations (like hardlink + /// creation) metadata can be updated concurrently from multiple threads + /// (file actually rewritten on disk). So additional RW lock is required for + /// metadata read and write, but not for create new metadata. + Metadata readMetadata(const String & path) const; + Metadata readMetadataUnlocked(const String & path, std::shared_lock &) const; + Metadata readUpdateAndStoreMetadata(const String & path, bool sync, MetadataUpdater updater); + Metadata readOrCreateUpdateAndStoreMetadata(const String & path, WriteMode mode, bool sync, MetadataUpdater updater); - Metadata createMeta(const String & path) const; - - Metadata readOrCreateMetaForWriting(const String & path, WriteMode mode); + Metadata createAndStoreMetadata(const String & path, bool sync); + Metadata createUpdateAndStoreMetadata(const String & path, bool sync, MetadataUpdater updater); UInt64 getTotalSpace() const override { return std::numeric_limits::max(); } @@ -94,13 +104,13 @@ public: void removeRecursive(const String & path) override { removeSharedRecursive(path, false); } - void removeSharedFile(const String & path, bool keep_in_remote_fs) override; + void removeSharedFile(const String & path, bool delete_metadata_only) override; - void removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) override; + void removeSharedFileIfExists(const String & path, bool delete_metadata_only) override; - void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool delete_metadata_only) override; - void removeSharedRecursive(const String & path, bool keep_in_remote_fs) override; + void removeSharedRecursive(const String & path, bool delete_metadata_only) override; void listFiles(const String & path, std::vector & file_names) override; @@ -139,38 +149,33 @@ public: static AsynchronousReaderPtr getThreadPoolReader(); static ThreadPool & getThreadPoolWriter(); - virtual std::unique_ptr readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const override; - - virtual std::unique_ptr writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) override; - - virtual void removeMetaFileIfExists( - const String & path) override; + DiskPtr getMetadataDiskIfExistsOrSelf() override { return metadata_disk; } UInt32 getRefCount(const String & path) const override; + /// Return metadata for each file path. Also, before serialization reset + /// ref_count for each metadata to zero. This function used only for remote + /// fetches/sends in replicated engines. That's why we reset ref_count to zero. + std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override; protected: Poco::Logger * log; const String name; const String remote_fs_root_path; DiskPtr metadata_disk; + FileCachePtr cache; private: - void removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); + void removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); - void removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); + void removeMetadataRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); bool tryReserve(UInt64 bytes); UInt64 reserved_bytes = 0; UInt64 reservation_count = 0; std::mutex reservation_mutex; + mutable std::shared_mutex metadata_mutex; }; using RemoteDiskPtr = std::shared_ptr; @@ -200,6 +205,7 @@ struct RemoteMetadata struct IDiskRemote::Metadata : RemoteMetadata { + using Updater = std::function; /// Metadata file version. static constexpr UInt32 VERSION_ABSOLUTE_PATHS = 1; static constexpr UInt32 VERSION_RELATIVE_PATHS = 2; @@ -211,22 +217,36 @@ struct IDiskRemote::Metadata : RemoteMetadata size_t total_size = 0; /// Number of references (hardlinks) to this metadata file. + /// + /// FIXME: Why we are tracking it explicetly, without + /// info from filesystem???? UInt32 ref_count = 0; /// Flag indicates that file is read only. bool read_only = false; - /// Load metadata by path or create empty if `create` flag is set. - Metadata(const String & remote_fs_root_path_, - DiskPtr metadata_disk_, - const String & metadata_file_path_, - bool create = false); + Metadata( + const String & remote_fs_root_path_, + DiskPtr metadata_disk_, + const String & metadata_file_path_); void addObject(const String & path, size_t size); + static Metadata readMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_); + static Metadata readUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, Updater updater); + + static Metadata createAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync); + static Metadata createUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, Updater updater); + static Metadata createAndStoreMetadataIfNotExists(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, bool overwrite); + + /// Serialize metadata to string (very same with saveToBuffer) + std::string serializeToString(); + +private: /// Fsync metadata file if 'sync' flag is set. void save(bool sync = false); - + void saveToBuffer(WriteBuffer & buffer, bool sync); + void load(); }; class DiskRemoteReservation final : public IReservation diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index 184fcfe6f8c..e693a8e9ea8 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -32,7 +32,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int ARGUMENT_OUT_OF_BOUND; } @@ -48,6 +48,11 @@ AsynchronousReadIndirectBufferFromRemoteFS::AsynchronousReadIndirectBufferFromRe , prefetch_buffer(settings_.remote_fs_buffer_size) , min_bytes_for_seek(min_bytes_for_seek_) , must_read_until_position(settings_.must_read_until_position) +#ifndef NDEBUG + , log(&Poco::Logger::get("AsynchronousBufferFromRemoteFS")) +#else + , log(&Poco::Logger::get("AsyncBuffer(" + impl->getFileName() + ")")) +#endif { ProfileEvents::increment(ProfileEvents::RemoteFSBuffers); } @@ -59,6 +64,12 @@ String AsynchronousReadIndirectBufferFromRemoteFS::getFileName() const } +String AsynchronousReadIndirectBufferFromRemoteFS::getInfoForLog() +{ + return impl->getInfoForLog(); +} + + bool AsynchronousReadIndirectBufferFromRemoteFS::hasPendingDataToRead() { /** @@ -76,8 +87,8 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::hasPendingDataToRead() return false; if (file_offset_of_buffer_end > *read_until_position) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Read beyond last offset ({} > {})", - file_offset_of_buffer_end, *read_until_position); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Read beyond last offset ({} > {}, info: {})", + file_offset_of_buffer_end, *read_until_position, impl->getInfoForLog()); } else if (must_read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, @@ -125,8 +136,11 @@ void AsynchronousReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t pos if (prefetch_future.valid()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Prefetch is valid in readUntilPosition"); - read_until_position = position; - impl->setReadUntilPosition(*read_until_position); + if (position > read_until_position) + { + read_until_position = position; + impl->setReadUntilPosition(*read_until_position); + } } @@ -146,125 +160,134 @@ bool AsynchronousReadIndirectBufferFromRemoteFS::nextImpl() return false; size_t size = 0; - if (prefetch_future.valid()) { ProfileEvents::increment(ProfileEvents::RemoteFSPrefetchedReads); - CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; - Stopwatch watch; + size_t offset = 0; { + Stopwatch watch; + CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; auto result = prefetch_future.get(); size = result.size; - auto offset = result.offset; - assert(offset < size); + offset = result.offset; + LOG_TEST(log, "Current size: {}, offset: {}", size, offset); - if (size) - { - memory.swap(prefetch_buffer); - size -= offset; - set(memory.data() + offset, size); - working_buffer.resize(size); - file_offset_of_buffer_end += size; - } + /// If prefetch_future is valid, size should always be greater than zero. + assert(offset < size); + ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); } - watch.stop(); - ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); + prefetch_buffer.swap(memory); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); } else { ProfileEvents::increment(ProfileEvents::RemoteFSUnprefetchedReads); + auto result = readInto(memory.data(), memory.size()).get(); size = result.size; auto offset = result.offset; + + LOG_TEST(log, "Current size: {}, offset: {}", size, offset); assert(offset < size); if (size) { - size -= offset; - set(memory.data() + offset, size); - working_buffer.resize(size); - file_offset_of_buffer_end += size; + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); } } - if (file_offset_of_buffer_end != impl->offset()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected equality {} == {}. It's a bug", file_offset_of_buffer_end, impl->offset()); + file_offset_of_buffer_end = impl->getFileOffsetOfBufferEnd(); + assert(file_offset_of_buffer_end == impl->getImplementationBufferOffset()); prefetch_future = {}; return size; } -off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) +off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset, int whence) { ProfileEvents::increment(ProfileEvents::RemoteFSSeeks); - if (whence == SEEK_CUR) + size_t new_pos; + if (whence == SEEK_SET) { - /// If position within current working buffer - shift pos. - if (!working_buffer.empty() && static_cast(getPosition() + offset_) < file_offset_of_buffer_end) - { - pos += offset_; - return getPosition(); - } - else - { - file_offset_of_buffer_end += offset_; - } + assert(offset >= 0); + new_pos = offset; } - else if (whence == SEEK_SET) + else if (whence == SEEK_CUR) { - /// If position is within current working buffer - shift pos. - if (!working_buffer.empty() - && static_cast(offset_) >= file_offset_of_buffer_end - working_buffer.size() - && size_t(offset_) < file_offset_of_buffer_end) - { - pos = working_buffer.end() - (file_offset_of_buffer_end - offset_); + new_pos = file_offset_of_buffer_end - (working_buffer.end() - pos) + offset; + } + else + { + throw Exception("ReadBufferFromFileDescriptor::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } + /// Position is unchanged. + if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end) + return new_pos; + + bool read_from_prefetch = false; + while (true) + { + if (file_offset_of_buffer_end - working_buffer.size() <= new_pos && new_pos <= file_offset_of_buffer_end) + { + /// Position is still inside the buffer. + /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. + + pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; assert(pos >= working_buffer.begin()); assert(pos <= working_buffer.end()); - return getPosition(); + return new_pos; } - else + else if (prefetch_future.valid()) { - file_offset_of_buffer_end = offset_; + /// Read from prefetch buffer and recheck if the new position is valid inside. + + if (nextImpl()) + { + read_from_prefetch = true; + continue; + } } - } - else - throw Exception("Only SEEK_SET or SEEK_CUR modes are allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - if (prefetch_future.valid()) - { - ProfileEvents::increment(ProfileEvents::RemoteFSCancelledPrefetches); - prefetch_future.wait(); - prefetch_future = {}; + /// Prefetch is cancelled because of seek. + if (read_from_prefetch) + ProfileEvents::increment(ProfileEvents::RemoteFSCancelledPrefetches); + + break; } + assert(!prefetch_future.valid()); + + /// First reset the buffer so the next read will fetch new data to the buffer. resetWorkingBuffer(); /** * Lazy ignore. Save number of bytes to ignore and ignore it either for prefetch buffer or current buffer. * Note: we read in range [file_offset_of_buffer_end, read_until_position). */ - off_t file_offset_before_seek = impl->offset(); if (impl->initialized() - && read_until_position && file_offset_of_buffer_end < *read_until_position - && static_cast(file_offset_of_buffer_end) > file_offset_before_seek - && static_cast(file_offset_of_buffer_end) < file_offset_before_seek + static_cast(min_bytes_for_seek)) + && read_until_position && new_pos < *read_until_position + && new_pos > file_offset_of_buffer_end + && new_pos < file_offset_of_buffer_end + min_bytes_for_seek) { ProfileEvents::increment(ProfileEvents::RemoteFSLazySeeks); - bytes_to_ignore = file_offset_of_buffer_end - file_offset_before_seek; + bytes_to_ignore = new_pos - file_offset_of_buffer_end; } else { ProfileEvents::increment(ProfileEvents::RemoteFSSeeksWithReset); impl->reset(); + file_offset_of_buffer_end = new_pos; } - return file_offset_of_buffer_end; + return new_pos; } diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h index c9b81c98e61..48c4ff3b4f0 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.h @@ -5,6 +5,7 @@ #include #include +namespace Poco { class Logger; } namespace DB { @@ -44,10 +45,12 @@ public: void prefetch() override; - void setReadUntilPosition(size_t position) override; + void setReadUntilPosition(size_t position) override; /// [..., position). void setReadUntilEnd() override; + String getInfoForLog() override; + private: bool nextImpl() override; @@ -76,6 +79,8 @@ private: std::optional read_until_position; bool must_read_until_position; + + Poco::Logger * log; }; } diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp new file mode 100644 index 00000000000..5cab2cb2995 --- /dev/null +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.cpp @@ -0,0 +1,763 @@ +#include "CachedReadBufferFromRemoteFS.h" + +#include +#include +#include +#include +#include + + +namespace ProfileEvents +{ + extern const Event RemoteFSReadBytes; + extern const Event RemoteFSCacheReadBytes; + extern const Event RemoteFSCacheDownloadBytes; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int LOGICAL_ERROR; +} + +CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS( + const String & remote_fs_object_path_, + FileCachePtr cache_, + RemoteFSFileReaderCreator remote_file_reader_creator_, + const ReadSettings & settings_, + size_t read_until_position_) + : SeekableReadBuffer(nullptr, 0) +#ifndef NDEBUG + , log(&Poco::Logger::get("CachedReadBufferFromRemoteFS(" + remote_fs_object_path_ + ")")) +#else + , log(&Poco::Logger::get("CachedReadBufferFromRemoteFS")) +#endif + , cache_key(cache_->hash(remote_fs_object_path_)) + , remote_fs_object_path(remote_fs_object_path_) + , cache(cache_) + , settings(settings_) + , read_until_position(read_until_position_) + , remote_file_reader_creator(remote_file_reader_creator_) +{ +} + +void CachedReadBufferFromRemoteFS::initialize(size_t offset, size_t size) +{ + file_segments_holder.emplace(cache->getOrSet(cache_key, offset, size)); + + /** + * Segments in returned list are ordered in ascending order and represent a full contiguous + * interval (no holes). Each segment in returned list has state: DOWNLOADED, DOWNLOADING or EMPTY. + */ + if (file_segments_holder->file_segments.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "List of file segments cannot be empty"); + + LOG_TEST(log, "Having {} file segments to read", file_segments_holder->file_segments.size()); + current_file_segment_it = file_segments_holder->file_segments.begin(); + + initialized = true; +} + +SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getCacheReadBuffer(size_t offset) const +{ + return std::make_shared(cache->getPathInLocalCache(cache_key, offset), settings.local_fs_buffer_size); +} + +SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getRemoteFSReadBuffer(FileSegmentPtr & file_segment, ReadType read_type_) +{ + switch (read_type_) + { + case ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE: + { + /** + * Each downloader is elected to download at most buffer_size bytes and then any other can + * continue. The one who continues download should reuse download buffer. + * + * TODO: Also implementation (s3, hdfs, web) buffer might be passed through file segments. + * E.g. consider for query1 and query2 we need intersecting ranges like this: + * + * [___________] -- read_range_1 for query1 + * [_______________] -- read_range_2 for query2 + * ^___________^______^ + * | segment1 | segment2 + * + * So query2 can reuse implementation buffer, which downloaded segment1. + * Implementation buffer from segment1 is passed to segment2 once segment1 is loaded. + */ + + auto remote_fs_segment_reader = file_segment->getRemoteFileReader(); + + if (remote_fs_segment_reader) + return remote_fs_segment_reader; + + remote_fs_segment_reader = remote_file_reader_creator(); + file_segment->setRemoteFileReader(remote_fs_segment_reader); + + ///TODO: add check for pending data + return remote_fs_segment_reader; + } + case ReadType::REMOTE_FS_READ_BYPASS_CACHE: + { + /// Result buffer is owned only by current buffer -- not shareable like in the case above. + + if (remote_file_reader && remote_file_reader->getFileOffsetOfBufferEnd() == file_offset_of_buffer_end) + return remote_file_reader; + + remote_file_reader = remote_file_reader_creator(); + return remote_file_reader; + } + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot use remote filesystem reader with read type: {}", toString(read_type)); + } +} + +SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getReadBufferForFileSegment(FileSegmentPtr & file_segment) +{ + auto range = file_segment->range(); + + /// Each wait() call has a timeout of 1 second. + size_t wait_download_max_tries = settings.remote_fs_cache_max_wait_sec; + size_t wait_download_tries = 0; + + auto download_state = file_segment->state(); + while (true) + { + switch (download_state) + { + case FileSegment::State::SKIP_CACHE: + { + read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; + return getRemoteFSReadBuffer(file_segment, read_type); + } + case FileSegment::State::EMPTY: + { + auto downloader_id = file_segment->getOrSetDownloader(); + if (downloader_id == file_segment->getCallerId()) + { + if (file_offset_of_buffer_end == file_segment->getDownloadOffset()) + { + read_type = ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + return getRemoteFSReadBuffer(file_segment, read_type); + } + else + { + /// segment{k} + /// cache: [______|___________ + /// ^ + /// download_offset + /// requested_range: [__________] + /// ^ + /// file_offset_of_buffer_end + assert(file_offset_of_buffer_end > file_segment->getDownloadOffset()); + bytes_to_predownload = file_offset_of_buffer_end - file_segment->getDownloadOffset(); + + read_type = ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + return getRemoteFSReadBuffer(file_segment, read_type); + } + } + else + { + download_state = file_segment->state(); + continue; + } + } + case FileSegment::State::DOWNLOADING: + { + size_t download_offset = file_segment->getDownloadOffset(); + bool can_start_from_cache = download_offset > file_offset_of_buffer_end; + + /// If file segment is being downloaded but we can already read from already downloaded part, do that. + if (can_start_from_cache) + { + /// segment{k} state: DOWNLOADING + /// cache: [______|___________ + /// ^ + /// download_offset (in progress) + /// requested_range: [__________] + /// ^ + /// file_offset_of_buffer_end + + read_type = ReadType::CACHED; + return getCacheReadBuffer(range.left); + } + + if (wait_download_tries++ < wait_download_max_tries) + { + download_state = file_segment->wait(); + } + else + { + download_state = FileSegment::State::SKIP_CACHE; + } + + continue; + } + case FileSegment::State::DOWNLOADED: + { + read_type = ReadType::CACHED; + return getCacheReadBuffer(range.left); + } + case FileSegment::State::PARTIALLY_DOWNLOADED: + { + auto downloader_id = file_segment->getOrSetDownloader(); + if (downloader_id == file_segment->getCallerId()) + { + size_t download_offset = file_segment->getDownloadOffset(); + bool can_start_from_cache = download_offset > file_offset_of_buffer_end; + + LOG_TEST(log, "Current download offset: {}, file offset of buffer end: {}", download_offset, file_offset_of_buffer_end); + + if (can_start_from_cache) + { + /// segment{k} + /// cache: [______|___________ + /// ^ + /// download_offset + /// requested_range: [__________] + /// ^ + /// file_offset_of_buffer_end + + read_type = ReadType::CACHED; + file_segment->resetDownloader(); + return getCacheReadBuffer(range.left); + } + + if (download_offset < file_offset_of_buffer_end) + { + /// segment{1} + /// cache: [_____|___________ + /// ^ + /// download_offset + /// requested_range: [__________] + /// ^ + /// file_offset_of_buffer_end + + assert(file_offset_of_buffer_end > file_segment->getDownloadOffset()); + bytes_to_predownload = file_offset_of_buffer_end - file_segment->getDownloadOffset(); + } + + download_offset = file_segment->getDownloadOffset(); + can_start_from_cache = download_offset > file_offset_of_buffer_end; + assert(!can_start_from_cache); + + read_type = ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + return getRemoteFSReadBuffer(file_segment, read_type); + } + + download_state = file_segment->state(); + continue; + } + case FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION: + { + size_t download_offset = file_segment->getDownloadOffset(); + bool can_start_from_cache = download_offset > file_offset_of_buffer_end; + + if (can_start_from_cache) + { + read_type = ReadType::CACHED; + return getCacheReadBuffer(range.left); + } + else + { + read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; + return getRemoteFSReadBuffer(file_segment, read_type); + } + } + } + } +} + +SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(FileSegmentPtr & file_segment) +{ + assert(!file_segment->isDownloader()); + assert(file_offset_of_buffer_end >= file_segment->range().left); + + auto range = file_segment->range(); + bytes_to_predownload = 0; + + auto read_buffer_for_file_segment = getReadBufferForFileSegment(file_segment); + + [[maybe_unused]] auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + assert(download_current_segment == file_segment->isDownloader()); + + assert(file_segment->range() == range); + assert(file_offset_of_buffer_end >= range.left && file_offset_of_buffer_end <= range.right); + + LOG_TEST(log, "Current file segment: {}, read type: {}, current file offset: {}", + range.toString(), toString(read_type), file_offset_of_buffer_end); + + read_buffer_for_file_segment->setReadUntilPosition(range.right + 1); /// [..., range.right] + + switch (read_type) + { + case ReadType::CACHED: + { + size_t seek_offset = file_offset_of_buffer_end - range.left; + read_buffer_for_file_segment->seek(seek_offset, SEEK_SET); + + auto * file_reader = assert_cast(read_buffer_for_file_segment.get()); + size_t file_size = file_reader->size(); + auto state = file_segment->state(); + + LOG_TEST(log, "Cache file: {}. Cached seek to: {}, file size: {}, file segment state: {}, download offset: {}", + file_reader->getFileName(), seek_offset, file_size, state, file_segment->getDownloadOffset()); + + assert(file_size > 0); + break; + } + case ReadType::REMOTE_FS_READ_BYPASS_CACHE: + { + read_buffer_for_file_segment->seek(file_offset_of_buffer_end, SEEK_SET); + break; + } + case ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE: + { + assert(file_segment->isDownloader()); + + if (bytes_to_predownload) + { + size_t download_offset = file_segment->getDownloadOffset(); + read_buffer_for_file_segment->seek(download_offset, SEEK_SET); + } + else + { + read_buffer_for_file_segment->seek(file_offset_of_buffer_end, SEEK_SET); + } + + auto impl_range = read_buffer_for_file_segment->getRemainingReadRange(); + auto download_offset = file_segment->getDownloadOffset(); + if (download_offset != static_cast(read_buffer_for_file_segment->getPosition())) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Buffer's offsets mismatch; cached buffer offset: {}, download_offset: {}, position: {}, implementation buffer offset: {}, " + "implementation buffer reading until: {}, file segment info: {}", + file_offset_of_buffer_end, download_offset, read_buffer_for_file_segment->getPosition(), + impl_range.left, *impl_range.right, file_segment->getInfoForLog()); + + break; + } + } + + return read_buffer_for_file_segment; +} + +bool CachedReadBufferFromRemoteFS::completeFileSegmentAndGetNext() +{ + LOG_TEST(log, "Completed segment: {}", (*current_file_segment_it)->range().toString()); + + auto file_segment_it = current_file_segment_it++; + auto & file_segment = *file_segment_it; + + [[maybe_unused]] const auto & range = file_segment->range(); + assert(file_offset_of_buffer_end > range.right); + + LOG_TEST(log, "Removing file segment: {}, downloader: {}, state: {}", + file_segment->range().toString(), file_segment->getDownloader(), file_segment->state()); + + /// Do not hold pointer to file segment if it is not needed anymore + /// so can become releasable and can be evicted from cache. + file_segments_holder->file_segments.erase(file_segment_it); + + if (current_file_segment_it == file_segments_holder->file_segments.end()) + return false; + + implementation_buffer = getImplementationBuffer(*current_file_segment_it); + + LOG_TEST(log, "New segment: {}", (*current_file_segment_it)->range().toString()); + return true; +} + +void CachedReadBufferFromRemoteFS::predownload(FileSegmentPtr & file_segment) +{ + if (bytes_to_predownload) + { + /// Consider this case. Some user needed segment [a, b] and downloaded it partially. + /// But before he called complete(state) or his holder called complete(), + /// some other user, who needed segment [a', b'], a < a' < b', started waiting on [a, b] to be + /// downloaded because it intersects with the range he needs. + /// But then first downloader fails and second must continue. In this case we need to + /// download from offset a'' < a', but return buffer from offset a'. + LOG_TEST(log, "Bytes to predownload: {}, caller_id: {}", bytes_to_predownload, FileSegment::getCallerId()); + + assert(implementation_buffer->getFileOffsetOfBufferEnd() == file_segment->getDownloadOffset()); + + while (true) + { + if (!bytes_to_predownload || implementation_buffer->eof()) + { + if (bytes_to_predownload) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Failed to predownload remaining {} bytes. Current file segment: {}, current download offset: {}, expected: {}, eof: {}", + file_segment->range().toString(), file_segment->getDownloadOffset(), file_offset_of_buffer_end, implementation_buffer->eof()); + + auto result = implementation_buffer->hasPendingData(); + + if (result) + { + nextimpl_working_buffer_offset = implementation_buffer->offset(); + + auto download_offset = file_segment->getDownloadOffset(); + if (download_offset != static_cast(implementation_buffer->getPosition()) || download_offset != file_offset_of_buffer_end) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Buffer's offsets mismatch after predownloading; download offset: {}, cached buffer offset: {}, implementation buffer offset: {}, " + "file segment info: {}", download_offset, file_offset_of_buffer_end, implementation_buffer->getPosition(), file_segment->getInfoForLog()); + } + + break; + } + + size_t current_predownload_size = std::min(implementation_buffer->buffer().size(), bytes_to_predownload); + + if (file_segment->reserve(current_predownload_size)) + { + LOG_TEST(log, "Left to predownload: {}, buffer size: {}", bytes_to_predownload, implementation_buffer->buffer().size()); + + file_segment->write(implementation_buffer->buffer().begin(), current_predownload_size); + + bytes_to_predownload -= current_predownload_size; + implementation_buffer->position() += current_predownload_size; + } + else + { + /// We were predownloading: + /// segment{1} + /// cache: [_____|___________ + /// ^ + /// download_offset + /// requested_range: [__________] + /// ^ + /// file_offset_of_buffer_end + /// But space reservation failed. + /// So get working and internal buffer from predownload buffer, get new download buffer, + /// return buffer back, seek to actual position. + /// We could reuse predownload buffer and just seek to needed position, but for now + /// seek is only allowed once for ReadBufferForS3 - before call to nextImpl. + /// TODO: allow seek more than once with seek avoiding. + + bytes_to_predownload = 0; + file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + + read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; + + swap(*implementation_buffer); + working_buffer.resize(0); + position() = working_buffer.end(); + + implementation_buffer = getRemoteFSReadBuffer(file_segment, read_type); + + swap(*implementation_buffer); + + implementation_buffer->seek(file_offset_of_buffer_end, SEEK_SET); + + LOG_TEST( + log, "Predownload failed because of space limit. Will read from remote filesystem starting from offset: {}", + file_offset_of_buffer_end); + + break; + } + } + } +} + +bool CachedReadBufferFromRemoteFS::updateImplementationBufferIfNeeded() +{ + auto & file_segment = *current_file_segment_it; + auto current_read_range = file_segment->range(); + auto current_state = file_segment->state(); + + assert(current_read_range.left <= file_offset_of_buffer_end); + assert(!file_segment->isDownloader()); + + if (file_offset_of_buffer_end > current_read_range.right) + { + return completeFileSegmentAndGetNext(); + } + + if (read_type == ReadType::CACHED && current_state != FileSegment::State::DOWNLOADED) + { + /// If current read_type is ReadType::CACHED and file segment is not DOWNLOADED, + /// it means the following case, e.g. we started from CacheReadBuffer and continue with RemoteFSReadBuffer. + /// segment{k} + /// cache: [______|___________ + /// ^ + /// download_offset + /// requested_range: [__________] + /// ^ + /// file_offset_of_buffer_end + + auto download_offset = file_segment->getDownloadOffset(); + if (download_offset == file_offset_of_buffer_end) + { + /// TODO: makes sense to reuse local file reader if we return here with CACHED read type again? + implementation_buffer = getImplementationBuffer(*current_file_segment_it); + + return true; + } + else if (download_offset < file_offset_of_buffer_end) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} >= {} ({})", download_offset, file_offset_of_buffer_end, getInfoForLog()); + } + + if (read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE) + { + /** + * ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE means that on previous getImplementationBuffer() call + * current buffer successfully called file_segment->getOrSetDownloader() and became a downloader + * for this file segment. However, the downloader's term has a lifespan of 1 nextImpl() call, + * e.g. downloader reads buffer_size byte and calls completeBatchAndResetDownloader() and some other + * thread can become a downloader if it calls getOrSetDownloader() faster. + * + * So downloader is committed to download only buffer_size bytes and then is not a downloader anymore, + * because there is no guarantee on a higher level, that current buffer will not disappear without + * being destructed till the end of query or without finishing the read range, which he was supposed + * to read by marks range given to him. Therefore, each nextImpl() call, in case of + * READ_AND_PUT_IN_CACHE, starts with getOrSetDownloader(). + */ + implementation_buffer = getImplementationBuffer(*current_file_segment_it); + } + + return true; +} + +bool CachedReadBufferFromRemoteFS::nextImpl() +{ + try + { + return nextImplStep(); + } + catch (Exception & e) + { + e.addMessage("Cache info: {}", getInfoForLog()); + throw; + } +} + +bool CachedReadBufferFromRemoteFS::nextImplStep() +{ + if (IFileCache::shouldBypassCache()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Using cache when not allowed"); + + if (!initialized) + initialize(file_offset_of_buffer_end, getTotalSizeToRead()); + + if (current_file_segment_it == file_segments_holder->file_segments.end()) + return false; + + SCOPE_EXIT({ + if (current_file_segment_it == file_segments_holder->file_segments.end()) + return; + + auto & file_segment = *current_file_segment_it; + + bool download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + if (download_current_segment) + { + try + { + bool file_segment_already_completed = !file_segment->isDownloader(); + if (!file_segment_already_completed) + file_segment->completeBatchAndResetDownloader(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + assert(!file_segment->isDownloader()); + }); + + bytes_to_predownload = 0; + + if (implementation_buffer) + { + bool can_read_further = updateImplementationBufferIfNeeded(); + if (!can_read_further) + return false; + } + else + { + implementation_buffer = getImplementationBuffer(*current_file_segment_it); + } + + assert(!internal_buffer.empty()); + swap(*implementation_buffer); + + auto & file_segment = *current_file_segment_it; + auto current_read_range = file_segment->range(); + + LOG_TEST(log, "Current segment: {}, downloader: {}, current count: {}, position: {}", + current_read_range.toString(), file_segment->getDownloader(), implementation_buffer->count(), implementation_buffer->getPosition()); + + assert(current_read_range.left <= file_offset_of_buffer_end); + assert(current_read_range.right >= file_offset_of_buffer_end); + + bool result = false; + size_t size = 0; + + size_t needed_to_predownload = bytes_to_predownload; + if (needed_to_predownload) + { + predownload(file_segment); + + result = implementation_buffer->hasPendingData(); + size = implementation_buffer->available(); + } + + auto download_current_segment = read_type == ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE; + if (download_current_segment != file_segment->isDownloader()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Incorrect segment state. Having read type: {}, Caller id: {}, downloader id: {}, file segment state: {}", + toString(read_type), file_segment->getCallerId(), file_segment->getDownloader(), file_segment->state()); + + if (!result) + { + result = implementation_buffer->next(); + size = implementation_buffer->buffer().size(); + } + + if (result) + { + if (download_current_segment) + { + assert(file_offset_of_buffer_end + size - 1 <= file_segment->range().right); + + if (file_segment->reserve(size)) + { + file_segment->write(needed_to_predownload ? implementation_buffer->position() : implementation_buffer->buffer().begin(), size); + } + else + { + download_current_segment = false; + file_segment->complete(FileSegment::State::PARTIALLY_DOWNLOADED_NO_CONTINUATION); + LOG_DEBUG(log, "No space left in cache, will continue without cache download"); + } + } + + switch (read_type) + { + case ReadType::CACHED: + { + ProfileEvents::increment(ProfileEvents::RemoteFSCacheReadBytes, size); + break; + } + case ReadType::REMOTE_FS_READ_BYPASS_CACHE: + { + ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, size); + break; + } + case ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE: + { + ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, size); + ProfileEvents::increment(ProfileEvents::RemoteFSCacheDownloadBytes, size); + break; + } + } + + if (std::next(current_file_segment_it) == file_segments_holder->file_segments.end()) + { + size_t remaining_size_to_read = std::min(current_read_range.right, read_until_position - 1) - file_offset_of_buffer_end + 1; + size = std::min(size, remaining_size_to_read); + implementation_buffer->buffer().resize(nextimpl_working_buffer_offset + size); + } + + file_offset_of_buffer_end += size; + } + + swap(*implementation_buffer); + + if (download_current_segment) + file_segment->completeBatchAndResetDownloader(); + + assert(!file_segment->isDownloader()); + + LOG_TEST(log, + "Key: {}. Returning with {} bytes, buffer position: {} (offset: {}, predownloaded: {}), " + "buffer available: {}, current range: {}, current offset: {}, file segment state: {}, download offset: {}, read_type: {}, " + "reading until position: {}, started with offset: {}, remaining ranges: {}", + getHexUIntLowercase(cache_key), working_buffer.size(), getPosition(), offset(), needed_to_predownload, + available(), current_read_range.toString(), + file_offset_of_buffer_end, FileSegment::stateToString(file_segment->state()), file_segment->getDownloadOffset(), toString(read_type), + read_until_position, first_offset, file_segments_holder->toString()); + + if (size == 0 && file_offset_of_buffer_end < read_until_position) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Having zero bytes, but range is not finished: file offset: {}, reading until: {}", + file_offset_of_buffer_end, read_until_position); + return result; +} + +off_t CachedReadBufferFromRemoteFS::seek(off_t offset, int whence) +{ + if (initialized) + throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, + "Seek is allowed only before first read attempt from the buffer"); + + if (whence != SEEK_SET) + throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Only SEEK_SET allowed"); + + first_offset = offset; + file_offset_of_buffer_end = offset; + size_t size = getTotalSizeToRead(); + initialize(offset, size); + + return offset; +} + +size_t CachedReadBufferFromRemoteFS::getTotalSizeToRead() +{ + /// Last position should be guaranteed to be set, as at least we always know file size. + if (!read_until_position) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Last position was not set"); + + /// On this level should be guaranteed that read size is non-zero. + if (file_offset_of_buffer_end >= read_until_position) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Read boundaries mismatch. Expected {} < {}", + file_offset_of_buffer_end, read_until_position); + + return read_until_position - file_offset_of_buffer_end; +} + +void CachedReadBufferFromRemoteFS::setReadUntilPosition(size_t) +{ + throw Exception(ErrorCodes::LOGICAL_ERROR, "Method `setReadUntilPosition()` not allowed"); +} + +off_t CachedReadBufferFromRemoteFS::getPosition() +{ + return file_offset_of_buffer_end - available(); +} + +std::optional CachedReadBufferFromRemoteFS::getLastNonDownloadedOffset() const +{ + if (!file_segments_holder) + throw Exception(ErrorCodes::LOGICAL_ERROR, "File segments holder not initialized"); + + const auto & file_segments = file_segments_holder->file_segments; + for (auto it = file_segments.rbegin(); it != file_segments.rend(); ++it) + { + const auto & file_segment = *it; + if (file_segment->state() != FileSegment::State::DOWNLOADED) + return file_segment->range().right; + } + + return std::nullopt; +} + +String CachedReadBufferFromRemoteFS::getInfoForLog() +{ + return fmt::format("Buffer path: {}, hash key: {}, file_offset_of_buffer_end: {}, internal buffer remaining read range: {}, file segment info: {}", + remote_fs_object_path, getHexUIntLowercase(cache_key), file_offset_of_buffer_end, + (implementation_buffer ? + std::to_string(implementation_buffer->getRemainingReadRange().left) + '-' + (implementation_buffer->getRemainingReadRange().right ? std::to_string(*implementation_buffer->getRemainingReadRange().right) : "None") + : "None"), + (current_file_segment_it == file_segments_holder->file_segments.end() ? "None" : (*current_file_segment_it)->getInfoForLog())); +} + +} diff --git a/src/Disks/IO/CachedReadBufferFromRemoteFS.h b/src/Disks/IO/CachedReadBufferFromRemoteFS.h new file mode 100644 index 00000000000..3d03debcd01 --- /dev/null +++ b/src/Disks/IO/CachedReadBufferFromRemoteFS.h @@ -0,0 +1,103 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +class CachedReadBufferFromRemoteFS : public SeekableReadBuffer +{ +public: + using RemoteFSFileReaderCreator = std::function; + + CachedReadBufferFromRemoteFS( + const String & remote_fs_object_path_, + FileCachePtr cache_, + RemoteFSFileReaderCreator remote_file_reader_creator_, + const ReadSettings & settings_, + size_t read_until_position_); + + bool nextImpl() override; + + off_t seek(off_t off, int whence) override; + + off_t getPosition() override; + + size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; } + + String getInfoForLog() override; + + void setReadUntilPosition(size_t position) override; + +private: + void initialize(size_t offset, size_t size); + + SeekableReadBufferPtr getImplementationBuffer(FileSegmentPtr & file_segment); + + SeekableReadBufferPtr getReadBufferForFileSegment(FileSegmentPtr & file_segment); + + SeekableReadBufferPtr getCacheReadBuffer(size_t offset) const; + + std::optional getLastNonDownloadedOffset() const; + + bool updateImplementationBufferIfNeeded(); + + void predownload(FileSegmentPtr & file_segment); + + bool nextImplStep(); + + enum class ReadType + { + CACHED, + REMOTE_FS_READ_BYPASS_CACHE, + REMOTE_FS_READ_AND_PUT_IN_CACHE, + }; + + SeekableReadBufferPtr getRemoteFSReadBuffer(FileSegmentPtr & file_segment, ReadType read_type_); + + size_t getTotalSizeToRead(); + bool completeFileSegmentAndGetNext(); + + Poco::Logger * log; + IFileCache::Key cache_key; + String remote_fs_object_path; + FileCachePtr cache; + ReadSettings settings; + + size_t read_until_position; + size_t file_offset_of_buffer_end = 0; + size_t bytes_to_predownload = 0; + + RemoteFSFileReaderCreator remote_file_reader_creator; + + /// Remote read buffer, which can only be owned by current buffer. + FileSegment::RemoteFileReaderPtr remote_file_reader; + + std::optional file_segments_holder; + FileSegments::iterator current_file_segment_it; + + SeekableReadBufferPtr implementation_buffer; + bool initialized = false; + + ReadType read_type = ReadType::REMOTE_FS_READ_BYPASS_CACHE; + + static String toString(ReadType type) + { + switch (type) + { + case ReadType::CACHED: + return "CACHED"; + case ReadType::REMOTE_FS_READ_BYPASS_CACHE: + return "REMOTE_FS_READ_BYPASS_CACHE"; + case ReadType::REMOTE_FS_READ_AND_PUT_IN_CACHE: + return "REMOTE_FS_READ_AND_PUT_IN_CACHE"; + } + } + size_t first_offset = 0; +}; + +} diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 4db0c9e3c71..8f91804bbbe 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -16,51 +16,79 @@ #include #endif +#include #include #include #include +#include namespace fs = std::filesystem; namespace DB { -#if USE_AWS_S3 -SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const String & path, size_t read_until_position_) const +namespace ErrorCodes { - return std::make_unique(client_ptr, bucket, - fs::path(metadata.remote_fs_root_path) / path, max_single_read_retries, settings, threadpool_read, read_until_position_); + extern const int LOGICAL_ERROR; +} + +#if USE_AWS_S3 +SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const String & path, size_t file_size) +{ + current_path = path; + + auto cache = settings.remote_fs_cache; + bool with_cache = cache && settings.remote_fs_enable_cache && !IFileCache::shouldBypassCache(); + + auto remote_file_reader_creator = [=, this]() + { + return std::make_unique( + client_ptr, bucket, fs::path(metadata.remote_fs_root_path) / path, max_single_read_retries, + settings, /* use_external_buffer */true, read_until_position, /* restricted_seek */true); + }; + + if (with_cache) + { + return std::make_shared( + path, cache, remote_file_reader_creator, settings, read_until_position ? read_until_position : file_size); + } + + return remote_file_reader_creator(); } #endif #if USE_AZURE_BLOB_STORAGE -SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBuffer(const String & path, size_t read_until_position_) const +SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBuffer(const String & path, size_t /* file_size */) { + current_path = path; return std::make_unique(blob_container_client, path, max_single_read_retries, - max_single_download_retries, settings.remote_fs_buffer_size, threadpool_read, read_until_position_); + max_single_download_retries, settings.remote_fs_buffer_size, /* use_external_buffer */true, read_until_position); } #endif -SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBuffer(const String & path, size_t read_until_position_) const +SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBuffer(const String & path, size_t /* file_size */) { - return std::make_unique(fs::path(uri) / path, context, settings, threadpool_read, read_until_position_); + current_path = path; + return std::make_unique(fs::path(uri) / path, context, settings, /* use_external_buffer */true, read_until_position); } #if USE_HDFS -SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBuffer(const String & path, size_t read_until_position_) const +SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBuffer(const String & path, size_t /* file_size */) { - return std::make_unique(hdfs_uri, fs::path(hdfs_directory) / path, config, buf_size, read_until_position_); + return std::make_unique(hdfs_uri, fs::path(hdfs_directory) / path, config, settings.remote_fs_buffer_size); } #endif -ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(const RemoteMetadata & metadata_, const String & path_) +ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(const RemoteMetadata & metadata_, const ReadSettings & settings_, const String & path_) : ReadBuffer(nullptr, 0) , metadata(metadata_) + , settings(settings_) , canonical_path(path_) + , log(&Poco::Logger::get("ReadBufferFromRemoteFSGather")) { } @@ -75,8 +103,8 @@ ReadBufferFromRemoteFSGather::ReadResult ReadBufferFromRemoteFSGather::readInto( file_offset_of_buffer_end = offset; bytes_to_ignore = ignore; - if (bytes_to_ignore) - assert(initialized()); + + assert(!bytes_to_ignore || initialized()); auto result = nextImpl(); @@ -100,11 +128,8 @@ void ReadBufferFromRemoteFSGather::initialize() /// Do not create a new buffer if we already have what we need. if (!current_buf || current_buf_idx != i) { - current_buf = createImplementationBuffer(file_path, read_until_position); current_buf_idx = i; - - if (auto * in = dynamic_cast(current_buf.get())) - in->setReadType(SeekableReadBufferWithSize::ReadType::DISK_READ); + current_buf = createImplementationBuffer(file_path, size); } current_buf->seek(current_buf_offset, SEEK_SET); @@ -133,22 +158,34 @@ bool ReadBufferFromRemoteFSGather::nextImpl() else return false; + if (!moveToNextBuffer()) + return false; + + return readImpl(); +} + + +bool ReadBufferFromRemoteFSGather::moveToNextBuffer() +{ /// If there is no available buffers - nothing to read. if (current_buf_idx + 1 >= metadata.remote_fs_objects.size()) return false; ++current_buf_idx; - const auto & current_path = metadata.remote_fs_objects[current_buf_idx].first; - current_buf = createImplementationBuffer(current_path, read_until_position); + const auto & [path, size] = metadata.remote_fs_objects[current_buf_idx]; + current_buf = createImplementationBuffer(path, size); - return readImpl(); + return true; } + bool ReadBufferFromRemoteFSGather::readImpl() { swap(*current_buf); + bool result = false; + /** * Lazy seek is performed here. * In asynchronous buffer when seeking to offset in range [pos, pos + min_bytes_for_seek] @@ -157,32 +194,50 @@ bool ReadBufferFromRemoteFSGather::readImpl() if (bytes_to_ignore) { current_buf->ignore(bytes_to_ignore); + result = current_buf->hasPendingData(); + file_offset_of_buffer_end += bytes_to_ignore; bytes_to_ignore = 0; } - bool result = current_buf->hasPendingData(); - if (result) + if (!result) + result = current_buf->next(); + + if (metadata.remote_fs_objects.size() == 1) { - /// bytes_to_ignore already added. - file_offset_of_buffer_end += current_buf->available(); + file_offset_of_buffer_end = current_buf->getFileOffsetOfBufferEnd(); } else { - result = current_buf->next(); - if (result) - file_offset_of_buffer_end += current_buf->buffer().size(); + /// For log family engines there are multiple s3 files for the same clickhouse file + file_offset_of_buffer_end += current_buf->available(); } swap(*current_buf); + /// Required for non-async reads. + if (result) + { + assert(available()); + nextimpl_working_buffer_offset = offset(); + } + return result; } +size_t ReadBufferFromRemoteFSGather::getFileOffsetOfBufferEnd() const +{ + return file_offset_of_buffer_end; +} + + void ReadBufferFromRemoteFSGather::setReadUntilPosition(size_t position) { - read_until_position = position; - reset(); + if (position != read_until_position) + { + read_until_position = position; + reset(); + } } @@ -193,7 +248,7 @@ void ReadBufferFromRemoteFSGather::reset() String ReadBufferFromRemoteFSGather::getFileName() const { - return canonical_path; + return current_path; } @@ -205,4 +260,21 @@ size_t ReadBufferFromRemoteFSGather::getFileSize() const return size; } +String ReadBufferFromRemoteFSGather::getInfoForLog() +{ + if (!current_buf) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get info: buffer not initialized"); + + return current_buf->getInfoForLog(); +} + +size_t ReadBufferFromRemoteFSGather::getImplementationBufferOffset() const +{ + if (!current_buf) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Buffer not initialized"); + + return current_buf->getFileOffsetOfBufferEnd(); +} + + } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index ddd651f47a1..25bfe0b7e16 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -9,13 +9,9 @@ #include #endif -namespace Aws -{ -namespace S3 -{ -class S3Client; -} -} +namespace Aws { namespace S3 { class S3Client; } } + +namespace Poco { class Logger; } namespace DB { @@ -29,7 +25,10 @@ class ReadBufferFromRemoteFSGather : public ReadBuffer friend class ReadIndirectBufferFromRemoteFS; public: - explicit ReadBufferFromRemoteFSGather(const RemoteMetadata & metadata_, const String & path_); + ReadBufferFromRemoteFSGather( + const RemoteMetadata & metadata_, + const ReadSettings & settings_, + const String & path_); String getFileName() const; @@ -47,15 +46,27 @@ public: size_t getFileSize() const; - size_t offset() const { return file_offset_of_buffer_end; } + size_t getFileOffsetOfBufferEnd() const; bool initialized() const { return current_buf != nullptr; } + String getInfoForLog(); + + size_t getImplementationBufferOffset() const; + protected: - virtual SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t read_until_position) const = 0; + virtual SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) = 0; RemoteMetadata metadata; + ReadSettings settings; + + bool use_external_buffer; + + size_t read_until_position = 0; + + String current_path; + private: bool nextImpl() override; @@ -63,6 +74,8 @@ private: bool readImpl(); + bool moveToNextBuffer(); + SeekableReadBufferPtr current_buf; size_t current_buf_idx = 0; @@ -76,9 +89,9 @@ private: */ size_t bytes_to_ignore = 0; - size_t read_until_position = 0; - String canonical_path; + + Poco::Logger * log; }; @@ -93,25 +106,20 @@ public: const String & bucket_, IDiskRemote::Metadata metadata_, size_t max_single_read_retries_, - const ReadSettings & settings_, - bool threadpool_read_ = false) - : ReadBufferFromRemoteFSGather(metadata_, path_) + const ReadSettings & settings_) + : ReadBufferFromRemoteFSGather(metadata_, settings_, path_) , client_ptr(std::move(client_ptr_)) , bucket(bucket_) , max_single_read_retries(max_single_read_retries_) - , settings(settings_) - , threadpool_read(threadpool_read_) { } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t read_until_position) const override; + SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; private: std::shared_ptr client_ptr; String bucket; UInt64 max_single_read_retries; - ReadSettings settings; - bool threadpool_read; }; #endif @@ -127,25 +135,20 @@ public: IDiskRemote::Metadata metadata_, size_t max_single_read_retries_, size_t max_single_download_retries_, - const ReadSettings & settings_, - bool threadpool_read_ = false) - : ReadBufferFromRemoteFSGather(metadata_, path_) + const ReadSettings & settings_) + : ReadBufferFromRemoteFSGather(metadata_, settings_, path_) , blob_container_client(blob_container_client_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) - , settings(settings_) - , threadpool_read(threadpool_read_) { } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t read_until_position) const override; + SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; private: std::shared_ptr blob_container_client; size_t max_single_read_retries; size_t max_single_download_retries; - ReadSettings settings; - bool threadpool_read; }; #endif @@ -158,23 +161,18 @@ public: const String & uri_, RemoteMetadata metadata_, ContextPtr context_, - size_t threadpool_read_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(metadata_, path_) + : ReadBufferFromRemoteFSGather(metadata_, settings_, path_) , uri(uri_) , context(context_) - , threadpool_read(threadpool_read_) - , settings(settings_) { } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t read_until_position) const override; + SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; private: String uri; ContextPtr context; - bool threadpool_read; - ReadSettings settings; }; @@ -188,23 +186,21 @@ public: const Poco::Util::AbstractConfiguration & config_, const String & hdfs_uri_, IDiskRemote::Metadata metadata_, - size_t buf_size_) - : ReadBufferFromRemoteFSGather(metadata_, path_) + const ReadSettings & settings_) + : ReadBufferFromRemoteFSGather(metadata_, settings_, path_) , config(config_) - , buf_size(buf_size_) { const size_t begin_of_path = hdfs_uri_.find('/', hdfs_uri_.find("//") + 2); hdfs_directory = hdfs_uri_.substr(begin_of_path); hdfs_uri = hdfs_uri_.substr(0, begin_of_path); } - SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t read_until_position) const override; + SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) override; private: const Poco::Util::AbstractConfiguration & config; String hdfs_uri; String hdfs_directory; - size_t buf_size; }; #endif diff --git a/src/Disks/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp index c7da6c2051b..e845f24b7d9 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -76,7 +76,7 @@ std::unique_ptr ReadBufferFromWebServer::initialize() read_settings, ReadWriteBufferFromHTTP::HTTPHeaderEntries{}, range, - context->getRemoteHostFilter(), + &context->getRemoteHostFilter(), /* delay_initialization */true, use_external_buffer); } diff --git a/src/Disks/IO/ReadBufferFromWebServer.h b/src/Disks/IO/ReadBufferFromWebServer.h index 7285a94b0d8..ea746fb75a1 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.h +++ b/src/Disks/IO/ReadBufferFromWebServer.h @@ -30,6 +30,8 @@ public: off_t getPosition() override; + size_t getFileOffsetOfBufferEnd() const override { return offset; } + private: std::unique_ptr initialize(); diff --git a/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp index cbf265ce741..699f8380cb8 100644 --- a/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp @@ -13,7 +13,9 @@ namespace ErrorCodes ReadIndirectBufferFromRemoteFS::ReadIndirectBufferFromRemoteFS( - std::shared_ptr impl_) : impl(std::move(impl_)) + std::shared_ptr impl_) + : ReadBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0) + , impl(impl_) { } @@ -30,6 +32,18 @@ String ReadIndirectBufferFromRemoteFS::getFileName() const } +void ReadIndirectBufferFromRemoteFS::setReadUntilPosition(size_t position) +{ + impl->setReadUntilPosition(position); +} + + +void ReadIndirectBufferFromRemoteFS::setReadUntilEnd() +{ + impl->setReadUntilPosition(impl->getFileSize()); +} + + off_t ReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) { if (whence == SEEK_CUR) @@ -66,6 +80,7 @@ off_t ReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) impl->reset(); resetWorkingBuffer(); + file_offset_of_buffer_end = impl->file_offset_of_buffer_end; return impl->file_offset_of_buffer_end; } @@ -74,11 +89,21 @@ bool ReadIndirectBufferFromRemoteFS::nextImpl() { /// Transfer current position and working_buffer to actual ReadBuffer swap(*impl); + + assert(!impl->hasPendingData()); /// Position and working_buffer will be updated in next() call auto result = impl->next(); /// and assigned to current buffer. swap(*impl); + if (result) + { + file_offset_of_buffer_end += available(); + BufferBase::set(working_buffer.begin() + offset(), available(), 0); + } + + assert(file_offset_of_buffer_end == impl->file_offset_of_buffer_end); + return result; } diff --git a/src/Disks/IO/ReadIndirectBufferFromRemoteFS.h b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.h index 0c8b1b4dd21..a0669be411f 100644 --- a/src/Disks/IO/ReadIndirectBufferFromRemoteFS.h +++ b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.h @@ -27,10 +27,16 @@ public: String getFileName() const override; + void setReadUntilPosition(size_t position) override; + + void setReadUntilEnd() override; + private: bool nextImpl() override; std::shared_ptr impl; + + size_t file_offset_of_buffer_end = 0; }; } diff --git a/src/IO/ThreadPoolReader.cpp b/src/Disks/IO/ThreadPoolReader.cpp similarity index 89% rename from src/IO/ThreadPoolReader.cpp rename to src/Disks/IO/ThreadPoolReader.cpp index 63bc8fe7c49..e39f6057445 100644 --- a/src/IO/ThreadPoolReader.cpp +++ b/src/Disks/IO/ThreadPoolReader.cpp @@ -1,4 +1,4 @@ -#include +#include "ThreadPoolReader.h" #include #include #include @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -176,7 +177,7 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheHitElapsedMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - promise.set_value({bytes_read, 0}); + promise.set_value({bytes_read, request.ignore}); return future; } } @@ -184,9 +185,26 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMiss); - auto task = std::make_shared>([request, fd] + ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() + ? CurrentThread::get().getThreadGroup() + : MainThreadStatus::getInstance().getThreadGroup(); + + ContextPtr query_context; + if (CurrentThread::isInitialized()) + query_context = CurrentThread::get().getQueryContext(); + + auto task = std::make_shared>([request, fd, running_group, query_context] { + ThreadStatus thread_status; + + if (query_context) + thread_status.attachQueryContext(query_context); + + if (running_group) + thread_status.attachQuery(running_group); + setThreadName("ThreadPoolRead"); + Stopwatch watch(CLOCK_MONOTONIC); size_t bytes_read = 0; @@ -219,7 +237,10 @@ std::future ThreadPoolReader::submit(Request reques ProfileEvents::increment(ProfileEvents::ThreadPoolReaderPageCacheMissElapsedMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - return Result{ .size = bytes_read, .offset = 0 }; + if (running_group) + thread_status.detachQuery(); + + return Result{ .size = bytes_read, .offset = request.ignore }; }); auto future = task->get_future(); diff --git a/src/IO/ThreadPoolReader.h b/src/Disks/IO/ThreadPoolReader.h similarity index 100% rename from src/IO/ThreadPoolReader.h rename to src/Disks/IO/ThreadPoolReader.h diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp index 4be55ff3ecf..bdb012a6376 100644 --- a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -41,9 +42,28 @@ ThreadPoolRemoteFSReader::ThreadPoolRemoteFSReader(size_t pool_size, size_t queu std::future ThreadPoolRemoteFSReader::submit(Request request) { - auto task = std::make_shared>([request] + ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() + ? CurrentThread::get().getThreadGroup() + : MainThreadStatus::getInstance().getThreadGroup(); + + ContextPtr query_context; + if (CurrentThread::isInitialized()) + query_context = CurrentThread::get().getQueryContext(); + + auto task = std::make_shared>([request, running_group, query_context] { + ThreadStatus thread_status; + + /// Save query context if any, because cache implementation needs it. + if (query_context) + thread_status.attachQueryContext(query_context); + + /// To be able to pass ProfileEvents. + if (running_group) + thread_status.attachQuery(running_group); + setThreadName("VFSRead"); + CurrentMetrics::Increment metric_increment{CurrentMetrics::Read}; auto * remote_fs_fd = assert_cast(request.descriptor.get()); @@ -54,6 +74,9 @@ std::future ThreadPoolRemoteFSReader::submit(Reques ProfileEvents::increment(ProfileEvents::RemoteFSReadMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::RemoteFSReadBytes, bytes_read); + if (running_group) + thread_status.detachQuery(); + return Result{ .size = bytes_read, .offset = offset }; }); diff --git a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp index 87453440693..9b604341da9 100644 --- a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp @@ -9,20 +9,18 @@ namespace DB { -template -WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( - std::unique_ptr impl_, - IDiskRemote::Metadata metadata_, - const String & remote_fs_path_) +WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( + std::unique_ptr impl_, + CreateMetadataCallback && create_callback_, + const String & metadata_file_path_) : WriteBufferFromFileDecorator(std::move(impl_)) - , metadata(std::move(metadata_)) - , remote_fs_path(remote_fs_path_) + , create_metadata_callback(std::move(create_callback_)) + , metadata_file_path(metadata_file_path_) { } -template -WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() +WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() { try { @@ -35,40 +33,11 @@ WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() } -template -void WriteIndirectBufferFromRemoteFS::finalizeImpl() +void WriteIndirectBufferFromRemoteFS::finalizeImpl() { WriteBufferFromFileDecorator::finalizeImpl(); - - metadata.addObject(remote_fs_path, count()); - metadata.save(); + create_metadata_callback(count()); } -template -void WriteIndirectBufferFromRemoteFS::sync() -{ - if (finalized) - metadata.save(true); -} - - -#if USE_AWS_S3 -template -class WriteIndirectBufferFromRemoteFS; -#endif - -#if USE_AZURE_BLOB_STORAGE -template -class WriteIndirectBufferFromRemoteFS; -#endif - -#if USE_HDFS -template -class WriteIndirectBufferFromRemoteFS; -#endif - -template -class WriteIndirectBufferFromRemoteFS; - } diff --git a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h index 8f88eff3b33..25a93e2fe07 100644 --- a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h @@ -9,28 +9,26 @@ namespace DB { +using CreateMetadataCallback = std::function; + /// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS. -template class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator { public: WriteIndirectBufferFromRemoteFS( - std::unique_ptr impl_, - IDiskRemote::Metadata metadata_, - const String & remote_fs_path_); + std::unique_ptr impl_, + CreateMetadataCallback && create_callback_, + const String & metadata_file_path_); - virtual ~WriteIndirectBufferFromRemoteFS() override; + ~WriteIndirectBufferFromRemoteFS() override; - void sync() override; - - String getFileName() const override { return metadata.metadata_file_path; } + String getFileName() const override { return metadata_file_path; } private: void finalizeImpl() override; - IDiskRemote::Metadata metadata; - - String remote_fs_path; + CreateMetadataCallback create_metadata_callback; + String metadata_file_path; }; } diff --git a/src/IO/createReadBufferFromFileBase.cpp b/src/Disks/IO/createReadBufferFromFileBase.cpp similarity index 98% rename from src/IO/createReadBufferFromFileBase.cpp rename to src/Disks/IO/createReadBufferFromFileBase.cpp index b83bfdbf3a8..4ff492e4013 100644 --- a/src/IO/createReadBufferFromFileBase.cpp +++ b/src/Disks/IO/createReadBufferFromFileBase.cpp @@ -1,9 +1,9 @@ -#include +#include #include #include #include #include -#include +#include #include #include diff --git a/src/IO/createReadBufferFromFileBase.h b/src/Disks/IO/createReadBufferFromFileBase.h similarity index 100% rename from src/IO/createReadBufferFromFileBase.h rename to src/Disks/IO/createReadBufferFromFileBase.h diff --git a/src/Disks/IStoragePolicy.cpp b/src/Disks/IStoragePolicy.cpp new file mode 100644 index 00000000000..2ba6df4be8f --- /dev/null +++ b/src/Disks/IStoragePolicy.cpp @@ -0,0 +1,34 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_VOLUME; + extern const int UNKNOWN_DISK; +} + +DiskPtr IStoragePolicy::getDiskByName(const String & disk_name) const +{ + auto disk = tryGetDiskByName(disk_name); + if (!disk) + throw Exception(ErrorCodes::UNKNOWN_DISK, + "No such disk {} in storage policy {}", backQuote(disk_name), backQuote(getName())); + + return disk; +} + +VolumePtr IStoragePolicy::getVolumeByName(const String & volume_name) const +{ + auto volume = tryGetVolumeByName(volume_name); + if (!volume) + throw Exception(ErrorCodes::UNKNOWN_VOLUME, + "No such volume {} in storage policy {}", backQuote(volume_name), backQuote(getName())); + + return volume; +} + +} diff --git a/src/Disks/IStoragePolicy.h b/src/Disks/IStoragePolicy.h index 8c6d56bdcfd..a5d1120f377 100644 --- a/src/Disks/IStoragePolicy.h +++ b/src/Disks/IStoragePolicy.h @@ -39,7 +39,8 @@ public: /// Used when it's not important, for example for /// mutations files virtual DiskPtr getAnyDisk() const = 0; - virtual DiskPtr getDiskByName(const String & disk_name) const = 0; + virtual DiskPtr tryGetDiskByName(const String & disk_name) const = 0; + DiskPtr getDiskByName(const String & disk_name) const; /// Get free space from most free disk virtual UInt64 getMaxUnreservedFreeSpace() const = 0; /// Reserves space on any volume with index > min_volume_index or returns nullptr @@ -53,7 +54,8 @@ public: virtual ReservationPtr makeEmptyReservationOnLargestDisk() const = 0; /// Get volume by index. virtual VolumePtr getVolume(size_t index) const = 0; - virtual VolumePtr getVolumeByName(const String & volume_name) const = 0; + virtual VolumePtr tryGetVolumeByName(const String & volume_name) const = 0; + VolumePtr getVolumeByName(const String & volume_name) const; /// Checks if storage policy can be replaced by another one. virtual void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const = 0; /// Find volume index, which contains disk diff --git a/src/Disks/LocalDirectorySyncGuard.h b/src/Disks/LocalDirectorySyncGuard.h index 34e4cb9e657..cb891461e85 100644 --- a/src/Disks/LocalDirectorySyncGuard.h +++ b/src/Disks/LocalDirectorySyncGuard.h @@ -17,8 +17,8 @@ class LocalDirectorySyncGuard final : public ISyncGuard public: /// NOTE: If you have already opened descriptor, it's preferred to use /// this constructor instead of constructor with path. - LocalDirectorySyncGuard(int fd_) : fd(fd_) {} - LocalDirectorySyncGuard(const String & full_path); + explicit LocalDirectorySyncGuard(int fd_) : fd(fd_) {} + explicit LocalDirectorySyncGuard(const String & full_path); ~LocalDirectorySyncGuard() override; private: diff --git a/src/Disks/RemoteDisksCommon.cpp b/src/Disks/RemoteDisksCommon.cpp index 1402e3f62c8..36f2aed3e7c 100644 --- a/src/Disks/RemoteDisksCommon.cpp +++ b/src/Disks/RemoteDisksCommon.cpp @@ -1,12 +1,13 @@ #include #include +#include +#include namespace DB { namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; +{extern const int BAD_ARGUMENTS; } std::shared_ptr wrapWithCache( @@ -26,6 +27,14 @@ std::shared_ptr wrapWithCache( return std::make_shared(disk, cache_disk, cache_file_predicate); } +static String getDiskMetadataPath( + const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context) +{ + return config.getString(config_prefix + ".metadata_path", context->getPath() + "disks/" + name + "/"); +} std::pair prepareForLocalMetadata( const String & name, @@ -34,10 +43,40 @@ std::pair prepareForLocalMetadata( ContextPtr context) { /// where the metadata files are stored locally - auto metadata_path = config.getString(config_prefix + ".metadata_path", context->getPath() + "disks/" + name + "/"); + auto metadata_path = getDiskMetadataPath(name, config, config_prefix, context); fs::create_directories(metadata_path); auto metadata_disk = std::make_shared(name + "-metadata", metadata_path, 0); return std::make_pair(metadata_path, metadata_disk); } + +FileCachePtr getCachePtrForDisk( + const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context) +{ + bool data_cache_enabled = config.getBool(config_prefix + ".data_cache_enabled", false); + if (!data_cache_enabled) + return nullptr; + + auto cache_base_path = config.getString(config_prefix + ".data_cache_path", fs::path(context->getPath()) / "disks" / name / "data_cache/"); + if (!fs::exists(cache_base_path)) + fs::create_directories(cache_base_path); + + LOG_INFO(&Poco::Logger::get("Disk(" + name + ")"), "Disk registered with cache path: {}", cache_base_path); + + auto metadata_path = getDiskMetadataPath(name, config, config_prefix, context); + if (metadata_path == cache_base_path) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Metadata path and cache base path must be different: {}", metadata_path); + + size_t max_cache_size = config.getUInt64(config_prefix + ".data_cache_max_size", 1024*1024*1024); + size_t max_cache_elements = config.getUInt64(config_prefix + ".data_cache_max_elements", REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_ELEMENTS); + size_t max_file_segment_size = config.getUInt64(config_prefix + ".max_file_segment_size", REMOTE_FS_OBJECTS_CACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE); + + auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, max_cache_size, max_cache_elements, max_file_segment_size); + cache->initialize(); + return cache; +} + } diff --git a/src/Disks/RemoteDisksCommon.h b/src/Disks/RemoteDisksCommon.h index 0d057b44d18..661d4e293df 100644 --- a/src/Disks/RemoteDisksCommon.h +++ b/src/Disks/RemoteDisksCommon.h @@ -21,4 +21,10 @@ std::pair prepareForLocalMetadata( const String & config_prefix, ContextPtr context); +FileCachePtr getCachePtrForDisk( + const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr context); + } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 5cfb4532b65..e46620d9d1f 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -153,10 +153,11 @@ DiskS3::DiskS3( String bucket_, String s3_root_path_, DiskPtr metadata_disk_, + FileCachePtr cache_, ContextPtr context_, SettingsPtr settings_, GetDiskSettings settings_getter_) - : IDiskRemote(name_, s3_root_path_, metadata_disk_, "DiskS3", settings_->thread_pool_size) + : IDiskRemote(name_, s3_root_path_, metadata_disk_, std::move(cache_), "DiskS3", settings_->thread_pool_size) , bucket(std::move(bucket_)) , current_settings(std::move(settings_)) , settings_getter(settings_getter_) @@ -218,22 +219,23 @@ void DiskS3::moveFile(const String & from_path, const String & to_path, bool sen std::unique_ptr DiskS3::readFile(const String & path, const ReadSettings & read_settings, std::optional, std::optional) const { auto settings = current_settings.get(); - auto metadata = readMeta(path); + auto metadata = readMetadata(path); LOG_TEST(log, "Read from file by path: {}. Existing S3 objects: {}", backQuote(metadata_disk->getPath() + path), metadata.remote_fs_objects.size()); - bool threadpool_read = read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; + ReadSettings disk_read_settings{read_settings}; + if (cache) + disk_read_settings.remote_fs_cache = cache; auto s3_impl = std::make_unique( - path, - settings->client, bucket, metadata, - settings->s3_max_single_read_retries, read_settings, threadpool_read); + path, settings->client, bucket, metadata, + settings->s3_max_single_read_retries, disk_read_settings); - if (threadpool_read) + if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { auto reader = getThreadPoolReader(); - return std::make_unique(reader, read_settings, std::move(s3_impl)); + return std::make_unique(reader, disk_read_settings, std::move(s3_impl)); } else { @@ -245,10 +247,9 @@ std::unique_ptr DiskS3::readFile(const String & path, co std::unique_ptr DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode) { auto settings = current_settings.get(); - auto metadata = readOrCreateMetaForWriting(path, mode); /// Path to store new S3 object. - auto s3_path = getRandomASCIIString(); + auto blob_name = getRandomASCIIString(); std::optional object_metadata; if (settings->send_metadata) @@ -257,15 +258,15 @@ std::unique_ptr DiskS3::writeFile(const String & path, object_metadata = { {"path", path} }; - s3_path = "r" + revisionToString(revision) + "-file-" + s3_path; + blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", - mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + s3_path); + mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + blob_name); - ScheduleFunc schedule = [pool = &getThreadPoolWriter()](auto callback) + ScheduleFunc schedule = [pool = &getThreadPoolWriter(), thread_group = CurrentThread::getGroup()](auto callback) { - pool->scheduleOrThrow([callback = std::move(callback), thread_group = CurrentThread::getGroup()]() + pool->scheduleOrThrow([callback = std::move(callback), thread_group]() { if (thread_group) CurrentThread::attachTo(thread_group); @@ -273,6 +274,17 @@ std::unique_ptr DiskS3::writeFile(const String & path, SCOPE_EXIT_SAFE( if (thread_group) CurrentThread::detachQueryIfNotDetached(); + + /// After we detached from the thread_group, parent for memory_tracker inside ThreadStatus will be reset to it's parent. + /// Typically, it may be changes from Process to User. + /// Usually it could be ok, because thread pool task is executed before user-level memory tracker is destroyed. + /// However, thread could stay alive inside the thread pool, and it's ThreadStatus as well. + /// When, finally, we destroy the thread (and the ThreadStatus), + /// it can use memory tracker in the ~ThreadStatus in order to alloc/free untracked_memory,\ + /// and by this time user-level memory tracker may be already destroyed. + /// + /// As a work-around, reset memory tracker to total, which is always alive. + CurrentThread::get().memory_tracker.setParent(&total_memory_tracker); ); callback(); }); @@ -281,16 +293,20 @@ std::unique_ptr DiskS3::writeFile(const String & path, auto s3_buffer = std::make_unique( settings->client, bucket, - metadata.remote_fs_root_path + s3_path, + remote_fs_root_path + blob_name, settings->s3_min_upload_part_size, settings->s3_upload_part_size_multiply_factor, settings->s3_upload_part_size_multiply_parts_count_threshold, settings->s3_max_single_part_upload_size, std::move(object_metadata), - buf_size, - std::move(schedule)); + buf_size, std::move(schedule)); - return std::make_unique>(std::move(s3_buffer), std::move(metadata), s3_path); + auto create_metadata_callback = [this, path, blob_name, mode] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; }); + }; + + return std::make_unique(std::move(s3_buffer), std::move(create_metadata_callback), path); } void DiskS3::createHardLink(const String & src_path, const String & dst_path) @@ -312,13 +328,7 @@ void DiskS3::createHardLink(const String & src_path, const String & dst_path, bo createFileOperationObject("hardlink", revision, object_metadata); } - /// Increment number of references. - auto src = readMeta(src_path); - ++src.ref_count; - src.save(); - - /// Create FS hardlink to metadata file. - metadata_disk->createHardLink(src_path, dst_path); + IDiskRemote::createHardLink(src_path, dst_path); } void DiskS3::shutdown() @@ -438,7 +448,7 @@ void DiskS3::migrateFileToRestorableSchema(const String & path) { LOG_TRACE(log, "Migrate file {} to restorable schema", metadata_disk->getPath() + path); - auto meta = readMeta(path); + auto meta = readMetadata(path); for (const auto & [key, _] : meta.remote_fs_objects) { @@ -894,15 +904,19 @@ void DiskS3::processRestoreFiles(const String & source_bucket, const String & so const auto & path = path_entry->second; createDirectories(directoryPath(path)); - auto metadata = createMeta(path); auto relative_key = shrinkKey(source_path, key); /// Copy object if we restore to different bucket / path. if (bucket != source_bucket || remote_fs_root_path != source_path) copyObject(source_bucket, key, bucket, remote_fs_root_path + relative_key, head_result); - metadata.addObject(relative_key, head_result.GetContentLength()); - metadata.save(); + auto updater = [relative_key, head_result] (Metadata & metadata) + { + metadata.addObject(relative_key, head_result.GetContentLength()); + return true; + }; + + createUpdateAndStoreMetadata(path, false, updater); LOG_TRACE(log, "Restored file {}", path); } diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 698fa6173c2..7e39c9d9b3c 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB @@ -73,6 +74,7 @@ public: String bucket_, String s3_root_path_, DiskPtr metadata_disk_, + FileCachePtr cache_, ContextPtr context_, SettingsPtr settings_, GetDiskSettings settings_getter_); @@ -103,6 +105,8 @@ public: bool supportZeroCopyReplication() const override { return true; } + bool supportParallelWrite() const override { return true; } + void shutdown() override; void startup() override; diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index e16626a009a..2b5fe3c5a81 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -19,6 +19,7 @@ #include "Disks/DiskRestartProxy.h" #include "Disks/DiskLocal.h" #include "Disks/RemoteDisksCommon.h" +#include namespace DB { @@ -176,16 +177,23 @@ void registerDiskS3(DiskFactory & factory) ContextPtr context, const DisksMap & /*map*/) -> DiskPtr { S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint"))); + + if (uri.key.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "No key in S3 uri: {}", uri.uri.toString()); + if (uri.key.back() != '/') - throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 path must ends with '/', but '{}' doesn't.", uri.key); auto [metadata_path, metadata_disk] = prepareForLocalMetadata(name, config, config_prefix, context); + FileCachePtr cache = getCachePtrForDisk(name, config, config_prefix, context); + std::shared_ptr s3disk = std::make_shared( name, uri.bucket, uri.key, metadata_disk, + std::move(cache), context, getSettings(config, config_prefix, context), getSettings); @@ -200,7 +208,16 @@ void registerDiskS3(DiskFactory & factory) s3disk->startup(); - if (config.getBool(config_prefix + ".cache_enabled", true)) + +#ifdef NDEBUG + bool use_cache = true; +#else + /// Current S3 cache implementation lead to allocations in destructor of + /// read buffer. + bool use_cache = false; +#endif + + if (config.getBool(config_prefix + ".cache_enabled", use_cache)) { String cache_path = config.getString(config_prefix + ".cache_path", context->getPath() + "disks/" + name + "/cache/"); s3disk = wrapWithCache(s3disk, "s3-cache", cache_path, metadata_path); diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index 4c77528f1fc..20192c3a29f 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -179,7 +179,7 @@ DiskPtr StoragePolicy::getAnyDisk() const } -DiskPtr StoragePolicy::getDiskByName(const String & disk_name) const +DiskPtr StoragePolicy::tryGetDiskByName(const String & disk_name) const { for (auto && volume : volumes) for (auto && disk : volume->getDisks()) @@ -265,11 +265,11 @@ VolumePtr StoragePolicy::getVolume(size_t index) const } -VolumePtr StoragePolicy::getVolumeByName(const String & volume_name) const +VolumePtr StoragePolicy::tryGetVolumeByName(const String & volume_name) const { auto it = volume_index_by_volume_name.find(volume_name); if (it == volume_index_by_volume_name.end()) - throw Exception("No such volume " + backQuote(volume_name) + " in storage policy " + backQuote(name), ErrorCodes::UNKNOWN_VOLUME); + return nullptr; return getVolume(it->second); } diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index b09baf09bda..c3f72e01ec8 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -52,7 +52,7 @@ public: /// mutations files DiskPtr getAnyDisk() const override; - DiskPtr getDiskByName(const String & disk_name) const override; + DiskPtr tryGetDiskByName(const String & disk_name) const override; /// Get free space from most free disk UInt64 getMaxUnreservedFreeSpace() const override; @@ -84,7 +84,7 @@ public: /// Get volume by index. VolumePtr getVolume(size_t index) const override; - VolumePtr getVolumeByName(const String & volume_name) const override; + VolumePtr tryGetVolumeByName(const String & volume_name) const override; /// Checks if storage policy can be replaced by another one. void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const override; diff --git a/src/Disks/TemporaryFileOnDisk.h b/src/Disks/TemporaryFileOnDisk.h index c854a600146..b82cb7d2254 100644 --- a/src/Disks/TemporaryFileOnDisk.h +++ b/src/Disks/TemporaryFileOnDisk.h @@ -15,7 +15,7 @@ using DiskPtr = std::shared_ptr; class TemporaryFileOnDisk { public: - TemporaryFileOnDisk(const DiskPtr & disk_, const String & prefix_ = "tmp"); + explicit TemporaryFileOnDisk(const DiskPtr & disk_, const String & prefix_ = "tmp"); ~TemporaryFileOnDisk(); DiskPtr getDisk() const { return disk; } diff --git a/src/Disks/tests/gtest_disk_encrypted.cpp b/src/Disks/tests/gtest_disk_encrypted.cpp index d03128a6b33..fd3cc1acbe5 100644 --- a/src/Disks/tests/gtest_disk_encrypted.cpp +++ b/src/Disks/tests/gtest_disk_encrypted.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 6e6aa6d4553..44883c271f4 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -1,21 +1,2 @@ -if (TARGET ch_contrib::avrocpp) - set(USE_AVRO 1) -endif() -if (TARGET ch_contrib::parquet) - set(USE_PARQUET 1) - set(USE_ARROW 1) - set(USE_ORC 1) -endif() -if (TARGET ch_contrib::snappy) - set(USE_SNAPPY 1) -endif() -if (TARGET ch_contrib::protobuf) - set(USE_PROTOBUF 1) -endif() -if (TARGET ch_contrib::msgpack) - set(USE_MSGPACK 1) -endif() -if (TARGET ch_contrib::capnp) - set(USE_CAPNP 1) -endif() +include(configure_config.cmake) configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) diff --git a/src/Formats/CapnProtoUtils.h b/src/Formats/CapnProtoUtils.h index 51c152de17f..47fe3ada7cd 100644 --- a/src/Formats/CapnProtoUtils.h +++ b/src/Formats/CapnProtoUtils.h @@ -18,14 +18,14 @@ struct DestructorCatcher { T impl; template - DestructorCatcher(Arg && ... args) : impl(kj::fwd(args)...) {} + explicit DestructorCatcher(Arg && ... args) : impl(kj::fwd(args)...) {} ~DestructorCatcher() noexcept try { } catch (...) { return; } }; class CapnProtoSchemaParser : public DestructorCatcher { public: - CapnProtoSchemaParser() {} + CapnProtoSchemaParser() = default; capnp::StructSchema getMessageSchema(const FormatSchemaInfo & schema_info); }; diff --git a/src/Formats/EscapingRuleUtils.cpp b/src/Formats/EscapingRuleUtils.cpp index 0a7747fc864..b0ea10abdb6 100644 --- a/src/Formats/EscapingRuleUtils.cpp +++ b/src/Formats/EscapingRuleUtils.cpp @@ -262,7 +262,7 @@ static bool evaluateConstantExpressionFromString(const StringRef & field, DataTy /// FIXME: Our parser cannot parse maps in the form of '{key : value}' that is used in text formats. bool parsed = parser.parse(token_iterator, ast, expected); - if (!parsed) + if (!parsed || !token_iterator->isEnd()) return false; try diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index be565a532bb..08554cf7e07 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -89,6 +89,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; format_settings.null_as_default = settings.input_format_null_as_default; + format_settings.use_lowercase_column_name = settings.input_format_use_lowercase_column_name; format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; format_settings.parquet.import_nested = settings.input_format_parquet_import_nested; @@ -278,9 +279,10 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( if (settings.output_format_parallel_formatting && getCreators(name).supports_parallel_formatting && !settings.output_format_json_array_of_rows) { - auto formatter_creator = [output_getter, sample, callback, format_settings] - (WriteBuffer & output) -> OutputFormatPtr - { return output_getter(output, sample, {std::move(callback)}, format_settings);}; + auto formatter_creator = [output_getter, sample, callback, format_settings] (WriteBuffer & output) -> OutputFormatPtr + { + return output_getter(output, sample, {callback}, format_settings); + }; ParallelFormattingOutputFormat::Params builder{buf, sample, formatter_creator, settings.max_threads}; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 265c879e768..4881c1a43c8 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -32,14 +32,16 @@ struct FormatSettings bool null_as_default = true; bool decimal_trailing_zeros = false; bool defaults_for_omitted_fields = true; + bool use_lowercase_column_name = false; bool seekable_read = true; UInt64 max_rows_to_read_for_schema_inference = 100; enum class DateTimeInputFormat { - Basic, /// Default format for fast parsing: YYYY-MM-DD hh:mm:ss (ISO-8601 without fractional part and timezone) or NNNNNNNNNN unix timestamp. - BestEffort /// Use sophisticated rules to parse whatever possible. + Basic, /// Default format for fast parsing: YYYY-MM-DD hh:mm:ss (ISO-8601 without fractional part and timezone) or NNNNNNNNNN unix timestamp. + BestEffort, /// Use sophisticated rules to parse whatever possible. + BestEffortUS /// Use sophisticated rules to parse American style: mm/dd/yyyy }; DateTimeInputFormat date_time_input_format = DateTimeInputFormat::Basic; diff --git a/src/Formats/JSONEachRowUtils.cpp b/src/Formats/JSONEachRowUtils.cpp index c63b8453634..66e0538fef1 100644 --- a/src/Formats/JSONEachRowUtils.cpp +++ b/src/Formats/JSONEachRowUtils.cpp @@ -9,9 +9,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include @@ -169,6 +169,10 @@ DataTypePtr getDataTypeFromJSONFieldImpl(const Element & field) value_type = type; } + + if (!value_type) + return nullptr; + return std::make_shared(std::make_shared(), value_type); } diff --git a/src/Formats/JSONEachRowUtils.h b/src/Formats/JSONEachRowUtils.h index 6f71baa8b40..8d304e2ffd8 100644 --- a/src/Formats/JSONEachRowUtils.h +++ b/src/Formats/JSONEachRowUtils.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include #include diff --git a/src/Formats/MarkInCompressedFile.h b/src/Formats/MarkInCompressedFile.h index ceefde43615..1cd545e1a03 100644 --- a/src/Formats/MarkInCompressedFile.h +++ b/src/Formats/MarkInCompressedFile.h @@ -33,7 +33,7 @@ struct MarkInCompressedFile return "(" + DB::toString(offset_in_compressed_file) + "," + DB::toString(offset_in_decompressed_block) + ")"; } - String toStringWithRows(size_t rows_num) + String toStringWithRows(size_t rows_num) const { return "(" + DB::toString(offset_in_compressed_file) + "," + DB::toString(offset_in_decompressed_block) + "," + DB::toString(rows_num) + ")"; } @@ -43,7 +43,7 @@ struct MarkInCompressedFile class MarksInCompressedFile : public PODArray { public: - MarksInCompressedFile(size_t n) : PODArray(n) {} + explicit MarksInCompressedFile(size_t n) : PODArray(n) {} void read(ReadBuffer & buffer, size_t from, size_t count) { diff --git a/src/Formats/MsgPackExtensionTypes.h b/src/Formats/MsgPackExtensionTypes.h index 139d2f9047b..2f7d28eb5bf 100644 --- a/src/Formats/MsgPackExtensionTypes.h +++ b/src/Formats/MsgPackExtensionTypes.h @@ -5,7 +5,7 @@ namespace DB enum class MsgPackExtensionTypes { - UUID = 0x02, + UUIDType = 0x02, }; } diff --git a/src/Formats/ParsedTemplateFormatString.h b/src/Formats/ParsedTemplateFormatString.h index c5617d0f0ef..5d7ee820f2f 100644 --- a/src/Formats/ParsedTemplateFormatString.h +++ b/src/Formats/ParsedTemplateFormatString.h @@ -28,7 +28,7 @@ struct ParsedTemplateFormatString /// For diagnostic info Strings column_names; - typedef std::function(const String &)> ColumnIdxGetter; + using ColumnIdxGetter = std::function(const String &)>; ParsedTemplateFormatString() = default; ParsedTemplateFormatString(const FormatSchemaInfo & schema, const ColumnIdxGetter & idx_by_name, bool allow_indexes = true); diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h index 0df139eeacd..2e2a71a7d11 100644 --- a/src/Formats/ProtobufReader.h +++ b/src/Formats/ProtobufReader.h @@ -16,7 +16,7 @@ class ReadBuffer; class ProtobufReader { public: - ProtobufReader(ReadBuffer & in_); + explicit ProtobufReader(ReadBuffer & in_); void startMessage(bool with_length_delimiter_); void endMessage(bool ignore_errors); diff --git a/src/Formats/ProtobufSchemas.cpp b/src/Formats/ProtobufSchemas.cpp index 9f25f830e37..249737d1838 100644 --- a/src/Formats/ProtobufSchemas.cpp +++ b/src/Formats/ProtobufSchemas.cpp @@ -24,7 +24,9 @@ ProtobufSchemas & ProtobufSchemas::instance() class ProtobufSchemas::ImporterWithSourceTree : public google::protobuf::compiler::MultiFileErrorCollector { public: - explicit ImporterWithSourceTree(const String & schema_directory) : importer(&disk_source_tree, this) + explicit ImporterWithSourceTree(const String & schema_directory, WithEnvelope with_envelope_) + : importer(&disk_source_tree, this) + , with_envelope(with_envelope_) { disk_source_tree.MapPath("", schema_directory); } @@ -39,16 +41,33 @@ public: return descriptor; const auto * file_descriptor = importer.Import(schema_path); - // If there are parsing errors AddError() throws an exception and in this case the following line + // If there are parsing errors, AddError() throws an exception and in this case the following line // isn't executed. assert(file_descriptor); - descriptor = file_descriptor->FindMessageTypeByName(message_name); - if (!descriptor) - throw Exception( - "Not found a message named '" + message_name + "' in the schema file '" + schema_path + "'", ErrorCodes::BAD_ARGUMENTS); + if (with_envelope == WithEnvelope::No) + { + const auto * message_descriptor = file_descriptor->FindMessageTypeByName(message_name); + if (!message_descriptor) + throw Exception( + "Could not find a message named '" + message_name + "' in the schema file '" + schema_path + "'", ErrorCodes::BAD_ARGUMENTS); - return descriptor; + return message_descriptor; + } + else + { + const auto * envelope_descriptor = file_descriptor->FindMessageTypeByName("Envelope"); + if (!envelope_descriptor) + throw Exception( + "Could not find a message named 'Envelope' in the schema file '" + schema_path + "'", ErrorCodes::BAD_ARGUMENTS); + + const auto * message_descriptor = envelope_descriptor->FindNestedTypeByName(message_name); // silly protobuf API disallows a restricting the field type to messages + if (!message_descriptor) + throw Exception( + "Could not find a message named '" + message_name + "' in the schema file '" + schema_path + "'", ErrorCodes::BAD_ARGUMENTS); + + return message_descriptor; + } } private: @@ -63,18 +82,16 @@ private: google::protobuf::compiler::DiskSourceTree disk_source_tree; google::protobuf::compiler::Importer importer; + const WithEnvelope with_envelope; }; -ProtobufSchemas::ProtobufSchemas() = default; -ProtobufSchemas::~ProtobufSchemas() = default; - -const google::protobuf::Descriptor * ProtobufSchemas::getMessageTypeForFormatSchema(const FormatSchemaInfo & info) +const google::protobuf::Descriptor * ProtobufSchemas::getMessageTypeForFormatSchema(const FormatSchemaInfo & info, WithEnvelope with_envelope) { std::lock_guard lock(mutex); auto it = importers.find(info.schemaDirectory()); if (it == importers.end()) - it = importers.emplace(info.schemaDirectory(), std::make_unique(info.schemaDirectory())).first; + it = importers.emplace(info.schemaDirectory(), std::make_unique(info.schemaDirectory(), with_envelope)).first; auto * importer = it->second.get(); return importer->import(info.schemaPath(), info.messageName()); } diff --git a/src/Formats/ProtobufSchemas.h b/src/Formats/ProtobufSchemas.h index 0a2eeea9893..40e386b4642 100644 --- a/src/Formats/ProtobufSchemas.h +++ b/src/Formats/ProtobufSchemas.h @@ -28,14 +28,36 @@ class FormatSchemaInfo; class ProtobufSchemas : private boost::noncopyable { public: - static ProtobufSchemas & instance(); + enum class WithEnvelope + { + // Return descriptor for a top-level message with a user-provided name. + // Example: In protobuf schema + // message MessageType { + // string colA = 1; + // int32 colB = 2; + // } + // message_name = "MessageType" returns a descriptor. Used by IO + // formats Protobuf and ProtobufSingle. + No, + // Return descriptor for a message with a user-provided name one level + // below a top-level message with the hardcoded name "Envelope". + // Example: In protobuf schema + // message Envelope { + // message MessageType { + // string colA = 1; + // int32 colB = 2; + // } + // } + // message_name = "MessageType" returns a descriptor. Used by IO format + // ProtobufList. + Yes + }; - ProtobufSchemas(); - ~ProtobufSchemas(); + static ProtobufSchemas & instance(); /// Parses the format schema, then parses the corresponding proto file, and returns the descriptor of the message type. /// The function never returns nullptr, it throws an exception if it cannot load or parse the file. - const google::protobuf::Descriptor * getMessageTypeForFormatSchema(const FormatSchemaInfo & info); + const google::protobuf::Descriptor * getMessageTypeForFormatSchema(const FormatSchemaInfo & info, WithEnvelope with_envelope); private: class ImporterWithSourceTree; diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 389d25a1f46..b44342d0ca5 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -2171,6 +2171,11 @@ namespace field_index_by_field_tag.emplace(field_infos[i].field_tag, i); } + void setHasEnvelopeAsParent() + { + has_envelope_as_parent = true; + } + void setColumns(const ColumnPtr * columns_, size_t num_columns_) override { if (!num_columns_) @@ -2217,7 +2222,7 @@ namespace void writeRow(size_t row_num) override { - if (parent_field_descriptor) + if (parent_field_descriptor || has_envelope_as_parent) writer->startNestedMessage(); else writer->startMessage(); @@ -2236,13 +2241,17 @@ namespace bool is_group = (parent_field_descriptor->type() == FieldTypeId::TYPE_GROUP); writer->endNestedMessage(parent_field_descriptor->number(), is_group, should_skip_if_empty); } + else if (has_envelope_as_parent) + { + writer->endNestedMessage(1, false, should_skip_if_empty); + } else writer->endMessage(with_length_delimiter); } void readRow(size_t row_num) override { - if (parent_field_descriptor) + if (parent_field_descriptor || has_envelope_as_parent) reader->startNestedMessage(); else reader->startMessage(with_length_delimiter); @@ -2285,7 +2294,7 @@ namespace } } - if (parent_field_descriptor) + if (parent_field_descriptor || has_envelope_as_parent) reader->endNestedMessage(); else reader->endMessage(false); @@ -2375,6 +2384,7 @@ namespace }; const FieldDescriptor * const parent_field_descriptor; + bool has_envelope_as_parent = false; const bool with_length_delimiter; const std::unique_ptr missing_columns_filler; const bool should_skip_if_empty; @@ -2388,6 +2398,86 @@ namespace size_t last_field_index = static_cast(-1); }; + /// Serializes a top-level envelope message in the protobuf schema. + /// "Envelope" means that the contained subtree of serializers is enclosed in a message just once, + /// i.e. only when the first and the last row read/write trigger a read/write of the msg header. + class ProtobufSerializerEnvelope : public ProtobufSerializer + { + public: + ProtobufSerializerEnvelope( + std::unique_ptr&& serializer_, + const ProtobufReaderOrWriter & reader_or_writer_) + : serializer(std::move(serializer_)) + , reader(reader_or_writer_.reader) + , writer(reader_or_writer_.writer) + { + // The inner serializer has a backreference of type protobuf::FieldDescriptor * to it's parent + // serializer. If it is unset, it considers itself the top-level message, otherwise a nested + // message and accordingly it makes start/endMessage() vs. startEndNestedMessage() calls into + // Protobuf(Writer|Reader). There is no field descriptor because Envelopes merely forward calls + // but don't contain data to be serialized. We must still force the inner serializer to act + // as nested message. + serializer->setHasEnvelopeAsParent(); + } + + void setColumns(const ColumnPtr * columns_, size_t num_columns_) override + { + serializer->setColumns(columns_, num_columns_); + } + + void setColumns(const MutableColumnPtr * columns_, size_t num_columns_) override + { + serializer->setColumns(columns_, num_columns_); + } + + void writeRow(size_t row_num) override + { + if (first_call_of_write_row) + { + writer->startMessage(); + first_call_of_write_row = false; + } + + serializer->writeRow(row_num); + } + + void finalizeWrite() override + { + writer->endMessage(/*with_length_delimiter = */ true); + } + + void readRow(size_t row_num) override + { + if (first_call_of_read_row) + { + reader->startMessage(/*with_length_delimiter = */ true); + first_call_of_read_row = false; + } + + int field_tag; + [[maybe_unused]] bool ret = reader->readFieldNumber(field_tag); + assert(ret); + + serializer->readRow(row_num); + } + + void insertDefaults(size_t row_num) override + { + serializer->insertDefaults(row_num); + } + + void describeTree(WriteBuffer & out, size_t indent) const override + { + writeIndent(out, indent) << "ProtobufSerializerEnvelope ->\n"; + serializer->describeTree(out, indent + 1); + } + + std::unique_ptr serializer; + ProtobufReader * const reader; + ProtobufWriter * const writer; + bool first_call_of_write_row = true; + bool first_call_of_read_row = true; + }; /// Serializes a tuple with explicit names as a nested message. class ProtobufSerializerTupleAsNestedMessage : public ProtobufSerializer @@ -2610,7 +2700,8 @@ namespace const DataTypes & data_types, std::vector & missing_column_indices, const MessageDescriptor & message_descriptor, - bool with_length_delimiter) + bool with_length_delimiter, + bool with_envelope) { root_serializer_ptr = std::make_shared(); get_root_desc_function = [root_serializer_ptr = root_serializer_ptr](size_t indent) -> String @@ -2648,13 +2739,23 @@ namespace boost::range::set_difference(collections::range(column_names.size()), used_column_indices_sorted, std::back_inserter(missing_column_indices)); - *root_serializer_ptr = message_serializer.get(); - + if (!with_envelope) + { + *root_serializer_ptr = message_serializer.get(); #if 0 - LOG_INFO(&Poco::Logger::get("ProtobufSerializer"), "Serialization tree:\n{}", get_root_desc_function(0)); + LOG_INFO(&Poco::Logger::get("ProtobufSerializer"), "Serialization tree:\n{}", get_root_desc_function(0)); #endif - - return message_serializer; + return message_serializer; + } + else + { + auto envelope_serializer = std::make_unique(std::move(message_serializer), reader_or_writer); + *root_serializer_ptr = envelope_serializer.get(); +#if 0 + LOG_INFO(&Poco::Logger::get("ProtobufSerializer"), "Serialization tree:\n{}", get_root_desc_function(0)); +#endif + return envelope_serializer; + } } private: @@ -3337,9 +3438,10 @@ std::unique_ptr ProtobufSerializer::create( std::vector & missing_column_indices, const google::protobuf::Descriptor & message_descriptor, bool with_length_delimiter, + bool with_envelope, ProtobufReader & reader) { - return ProtobufSerializerBuilder(reader).buildMessageSerializer(column_names, data_types, missing_column_indices, message_descriptor, with_length_delimiter); + return ProtobufSerializerBuilder(reader).buildMessageSerializer(column_names, data_types, missing_column_indices, message_descriptor, with_length_delimiter, with_envelope); } std::unique_ptr ProtobufSerializer::create( @@ -3347,10 +3449,11 @@ std::unique_ptr ProtobufSerializer::create( const DataTypes & data_types, const google::protobuf::Descriptor & message_descriptor, bool with_length_delimiter, + bool with_envelope, ProtobufWriter & writer) { std::vector missing_column_indices; - return ProtobufSerializerBuilder(writer).buildMessageSerializer(column_names, data_types, missing_column_indices, message_descriptor, with_length_delimiter); + return ProtobufSerializerBuilder(writer).buildMessageSerializer(column_names, data_types, missing_column_indices, message_descriptor, with_length_delimiter, with_envelope); } NamesAndTypesList protobufSchemaToCHSchema(const google::protobuf::Descriptor * message_descriptor) diff --git a/src/Formats/ProtobufSerializer.h b/src/Formats/ProtobufSerializer.h index d9bed913517..6ebb50cc636 100644 --- a/src/Formats/ProtobufSerializer.h +++ b/src/Formats/ProtobufSerializer.h @@ -26,6 +26,7 @@ public: virtual void setColumns(const ColumnPtr * columns, size_t num_columns) = 0; virtual void writeRow(size_t row_num) = 0; + virtual void finalizeWrite() {} virtual void setColumns(const MutableColumnPtr * columns, size_t num_columns) = 0; virtual void readRow(size_t row_num) = 0; @@ -39,6 +40,7 @@ public: std::vector & missing_column_indices, const google::protobuf::Descriptor & message_descriptor, bool with_length_delimiter, + bool with_envelope, ProtobufReader & reader); static std::unique_ptr create( @@ -46,6 +48,7 @@ public: const DataTypes & data_types, const google::protobuf::Descriptor & message_descriptor, bool with_length_delimiter, + bool with_envelope, ProtobufWriter & writer); }; diff --git a/src/Formats/ProtobufWriter.h b/src/Formats/ProtobufWriter.h index c564db110cc..1dcc8f4ef7c 100644 --- a/src/Formats/ProtobufWriter.h +++ b/src/Formats/ProtobufWriter.h @@ -16,7 +16,7 @@ class WriteBuffer; class ProtobufWriter { public: - ProtobufWriter(WriteBuffer & out_); + explicit ProtobufWriter(WriteBuffer & out_); ~ProtobufWriter(); void startMessage(); diff --git a/src/Formats/RowInputMissingColumnsFiller.h b/src/Formats/RowInputMissingColumnsFiller.h index 0eaefd4e814..9785d8bed62 100644 --- a/src/Formats/RowInputMissingColumnsFiller.h +++ b/src/Formats/RowInputMissingColumnsFiller.h @@ -14,7 +14,7 @@ class RowInputMissingColumnsFiller { public: /// Makes a column filler which checks nested structures while adding default values to columns. - RowInputMissingColumnsFiller(const NamesAndTypesList & names_and_types); + explicit RowInputMissingColumnsFiller(const NamesAndTypesList & names_and_types); RowInputMissingColumnsFiller(const Names & names, const DataTypes & types); RowInputMissingColumnsFiller(size_t count, const std::string_view * names, const DataTypePtr * types); diff --git a/src/Formats/configure_config.cmake b/src/Formats/configure_config.cmake new file mode 100644 index 00000000000..3a11f3c6448 --- /dev/null +++ b/src/Formats/configure_config.cmake @@ -0,0 +1,20 @@ +if (TARGET ch_contrib::avrocpp) + set(USE_AVRO 1) +endif() +if (TARGET ch_contrib::parquet) + set(USE_PARQUET 1) + set(USE_ARROW 1) + set(USE_ORC 1) +endif() +if (TARGET ch_contrib::snappy) + set(USE_SNAPPY 1) +endif() +if (TARGET ch_contrib::protobuf) + set(USE_PROTOBUF 1) +endif() +if (TARGET ch_contrib::msgpack) + set(USE_MSGPACK 1) +endif() +if (TARGET ch_contrib::capnp) + set(USE_CAPNP 1) +endif() diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp index b7b3b51cd7b..210ef1953b1 100644 --- a/src/Formats/registerFormats.cpp +++ b/src/Formats/registerFormats.cpp @@ -36,6 +36,8 @@ void registerInputFormatJSONCompactEachRow(FormatFactory & factory); void registerOutputFormatJSONCompactEachRow(FormatFactory & factory); void registerInputFormatProtobuf(FormatFactory & factory); void registerOutputFormatProtobuf(FormatFactory & factory); +void registerInputFormatProtobufList(FormatFactory & factory); +void registerOutputFormatProtobufList(FormatFactory & factory); void registerInputFormatTemplate(FormatFactory & factory); void registerOutputFormatTemplate(FormatFactory & factory); void registerInputFormatMsgPack(FormatFactory & factory); @@ -74,6 +76,7 @@ void registerOutputFormatCapnProto(FormatFactory & factory); void registerInputFormatRegexp(FormatFactory & factory); void registerInputFormatJSONAsString(FormatFactory & factory); +void registerInputFormatJSONAsObject(FormatFactory & factory); void registerInputFormatLineAsString(FormatFactory & factory); void registerInputFormatCapnProto(FormatFactory & factory); @@ -84,6 +87,7 @@ void registerInputFormatHiveText(FormatFactory & factory); /// Non trivial prefix and suffix checkers for disabling parallel parsing. void registerNonTrivialPrefixAndSuffixCheckerJSONEachRow(FormatFactory & factory); void registerNonTrivialPrefixAndSuffixCheckerJSONAsString(FormatFactory & factory); +void registerNonTrivialPrefixAndSuffixCheckerJSONAsObject(FormatFactory & factory); void registerArrowSchemaReader(FormatFactory & factory); void registerParquetSchemaReader(FormatFactory & factory); @@ -96,6 +100,7 @@ void registerNativeSchemaReader(FormatFactory & factory); void registerRowBinaryWithNamesAndTypesSchemaReader(FormatFactory & factory); void registerAvroSchemaReader(FormatFactory & factory); void registerProtobufSchemaReader(FormatFactory & factory); +void registerProtobufListSchemaReader(FormatFactory & factory); void registerLineAsStringSchemaReader(FormatFactory & factory); void registerJSONAsStringSchemaReader(FormatFactory & factory); void registerRawBLOBSchemaReader(FormatFactory & factory); @@ -138,6 +143,8 @@ void registerFormats() registerInputFormatJSONCompactEachRow(factory); registerOutputFormatJSONCompactEachRow(factory); registerInputFormatProtobuf(factory); + registerOutputFormatProtobufList(factory); + registerInputFormatProtobufList(factory); registerOutputFormatProtobuf(factory); registerInputFormatTemplate(factory); registerOutputFormatTemplate(factory); @@ -175,6 +182,7 @@ void registerFormats() registerInputFormatRegexp(factory); registerInputFormatJSONAsString(factory); registerInputFormatLineAsString(factory); + registerInputFormatJSONAsObject(factory); #if USE_HIVE registerInputFormatHiveText(factory); #endif @@ -183,6 +191,7 @@ void registerFormats() registerNonTrivialPrefixAndSuffixCheckerJSONEachRow(factory); registerNonTrivialPrefixAndSuffixCheckerJSONAsString(factory); + registerNonTrivialPrefixAndSuffixCheckerJSONAsObject(factory); registerArrowSchemaReader(factory); registerParquetSchemaReader(factory); @@ -195,6 +204,7 @@ void registerFormats() registerRowBinaryWithNamesAndTypesSchemaReader(factory); registerAvroSchemaReader(factory); registerProtobufSchemaReader(factory); + registerProtobufListSchemaReader(factory); registerLineAsStringSchemaReader(factory); registerJSONAsStringSchemaReader(factory); registerRawBLOBSchemaReader(factory); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index b7020ea128e..2596b10503f 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -48,7 +48,7 @@ endif() option(STRIP_DEBUG_SYMBOLS_FUNCTIONS "Do not generate debugger info for ClickHouse functions" ${STRIP_DSF_DEFAULT}) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - message(WARNING "Not generating debugger info for ClickHouse functions") + message(INFO "Not generating debugger info for ClickHouse functions") target_compile_options(clickhouse_functions PRIVATE "-g0") else() message(STATUS "Generating debugger info for ClickHouse functions") diff --git a/src/Functions/CastOverloadResolver.h b/src/Functions/CastOverloadResolver.h index ffd5dda4af3..cff17d810fe 100644 --- a/src/Functions/CastOverloadResolver.h +++ b/src/Functions/CastOverloadResolver.h @@ -33,22 +33,27 @@ public: ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - explicit CastOverloadResolverImpl(std::optional diagnostic_, bool keep_nullable_) - : diagnostic(std::move(diagnostic_)), keep_nullable(keep_nullable_) + explicit CastOverloadResolverImpl(std::optional diagnostic_, bool keep_nullable_, bool cast_ipv4_ipv6_default_on_conversion_error_) + : diagnostic(std::move(diagnostic_)) + , keep_nullable(keep_nullable_) + , cast_ipv4_ipv6_default_on_conversion_error(cast_ipv4_ipv6_default_on_conversion_error_) { } static FunctionOverloadResolverPtr create(ContextPtr context) { + const auto & settings_ref = context->getSettingsRef(); + if constexpr (internal) - return createImpl(); - return createImpl({}, context->getSettingsRef().cast_keep_nullable); + return createImpl({}, false /*keep_nullable*/, false /*cast_ipv4_ipv6_default_on_conversion_error*/); + + return createImpl({}, settings_ref.cast_keep_nullable, settings_ref.cast_ipv4_ipv6_default_on_conversion_error); } - static FunctionOverloadResolverPtr createImpl(std::optional diagnostic = {}, bool keep_nullable = false) + static FunctionOverloadResolverPtr createImpl(std::optional diagnostic = {}, bool keep_nullable = false, bool cast_ipv4_ipv6_default_on_conversion_error = false) { assert(!internal || !keep_nullable); - return std::make_unique(std::move(diagnostic), keep_nullable); + return std::make_unique(std::move(diagnostic), keep_nullable, cast_ipv4_ipv6_default_on_conversion_error); } protected: @@ -61,7 +66,7 @@ protected: data_types[i] = arguments[i].type; auto monotonicity = MonotonicityHelper::getMonotonicityInformation(arguments.front().type, return_type.get()); - return std::make_unique>(name, std::move(monotonicity), data_types, return_type, diagnostic, cast_type); + return std::make_unique>(name, std::move(monotonicity), data_types, return_type, diagnostic, cast_type, cast_ipv4_ipv6_default_on_conversion_error); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -98,6 +103,7 @@ protected: private: std::optional diagnostic; bool keep_nullable; + bool cast_ipv4_ipv6_default_on_conversion_error; }; @@ -115,7 +121,10 @@ struct CastInternalOverloadName static constexpr auto accurate_cast_or_null_name = "accurate_CastOrNull"; }; -template using CastOverloadResolver = CastOverloadResolverImpl; -template using CastInternalOverloadResolver = CastOverloadResolverImpl; +template +using CastOverloadResolver = CastOverloadResolverImpl; + +template +using CastInternalOverloadResolver = CastOverloadResolverImpl; } diff --git a/src/Functions/CountSubstringsImpl.h b/src/Functions/CountSubstringsImpl.h index 6668ca0a392..fc6e4a0e671 100644 --- a/src/Functions/CountSubstringsImpl.h +++ b/src/Functions/CountSubstringsImpl.h @@ -83,7 +83,7 @@ struct CountSubstringsImpl { res = 0; - if (needle.size() == 0) + if (needle.empty()) return; auto start = std::max(start_pos, UInt64(1)); diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index a7f06689820..bc1ae807e7d 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -41,6 +41,11 @@ namespace ErrorCodes throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } + static inline UInt32 dateTimeIsNotSupported(const char * name) + { + throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { @@ -311,6 +316,133 @@ struct ToStartOfSecondImpl using FactorTransform = ZeroTransform; }; +struct ToStartOfMillisecondImpl +{ + static constexpr auto name = "toStartOfMillisecond"; + + static inline DateTime64 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl &) + { + // given that scale is 6, scale_multiplier is 1000000 + // for DateTime64 value of 123.456789: + // 123456789 - 789 = 123456000 + // for DateTime64 value of -123.456789: + // -123456789 - (1000 + (-789)) = -123457000 + + if (scale_multiplier == 1000) + { + return datetime64; + } + else if (scale_multiplier <= 1000) + { + return datetime64 * (1000 / scale_multiplier); + } + else + { + auto droppable_part_with_sign = DecimalUtils::getFractionalPartWithScaleMultiplier(datetime64, scale_multiplier / 1000); + + if (droppable_part_with_sign < 0) + droppable_part_with_sign += scale_multiplier; + + return datetime64 - droppable_part_with_sign; + } + } + + static inline UInt32 execute(UInt32, const DateLUTImpl &) + { + throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + static inline UInt32 execute(Int32, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + static inline UInt32 execute(UInt16, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + + using FactorTransform = ZeroTransform; +}; + +struct ToStartOfMicrosecondImpl +{ + static constexpr auto name = "toStartOfMicrosecond"; + + static inline DateTime64 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl &) + { + // @see ToStartOfMillisecondImpl + + if (scale_multiplier == 1000000) + { + return datetime64; + } + else if (scale_multiplier <= 1000000) + { + return datetime64 * (1000000 / scale_multiplier); + } + else + { + auto droppable_part_with_sign = DecimalUtils::getFractionalPartWithScaleMultiplier(datetime64, scale_multiplier / 1000000); + + if (droppable_part_with_sign < 0) + droppable_part_with_sign += scale_multiplier; + + return datetime64 - droppable_part_with_sign; + } + } + + static inline UInt32 execute(UInt32, const DateLUTImpl &) + { + throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + static inline UInt32 execute(Int32, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + static inline UInt32 execute(UInt16, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + + using FactorTransform = ZeroTransform; +}; + +struct ToStartOfNanosecondImpl +{ + static constexpr auto name = "toStartOfNanosecond"; + + static inline DateTime64 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl &) + { + // @see ToStartOfMillisecondImpl + if (scale_multiplier == 1000000000) + { + return datetime64; + } + else if (scale_multiplier <= 1000000000) + { + return datetime64 * (1000000000 / scale_multiplier); + } + else + { + throw Exception("Illegal type of argument for function " + std::string(name) + ", DateTime64 expected", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + + static inline UInt32 execute(UInt32, const DateLUTImpl &) + { + throw Exception("Illegal type DateTime of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + static inline UInt32 execute(Int32, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + static inline UInt32 execute(UInt16, const DateLUTImpl &) + { + return dateIsNotSupported(name); + } + + using FactorTransform = ZeroTransform; +}; + struct ToStartOfFiveMinuteImpl { static constexpr auto name = "toStartOfFiveMinute"; diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index 2e601888ecc..c246f7fd31a 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -6,6 +6,7 @@ #include #include +#include "config_core.h" #include diff --git a/src/Functions/EmptyImpl.h b/src/Functions/EmptyImpl.h index 60daa66ea03..6f5c4f7a7dc 100644 --- a/src/Functions/EmptyImpl.h +++ b/src/Functions/EmptyImpl.h @@ -2,6 +2,7 @@ #include #include +#include #include diff --git a/src/Functions/FunctionBitTestMany.h b/src/Functions/FunctionBitTestMany.h index 808c3711631..e49af4c166f 100644 --- a/src/Functions/FunctionBitTestMany.h +++ b/src/Functions/FunctionBitTestMany.h @@ -5,6 +5,7 @@ #include #include #include +#include #include diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 542062151ce..6ed751fd889 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 8f6b1370935..fbfc9e9bc1f 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -1,4 +1,6 @@ #pragma once +#include +#include #include #include @@ -22,9 +24,10 @@ namespace DB namespace ErrorCodes { - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int DECIMAL_OVERFLOW; extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /// Type of first argument of 'execute' function overload defines what INPUT DataType it is used for. @@ -37,26 +40,158 @@ namespace ErrorCodes /// - 'AddSecondsImpl::execute(UInt32, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(DateTime, ...) -> DateTime' /// - 'AddSecondsImpl::execute(UInt16, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(Date, ...) -> DateTime' +struct AddNanosecondsImpl +{ + static constexpr auto name = "addNanoseconds"; + + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &, UInt16 scale = DataTypeDateTime64::default_scale) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(9 - scale); + auto division = std::div(t.fractional * multiplier + delta, static_cast(1000000000)); + return {t.whole * multiplier + division.quot, t.fractional * multiplier + delta}; + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(9 - scale); + return t * multiplier + delta; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &, UInt16 = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(9); + return t * multiplier + delta; + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 execute(UInt16, Int64, const DateLUTImpl &, UInt16 = 0) + { + throw Exception("addNanoSeconds() cannot be used with Date", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 execute(Int32, Int64, const DateLUTImpl &, UInt16 = 0) + { + throw Exception("addNanoSeconds() cannot be used with Date32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } +}; + +struct AddMicrosecondsImpl +{ + static constexpr auto name = "addMicroseconds"; + + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(std::abs(6 - scale)); + if (scale <= 6) + { + auto division = std::div((t.fractional + delta), static_cast(10e6)); + return {t.whole * multiplier + division.quot, division.rem}; + } + else + { + auto division = std::div((t.fractional + delta * multiplier), static_cast(10e6 * multiplier)); + return {t.whole + division.quot, division.rem}; + } + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(std::abs(6 - scale)); + return scale <= 6 ? t * multiplier + delta : t + delta * multiplier; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &, UInt16 = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(6); + return t * multiplier + delta; + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 execute(UInt16, Int64, const DateLUTImpl &, UInt16 = 0) + { + throw Exception("addMicroSeconds() cannot be used with Date", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 execute(Int32, Int64, const DateLUTImpl &, UInt16 = 0) + { + throw Exception("addMicroSeconds() cannot be used with Date32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } +}; + +struct AddMillisecondsImpl +{ + static constexpr auto name = "addMilliseconds"; + + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &, UInt16 scale = DataTypeDateTime64::default_scale) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(std::abs(3 - scale)); + if (scale <= 3) + { + auto division = std::div((t.fractional + delta), static_cast(1000)); + return {t.whole * multiplier + division.quot, division.rem}; + } + else + { + auto division = std::div((t.fractional + delta * multiplier), static_cast(1000 * multiplier)); + return {t.whole + division.quot,division.rem}; + } + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(std::abs(3 - scale)); + return scale <= 3 ? t * multiplier + delta : t + delta * multiplier; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &, UInt16 = 0) + { + Int64 multiplier = DecimalUtils::scaleMultiplier(3); + return t * multiplier + delta; + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 execute(UInt16, Int64, const DateLUTImpl &, UInt16 = 0) + { + throw Exception("addMilliSeconds() cannot be used with Date", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + static inline NO_SANITIZE_UNDEFINED DateTime64 execute(Int32, Int64, const DateLUTImpl &, UInt16 = 0) + { + throw Exception("addMilliSeconds() cannot be used with Date32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } +}; + struct AddSecondsImpl { static constexpr auto name = "addSeconds"; static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return {t.whole + delta, t.fractional}; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + return t + delta * DecimalUtils::scaleMultiplier(scale); + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return t + delta; } - static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone) + + static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { // use default datetime64 scale return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta) * 1000; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.fromDayNum(DayNum(d)) + delta; } @@ -67,21 +202,29 @@ struct AddMinutesImpl static constexpr auto name = "addMinutes"; static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return {t.whole + delta * 60, t.fractional}; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + return t + 60 * delta * DecimalUtils::scaleMultiplier(scale); + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return t + delta * 60; } - static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone) + + static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { // use default datetime64 scale return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60) * 1000; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.fromDayNum(DayNum(d)) + delta * 60; } @@ -92,20 +235,29 @@ struct AddHoursImpl static constexpr auto name = "addHours"; static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return {t.whole + delta * 3600, t.fractional}; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) + + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl &, UInt16 scale = 0) + { + return t + 3600 * delta * DecimalUtils::scaleMultiplier(scale); + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return t + delta * 3600; } - static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone) + + static inline NO_SANITIZE_UNDEFINED Int64 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { // use default datetime64 scale return (time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600) * 1000; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.fromDayNum(DayNum(d)) + delta * 3600; } @@ -116,22 +268,30 @@ struct AddDaysImpl static constexpr auto name = "addDays"; static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return {time_zone.addDays(t.whole, delta), t.fractional}; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale = 0) + { + auto multiplier = DecimalUtils::scaleMultiplier(scale); + auto d = std::div(t, multiplier); + return time_zone.addDays(d.quot, delta) * multiplier + d.rem; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addDays(t, delta); } - static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl &) + static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return d + delta; } - static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int64 delta, const DateLUTImpl &) + static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int64 delta, const DateLUTImpl &, UInt16 = 0) { return d + delta; } @@ -142,22 +302,30 @@ struct AddWeeksImpl static constexpr auto name = "addWeeks"; static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int32 delta, const DateLUTImpl & time_zone) + execute(DecimalUtils::DecimalComponents t, Int32 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return {time_zone.addWeeks(t.whole, delta), t.fractional}; } - static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int32 delta, const DateLUTImpl & time_zone, UInt16 scale = 0) + { + auto multiplier = DecimalUtils::scaleMultiplier(scale); + auto d = std::div(t, multiplier); + return time_zone.addDays(d.quot, delta * 7) * multiplier + d.rem; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addWeeks(t, delta); } - static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl &) + static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl &, UInt16 = 0) { return d + delta * 7; } - static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int32 delta, const DateLUTImpl &) + static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int32 delta, const DateLUTImpl &, UInt16 = 0) { return d + delta * 7; } @@ -167,23 +335,31 @@ struct AddMonthsImpl { static constexpr auto name = "addMonths"; - static inline DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return {time_zone.addMonths(t.whole, delta), t.fractional}; } - static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale = 0) + { + auto multiplier = DecimalUtils::scaleMultiplier(scale); + auto d = std::div(t, multiplier); + return time_zone.addMonths(d.quot, delta) * multiplier + d.rem; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addMonths(t, delta); } - static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addMonths(DayNum(d), delta); } - static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addMonths(ExtendedDayNum(d), delta); } @@ -194,22 +370,30 @@ struct AddQuartersImpl static constexpr auto name = "addQuarters"; static inline DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int32 delta, const DateLUTImpl & time_zone) + execute(DecimalUtils::DecimalComponents t, Int32 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return {time_zone.addQuarters(t.whole, delta), t.fractional}; } - static inline UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int32 delta, const DateLUTImpl & time_zone, UInt16 scale = 0) + { + auto multiplier = DecimalUtils::scaleMultiplier(scale); + auto d = std::div(t, multiplier); + return time_zone.addQuarters(d.quot, delta) * multiplier + d.rem; + } + + static inline UInt32 execute(UInt32 t, Int32 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addQuarters(t, delta); } - static inline UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl & time_zone) + static inline UInt16 execute(UInt16 d, Int32 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addQuarters(DayNum(d), delta); } - static inline Int32 execute(Int32 d, Int32 delta, const DateLUTImpl & time_zone) + static inline Int32 execute(Int32 d, Int32 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addQuarters(ExtendedDayNum(d), delta); } @@ -219,23 +403,31 @@ struct AddYearsImpl { static constexpr auto name = "addYears"; - static inline DecimalUtils::DecimalComponents - execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents + execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return {time_zone.addYears(t.whole, delta), t.fractional}; } - static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DateTime64 + execute(DateTime64 t, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale = 0) + { + auto multiplier = DecimalUtils::scaleMultiplier(scale); + auto d = std::div(t, multiplier); + return time_zone.addYears(d.quot, delta) * multiplier + d.rem; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addYears(t, delta); } - static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addYears(DayNum(d), delta); } - static inline Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED Int32 execute(Int32 d, Int64 delta, const DateLUTImpl & time_zone, UInt16 = 0) { return time_zone.addYears(ExtendedDayNum(d), delta); } @@ -247,13 +439,16 @@ struct SubtractIntervalImpl : public Transform using Transform::Transform; template - inline NO_SANITIZE_UNDEFINED auto execute(T t, Int64 delta, const DateLUTImpl & time_zone) const + inline NO_SANITIZE_UNDEFINED auto execute(T t, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale) const { /// Signed integer overflow is Ok. - return Transform::execute(t, -delta, time_zone); + return Transform::execute(t, -delta, time_zone, scale); } }; +struct SubtractNanosecondsImpl : SubtractIntervalImpl { static constexpr auto name = "subtractNanoseconds"; }; +struct SubtractMicrosecondsImpl : SubtractIntervalImpl { static constexpr auto name = "subtractMicroseconds"; }; +struct SubtractMillisecondsImpl : SubtractIntervalImpl { static constexpr auto name = "subtractMilliseconds"; }; struct SubtractSecondsImpl : SubtractIntervalImpl { static constexpr auto name = "subtractSeconds"; }; struct SubtractMinutesImpl : SubtractIntervalImpl { static constexpr auto name = "subtractMinutes"; }; struct SubtractHoursImpl : SubtractIntervalImpl { static constexpr auto name = "subtractHours"; }; @@ -274,17 +469,17 @@ struct Adder {} template - void NO_INLINE vectorConstant(const FromVectorType & vec_from, ToVectorType & vec_to, Int64 delta, const DateLUTImpl & time_zone) const + void NO_INLINE vectorConstant(const FromVectorType & vec_from, ToVectorType & vec_to, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale) const { size_t size = vec_from.size(); vec_to.resize(size); for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], delta, time_zone); + vec_to[i] = transform.execute(vec_from[i], checkOverflow(delta), time_zone, scale); } template - void vectorVector(const FromVectorType & vec_from, ToVectorType & vec_to, const IColumn & delta, const DateLUTImpl & time_zone) const + void vectorVector(const FromVectorType & vec_from, ToVectorType & vec_to, const IColumn & delta, const DateLUTImpl & time_zone, UInt16 scale) const { size_t size = vec_from.size(); vec_to.resize(size); @@ -293,11 +488,11 @@ struct Adder ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64, ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64, ColumnFloat32, ColumnFloat64>( - &delta, [&](const auto & column){ vectorVector(vec_from, vec_to, column, time_zone, size); return true; }); + &delta, [&](const auto & column){ vectorVector(vec_from, vec_to, column, time_zone, scale, size); return true; }); } template - void constantVector(const FromType & from, ToVectorType & vec_to, const IColumn & delta, const DateLUTImpl & time_zone) const + void constantVector(const FromType & from, ToVectorType & vec_to, const IColumn & delta, const DateLUTImpl & time_zone, UInt16 scale) const { size_t size = delta.size(); vec_to.resize(size); @@ -306,24 +501,34 @@ struct Adder ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64, ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64, ColumnFloat32, ColumnFloat64>( - &delta, [&](const auto & column){ constantVector(from, vec_to, column, time_zone, size); return true; }); + &delta, [&](const auto & column){ constantVector(from, vec_to, column, time_zone, scale, size); return true; }); } private: + + template + static Int64 checkOverflow(Value val) + { + Int64 result; + if (accurate::convertNumeric(val, result)) + return result; + throw DB::Exception("Numeric overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + template NO_INLINE NO_SANITIZE_UNDEFINED void vectorVector( - const FromVectorType & vec_from, ToVectorType & vec_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, size_t size) const + const FromVectorType & vec_from, ToVectorType & vec_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, UInt16 scale, size_t size) const { for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(vec_from[i], delta.getData()[i], time_zone); + vec_to[i] = transform.execute(vec_from[i], checkOverflow(delta.getData()[i]), time_zone, scale); } template NO_INLINE NO_SANITIZE_UNDEFINED void constantVector( - const FromType & from, ToVectorType & vec_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, size_t size) const + const FromType & from, ToVectorType & vec_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, UInt16 scale, size_t size) const { for (size_t i = 0; i < size; ++i) - vec_to[i] = transform.execute(from, delta.getData()[i], time_zone); + vec_to[i] = transform.execute(from, checkOverflow(delta.getData()[i]), time_zone, scale); } }; @@ -331,7 +536,7 @@ private: template struct DateTimeAddIntervalImpl { - static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) + static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, UInt16 scale = 0) { using FromValueType = typename FromDataType::FieldType; using FromColumnType = typename FromDataType::ColumnType; @@ -346,27 +551,24 @@ struct DateTimeAddIntervalImpl auto result_col = result_type->createColumn(); auto col_to = assert_cast(result_col.get()); + const IColumn & delta_column = *arguments[1].column; if (const auto * sources = checkAndGetColumn(source_col.get())) { - const IColumn & delta_column = *arguments[1].column; - if (const auto * delta_const_column = typeid_cast(&delta_column)) - op.vectorConstant(sources->getData(), col_to->getData(), delta_const_column->getInt(0), time_zone); + op.vectorConstant(sources->getData(), col_to->getData(), delta_const_column->getInt(0), time_zone, scale); else - op.vectorVector(sources->getData(), col_to->getData(), delta_column, time_zone); + op.vectorVector(sources->getData(), col_to->getData(), delta_column, time_zone, scale); } else if (const auto * sources_const = checkAndGetColumnConst(source_col.get())) { op.constantVector( sources_const->template getValue(), - col_to->getData(), - *arguments[1].column, time_zone); + col_to->getData(), delta_column, time_zone, scale); } else { - throw Exception("Illegal column " + arguments[0].column->getName() - + " of first argument of function " + Transform::name, - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", + arguments[0].column->getName(), Transform::name); } return result_col; @@ -452,18 +654,10 @@ public: } } - // TransformDateTime64 helps choosing correct overload of exec and does some transformations - // on input and output parameters to simplify support of DateTime64 in concrete Transform. - template - using TransformType = std::conditional_t< - std::is_same_v, - TransformDateTime64, - Transform>; - /// Helper templates to deduce return type based on argument type, since some overloads may promote or denote types, /// e.g. addSeconds(Date, 1) => DateTime template - using TransformExecuteReturnType = decltype(std::declval>().execute(FieldType(), 0, std::declval())); + using TransformExecuteReturnType = decltype(std::declval().execute(FieldType(), 0, std::declval(), 0)); // Deduces RETURN DataType from INPUT DataType, based on return type of Transform{}.execute(INPUT_TYPE, UInt64, DateLUTImpl). // e.g. for Transform-type that has execute()-overload with 'UInt16' input and 'UInt32' return, @@ -489,11 +683,33 @@ public: if (typeid_cast(arguments[0].type.get())) { const auto & datetime64_type = assert_cast(*arguments[0].type); - return std::make_shared(datetime64_type.getScale(), extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); + + auto from_scale = datetime64_type.getScale(); + auto scale = from_scale; + + if (std::is_same_v) + scale = 9; + else if (std::is_same_v) + scale = 6; + else if (std::is_same_v) + scale = 3; + + scale = std::max(scale, from_scale); + + return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); } else { - return std::make_shared(DataTypeDateTime64::default_scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); + auto scale = DataTypeDateTime64::default_scale; + + if (std::is_same_v) + scale = 9; + else if (std::is_same_v) + scale = 6; + else if (std::is_same_v) + scale = 3; + + return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); } } else @@ -530,9 +746,9 @@ public: } else if (const auto * datetime64_type = assert_cast(from_type)) { - using WrappedTransformType = TransformType; - return DateTimeAddIntervalImpl, WrappedTransformType>::execute( - WrappedTransformType{datetime64_type->getScale()}, arguments, result_type); + auto from_scale = datetime64_type->getScale(); + return DateTimeAddIntervalImpl, Transform>::execute( + Transform{}, arguments, result_type, from_scale); } else throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(), diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index 00678e65364..5269eecea37 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -88,6 +88,20 @@ public: Int64 scale = DataTypeDateTime64::default_scale; if (const auto * dt64 = checkAndGetDataType(arguments[0].type.get())) scale = dt64->getScale(); + auto source_scale = scale; + + if constexpr (std::is_same_v) + { + scale = std::max(source_scale, static_cast(3)); + } + else if constexpr (std::is_same_v) + { + scale = std::max(source_scale, static_cast(6)); + } + else if constexpr (std::is_same_v) + { + scale = std::max(source_scale, static_cast(9)); + } return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 1, 0)); } diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index df131538275..da1061e4b3e 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -74,9 +74,9 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp { if (arguments.size() < 3) throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) - + ", should be greater or equal to 3", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function '{}' doesn't match: passed {}, should be greater or equal to 3", + getName() , arguments.size()); auto [storage_join, attr_name] = getJoin(arguments, getContext()); DataTypes data_types(arguments.size() - 2); DataTypes argument_types(arguments.size()); @@ -86,9 +86,13 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp data_types[i - 2] = arguments[i].type; argument_types[i] = arguments[i].type; } - auto return_type = storage_join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null); + + auto return_type = storage_join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null || storage_join->useNulls()); auto table_lock = storage_join->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + if (storage_join->useNulls()) + return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); + return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index d860da62b9d..e45951e3ec5 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -8,13 +8,13 @@ #include #include #include -#include +#include #include #include #include #include -#include -#include +#include +#include #include #include #include @@ -242,7 +242,7 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - Element res; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { if (status == VisitorStatus::Ok) diff --git a/src/Functions/FunctionSnowflake.h b/src/Functions/FunctionSnowflake.h index 1ba15433e94..f4a62e509ed 100644 --- a/src/Functions/FunctionSnowflake.h +++ b/src/Functions/FunctionSnowflake.h @@ -24,7 +24,7 @@ namespace ErrorCodes * https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake * https://ws-dl.blogspot.com/2019/08/2019-08-03-tweetedat-finding-tweet.html */ -static constexpr long snowflake_epoch = 1288834974657L; +static constexpr size_t snowflake_epoch = 1288834974657L; static constexpr int time_shift = 22; class FunctionDateTimeToSnowflake : public IFunction @@ -33,7 +33,7 @@ private: const char * name; public: - FunctionDateTimeToSnowflake(const char * name_) : name(name_) { } + explicit FunctionDateTimeToSnowflake(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } @@ -74,7 +74,7 @@ private: const char * name; public: - FunctionSnowflakeToDateTime(const char * name_) : name(name_) { } + explicit FunctionSnowflakeToDateTime(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -84,7 +84,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() < 1 || arguments.size() > 2) + if (arguments.empty() || arguments.size() > 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); if (!typeid_cast(arguments[0].type.get())) @@ -122,7 +122,7 @@ private: const char * name; public: - FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { } + explicit FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } @@ -163,7 +163,7 @@ private: const char * name; public: - FunctionSnowflakeToDateTime64(const char * name_) : name(name_) { } + explicit FunctionSnowflakeToDateTime64(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -173,7 +173,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() < 1 || arguments.size() > 2) + if (arguments.empty() || arguments.size() > 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); if (!typeid_cast(arguments[0].type.get())) diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index 5a3aba62f26..bbe1631fdf9 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -1,4 +1,6 @@ #pragma once +#include + #include #include #include @@ -7,7 +9,9 @@ #include #include #include +#include #include +#include namespace DB { @@ -17,6 +21,7 @@ using namespace GatherUtils; namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -59,16 +64,65 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (!isStringOrFixedString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (isStringOrFixedString(arguments[0]) && isStringOrFixedString(arguments[1])) + return std::make_shared(); - if (!isStringOrFixedString(arguments[1])) - throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (isArray(arguments[0]) && isArray(arguments[1])) + return std::make_shared(); - return std::make_shared(); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal types {} {} of arguments of function {}. Both must be String or Array", + arguments[0]->getName(), arguments[1]->getName(), getName()); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto data_type = arguments[0].type; + if (isStringOrFixedString(*data_type)) + return executeImplString(arguments, {}, input_rows_count); + if (isArray(data_type)) + return executeImplArray(arguments, {}, input_rows_count); + return {}; + } + +private: + ColumnPtr executeImplArray(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const + { + DataTypePtr common_type = getLeastSupertype(collections::map(arguments, [](auto & arg) { return arg.type; })); + + Columns preprocessed_columns(2); + for (size_t i = 0; i < 2; ++i) + preprocessed_columns[i] = castColumn(arguments[i], common_type); + + std::vector> sources; + for (auto & argument_column : preprocessed_columns) + { + bool is_const = false; + + if (const auto * argument_column_const = typeid_cast(argument_column.get())) + { + is_const = true; + argument_column = argument_column_const->getDataColumnPtr(); + } + + if (const auto * argument_column_array = typeid_cast(argument_column.get())) + sources.emplace_back(GatherUtils::createArraySource(*argument_column_array, is_const, input_rows_count)); + else + throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR}; + } + + auto result_column = ColumnUInt8::create(input_rows_count); + auto * result_column_ptr = typeid_cast(result_column.get()); + + if constexpr (std::is_same_v) + GatherUtils::sliceHas(*sources[0], *sources[1], GatherUtils::ArraySearchType::StartsWith, *result_column_ptr); + else + GatherUtils::sliceHas(*sources[0], *sources[1], GatherUtils::ArraySearchType::EndsWith, *result_column_ptr); + + return result_column; + } + + ColumnPtr executeImplString(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const { const IColumn * haystack_column = arguments[0].column.get(); const IColumn * needle_column = arguments[1].column.get(); @@ -92,7 +146,6 @@ public: return col_res; } -private: template void dispatch(HaystackSource haystack_source, const IColumn * needle_column, PaddedPODArray & res_data) const { diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h index 3bf1f0a5d34..cda5da5c177 100644 --- a/src/Functions/FunctionStringOrArrayToT.h +++ b/src/Functions/FunctionStringOrArrayToT.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h index 775a39f4d08..1e48588892a 100644 --- a/src/Functions/FunctionsBitmap.h +++ b/src/Functions/FunctionsBitmap.h @@ -421,7 +421,7 @@ private: for (size_t i = 0; i < input_rows_count; ++i) { - const AggregateDataPtr data_ptr_0 = is_column_const[0] ? (*container0)[0] : (*container0)[i]; + AggregateDataPtr data_ptr_0 = is_column_const[0] ? (*container0)[0] : (*container0)[i]; const AggregateFunctionGroupBitmapData & bitmap_data_0 = *reinterpret_cast*>(data_ptr_0); const UInt64 range_start = is_column_const[1] ? (*container1)[0] : (*container1)[i]; @@ -615,7 +615,7 @@ private: size_t to_end; for (size_t i = 0; i < input_rows_count; ++i) { - const AggregateDataPtr data_ptr_0 = is_column_const[0] ? (*container0)[0] : (*container0)[i]; + AggregateDataPtr data_ptr_0 = is_column_const[0] ? (*container0)[0] : (*container0)[i]; const AggregateFunctionGroupBitmapData & bitmap_data_0 = *reinterpret_cast *>(data_ptr_0); if (is_column_const[1]) @@ -923,7 +923,7 @@ private: for (size_t i = 0; i < input_rows_count; ++i) { - const AggregateDataPtr data_ptr_0 = is_column_const[0] ? (*container0)[0] : (*container0)[i]; + AggregateDataPtr data_ptr_0 = is_column_const[0] ? (*container0)[0] : (*container0)[i]; const UInt64 data1 = is_column_const[1] ? (*container1)[0] : (*container1)[i]; const AggregateFunctionGroupBitmapData & bitmap_data_0 = *reinterpret_cast *>(data_ptr_0); @@ -1030,8 +1030,8 @@ private: for (size_t i = 0; i < input_rows_count; ++i) { - const AggregateDataPtr data_ptr_0 = is_column_const[0] ? container0[0] : container0[i]; - const AggregateDataPtr data_ptr_1 = is_column_const[1] ? container1[0] : container1[i]; + AggregateDataPtr data_ptr_0 = is_column_const[0] ? container0[0] : container0[i]; + AggregateDataPtr data_ptr_1 = is_column_const[1] ? container1[0] : container1[i]; const AggregateFunctionGroupBitmapData & bitmap_data_1 = *reinterpret_cast *>(data_ptr_0); const AggregateFunctionGroupBitmapData & bitmap_data_2 @@ -1178,8 +1178,8 @@ private: for (size_t i = 0; i < input_rows_count; ++i) { - const AggregateDataPtr data_ptr_0 = is_column_const[0] ? container0[0] : container0[i]; - const AggregateDataPtr data_ptr_1 = is_column_const[1] ? container1[0] : container1[i]; + AggregateDataPtr data_ptr_0 = is_column_const[0] ? container0[0] : container0[i]; + AggregateDataPtr data_ptr_1 = is_column_const[1] ? container1[0] : container1[i]; // bitmapAnd(RoaringBitMap, SmallSet) is slower than bitmapAnd(SmallSet, RoaringBitMap), so we can exchange the position of two arguments for the speed auto * bm_1 = reinterpret_cast *>(data_ptr_0); diff --git a/src/Functions/FunctionsCharsetClassification.cpp b/src/Functions/FunctionsCharsetClassification.cpp index d29dc14fa9f..af6c1de2768 100644 --- a/src/Functions/FunctionsCharsetClassification.cpp +++ b/src/Functions/FunctionsCharsetClassification.cpp @@ -8,26 +8,21 @@ namespace DB { -/* Determine language and charset of text data. For each text, we build the distribution of bigrams bytes. - * Then we use marked-up dictionaries with distributions of bigram bytes of various languages ​​and charsets. - * Using a naive Bayesian classifier, find the most likely charset and language and return it - */ - -template -struct CharsetClassificationImpl +namespace { /* We need to solve zero-frequency problem for Naive Bayes Classifier * If the bigram is not found in the text, we assume that the probability of its meeting is 1e-06. * 1e-06 is minimal value in our marked-up dictionary. */ - static constexpr Float64 zero_frequency = 1e-06; + constexpr Float64 zero_frequency = 1e-06; /// If the data size is bigger than this, behaviour is unspecified for this function. - static constexpr size_t max_string_size = 1u << 15; + constexpr size_t max_string_size = 1UL << 15; - static ALWAYS_INLINE inline Float64 naiveBayes( + template + ALWAYS_INLINE inline Float64 naiveBayes( const FrequencyHolder::EncodingMap & standard, - const HashMap & model, + const ModelMap & model, Float64 max_result) { Float64 res = 0; @@ -52,10 +47,11 @@ struct CharsetClassificationImpl } /// Сount how many times each bigram occurs in the text. - static ALWAYS_INLINE inline void calculateStats( + template + ALWAYS_INLINE inline void calculateStats( const UInt8 * data, const size_t size, - HashMap & model) + ModelMap & model) { UInt16 hash = 0; for (size_t i = 0; i < size; ++i) @@ -65,7 +61,15 @@ struct CharsetClassificationImpl ++model[hash]; } } +} +/* Determine language and charset of text data. For each text, we build the distribution of bigrams bytes. + * Then we use marked-up dictionaries with distributions of bigram bytes of various languages ​​and charsets. + * Using a naive Bayesian classifier, find the most likely charset and language and return it + */ +template +struct CharsetClassificationImpl +{ static void vector( const ColumnString::Chars & data, const ColumnString::Offsets & offsets, @@ -74,7 +78,7 @@ struct CharsetClassificationImpl { const auto & encodings_freq = FrequencyHolder::getInstance().getEncodingsFrequency(); - if (detect_language) + if constexpr (detect_language) /// 2 chars for ISO code + 1 zero byte res_data.reserve(offsets.size() * 3); else @@ -83,37 +87,43 @@ struct CharsetClassificationImpl res_offsets.resize(offsets.size()); - size_t res_offset = 0; + size_t current_result_offset = 0; + + double zero_frequency_log = log(zero_frequency); for (size_t i = 0; i < offsets.size(); ++i) { const UInt8 * str = data.data() + offsets[i - 1]; const size_t str_len = offsets[i] - offsets[i - 1] - 1; - std::string_view res; - - HashMap model; + HashMapWithStackMemory, 4> model; calculateStats(str, str_len, model); + std::string_view result_value; + /// Go through the dictionary and find the charset with the highest weight - Float64 max_result = log(zero_frequency) * (max_string_size); + Float64 max_result = zero_frequency_log * (max_string_size); for (const auto & item : encodings_freq) { Float64 score = naiveBayes(item.map, model, max_result); if (max_result < score) { max_result = score; - res = detect_language ? item.lang : item.name; + + if constexpr (detect_language) + result_value = item.lang; + else + result_value = item.name; } } - res_data.resize(res_offset + res.size() + 1); - memcpy(&res_data[res_offset], res.data(), res.size()); + size_t result_value_size = result_value.size(); + res_data.resize(current_result_offset + result_value_size + 1); + memcpy(&res_data[current_result_offset], result_value.data(), result_value_size); + res_data[current_result_offset + result_value_size] = '\0'; + current_result_offset += result_value_size + 1; - res_data[res_offset + res.size()] = 0; - res_offset += res.size() + 1; - - res_offsets[i] = res_offset; + res_offsets[i] = current_result_offset; } } }; diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index 3e7c8bff4d5..de814529d03 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -2,12 +2,15 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" #endif +#include + #include #include #include #include #include #include +#include #include #include #include @@ -17,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -239,17 +242,19 @@ private: } }; - +template class FunctionIPv6StringToNum : public IFunction { public: - static constexpr auto name = "IPv6StringToNum"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static constexpr auto name = exception_mode == IPStringToNumExceptionMode::Throw + ? "IPv6StringToNum" + : (exception_mode == IPStringToNumExceptionMode::Default ? "IPv6StringToNumOrDefault" : "IPv6StringToNumOrNull"); - static inline bool tryParseIPv4(const char * pos) + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + + explicit FunctionIPv6StringToNum(ContextPtr context) + : cast_ipv4_ipv6_default_on_conversion_error(context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error) { - UInt32 result = 0; - return DB::parseIPv4(pos, reinterpret_cast(&result)); } String getName() const override { return name; } @@ -258,62 +263,43 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (!isString(arguments[0])) + if (!isStringOrFixedString(arguments[0])) + { throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); + } - return std::make_shared(IPV6_BINARY_LENGTH); + auto result_type = std::make_shared(IPV6_BINARY_LENGTH); + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + return makeNullable(result_type); + } + + return result_type; } - bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; - if (const auto * col_in = checkAndGetColumn(column.get())) + if constexpr (exception_mode == IPStringToNumExceptionMode::Throw) { - auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH); - - auto & vec_res = col_res->getChars(); - vec_res.resize(col_in->size() * IPV6_BINARY_LENGTH); - - const ColumnString::Chars & vec_src = col_in->getChars(); - const ColumnString::Offsets & offsets_src = col_in->getOffsets(); - size_t src_offset = 0; - char src_ipv4_buf[sizeof("::ffff:") + IPV4_MAX_TEXT_LENGTH + 1] = "::ffff:"; - - for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += IPV6_BINARY_LENGTH, ++i) + if (cast_ipv4_ipv6_default_on_conversion_error) { - /// For both cases below: In case of failure, the function parseIPv6 fills vec_res with zero bytes. - - /// If the source IP address is parsable as an IPv4 address, then transform it into a valid IPv6 address. - /// Keeping it simple by just prefixing `::ffff:` to the IPv4 address to represent it as a valid IPv6 address. - if (tryParseIPv4(reinterpret_cast(&vec_src[src_offset]))) - { - std::memcpy( - src_ipv4_buf + std::strlen("::ffff:"), - reinterpret_cast(&vec_src[src_offset]), - std::min(offsets_src[i] - src_offset, IPV4_MAX_TEXT_LENGTH + 1)); - parseIPv6(src_ipv4_buf, reinterpret_cast(&vec_res[out_offset])); - } - else - { - parseIPv6( - reinterpret_cast(&vec_src[src_offset]), reinterpret_cast(&vec_res[out_offset])); - } - src_offset = offsets_src[i]; + return convertToIPv6(column); } - - return col_res; } - else - throw Exception("Illegal column " + arguments[0].column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + + return convertToIPv6(column); } + +private: + bool cast_ipv4_ipv6_default_on_conversion_error = false; }; @@ -381,69 +367,64 @@ public: } }; - +template class FunctionIPv4StringToNum : public IFunction { public: - static constexpr auto name = "IPv4StringToNum"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static constexpr auto name = exception_mode == IPStringToNumExceptionMode::Throw + ? "IPv4StringToNum" + : (exception_mode == IPStringToNumExceptionMode::Default ? "IPv4StringToNumOrDefault" : "IPv4StringToNumOrNull"); - String getName() const override + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + + explicit FunctionIPv4StringToNum(ContextPtr context) + : cast_ipv4_ipv6_default_on_conversion_error(context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error) { - return name; } + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); + } - return std::make_shared(); + auto result_type = std::make_shared(); + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + return makeNullable(result_type); + } + + return result_type; } - static inline UInt32 parseIPv4(const char * pos) - { - UInt32 result = 0; - DB::parseIPv4(pos, reinterpret_cast(&result)); - - return result; - } - - bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnPtr & column = arguments[0].column; - if (const ColumnString * col = checkAndGetColumn(column.get())) + if constexpr (exception_mode == IPStringToNumExceptionMode::Throw) { - auto col_res = ColumnUInt32::create(); - - ColumnUInt32::Container & vec_res = col_res->getData(); - vec_res.resize(col->size()); - - const ColumnString::Chars & vec_src = col->getChars(); - const ColumnString::Offsets & offsets_src = col->getOffsets(); - size_t prev_offset = 0; - - for (size_t i = 0; i < vec_res.size(); ++i) + if (cast_ipv4_ipv6_default_on_conversion_error) { - vec_res[i] = parseIPv4(reinterpret_cast(&vec_src[prev_offset])); - prev_offset = offsets_src[i]; + return convertToIPv4(column); } - - return col_res; } - else - throw Exception("Illegal column " + arguments[0].column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + + return convertToIPv4(column); } + +private: + bool cast_ipv4_ipv6_default_on_conversion_error = false; }; @@ -503,16 +484,21 @@ private: } }; -class FunctionToIPv4 : public FunctionIPv4StringToNum +template +class FunctionToIPv4 : public FunctionIPv4StringToNum { public: - static constexpr auto name = "toIPv4"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + using Base = FunctionIPv4StringToNum; - String getName() const override - { - return name; - } + static constexpr auto name = exception_mode == IPStringToNumExceptionMode::Throw + ? "toIPv4" + : (exception_mode == IPStringToNumExceptionMode::Default ? "toIPv4OrDefault" : "toIPv4OrNull"); + + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + + explicit FunctionToIPv4(ContextPtr context) : Base(context) { } + + String getName() const override { return name; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -521,18 +507,35 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); + } - return DataTypeFactory::instance().get("IPv4"); + auto result_type = DataTypeFactory::instance().get("IPv4"); + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + return makeNullable(result_type); + } + + return result_type; } }; -class FunctionToIPv6 : public FunctionIPv6StringToNum +template +class FunctionToIPv6 : public FunctionIPv6StringToNum { public: - static constexpr auto name = "toIPv6"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + using Base = FunctionIPv6StringToNum; + + static constexpr auto name = exception_mode == IPStringToNumExceptionMode::Throw + ? "toIPv6" + : (exception_mode == IPStringToNumExceptionMode::Default ? "toIPv6OrDefault" : "toIPv6OrNull"); + + static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + + explicit FunctionToIPv6(ContextPtr context) : Base(context) { } String getName() const override { return name; } @@ -540,11 +543,20 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (!isString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!isStringOrFixedString(arguments[0])) + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); + } - return DataTypeFactory::instance().get("IPv6"); + auto result_type = DataTypeFactory::instance().get("IPv6"); + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + return makeNullable(result_type); + } + + return result_type; } }; @@ -971,7 +983,7 @@ public: } }; -class FunctionIsIPv4String : public FunctionIPv4StringToNum +class FunctionIsIPv4String : public IFunction { public: static constexpr auto name = "isIPv4String"; @@ -980,46 +992,51 @@ public: String getName() const override { return name; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + size_t getNumberOfArguments() const override { return 1; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - const ColumnPtr & column = arguments[0].column; - if (const ColumnString * col = checkAndGetColumn(column.get())) + const ColumnString * input_column = checkAndGetColumn(arguments[0].column.get()); + + if (!input_column) { - auto col_res = ColumnUInt8::create(); - - ColumnUInt8::Container & vec_res = col_res->getData(); - vec_res.resize(col->size()); - - const ColumnString::Chars & vec_src = col->getChars(); - const ColumnString::Offsets & offsets_src = col->getOffsets(); - size_t prev_offset = 0; - UInt32 result = 0; - - for (size_t i = 0; i < vec_res.size(); ++i) - { - vec_res[i] = DB::parseIPv4(reinterpret_cast(&vec_src[prev_offset]), reinterpret_cast(&result)); - prev_offset = offsets_src[i]; - } - return col_res; + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()); } - else - throw Exception("Illegal column " + arguments[0].column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + + auto col_res = ColumnUInt8::create(); + + ColumnUInt8::Container & vec_res = col_res->getData(); + vec_res.resize(input_column->size()); + + const ColumnString::Chars & vec_src = input_column->getChars(); + const ColumnString::Offsets & offsets_src = input_column->getOffsets(); + size_t prev_offset = 0; + UInt32 result = 0; + + for (size_t i = 0; i < vec_res.size(); ++i) + { + vec_res[i] = DB::parseIPv4(reinterpret_cast(&vec_src[prev_offset]), reinterpret_cast(&result)); + prev_offset = offsets_src[i]; + } + + return col_res; } }; -class FunctionIsIPv6String : public FunctionIPv6StringToNum +class FunctionIsIPv6String : public IFunction { public: static constexpr auto name = "isIPv6String"; @@ -1028,44 +1045,49 @@ public: String getName() const override { return name; } - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + size_t getNumberOfArguments() const override { return 1; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); + } return std::make_shared(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - const ColumnPtr & column = arguments[0].column; - - if (const ColumnString * col = checkAndGetColumn(column.get())) + const ColumnString * input_column = checkAndGetColumn(arguments[0].column.get()); + if (!input_column) { - auto col_res = ColumnUInt8::create(); - - ColumnUInt8::Container & vec_res = col_res->getData(); - vec_res.resize(col->size()); - - const ColumnString::Chars & vec_src = col->getChars(); - const ColumnString::Offsets & offsets_src = col->getOffsets(); - size_t prev_offset = 0; - char v[IPV6_BINARY_LENGTH]; - - for (size_t i = 0; i < vec_res.size(); ++i) - { - vec_res[i] = DB::parseIPv6(reinterpret_cast(&vec_src[prev_offset]), reinterpret_cast(v)); - prev_offset = offsets_src[i]; - } - return col_res; + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()); } - else - throw Exception("Illegal column " + arguments[0].column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + + auto col_res = ColumnUInt8::create(); + + ColumnUInt8::Container & vec_res = col_res->getData(); + vec_res.resize(input_column->size()); + + const ColumnString::Chars & vec_src = input_column->getChars(); + const ColumnString::Offsets & offsets_src = input_column->getOffsets(); + size_t prev_offset = 0; + char buffer[IPV6_BINARY_LENGTH]; + + for (size_t i = 0; i < vec_res.size(); ++i) + { + vec_res[i] = DB::parseIPv6(reinterpret_cast(&vec_src[prev_offset]), reinterpret_cast(buffer)); + prev_offset = offsets_src[i]; + } + + return col_res; } }; @@ -1079,8 +1101,6 @@ void registerFunctionsCoding(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction>(); factory.registerFunction>(); - factory.registerFunction(); - factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); @@ -1089,14 +1109,26 @@ void registerFunctionsCoding(FunctionFactory & factory) factory.registerFunction>(); factory.registerFunction>(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); - /// MysQL compatibility aliases: - factory.registerAlias("INET_ATON", FunctionIPv4StringToNum::name, FunctionFactory::CaseInsensitive); + factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + + + /// MySQL compatibility aliases: + factory.registerAlias("INET_ATON", FunctionIPv4StringToNum::name, FunctionFactory::CaseInsensitive); factory.registerAlias("INET6_NTOA", FunctionIPv6NumToString::name, FunctionFactory::CaseInsensitive); - factory.registerAlias("INET6_ATON", FunctionIPv6StringToNum::name, FunctionFactory::CaseInsensitive); + factory.registerAlias("INET6_ATON", FunctionIPv6StringToNum::name, FunctionFactory::CaseInsensitive); factory.registerAlias("INET_NTOA", NameFunctionIPv4NumToString::name, FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/FunctionsCodingIP.h b/src/Functions/FunctionsCodingIP.h new file mode 100644 index 00000000000..246e62d965c --- /dev/null +++ b/src/Functions/FunctionsCodingIP.h @@ -0,0 +1,212 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING; + extern const int ILLEGAL_COLUMN; +} + +enum class IPStringToNumExceptionMode : uint8_t +{ + Throw, + Default, + Null +}; + +static inline bool tryParseIPv4(const char * pos, UInt32 & result_value) +{ + return parseIPv4(pos, reinterpret_cast(&result_value)); +} + +namespace detail +{ + template + ColumnPtr convertToIPv6(const StringColumnType & string_column) + { + size_t column_size = string_column.size(); + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to = nullptr; + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + col_null_map_to = ColumnUInt8::create(column_size, false); + vec_null_map_to = &col_null_map_to->getData(); + } + + auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH); + + auto & vec_res = col_res->getChars(); + vec_res.resize(column_size * IPV6_BINARY_LENGTH); + + using Chars = typename StringColumnType::Chars; + const Chars & vec_src = string_column.getChars(); + + size_t src_offset = 0; + char src_ipv4_buf[sizeof("::ffff:") + IPV4_MAX_TEXT_LENGTH + 1] = "::ffff:"; + + /// ColumnFixedString contains not null terminated strings. But functions parseIPv6, parseIPv4 expect null terminated string. + std::string fixed_string_buffer; + + if constexpr (std::is_same_v) + { + fixed_string_buffer.resize(string_column.getN()); + } + + for (size_t out_offset = 0, i = 0; out_offset < vec_res.size(); out_offset += IPV6_BINARY_LENGTH, ++i) + { + size_t src_next_offset = src_offset; + + const char * src_value = nullptr; + unsigned char * res_value = reinterpret_cast(&vec_res[out_offset]); + + if constexpr (std::is_same_v) + { + src_value = reinterpret_cast(&vec_src[src_offset]); + src_next_offset = string_column.getOffsets()[i]; + } + else if constexpr (std::is_same_v) + { + size_t fixed_string_size = string_column.getN(); + + std::memcpy(fixed_string_buffer.data(), reinterpret_cast(&vec_src[src_offset]), fixed_string_size); + src_value = fixed_string_buffer.data(); + + src_next_offset += fixed_string_size; + } + + bool parse_result = false; + UInt32 dummy_result = 0; + + /// For both cases below: In case of failure, the function parseIPv6 fills vec_res with zero bytes. + + /// If the source IP address is parsable as an IPv4 address, then transform it into a valid IPv6 address. + /// Keeping it simple by just prefixing `::ffff:` to the IPv4 address to represent it as a valid IPv6 address. + if (tryParseIPv4(src_value, dummy_result)) + { + std::memcpy( + src_ipv4_buf + std::strlen("::ffff:"), + src_value, + std::min(src_next_offset - src_offset, IPV4_MAX_TEXT_LENGTH + 1)); + parse_result = parseIPv6(src_ipv4_buf, res_value); + } + else + { + parse_result = parseIPv6(src_value, res_value); + } + + if (!parse_result) + { + if constexpr (exception_mode == IPStringToNumExceptionMode::Throw) + throw Exception("Invalid IPv6 value", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); + else if constexpr (exception_mode == IPStringToNumExceptionMode::Default) + vec_res[i] = 0; + else if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + (*vec_null_map_to)[i] = true; + } + + src_offset = src_next_offset; + } + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + return ColumnNullable::create(std::move(col_res), std::move(col_null_map_to)); + + return col_res; + } +} + +template +ColumnPtr convertToIPv6(ColumnPtr column) +{ + size_t column_size = column->size(); + + auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH); + + auto & vec_res = col_res->getChars(); + vec_res.resize(column_size * IPV6_BINARY_LENGTH); + + if (const auto * column_input_string = checkAndGetColumn(column.get())) + { + return detail::convertToIPv6(*column_input_string); + } + else if (const auto * column_input_fixed_string = checkAndGetColumn(column.get())) + { + return detail::convertToIPv6(*column_input_fixed_string); + } + else + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column type {}. Expected String or FixedString", column->getName()); + } +} + +template +ColumnPtr convertToIPv4(ColumnPtr column) +{ + const ColumnString * column_string = checkAndGetColumn(column.get()); + + if (!column_string) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column type {}. Expected String.", column->getName()); + } + + size_t column_size = column_string->size(); + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to = nullptr; + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + col_null_map_to = ColumnUInt8::create(column_size, false); + vec_null_map_to = &col_null_map_to->getData(); + } + + auto col_res = ColumnUInt32::create(); + + ColumnUInt32::Container & vec_res = col_res->getData(); + vec_res.resize(column_size); + + const ColumnString::Chars & vec_src = column_string->getChars(); + const ColumnString::Offsets & offsets_src = column_string->getOffsets(); + size_t prev_offset = 0; + + for (size_t i = 0; i < vec_res.size(); ++i) + { + bool parse_result = tryParseIPv4(reinterpret_cast(&vec_src[prev_offset]), vec_res[i]); + + if (!parse_result) + { + if constexpr (exception_mode == IPStringToNumExceptionMode::Throw) + { + throw Exception("Invalid IPv4 value", ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING); + } + else if constexpr (exception_mode == IPStringToNumExceptionMode::Default) + { + vec_res[i] = 0; + } + else if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + { + (*vec_null_map_to)[i] = true; + vec_res[i] = 0; + } + } + + prev_offset = offsets_src[i]; + } + + if constexpr (exception_mode == IPStringToNumExceptionMode::Null) + return ColumnNullable::create(std::move(col_res), std::move(col_null_map_to)); + + return col_res; +} + +} diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index a0c7fc643d2..48170d6f564 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -137,7 +137,7 @@ struct NumComparisonImpl template struct StringComparisonImpl { - static void NO_INLINE string_vector_string_vector( + static void NO_INLINE string_vector_string_vector( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) @@ -157,7 +157,7 @@ struct StringComparisonImpl } } - static void NO_INLINE string_vector_fixed_string_vector( + static void NO_INLINE string_vector_fixed_string_vector( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) @@ -175,7 +175,7 @@ struct StringComparisonImpl } } - static void NO_INLINE string_vector_constant( + static void NO_INLINE string_vector_constant( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -193,7 +193,7 @@ struct StringComparisonImpl } } - static void fixed_string_vector_string_vector( + static void fixed_string_vector_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) @@ -201,7 +201,7 @@ struct StringComparisonImpl StringComparisonImpl::string_vector_fixed_string_vector(b_data, b_offsets, a_data, a_n, c); } - static void NO_INLINE fixed_string_vector_fixed_string_vector_16( + static void NO_INLINE fixed_string_vector_fixed_string_vector_16( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Chars & b_data, PaddedPODArray & c) @@ -212,7 +212,7 @@ struct StringComparisonImpl c[j] = Op::apply(memcmp16(&a_data[i], &b_data[i]), 0); } - static void NO_INLINE fixed_string_vector_constant_16( + static void NO_INLINE fixed_string_vector_constant_16( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Chars & b_data, PaddedPODArray & c) @@ -223,7 +223,7 @@ struct StringComparisonImpl c[j] = Op::apply(memcmp16(&a_data[i], &b_data[0]), 0); } - static void NO_INLINE fixed_string_vector_fixed_string_vector( + static void NO_INLINE fixed_string_vector_fixed_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) @@ -250,7 +250,7 @@ struct StringComparisonImpl } } - static void NO_INLINE fixed_string_vector_constant( + static void NO_INLINE fixed_string_vector_constant( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -273,7 +273,7 @@ struct StringComparisonImpl } } - static void constant_string_vector( + static void constant_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_size, const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) @@ -281,7 +281,7 @@ struct StringComparisonImpl StringComparisonImpl::string_vector_constant(b_data, b_offsets, a_data, a_size, c); } - static void constant_fixed_string_vector( + static void constant_fixed_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_size, const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) @@ -295,7 +295,7 @@ struct StringComparisonImpl template struct StringEqualsImpl { - static void NO_INLINE string_vector_string_vector( + static void NO_INLINE string_vector_string_vector( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) @@ -318,7 +318,7 @@ struct StringEqualsImpl } } - static void NO_INLINE string_vector_fixed_string_vector( + static void NO_INLINE string_vector_fixed_string_vector( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) @@ -338,7 +338,7 @@ struct StringEqualsImpl } } - static void NO_INLINE string_vector_constant( + static void NO_INLINE string_vector_constant( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -358,7 +358,7 @@ struct StringEqualsImpl } } - static void NO_INLINE fixed_string_vector_fixed_string_vector_16( + static void NO_INLINE fixed_string_vector_fixed_string_vector_16( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Chars & b_data, PaddedPODArray & c) @@ -371,7 +371,7 @@ struct StringEqualsImpl b_data.data() + i * 16); } - static void NO_INLINE fixed_string_vector_constant_16( + static void NO_INLINE fixed_string_vector_constant_16( /// NOLINT const ColumnString::Chars & a_data, const ColumnString::Chars & b_data, PaddedPODArray & c) @@ -384,7 +384,7 @@ struct StringEqualsImpl b_data.data()); } - static void NO_INLINE fixed_string_vector_fixed_string_vector( + static void NO_INLINE fixed_string_vector_fixed_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) @@ -410,7 +410,7 @@ struct StringEqualsImpl } } - static void NO_INLINE fixed_string_vector_constant( + static void NO_INLINE fixed_string_vector_constant( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, ColumnString::Offset b_size, PaddedPODArray & c) @@ -427,7 +427,7 @@ struct StringEqualsImpl } } - static void fixed_string_vector_string_vector( + static void fixed_string_vector_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_n, const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) @@ -435,7 +435,7 @@ struct StringEqualsImpl string_vector_fixed_string_vector(b_data, b_offsets, a_data, a_n, c); } - static void constant_string_vector( + static void constant_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_size, const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets, PaddedPODArray & c) @@ -443,7 +443,7 @@ struct StringEqualsImpl string_vector_constant(b_data, b_offsets, a_data, a_size, c); } - static void constant_fixed_string_vector( + static void constant_fixed_string_vector( /// NOLINT const ColumnString::Chars & a_data, ColumnString::Offset a_size, const ColumnString::Chars & b_data, ColumnString::Offset b_n, PaddedPODArray & c) @@ -1055,7 +1055,7 @@ private: ColumnPtr executeGeneric(const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1) const { - DataTypePtr common_type = getLeastSupertype({c0.type, c1.type}); + DataTypePtr common_type = getLeastSupertype(DataTypes{c0.type, c1.type}); ColumnPtr c0_converted = castColumn(c0, common_type); ColumnPtr c1_converted = castColumn(c1, common_type); @@ -1228,7 +1228,7 @@ public: // Comparing Date/Date32 and DateTime64 requires implicit conversion, if (date_and_datetime && (isDateOrDate32(left_type) || isDateOrDate32(right_type))) { - DataTypePtr common_type = getLeastSupertype({left_type, right_type}); + DataTypePtr common_type = getLeastSupertype(DataTypes{left_type, right_type}); ColumnPtr c0_converted = castColumn(col_with_type_and_name_left, common_type); ColumnPtr c1_converted = castColumn(col_with_type_and_name_right, common_type); return executeDecimal({c0_converted, common_type, "left"}, {c1_converted, common_type, "right"}); @@ -1258,7 +1258,7 @@ public: } else if (date_and_datetime) { - DataTypePtr common_type = getLeastSupertype({left_type, right_type}); + DataTypePtr common_type = getLeastSupertype(DataTypes{left_type, right_type}); ColumnPtr c0_converted = castColumn(col_with_type_and_name_left, common_type); ColumnPtr c1_converted = castColumn(col_with_type_and_name_right, common_type); if (!((res = executeNumLeftType(c0_converted.get(), c1_converted.get())) diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 4f5f6ae483f..7f8e9148032 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -112,6 +112,9 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); factory.registerFunction>(); factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 909803d7cd7..e098378f51a 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -25,6 +25,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -34,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -542,7 +547,7 @@ struct ToDateTime64TransformUnsigned const DateTime64::NativeType scale_multiplier = 1; - ToDateTime64TransformUnsigned(UInt32 scale = 0) + ToDateTime64TransformUnsigned(UInt32 scale = 0) /// NOLINT : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) {} @@ -559,7 +564,7 @@ struct ToDateTime64TransformSigned const DateTime64::NativeType scale_multiplier = 1; - ToDateTime64TransformSigned(UInt32 scale = 0) + ToDateTime64TransformSigned(UInt32 scale = 0) /// NOLINT : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) {} @@ -577,7 +582,7 @@ struct ToDateTime64TransformFloat const UInt32 scale = 1; - ToDateTime64TransformFloat(UInt32 scale_ = 0) + ToDateTime64TransformFloat(UInt32 scale_ = 0) /// NOLINT : scale(scale_) {} @@ -615,7 +620,7 @@ struct FromDateTime64Transform const DateTime64::NativeType scale_multiplier = 1; - FromDateTime64Transform(UInt32 scale) + FromDateTime64Transform(UInt32 scale) /// NOLINT : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) {} @@ -639,7 +644,7 @@ struct ToDateTime64Transform const DateTime64::NativeType scale_multiplier = 1; - ToDateTime64Transform(UInt32 scale = 0) + ToDateTime64Transform(UInt32 scale = 0) /// NOLINT : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) {} @@ -786,7 +791,8 @@ static ColumnUInt8::MutablePtr copyNullMap(ColumnPtr col) } template -struct ConvertImpl, DataTypeString>, Name, ConvertDefaultBehaviorTag> +requires (!std::is_same_v) +struct ConvertImpl { using FromFieldType = typename FromDataType::FieldType; using ColVecType = ColumnVectorOrDecimal; @@ -906,6 +912,41 @@ struct ConvertImplGenericToString } }; +/** Conversion of time_t to UInt16, Int32, UInt32 + */ +template +void convertFromTime(typename DataType::FieldType & x, time_t & time) +{ + x = time; +} + +template <> +inline void convertFromTime(DataTypeDate::FieldType & x, time_t & time) +{ + if (unlikely(time < 0)) + x = 0; + else if (unlikely(time > 0xFFFF)) + x = 0xFFFF; + else + x = time; +} + +template <> +inline void convertFromTime(DataTypeDate32::FieldType & x, time_t & time) +{ + x = time; +} + +template <> +inline void convertFromTime(DataTypeDateTime::FieldType & x, time_t & time) +{ + if (unlikely(time < 0)) + x = 0; + else if (unlikely(time > 0xFFFFFFFF)) + x = 0xFFFFFFFF; + else + x = time; +} /** Conversion of strings to numbers, dates, datetimes: through parsing. */ @@ -931,18 +972,16 @@ inline void parseImpl(DataTypeDate32::FieldType & x, ReadBuffer x = tmp; } + // NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code. template <> inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone) { time_t time = 0; readDateTimeText(time, rb, *time_zone); - if (time < 0) - time = 0; - x = time; + convertFromTime(x, time); } - template <> inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { @@ -951,7 +990,6 @@ inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb x = tmp.toUnderType(); } - template bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { @@ -1178,7 +1216,7 @@ struct ConvertThroughParsing { time_t res; parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + convertFromTime(vec_to[i], res); } } else if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffortUS) @@ -1193,7 +1231,7 @@ struct ConvertThroughParsing { time_t res; parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + convertFromTime(vec_to[i], res); } } else @@ -1232,14 +1270,14 @@ struct ConvertThroughParsing { time_t res; parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + convertFromTime(vec_to[i],res); } } else if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffortUS) { time_t res; parsed = tryParseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + convertFromTime(vec_to[i],res); } else { @@ -1287,19 +1325,23 @@ struct ConvertThroughParsing template -struct ConvertImpl, DataTypeString>, ToDataType, Name, ConvertDefaultBehaviorTag> +requires (!std::is_same_v) +struct ConvertImpl : ConvertThroughParsing {}; template -struct ConvertImpl, DataTypeFixedString>, ToDataType, Name, ConvertDefaultBehaviorTag> +requires (!std::is_same_v) +struct ConvertImpl : ConvertThroughParsing {}; template -struct ConvertImpl, DataTypeString>, ToDataType, Name, ConvertReturnNullOnErrorTag> +requires (!std::is_same_v) +struct ConvertImpl : ConvertThroughParsing {}; template -struct ConvertImpl, DataTypeFixedString>, ToDataType, Name, ConvertReturnNullOnErrorTag> +requires (!std::is_same_v) +struct ConvertImpl : ConvertThroughParsing {}; /// Generic conversion of any type from String. Used for complex types: Array and Tuple or types with custom serialization. @@ -1354,7 +1396,8 @@ struct ConvertImpl -struct ConvertImpl, T, Name, ConvertDefaultBehaviorTag> +requires (!T::is_parametric) +struct ConvertImpl { template static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, @@ -1444,6 +1487,9 @@ struct NameToDecimal256 { static constexpr auto name = "toDecimal256"; }; static constexpr auto kind = IntervalKind::INTERVAL_KIND; \ }; +DEFINE_NAME_TO_INTERVAL(Nanosecond) +DEFINE_NAME_TO_INTERVAL(Microsecond) +DEFINE_NAME_TO_INTERVAL(Millisecond) DEFINE_NAME_TO_INTERVAL(Second) DEFINE_NAME_TO_INTERVAL(Minute) DEFINE_NAME_TO_INTERVAL(Hour) @@ -2500,10 +2546,12 @@ public: , const DataTypes & argument_types_ , const DataTypePtr & return_type_ , std::optional diagnostic_ - , CastType cast_type_) + , CastType cast_type_ + , bool cast_ipv4_ipv6_default_on_conversion_error_) : cast_name(cast_name_), monotonicity_for_range(std::move(monotonicity_for_range_)) , argument_types(argument_types_), return_type(return_type_), diagnostic(std::move(diagnostic_)) , cast_type(cast_type_) + , cast_ipv4_ipv6_default_on_conversion_error(cast_ipv4_ipv6_default_on_conversion_error_) { } @@ -2552,6 +2600,7 @@ private: std::optional diagnostic; CastType cast_type; + bool cast_ipv4_ipv6_default_on_conversion_error; static WrapperType createFunctionAdaptor(FunctionPtr function, const DataTypePtr & from_type) { @@ -2657,13 +2706,10 @@ private: return createWrapper(from_type, to_type, requested_result_is_nullable); } - WrapperType createUInt8ToUInt8Wrapper(const DataTypePtr from_type, const DataTypePtr to_type) const + WrapperType createUInt8ToBoolWrapper(const DataTypePtr from_type, const DataTypePtr to_type) const { return [from_type, to_type] (ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable *, size_t /*input_rows_count*/) -> ColumnPtr { - if (isBool(from_type) || !isBool(to_type)) - return arguments.front().column; - /// Special case when we convert UInt8 column to Bool column. /// both columns have type UInt8, but we shouldn't use identity wrapper, /// because Bool column can contain only 0 and 1. @@ -2699,8 +2745,8 @@ private: } template - std::enable_if_t, WrapperType> - createDecimalWrapper(const DataTypePtr & from_type, const ToDataType * to_type, bool requested_result_is_nullable) const + requires IsDataTypeDecimal + WrapperType createDecimalWrapper(const DataTypePtr & from_type, const ToDataType * to_type, bool requested_result_is_nullable) const { TypeIndex type_index = from_type->getTypeId(); UInt32 scale = to_type->getScale(); @@ -2902,20 +2948,61 @@ private: throw Exception{"CAST AS Tuple can only be performed between tuple types or from String.\nLeft type: " + from_type_untyped->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; - if (from_type->getElements().size() != to_type->getElements().size()) - throw Exception{"CAST AS Tuple can only be performed between tuple types with the same number of elements or from String.\n" - "Left type: " + from_type->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; - const auto & from_element_types = from_type->getElements(); const auto & to_element_types = to_type->getElements(); - auto element_wrappers = getElementWrappers(from_element_types, to_element_types); - return [element_wrappers, from_element_types, to_element_types] + std::vector element_wrappers; + std::vector> to_reverse_index; + + /// For named tuples allow conversions for tuples with + /// different sets of elements. If element exists in @to_type + /// and doesn't exist in @to_type it will be filled by default values. + if (from_type->haveExplicitNames() && from_type->serializeNames() + && to_type->haveExplicitNames() && to_type->serializeNames()) + { + const auto & from_names = from_type->getElementNames(); + std::unordered_map from_positions; + from_positions.reserve(from_names.size()); + for (size_t i = 0; i < from_names.size(); ++i) + from_positions[from_names[i]] = i; + + const auto & to_names = to_type->getElementNames(); + element_wrappers.reserve(to_names.size()); + to_reverse_index.reserve(from_names.size()); + + for (size_t i = 0; i < to_names.size(); ++i) + { + auto it = from_positions.find(to_names[i]); + if (it != from_positions.end()) + { + element_wrappers.emplace_back(prepareUnpackDictionaries(from_element_types[it->second], to_element_types[i])); + to_reverse_index.emplace_back(it->second); + } + else + { + element_wrappers.emplace_back(); + to_reverse_index.emplace_back(); + } + } + } + else + { + if (from_element_types.size() != to_element_types.size()) + throw Exception{"CAST AS Tuple can only be performed between tuple types with the same number of elements or from String.\n" + "Left type: " + from_type->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; + + element_wrappers = getElementWrappers(from_element_types, to_element_types); + to_reverse_index.reserve(to_element_types.size()); + for (size_t i = 0; i < to_element_types.size(); ++i) + to_reverse_index.emplace_back(i); + } + + return [element_wrappers, from_element_types, to_element_types, to_reverse_index] (ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable * nullable_source, size_t input_rows_count) -> ColumnPtr { const auto * col = arguments.front().column.get(); - size_t tuple_size = from_element_types.size(); + size_t tuple_size = to_element_types.size(); const ColumnTuple & column_tuple = typeid_cast(*col); Columns converted_columns(tuple_size); @@ -2923,8 +3010,16 @@ private: /// invoke conversion for each element for (size_t i = 0; i < tuple_size; ++i) { - ColumnsWithTypeAndName element = {{column_tuple.getColumns()[i], from_element_types[i], "" }}; - converted_columns[i] = element_wrappers[i](element, to_element_types[i], nullable_source, input_rows_count); + if (to_reverse_index[i]) + { + size_t from_idx = *to_reverse_index[i]; + ColumnsWithTypeAndName element = {{column_tuple.getColumns()[from_idx], from_element_types[from_idx], "" }}; + converted_columns[i] = element_wrappers[i](element, to_element_types[i], nullable_source, input_rows_count); + } + else + { + converted_columns[i] = to_element_types[i]->createColumn()->cloneResized(input_rows_count); + } } return ColumnTuple::create(converted_columns); @@ -3045,6 +3140,68 @@ private: } } + WrapperType createObjectWrapper(const DataTypePtr & from_type, const DataTypeObject * to_type) const + { + if (const auto * from_tuple = checkAndGetDataType(from_type.get())) + { + if (!from_tuple->haveExplicitNames()) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Cast to Object can be performed only from flatten Named Tuple. Got: {}", from_type->getName()); + + PathsInData paths; + DataTypes from_types; + + std::tie(paths, from_types) = flattenTuple(from_type); + auto to_types = from_types; + + for (auto & type : to_types) + { + if (isTuple(type) || isNested(type)) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Cast to Object can be performed only from flatten Named Tuple. Got: {}", from_type->getName()); + + type = recursiveRemoveLowCardinality(type); + } + + return [element_wrappers = getElementWrappers(from_types, to_types), + has_nullable_subcolumns = to_type->hasNullableSubcolumns(), from_types, to_types, paths] + (ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable * nullable_source, size_t input_rows_count) + { + size_t tuple_size = to_types.size(); + auto flattened_column = flattenTuple(arguments.front().column); + const auto & column_tuple = assert_cast(*flattened_column); + + if (tuple_size != column_tuple.getColumns().size()) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Expected tuple with {} subcolumn, but got {} subcolumns", + tuple_size, column_tuple.getColumns().size()); + + auto res = ColumnObject::create(has_nullable_subcolumns); + for (size_t i = 0; i < tuple_size; ++i) + { + ColumnsWithTypeAndName element = {{column_tuple.getColumns()[i], from_types[i], "" }}; + auto converted_column = element_wrappers[i](element, to_types[i], nullable_source, input_rows_count); + res->addSubcolumn(paths[i], converted_column->assumeMutable()); + } + + return res; + }; + } + else if (checkAndGetDataType(from_type.get())) + { + return [] (ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * nullable_source, size_t input_rows_count) + { + auto res = ConvertImplGenericFromString::execute(arguments, result_type, nullable_source, input_rows_count); + auto & res_object = assert_cast(res->assumeMutableRef()); + res_object.finalize(); + return res; + }; + } + + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Cast to Object can be performed only from flatten named tuple or string. Got: {}", from_type->getName()); + } + template WrapperType createEnumWrapper(const DataTypePtr & from_type, const DataTypeEnum * to_type) const { @@ -3349,13 +3506,19 @@ private: /// 'requested_result_is_nullable' is true if CAST to Nullable type is requested. WrapperType prepareImpl(const DataTypePtr & from_type, const DataTypePtr & to_type, bool requested_result_is_nullable) const { - if (from_type->equals(*to_type)) - { - if (isUInt8(from_type)) - return createUInt8ToUInt8Wrapper(from_type, to_type); + if (isUInt8(from_type) && isBool(to_type)) + return createUInt8ToBoolWrapper(from_type, to_type); + /// We can cast IPv6 into IPv6, IPv4 into IPv4, but we should not allow to cast FixedString(16) into IPv6 as part of identity cast + bool safe_convert_custom_types = true; + + if (const auto * to_type_custom_name = to_type->getCustomName()) + safe_convert_custom_types = from_type->getCustomName() && from_type->getCustomName()->getName() == to_type_custom_name->getName(); + else if (const auto * from_type_custom_name = from_type->getCustomName()) + safe_convert_custom_types = to_type->getCustomName() && from_type_custom_name->getName() == to_type->getCustomName()->getName(); + + if (from_type->equals(*to_type) && safe_convert_custom_types) return createIdentityWrapper(from_type); - } else if (WhichDataType(from_type).isNothing()) return createNothingWrapper(to_type.get()); @@ -3417,7 +3580,9 @@ private: return false; }; - auto make_custom_serialization_wrapper = [&](const auto & types) -> bool + bool cast_ipv4_ipv6_default_on_conversion_error_value = cast_ipv4_ipv6_default_on_conversion_error; + + auto make_custom_serialization_wrapper = [&, cast_ipv4_ipv6_default_on_conversion_error_value](const auto & types) -> bool { using Types = std::decay_t; using ToDataType = typename Types::RightType; @@ -3425,8 +3590,45 @@ private: if constexpr (WhichDataType(FromDataType::type_id).isStringOrFixedString()) { - if (to_type->getCustomSerialization()) + if (to_type->getCustomSerialization() && to_type->getCustomName()) { + if (to_type->getCustomName()->getName() == "IPv4") + { + ret = [cast_ipv4_ipv6_default_on_conversion_error_value]( + ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable *, size_t) + -> ColumnPtr + { + if (!WhichDataType(result_type).isUInt32()) + throw Exception(ErrorCodes::TYPE_MISMATCH, "Wrong result type {}. Expected UInt32", result_type->getName()); + + if (cast_ipv4_ipv6_default_on_conversion_error_value) + return convertToIPv4(arguments[0].column); + else + return convertToIPv4(arguments[0].column); + }; + + return true; + } + + if (to_type->getCustomName()->getName() == "IPv6") + { + ret = [cast_ipv4_ipv6_default_on_conversion_error_value]( + ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable *, size_t) + -> ColumnPtr + { + if (!WhichDataType(result_type).isFixedString()) + throw Exception( + ErrorCodes::TYPE_MISMATCH, "Wrong result type {}. Expected FixedString", result_type->getName()); + + if (cast_ipv4_ipv6_default_on_conversion_error_value) + return convertToIPv6(arguments[0].column); + else + return convertToIPv6(arguments[0].column); + }; + + return true; + } + ret = &ConvertImplGenericFromString::execute; return true; } @@ -3464,6 +3666,8 @@ private: return createTupleWrapper(from_type, checkAndGetDataType(to_type.get())); case TypeIndex::Map: return createMapWrapper(from_type, checkAndGetDataType(to_type.get())); + case TypeIndex::Object: + return createObjectWrapper(from_type, checkAndGetDataType(to_type.get())); case TypeIndex::AggregateFunction: return createAggregateFunctionWrapper(from_type, checkAndGetDataType(to_type.get())); default: diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 0f75750354a..c6ea886b4a8 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -593,7 +593,7 @@ public: size_t getNumberOfArguments() const override { return 0; } /// For the purpose of query optimization, we assume this function to be injective - /// even in face of fact that there are many different cities named Moscow. + /// even in face of fact that there are many different cities named Paris. bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index fb0dbdfff5c..6a701d7b864 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -90,6 +90,22 @@ public: return getDictionary(dict_name_col->getValue()); } + static const DictionaryAttribute & getDictionaryHierarchicalAttribute(const std::shared_ptr & dictionary) + { + const auto & dictionary_structure = dictionary->getStructure(); + auto hierarchical_attribute_index_optional = dictionary_structure.hierarchical_attribute_index; + + if (!dictionary->hasHierarchy() || !hierarchical_attribute_index_optional.has_value()) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "Dictionary {} does not support hierarchy", + dictionary->getFullName()); + + size_t hierarchical_attribute_index = *hierarchical_attribute_index_optional; + const auto & hierarchical_attribute = dictionary_structure.attributes[hierarchical_attribute_index]; + + return hierarchical_attribute; + } + bool isDictGetFunctionInjective(const Block & sample_columns) { /// Assume non-injective by default @@ -881,7 +897,9 @@ private: result = std::move(dictionary_get_result_column); } else - result = ColumnNullable::create(std::move(dictionary_get_result_column), std::move(is_key_in_dictionary_column_mutable)); + { + result = ColumnNullable::create(dictionary_get_result_column, std::move(is_key_in_dictionary_column_mutable)); + } } return result; @@ -939,39 +957,38 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!isString(arguments[0])) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type of first argument of function {}. Expected String. Actual type {}", - getName(), - arguments[0]->getName()); - - if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type of second argument of function {}. Expected UInt64. Actual type {}", - getName(), - arguments[1]->getName()); - - return std::make_shared(std::make_shared()); - } - bool isDeterministic() const override { return false; } + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + String dictionary_name; + if (const auto * name_col = checkAndGetColumnConst(arguments[0].column.get())) + dictionary_name = name_col->getValue(); + else + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of first argument of function {}, expected a const string.", + arguments[0].type->getName(), + getName()); + + auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); + + return std::make_shared(hierarchical_attribute.type); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (input_rows_count == 0) return result_type->createColumn(); auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); - if (!dictionary->hasHierarchy()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Dictionary {} does not support hierarchy", - dictionary->getFullName()); + auto key_column = ColumnWithTypeAndName{arguments[1].column, arguments[1].type, arguments[1].name}; + auto key_column_casted = castColumnAccurate(key_column, hierarchical_attribute.type); + + ColumnPtr result = dictionary->getHierarchy(key_column_casted, hierarchical_attribute.type); - ColumnPtr result = dictionary->getHierarchy(arguments[1].column, std::make_shared()); return result; } @@ -1009,18 +1026,6 @@ private: getName(), arguments[0]->getName()); - if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type of second argument of function {}. Expected UInt64. Actual type {}", - getName(), - arguments[1]->getName()); - - if (!WhichDataType(arguments[2]).isUInt64()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type of third argument of function {}. Expected UInt64. Actual type {}", - getName(), - arguments[2]->getName()); - return std::make_shared(); } @@ -1031,16 +1036,18 @@ private: if (input_rows_count == 0) return result_type->createColumn(); - auto dict = helper.getDictionary(arguments[0].column); + auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); - if (!dict->hasHierarchy()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Dictionary {} does not support hierarchy", - dict->getFullName()); + auto key_column = ColumnWithTypeAndName{arguments[1].column->convertToFullColumnIfConst(), arguments[1].type, arguments[2].name}; + auto in_key_column = ColumnWithTypeAndName{arguments[2].column->convertToFullColumnIfConst(), arguments[2].type, arguments[2].name}; - ColumnPtr res = dict->isInHierarchy(arguments[1].column, arguments[2].column, std::make_shared()); + auto key_column_casted = castColumnAccurate(key_column, hierarchical_attribute.type); + auto in_key_column_casted = castColumnAccurate(in_key_column, hierarchical_attribute.type); - return res; + ColumnPtr result = dictionary->isInHierarchy(key_column_casted, in_key_column_casted, hierarchical_attribute.type); + + return result; } mutable FunctionDictHelper helper; @@ -1069,21 +1076,18 @@ private: bool isDeterministic() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (!isString(arguments[0])) + if (!isString(arguments[0].type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type of first argument of function {}. Expected String. Actual type {}", getName(), - arguments[0]->getName()); + arguments[0].type->getName()); - if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type of second argument of function {}. Expected UInt64. Actual type {}", - getName(), - arguments[1]->getName()); + auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); - return std::make_shared(std::make_shared()); + return std::make_shared(hierarchical_attribute.type); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override @@ -1092,13 +1096,12 @@ private: return result_type->createColumn(); auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); - if (!dictionary->hasHierarchy()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Dictionary {} does not support hierarchy", - dictionary->getFullName()); + auto key_column = ColumnWithTypeAndName{arguments[1].column->convertToFullColumnIfConst(), arguments[1].type, arguments[1].name}; + auto key_column_casted = castColumnAccurate(key_column, hierarchical_attribute.type); - ColumnPtr result = dictionary->getDescendants(arguments[1].column, std::make_shared(), 1); + ColumnPtr result = dictionary->getDescendants(key_column_casted, hierarchical_attribute.type, 1); return result; } @@ -1126,12 +1129,11 @@ private: bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const final { return true; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 2}; } bool isDeterministic() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { size_t arguments_size = arguments.size(); if (arguments_size < 2 || arguments_size > 3) @@ -1142,27 +1144,24 @@ private: arguments_size); } - if (!isString(arguments[0])) + if (!isString(arguments[0].type)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type of first argument of function {}. Expected const String. Actual type {}", getName(), - arguments[0]->getName()); + arguments[0].type->getName()); - if (!WhichDataType(arguments[1]).isUInt64()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type of second argument of function {}. Expected UInt64. Actual type {}", - getName(), - arguments[1]->getName()); - - if (arguments.size() == 3 && !isUnsignedInteger(arguments[2])) + if (arguments.size() == 3 && !isInteger(arguments[2].type)) { throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type of third argument of function {}. Expected const unsigned integer. Actual type {}", getName(), - arguments[2]->getName()); + arguments[2].type->getName()); } - return std::make_shared(std::make_shared()); + auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); + + return std::make_shared(hierarchical_attribute.type); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override @@ -1171,6 +1170,7 @@ private: return result_type->createColumn(); auto dictionary = helper.getDictionary(arguments[0].column); + const auto & hierarchical_attribute = helper.getDictionaryHierarchicalAttribute(dictionary); size_t level = 0; @@ -1181,17 +1181,21 @@ private: "Illegal type of third argument of function {}. Expected const unsigned integer.", getName()); - level = static_cast(arguments[2].column->get64(0)); + auto value = static_cast(arguments[2].column->getInt(0)); + if (value < 0) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type of third argument of function {}. Expected const unsigned integer.", + getName()); + + level = static_cast(value); } - if (!dictionary->hasHierarchy()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Dictionary {} does not support hierarchy", - dictionary->getFullName()); + auto key_column = ColumnWithTypeAndName{arguments[1].column->convertToFullColumnIfConst(), arguments[1].type, arguments[1].name}; + auto key_column_casted = castColumnAccurate(key_column, hierarchical_attribute.type); - ColumnPtr res = dictionary->getDescendants(arguments[1].column, std::make_shared(), level); + ColumnPtr result = dictionary->getDescendants(key_column_casted, hierarchical_attribute.type, level); - return res; + return result; } mutable FunctionDictHelper helper; diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index d542f023625..e8f9f73b805 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -35,9 +35,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 0dee048dae3..c709cd22880 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -611,7 +611,7 @@ template ColumnPtr FunctionAnyArityLogical::executeImpl( const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const { - ColumnsWithTypeAndName arguments = std::move(args); + ColumnsWithTypeAndName arguments = args; /// Special implementation for short-circuit arguments. if (checkShortCircuitArguments(arguments) != -1) diff --git a/src/Functions/FunctionsLogical.h b/src/Functions/FunctionsLogical.h index 7d4f5489e86..140981faf9f 100644 --- a/src/Functions/FunctionsLogical.h +++ b/src/Functions/FunctionsLogical.h @@ -7,6 +7,7 @@ #include #include #include +#include #if USE_EMBEDDED_COMPILER @@ -147,7 +148,6 @@ public: static constexpr auto name = Name::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } -public: String getName() const override { return name; @@ -189,7 +189,7 @@ public: result = Impl::apply(b, result, nativeBoolCast(b, types[i], values[i])); return b.CreateSelect(result, b.getInt8(1), b.getInt8(0)); } - constexpr bool breakOnTrue = Impl::isSaturatedValue(true); + constexpr bool break_on_true = Impl::isSaturatedValue(true); auto * next = b.GetInsertBlock(); auto * stop = llvm::BasicBlock::Create(next->getContext(), "", next->getParent()); b.SetInsertPoint(stop); @@ -205,7 +205,7 @@ public: if (i + 1 < types.size()) { next = llvm::BasicBlock::Create(next->getContext(), "", next->getParent()); - b.CreateCondBr(truth, breakOnTrue ? stop : next, breakOnTrue ? next : stop); + b.CreateCondBr(truth, break_on_true ? stop : next, break_on_true ? next : stop); } } b.CreateBr(stop); @@ -223,7 +223,6 @@ public: static constexpr auto name = Name::name; static FunctionPtr create(ContextPtr) { return std::make_shared(); } -public: String getName() const override { return name; diff --git a/src/Functions/FunctionsRound.h b/src/Functions/FunctionsRound.h index 9e2728e5926..518b969d441 100644 --- a/src/Functions/FunctionsRound.h +++ b/src/Functions/FunctionsRound.h @@ -159,7 +159,6 @@ struct IntegerRoundingComputation switch (scale_mode) { case ScaleMode::Zero: - return x; case ScaleMode::Positive: return x; case ScaleMode::Negative: @@ -171,10 +170,15 @@ struct IntegerRoundingComputation static ALWAYS_INLINE void compute(const T * __restrict in, size_t scale, T * __restrict out) { - if (sizeof(T) <= sizeof(scale) && scale > size_t(std::numeric_limits::max())) - *out = 0; - else - *out = compute(*in, scale); + if constexpr (sizeof(T) <= sizeof(scale) && scale_mode == ScaleMode::Negative) + { + if (scale > size_t(std::numeric_limits::max())) + { + *out = 0; + return; + } + } + *out = compute(*in, scale); } }; @@ -659,7 +663,7 @@ public: throw Exception{"Elements of array of second argument of function " + getName() + " must be numeric type.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; } - return getLeastSupertype({type_x, type_arr_nested}); + return getLeastSupertype(DataTypes{type_x, type_arr_nested}); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t) const override diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h index b1de017120c..a1256598f1b 100644 --- a/src/Functions/FunctionsStringArray.h +++ b/src/Functions/FunctionsStringArray.h @@ -93,7 +93,7 @@ public: } /// Returns the position of the argument, that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 0; } @@ -152,7 +152,7 @@ public: } /// Returns the position of the argument, that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 0; } @@ -211,7 +211,7 @@ public: } /// Returns the position of the argument, that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 0; } @@ -328,7 +328,7 @@ public: } /// Returns the position of the argument, that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 1; } @@ -399,7 +399,7 @@ public: } /// Returns the position of the argument that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 1; } @@ -482,7 +482,7 @@ public: } /// Returns the position of the argument that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 1; } @@ -567,7 +567,7 @@ public: } /// Returns the position of the argument that is the column of strings - size_t getStringsArgumentPosition() + static size_t getStringsArgumentPosition() { return 0; } diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 79ce7356ee7..76844e2e6fb 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; + extern const int SYNTAX_ERROR; } namespace @@ -167,6 +168,13 @@ struct TimeWindowImpl switch (std::get<0>(interval)) { + //TODO: add proper support for fractional seconds +// case IntervalKind::Nanosecond: +// return executeTumble(*time_column_vec, std::get<1>(interval), time_zone); +// case IntervalKind::Microsecond: +// return executeTumble(*time_column_vec, std::get<1>(interval), time_zone); +// case IntervalKind::Millisecond: +// return executeTumble(*time_column_vec, std::get<1>(interval), time_zone); case IntervalKind::Second: return executeTumble(*time_column_vec, std::get<1>(interval), time_zone); case IntervalKind::Minute: @@ -183,6 +191,8 @@ struct TimeWindowImpl return executeTumble(*time_column_vec, std::get<1>(interval), time_zone); case IntervalKind::Year: return executeTumble(*time_column_vec, std::get<1>(interval), time_zone); + default: + throw Exception("Fraction seconds are unsupported by windows yet", ErrorCodes::SYNTAX_ERROR); } __builtin_unreachable(); } @@ -350,6 +360,16 @@ struct TimeWindowImpl switch (std::get<0>(window_interval)) { + //TODO: add proper support for fractional seconds +// case IntervalKind::Nanosecond: +// return executeHop( +// *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); +// case IntervalKind::Microsecond: +// return executeHop( +// *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); +// case IntervalKind::Millisecond: +// return executeHop( +// *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); case IntervalKind::Second: return executeHop( *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); @@ -374,6 +394,8 @@ struct TimeWindowImpl case IntervalKind::Year: return executeHop( *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); + default: + throw Exception("Fraction seconds are unsupported by windows yet", ErrorCodes::SYNTAX_ERROR); } __builtin_unreachable(); } @@ -487,6 +509,16 @@ struct TimeWindowImpl switch (std::get<0>(window_interval)) { + //TODO: add proper support for fractional seconds +// case IntervalKind::Nanosecond: +// return executeHopSlice( +// *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); +// case IntervalKind::Microsecond: +// return executeHopSlice( +// *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); +// case IntervalKind::Millisecond: +// return executeHopSlice( +// *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); case IntervalKind::Second: return executeHopSlice( *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); @@ -511,6 +543,8 @@ struct TimeWindowImpl case IntervalKind::Year: return executeHopSlice( *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone); + default: + throw Exception("Fraction seconds are unsupported by windows yet", ErrorCodes::SYNTAX_ERROR); } __builtin_unreachable(); } diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index 313de10702d..3ea397e4c7d 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -80,7 +80,32 @@ struct ToStartOfTransform; TRANSFORM_TIME(Hour) TRANSFORM_TIME(Minute) TRANSFORM_TIME(Second) -#undef TRANSFORM_DATE +#undef TRANSFORM_TIME + +#define TRANSFORM_SUBSECONDS(INTERVAL_KIND, DEF_SCALE) \ +template<> \ + struct ToStartOfTransform \ + { \ + static Int64 execute(Int64 t, UInt64 delta, const UInt32 scale) \ + { \ + if (scale <= DEF_SCALE) \ + { \ + auto val = t * DecimalUtils::scaleMultiplier(DEF_SCALE - scale); \ + if (delta == 1) \ + return val; \ + else \ + return val - (val % delta); \ + } \ + else \ + { \ + return t - (t % (delta * DecimalUtils::scaleMultiplier(scale - DEF_SCALE))) ; \ + } \ + } \ + }; + TRANSFORM_SUBSECONDS(Millisecond, 3) + TRANSFORM_SUBSECONDS(Microsecond, 6) + TRANSFORM_SUBSECONDS(Nanosecond, 9) +#undef TRANSFORM_SUBSECONDS template struct AddTime; @@ -117,6 +142,25 @@ struct ToStartOfTransform; ADD_TIME(Second, 1) #undef ADD_TIME +#define ADD_SUBSECONDS(INTERVAL_KIND, DEF_SCALE) \ +template <> \ + struct AddTime \ + { \ + static inline NO_SANITIZE_UNDEFINED Int64 execute(Int64 t, UInt64 delta, const UInt32 scale) \ + { \ + if (scale < DEF_SCALE) \ + { \ + return t + delta * DecimalUtils::scaleMultiplier(DEF_SCALE - scale); \ + } \ + else \ + return t + delta * DecimalUtils::scaleMultiplier(scale - DEF_SCALE); \ + } \ + }; + ADD_SUBSECONDS(Millisecond, 3) + ADD_SUBSECONDS(Microsecond, 6) + ADD_SUBSECONDS(Nanosecond, 9) +#undef ADD_SUBSECONDS + template struct TimeWindowImpl { diff --git a/src/Functions/FunctionsTonalityClassification.cpp b/src/Functions/FunctionsTonalityClassification.cpp index 5dbd6d0356d..2d8e47b9bcb 100644 --- a/src/Functions/FunctionsTonalityClassification.cpp +++ b/src/Functions/FunctionsTonalityClassification.cpp @@ -24,7 +24,7 @@ struct FunctionDetectTonalityImpl UInt64 count_words = 0; String word; - /// Select all Russian words from the string + /// Select all words from the string for (size_t ind = 0; ind < str_len; ++ind) { /// Split words by whitespaces and punctuation signs @@ -36,7 +36,7 @@ struct FunctionDetectTonalityImpl word.push_back(str[ind]); ++ind; } - /// Try to find a russian word in the tonality dictionary + /// Try to find a word in the tonality dictionary const auto * it = emotional_dict.find(word); if (it != emotional_dict.end()) { diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index 046e2dcf70f..d08248e71fc 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -203,7 +203,7 @@ void concat(const std::vector> & array_sources, Si size_t sources_num = array_sources.size(); std::vector is_const(sources_num); - auto checkAndGetSizeToReserve = [] (auto source, IArraySource * array_source) + auto check_and_get_size_to_reserve = [] (auto source, IArraySource * array_source) { if (source == nullptr) throw Exception("Concat function expected " + demangle(typeid(Source).name()) + " or " @@ -215,17 +215,17 @@ void concat(const std::vector> & array_sources, Si size_t size_to_reserve = 0; for (auto i : collections::range(0, sources_num)) { - auto & source = array_sources[i]; + const auto & source = array_sources[i]; is_const[i] = source->isConst(); if (is_const[i]) - size_to_reserve += checkAndGetSizeToReserve(typeid_cast *>(source.get()), source.get()); + size_to_reserve += check_and_get_size_to_reserve(typeid_cast *>(source.get()), source.get()); else - size_to_reserve += checkAndGetSizeToReserve(typeid_cast(source.get()), source.get()); + size_to_reserve += check_and_get_size_to_reserve(typeid_cast(source.get()), source.get()); } sink.reserve(size_to_reserve); - auto writeNext = [& sink] (auto source) + auto write_next = [& sink] (auto source) { writeSlice(source->getWhole(), sink); source->next(); @@ -235,11 +235,11 @@ void concat(const std::vector> & array_sources, Si { for (auto i : collections::range(0, sources_num)) { - auto & source = array_sources[i]; + const auto & source = array_sources[i]; if (is_const[i]) - writeNext(static_cast *>(source.get())); + write_next(static_cast *>(source.get())); else - writeNext(static_cast(source.get())); + write_next(static_cast(source.get())); } sink.next(); } @@ -496,6 +496,31 @@ bool sliceHasImplAnyAll(const FirstSliceType & first, const SecondSliceType & se return search_type == ArraySearchType::All; } +template < + ArraySearchType search_type, + typename FirstSliceType, + typename SecondSliceType, + bool (*isEqual)(const FirstSliceType &, const SecondSliceType &, size_t, size_t)> +bool sliceHasImplStartsEndsWith(const FirstSliceType & first, const SecondSliceType & second, const UInt8 * first_null_map, const UInt8 * second_null_map) +{ + const bool has_first_null_map = first_null_map != nullptr; + const bool has_second_null_map = second_null_map != nullptr; + + if (first.size < second.size) + return false; + + size_t first_index = (search_type == ArraySearchType::StartsWith) ? 0 : first.size - second.size; + for (size_t second_index = 0; second_index < second.size; ++second_index, ++first_index) + { + const bool is_first_null = has_first_null_map && first_null_map[first_index]; + const bool is_second_null = has_second_null_map && second_null_map[second_index]; + if (is_first_null != is_second_null) + return false; + if (!is_first_null && !is_second_null && !isEqual(first, second, first_index, second_index)) + return false; + } + return true; +} /// For details of Knuth-Morris-Pratt string matching algorithm see /// https://en.wikipedia.org/wiki/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm. @@ -551,31 +576,31 @@ bool sliceHasImplSubstr(const FirstSliceType & first, const SecondSliceType & se [](const SecondSliceType & pattern, size_t i, size_t j) { return isEqualUnary(pattern, i, j); }); } - size_t firstCur = 0; - size_t secondCur = 0; - while (firstCur < first.size && secondCur < second.size) + size_t first_cur = 0; + size_t second_cur = 0; + while (first_cur < first.size && second_cur < second.size) { - const bool is_first_null = has_first_null_map && first_null_map[firstCur]; - const bool is_second_null = has_second_null_map && second_null_map[secondCur]; + const bool is_first_null = has_first_null_map && first_null_map[first_cur]; + const bool is_second_null = has_second_null_map && second_null_map[second_cur]; const bool cond_both_null_match = is_first_null && is_second_null; const bool cond_both_not_null = !is_first_null && !is_second_null; - if (cond_both_null_match || (cond_both_not_null && isEqual(first, second, firstCur, secondCur))) + if (cond_both_null_match || (cond_both_not_null && isEqual(first, second, first_cur, second_cur))) { - ++firstCur; - ++secondCur; + ++first_cur; + ++second_cur; } - else if (secondCur > 0) + else if (second_cur > 0) { - secondCur = prefix_function[secondCur - 1]; + second_cur = prefix_function[second_cur - 1]; } else { - ++firstCur; + ++first_cur; } } - return secondCur == second.size; + return second_cur == second.size; } @@ -589,6 +614,8 @@ bool sliceHasImpl(const FirstSliceType & first, const SecondSliceType & second, { if constexpr (search_type == ArraySearchType::Substr) return sliceHasImplSubstr(first, second, first_null_map, second_null_map); + else if constexpr (search_type == ArraySearchType::StartsWith || search_type == ArraySearchType::EndsWith) + return sliceHasImplStartsEndsWith(first, second, first_null_map, second_null_map); else return sliceHasImplAnyAll(first, second, first_null_map, second_null_map); } diff --git a/src/Functions/GatherUtils/GatherUtils.h b/src/Functions/GatherUtils/GatherUtils.h index 8a623caa297..52a01b6ff62 100644 --- a/src/Functions/GatherUtils/GatherUtils.h +++ b/src/Functions/GatherUtils/GatherUtils.h @@ -34,7 +34,9 @@ enum class ArraySearchType { Any, // Corresponds to the hasAny array function All, // Corresponds to the hasAll array function - Substr // Corresponds to the hasSubstr array function + Substr, // Corresponds to the hasSubstr array function + StartsWith, + EndsWith }; std::unique_ptr createArraySource(const ColumnArray & col, bool is_const, size_t total_rows); @@ -58,6 +60,8 @@ ColumnArray::MutablePtr sliceFromRightDynamicLength(IArraySource & src, const IC void sliceHasAny(IArraySource & first, IArraySource & second, ColumnUInt8 & result); void sliceHasAll(IArraySource & first, IArraySource & second, ColumnUInt8 & result); void sliceHasSubstr(IArraySource & first, IArraySource & second, ColumnUInt8 & result); +void sliceHasStartsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result); +void sliceHasEndsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result); inline void sliceHas(IArraySource & first, IArraySource & second, ArraySearchType search_type, ColumnUInt8 & result) { @@ -72,7 +76,12 @@ inline void sliceHas(IArraySource & first, IArraySource & second, ArraySearchTyp case ArraySearchType::Substr: sliceHasSubstr(first, second, result); break; - + case ArraySearchType::StartsWith: + sliceHasStartsWith(first, second, result); + break; + case ArraySearchType::EndsWith: + sliceHasEndsWith(first, second, result); + break; } } diff --git a/src/Functions/GatherUtils/Selectors.h b/src/Functions/GatherUtils/Selectors.h index bbe631a6a3a..5793701e93a 100644 --- a/src/Functions/GatherUtils/Selectors.h +++ b/src/Functions/GatherUtils/Selectors.h @@ -131,7 +131,7 @@ struct ArrayAndValueSourceSelectorBySink : public ArraySinkSelector).name()) + " but got " + demangle(typeid(*source_ptr).name()), ErrorCodes::LOGICAL_ERROR); }; - auto checkTypeAndCallConcat = [& sink, & checkType, & args ...] (auto array_source_ptr, auto value_source_ptr) + auto check_type_and_call_concat = [& sink, & check_type, & args ...] (auto array_source_ptr, auto value_source_ptr) { - checkType(array_source_ptr); - checkType(value_source_ptr); + check_type(array_source_ptr); + check_type(value_source_ptr); Base::selectArrayAndValueSourceBySink(*array_source_ptr, *value_source_ptr, sink, args ...); }; if (array_source.isConst() && value_source.isConst()) - checkTypeAndCallConcat(typeid_cast *>(&array_source), + check_type_and_call_concat(typeid_cast *>(&array_source), typeid_cast *>(&value_source)); else if (array_source.isConst()) - checkTypeAndCallConcat(typeid_cast *>(&array_source), + check_type_and_call_concat(typeid_cast *>(&array_source), typeid_cast(&value_source)); else if (value_source.isConst()) - checkTypeAndCallConcat(typeid_cast(&array_source), + check_type_and_call_concat(typeid_cast(&array_source), typeid_cast *>(&value_source)); else - checkTypeAndCallConcat(typeid_cast(&array_source), + check_type_and_call_concat(typeid_cast(&array_source), typeid_cast(&value_source)); } }; diff --git a/src/Functions/GatherUtils/Slices.h b/src/Functions/GatherUtils/Slices.h index 7951178497a..22f475adf59 100644 --- a/src/Functions/GatherUtils/Slices.h +++ b/src/Functions/GatherUtils/Slices.h @@ -26,7 +26,7 @@ struct NullableSlice : public Slice const UInt8 * null_map = nullptr; NullableSlice() = default; - NullableSlice(const Slice & base) : Slice(base) {} + NullableSlice(const Slice & base) : Slice(base) {} /// NOLINT }; template diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h index 7d1241be7d1..13e3de99552 100644 --- a/src/Functions/GatherUtils/Sources.h +++ b/src/Functions/GatherUtils/Sources.h @@ -184,7 +184,7 @@ struct ConstSource : public Base virtual void accept(ArraySourceVisitor & visitor) // override { - if constexpr (std::is_base_of::value) + if constexpr (std::is_base_of_v) visitor.visit(*this); else throw Exception( @@ -194,7 +194,7 @@ struct ConstSource : public Base virtual void accept(ValueSourceVisitor & visitor) // override { - if constexpr (std::is_base_of::value) + if constexpr (std::is_base_of_v) visitor.visit(*this); else throw Exception( diff --git a/src/Functions/GatherUtils/ends_with.cpp b/src/Functions/GatherUtils/ends_with.cpp new file mode 100644 index 00000000000..579d903005a --- /dev/null +++ b/src/Functions/GatherUtils/ends_with.cpp @@ -0,0 +1,71 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +namespace +{ + +struct ArrayEndsWithSelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } + } +}; + +} + +void sliceHasEndsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayEndsWithSelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/GatherUtils/starts_with.cpp b/src/Functions/GatherUtils/starts_with.cpp new file mode 100644 index 00000000000..813294bc092 --- /dev/null +++ b/src/Functions/GatherUtils/starts_with.cpp @@ -0,0 +1,71 @@ +#include "GatherUtils.h" +#include "Selectors.h" +#include "Algorithms.h" + +namespace DB::GatherUtils +{ + +namespace +{ + +struct ArrayStartsWithSelectArraySourcePair : public ArraySourcePairSelector +{ + template + static void callFunction(FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_second_nullable) + { + using NullableSource = NullableArraySource; + + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, static_cast(second), result); + } + else + { + if (is_second_const) + arrayAllAny(first, static_cast &>(second), result); + else + arrayAllAny(first, second, result); + } + } + + template + static void selectSourcePair(bool is_first_const, bool is_first_nullable, FirstSource && first, + bool is_second_const, bool is_second_nullable, SecondSource && second, + ColumnUInt8 & result) + { + using SourceType = typename std::decay::type; + + if (is_first_nullable) + { + using NullableSource = NullableArraySource; + + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(static_cast(first), is_second_const, is_second_nullable, second, result); + } + else + { + if (is_first_const) + callFunction(static_cast &>(first), is_second_const, is_second_nullable, second, result); + else + callFunction(first, is_second_const, is_second_nullable, second, result); + } + } +}; + +} + +void sliceHasStartsWith(IArraySource & first, IArraySource & second, ColumnUInt8 & result) +{ + ArrayStartsWithSelectArraySourcePair::select(first, second, result); +} + +} diff --git a/src/Functions/GeoHash.h b/src/Functions/GeoHash.h index d97eda31cef..071bc5072a4 100644 --- a/src/Functions/GeoHash.h +++ b/src/Functions/GeoHash.h @@ -37,8 +37,8 @@ struct GeohashesInBoxPreparedArgs }; GeohashesInBoxPreparedArgs geohashesInBoxPrepare( - const Float64 longitude_min, - const Float64 latitude_min, + Float64 longitude_min, + Float64 latitude_min, Float64 longitude_max, Float64 latitude_max, uint8_t precision); diff --git a/src/Functions/GregorianDate.h b/src/Functions/GregorianDate.h index b44b6c0dd13..ef2b9e6eede 100644 --- a/src/Functions/GregorianDate.h +++ b/src/Functions/GregorianDate.h @@ -32,13 +32,13 @@ namespace DB /** Construct from date in text form 'YYYY-MM-DD' by reading from * ReadBuffer. */ - GregorianDate(ReadBuffer & in); + explicit GregorianDate(ReadBuffer & in); /** Construct from Modified Julian Day. The type T is an * integral type which should be at least 32 bits wide, and * should preferably signed. */ - GregorianDate(is_integer auto mjd); + explicit GregorianDate(is_integer auto mjd); /** Convert to Modified Julian Day. The type T is an integral type * which should be at least 32 bits wide, and should preferably @@ -65,15 +65,15 @@ namespace DB return month_; } - uint8_t day_of_month() const noexcept + uint8_t day_of_month() const noexcept /// NOLINT { return day_of_month_; } private: - YearT year_; - uint8_t month_; - uint8_t day_of_month_; + YearT year_; /// NOLINT + uint8_t month_; /// NOLINT + uint8_t day_of_month_; /// NOLINT }; /** ISO 8601 Ordinal Date. YearT is an integral type which should @@ -89,7 +89,7 @@ namespace DB * integral type which should be at least 32 bits wide, and * should preferably signed. */ - OrdinalDate(is_integer auto mjd); + explicit OrdinalDate(is_integer auto mjd); /** Convert to Modified Julian Day. The type T is an integral * type which should be at least 32 bits wide, and should @@ -109,8 +109,8 @@ namespace DB } private: - YearT year_; - uint16_t day_of_year_; + YearT year_; /// NOLINT + uint16_t day_of_year_; /// NOLINT }; class MonthDay @@ -134,14 +134,14 @@ namespace DB return month_; } - uint8_t day_of_month() const noexcept + uint8_t day_of_month() const noexcept /// NOLINT { return day_of_month_; } private: - uint8_t month_; - uint8_t day_of_month_; + uint8_t month_; /// NOLINT + uint8_t day_of_month_; /// NOLINT }; } @@ -183,13 +183,13 @@ namespace gd template static inline constexpr I div(I x, J y) { - const auto y_ = static_cast(y); - if (x > 0 && y_ < 0) - return ((x - 1) / y_) - 1; - else if (x < 0 && y_ > 0) - return ((x + 1) / y_) - 1; + const auto y_cast = static_cast(y); + if (x > 0 && y_cast < 0) + return ((x - 1) / y_cast) - 1; + else if (x < 0 && y_cast > 0) + return ((x + 1) / y_cast) - 1; else - return x / y_; + return x / y_cast; } /** Integer modulus, satisfying div(x, y)*y + mod(x, y) == x. @@ -197,10 +197,10 @@ namespace gd template static inline constexpr I mod(I x, J y) { - const auto y_ = static_cast(y); - const auto r = x % y_; - if ((x > 0 && y_ < 0) || (x < 0 && y_ > 0)) - return r == 0 ? static_cast(0) : r + y_; + const auto y_cast = static_cast(y); + const auto r = x % y_cast; + if ((x > 0 && y_cast < 0) || (x < 0 && y_cast > 0)) + return r == 0 ? static_cast(0) : r + y_cast; else return r; } @@ -210,8 +210,8 @@ namespace gd template static inline constexpr I min(I x, J y) { - const auto y_ = static_cast(y); - return x < y_ ? x : y_; + const auto y_cast = static_cast(y); + return x < y_cast ? x : y_cast; } static inline char readDigit(ReadBuffer & in) diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 8063ad77ad0..7b272fef53d 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -120,7 +120,7 @@ public: virtual ~IFunctionBase() = default; - virtual ColumnPtr execute( + virtual ColumnPtr execute( /// NOLINT const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const { return prepare(arguments)->execute(arguments, result_type, input_rows_count, dry_run); @@ -267,7 +267,7 @@ public: */ virtual Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const { - throw Exception("Function " + getName() + " has no information about its monotonicity.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Function " + getName() + " has no information about its monotonicity", ErrorCodes::NOT_IMPLEMENTED); } }; @@ -452,7 +452,7 @@ public: using Monotonicity = IFunctionBase::Monotonicity; virtual Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const { - throw Exception("Function " + getName() + " has no information about its monotonicity.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Function " + getName() + " has no information about its monotonicity", ErrorCodes::NOT_IMPLEMENTED); } /// For non-variadic functions, return number of arguments; otherwise return zero (that should be ignored). diff --git a/src/Functions/ITupleFunction.h b/src/Functions/ITupleFunction.h index 836e5d273fc..0dbbb81aab9 100644 --- a/src/Functions/ITupleFunction.h +++ b/src/Functions/ITupleFunction.h @@ -1,6 +1,11 @@ #pragma once +#include +#include #include +#include +#include + namespace DB { diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h index 2c9482b665e..3a5e121b989 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h @@ -11,7 +11,6 @@ public: ASTPtr clone() const override { return std::make_shared(*this); } -public: /// Member name to lookup in json document (in path: $.some_key.another_key. ...) String member_name; }; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h index 746c6211f29..083d4b8e3ab 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -12,7 +12,6 @@ public: ASTPtr clone() const override { return std::make_shared(*this); } -public: /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) /// Range is represented as /// Single index is represented as diff --git a/src/Functions/JSONPath/Generator/GeneratorJSONPath.h b/src/Functions/JSONPath/Generator/GeneratorJSONPath.h index 291150f6df4..fe00f06bbbf 100644 --- a/src/Functions/JSONPath/Generator/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generator/GeneratorJSONPath.h @@ -25,7 +25,7 @@ public: * Traverses children ASTs of ASTJSONPathQuery and creates a vector of corresponding visitors * @param query_ptr_ pointer to ASTJSONPathQuery */ - GeneratorJSONPath(ASTPtr query_ptr_) + explicit GeneratorJSONPath(ASTPtr query_ptr_) { query_ptr = query_ptr_; const auto * path = query_ptr->as(); diff --git a/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h index 5fe35e75a84..8446e1ff3be 100644 --- a/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h @@ -10,7 +10,7 @@ template class VisitorJSONPathMemberAccess : public IVisitor { public: - VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) + explicit VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_->as()) { } const char * getName() const override { return "VisitorJSONPathMemberAccess"; } diff --git a/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h index 40d4f6ad95e..708a71f7cf4 100644 --- a/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h @@ -10,7 +10,7 @@ template class VisitorJSONPathRange : public IVisitor { public: - VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_->as()) + explicit VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_->as()) { current_range = 0; current_index = range_ptr->ranges[current_range].first; @@ -20,7 +20,6 @@ public: VisitorStatus apply(typename JSONParser::Element & element) const override { - typename JSONParser::Element result; typename JSONParser::Array array = element.getArray(); element = array[current_index]; return VisitorStatus::Ok; diff --git a/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h b/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h index 5c48c12782f..71569d3c0a0 100644 --- a/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h @@ -10,7 +10,7 @@ template class VisitorJSONPathRoot : public IVisitor { public: - VisitorJSONPathRoot(ASTPtr) { } + explicit VisitorJSONPathRoot(ASTPtr) { } const char * getName() const override { return "VisitorJSONPathRoot"; } diff --git a/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h b/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h index 4a54a76c199..0c297f64316 100644 --- a/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h @@ -10,7 +10,7 @@ template class VisitorJSONPathStar : public IVisitor { public: - VisitorJSONPathStar(ASTPtr) + explicit VisitorJSONPathStar(ASTPtr) { current_index = 0; } @@ -19,7 +19,6 @@ public: VisitorStatus apply(typename JSONParser::Element & element) const override { - typename JSONParser::Element result; typename JSONParser::Array array = element.getArray(); element = array[current_index]; return VisitorStatus::Ok; diff --git a/src/Functions/LeftRight.h b/src/Functions/LeftRight.h index 054e76b7792..a82182a52e7 100644 --- a/src/Functions/LeftRight.h +++ b/src/Functions/LeftRight.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/LowerUpperImpl.h b/src/Functions/LowerUpperImpl.h index cf614850e66..a7c38a7f904 100644 --- a/src/Functions/LowerUpperImpl.h +++ b/src/Functions/LowerUpperImpl.h @@ -31,7 +31,7 @@ private: #ifdef __SSE2__ const auto bytes_sse = sizeof(__m128i); - const auto src_end_sse = src_end - (src_end - src) % bytes_sse; + const auto * src_end_sse = src_end - (src_end - src) % bytes_sse; const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); diff --git a/src/Functions/LowerUpperUTF8Impl.h b/src/Functions/LowerUpperUTF8Impl.h index 4c155034b3d..a7475870dab 100644 --- a/src/Functions/LowerUpperUTF8Impl.h +++ b/src/Functions/LowerUpperUTF8Impl.h @@ -16,61 +16,58 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -namespace +/// xor or do nothing +template +UInt8 xor_or_identity(const UInt8 c, const int mask) { - /// xor or do nothing - template - UInt8 xor_or_identity(const UInt8 c, const int mask) - { - return c ^ mask; - } + return c ^ mask; +} - template <> - inline UInt8 xor_or_identity(const UInt8 c, const int) - { - return c; - } +template <> +inline UInt8 xor_or_identity(const UInt8 c, const int) +{ + return c; +} - /// It is caller's responsibility to ensure the presence of a valid cyrillic sequence in array - template - inline void UTF8CyrillicToCase(const UInt8 *& src, UInt8 *& dst) +/// It is caller's responsibility to ensure the presence of a valid cyrillic sequence in array +template +inline void UTF8CyrillicToCase(const UInt8 *& src, UInt8 *& dst) +{ + if (src[0] == 0xD0u && (src[1] >= 0x80u && src[1] <= 0x8Fu)) { - if (src[0] == 0xD0u && (src[1] >= 0x80u && src[1] <= 0x8Fu)) - { - /// ЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏ - *dst++ = xor_or_identity(*src++, 0x1); - *dst++ = xor_or_identity(*src++, 0x10); - } - else if (src[0] == 0xD1u && (src[1] >= 0x90u && src[1] <= 0x9Fu)) - { - /// ѐёђѓєѕіїјљњћќѝўџ - *dst++ = xor_or_identity(*src++, 0x1); - *dst++ = xor_or_identity(*src++, 0x10); - } - else if (src[0] == 0xD0u && (src[1] >= 0x90u && src[1] <= 0x9Fu)) - { - /// А-П - *dst++ = *src++; - *dst++ = xor_or_identity(*src++, 0x20); - } - else if (src[0] == 0xD0u && (src[1] >= 0xB0u && src[1] <= 0xBFu)) - { - /// а-п - *dst++ = *src++; - *dst++ = xor_or_identity(*src++, 0x20); - } - else if (src[0] == 0xD0u && (src[1] >= 0xA0u && src[1] <= 0xAFu)) - { - /// Р-Я - *dst++ = xor_or_identity(*src++, 0x1); - *dst++ = xor_or_identity(*src++, 0x20); - } - else if (src[0] == 0xD1u && (src[1] >= 0x80u && src[1] <= 0x8Fu)) - { - /// р-я - *dst++ = xor_or_identity(*src++, 0x1); - *dst++ = xor_or_identity(*src++, 0x20); - } + /// ЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏ + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x10); + } + else if (src[0] == 0xD1u && (src[1] >= 0x90u && src[1] <= 0x9Fu)) + { + /// ѐёђѓєѕіїјљњћќѝўџ + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x10); + } + else if (src[0] == 0xD0u && (src[1] >= 0x90u && src[1] <= 0x9Fu)) + { + /// А-П + *dst++ = *src++; + *dst++ = xor_or_identity(*src++, 0x20); + } + else if (src[0] == 0xD0u && (src[1] >= 0xB0u && src[1] <= 0xBFu)) + { + /// а-п + *dst++ = *src++; + *dst++ = xor_or_identity(*src++, 0x20); + } + else if (src[0] == 0xD0u && (src[1] >= 0xA0u && src[1] <= 0xAFu)) + { + /// Р-Я + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x20); + } + else if (src[0] == 0xD1u && (src[1] >= 0x80u && src[1] <= 0x8Fu)) + { + /// р-я + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x20); } } @@ -171,7 +168,7 @@ private: { #ifdef __SSE2__ static constexpr auto bytes_sse = sizeof(__m128i); - auto src_end_sse = src + (src_end - src) / bytes_sse * bytes_sse; + const auto * src_end_sse = src + (src_end - src) / bytes_sse * bytes_sse; /// SSE2 packed comparison operate on signed types, hence compare (c < 0) instead of (c > 0x7f) const auto v_zero = _mm_setzero_si128(); @@ -216,7 +213,7 @@ private: else { /// UTF-8 - const auto expected_end = src + bytes_sse; + const auto * expected_end = src + bytes_sse; while (src < expected_end) toCase(src, src_end, dst); diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index c2e64671d1f..f3e67008707 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "Regexps.h" #include "config_functions.h" diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h index 9ef6454d085..bcc195e988e 100644 --- a/src/Functions/PerformanceAdaptors.h +++ b/src/Functions/PerformanceAdaptors.h @@ -72,7 +72,7 @@ namespace detail return size() == 0; } - void emplace_back() + void emplace_back() /// NOLINT { data.emplace_back(); } @@ -198,7 +198,7 @@ class ImplementationSelector : WithContext public: using ImplementationPtr = std::shared_ptr; - ImplementationSelector(ContextPtr context_) : WithContext(context_) {} + explicit ImplementationSelector(ContextPtr context_) : WithContext(context_) {} /* Select the best implementation based on previous runs. * If FunctionInterface is IFunction, then "executeImpl" method of the implementation will be called diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index 1a340c517dc..de4bb2d48de 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -53,14 +53,14 @@ UInt64 getPolygonAllocatedBytes(const Polygon & polygon) using RingType = typename Polygon::ring_type; using ValueType = typename RingType::value_type; - auto sizeOfRing = [](const RingType & ring) { return sizeof(ring) + ring.capacity() * sizeof(ValueType); }; + auto size_of_ring = [](const RingType & ring) { return sizeof(ring) + ring.capacity() * sizeof(ValueType); }; - size += sizeOfRing(polygon.outer()); + size += size_of_ring(polygon.outer()); const auto & inners = polygon.inners(); size += sizeof(inners) + inners.capacity() * sizeof(RingType); for (auto & inner : inners) - size += sizeOfRing(inner); + size += size_of_ring(inner); return size; } diff --git a/src/Functions/ReplaceRegexpImpl.h b/src/Functions/ReplaceRegexpImpl.h index 5d2549239c8..549edf70dff 100644 --- a/src/Functions/ReplaceRegexpImpl.h +++ b/src/Functions/ReplaceRegexpImpl.h @@ -33,8 +33,8 @@ struct ReplaceRegexpImpl /// Otherwise - paste this string verbatim. std::string literal; - Instruction(int substitution_num_) : substitution_num(substitution_num_) {} - Instruction(std::string literal_) : literal(std::move(literal_)) {} + Instruction(int substitution_num_) : substitution_num(substitution_num_) {} /// NOLINT + Instruction(std::string literal_) : literal(std::move(literal_)) {} /// NOLINT }; using Instructions = std::vector; @@ -137,8 +137,14 @@ struct ReplaceRegexpImpl if (replace_one) can_finish_current_string = true; - else if (match.length() == 0) - ++match_pos; /// Step one character to avoid infinite loop. + + if (match.length() == 0) + { + /// Step one character to avoid infinite loop + ++match_pos; + if (match_pos >= static_cast(input.length())) + can_finish_current_string = true; + } } else can_finish_current_string = true; diff --git a/src/Functions/SubtractSubSeconds.cpp b/src/Functions/SubtractSubSeconds.cpp new file mode 100644 index 00000000000..5eeb24c8748 --- /dev/null +++ b/src/Functions/SubtractSubSeconds.cpp @@ -0,0 +1,28 @@ +#include +#include + + +namespace DB +{ + +using FunctionSubtractNanoseconds = FunctionDateOrDateTimeAddInterval; +void registerFunctionSubtractNanoseconds(FunctionFactory & factory) +{ + factory.registerFunction(); +}; + +using FunctionSubtractMicroseconds = FunctionDateOrDateTimeAddInterval; +void registerFunctionSubtractMicroseconds(FunctionFactory & factory) +{ + factory.registerFunction(); +}; + +using FunctionSubtractMilliseconds = FunctionDateOrDateTimeAddInterval; +void registerFunctionSubtractMilliseconds(FunctionFactory & factory) +{ + factory.registerFunction(); +}; + +} + + diff --git a/src/Functions/TargetSpecific.h b/src/Functions/TargetSpecific.h index fa230a56fb7..d7fa55fbb08 100644 --- a/src/Functions/TargetSpecific.h +++ b/src/Functions/TargetSpecific.h @@ -89,6 +89,7 @@ String toString(TargetArch arch); #if ENABLE_MULTITARGET_CODE && defined(__GNUC__) && defined(__x86_64__) +/// NOLINTNEXTLINE #define USE_MULTITARGET_CODE 1 #if defined(__clang__) @@ -183,6 +184,7 @@ namespace TargetSpecific::Default { \ __VA_ARGS__ \ } +/// NOLINTNEXTLINE #define DECLARE_MULTITARGET_CODE(...) \ DECLARE_DEFAULT_CODE (__VA_ARGS__) \ DECLARE_SSE42_SPECIFIC_CODE (__VA_ARGS__) \ @@ -191,23 +193,23 @@ DECLARE_AVX2_SPECIFIC_CODE (__VA_ARGS__) \ DECLARE_AVX512F_SPECIFIC_CODE(__VA_ARGS__) DECLARE_DEFAULT_CODE( - constexpr auto BuildArch = TargetArch::Default; + constexpr auto BuildArch = TargetArch::Default; /// NOLINT ) // DECLARE_DEFAULT_CODE DECLARE_SSE42_SPECIFIC_CODE( - constexpr auto BuildArch = TargetArch::SSE42; + constexpr auto BuildArch = TargetArch::SSE42; /// NOLINT ) // DECLARE_SSE42_SPECIFIC_CODE DECLARE_AVX_SPECIFIC_CODE( - constexpr auto BuildArch = TargetArch::AVX; + constexpr auto BuildArch = TargetArch::AVX; /// NOLINT ) // DECLARE_AVX_SPECIFIC_CODE DECLARE_AVX2_SPECIFIC_CODE( - constexpr auto BuildArch = TargetArch::AVX2; + constexpr auto BuildArch = TargetArch::AVX2; /// NOLINT ) // DECLARE_AVX2_SPECIFIC_CODE DECLARE_AVX512F_SPECIFIC_CODE( - constexpr auto BuildArch = TargetArch::AVX512F; + constexpr auto BuildArch = TargetArch::AVX512F; /// NOLINT ) // DECLARE_AVX512F_SPECIFIC_CODE } diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h index 4eab2a491c7..9ac28118b8f 100644 --- a/src/Functions/TransformDateTime64.h +++ b/src/Functions/TransformDateTime64.h @@ -13,7 +13,7 @@ namespace DB * * DateTime64 value and scale factor (2) * * DateTime64 broken down to components, result of execute is then re-assembled back into DateTime64 value (3) * - * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, + * Suitable Transform-types are commonly used in Date/DateTime manipulation functions, * and should implement static (or const) function with following signatures: * 1: * R execute(Int64 whole_value, ... ) @@ -44,7 +44,7 @@ public: static constexpr auto name = Transform::name; // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. - TransformDateTime64(UInt32 scale_ = 0) + TransformDateTime64(UInt32 scale_ = 0) /// NOLINT : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) {} diff --git a/src/Functions/URL/ExtractFirstSignificantSubdomain.h b/src/Functions/URL/ExtractFirstSignificantSubdomain.h index 4f9b1ec3c6c..70c9c25e4f3 100644 --- a/src/Functions/URL/ExtractFirstSignificantSubdomain.h +++ b/src/Functions/URL/ExtractFirstSignificantSubdomain.h @@ -49,11 +49,11 @@ struct ExtractFirstSignificantSubdomain res_data = tmp; res_size = domain_length; - auto begin = tmp; - auto end = begin + domain_length; + const auto * begin = tmp; + const auto * end = begin + domain_length; const char * last_3_periods[3]{}; - auto pos = find_first_symbols<'.'>(begin, end); + const auto * pos = find_first_symbols<'.'>(begin, end); while (pos < end) { last_3_periods[2] = last_3_periods[1]; @@ -74,7 +74,7 @@ struct ExtractFirstSignificantSubdomain if (!last_3_periods[2]) last_3_periods[2] = begin - 1; - auto end_of_level_domain = find_first_symbols<'/'>(last_3_periods[0], end); + const auto * end_of_level_domain = find_first_symbols<'/'>(last_3_periods[0], end); if (!end_of_level_domain) { end_of_level_domain = end; @@ -117,12 +117,12 @@ struct ExtractFirstSignificantSubdomain res_data = tmp; res_size = domain_length; - auto begin = tmp; - auto end = begin + domain_length; + const auto * begin = tmp; + const auto * end = begin + domain_length; const char * last_2_periods[2]{}; const char * prev = begin - 1; - auto pos = find_first_symbols<'.'>(begin, end); + const auto * pos = find_first_symbols<'.'>(begin, end); while (pos < end) { if (lookup(pos + 1, end - pos - 1)) diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h index 8a76d52741b..5d78500c252 100644 --- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h +++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h @@ -20,7 +20,7 @@ namespace ErrorCodes struct FirstSignificantSubdomainCustomLookup { const TLDList & tld_list; - FirstSignificantSubdomainCustomLookup(const std::string & tld_list_name) + explicit FirstSignificantSubdomainCustomLookup(const std::string & tld_list_name) : tld_list(TLDListsHolder::getInstance().getTldList(tld_list_name)) { } diff --git a/src/Functions/URL/decodeURLComponent.cpp b/src/Functions/URL/decodeURLComponent.cpp index 9ed290b1832..a4c9e1f4eec 100644 --- a/src/Functions/URL/decodeURLComponent.cpp +++ b/src/Functions/URL/decodeURLComponent.cpp @@ -11,8 +11,33 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } +static size_t encodeURL(const char * __restrict src, size_t src_size, char * __restrict dst, bool space_as_plus) +{ + char * dst_pos = dst; + for (size_t i = 0; i < src_size; i++) + { + if ((src[i] >= '0' && src[i] <= '9') || (src[i] >= 'a' && src[i] <= 'z') || (src[i] >= 'A' && src[i] <= 'Z') + || src[i] == '-' || src[i] == '_' || src[i] == '.' || src[i] == '~') + { + *dst_pos++ = src[i]; + } + else if (src[i] == ' ' && space_as_plus) + { + *dst_pos++ = '+'; + } + else + { + *dst_pos++ = '%'; + *dst_pos++ = hexDigitUppercase(src[i] >> 4); + *dst_pos++ = hexDigitUppercase(src[i] & 0xf); + } + } + *dst_pos++ = src[src_size]; + return dst_pos - dst; +} + /// We assume that size of the dst buf isn't less than src_size. -static size_t decodeURL(const char * src, size_t src_size, char * dst, bool plus_as_space) +static size_t decodeURL(const char * __restrict src, size_t src_size, char * __restrict dst, bool plus_as_space) { const char * src_prev_pos = src; const char * src_curr_pos = src; @@ -81,15 +106,25 @@ static size_t decodeURL(const char * src, size_t src_size, char * dst, bool plus return dst_pos - dst; } +enum URLCodeStrategy +{ + encode, + decode +}; /// Percent decode of URL data. -template -struct DecodeURLComponentImpl +template +struct CodeURLComponentImpl { static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) { - res_data.resize(data.size()); + if (code_strategy == encode) + //the destination(res_data) string is at most three times the length of the source string + res_data.resize(data.size() * 3); + else + res_data.resize(data.size()); + size_t size = offsets.size(); res_offsets.resize(size); @@ -100,7 +135,18 @@ struct DecodeURLComponentImpl { const char * src_data = reinterpret_cast(&data[prev_offset]); size_t src_size = offsets[i] - prev_offset; - size_t dst_size = decodeURL(src_data, src_size, reinterpret_cast(res_data.data() + res_offset), plus_as_space); + size_t dst_size; + + if constexpr (code_strategy == encode) + { + /// Skip encoding of zero terminated character + size_t src_encode_size = src_size - 1; + dst_size = encodeURL(src_data, src_encode_size, reinterpret_cast(res_data.data() + res_offset), space_as_plus); + } + else + { + dst_size = decodeURL(src_data, src_size, reinterpret_cast(res_data.data() + res_offset), space_as_plus); + } res_offset += dst_size; res_offsets[i] = res_offset; @@ -118,14 +164,20 @@ struct DecodeURLComponentImpl struct NameDecodeURLComponent { static constexpr auto name = "decodeURLComponent"; }; +struct NameEncodeURLComponent { static constexpr auto name = "encodeURLComponent"; }; struct NameDecodeURLFormComponent { static constexpr auto name = "decodeURLFormComponent"; }; -using FunctionDecodeURLComponent = FunctionStringToString, NameDecodeURLComponent>; -using FunctionDecodeURLFormComponent = FunctionStringToString, NameDecodeURLFormComponent>; +struct NameEncodeURLFormComponent { static constexpr auto name = "encodeURLFormComponent"; }; +using FunctionDecodeURLComponent = FunctionStringToString, NameDecodeURLComponent>; +using FunctionEncodeURLComponent = FunctionStringToString, NameEncodeURLComponent>; +using FunctionDecodeURLFormComponent = FunctionStringToString, NameDecodeURLFormComponent>; +using FunctionEncodeURLFormComponent = FunctionStringToString, NameEncodeURLFormComponent>; -void registerFunctionDecodeURLComponent(FunctionFactory & factory) +void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory) { factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/URL/domain.h b/src/Functions/URL/domain.h index d43be198043..18efe969216 100644 --- a/src/Functions/URL/domain.h +++ b/src/Functions/URL/domain.h @@ -8,9 +8,6 @@ namespace DB { -namespace -{ - inline StringRef checkAndReturnHost(const Pos & pos, const Pos & dot_pos, const Pos & start_of_host) { if (!dot_pos || start_of_host >= pos || pos - dot_pos == 1) @@ -23,8 +20,6 @@ inline StringRef checkAndReturnHost(const Pos & pos, const Pos & dot_pos, const return StringRef(start_of_host, pos - start_of_host); } -} - /// Extracts host from given url. /// /// @return empty StringRef if the host is not valid (i.e. it does not have dot, or there no symbol after dot). @@ -79,7 +74,7 @@ exloop: if ((scheme_end - pos) > 2 && *pos == ':' && *(pos + 1) == '/' && *(pos } Pos dot_pos = nullptr; - auto start_of_host = pos; + const auto * start_of_host = pos; for (; pos < end; ++pos) { switch (*pos) diff --git a/src/Functions/URL/registerFunctionsURL.cpp b/src/Functions/URL/registerFunctionsURL.cpp index 91118074b7a..91142a593f2 100644 --- a/src/Functions/URL/registerFunctionsURL.cpp +++ b/src/Functions/URL/registerFunctionsURL.cpp @@ -27,7 +27,7 @@ void registerFunctionCutQueryString(FunctionFactory & factory); void registerFunctionCutFragment(FunctionFactory & factory); void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory); void registerFunctionCutURLParameter(FunctionFactory & factory); -void registerFunctionDecodeURLComponent(FunctionFactory & factory); +void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory); void registerFunctionNetloc(FunctionFactory & factory); void registerFunctionsURL(FunctionFactory & factory) @@ -56,7 +56,7 @@ void registerFunctionsURL(FunctionFactory & factory) registerFunctionCutFragment(factory); registerFunctionCutQueryStringAndFragment(factory); registerFunctionCutURLParameter(factory); - registerFunctionDecodeURLComponent(factory); + registerFunctionEncodeAndDecodeURLComponent(factory); registerFunctionNetloc(factory); } diff --git a/src/Functions/VectorExtension.h b/src/Functions/VectorExtension.h index cb4347e3031..fbcbae6b0b6 100644 --- a/src/Functions/VectorExtension.h +++ b/src/Functions/VectorExtension.h @@ -6,27 +6,27 @@ namespace DB::VectorExtension { -typedef UInt64 UInt64x2 __attribute__ ((vector_size (sizeof(UInt64) * 2))); -typedef UInt64 UInt64x4 __attribute__ ((vector_size (sizeof(UInt64) * 4))); -typedef UInt64 UInt64x8 __attribute__ ((vector_size (sizeof(UInt64) * 8))); +using UInt64x2 = UInt64 __attribute__ ((vector_size (sizeof(UInt64) * 2))); +using UInt64x4 = UInt64 __attribute__ ((vector_size (sizeof(UInt64) * 4))); +using UInt64x8 = UInt64 __attribute__ ((vector_size (sizeof(UInt64) * 8))); -typedef UInt32 UInt32x2 __attribute__ ((vector_size (sizeof(UInt32) * 2))); -typedef UInt32 UInt32x4 __attribute__ ((vector_size (sizeof(UInt32) * 4))); -typedef UInt32 UInt32x8 __attribute__ ((vector_size (sizeof(UInt32) * 8))); -typedef UInt32 UInt32x16 __attribute__ ((vector_size (sizeof(UInt32) * 16))); +using UInt32x2 = UInt32 __attribute__ ((vector_size (sizeof(UInt32) * 2))); +using UInt32x4 = UInt32 __attribute__ ((vector_size (sizeof(UInt32) * 4))); +using UInt32x8 = UInt32 __attribute__ ((vector_size (sizeof(UInt32) * 8))); +using UInt32x16 = UInt32 __attribute__ ((vector_size (sizeof(UInt32) * 16))); -typedef UInt16 UInt16x2 __attribute__ ((vector_size (sizeof(UInt16) * 2))); -typedef UInt16 UInt16x4 __attribute__ ((vector_size (sizeof(UInt16) * 4))); -typedef UInt16 UInt16x8 __attribute__ ((vector_size (sizeof(UInt16) * 8))); -typedef UInt16 UInt16x16 __attribute__ ((vector_size (sizeof(UInt16) * 16))); -typedef UInt16 UInt16x32 __attribute__ ((vector_size (sizeof(UInt16) * 32))); +using UInt16x2 = UInt16 __attribute__ ((vector_size (sizeof(UInt16) * 2))); +using UInt16x4 = UInt16 __attribute__ ((vector_size (sizeof(UInt16) * 4))); +using UInt16x8 = UInt16 __attribute__ ((vector_size (sizeof(UInt16) * 8))); +using UInt16x16 = UInt16 __attribute__ ((vector_size (sizeof(UInt16) * 16))); +using UInt16x32 = UInt16 __attribute__ ((vector_size (sizeof(UInt16) * 32))); -typedef UInt8 UInt8x2 __attribute__ ((vector_size (sizeof(UInt8) * 2))); -typedef UInt8 UInt8x4 __attribute__ ((vector_size (sizeof(UInt8) * 4))); -typedef UInt8 UInt8x8 __attribute__ ((vector_size (sizeof(UInt8) * 8))); -typedef UInt8 UInt8x16 __attribute__ ((vector_size (sizeof(UInt8) * 16))); -typedef UInt8 UInt8x32 __attribute__ ((vector_size (sizeof(UInt8) * 32))); -typedef UInt8 UInt8x64 __attribute__ ((vector_size (sizeof(UInt8) * 64))); +using UInt8x2 = UInt8 __attribute__ ((vector_size (sizeof(UInt8) * 2))); +using UInt8x4 = UInt8 __attribute__ ((vector_size (sizeof(UInt8) * 4))); +using UInt8x8 = UInt8 __attribute__ ((vector_size (sizeof(UInt8) * 8))); +using UInt8x16 = UInt8 __attribute__ ((vector_size (sizeof(UInt8) * 16))); +using UInt8x32 = UInt8 __attribute__ ((vector_size (sizeof(UInt8) * 32))); +using UInt8x64 = UInt8 __attribute__ ((vector_size (sizeof(UInt8) * 64))); namespace detail { diff --git a/src/Functions/addSubSeconds.cpp b/src/Functions/addSubSeconds.cpp new file mode 100644 index 00000000000..f58f8b20b99 --- /dev/null +++ b/src/Functions/addSubSeconds.cpp @@ -0,0 +1,28 @@ +#include +#include + + +namespace DB +{ + +using FunctionAddNanoseconds = FunctionDateOrDateTimeAddInterval; +void registerFunctionAddNanoseconds(FunctionFactory & factory) +{ + factory.registerFunction(); +}; + +using FunctionAddMicroseconds = FunctionDateOrDateTimeAddInterval; +void registerFunctionAddMicroseconds(FunctionFactory & factory) +{ + factory.registerFunction(); +}; + +using FunctionAddMilliseconds = FunctionDateOrDateTimeAddInterval; +void registerFunctionAddMilliseconds(FunctionFactory & factory) +{ + factory.registerFunction(); +}; + +} + + diff --git a/src/Functions/addressToLineWithInlines.cpp b/src/Functions/addressToLineWithInlines.cpp index c3e62bd802e..4877268989d 100644 --- a/src/Functions/addressToLineWithInlines.cpp +++ b/src/Functions/addressToLineWithInlines.cpp @@ -75,7 +75,7 @@ private: writeChar(':', out); writeIntText(location.line, out); - if (frame) + if (frame && frame->name != nullptr) { writeChar(':', out); int status = 0; diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 029e33db0cf..58e6db86f75 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -1,18 +1,29 @@ #pragma once +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + #include #include #include -#include -#include -#include -#include -#include -#include +#include + #include -#include +#include + #include +#include + namespace DB { @@ -21,11 +32,38 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int LOGICAL_ERROR; extern const int SIZES_OF_ARRAYS_DOESNT_MATCH; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } +template +ColumnPtr getOffsetsPtr(const T & column) +{ + if constexpr (std::is_same_v) + { + return column.getOffsetsPtr(); + } + else // ColumnMap + { + return column.getNestedColumn().getOffsetsPtr(); + } +} + +template +const IColumn::Offsets & getOffsets(const T & column) +{ + if constexpr (std::is_same_v) + { + return column.getOffsets(); + } + else // ColumnMap + { + return column.getNestedColumn().getOffsets(); + } +} + /** Higher-order functions for arrays. * These functions optionally apply a map (transform) to array (or multiple arrays of identical size) by lambda function, * and return some result based on that transformation. @@ -60,29 +98,42 @@ public: void getLambdaArgumentTypes(DataTypes & arguments) const override { if (arguments.empty()) - throw Exception("Function " + getName() + " needs at least one argument; passed " - + toString(arguments.size()) + ".", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs at least one argument, passed {}", getName(), arguments.size()); if (arguments.size() == 1) - throw Exception("Function " + getName() + " needs at least one array argument.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs at least one argument with data", getName()); - DataTypes nested_types(arguments.size() - 1); - for (size_t i = 0; i < nested_types.size(); ++i) + if (arguments.size() > 2 && Impl::needOneArray()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs one argument with data", getName()); + + size_t nested_types_count = std::is_same_v ? (arguments.size() - 1) * 2 : (arguments.size() - 1); + DataTypes nested_types(nested_types_count); + for (size_t i = 0; i < arguments.size() - 1; ++i) { - const DataTypeArray * array_type = checkAndGetDataType(&*arguments[i + 1]); + const auto * array_type = checkAndGetDataType(&*arguments[i + 1]); if (!array_type) throw Exception("Argument " + toString(i + 2) + " of function " + getName() + " must be array. Found " + arguments[i + 1]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType()); + if constexpr (std::is_same_v) + { + nested_types[2 * i] = recursiveRemoveLowCardinality(array_type->getKeyType()); + nested_types[2 * i + 1] = recursiveRemoveLowCardinality(array_type->getValueType()); + } + else if constexpr (std::is_same_v) + { + nested_types[i] = recursiveRemoveLowCardinality(array_type->getNestedType()); + } } const DataTypeFunction * function_type = checkAndGetDataType(arguments[0].get()); if (!function_type || function_type->getArgumentTypes().size() != nested_types.size()) - throw Exception("First argument for this overload of " + getName() + " must be a function with " - + toString(nested_types.size()) + " arguments. Found " - + arguments[0]->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "First argument for this overload of {} must be a function with {} arguments, found {} instead", + getName(), nested_types.size(), arguments[0]->getName()); arguments[0] = std::make_shared(nested_types); } @@ -91,37 +142,39 @@ public: { size_t min_args = Impl::needExpression() ? 2 : 1; if (arguments.size() < min_args) - throw Exception("Function " + getName() + " needs at least " - + toString(min_args) + " argument; passed " - + toString(arguments.size()) + ".", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} needs at least {} argument, passed {}", + getName(), min_args, arguments.size()); - if (arguments.size() == 1) + if ((arguments.size() == 1) && std::is_same_v) { - const auto * array_type = checkAndGetDataType(arguments[0].type.get()); + const auto * data_type = checkAndGetDataType(arguments[0].type.get()); - if (!array_type) + if (!data_type) throw Exception("The only argument for function " + getName() + " must be array. Found " - + arguments[0].type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + arguments[0].type->getName() + " instead", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - DataTypePtr nested_type = array_type->getNestedType(); + DataTypePtr nested_type = data_type->getNestedType(); if (Impl::needBoolean() && !WhichDataType(nested_type).isUInt8()) throw Exception("The only argument for function " + getName() + " must be array of UInt8. Found " - + arguments[0].type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + arguments[0].type->getName() + " instead", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return Impl::getReturnType(nested_type, nested_type); + if constexpr (std::is_same_v) + return Impl::getReturnType(nested_type, nested_type); + else + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Unreachable code reached"); } else { if (arguments.size() > 2 && Impl::needOneArray()) - throw Exception("Function " + getName() + " needs one array argument.", + throw Exception("Function " + getName() + " needs one argument with data", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); const auto * data_type_function = checkAndGetDataType(arguments[0].type.get()); if (!data_type_function) - throw Exception("First argument for function " + getName() + " must be a function.", + throw Exception("First argument for function " + getName() + " must be a function", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); /// The types of the remaining arguments are already checked in getLambdaArgumentTypes. @@ -131,9 +184,28 @@ public: throw Exception("Expression for function " + getName() + " must return UInt8, found " + return_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); + static_assert( + std::is_same_v || + std::is_same_v, + "unsupported type"); - return Impl::getReturnType(return_type, first_array_type->getNestedType()); + if (arguments.size() < 2) + { + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "{}", arguments.size()); + } + + const auto * first_array_type = checkAndGetDataType(arguments[1].type.get()); + + if (!first_array_type) + throw DB::Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Unsupported type {}", arguments[1].type->getName()); + + if constexpr (std::is_same_v) + return Impl::getReturnType(return_type, first_array_type->getNestedType()); + + if constexpr (std::is_same_v) + return Impl::getReturnType(return_type, first_array_type->getKeyValueTypes()); + + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Unreachable code reached"); } } @@ -142,18 +214,25 @@ public: if (arguments.size() == 1) { ColumnPtr column_array_ptr = arguments[0].column; - const auto * column_array = checkAndGetColumn(column_array_ptr.get()); + const auto * column_array = checkAndGetColumn(column_array_ptr.get()); if (!column_array) { - const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); + const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = column_const_array->convertToFullColumn(); - column_array = assert_cast(column_array_ptr.get()); + column_array = assert_cast(column_array_ptr.get()); } - return Impl::execute(*column_array, column_array->getDataPtr()); + if constexpr (std::is_same_v) + { + return Impl::execute(*column_array, column_array->getNestedColumn().getDataPtr()); + } + else + { + return Impl::execute(*column_array, column_array->getDataPtr()); + } } else { @@ -172,7 +251,7 @@ public: ColumnPtr offsets_column; ColumnPtr column_first_array_ptr; - const ColumnArray * column_first_array = nullptr; + const typename Impl::column_type * column_first_array = nullptr; ColumnsWithTypeAndName arrays; arrays.reserve(arguments.size() - 1); @@ -182,18 +261,18 @@ public: const auto & array_with_type_and_name = arguments[i]; ColumnPtr column_array_ptr = array_with_type_and_name.column; - const auto * column_array = checkAndGetColumn(column_array_ptr.get()); + const auto * column_array = checkAndGetColumn(column_array_ptr.get()); const DataTypePtr & array_type_ptr = array_with_type_and_name.type; - const auto * array_type = checkAndGetDataType(array_type_ptr.get()); + const auto * array_type = checkAndGetDataType(array_type_ptr.get()); if (!column_array) { - const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); + const ColumnConst * column_const_array = checkAndGetColumnConst(column_array_ptr.get()); if (!column_const_array) throw Exception("Expected array column, found " + column_array_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); column_array_ptr = recursiveRemoveLowCardinality(column_const_array->convertToFullColumn()); - column_array = checkAndGetColumn(column_array_ptr.get()); + column_array = checkAndGetColumn(column_array_ptr.get()); } if (!array_type) @@ -201,13 +280,13 @@ public: if (!offsets_column) { - offsets_column = column_array->getOffsetsPtr(); + offsets_column = getOffsetsPtr(*column_array); } else { /// The first condition is optimization: do not compare data if the pointers are equal. - if (column_array->getOffsetsPtr() != offsets_column - && column_array->getOffsets() != typeid_cast(*offsets_column).getData()) + if (getOffsetsPtr(*column_array) != offsets_column + && getOffsets(*column_array) != typeid_cast(*offsets_column).getData()) throw Exception("Arrays passed to " + getName() + " must have equal size", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } @@ -217,13 +296,23 @@ public: column_first_array = column_array; } - arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), - recursiveRemoveLowCardinality(array_type->getNestedType()), - array_with_type_and_name.name)); + if constexpr (std::is_same_v) + { + arrays.emplace_back(ColumnWithTypeAndName( + column_array->getNestedData().getColumnPtr(0), recursiveRemoveLowCardinality(array_type->getKeyType()), array_with_type_and_name.name+".key")); + arrays.emplace_back(ColumnWithTypeAndName( + column_array->getNestedData().getColumnPtr(1), recursiveRemoveLowCardinality(array_type->getValueType()), array_with_type_and_name.name+".value")); + } + else + { + arrays.emplace_back(ColumnWithTypeAndName(column_array->getDataPtr(), + recursiveRemoveLowCardinality(array_type->getNestedType()), + array_with_type_and_name.name)); + } } /// Put all the necessary columns multiplied by the sizes of arrays into the columns. - auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(getOffsets(*column_first_array))); auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(arrays); diff --git a/src/Functions/array/arrayAggregation.cpp b/src/Functions/array/arrayAggregation.cpp index ee08c4f7f37..97a2f9c4c17 100644 --- a/src/Functions/array/arrayAggregation.cpp +++ b/src/Functions/array/arrayAggregation.cpp @@ -1,12 +1,18 @@ -#include -#include -#include -#include -#include -#include "FunctionArrayMapped.h" -#include #include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "FunctionArrayMapped.h" + namespace DB { @@ -83,6 +89,9 @@ using ArrayAggregateResult = typename ArrayAggregateResultImpl struct ArrayAggregateImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayAll.cpp b/src/Functions/array/arrayAll.cpp index 34deafdffdf..0f7ae797dc9 100644 --- a/src/Functions/array/arrayAll.cpp +++ b/src/Functions/array/arrayAll.cpp @@ -1,8 +1,8 @@ -#include #include -#include "FunctionArrayMapped.h" +#include #include +#include "FunctionArrayMapped.h" namespace DB { @@ -16,6 +16,9 @@ namespace ErrorCodes */ struct ArrayAllImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayCompact.cpp b/src/Functions/array/arrayCompact.cpp index c2908e37e12..8abce7288d2 100644 --- a/src/Functions/array/arrayCompact.cpp +++ b/src/Functions/array/arrayCompact.cpp @@ -1,10 +1,13 @@ -#include -#include -#include #include +#include + #include -#include + +#include +#include + #include +#include namespace DB @@ -16,13 +19,16 @@ namespace ErrorCodes struct ArrayCompactImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } - static DataTypePtr getReturnType(const DataTypePtr & nested_type, const DataTypePtr &) + static DataTypePtr getReturnType(const DataTypePtr & , const DataTypePtr & array_element) { - return std::make_shared(nested_type); + return std::make_shared(array_element); } template @@ -30,14 +36,16 @@ struct ArrayCompactImpl { using ColVecType = ColumnVectorOrDecimal; - const ColVecType * src_values_column = checkAndGetColumn(mapped.get()); + const ColVecType * check_values_column = checkAndGetColumn(mapped.get()); + const ColVecType * src_values_column = checkAndGetColumn(array.getData()); - if (!src_values_column) + if (!src_values_column || !check_values_column) return false; const IColumn::Offsets & src_offsets = array.getOffsets(); - const typename ColVecType::Container & src_values = src_values_column->getData(); + const auto & src_values = src_values_column->getData(); + const auto & check_values = check_values_column->getData(); typename ColVecType::MutablePtr res_values_column; if constexpr (is_decimal) res_values_column = ColVecType::create(src_values.size(), src_values_column->getScale()); @@ -45,6 +53,7 @@ struct ArrayCompactImpl res_values_column = ColVecType::create(src_values.size()); typename ColVecType::Container & res_values = res_values_column->getData(); + size_t src_offsets_size = src_offsets.size(); auto res_offsets_column = ColumnArray::ColumnOffsets::create(src_offsets_size); IColumn::Offsets & res_offsets = res_offsets_column->getData(); @@ -67,7 +76,7 @@ struct ArrayCompactImpl ++res_pos; for (; src_pos < src_offset; ++src_pos) { - if (!bitEquals(src_values[src_pos], src_values[src_pos - 1])) + if (!bitEquals(check_values[src_pos], check_values[src_pos - 1])) { res_values[res_pos] = src_values[src_pos]; ++res_pos; @@ -86,8 +95,9 @@ struct ArrayCompactImpl { const IColumn::Offsets & src_offsets = array.getOffsets(); - auto res_values_column = mapped->cloneEmpty(); - res_values_column->reserve(mapped->size()); + const auto & src_values = array.getData(); + auto res_values_column = src_values.cloneEmpty(); + res_values_column->reserve(src_values.size()); size_t src_offsets_size = src_offsets.size(); auto res_offsets_column = ColumnArray::ColumnOffsets::create(src_offsets_size); @@ -104,7 +114,7 @@ struct ArrayCompactImpl if (src_pos < src_offset) { /// Insert first element unconditionally. - res_values_column->insertFrom(*mapped, src_pos); + res_values_column->insertFrom(src_values, src_pos); /// For the rest of elements, insert if the element is different from the previous. ++src_pos; @@ -113,7 +123,7 @@ struct ArrayCompactImpl { if (mapped->compareAt(src_pos - 1, src_pos, *mapped, 1)) { - res_values_column->insertFrom(*mapped, src_pos); + res_values_column->insertFrom(src_values, src_pos); ++res_pos; } } diff --git a/src/Functions/array/arrayCount.cpp b/src/Functions/array/arrayCount.cpp index 377a6eb8fb1..df45783323b 100644 --- a/src/Functions/array/arrayCount.cpp +++ b/src/Functions/array/arrayCount.cpp @@ -1,8 +1,9 @@ -#include #include -#include "FunctionArrayMapped.h" +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -16,6 +17,9 @@ namespace ErrorCodes */ struct ArrayCountImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayCumSum.cpp b/src/Functions/array/arrayCumSum.cpp index 467d9ad3951..98ffa09820b 100644 --- a/src/Functions/array/arrayCumSum.cpp +++ b/src/Functions/array/arrayCumSum.cpp @@ -1,10 +1,11 @@ -#include -#include -#include #include -#include "FunctionArrayMapped.h" +#include +#include +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -17,6 +18,9 @@ namespace ErrorCodes struct ArrayCumSumImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index 476bbd08163..cd8393b7a5f 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -1,10 +1,10 @@ -#include -#include -#include #include -#include "FunctionArrayMapped.h" +#include +#include +#include #include +#include "FunctionArrayMapped.h" namespace DB { @@ -19,6 +19,9 @@ namespace ErrorCodes */ struct ArrayCumSumNonNegativeImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayDifference.cpp b/src/Functions/array/arrayDifference.cpp index c5fdf27100b..8af0e8b04f9 100644 --- a/src/Functions/array/arrayDifference.cpp +++ b/src/Functions/array/arrayDifference.cpp @@ -1,10 +1,11 @@ -#include -#include -#include #include -#include "FunctionArrayMapped.h" +#include +#include +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -20,6 +21,9 @@ namespace ErrorCodes */ struct ArrayDifferenceImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 4d03c52460f..d6a62a966ae 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -252,7 +252,7 @@ ColumnPtr FunctionArrayEnumerateRankedExtended::executeImpl( ColumnPtr result_nested_array = std::move(res_nested); for (ssize_t depth = arrays_depths.max_array_depth - 1; depth >= 0; --depth) - result_nested_array = ColumnArray::create(std::move(result_nested_array), offsetsptr_by_depth[depth]); + result_nested_array = ColumnArray::create(result_nested_array, offsetsptr_by_depth[depth]); return result_nested_array; } diff --git a/src/Functions/array/arrayExists.cpp b/src/Functions/array/arrayExists.cpp index 34ea71af259..ea39cc0dc0b 100644 --- a/src/Functions/array/arrayExists.cpp +++ b/src/Functions/array/arrayExists.cpp @@ -1,8 +1,9 @@ -#include #include -#include "FunctionArrayMapped.h" +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -16,6 +17,9 @@ namespace ErrorCodes */ struct ArrayExistsImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return true; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayFill.cpp b/src/Functions/array/arrayFill.cpp index d4b36a89ba5..22b9e9a657b 100644 --- a/src/Functions/array/arrayFill.cpp +++ b/src/Functions/array/arrayFill.cpp @@ -1,8 +1,9 @@ -#include #include -#include "FunctionArrayMapped.h" +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -19,6 +20,9 @@ namespace ErrorCodes template struct ArrayFillImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayFilter.cpp b/src/Functions/array/arrayFilter.cpp index 1291989f9a2..89a9de44532 100644 --- a/src/Functions/array/arrayFilter.cpp +++ b/src/Functions/array/arrayFilter.cpp @@ -1,8 +1,9 @@ -#include #include -#include "FunctionArrayMapped.h" +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -15,6 +16,9 @@ namespace ErrorCodes */ struct ArrayFilterImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayFirst.cpp b/src/Functions/array/arrayFirstLast.cpp similarity index 56% rename from src/Functions/array/arrayFirst.cpp rename to src/Functions/array/arrayFirstLast.cpp index edbf7ef6269..c9a8d2ba497 100644 --- a/src/Functions/array/arrayFirst.cpp +++ b/src/Functions/array/arrayFirstLast.cpp @@ -1,8 +1,10 @@ -#include #include -#include "FunctionArrayMapped.h" +#include +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -11,21 +13,33 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -enum class ArrayFirstLastStrategy +enum class ArrayFirstLastStrategy : uint8_t { First, Last }; -template +enum class ArrayFirstLastElementNotExistsStrategy : uint8_t +{ + Default, + Null +}; + +template struct ArrayFirstLastImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return true; } static bool needOneArray() { return false; } static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypePtr & array_element) { + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + return makeNullable(array_element); + return array_element; } @@ -48,6 +62,16 @@ struct ArrayFirstLastImpl out->reserve(data.size()); size_t offsets_size = offsets.size(); + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to = nullptr; + + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + { + col_null_map_to = ColumnUInt8::create(offsets_size, false); + vec_null_map_to = &col_null_map_to->getData(); + } + for (size_t offset_index = 0; offset_index < offsets_size; ++offset_index) { size_t start_offset = offsets[offset_index - 1]; @@ -63,16 +87,29 @@ struct ArrayFirstLastImpl else { out->insertDefault(); + + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + (*vec_null_map_to)[offset_index] = true; } } + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + return ColumnNullable::create(std::move(out), std::move(col_null_map_to)); + return out; } else { auto out = array.getData().cloneEmpty(); - out->insertDefault(); - return out->replicate(IColumn::Offsets(1, array.size())); + out->insertManyDefaults(array.size()); + + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + { + auto col_null_map_to = ColumnUInt8::create(out->size(), true); + return ColumnNullable::create(std::move(out), std::move(col_null_map_to)); + } + + return out; } } @@ -83,6 +120,16 @@ struct ArrayFirstLastImpl out->reserve(data.size()); size_t offsets_size = offsets.size(); + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to = nullptr; + + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + { + col_null_map_to = ColumnUInt8::create(offsets_size, false); + vec_null_map_to = &col_null_map_to->getData(); + } + for (size_t offset_index = 0; offset_index < offsets_size; ++offset_index) { size_t start_offset = offsets[offset_index - 1]; @@ -116,25 +163,43 @@ struct ArrayFirstLastImpl } if (!exists) + { out->insertDefault(); + + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + (*vec_null_map_to)[offset_index] = true; + } } + if constexpr (element_not_exists_strategy == ArrayFirstLastElementNotExistsStrategy::Null) + return ColumnNullable::create(std::move(out), std::move(col_null_map_to)); + return out; } }; struct NameArrayFirst { static constexpr auto name = "arrayFirst"; }; -using ArrayFirstImpl = ArrayFirstLastImpl; +using ArrayFirstImpl = ArrayFirstLastImpl; using FunctionArrayFirst = FunctionArrayMapped; +struct NameArrayFirstOrNull { static constexpr auto name = "arrayFirstOrNull"; }; +using ArrayFirstOrNullImpl = ArrayFirstLastImpl; +using FunctionArrayFirstOrNull = FunctionArrayMapped; + struct NameArrayLast { static constexpr auto name = "arrayLast"; }; -using ArrayLastImpl = ArrayFirstLastImpl; +using ArrayLastImpl = ArrayFirstLastImpl; using FunctionArrayLast = FunctionArrayMapped; +struct NameArrayLastOrNull { static constexpr auto name = "arrayLastOrNull"; }; +using ArrayLastOrNullImpl = ArrayFirstLastImpl; +using FunctionArrayLastOrNull = FunctionArrayMapped; + void registerFunctionArrayFirst(FunctionFactory & factory) { factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/array/arrayFirstLastIndex.cpp b/src/Functions/array/arrayFirstLastIndex.cpp index 467678f3faa..9392cbdc840 100644 --- a/src/Functions/array/arrayFirstLastIndex.cpp +++ b/src/Functions/array/arrayFirstLastIndex.cpp @@ -1,8 +1,9 @@ #include #include -#include "FunctionArrayMapped.h" #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -20,6 +21,9 @@ enum class ArrayFirstLastIndexStrategy template struct ArrayFirstLastIndexImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return true; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index c231ddbb373..8b42b99cd69 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -432,7 +432,7 @@ public: const auto & map_array_column = map_column.getNestedColumn(); auto offsets = map_array_column.getOffsetsPtr(); auto keys = map_column.getNestedData().getColumnPtr(0); - auto array_column = ColumnArray::create(std::move(keys), std::move(offsets)); + auto array_column = ColumnArray::create(keys, offsets); const auto & type_map = assert_cast(*arguments[0].type); auto array_type = std::make_shared(type_map.getKeyType()); @@ -474,7 +474,7 @@ private: auto arg_decayed = removeNullable(removeLowCardinality(arg)); return ((isNativeNumber(inner_type_decayed) || isEnum(inner_type_decayed)) && isNativeNumber(arg_decayed)) - || getLeastSupertype({inner_type_decayed, arg_decayed}); + || getLeastSupertype(DataTypes{inner_type_decayed, arg_decayed}); } /** @@ -1045,7 +1045,7 @@ private: DataTypePtr array_elements_type = assert_cast(*arguments[0].type).getNestedType(); const DataTypePtr & index_type = arguments[1].type; - DataTypePtr common_type = getLeastSupertype({array_elements_type, index_type}); + DataTypePtr common_type = getLeastSupertype(DataTypes{array_elements_type, index_type}); ColumnPtr col_nested = castColumn({ col->getDataPtr(), array_elements_type, "" }, common_type); diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index f2779a2fe58..f1b849b64f0 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -477,7 +477,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable columns.reserve(args); for (const auto & arg : arrays.args) { - if constexpr (std::is_same::value) + if constexpr (std::is_same_v) columns.push_back(arg.nested_column); else columns.push_back(checkAndGetColumn(arg.nested_column)); @@ -530,7 +530,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable { value = &map[columns[arg_num]->getElement(i)]; } - else if constexpr (std::is_same::value || std::is_same::value) + else if constexpr (std::is_same_v || std::is_same_v) value = &map[columns[arg_num]->getDataAt(i)]; else { @@ -566,7 +566,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable ++result_offset; if constexpr (is_numeric_column) result_data.insertValue(pair.getKey()); - else if constexpr (std::is_same::value || std::is_same::value) + else if constexpr (std::is_same_v || std::is_same_v) result_data.insertData(pair.getKey().data, pair.getKey().size); else result_data.deserializeAndInsertFromArena(pair.getKey().data); diff --git a/src/Functions/array/arrayMap.cpp b/src/Functions/array/arrayMap.cpp index e3afaf7fb66..ec1973d573b 100644 --- a/src/Functions/array/arrayMap.cpp +++ b/src/Functions/array/arrayMap.cpp @@ -1,14 +1,18 @@ -#include "FunctionArrayMapped.h" #include +#include "FunctionArrayMapped.h" + namespace DB { -/** arrayMap(x1,...,xn -> expression, array1,...,arrayn) - apply the expression to each element of the array (or set of parallel arrays). +/** arrayMap(x1, ..., xn -> expression, array1, ..., arrayn) - apply the expression to each element of the array (or set of parallel arrays). */ struct ArrayMapImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + /// true if the expression (for an overload of f(expression, arrays)) or an array (for f(array)) should be boolean. static bool needBoolean() { return false; } /// true if the f(array) overload is unavailable. diff --git a/src/Functions/array/arrayResize.cpp b/src/Functions/array/arrayResize.cpp index 9d2a29b2fb4..1e6dcfbf069 100644 --- a/src/Functions/array/arrayResize.cpp +++ b/src/Functions/array/arrayResize.cpp @@ -62,7 +62,7 @@ public: if (number_of_arguments == 2) return arguments[0]; else /* if (number_of_arguments == 3) */ - return std::make_shared(getLeastSupertype({array_type->getNestedType(), arguments[2]})); + return std::make_shared(getLeastSupertype(DataTypes{array_type->getNestedType(), arguments[2]})); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type, size_t input_rows_count) const override diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h index 87161038d4c..4e3eab2faf8 100644 --- a/src/Functions/array/arrayScalarProduct.h +++ b/src/Functions/array/arrayScalarProduct.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 476dfb46f07..5421185211e 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -1,8 +1,8 @@ #include "FunctionArrayMapped.h" + #include #include - namespace DB { @@ -11,6 +11,9 @@ namespace DB template struct ArraySortImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return false; } static bool needExpression() { return false; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/arraySplit.cpp b/src/Functions/array/arraySplit.cpp index 2e5f2d8432e..c818be97f60 100644 --- a/src/Functions/array/arraySplit.cpp +++ b/src/Functions/array/arraySplit.cpp @@ -1,8 +1,9 @@ -#include #include -#include "FunctionArrayMapped.h" +#include #include +#include "FunctionArrayMapped.h" + namespace DB { @@ -14,6 +15,9 @@ namespace ErrorCodes template struct ArraySplitImpl { + using column_type = ColumnArray; + using data_type = DataTypeArray; + static bool needBoolean() { return true; } static bool needExpression() { return true; } static bool needOneArray() { return false; } diff --git a/src/Functions/array/hasAllAny.h b/src/Functions/array/hasAllAny.h index cd55fea3521..3ba8bb6156f 100644 --- a/src/Functions/array/hasAllAny.h +++ b/src/Functions/array/hasAllAny.h @@ -44,7 +44,7 @@ public: { for (auto i : collections::range(0, arguments.size())) { - auto array_type = typeid_cast(arguments[i].get()); + const auto * array_type = typeid_cast(arguments[i].get()); if (!array_type) throw Exception("Argument " + std::to_string(i) + " for function " + getName() + " must be an array but it has type " + arguments[i]->getName() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); diff --git a/src/Functions/array/mapOp.cpp b/src/Functions/array/mapOp.cpp index b928254e454..f743cfb5b5d 100644 --- a/src/Functions/array/mapOp.cpp +++ b/src/Functions/array/mapOp.cpp @@ -225,7 +225,7 @@ private: for (size_t j = 0; j < len; ++j) { KeyType key; - if constexpr (std::is_same::value) + if constexpr (std::is_same_v) { if (const auto * col_fixed = checkAndGetColumn(arg.key_column.get())) key = col_fixed->getDataAt(offset + j).toString(); diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index 17269f8dfe1..8b4a1dda197 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -379,8 +379,7 @@ private: if (!max_key_column_type->equals(*input.key_series_type)) { ColumnWithTypeAndName column_to_cast = {max_key_column, max_key_column_type, ""}; - auto casted_column = castColumnAccurate(std::move(column_to_cast), input.key_series_type); - max_key_column = std::move(casted_column); + max_key_column = castColumnAccurate(column_to_cast, input.key_series_type); } } diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp index 95046d95176..628ac57f34d 100644 --- a/src/Functions/castOrDefault.cpp +++ b/src/Functions/castOrDefault.cpp @@ -99,7 +99,7 @@ public: { const ColumnWithTypeAndName & column_to_cast = arguments[0]; auto non_const_column_to_cast = column_to_cast.column->convertToFullColumnIfConst(); - ColumnWithTypeAndName column_to_cast_non_const { std::move(non_const_column_to_cast), column_to_cast.type, column_to_cast.name }; + ColumnWithTypeAndName column_to_cast_non_const { non_const_column_to_cast, column_to_cast.type, column_to_cast.name }; auto cast_result = castColumnAccurateOrNull(column_to_cast_non_const, return_type); diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index fa75e305af4..057dedab6e4 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -55,7 +55,7 @@ public: static constexpr auto Kind = Impl::Kind; static constexpr auto name = Impl::Name; - FunctionExtractAllGroups(ContextPtr context_) + explicit FunctionExtractAllGroups(ContextPtr context_) : context(context_) {} diff --git a/src/Functions/formatReadable.h b/src/Functions/formatReadable.h index 7c0d6c5c817..0378e1f82f2 100644 --- a/src/Functions/formatReadable.h +++ b/src/Functions/formatReadable.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 18951d1a03f..ce0bdee74a3 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -20,6 +21,8 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int INCORRECT_DATA; + extern const int ILLEGAL_COLUMN; + extern const int ARGUMENT_OUT_OF_BOUND; } namespace @@ -68,9 +71,39 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lon = arguments[0].column.get(); - const auto * col_lat = arguments[1].column.get(); - const auto * col_res = arguments[2].column.get(); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lon = checkAndGetColumn(non_const_arguments[0].column.get()); + if (!col_lon) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64.", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_lon = col_lon->getData(); + + const auto * col_lat = checkAndGetColumn(non_const_arguments[1].column.get()); + if (!col_lat) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64.", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_lat = col_lat->getData(); + + const auto * col_res = checkAndGetColumn(non_const_arguments[2].column.get()); + if (!col_res) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt8.", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_res = col_res->getData(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); @@ -78,9 +111,17 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const double lon = col_lon->getFloat64(row); - const double lat = col_lat->getFloat64(row); - const UInt8 res = col_res->getUInt(row); + const double lon = data_lon[row]; + const double lat = data_lat[row]; + const UInt8 res = data_res[row]; + + if (res > MAX_H3_RES) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + toString(res), + getName(), + MAX_H3_RES); LatLng coord; coord.lng = degsToRads(lon); diff --git a/src/Functions/geoToS2.cpp b/src/Functions/geoToS2.cpp index d69c15bdbe0..8d90552652a 100644 --- a/src/Functions/geoToS2.cpp +++ b/src/Functions/geoToS2.cpp @@ -67,7 +67,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lon = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lon = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lon) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -77,7 +81,7 @@ public: getName()); const auto & data_col_lon = col_lon->getData(); - const auto * col_lat = checkAndGetColumn(arguments[1].column.get()); + const auto * col_lat = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_lat) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/getMacro.cpp b/src/Functions/getMacro.cpp index fb48a06aabc..29e706ce2a0 100644 --- a/src/Functions/getMacro.cpp +++ b/src/Functions/getMacro.cpp @@ -27,15 +27,19 @@ class FunctionGetMacro : public IFunction { private: MultiVersion::Version macros; + bool is_distributed; public: static constexpr auto name = "getMacro"; static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getMacros()); + return std::make_shared(context->getMacros(), context->isDistributed()); } - explicit FunctionGetMacro(MultiVersion::Version macros_) : macros(std::move(macros_)) {} + explicit FunctionGetMacro(MultiVersion::Version macros_, bool is_distributed_) + : macros(std::move(macros_)), is_distributed(is_distributed_) + { + } String getName() const override { @@ -48,9 +52,12 @@ public: bool isDeterministicInScopeOfQuery() const override { - return false; + return true; } + /// getMacro may return different values on different shards/replicas, so it's not constant for distributed query + bool isSuitableForConstantFolding() const override { return !is_distributed; } + size_t getNumberOfArguments() const override { return 1; @@ -63,9 +70,6 @@ public: return std::make_shared(); } - /** convertToFullColumn needed because in distributed query processing, - * each server returns its own value. - */ ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const IColumn * arg_column = arguments[0].column.get(); @@ -74,8 +78,7 @@ public: if (!arg_string) throw Exception("The argument of function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_COLUMN); - return result_type->createColumnConst( - input_rows_count, macros->getValue(arg_string->getDataAt(0).toString()))->convertToFullColumnIfConst(); + return result_type->createColumnConst(input_rows_count, macros->getValue(arg_string->getDataAt(0).toString())); } }; diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index 1f7be1a6374..f0743486584 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -179,7 +179,7 @@ float distance(float lon1deg, float lat1deg, float lon2deg, float lat2deg) /// Why comparing only difference in longitude? /// If longitudes are different enough, there is a big difference between great circle line and a line with constant latitude. - /// (Remember how a plane flies from Moscow to New York) + /// (Remember how a plane flies from Amsterdam to New York) /// But if longitude is close but latitude is different enough, there is no difference between meridian and great circle line. float latitude_midpoint = (lat1deg + lat2deg + 180) * METRIC_LUT_SIZE / 360; // [-90, 90] degrees -> [0, METRIC_LUT_SIZE] indexes @@ -326,4 +326,3 @@ void registerFunctionGeoDistance(FunctionFactory & factory) } } - diff --git a/src/Functions/h3CellAreaM2.cpp b/src/Functions/h3CellAreaM2.cpp index d110d0d92f9..e5fb5aa2ed7 100644 --- a/src/Functions/h3CellAreaM2.cpp +++ b/src/Functions/h3CellAreaM2.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; +extern const int INCORRECT_DATA; } namespace @@ -52,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -70,6 +75,12 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { const UInt64 index = data[row]; + + CellBoundary boundary{}; + auto err = cellToBoundary(index, &boundary); + if (err) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect H3 index: {}, error: {}", index, err); + Float64 res = cellAreaM2(index); dst_data[row] = res; } diff --git a/src/Functions/h3CellAreaRads2.cpp b/src/Functions/h3CellAreaRads2.cpp index 1a257b0d9d3..15d18613b72 100644 --- a/src/Functions/h3CellAreaRads2.cpp +++ b/src/Functions/h3CellAreaRads2.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; +extern const int INCORRECT_DATA; } namespace @@ -52,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -60,7 +65,6 @@ public: arguments[0].type->getName(), 1, getName()); - const auto & data = column->getData(); auto dst = ColumnVector::create(); @@ -70,6 +74,12 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { const UInt64 index = data[row]; + + CellBoundary boundary{}; + auto err = cellToBoundary(index, &boundary); + if (err) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect H3 index: {}, error: {}", index, err); + Float64 res = cellAreaRads2(index); dst_data[row] = res; } diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index a65fde285f1..8240f092ee8 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp index 0cc485e93b1..b7072a2f309 100644 --- a/src/Functions/h3EdgeLengthKm.cpp +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 08b78517be9..2c934bc6c05 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -58,7 +58,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ExactEdgeLengthKm.cpp b/src/Functions/h3ExactEdgeLengthKm.cpp index 7aa9e573bed..f37d93fd715 100644 --- a/src/Functions/h3ExactEdgeLengthKm.cpp +++ b/src/Functions/h3ExactEdgeLengthKm.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ExactEdgeLengthM.cpp b/src/Functions/h3ExactEdgeLengthM.cpp index 5b7cb91e427..99acbb757c3 100644 --- a/src/Functions/h3ExactEdgeLengthM.cpp +++ b/src/Functions/h3ExactEdgeLengthM.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ExactEdgeLengthRads.cpp b/src/Functions/h3ExactEdgeLengthRads.cpp index d2b9345c989..a2937e85c65 100644 --- a/src/Functions/h3ExactEdgeLengthRads.cpp +++ b/src/Functions/h3ExactEdgeLengthRads.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index 7865f454815..a3023561824 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3GetFaces.cpp b/src/Functions/h3GetFaces.cpp index b85ad37c04f..0344ccc7944 100644 --- a/src/Functions/h3GetFaces.cpp +++ b/src/Functions/h3GetFaces.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3GetPentagonIndexes.cpp b/src/Functions/h3GetPentagonIndexes.cpp new file mode 100644 index 00000000000..bc90187bb3a --- /dev/null +++ b/src/Functions/h3GetPentagonIndexes.cpp @@ -0,0 +1,118 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + +class FunctionH3GetPentagonIndexes : public IFunction +{ +public: + static constexpr auto name = "h3GetPentagonIndexes"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt8()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 1, getName()); + + return std::make_shared(std::make_shared()); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt8.", + arguments[0].type->getName(), + 1, + getName()); + const auto & data = column->getData(); + + auto result_column_data = ColumnUInt64::create(); + auto & result_data = result_column_data->getData(); + + auto result_column_offsets = ColumnArray::ColumnOffsets::create(); + auto & result_offsets = result_column_offsets->getData(); + result_offsets.resize(input_rows_count); + + auto current_offset = 0; + std::vector hindex_vec; + result_data.reserve(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + if (data[row] > MAX_H3_RES) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + toString(data[row]), + getName(), + MAX_H3_RES); + + const auto vec_size = pentagonCount(); + hindex_vec.resize(vec_size); + getPentagons(data[row], hindex_vec.data()); + + for (auto & i : hindex_vec) + { + ++current_offset; + result_data.emplace_back(i); + } + + result_offsets[row] = current_offset; + hindex_vec.clear(); + } + + return ColumnArray::create(std::move(result_column_data), std::move(result_column_offsets)); + } +}; + +} + +void registerFunctionH3GetPentagonIndexes(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/h3GetRes0Indexes.cpp b/src/Functions/h3GetRes0Indexes.cpp new file mode 100644 index 00000000000..7347b0fcf7f --- /dev/null +++ b/src/Functions/h3GetRes0Indexes.cpp @@ -0,0 +1,72 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace +{ + +class FunctionH3GetRes0Indexes final : public IFunction +{ +public: + static constexpr auto name = "h3GetRes0Indexes"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(std::make_shared()); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr & result_type, size_t input_rows_count) const override + { + if (input_rows_count == 0) + return result_type->createColumn(); + + std::vector res0_indexes; + const auto cell_count = res0CellCount(); + res0_indexes.resize(cell_count); + getRes0Cells(res0_indexes.data()); + + auto res = ColumnArray::create(ColumnUInt64::create()); + + Array res_indexes; + res_indexes.insert(res_indexes.end(), res0_indexes.begin(), res0_indexes.end()); + res->insert(res_indexes); + + return result_type->createColumnConst(input_rows_count, res_indexes); + } +}; + +} + +void registerFunctionH3GetRes0Indexes(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index 7cc7dab8916..4d0a92a1414 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3HexAreaKm2.cpp b/src/Functions/h3HexAreaKm2.cpp index 74b74a351a1..c4a3c1258ab 100644 --- a/src/Functions/h3HexAreaKm2.cpp +++ b/src/Functions/h3HexAreaKm2.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index ff68d01bf68..f8599651b0f 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index 97b8461fc5a..ee603f7de49 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -57,7 +57,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex_origin = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex_origin = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex_origin) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -68,7 +72,7 @@ public: const auto & data_hindex_origin = col_hindex_origin->getData(); - const auto * col_hindex_dest = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hindex_dest = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hindex_dest) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IsPentagon.cpp b/src/Functions/h3IsPentagon.cpp index 78c93be75a6..b7374bc1d8d 100644 --- a/src/Functions/h3IsPentagon.cpp +++ b/src/Functions/h3IsPentagon.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index 3fd9e1e62b0..08025c966cd 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -50,7 +50,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index 7c97e77250c..b6701d89de6 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -49,7 +49,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index 4336e441cfb..009ff182940 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3PointDist.cpp b/src/Functions/h3PointDist.cpp new file mode 100644 index 00000000000..463050c9b81 --- /dev/null +++ b/src/Functions/h3PointDist.cpp @@ -0,0 +1,153 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} + +namespace +{ +template +class FunctionH3PointDist final : public IFunction +{ +public: + static constexpr auto name = Impl::name; + static constexpr auto function = Impl::function; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 4; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const auto * arg = arguments[i].get(); + if (!WhichDataType(arg).isFloat64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), i, getName()); + } + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lat1 = checkAndGetColumn(non_const_arguments[0].column.get()); + if (!col_lat1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_lat1 = col_lat1->getData(); + + const auto * col_lon1 = checkAndGetColumn(non_const_arguments[1].column.get()); + if (!col_lon1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_lon1 = col_lon1->getData(); + + const auto * col_lat2 = checkAndGetColumn(non_const_arguments[2].column.get()); + if (!col_lat2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_lat2 = col_lat2->getData(); + + const auto * col_lon2 = checkAndGetColumn(non_const_arguments[3].column.get()); + if (!col_lon2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[3].type->getName(), + 4, + getName()); + const auto & data_lon2 = col_lon2->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const double lat1 = data_lat1[row]; + const double lon1 = data_lon1[row]; + const auto lat2 = data_lat2[row]; + const auto lon2 = data_lon2[row]; + + LatLng point1 = {degsToRads(lat1), degsToRads(lon1)}; + LatLng point2 = {degsToRads(lat2), degsToRads(lon2)}; + + // function will be equivalent to distanceM or distanceKm or distanceRads + Float64 res = function(&point1, &point2); + dst_data[row] = res; + } + + return dst; + } +}; + +} + +struct H3PointDistM +{ + static constexpr auto name = "h3PointDistM"; + static constexpr auto function = distanceM; +}; + +struct H3PointDistKm +{ + static constexpr auto name = "h3PointDistKm"; + static constexpr auto function = distanceKm; +}; + +struct H3PointDistRads +{ + static constexpr auto name = "h3PointDistRads"; + static constexpr auto function = distanceRads; +}; + + +void registerFunctionH3PointDistM(FunctionFactory & factory) { factory.registerFunction>(); } +void registerFunctionH3PointDistKm(FunctionFactory & factory) { factory.registerFunction>(); } +void registerFunctionH3PointDistRads(FunctionFactory & factory) { factory.registerFunction>(); } + +} + +#endif diff --git a/src/Functions/h3ToCenterChild.cpp b/src/Functions/h3ToCenterChild.cpp index d9bc31b8f19..6b9724c297c 100644 --- a/src/Functions/h3ToCenterChild.cpp +++ b/src/Functions/h3ToCenterChild.cpp @@ -61,7 +61,11 @@ namespace ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -71,7 +75,7 @@ namespace getName()); const auto & data_hindex = col_hindex->getData(); - const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + const auto * col_resolution = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_resolution) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index a825255495d..26d3e3dc204 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -66,7 +66,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -77,7 +81,7 @@ public: const auto & data_hindex = col_hindex->getData(); - const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + const auto * col_resolution = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_resolution) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToGeoBoundary.cpp b/src/Functions/h3ToGeoBoundary.cpp index 2c892fb59ae..acf1ec64020 100644 --- a/src/Functions/h3ToGeoBoundary.cpp +++ b/src/Functions/h3ToGeoBoundary.cpp @@ -52,7 +52,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index b2262d3ac22..8fd9931b81b 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -60,7 +60,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -71,7 +75,7 @@ public: const auto & data_hindex = col_hindex->getData(); - const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + const auto * col_resolution = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_resolution) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index 0152dd69b6a..7ca46c3b064 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -53,7 +53,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index baa74b6698b..a801fd299d6 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -64,7 +64,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_hindex = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_hindex) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -76,7 +80,7 @@ public: const auto & data_hindex = col_hindex->getData(); /// ColumnUInt16 is sufficient as the max value of 2nd arg is checked (arg > 0 < 10000) in implementation below - const auto * col_k = checkAndGetColumn(arguments[1].column.get()); + const auto * col_k = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_k) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index 403bcc3c3d6..9c5eb1b11bf 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -59,7 +59,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * column = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * column = checkAndGetColumn(non_const_arguments[0].column.get()); if (!column) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 6841098ebcf..82448966b8c 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -632,7 +632,7 @@ private: const ColumnWithTypeAndName & arg1 = arguments[1]; const ColumnWithTypeAndName & arg2 = arguments[2]; - DataTypePtr common_type = getLeastSupertype({arg1.type, arg2.type}); + DataTypePtr common_type = getLeastSupertype(DataTypes{arg1.type, arg2.type}); ColumnPtr col_then = castColumn(arg1, common_type); ColumnPtr col_else = castColumn(arg2, common_type); @@ -894,13 +894,20 @@ private: /// If then is NULL, we create Nullable column with null mask OR-ed with condition. if (then_is_null) { + ColumnPtr arg_else_column; + /// In case when arg_else column type differs with result + /// column type we should cast it to result type. + if (removeNullable(arg_else.type)->getName() != removeNullable(result_type)->getName()) + arg_else_column = castColumn(arg_else, result_type); + else + arg_else_column = arg_else.column; + if (cond_col) { - auto arg_else_column = arg_else.column; auto result_column = IColumn::mutate(std::move(arg_else_column)); if (else_is_short) result_column->expand(cond_col->getData(), true); - if (isColumnNullable(*arg_else.column)) + if (isColumnNullable(*result_column)) { assert_cast(*result_column).applyNullMap(assert_cast(*arg_cond.column)); return result_column; @@ -913,7 +920,7 @@ private: if (cond_const_col->getValue()) return result_type->createColumn()->cloneResized(input_rows_count); else - return makeNullableColumnIfNot(arg_else.column); + return makeNullableColumnIfNot(arg_else_column); } else throw Exception("Illegal column " + arg_cond.column->getName() + " of first argument of function " + getName() @@ -924,14 +931,21 @@ private: /// If else is NULL, we create Nullable column with null mask OR-ed with negated condition. if (else_is_null) { + ColumnPtr arg_then_column; + /// In case when arg_then column type differs with result + /// column type we should cast it to result type. + if (removeNullable(arg_then.type)->getName() != removeNullable(result_type)->getName()) + arg_then_column = castColumn(arg_then, result_type); + else + arg_then_column = arg_then.column; + if (cond_col) { - auto arg_then_column = arg_then.column; auto result_column = IColumn::mutate(std::move(arg_then_column)); if (then_is_short) result_column->expand(cond_col->getData(), false); - if (isColumnNullable(*arg_then.column)) + if (isColumnNullable(*result_column)) { assert_cast(*result_column).applyNegatedNullMap(assert_cast(*arg_cond.column)); return result_column; @@ -954,7 +968,7 @@ private: else if (cond_const_col) { if (cond_const_col->getValue()) - return makeNullableColumnIfNot(arg_then.column); + return makeNullableColumnIfNot(arg_then_column); else return result_type->createColumn()->cloneResized(input_rows_count); } @@ -1022,12 +1036,12 @@ public: throw Exception("Illegal type " + arguments[0]->getName() + " of first argument (condition) of function if. Must be UInt8.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return getLeastSupertype({arguments[1], arguments[2]}); + return getLeastSupertype(DataTypes{arguments[1], arguments[2]}); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const override { - ColumnsWithTypeAndName arguments = std::move(args); + ColumnsWithTypeAndName arguments = args; executeShortCircuitArguments(arguments); ColumnPtr res; if ( (res = executeForConstAndNullableCondition(arguments, result_type, input_rows_count)) diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp index 31880b81a41..ab8e2677d28 100644 --- a/src/Functions/ifNull.cpp +++ b/src/Functions/ifNull.cpp @@ -47,7 +47,7 @@ public: if (!arguments[0]->isNullable()) return arguments[0]; - return getLeastSupertype({removeNullable(arguments[0]), arguments[1]}); + return getLeastSupertype(DataTypes{removeNullable(arguments[0]), arguments[1]}); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 87e5886b247..469b98ed00d 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -121,7 +121,8 @@ public: auto set = column_set->getData(); auto set_types = set->getDataTypes(); - if (tuple && (set_types.size() != 1 || !set_types[0]->equals(*type_tuple))) + + if (tuple && set_types.size() != 1 && set_types.size() == tuple->tupleSize()) { const auto & tuple_columns = tuple->getColumns(); const DataTypes & tuple_types = type_tuple->getElements(); diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 4e242c4348b..471d6fc575c 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -518,6 +518,115 @@ public: } }; +class FunctionMapUpdate : public IFunction +{ +public: + static constexpr auto name = "mapUpdate"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 2) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + toString(arguments.size()) + ", should be 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeMap * left = checkAndGetDataType(arguments[0].type.get()); + const DataTypeMap * right = checkAndGetDataType(arguments[1].type.get()); + + if (!left || !right) + throw Exception{"The two arguments for function " + getName() + " must be both Map type", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + if (!left->getKeyType()->equals(*right->getKeyType()) || !left->getValueType()->equals(*right->getValueType())) + throw Exception{"The Key And Value type of Map for function " + getName() + " must be the same", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + return std::make_shared(left->getKeyType(), left->getValueType()); + } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const ColumnMap * col_map_left = typeid_cast(arguments[0].column.get()); + const auto * col_const_map_left = checkAndGetColumnConst(arguments[0].column.get()); + if (col_const_map_left) + col_map_left = typeid_cast(&col_const_map_left->getDataColumn()); + if (!col_map_left) + return nullptr; + + const ColumnMap * col_map_right = typeid_cast(arguments[1].column.get()); + const auto * col_const_map_right = checkAndGetColumnConst(arguments[1].column.get()); + if (col_const_map_right) + col_map_right = typeid_cast(&col_const_map_right->getDataColumn()); + if (!col_map_right) + return nullptr; + + const auto & nested_column_left = col_map_left->getNestedColumn(); + const auto & keys_data_left = col_map_left->getNestedData().getColumn(0); + const auto & values_data_left = col_map_left->getNestedData().getColumn(1); + const auto & offsets_left = nested_column_left.getOffsets(); + + const auto & nested_column_right = col_map_right->getNestedColumn(); + const auto & keys_data_right = col_map_right->getNestedData().getColumn(0); + const auto & values_data_right = col_map_right->getNestedData().getColumn(1); + const auto & offsets_right = nested_column_right.getOffsets(); + + const auto & result_type_map = static_cast(*result_type); + const DataTypePtr & key_type = result_type_map.getKeyType(); + const DataTypePtr & value_type = result_type_map.getValueType(); + MutableColumnPtr keys_data = key_type->createColumn(); + MutableColumnPtr values_data = value_type->createColumn(); + MutableColumnPtr offsets = DataTypeNumber().createColumn(); + + IColumn::Offset current_offset = 0; + for (size_t idx = 0; idx < input_rows_count; ++idx) + { + for (size_t i = offsets_left[idx - 1]; i < offsets_left[idx]; ++i) + { + bool matched = false; + auto key = keys_data_left.getDataAt(i); + for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j) + { + if (keys_data_right.getDataAt(j).toString() == key.toString()) + { + matched = true; + break; + } + } + if (!matched) + { + keys_data->insertFrom(keys_data_left, i); + values_data->insertFrom(values_data_left, i); + ++current_offset; + } + } + for (size_t j = offsets_right[idx - 1]; j < offsets_right[idx]; ++j) + { + keys_data->insertFrom(keys_data_right, j); + values_data->insertFrom(values_data_right, j); + ++current_offset; + } + offsets->insert(current_offset); + } + + auto nested_column = ColumnArray::create( + ColumnTuple::create(Columns{std::move(keys_data), std::move(values_data)}), + std::move(offsets)); + + return ColumnMap::create(nested_column); + } +}; + } void registerFunctionsMap(FunctionFactory & factory) @@ -528,6 +637,7 @@ void registerFunctionsMap(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/mapFilter.cpp b/src/Functions/mapFilter.cpp new file mode 100644 index 00000000000..f38f8f8b4d1 --- /dev/null +++ b/src/Functions/mapFilter.cpp @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +/** Higher-order functions for map. + * These functions optionally apply a map by lambda function, + * and return some result based on that transformation. + */ + + +/** mapFilter((k, v) -> predicate, map) - leave in the map only the kv elements for which the expression is true. + */ +struct MapFilterImpl +{ + using data_type = DataTypeMap; + using column_type = ColumnMap; + + static constexpr auto name = "mapFilter"; + + static bool needBoolean() { return true; } + static bool needExpression() { return true; } + static bool needOneArray() { return true; } + + static DataTypePtr getReturnType(const DataTypePtr & /*expression_return*/, const DataTypes & elems) + { + return std::make_shared(elems); + } + + /// If there are several arrays, the first one is passed here. + static ColumnPtr execute(const ColumnMap & map_column, ColumnPtr mapped) + { + const ColumnUInt8 * column_filter = typeid_cast(&*mapped); + + if (!column_filter) + { + const auto * column_filter_const = checkAndGetColumnConst(&*mapped); + + if (!column_filter_const) + throw Exception("Unexpected type of filter column", ErrorCodes::ILLEGAL_COLUMN); + + if (column_filter_const->getValue()) + return map_column.clone(); + else + { + const auto * column_array = typeid_cast(map_column.getNestedColumnPtr().get()); + const auto * column_tuple = typeid_cast(column_array->getDataPtr().get()); + ColumnPtr keys = column_tuple->getColumnPtr(0)->cloneEmpty(); + ColumnPtr values = column_tuple->getColumnPtr(1)->cloneEmpty(); + return ColumnMap::create(keys, values, ColumnArray::ColumnOffsets::create(map_column.size(), 0)); + } + } + + const IColumn::Filter & filter = column_filter->getData(); + ColumnPtr filtered = map_column.getNestedColumn().getData().filter(filter, -1); + + const IColumn::Offsets & in_offsets = map_column.getNestedColumn().getOffsets(); + auto column_offsets = ColumnArray::ColumnOffsets::create(in_offsets.size()); + IColumn::Offsets & out_offsets = column_offsets->getData(); + + size_t in_pos = 0; + size_t out_pos = 0; + for (size_t i = 0; i < in_offsets.size(); ++i) + { + for (; in_pos < in_offsets[i]; ++in_pos) + { + if (filter[in_pos]) + ++out_pos; + } + out_offsets[i] = out_pos; + } + + return ColumnMap::create(ColumnArray::create(filtered, std::move(column_offsets))); + } +}; + + +/** mapApply((k,v) -> expression, map) - apply the expression to the map. + */ +struct MapApplyImpl +{ + using data_type = DataTypeMap; + using column_type = ColumnMap; + + static constexpr auto name = "mapApply"; + + /// true if the expression (for an overload of f(expression, maps)) or a map (for f(map)) should be boolean. + static bool needBoolean() { return false; } + static bool needExpression() { return true; } + static bool needOneArray() { return true; } + + static DataTypePtr getReturnType(const DataTypePtr & expression_return, const DataTypes & /*elems*/) + { + const auto * tuple_types = typeid_cast(expression_return.get()); + if (!tuple_types) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Expected return type is tuple, got {}", expression_return->getName()); + if (tuple_types->getElements().size() != 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Expected 2 columns as map's key and value, but found {}", tuple_types->getElements().size()); + + return std::make_shared(tuple_types->getElements()); + } + + static ColumnPtr execute(const ColumnMap & map, ColumnPtr mapped) + { + const auto * column_tuple = checkAndGetColumn(mapped.get()); + if (!column_tuple) + { + const ColumnConst * column_const_tuple = checkAndGetColumnConst(mapped.get()); + if (!column_const_tuple) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Expected tuple column, found {}", mapped->getName()); + auto cols = convertConstTupleToConstantElements(*column_const_tuple); + return ColumnMap::create(cols[0]->convertToFullColumnIfConst(), cols[1]->convertToFullColumnIfConst(), map.getNestedColumn().getOffsetsPtr()); + } + + return ColumnMap::create(column_tuple->getColumnPtr(0), column_tuple->getColumnPtr(1), + map.getNestedColumn().getOffsetsPtr()); + } +}; + +void registerFunctionMapApply(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} + +} + + diff --git a/src/Functions/minSampleSize.cpp b/src/Functions/minSampleSize.cpp new file mode 100644 index 00000000000..02a94c743e8 --- /dev/null +++ b/src/Functions/minSampleSize.cpp @@ -0,0 +1,291 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionMinSampleSize : public IFunction +{ +public: + static constexpr auto name = Impl::name; + + static FunctionPtr create(ContextPtr) { return std::make_shared>(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return Impl::num_args; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override + { + return ColumnNumbers(std::begin(Impl::const_args), std::end(Impl::const_args)); + } + + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + static DataTypePtr getReturnType() + { + auto float_64_type = std::make_shared>(); + + DataTypes types{ + float_64_type, + float_64_type, + float_64_type, + }; + + Strings names{ + "minimum_sample_size", + "detect_range_lower", + "detect_range_upper", + }; + + return std::make_shared(std::move(types), std::move(names)); + } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + Impl::validateArguments(arguments); + return getReturnType(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + return Impl::execute(arguments, input_rows_count); + } +}; + +static bool isBetweenZeroAndOne(Float64 v) +{ + return v >= 0.0 && v <= 1.0 && fabs(v - 0.0) >= DBL_EPSILON && fabs(v - 1.0) >= DBL_EPSILON; +} + +struct ContinousImpl +{ + static constexpr auto name = "minSampleSizeContinous"; + static constexpr size_t num_args = 5; + static constexpr size_t const_args[] = {2, 3, 4}; + + static void validateArguments(const DataTypes & arguments) + { + for (size_t i = 0; i < arguments.size(); ++i) + { + if (!isNativeNumber(arguments[i])) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The {}th Argument of function {} must be a number.", i + 1, name); + } + } + } + + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) + { + auto float_64_type = std::make_shared(); + auto baseline_argument = arguments[0]; + baseline_argument.column = baseline_argument.column->convertToFullColumnIfConst(); + auto baseline_column_untyped = castColumnAccurate(baseline_argument, float_64_type); + const auto * baseline_column = checkAndGetColumn>(*baseline_column_untyped); + const auto & baseline_column_data = baseline_column->getData(); + + auto sigma_argument = arguments[1]; + sigma_argument.column = sigma_argument.column->convertToFullColumnIfConst(); + auto sigma_column_untyped = castColumnAccurate(sigma_argument, float_64_type); + const auto * sigma_column = checkAndGetColumn>(*sigma_column_untyped); + const auto & sigma_column_data = sigma_column->getData(); + + const IColumn & col_mde = *arguments[2].column; + const IColumn & col_power = *arguments[3].column; + const IColumn & col_alpha = *arguments[4].column; + + auto res_min_sample_size = ColumnFloat64::create(); + auto & data_min_sample_size = res_min_sample_size->getData(); + data_min_sample_size.reserve(input_rows_count); + + auto res_detect_lower = ColumnFloat64::create(); + auto & data_detect_lower = res_detect_lower->getData(); + data_detect_lower.reserve(input_rows_count); + + auto res_detect_upper = ColumnFloat64::create(); + auto & data_detect_upper = res_detect_upper->getData(); + data_detect_upper.reserve(input_rows_count); + + /// Minimal Detectable Effect + const Float64 mde = col_mde.getFloat64(0); + /// Sufficient statistical power to detect a treatment effect + const Float64 power = col_power.getFloat64(0); + /// Significance level + const Float64 alpha = col_alpha.getFloat64(0); + + boost::math::normal_distribution<> nd(0.0, 1.0); + + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + /// Mean of control-metric + Float64 baseline = baseline_column_data[row_num]; + /// Standard deviation of conrol-metric + Float64 sigma = sigma_column_data[row_num]; + + if (!std::isfinite(baseline) || !std::isfinite(sigma) || !isBetweenZeroAndOne(mde) || !isBetweenZeroAndOne(power) + || !isBetweenZeroAndOne(alpha)) + { + data_min_sample_size.emplace_back(std::numeric_limits::quiet_NaN()); + data_detect_lower.emplace_back(std::numeric_limits::quiet_NaN()); + data_detect_upper.emplace_back(std::numeric_limits::quiet_NaN()); + continue; + } + + Float64 delta = baseline * mde; + + using namespace boost::math; + /// https://towardsdatascience.com/required-sample-size-for-a-b-testing-6f6608dd330a + /// \frac{2\sigma^{2} * (Z_{1 - alpha /2} + Z_{power})^{2}}{\Delta^{2}} + Float64 min_sample_size + = 2 * std::pow(sigma, 2) * std::pow(quantile(nd, 1.0 - alpha / 2) + quantile(nd, power), 2) / std::pow(delta, 2); + + data_min_sample_size.emplace_back(min_sample_size); + data_detect_lower.emplace_back(baseline - delta); + data_detect_upper.emplace_back(baseline + delta); + } + + return ColumnTuple::create(Columns{std::move(res_min_sample_size), std::move(res_detect_lower), std::move(res_detect_upper)}); + } +}; + + +struct ConversionImpl +{ + static constexpr auto name = "minSampleSizeConversion"; + static constexpr size_t num_args = 4; + static constexpr size_t const_args[] = {1, 2, 3}; + + static void validateArguments(const DataTypes & arguments) + { + size_t arguments_size = arguments.size(); + for (size_t i = 0; i < arguments_size; ++i) + { + if (!isFloat(arguments[i])) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The {}th argument of function {} must be a float.", i + 1, name); + } + } + } + + static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) + { + auto first_argument_column = castColumnAccurate(arguments[0], std::make_shared()); + + if (const ColumnConst * const col_p1_const = checkAndGetColumnConst>(first_argument_column.get())) + { + const Float64 left_value = col_p1_const->template getValue(); + return process(arguments, &left_value, input_rows_count); + } + else if (const ColumnVector * const col_p1 = checkAndGetColumn>(first_argument_column.get())) + { + return process(arguments, col_p1->getData().data(), input_rows_count); + } + else + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument of function {} must be a float.", name); + } + } + + template + static ColumnPtr process(const ColumnsWithTypeAndName & arguments, const Float64 * col_p1, const size_t input_rows_count) + { + const IColumn & col_mde = *arguments[1].column; + const IColumn & col_power = *arguments[2].column; + const IColumn & col_alpha = *arguments[3].column; + + auto res_min_sample_size = ColumnFloat64::create(); + auto & data_min_sample_size = res_min_sample_size->getData(); + data_min_sample_size.reserve(input_rows_count); + + auto res_detect_lower = ColumnFloat64::create(); + auto & data_detect_lower = res_detect_lower->getData(); + data_detect_lower.reserve(input_rows_count); + + auto res_detect_upper = ColumnFloat64::create(); + auto & data_detect_upper = res_detect_upper->getData(); + data_detect_upper.reserve(input_rows_count); + + /// Minimal Detectable Effect + const Float64 mde = col_mde.getFloat64(0); + /// Sufficient statistical power to detect a treatment effect + const Float64 power = col_power.getFloat64(0); + /// Significance level + const Float64 alpha = col_alpha.getFloat64(0); + + boost::math::normal_distribution<> nd(0.0, 1.0); + + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + /// Proportion of control-metric + Float64 p1; + + if constexpr (const_p1) + { + p1 = col_p1[0]; + } + else if constexpr (!const_p1) + { + p1 = col_p1[row_num]; + } + + if (!std::isfinite(p1) || !isBetweenZeroAndOne(mde) || !isBetweenZeroAndOne(power) || !isBetweenZeroAndOne(alpha)) + { + data_min_sample_size.emplace_back(std::numeric_limits::quiet_NaN()); + data_detect_lower.emplace_back(std::numeric_limits::quiet_NaN()); + data_detect_upper.emplace_back(std::numeric_limits::quiet_NaN()); + continue; + } + + Float64 q1 = 1.0 - p1; + Float64 p2 = p1 + mde; + Float64 q2 = 1.0 - p2; + Float64 p_bar = (p1 + p2) / 2.0; + Float64 q_bar = 1.0 - p_bar; + + using namespace boost::math; + /// https://towardsdatascience.com/required-sample-size-for-a-b-testing-6f6608dd330a + /// \frac{(Z_{1-alpha/2} * \sqrt{2*\bar{p}*\bar{q}} + Z_{power} * \sqrt{p1*q1+p2*q2})^{2}}{\Delta^{2}} + Float64 min_sample_size + = std::pow( + quantile(nd, 1.0 - alpha / 2.0) * std::sqrt(2.0 * p_bar * q_bar) + quantile(nd, power) * std::sqrt(p1 * q1 + p2 * q2), + 2) + / std::pow(mde, 2); + + data_min_sample_size.emplace_back(min_sample_size); + data_detect_lower.emplace_back(p1 - mde); + data_detect_upper.emplace_back(p1 + mde); + } + + return ColumnTuple::create(Columns{std::move(res_min_sample_size), std::move(res_detect_lower), std::move(res_detect_upper)}); + } +}; + + +void registerFunctionMinSampleSize(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 070a7c2f05e..7ed0ee00954 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -117,7 +117,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const override { - ColumnsWithTypeAndName arguments = std::move(args); + ColumnsWithTypeAndName arguments = args; executeShortCircuitArguments(arguments); /** We will gather values from columns in branches to result column, * depending on values of conditions. diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index a1254446e01..ab447e61aed 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -78,7 +78,7 @@ public: // check that default value column has supertype with first argument if (number_of_arguments == 3) - return getLeastSupertype({arguments[0], arguments[2]}); + return getLeastSupertype(DataTypes{arguments[0], arguments[2]}); return arguments[0]; } diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp index 2fe6a1159af..55eee90f136 100644 --- a/src/Functions/normalizeString.cpp +++ b/src/Functions/normalizeString.cpp @@ -98,8 +98,6 @@ struct NormalizeUTF8Impl ColumnString::Offset current_from_offset = 0; ColumnString::Offset current_to_offset = 0; - icu::UnicodeString to_string; - PODArray from_uchars; PODArray to_uchars; diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index bd1038b1fc6..e7d9011db53 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -152,7 +152,7 @@ public: for (const auto & arg : arguments) arg_types.push_back(arg.type); - return std::make_unique(nowSubsecond(scale), std::move(arg_types), std::move(result_type)); + return std::make_unique(nowSubsecond(scale), std::move(arg_types), result_type); } }; diff --git a/src/Functions/nullIf.cpp b/src/Functions/nullIf.cpp index c54bbc08bcd..0b4d024c91c 100644 --- a/src/Functions/nullIf.cpp +++ b/src/Functions/nullIf.cpp @@ -61,7 +61,7 @@ public: auto func_if = FunctionFactory::instance().get("if", context)->build(if_columns); auto if_res = func_if->execute(if_columns, result_type, input_rows_count); - return makeNullable(std::move(if_res)); + return makeNullable(if_res); } }; diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 65e332783d8..82f72d7a1fa 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -56,6 +56,7 @@ void registerFunctionTid(FunctionFactory & factory); void registerFunctionLogTrace(FunctionFactory & factory); void registerFunctionsTimeWindow(FunctionFactory &); void registerFunctionToBool(FunctionFactory &); +void registerFunctionMinSampleSize(FunctionFactory &); #if USE_SSL void registerFunctionEncrypt(FunctionFactory & factory); @@ -118,6 +119,7 @@ void registerFunctions() registerFunctionsSnowflake(factory); registerFunctionsTimeWindow(factory); registerFunctionToBool(factory); + registerFunctionMinSampleSize(factory); #if USE_SSL registerFunctionEncrypt(factory); diff --git a/src/Functions/registerFunctionsArithmetic.cpp b/src/Functions/registerFunctionsArithmetic.cpp index d3d82ca0dd8..96e77d34882 100644 --- a/src/Functions/registerFunctionsArithmetic.cpp +++ b/src/Functions/registerFunctionsArithmetic.cpp @@ -41,6 +41,7 @@ void registerFunctionBitBoolMaskOr(FunctionFactory & factory); void registerFunctionBitBoolMaskAnd(FunctionFactory & factory); void registerFunctionBitWrapperFunc(FunctionFactory & factory); void registerFunctionBitSwapLastTwo(FunctionFactory & factory); +void registerFunctionZTest(FunctionFactory & factory); void registerFunctionsArithmetic(FunctionFactory & factory) @@ -84,6 +85,8 @@ void registerFunctionsArithmetic(FunctionFactory & factory) registerFunctionBitBoolMaskAnd(factory); registerFunctionBitWrapperFunc(factory); registerFunctionBitSwapLastTwo(factory); + + registerFunctionZTest(factory); } } diff --git a/src/Functions/registerFunctionsDateTime.cpp b/src/Functions/registerFunctionsDateTime.cpp index 5211a62ff1e..dd7b67c47ac 100644 --- a/src/Functions/registerFunctionsDateTime.cpp +++ b/src/Functions/registerFunctionsDateTime.cpp @@ -11,6 +11,9 @@ void registerFunctionToDayOfWeek(FunctionFactory &); void registerFunctionToDayOfYear(FunctionFactory &); void registerFunctionToHour(FunctionFactory &); void registerFunctionToMinute(FunctionFactory &); +void registerFunctionToStartOfNanosecond(FunctionFactory &); +void registerFunctionToStartOfMicrosecond(FunctionFactory &); +void registerFunctionToStartOfMillisecond(FunctionFactory &); void registerFunctionToStartOfSecond(FunctionFactory &); void registerFunctionToSecond(FunctionFactory &); void registerFunctionToStartOfDay(FunctionFactory &); @@ -47,6 +50,9 @@ void registerFunctionTimeSlots(FunctionFactory &); void registerFunctionToYYYYMM(FunctionFactory &); void registerFunctionToYYYYMMDD(FunctionFactory &); void registerFunctionToYYYYMMDDhhmmss(FunctionFactory &); +void registerFunctionAddNanoseconds(FunctionFactory &); +void registerFunctionAddMicroseconds(FunctionFactory &); +void registerFunctionAddMilliseconds(FunctionFactory &); void registerFunctionAddSeconds(FunctionFactory &); void registerFunctionAddMinutes(FunctionFactory &); void registerFunctionAddHours(FunctionFactory &); @@ -55,6 +61,9 @@ void registerFunctionAddWeeks(FunctionFactory &); void registerFunctionAddMonths(FunctionFactory &); void registerFunctionAddQuarters(FunctionFactory &); void registerFunctionAddYears(FunctionFactory &); +void registerFunctionSubtractNanoseconds(FunctionFactory &); +void registerFunctionSubtractMicroseconds(FunctionFactory &); +void registerFunctionSubtractMilliseconds(FunctionFactory &); void registerFunctionSubtractSeconds(FunctionFactory &); void registerFunctionSubtractMinutes(FunctionFactory &); void registerFunctionSubtractHours(FunctionFactory &); @@ -93,6 +102,9 @@ void registerFunctionsDateTime(FunctionFactory & factory) registerFunctionToStartOfMonth(factory); registerFunctionToStartOfQuarter(factory); registerFunctionToStartOfYear(factory); + registerFunctionToStartOfNanosecond(factory); + registerFunctionToStartOfMicrosecond(factory); + registerFunctionToStartOfMillisecond(factory); registerFunctionToStartOfSecond(factory); registerFunctionToStartOfMinute(factory); registerFunctionToStartOfFiveMinute(factory); @@ -119,6 +131,9 @@ void registerFunctionsDateTime(FunctionFactory & factory) registerFunctionToYYYYMM(factory); registerFunctionToYYYYMMDD(factory); registerFunctionToYYYYMMDDhhmmss(factory); + registerFunctionAddNanoseconds(factory); + registerFunctionAddMicroseconds(factory); + registerFunctionAddMilliseconds(factory); registerFunctionAddSeconds(factory); registerFunctionAddMinutes(factory); registerFunctionAddHours(factory); @@ -127,6 +142,9 @@ void registerFunctionsDateTime(FunctionFactory & factory) registerFunctionAddMonths(factory); registerFunctionAddQuarters(factory); registerFunctionAddYears(factory); + registerFunctionSubtractNanoseconds(factory); + registerFunctionSubtractMicroseconds(factory); + registerFunctionSubtractMilliseconds(factory); registerFunctionSubtractSeconds(factory); registerFunctionSubtractMinutes(factory); registerFunctionSubtractHours(factory); diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 0501b603c57..9cbe1ed96cf 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -52,6 +52,11 @@ void registerFunctionH3HexAreaKm2(FunctionFactory &); void registerFunctionH3CellAreaM2(FunctionFactory &); void registerFunctionH3CellAreaRads2(FunctionFactory &); void registerFunctionH3NumHexagons(FunctionFactory &); +void registerFunctionH3PointDistM(FunctionFactory &); +void registerFunctionH3PointDistKm(FunctionFactory &); +void registerFunctionH3PointDistRads(FunctionFactory &); +void registerFunctionH3GetRes0Indexes(FunctionFactory &); +void registerFunctionH3GetPentagonIndexes(FunctionFactory &); #endif @@ -118,6 +123,11 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3CellAreaM2(factory); registerFunctionH3CellAreaRads2(factory); registerFunctionH3NumHexagons(factory); + registerFunctionH3PointDistM(factory); + registerFunctionH3PointDistKm(factory); + registerFunctionH3PointDistRads(factory); + registerFunctionH3GetRes0Indexes(factory); + registerFunctionH3GetPentagonIndexes(factory); #endif #if USE_S2_GEOMETRY diff --git a/src/Functions/registerFunctionsHigherOrder.cpp b/src/Functions/registerFunctionsHigherOrder.cpp index d3621a03ecd..00bea58b918 100644 --- a/src/Functions/registerFunctionsHigherOrder.cpp +++ b/src/Functions/registerFunctionsHigherOrder.cpp @@ -18,6 +18,7 @@ void registerFunctionsArraySort(FunctionFactory & factory); void registerFunctionArrayCumSum(FunctionFactory & factory); void registerFunctionArrayCumSumNonNegative(FunctionFactory & factory); void registerFunctionArrayDifference(FunctionFactory & factory); +void registerFunctionMapApply(FunctionFactory & factory); void registerFunctionsHigherOrder(FunctionFactory & factory) { @@ -36,6 +37,7 @@ void registerFunctionsHigherOrder(FunctionFactory & factory) registerFunctionArrayCumSum(factory); registerFunctionArrayCumSumNonNegative(factory); registerFunctionArrayDifference(factory); + registerFunctionMapApply(factory); } } diff --git a/src/Functions/roundToExp2.cpp b/src/Functions/roundToExp2.cpp index 37f0637c79a..846890bc5c8 100644 --- a/src/Functions/roundToExp2.cpp +++ b/src/Functions/roundToExp2.cpp @@ -14,36 +14,36 @@ namespace { template -inline std::enable_if_t && (sizeof(T) <= sizeof(UInt32)), T> -roundDownToPowerOfTwo(T x) +requires std::is_integral_v && (sizeof(T) <= sizeof(UInt32)) +inline T roundDownToPowerOfTwo(T x) { return x <= 0 ? 0 : (T(1) << (31 - __builtin_clz(x))); } template -inline std::enable_if_t && (sizeof(T) == sizeof(UInt64)), T> -roundDownToPowerOfTwo(T x) +requires std::is_integral_v && (sizeof(T) == sizeof(UInt64)) +inline T roundDownToPowerOfTwo(T x) { return x <= 0 ? 0 : (T(1) << (63 - __builtin_clzll(x))); } template -inline std::enable_if_t, T> -roundDownToPowerOfTwo(T x) +requires std::is_same_v +inline T roundDownToPowerOfTwo(T x) { return bit_cast(bit_cast(x) & ~((1ULL << 23) - 1)); } template -inline std::enable_if_t, T> -roundDownToPowerOfTwo(T x) +requires std::is_same_v +inline T roundDownToPowerOfTwo(T x) { return bit_cast(bit_cast(x) & ~((1ULL << 52) - 1)); } template -inline std::enable_if_t, T> -roundDownToPowerOfTwo(T) +requires is_big_int_v +inline T roundDownToPowerOfTwo(T) { throw Exception("roundToExp2() for big integers is not implemented", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Functions/s2CapContains.cpp b/src/Functions/s2CapContains.cpp index 6604ff9707e..482c4a22c63 100644 --- a/src/Functions/s2CapContains.cpp +++ b/src/Functions/s2CapContains.cpp @@ -84,7 +84,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_center = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_center = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_center) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -94,7 +98,7 @@ public: getName()); const auto & data_center = col_center->getData(); - const auto * col_degrees = checkAndGetColumn(arguments[1].column.get()); + const auto * col_degrees = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_degrees) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -105,7 +109,7 @@ public: const auto & data_degrees = col_degrees->getData(); - const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + const auto * col_point = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_point) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2CapUnion.cpp b/src/Functions/s2CapUnion.cpp index 7af6324a7d5..ea1f2f7534d 100644 --- a/src/Functions/s2CapUnion.cpp +++ b/src/Functions/s2CapUnion.cpp @@ -82,7 +82,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_center1 = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_center1 = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_center1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -92,7 +96,7 @@ public: getName()); const auto & data_center1 = col_center1->getData(); - const auto * col_radius1 = checkAndGetColumn(arguments[1].column.get()); + const auto * col_radius1 = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_radius1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -102,7 +106,7 @@ public: getName()); const auto & data_radius1 = col_radius1->getData(); - const auto * col_center2 = checkAndGetColumn(arguments[2].column.get()); + const auto * col_center2 = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_center2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -112,7 +116,7 @@ public: getName()); const auto & data_center2 = col_center2->getData(); - const auto * col_radius2 = checkAndGetColumn(arguments[3].column.get()); + const auto * col_radius2 = checkAndGetColumn(non_const_arguments[3].column.get()); if (!col_radius2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2CellsIntersect.cpp b/src/Functions/s2CellsIntersect.cpp index 5d9796ea26c..617910e1a59 100644 --- a/src/Functions/s2CellsIntersect.cpp +++ b/src/Functions/s2CellsIntersect.cpp @@ -66,7 +66,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id_first = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_id_first = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_id_first) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -76,7 +80,7 @@ public: getName()); const auto & data_id_first = col_id_first->getData(); - const auto * col_id_second = checkAndGetColumn(arguments[1].column.get()); + const auto * col_id_second = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_id_second) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2GetNeighbors.cpp b/src/Functions/s2GetNeighbors.cpp index 32eaff90740..f16e531a645 100644 --- a/src/Functions/s2GetNeighbors.cpp +++ b/src/Functions/s2GetNeighbors.cpp @@ -67,7 +67,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_id = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_id) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectAdd.cpp b/src/Functions/s2RectAdd.cpp index 75da7de8f7e..44f240588be 100644 --- a/src/Functions/s2RectAdd.cpp +++ b/src/Functions/s2RectAdd.cpp @@ -65,7 +65,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -75,7 +79,7 @@ public: getName()); const auto & data_low = col_lo->getData(); - const auto * col_hi = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -85,7 +89,7 @@ public: getName()); const auto & data_hi = col_hi->getData(); - const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + const auto * col_point = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_point) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectContains.cpp b/src/Functions/s2RectContains.cpp index be46253f70e..84218704ef1 100644 --- a/src/Functions/s2RectContains.cpp +++ b/src/Functions/s2RectContains.cpp @@ -63,7 +63,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -73,7 +77,7 @@ public: getName()); const auto & data_low = col_lo->getData(); - const auto * col_hi = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -83,7 +87,7 @@ public: getName()); const auto & data_hi = col_hi->getData(); - const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + const auto * col_point = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_point) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectIntersection.cpp b/src/Functions/s2RectIntersection.cpp index d4339b4d601..064dad4166d 100644 --- a/src/Functions/s2RectIntersection.cpp +++ b/src/Functions/s2RectIntersection.cpp @@ -68,7 +68,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo1 = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo1 = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -78,7 +82,7 @@ public: getName()); const auto & data_lo1 = col_lo1->getData(); - const auto * col_hi1 = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi1 = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -88,7 +92,7 @@ public: getName()); const auto & data_hi1 = col_hi1->getData(); - const auto * col_lo2 = checkAndGetColumn(arguments[2].column.get()); + const auto * col_lo2 = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_lo2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -98,7 +102,7 @@ public: getName()); const auto & data_lo2 = col_lo2->getData(); - const auto * col_hi2 = checkAndGetColumn(arguments[3].column.get()); + const auto * col_hi2 = checkAndGetColumn(non_const_arguments[3].column.get()); if (!col_hi2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2RectUnion.cpp b/src/Functions/s2RectUnion.cpp index 047d331e711..91664d3b65f 100644 --- a/src/Functions/s2RectUnion.cpp +++ b/src/Functions/s2RectUnion.cpp @@ -66,7 +66,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo1 = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_lo1 = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_lo1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -76,7 +80,7 @@ public: getName()); const auto & data_lo1 = col_lo1->getData(); - const auto * col_hi1 = checkAndGetColumn(arguments[1].column.get()); + const auto * col_hi1 = checkAndGetColumn(non_const_arguments[1].column.get()); if (!col_hi1) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -86,7 +90,7 @@ public: getName()); const auto & data_hi1 = col_hi1->getData(); - const auto * col_lo2 = checkAndGetColumn(arguments[2].column.get()); + const auto * col_lo2 = checkAndGetColumn(non_const_arguments[2].column.get()); if (!col_lo2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, @@ -96,7 +100,7 @@ public: getName()); const auto & data_lo2 = col_lo2->getData(); - const auto * col_hi2 = checkAndGetColumn(arguments[3].column.get()); + const auto * col_hi2 = checkAndGetColumn(non_const_arguments[3].column.get()); if (!col_hi2) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/s2ToGeo.cpp b/src/Functions/s2ToGeo.cpp index 3d11c21a353..082c300536d 100644 --- a/src/Functions/s2ToGeo.cpp +++ b/src/Functions/s2ToGeo.cpp @@ -68,7 +68,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id = checkAndGetColumn(arguments[0].column.get()); + auto non_const_arguments = arguments; + for (auto & argument : non_const_arguments) + argument.column = argument.column->convertToFullColumnIfConst(); + + const auto * col_id = checkAndGetColumn(non_const_arguments[0].column.get()); if (!col_id) throw Exception( ErrorCodes::ILLEGAL_COLUMN, diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index 7533e30c9b9..d2c1e7d2d55 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -131,8 +131,10 @@ public: message.value_or("Value passed to '" + getName() + "' function is non zero")); } + size_t result_size = in_untyped->size(); + /// We return non constant to avoid constant folding. - return ColumnUInt8::create(in_data.size(), 0); + return ColumnUInt8::create(result_size, 0); } return nullptr; diff --git a/src/Functions/timezoneOf.cpp b/src/Functions/timezoneOf.cpp index 03c9e27a3a8..97e025bc0e0 100644 --- a/src/Functions/timezoneOf.cpp +++ b/src/Functions/timezoneOf.cpp @@ -21,7 +21,7 @@ namespace /** timezoneOf(x) - get the name of the timezone of DateTime data type. - * Example: Europe/Moscow. + * Example: Pacific/Pitcairn. */ class FunctionTimezoneOf : public IFunction { @@ -74,4 +74,3 @@ void registerFunctionTimezoneOf(FunctionFactory & factory) } } - diff --git a/src/Functions/toFixedString.h b/src/Functions/toFixedString.h index 129e3e0e8b2..cbd29784271 100644 --- a/src/Functions/toFixedString.h +++ b/src/Functions/toFixedString.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 09b7931de8d..bff33f9b061 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -33,184 +33,273 @@ namespace template <> struct Transform { - static constexpr auto name = function_name; - - static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone) + static UInt16 execute(UInt16 d, Int64 years, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfYearInterval(DayNum(d), years); } - static UInt16 execute(Int32 d, UInt64 years, const DateLUTImpl & time_zone) + static UInt16 execute(Int32 d, Int64 years, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); } - static UInt16 execute(UInt32 t, UInt64 years, const DateLUTImpl & time_zone) + static UInt16 execute(UInt32 t, Int64 years, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); } - static UInt16 execute(Int64 t, UInt64 years, const DateLUTImpl & time_zone) + static UInt16 execute(Int64 t, Int64 years, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t / scale_multiplier), years); } }; template <> struct Transform { - static constexpr auto name = function_name; - - static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone) + static UInt16 execute(UInt16 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); } - static UInt16 execute(Int32 d, UInt64 quarters, const DateLUTImpl & time_zone) + static UInt16 execute(Int32 d, Int64 quarters, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); } - static UInt16 execute(UInt32 t, UInt64 quarters, const DateLUTImpl & time_zone) + static UInt16 execute(UInt32 t, Int64 quarters, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); } - static UInt16 execute(Int64 t, UInt64 quarters, const DateLUTImpl & time_zone) + static UInt16 execute(Int64 t, Int64 quarters, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t / scale_multiplier), quarters); } }; template <> struct Transform { - static constexpr auto name = function_name; - - static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone) + static UInt16 execute(UInt16 d, Int64 months, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfMonthInterval(DayNum(d), months); } - static UInt16 execute(Int32 d, UInt64 months, const DateLUTImpl & time_zone) + static UInt16 execute(Int32 d, Int64 months, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); } - static UInt16 execute(UInt32 t, UInt64 months, const DateLUTImpl & time_zone) + static UInt16 execute(UInt32 t, Int64 months, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); } - static UInt16 execute(Int64 t, UInt64 months, const DateLUTImpl & time_zone) + static UInt16 execute(Int64 t, Int64 months, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t / scale_multiplier), months); } }; template <> struct Transform { - static constexpr auto name = function_name; - - static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone) + static UInt16 execute(UInt16 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfWeekInterval(DayNum(d), weeks); } - static UInt16 execute(Int32 d, UInt64 weeks, const DateLUTImpl & time_zone) + static UInt16 execute(Int32 d, Int64 weeks, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); } - static UInt16 execute(UInt32 t, UInt64 weeks, const DateLUTImpl & time_zone) + static UInt16 execute(UInt32 t, Int64 weeks, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); } - static UInt16 execute(Int64 t, UInt64 weeks, const DateLUTImpl & time_zone) + static UInt16 execute(Int64 t, Int64 weeks, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t / scale_multiplier), weeks); } }; template <> struct Transform { - static constexpr auto name = function_name; - - static UInt32 execute(UInt16 d, UInt64 days, const DateLUTImpl & time_zone) + static UInt32 execute(UInt16 d, Int64 days, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfDayInterval(ExtendedDayNum(d), days); } - static UInt32 execute(Int32 d, UInt64 days, const DateLUTImpl & time_zone) + static UInt32 execute(Int32 d, Int64 days, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfDayInterval(ExtendedDayNum(d), days); } - static UInt32 execute(UInt32 t, UInt64 days, const DateLUTImpl & time_zone) + static UInt32 execute(UInt32 t, Int64 days, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); } - static UInt32 execute(Int64 t, UInt64 days, const DateLUTImpl & time_zone) + static Int64 execute(Int64 t, Int64 days, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); + return time_zone.toStartOfDayInterval(time_zone.toDayNum(t / scale_multiplier), days); } }; template <> struct Transform { - static constexpr auto name = function_name; + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - static UInt32 execute(Int32, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt32 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } - static UInt32 execute(Int64 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(UInt32 t, Int64 hours, const DateLUTImpl & time_zone, Int64) + { + return time_zone.toStartOfHourInterval(t, hours); + } + + static UInt32 execute(Int64 t, Int64 hours, const DateLUTImpl & time_zone, Int64 scale_multiplier) + { + return time_zone.toStartOfHourInterval(t / scale_multiplier, hours); + } }; template <> struct Transform { - static constexpr auto name = function_name; + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } - static UInt32 execute(Int32, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, UInt64 minutes, const DateLUTImpl & time_zone) + static UInt32 execute(UInt32 t, Int64 minutes, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfMinuteInterval(t, minutes); } - static UInt32 execute(Int64 t, UInt64 minutes, const DateLUTImpl & time_zone) + static UInt32 execute(Int64 t, Int64 minutes, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfMinuteInterval(t, minutes); + return time_zone.toStartOfMinuteInterval(t / scale_multiplier, minutes); } }; template <> struct Transform { - static constexpr auto name = function_name; + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } - static UInt32 execute(Int32, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - - static UInt32 execute(UInt32 t, UInt64 seconds, const DateLUTImpl & time_zone) + static UInt32 execute(UInt32 t, Int64 seconds, const DateLUTImpl & time_zone, Int64) { return time_zone.toStartOfSecondInterval(t, seconds); } - static Int64 execute(Int64 t, UInt64 seconds, const DateLUTImpl & time_zone) + static UInt32 execute(Int64 t, Int64 seconds, const DateLUTImpl & time_zone, Int64 scale_multiplier) { - return time_zone.toStartOfSecondInterval(t, seconds); + return time_zone.toStartOfSecondInterval(t / scale_multiplier, seconds); } }; + template <> + struct Transform + { + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { return dateTimeIsNotSupported(function_name); } + + static Int64 execute(Int64 t, Int64 milliseconds, const DateLUTImpl &, Int64 scale_multiplier) + { + if (scale_multiplier < 1000) + { + Int64 t_milliseconds = t * (static_cast(1000) / scale_multiplier); + if (likely(t >= 0)) + return t_milliseconds / milliseconds * milliseconds; + else + return ((t_milliseconds + 1) / milliseconds - 1) * milliseconds; + } + else if (scale_multiplier > 1000) + { + Int64 scale_diff = scale_multiplier / static_cast(1000); + if (likely(t >= 0)) + return t / milliseconds / scale_diff * milliseconds; + else + return ((t + 1) / milliseconds / scale_diff - 1) * milliseconds; + } + else + if (likely(t >= 0)) + return t / milliseconds * milliseconds; + else + return ((t + 1) / milliseconds - 1) * milliseconds; + } + }; + + template <> + struct Transform + { + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { return dateTimeIsNotSupported(function_name); } + + static Int64 execute(Int64 t, Int64 microseconds, const DateLUTImpl &, Int64 scale_multiplier) + { + if (scale_multiplier < 1000000) + { + Int64 t_microseconds = t * (static_cast(1000000) / scale_multiplier); + if (likely(t >= 0)) + return t_microseconds / microseconds * microseconds; + else + return ((t_microseconds + 1) / microseconds - 1) * microseconds; + } + else if (scale_multiplier > 1000000) + { + Int64 scale_diff = scale_multiplier / static_cast(1000000); + if (likely(t >= 0)) + return t / microseconds / scale_diff * microseconds; + else + return ((t + 1) / microseconds / scale_diff - 1) * microseconds; + } + else + if (likely(t >= 0)) + return t / microseconds * microseconds; + else + return ((t + 1) / microseconds - 1) * microseconds; + } + }; + + template <> + struct Transform + { + static UInt32 execute(UInt16, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(Int32, Int64, const DateLUTImpl &, Int64) { return dateIsNotSupported(function_name); } + + static UInt32 execute(UInt32, Int64, const DateLUTImpl &, Int64) { return dateTimeIsNotSupported(function_name); } + + static Int64 execute(Int64 t, Int64 nanoseconds, const DateLUTImpl &, Int64 scale_multiplier) + { + if (scale_multiplier < 1000000000) + { + Int64 t_nanoseconds = t * (static_cast(1000000000) / scale_multiplier); + if (likely(t >= 0)) + return t_nanoseconds / nanoseconds * nanoseconds; + else + return ((t_nanoseconds + 1) / nanoseconds - 1) * nanoseconds; + } + else + if (likely(t >= 0)) + return t / nanoseconds * nanoseconds; + else + return ((t + 1) / nanoseconds - 1) * nanoseconds; + } + }; class FunctionToStartOfInterval : public IFunction { @@ -240,6 +329,7 @@ public: const DataTypeInterval * interval_type = nullptr; bool result_type_is_date = false; + bool result_type_is_datetime = false; auto check_interval_argument = [&] { interval_type = checkAndGetDataType(arguments[1].type.get()); @@ -251,6 +341,8 @@ public: result_type_is_date = (interval_type->getKind() == IntervalKind::Year) || (interval_type->getKind() == IntervalKind::Quarter) || (interval_type->getKind() == IntervalKind::Month) || (interval_type->getKind() == IntervalKind::Week); + result_type_is_datetime = (interval_type->getKind() == IntervalKind::Day) || (interval_type->getKind() == IntervalKind::Hour) + || (interval_type->getKind() == IntervalKind::Minute) || (interval_type->getKind() == IntervalKind::Second); }; auto check_timezone_argument = [&] @@ -263,7 +355,7 @@ public: if (first_argument_is_date && result_type_is_date) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The timezone argument of function {} with interval type {} is allowed only when the 1st argument " - "has the type DateTime", + "has the type DateTime or DateTime64", getName(), interval_type->getKind().toString()); }; @@ -288,19 +380,33 @@ public: if (result_type_is_date) return std::make_shared(); - else + else if (result_type_is_datetime) return std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); + else + { + auto scale = 0; + + if (interval_type->getKind() == IntervalKind::Nanosecond) + scale = 9; + else if (interval_type->getKind() == IntervalKind::Microsecond) + scale = 6; + else if (interval_type->getKind() == IntervalKind::Millisecond) + scale = 3; + + return std::make_shared(scale, extractTimeZoneNameFromFunctionArguments(arguments, 2, 0)); + } + } bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /* input_rows_count */) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override { const auto & time_column = arguments[0]; const auto & interval_column = arguments[1]; const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); - auto result_column = dispatchForColumns(time_column, interval_column, time_zone); + auto result_column = dispatchForColumns(time_column, interval_column, result_type, time_zone); return result_column; } @@ -316,33 +422,36 @@ public: private: ColumnPtr dispatchForColumns( - const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const DateLUTImpl & time_zone) const + const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column, const DataTypePtr & result_type, const DateLUTImpl & time_zone) const { const auto & from_datatype = *time_column.type.get(); const auto which_type = WhichDataType(from_datatype); + + if (which_type.isDateTime64()) + { + const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); + auto scale = assert_cast(from_datatype).getScale(); + + if (time_column_vec) + return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone, scale); + } if (which_type.isDateTime()) { const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, time_zone); + return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone); } if (which_type.isDate()) { const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, time_zone); + return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone); } if (which_type.isDate32()) { const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, time_zone); - } - if (which_type.isDateTime64()) - { - const auto * time_column_vec = checkAndGetColumn(time_column.column.get()); - if (time_column_vec) - return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, time_zone); + return dispatchForIntervalColumn(assert_cast(from_datatype), *time_column_vec, interval_column, result_type, time_zone); } throw Exception( "Illegal column for first argument of function " + getName() + ". Must contain dates or dates with time", @@ -351,7 +460,8 @@ private: template ColumnPtr dispatchForIntervalColumn( - const FromDataType & from, const ColumnType & time_column, const ColumnWithTypeAndName & interval_column, const DateLUTImpl & time_zone) const + const FromDataType & from, const ColumnType & time_column, const ColumnWithTypeAndName & interval_column, + const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale = 1) const { const auto * interval_type = checkAndGetDataType(interval_column.type.get()); if (!interval_type) @@ -368,49 +478,52 @@ private: switch (interval_type->getKind()) { + case IntervalKind::Nanosecond: + return execute(from, time_column, num_units, result_type, time_zone, scale); + case IntervalKind::Microsecond: + return execute(from, time_column, num_units, result_type, time_zone, scale); + case IntervalKind::Millisecond: + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Second: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Minute: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Hour: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Day: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Week: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Month: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Quarter: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); case IntervalKind::Year: - return execute(from, time_column, num_units, time_zone); + return execute(from, time_column, num_units, result_type, time_zone, scale); } __builtin_unreachable(); } - - template - ColumnPtr execute(const FromDataType & from_datatype, const ColumnType & time_column, UInt64 num_units, const DateLUTImpl & time_zone) const + template + ColumnPtr execute(const FromDataType &, const ColumnType & time_column_type, Int64 num_units, const DataTypePtr & result_type, const DateLUTImpl & time_zone, const UInt16 scale) const { - const auto & time_data = time_column.getData(); - size_t size = time_column.size(); - auto result = ColumnVector::create(); - auto & result_data = result->getData(); + using ToColumnType = typename ToDataType::ColumnType; + + const auto & time_data = time_column_type.getData(); + size_t size = time_data.size(); + + auto result_col = result_type->createColumn(); + auto *col_to = assert_cast(result_col.get()); + auto & result_data = col_to->getData(); result_data.resize(size); - if constexpr (std::is_same_v) - { - const auto transform = TransformDateTime64>{from_datatype.getScale()}; - for (size_t i = 0; i != size; ++i) - result_data[i] = transform.execute(time_data[i], num_units, time_zone); - } - else - { - for (size_t i = 0; i != size; ++i) - result_data[i] = Transform::execute(time_data[i], num_units, time_zone); - } - return result; + Int64 scale_multiplier = DecimalUtils::scaleMultiplier(scale); + + for (size_t i = 0; i != size; ++i) + result_data[i] = Transform::execute(time_data[i], num_units, time_zone, scale_multiplier); + + return result_col; } }; diff --git a/src/Functions/toStartOfSubsecond.cpp b/src/Functions/toStartOfSubsecond.cpp new file mode 100644 index 00000000000..b2257c5e3cd --- /dev/null +++ b/src/Functions/toStartOfSubsecond.cpp @@ -0,0 +1,30 @@ +#include +#include +#include + + +namespace DB +{ + +using FunctionToStartOfMillisecond = FunctionDateOrDateTimeToSomething; + +void registerFunctionToStartOfMillisecond(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +using FunctionToStartOfMicrosecond = FunctionDateOrDateTimeToSomething; + +void registerFunctionToStartOfMicrosecond(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +using FunctionToStartOfNanosecond = FunctionDateOrDateTimeToSomething; + +void registerFunctionToStartOfNanosecond(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index b7e1db59c23..de9f1a5ba05 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -117,7 +117,7 @@ public: + " has signature: transform(T, Array(T), Array(U), U) -> U; or transform(T, Array(T), Array(T)) -> T; where T and U are types.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - return getLeastSupertype({type_x, type_arr_to_nested}); + return getLeastSupertype(DataTypes{type_x, type_arr_to_nested}); } else { @@ -140,7 +140,7 @@ public: if (type_arr_to_nested->isValueRepresentedByNumber() && type_default->isValueRepresentedByNumber()) { /// We take the smallest common type for the elements of the array of values `to` and for `default`. - return getLeastSupertype({type_arr_to_nested, type_default}); + return getLeastSupertype(DataTypes{type_arr_to_nested, type_default}); } /// TODO More checks. diff --git a/src/Functions/ztest.cpp b/src/Functions/ztest.cpp new file mode 100644 index 00000000000..c80b92960e9 --- /dev/null +++ b/src/Functions/ztest.cpp @@ -0,0 +1,225 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + + +class FunctionTwoSampleProportionsZTest : public IFunction +{ +public: + static constexpr auto POOLED = "pooled"; + static constexpr auto UNPOOLED = "unpooled"; + + static constexpr auto name = "proportionsZTest"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 6; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {5}; } + + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + static DataTypePtr getReturnType() + { + auto float_data_type = std::make_shared>(); + DataTypes types(4, float_data_type); + + Strings names{"z_statistic", "p_value", "confidence_interval_low", "confidence_interval_high"}; + + return std::make_shared(std::move(types), std::move(names)); + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + for (size_t i = 0; i < 4; ++i) + { + if (!isUnsignedInteger(arguments[i].type)) + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The {}th Argument of function {} must be an unsigned integer.", + i + 1, + getName()); + } + } + + if (!isFloat(arguments[4].type)) + { + throw Exception{ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The fifth argument {} of function {} should be a float,", + arguments[4].type->getName(), + getName()}; + } + + /// There is an additional check for constancy in ExecuteImpl + if (!isString(arguments[5].type) || !arguments[5].column) + { + throw Exception{ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "The sixth argument {} of function {} should be a constant string", + arguments[5].type->getName(), + getName()}; + } + + return getReturnType(); + } + + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & const_arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto arguments = const_arguments; + /// Only last argument have to be constant + for (size_t i = 0; i < 5; ++i) + arguments[i].column = arguments[i].column->convertToFullColumnIfConst(); + + static const auto uint64_data_type = std::make_shared>(); + + auto column_successes_x = castColumnAccurate(arguments[0], uint64_data_type); + const auto & data_successes_x = checkAndGetColumn>(column_successes_x.get())->getData(); + + auto column_successes_y = castColumnAccurate(arguments[1], uint64_data_type); + const auto & data_successes_y = checkAndGetColumn>(column_successes_y.get())->getData(); + + auto column_trials_x = castColumnAccurate(arguments[2], uint64_data_type); + const auto & data_trials_x = checkAndGetColumn>(column_trials_x.get())->getData(); + + auto column_trials_y = castColumnAccurate(arguments[3], uint64_data_type); + const auto & data_trials_y = checkAndGetColumn>(column_trials_y.get())->getData(); + + static const auto float64_data_type = std::make_shared>(); + + auto column_confidence_level = castColumnAccurate(arguments[4], float64_data_type); + const auto & data_confidence_level = checkAndGetColumn>(column_confidence_level.get())->getData(); + + String usevar = checkAndGetColumnConst(arguments[5].column.get())->getValue(); + + if (usevar != UNPOOLED && usevar != POOLED) + throw Exception{ErrorCodes::BAD_ARGUMENTS, + "The sixth argument {} of function {} must be equal to `pooled` or `unpooled`", + arguments[5].type->getName(), + getName()}; + + const bool is_unpooled = (usevar == UNPOOLED); + + auto res_z_statistic = ColumnFloat64::create(); + auto & data_z_statistic = res_z_statistic->getData(); + data_z_statistic.reserve(input_rows_count); + + auto res_p_value = ColumnFloat64::create(); + auto & data_p_value = res_p_value->getData(); + data_p_value.reserve(input_rows_count); + + auto res_ci_lower = ColumnFloat64::create(); + auto & data_ci_lower = res_ci_lower->getData(); + data_ci_lower.reserve(input_rows_count); + + auto res_ci_upper = ColumnFloat64::create(); + auto & data_ci_upper = res_ci_upper->getData(); + data_ci_upper.reserve(input_rows_count); + + auto insert_values_into_result = [&data_z_statistic, &data_p_value, &data_ci_lower, &data_ci_upper]( + Float64 z_stat, Float64 p_value, Float64 lower, Float64 upper) + { + data_z_statistic.emplace_back(z_stat); + data_p_value.emplace_back(p_value); + data_ci_lower.emplace_back(lower); + data_ci_upper.emplace_back(upper); + }; + + static constexpr Float64 nan = std::numeric_limits::quiet_NaN(); + + boost::math::normal_distribution<> nd(0.0, 1.0); + + for (size_t row_num = 0; row_num < input_rows_count; ++row_num) + { + const UInt64 successes_x = data_successes_x[row_num]; + const UInt64 successes_y = data_successes_y[row_num]; + const UInt64 trials_x = data_trials_x[row_num]; + const UInt64 trials_y = data_trials_y[row_num]; + const Float64 confidence_level = data_confidence_level[row_num]; + + const Float64 props_x = static_cast(successes_x) / trials_x; + const Float64 props_y = static_cast(successes_y) / trials_y; + const Float64 diff = props_x - props_y; + const UInt64 trials_total = trials_x + trials_y; + + if (successes_x == 0 || successes_y == 0 || successes_x > trials_x || successes_y > trials_y || trials_total == 0 + || !std::isfinite(confidence_level) || confidence_level < 0.0 || confidence_level > 1.0) + { + insert_values_into_result(nan, nan, nan, nan); + continue; + } + + Float64 se = std::sqrt(props_x * (1.0 - props_x) / trials_x + props_y * (1.0 - props_y) / trials_y); + + /// z-statistics + /// z = \frac{ \bar{p_{1}} - \bar{p_{2}} }{ \sqrt{ \frac{ \bar{p_{1}} \left ( 1 - \bar{p_{1}} \right ) }{ n_{1} } \frac{ \bar{p_{2}} \left ( 1 - \bar{p_{2}} \right ) }{ n_{2} } } } + Float64 zstat; + if (is_unpooled) + { + zstat = (props_x - props_y) / se; + } + else + { + UInt64 successes_total = successes_x + successes_y; + Float64 p_pooled = static_cast(successes_total) / trials_total; + Float64 trials_fact = 1.0 / trials_x + 1.0 / trials_y; + zstat = diff / std::sqrt(p_pooled * (1.0 - p_pooled) * trials_fact); + } + + if (!std::isfinite(zstat)) + { + insert_values_into_result(nan, nan, nan, nan); + continue; + } + + // pvalue + Float64 pvalue = 0; + Float64 one_side = 1 - boost::math::cdf(nd, std::abs(zstat)); + pvalue = one_side * 2; + + // Confidence intervals + Float64 d = props_x - props_y; + Float64 z = -boost::math::quantile(nd, (1.0 - confidence_level) / 2.0); + Float64 dist = z * se; + Float64 ci_low = d - dist; + Float64 ci_high = d + dist; + + insert_values_into_result(zstat, pvalue, ci_low, ci_high); + } + + return ColumnTuple::create( + Columns{std::move(res_z_statistic), std::move(res_p_value), std::move(res_ci_lower), std::move(res_ci_upper)}); + } +}; + + +void registerFunctionZTest(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/IO/AIO.cpp b/src/IO/AIO.cpp index 97e5a470463..fb762271e4d 100644 --- a/src/IO/AIO.cpp +++ b/src/IO/AIO.cpp @@ -55,12 +55,12 @@ AIOContext::~AIOContext() io_destroy(ctx); } -AIOContext::AIOContext(AIOContext && rhs) +AIOContext::AIOContext(AIOContext && rhs) noexcept { *this = std::move(rhs); } -AIOContext & AIOContext::operator=(AIOContext && rhs) +AIOContext & AIOContext::operator=(AIOContext && rhs) noexcept { std::swap(ctx, rhs.ctx); return *this; diff --git a/src/IO/AIO.h b/src/IO/AIO.h index 5149aa2eb71..202939638b7 100644 --- a/src/IO/AIO.h +++ b/src/IO/AIO.h @@ -26,20 +26,20 @@ int io_setup(unsigned nr, aio_context_t * ctxp); int io_destroy(aio_context_t ctx); /// last argument is an array of pointers technically speaking -int io_submit(aio_context_t ctx, long nr, struct iocb * iocbpp[]); +int io_submit(aio_context_t ctx, long nr, struct iocb * iocbpp[]); /// NOLINT -int io_getevents(aio_context_t ctx, long min_nr, long max_nr, io_event * events, struct timespec * timeout); +int io_getevents(aio_context_t ctx, long min_nr, long max_nr, io_event * events, struct timespec * timeout); /// NOLINT struct AIOContext : private boost::noncopyable { aio_context_t ctx = 0; - AIOContext() {} - AIOContext(unsigned int nr_events); + AIOContext() = default; + explicit AIOContext(unsigned int nr_events); ~AIOContext(); - AIOContext(AIOContext && rhs); - AIOContext & operator=(AIOContext && rhs); + AIOContext(AIOContext && rhs) noexcept; + AIOContext & operator=(AIOContext && rhs) noexcept; }; #elif defined(OS_FREEBSD) diff --git a/src/IO/Archives/IArchiveReader.h b/src/IO/Archives/IArchiveReader.h index 584e80a7d09..b5c474977bf 100644 --- a/src/IO/Archives/IArchiveReader.h +++ b/src/IO/Archives/IArchiveReader.h @@ -23,7 +23,6 @@ public: { UInt64 uncompressed_size; UInt64 compressed_size; - int compression_method; bool is_encrypted; }; diff --git a/src/IO/Archives/IArchiveWriter.h b/src/IO/Archives/IArchiveWriter.h index 6879d470b62..c6330509f54 100644 --- a/src/IO/Archives/IArchiveWriter.h +++ b/src/IO/Archives/IArchiveWriter.h @@ -29,7 +29,7 @@ public: /// Sets compression method and level. /// Changing them will affect next file in the archive. - virtual void setCompression(int /* compression_method */, int /* compression_level */ = kDefaultCompressionLevel) {} + virtual void setCompression(const String & /* compression_method */, int /* compression_level */ = kDefaultCompressionLevel) {} /// Sets password. If the password is not empty it will enable encryption in the archive. virtual void setPassword(const String & /* password */) {} diff --git a/src/IO/Archives/ZipArchiveReader.cpp b/src/IO/Archives/ZipArchiveReader.cpp index 16604da62dc..68726248dc4 100644 --- a/src/IO/Archives/ZipArchiveReader.cpp +++ b/src/IO/Archives/ZipArchiveReader.cpp @@ -1,6 +1,7 @@ #include #if USE_MINIZIP +#include #include #include #include @@ -18,6 +19,20 @@ namespace ErrorCodes using RawHandle = unzFile; +namespace +{ + void checkCompressionMethodIsEnabled(int compression_method_) + { + ZipArchiveWriter::checkCompressionMethodIsEnabled(compression_method_); + } + + void checkEncryptionIsEnabled() + { + ZipArchiveWriter::checkEncryptionIsEnabled(); + } +} + + /// Holds a raw handle, calls acquireRawHandle() in the constructor and releaseRawHandle() in the destructor. class ZipArchiveReader::HandleHolder { @@ -42,12 +57,12 @@ public: } } - HandleHolder(HandleHolder && src) + HandleHolder(HandleHolder && src) noexcept { *this = std::move(src); } - HandleHolder & operator =(HandleHolder && src) + HandleHolder & operator=(HandleHolder && src) noexcept { reader = std::exchange(src.reader, nullptr); raw_handle = std::exchange(src.raw_handle, nullptr); @@ -108,7 +123,7 @@ public: return *file_name; } - const FileInfo & getFileInfo() const + const FileInfoImpl & getFileInfo() const { if (!file_info) retrieveFileInfo(); @@ -161,7 +176,7 @@ private: std::shared_ptr reader; RawHandle raw_handle = nullptr; mutable std::optional file_name; - mutable std::optional file_info; + mutable std::optional file_info; }; @@ -174,7 +189,7 @@ public: , handle(std::move(handle_)) { const auto & file_info = handle.getFileInfo(); - checkCompressionMethodIsEnabled(static_cast(file_info.compression_method)); + checkCompressionMethodIsEnabled(file_info.compression_method); const char * password_cstr = nullptr; if (file_info.is_encrypted) @@ -227,7 +242,7 @@ public: if (new_pos > static_cast(file_info.uncompressed_size)) throw Exception("Seek position is out of bound", ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); - if (file_info.compression_method == static_cast(CompressionMethod::kStore)) + if (file_info.compression_method == MZ_COMPRESS_METHOD_STORE) { /// unzSeek64() works only for non-compressed files. checkResult(unzSeek64(raw_handle, off, whence)); diff --git a/src/IO/Archives/ZipArchiveReader.h b/src/IO/Archives/ZipArchiveReader.h index 6932a93e23f..7236b0b660c 100644 --- a/src/IO/Archives/ZipArchiveReader.h +++ b/src/IO/Archives/ZipArchiveReader.h @@ -4,7 +4,6 @@ #if USE_MINIZIP #include -#include #include #include #include @@ -20,8 +19,6 @@ class SeekableReadBuffer; class ZipArchiveReader : public shared_ptr_helper, public IArchiveReader { public: - using CompressionMethod = ZipArchiveWriter::CompressionMethod; - ~ZipArchiveReader() override; /// Returns true if there is a specified file in the archive. @@ -45,11 +42,6 @@ public: /// Sets password used to decrypt the contents of the files in the archive. void setPassword(const String & password_) override; - /// Utility functions. - static CompressionMethod parseCompressionMethod(const String & str) { return ZipArchiveWriter::parseCompressionMethod(str); } - static void checkCompressionMethodIsEnabled(CompressionMethod method) { ZipArchiveWriter::checkCompressionMethodIsEnabled(method); } - static void checkEncryptionIsEnabled() { ZipArchiveWriter::checkEncryptionIsEnabled(); } - private: /// Constructs an archive's reader that will read from a file in the local filesystem. explicit ZipArchiveReader(const String & path_to_archive_); @@ -66,6 +58,11 @@ private: void init(); + struct FileInfoImpl : public FileInfo + { + int compression_method; + }; + HandleHolder acquireHandle(); RawHandle acquireRawHandle(); void releaseRawHandle(RawHandle handle_); diff --git a/src/IO/Archives/ZipArchiveWriter.cpp b/src/IO/Archives/ZipArchiveWriter.cpp index f5ecea5e5aa..dbfd66a6293 100644 --- a/src/IO/Archives/ZipArchiveWriter.cpp +++ b/src/IO/Archives/ZipArchiveWriter.cpp @@ -46,12 +46,12 @@ public: } } - HandleHolder(HandleHolder && src) + HandleHolder(HandleHolder && src) noexcept { *this = std::move(src); } - HandleHolder & operator =(HandleHolder && src) + HandleHolder & operator=(HandleHolder && src) noexcept { writer = std::exchange(src.writer, nullptr); raw_handle = std::exchange(src.raw_handle, nullptr); @@ -80,7 +80,7 @@ public: { auto compress_method = handle.getWriter()->compression_method; auto compress_level = handle.getWriter()->compression_level; - checkCompressionMethodIsEnabled(static_cast(compress_method)); + checkCompressionMethodIsEnabled(compress_method); const char * password_cstr = nullptr; const String & password_str = handle.getWriter()->password; @@ -238,7 +238,7 @@ ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_) } ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_, std::unique_ptr archive_write_buffer_) - : path_to_archive(path_to_archive_) + : path_to_archive(path_to_archive_), compression_method(MZ_COMPRESS_METHOD_DEFLATE) { if (archive_write_buffer_) handle = StreamFromWriteBuffer::open(std::move(archive_write_buffer_)); @@ -246,6 +246,7 @@ ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_, std::unique_ handle = zipOpen64(path_to_archive.c_str(), /* append= */ false); if (!handle) throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't create zip archive {}", quoteString(path_to_archive)); + } ZipArchiveWriter::~ZipArchiveWriter() @@ -274,10 +275,10 @@ bool ZipArchiveWriter::isWritingFile() const return !handle; } -void ZipArchiveWriter::setCompression(int compression_method_, int compression_level_) +void ZipArchiveWriter::setCompression(const String & compression_method_, int compression_level_) { std::lock_guard lock{mutex}; - compression_method = compression_method_; + compression_method = compressionMethodToInt(compression_method_); compression_level = compression_level_; } @@ -287,48 +288,62 @@ void ZipArchiveWriter::setPassword(const String & password_) password = password_; } -ZipArchiveWriter::CompressionMethod ZipArchiveWriter::parseCompressionMethod(const String & str) +int ZipArchiveWriter::compressionMethodToInt(const String & compression_method_) { - if (str.empty()) - return CompressionMethod::kDeflate; /// Default compression method is DEFLATE. - else if (boost::iequals(str, "store")) - return CompressionMethod::kStore; - else if (boost::iequals(str, "deflate")) - return CompressionMethod::kDeflate; - else if (boost::iequals(str, "bzip2")) - return CompressionMethod::kBzip2; - else if (boost::iequals(str, "lzma")) - return CompressionMethod::kLzma; - else if (boost::iequals(str, "zstd")) - return CompressionMethod::kZstd; - else if (boost::iequals(str, "xz")) - return CompressionMethod::kXz; + if (compression_method_.empty()) + return MZ_COMPRESS_METHOD_DEFLATE; /// By default the compression method is "deflate". + else if (compression_method_ == kStore) + return MZ_COMPRESS_METHOD_STORE; + else if (compression_method_ == kDeflate) + return MZ_COMPRESS_METHOD_DEFLATE; + else if (compression_method_ == kBzip2) + return MZ_COMPRESS_METHOD_BZIP2; + else if (compression_method_ == kLzma) + return MZ_COMPRESS_METHOD_LZMA; + else if (compression_method_ == kZstd) + return MZ_COMPRESS_METHOD_ZSTD; + else if (compression_method_ == kXz) + return MZ_COMPRESS_METHOD_XZ; else - throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", str); + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", compression_method_); +} + +String ZipArchiveWriter::intToCompressionMethod(int compression_method_) +{ + switch (compression_method_) + { + case MZ_COMPRESS_METHOD_STORE: return kStore; + case MZ_COMPRESS_METHOD_DEFLATE: return kDeflate; + case MZ_COMPRESS_METHOD_BZIP2: return kBzip2; + case MZ_COMPRESS_METHOD_LZMA: return kLzma; + case MZ_COMPRESS_METHOD_ZSTD: return kZstd; + case MZ_COMPRESS_METHOD_XZ: return kXz; + } + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", compression_method_); } /// Checks that a passed compression method can be used. -void ZipArchiveWriter::checkCompressionMethodIsEnabled(CompressionMethod method) +void ZipArchiveWriter::checkCompressionMethodIsEnabled(int compression_method_) { - switch (method) + switch (compression_method_) { - case CompressionMethod::kStore: [[fallthrough]]; - case CompressionMethod::kDeflate: - case CompressionMethod::kLzma: - case CompressionMethod::kXz: - case CompressionMethod::kZstd: + case MZ_COMPRESS_METHOD_STORE: [[fallthrough]]; + case MZ_COMPRESS_METHOD_DEFLATE: + case MZ_COMPRESS_METHOD_LZMA: + case MZ_COMPRESS_METHOD_ZSTD: + case MZ_COMPRESS_METHOD_XZ: return; - case CompressionMethod::kBzip2: + case MZ_COMPRESS_METHOD_BZIP2: { #if USE_BZIP2 return; #else - throw Exception("BZIP2 compression method is disabled", ErrorCodes::SUPPORT_IS_DISABLED); + throw Exception("bzip2 compression method is disabled", ErrorCodes::SUPPORT_IS_DISABLED); #endif } } - throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", static_cast(method)); + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", compression_method_); } /// Checks that encryption is enabled. diff --git a/src/IO/Archives/ZipArchiveWriter.h b/src/IO/Archives/ZipArchiveWriter.h index 76f8dd8e9e5..58df4902434 100644 --- a/src/IO/Archives/ZipArchiveWriter.h +++ b/src/IO/Archives/ZipArchiveWriter.h @@ -31,16 +31,12 @@ public: bool isWritingFile() const override; /// Supported compression methods. - enum class CompressionMethod - { - /// See mz.h - kStore = 0, - kDeflate = 8, - kBzip2 = 12, - kLzma = 14, - kZstd = 93, - kXz = 95, - }; + static constexpr const char kStore[] = "store"; + static constexpr const char kDeflate[] = "deflate"; + static constexpr const char kBzip2[] = "bzip2"; + static constexpr const char kLzma[] = "lzma"; + static constexpr const char kZstd[] = "zstd"; + static constexpr const char kXz[] = "xz"; /// Some compression levels. enum class CompressionLevels @@ -53,7 +49,7 @@ public: /// Sets compression method and level. /// Changing them will affect next file in the archive. - void setCompression(int compression_method_, int compression_level_) override; + void setCompression(const String & compression_method_, int compression_level_) override; /// Sets password. Only contents of the files are encrypted, /// names of files are not encrypted. @@ -61,8 +57,9 @@ public: void setPassword(const String & password_) override; /// Utility functions. - static CompressionMethod parseCompressionMethod(const String & str); - static void checkCompressionMethodIsEnabled(CompressionMethod method); + static int compressionMethodToInt(const String & compression_method_); + static String intToCompressionMethod(int compression_method_); + static void checkCompressionMethodIsEnabled(int compression_method_); static void checkEncryptionIsEnabled(); private: @@ -85,7 +82,7 @@ private: [[noreturn]] void showError(const String & message) const; const String path_to_archive; - int compression_method = static_cast(CompressionMethod::kDeflate); + int compression_method; /// By default the compression method is "deflate". int compression_level = kDefaultCompressionLevel; String password; RawHandle handle = nullptr; diff --git a/src/IO/Archives/hasRegisteredArchiveFileExtension.cpp b/src/IO/Archives/hasRegisteredArchiveFileExtension.cpp new file mode 100644 index 00000000000..6b2ef29d054 --- /dev/null +++ b/src/IO/Archives/hasRegisteredArchiveFileExtension.cpp @@ -0,0 +1,12 @@ +#include + + +namespace DB +{ + +bool hasRegisteredArchiveFileExtension(const String & path) +{ + return path.ends_with(".zip") || path.ends_with(".zipx"); +} + +} diff --git a/src/IO/Archives/hasRegisteredArchiveFileExtension.h b/src/IO/Archives/hasRegisteredArchiveFileExtension.h new file mode 100644 index 00000000000..cab938aa0b4 --- /dev/null +++ b/src/IO/Archives/hasRegisteredArchiveFileExtension.h @@ -0,0 +1,12 @@ +#pragma once + +#include + + +namespace DB +{ + +/// Returns true if a specified path has one of the registered file extensions for an archive. +bool hasRegisteredArchiveFileExtension(const String & path); + +} diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp index 877702f9705..a8c3e23815d 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp @@ -26,6 +26,7 @@ namespace DB namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; + extern const int LOGICAL_ERROR; } @@ -43,6 +44,8 @@ std::future AsynchronousReadBufferFromFileDescripto request.size = size; request.offset = file_offset_of_buffer_end; request.priority = priority; + request.ignore = bytes_to_ignore; + bytes_to_ignore = 0; /// This is a workaround of a read pass EOF bug in linux kernel with pread() if (file_size.has_value() && file_offset_of_buffer_end >= *file_size) @@ -75,11 +78,14 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() /// Read request already in flight. Wait for its completion. size_t size = 0; + size_t offset = 0; { Stopwatch watch; CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; auto result = prefetch_future.get(); size = result.size; + offset = result.offset; + assert(offset < size || size == 0); ProfileEvents::increment(ProfileEvents::AsynchronousReadWaitMicroseconds, watch.elapsedMicroseconds()); } @@ -89,8 +95,8 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() if (size) { prefetch_buffer.swap(memory); - set(memory.data(), memory.size()); - working_buffer.resize(size); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); return true; } @@ -100,13 +106,13 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() { /// No pending request. Do synchronous read. - auto [size, _] = readInto(memory.data(), memory.size()).get(); + auto [size, offset] = readInto(memory.data(), memory.size()).get(); file_offset_of_buffer_end += size; if (size) { - set(memory.data(), memory.size()); - working_buffer.resize(size); + /// Adjust the working buffer so that it ignores `offset` bytes. + setWithBytesToIgnore(memory.data(), size, offset); return true; } @@ -125,6 +131,30 @@ void AsynchronousReadBufferFromFileDescriptor::finalize() } +AsynchronousReadBufferFromFileDescriptor::AsynchronousReadBufferFromFileDescriptor( + AsynchronousReaderPtr reader_, + Int32 priority_, + int fd_, + size_t buf_size, + char * existing_memory, + size_t alignment, + std::optional file_size_) + : ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_) + , reader(std::move(reader_)) + , priority(priority_) + , required_alignment(alignment) + , fd(fd_) +{ + if (required_alignment > buf_size) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Too large alignment. Cannot have required_alignment greater than buf_size: {} > {}. It is a bug", + required_alignment, + buf_size); + + prefetch_buffer.alignment = alignment; +} + AsynchronousReadBufferFromFileDescriptor::~AsynchronousReadBufferFromFileDescriptor() { finalize(); @@ -153,46 +183,48 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end) return new_pos; - if (file_offset_of_buffer_end - working_buffer.size() <= static_cast(new_pos) - && new_pos <= file_offset_of_buffer_end) + while (true) { - /// Position is still inside the buffer. - /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. - - pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; - assert(pos >= working_buffer.begin()); - assert(pos <= working_buffer.end()); - - return new_pos; - } - else - { - if (prefetch_future.valid()) + if (file_offset_of_buffer_end - working_buffer.size() <= new_pos && new_pos <= file_offset_of_buffer_end) { - //std::cerr << "Ignoring prefetched data" << "\n"; - prefetch_future.wait(); - prefetch_future = {}; + /// Position is still inside the buffer. + /// Probably it is at the end of the buffer - then we will load data on the following 'next' call. + + pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; + assert(pos >= working_buffer.begin()); + assert(pos <= working_buffer.end()); + + return new_pos; + } + else if (prefetch_future.valid()) + { + /// Read from prefetch buffer and recheck if the new position is valid inside. + + if (nextImpl()) + continue; } - /// Position is out of the buffer, we need to do real seek. - off_t seek_pos = required_alignment > 1 - ? new_pos / required_alignment * required_alignment - : new_pos; - - off_t offset_after_seek_pos = new_pos - seek_pos; - - /// First reset the buffer so the next read will fetch new data to the buffer. - resetWorkingBuffer(); - - /// Just update the info about the next position in file. - - file_offset_of_buffer_end = seek_pos; - - if (offset_after_seek_pos > 0) - ignore(offset_after_seek_pos); - - return seek_pos; + break; } + + assert(!prefetch_future.valid()); + + /// Position is out of the buffer, we need to do real seek. + off_t seek_pos = required_alignment > 1 + ? new_pos / required_alignment * required_alignment + : new_pos; + + /// First reset the buffer so the next read will fetch new data to the buffer. + resetWorkingBuffer(); + + /// Just update the info about the next position in file. + + file_offset_of_buffer_end = seek_pos; + bytes_to_ignore = new_pos - seek_pos; + + assert(bytes_to_ignore < internal_buffer.size()); + + return seek_pos; } diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.h b/src/IO/AsynchronousReadBufferFromFileDescriptor.h index 2a16148812e..a74f24d62a0 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.h +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.h @@ -24,6 +24,7 @@ protected: const size_t required_alignment = 0; /// For O_DIRECT both file offsets and memory addresses have to be aligned. size_t file_offset_of_buffer_end = 0; /// What offset in file corresponds to working_buffer.end(). + size_t bytes_to_ignore = 0; /// How many bytes should we ignore upon a new read request. int fd; bool nextImpl() override; @@ -41,15 +42,7 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0, - std::optional file_size_ = std::nullopt) - : ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_) - , reader(std::move(reader_)) - , priority(priority_) - , required_alignment(alignment) - , fd(fd_) - { - prefetch_buffer.alignment = alignment; - } + std::optional file_size_ = std::nullopt); ~AsynchronousReadBufferFromFileDescriptor() override; diff --git a/src/IO/AsynchronousReader.h b/src/IO/AsynchronousReader.h index e79e72f3bec..4583f594c37 100644 --- a/src/IO/AsynchronousReader.h +++ b/src/IO/AsynchronousReader.h @@ -32,7 +32,7 @@ public: struct LocalFileDescriptor : public IFileDescriptor { - LocalFileDescriptor(int fd_) : fd(fd_) {} + explicit LocalFileDescriptor(int fd_) : fd(fd_) {} int fd; }; diff --git a/src/IO/BitHelpers.h b/src/IO/BitHelpers.h index d15297637a3..b96f43bdeff 100644 --- a/src/IO/BitHelpers.h +++ b/src/IO/BitHelpers.h @@ -52,8 +52,7 @@ public: bits_count(0) {} - ~BitReader() - {} + ~BitReader() = default; // reads bits_to_read high-bits from bits_buffer inline UInt64 readBits(UInt8 bits_to_read) diff --git a/src/IO/BrotliReadBuffer.cpp b/src/IO/BrotliReadBuffer.cpp index 77069746153..d2e954173a4 100644 --- a/src/IO/BrotliReadBuffer.cpp +++ b/src/IO/BrotliReadBuffer.cpp @@ -50,24 +50,29 @@ bool BrotliReadBuffer::nextImpl() if (eof_flag) return false; - if (!in_available) + do { - in->nextIfAtEnd(); - in_available = in->buffer().end() - in->position(); - in_data = reinterpret_cast(in->position()); + if (!in_available) + { + in->nextIfAtEnd(); + in_available = in->buffer().end() - in->position(); + in_data = reinterpret_cast(in->position()); + } + + if (brotli->result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && (!in_available || in->eof())) + { + throw Exception("brotli decode error", ErrorCodes::BROTLI_READ_FAILED); + } + + out_capacity = internal_buffer.size(); + out_data = reinterpret_cast(internal_buffer.begin()); + + brotli->result = BrotliDecoderDecompressStream(brotli->state, &in_available, &in_data, &out_capacity, &out_data, nullptr); + + in->position() = in->buffer().end() - in_available; } + while (brotli->result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && out_capacity == internal_buffer.size()); - if (brotli->result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && (!in_available || in->eof())) - { - throw Exception("brotli decode error", ErrorCodes::BROTLI_READ_FAILED); - } - - out_capacity = internal_buffer.size(); - out_data = reinterpret_cast(internal_buffer.begin()); - - brotli->result = BrotliDecoderDecompressStream(brotli->state, &in_available, &in_data, &out_capacity, &out_data, nullptr); - - in->position() = in->buffer().end() - in_available; working_buffer.resize(internal_buffer.size() - out_capacity); if (brotli->result == BROTLI_DECODER_RESULT_SUCCESS) diff --git a/src/IO/BrotliReadBuffer.h b/src/IO/BrotliReadBuffer.h index 44a7dc7ddbd..cbb919e15ae 100644 --- a/src/IO/BrotliReadBuffer.h +++ b/src/IO/BrotliReadBuffer.h @@ -10,7 +10,7 @@ namespace DB class BrotliReadBuffer : public BufferWithOwnMemory { public: - BrotliReadBuffer( + explicit BrotliReadBuffer( std::unique_ptr in_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/Bzip2ReadBuffer.cpp b/src/IO/Bzip2ReadBuffer.cpp index c2060612757..1f35ad08216 100644 --- a/src/IO/Bzip2ReadBuffer.cpp +++ b/src/IO/Bzip2ReadBuffer.cpp @@ -53,19 +53,25 @@ bool Bzip2ReadBuffer::nextImpl() if (eof_flag) return false; - if (!bz->stream.avail_in) + int ret; + do { - in->nextIfAtEnd(); - bz->stream.avail_in = in->buffer().end() - in->position(); - bz->stream.next_in = in->position(); + if (!bz->stream.avail_in) + { + in->nextIfAtEnd(); + bz->stream.avail_in = in->buffer().end() - in->position(); + bz->stream.next_in = in->position(); + } + + bz->stream.avail_out = internal_buffer.size(); + bz->stream.next_out = internal_buffer.begin(); + + ret = BZ2_bzDecompress(&bz->stream); + + in->position() = in->buffer().end() - bz->stream.avail_in; } + while (bz->stream.avail_out == internal_buffer.size() && ret == BZ_OK && !in->eof()); - bz->stream.avail_out = internal_buffer.size(); - bz->stream.next_out = internal_buffer.begin(); - - int ret = BZ2_bzDecompress(&bz->stream); - - in->position() = in->buffer().end() - bz->stream.avail_in; working_buffer.resize(internal_buffer.size() - bz->stream.avail_out); if (ret == BZ_STREAM_END) diff --git a/src/IO/Bzip2ReadBuffer.h b/src/IO/Bzip2ReadBuffer.h index de1e61ee388..cd5fadf9c82 100644 --- a/src/IO/Bzip2ReadBuffer.h +++ b/src/IO/Bzip2ReadBuffer.h @@ -10,7 +10,7 @@ namespace DB class Bzip2ReadBuffer : public BufferWithOwnMemory { public: - Bzip2ReadBuffer( + explicit Bzip2ReadBuffer( std::unique_ptr in_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/CascadeWriteBuffer.h b/src/IO/CascadeWriteBuffer.h index db0d1e7a5a8..ebd4f262aa2 100644 --- a/src/IO/CascadeWriteBuffer.h +++ b/src/IO/CascadeWriteBuffer.h @@ -31,7 +31,7 @@ public: using WriteBufferConstructor = std::function; using WriteBufferConstructors = std::vector; - CascadeWriteBuffer(WriteBufferPtrs && prepared_sources_, WriteBufferConstructors && lazy_sources_ = {}); + explicit CascadeWriteBuffer(WriteBufferPtrs && prepared_sources_, WriteBufferConstructors && lazy_sources_ = {}); void nextImpl() override; diff --git a/src/IO/CompressionMethod.cpp b/src/IO/CompressionMethod.cpp index eaab7560e6a..f6daec78170 100644 --- a/src/IO/CompressionMethod.cpp +++ b/src/IO/CompressionMethod.cpp @@ -65,7 +65,13 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s file_extension = path.substr(pos + 1, std::string::npos); } - std::string method_str = file_extension.empty() ? hint : std::move(file_extension); + std::string method_str; + + if (file_extension.empty()) + method_str = hint; + else + method_str = std::move(file_extension); + boost::algorithm::to_lower(method_str); if (method_str == "gzip" || method_str == "gz") diff --git a/src/IO/ConcatReadBuffer.h b/src/IO/ConcatReadBuffer.h index 4ef8d04d4c9..3f44181a6e9 100644 --- a/src/IO/ConcatReadBuffer.h +++ b/src/IO/ConcatReadBuffer.h @@ -23,6 +23,12 @@ public: assert(!buffers.empty()); } + ConcatReadBuffer(std::unique_ptr buf1, std::unique_ptr buf2) : ConcatReadBuffer() + { + appendBuffer(std::move(buf1)); + appendBuffer(std::move(buf2)); + } + ConcatReadBuffer(ReadBuffer & buf1, ReadBuffer & buf2) : ConcatReadBuffer() { appendBuffer(wrapReadBufferReference(buf1)); diff --git a/src/IO/DoubleConverter.h b/src/IO/DoubleConverter.h index 0896aca717e..75429967390 100644 --- a/src/IO/DoubleConverter.h +++ b/src/IO/DoubleConverter.h @@ -29,9 +29,6 @@ template <> struct DoubleToStringConverterFlags template class DoubleConverter : private boost::noncopyable { - DoubleConverter(const DoubleConverter &) = delete; - DoubleConverter & operator=(const DoubleConverter &) = delete; - DoubleConverter() = default; public: diff --git a/src/IO/FileEncryptionCommon.h b/src/IO/FileEncryptionCommon.h index 28d924e6d81..bb6c8d14893 100644 --- a/src/IO/FileEncryptionCommon.h +++ b/src/IO/FileEncryptionCommon.h @@ -56,7 +56,7 @@ public: /// Adds a specified offset to the counter. InitVector & operator++() { ++counter; return *this; } - InitVector operator++(int) { InitVector res = *this; ++counter; return res; } + InitVector operator++(int) { InitVector res = *this; ++counter; return res; } /// NOLINT InitVector & operator+=(size_t offset) { counter += offset; return *this; } InitVector operator+(size_t offset) const { InitVector res = *this; return res += offset; } diff --git a/src/IO/HadoopSnappyReadBuffer.cpp b/src/IO/HadoopSnappyReadBuffer.cpp index 324df67e900..cac05b4827b 100644 --- a/src/IO/HadoopSnappyReadBuffer.cpp +++ b/src/IO/HadoopSnappyReadBuffer.cpp @@ -11,7 +11,6 @@ #include "HadoopSnappyReadBuffer.h" - namespace DB { namespace ErrorCodes @@ -32,11 +31,11 @@ inline bool HadoopSnappyDecoder::checkAvailIn(size_t avail_in, int min) inline void HadoopSnappyDecoder::copyToBuffer(size_t * avail_in, const char ** next_in) { - assert(*avail_in <= sizeof(buffer)); + assert(*avail_in + buffer_length <= sizeof(buffer)); - memcpy(buffer, *next_in, *avail_in); + memcpy(buffer + buffer_length, *next_in, *avail_in); - buffer_length = *avail_in; + buffer_length += *avail_in; *next_in += *avail_in; *avail_in = 0; } @@ -78,14 +77,21 @@ inline HadoopSnappyDecoder::Status HadoopSnappyDecoder::readLength(size_t * avai inline HadoopSnappyDecoder::Status HadoopSnappyDecoder::readBlockLength(size_t * avail_in, const char ** next_in) { if (block_length < 0) + { return readLength(avail_in, next_in, &block_length); + } return Status::OK; } inline HadoopSnappyDecoder::Status HadoopSnappyDecoder::readCompressedLength(size_t * avail_in, const char ** next_in) { if (compressed_length < 0) - return readLength(avail_in, next_in, &compressed_length); + { + auto status = readLength(avail_in, next_in, &compressed_length); + if (unlikely(compressed_length > 0 && static_cast(compressed_length) > sizeof(buffer))) + throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "Too large snappy compressed block. buffer size: {}, compressed block size: {}", sizeof(buffer), compressed_length); + return status; + } return Status::OK; } @@ -111,7 +117,6 @@ HadoopSnappyDecoder::readCompressedData(size_t * avail_in, const char ** next_in { compressed = const_cast(*next_in); } - size_t uncompressed_length = *avail_out; auto status = snappy_uncompress(compressed, compressed_length, *next_out, &uncompressed_length); if (status != SNAPPY_OK) @@ -154,7 +159,9 @@ HadoopSnappyDecoder::Status HadoopSnappyDecoder::readBlock(size_t * avail_in, co return status; } if (total_uncompressed_length != block_length) + { return Status::INVALID_INPUT; + } return Status::OK; } diff --git a/src/IO/HashingWriteBuffer.h b/src/IO/HashingWriteBuffer.h index bd00a2b12da..bf636deeb07 100644 --- a/src/IO/HashingWriteBuffer.h +++ b/src/IO/HashingWriteBuffer.h @@ -17,7 +17,7 @@ class IHashingBuffer : public BufferWithOwnMemory public: using uint128 = CityHash_v1_0_2::uint128; - IHashingBuffer(size_t block_size_ = DBMS_DEFAULT_HASHING_BLOCK_SIZE) + explicit IHashingBuffer(size_t block_size_ = DBMS_DEFAULT_HASHING_BLOCK_SIZE) : BufferWithOwnMemory(block_size_), block_pos(0), block_size(block_size_), state(0, 0) { } @@ -66,7 +66,7 @@ private: } public: - HashingWriteBuffer( + explicit HashingWriteBuffer( WriteBuffer & out_, size_t block_size_ = DBMS_DEFAULT_HASHING_BLOCK_SIZE) : IHashingBuffer(block_size_), out(out_) diff --git a/src/IO/IOThreadPool.cpp b/src/IO/IOThreadPool.cpp new file mode 100644 index 00000000000..4014d00d8b8 --- /dev/null +++ b/src/IO/IOThreadPool.cpp @@ -0,0 +1,34 @@ +#include +#include "Core/Field.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +std::unique_ptr IOThreadPool::instance; + +void IOThreadPool::initialize(size_t max_threads, size_t max_free_threads, size_t queue_size) +{ + if (instance) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "The IO thread pool is initialized twice"); + } + + instance = std::make_unique(max_threads, max_free_threads, queue_size, false /*shutdown_on_exception*/); +} + +ThreadPool & IOThreadPool::get() +{ + if (!instance) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "The IO thread pool is not initialized"); + } + + return *instance; +} + +} diff --git a/src/IO/IOThreadPool.h b/src/IO/IOThreadPool.h new file mode 100644 index 00000000000..4fcf99b6048 --- /dev/null +++ b/src/IO/IOThreadPool.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace DB +{ + +/* + * ThreadPool used for the IO. + */ +class IOThreadPool +{ + static std::unique_ptr instance; + +public: + static void initialize(size_t max_threads, size_t max_free_threads, size_t queue_size); + static ThreadPool & get(); +}; + +} diff --git a/src/IO/LZMAInflatingReadBuffer.cpp b/src/IO/LZMAInflatingReadBuffer.cpp index 80da7421fc3..19c0da5a808 100644 --- a/src/IO/LZMAInflatingReadBuffer.cpp +++ b/src/IO/LZMAInflatingReadBuffer.cpp @@ -6,6 +6,7 @@ namespace ErrorCodes { extern const int LZMA_STREAM_DECODER_FAILED; } + LZMAInflatingReadBuffer::LZMAInflatingReadBuffer(std::unique_ptr in_, size_t buf_size, char * existing_memory, size_t alignment) : BufferWithOwnMemory(buf_size, existing_memory, alignment), in(std::move(in_)), eof_flag(false) { @@ -40,24 +41,30 @@ bool LZMAInflatingReadBuffer::nextImpl() return false; lzma_action action = LZMA_RUN; + lzma_ret ret; - if (!lstr.avail_in) + do { - in->nextIfAtEnd(); - lstr.next_in = reinterpret_cast(in->position()); - lstr.avail_in = in->buffer().end() - in->position(); + if (!lstr.avail_in) + { + in->nextIfAtEnd(); + lstr.next_in = reinterpret_cast(in->position()); + lstr.avail_in = in->buffer().end() - in->position(); + } + + if (in->eof()) + { + action = LZMA_FINISH; + } + + lstr.next_out = reinterpret_cast(internal_buffer.begin()); + lstr.avail_out = internal_buffer.size(); + + ret = lzma_code(&lstr, action); + in->position() = in->buffer().end() - lstr.avail_in; } + while (ret == LZMA_OK && lstr.avail_out == internal_buffer.size()); - if (in->eof()) - { - action = LZMA_FINISH; - } - - lstr.next_out = reinterpret_cast(internal_buffer.begin()); - lstr.avail_out = internal_buffer.size(); - - lzma_ret ret = lzma_code(&lstr, action); - in->position() = in->buffer().end() - lstr.avail_in; working_buffer.resize(internal_buffer.size() - lstr.avail_out); if (ret == LZMA_STREAM_END) diff --git a/src/IO/LZMAInflatingReadBuffer.h b/src/IO/LZMAInflatingReadBuffer.h index 2d676eeeeb3..920345ee09c 100644 --- a/src/IO/LZMAInflatingReadBuffer.h +++ b/src/IO/LZMAInflatingReadBuffer.h @@ -11,7 +11,7 @@ namespace DB class LZMAInflatingReadBuffer : public BufferWithOwnMemory { public: - LZMAInflatingReadBuffer( + explicit LZMAInflatingReadBuffer( std::unique_ptr in_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/Lz4DeflatingWriteBuffer.h b/src/IO/Lz4DeflatingWriteBuffer.h index a27cb42a6e7..68873b5f8ee 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.h +++ b/src/IO/Lz4DeflatingWriteBuffer.h @@ -29,7 +29,7 @@ private: void finalizeBefore() override; void finalizeAfter() override; - LZ4F_preferences_t kPrefs; + LZ4F_preferences_t kPrefs; /// NOLINT LZ4F_compressionContext_t ctx; void * in_data; diff --git a/src/IO/Lz4InflatingReadBuffer.cpp b/src/IO/Lz4InflatingReadBuffer.cpp index 5639badbe0e..0cdde54497c 100644 --- a/src/IO/Lz4InflatingReadBuffer.cpp +++ b/src/IO/Lz4InflatingReadBuffer.cpp @@ -35,26 +35,39 @@ bool Lz4InflatingReadBuffer::nextImpl() if (eof_flag) return false; - if (!in_available) + bool need_more_input = false; + size_t ret; + + do { - in->nextIfAtEnd(); - in_available = in->buffer().end() - in->position(); + if (!in_available) + { + in->nextIfAtEnd(); + in_available = in->buffer().end() - in->position(); + } + + in_data = reinterpret_cast(in->position()); + out_data = reinterpret_cast(internal_buffer.begin()); + + out_available = internal_buffer.size(); + + size_t bytes_read = in_available; + size_t bytes_written = out_available; + + ret = LZ4F_decompress(dctx, out_data, &bytes_written, in_data, &bytes_read, /* LZ4F_decompressOptions_t */ nullptr); + + in_available -= bytes_read; + out_available -= bytes_written; + + /// It may happen that we didn't get new uncompressed data + /// (for example if we read the end of frame). Load new data + /// in this case. + need_more_input = bytes_written == 0; + + in->position() = in->buffer().end() - in_available; } + while (need_more_input && !LZ4F_isError(ret) && !in->eof()); - in_data = reinterpret_cast(in->position()); - out_data = reinterpret_cast(internal_buffer.begin()); - - out_available = internal_buffer.size(); - - size_t bytes_read = in_available; - size_t bytes_written = out_available; - - size_t ret = LZ4F_decompress(dctx, out_data, &bytes_written, in_data, &bytes_read, /* LZ4F_decompressOptions_t */ nullptr); - - in_available -= bytes_read; - out_available -= bytes_written; - - in->position() = in->buffer().end() - in_available; working_buffer.resize(internal_buffer.size() - out_available); if (LZ4F_isError(ret)) @@ -70,12 +83,6 @@ bool Lz4InflatingReadBuffer::nextImpl() return !working_buffer.empty(); } - /// It may happen that we didn't get new uncompressed data - /// (for example if we read the end of frame). Load new data - /// in this case. - if (working_buffer.empty()) - return nextImpl(); - return true; } } diff --git a/src/IO/Lz4InflatingReadBuffer.h b/src/IO/Lz4InflatingReadBuffer.h index d4d81f8765c..9921939d453 100644 --- a/src/IO/Lz4InflatingReadBuffer.h +++ b/src/IO/Lz4InflatingReadBuffer.h @@ -14,7 +14,7 @@ namespace DB class Lz4InflatingReadBuffer : public BufferWithOwnMemory { public: - Lz4InflatingReadBuffer( + explicit Lz4InflatingReadBuffer( std::unique_ptr in_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/MMapReadBufferFromFileDescriptor.h b/src/IO/MMapReadBufferFromFileDescriptor.h index 03718a61a6c..1715c2200fb 100644 --- a/src/IO/MMapReadBufferFromFileDescriptor.h +++ b/src/IO/MMapReadBufferFromFileDescriptor.h @@ -18,7 +18,7 @@ public: off_t seek(off_t off, int whence) override; protected: - MMapReadBufferFromFileDescriptor() {} + MMapReadBufferFromFileDescriptor() = default; void init(); MMappedFileDescriptor mapped; diff --git a/src/IO/MMappedFileCache.h b/src/IO/MMappedFileCache.h index adbb85a18cf..fe5e7e8e1f7 100644 --- a/src/IO/MMappedFileCache.h +++ b/src/IO/MMappedFileCache.h @@ -27,7 +27,7 @@ private: using Base = LRUCache; public: - MMappedFileCache(size_t max_size_in_bytes) + explicit MMappedFileCache(size_t max_size_in_bytes) : Base(max_size_in_bytes) {} /// Calculate key from path to file and offset. diff --git a/src/IO/MMappedFileDescriptor.h b/src/IO/MMappedFileDescriptor.h index 01dc7e1866c..2611093643f 100644 --- a/src/IO/MMappedFileDescriptor.h +++ b/src/IO/MMappedFileDescriptor.h @@ -22,7 +22,7 @@ public: MMappedFileDescriptor(int fd_, size_t offset_); /// Makes empty object that can be initialized with `set`. - MMappedFileDescriptor() {} + MMappedFileDescriptor() = default; virtual ~MMappedFileDescriptor(); @@ -40,10 +40,11 @@ public: void set(int fd_, size_t offset_, size_t length_); void set(int fd_, size_t offset_); -protected: MMappedFileDescriptor(const MMappedFileDescriptor &) = delete; MMappedFileDescriptor(MMappedFileDescriptor &&) = delete; +protected: + void init(); int fd = -1; diff --git a/src/IO/MemoryReadWriteBuffer.h b/src/IO/MemoryReadWriteBuffer.h index f9c11084f62..bcaf9a9a965 100644 --- a/src/IO/MemoryReadWriteBuffer.h +++ b/src/IO/MemoryReadWriteBuffer.h @@ -18,7 +18,7 @@ class MemoryWriteBuffer : public WriteBuffer, public IReadableWriteBuffer, boost public: /// Use max_total_size_ = 0 for unlimited storage - MemoryWriteBuffer( + explicit MemoryWriteBuffer( size_t max_total_size_ = 0, size_t initial_chunk_size_ = DBMS_DEFAULT_BUFFER_SIZE, double growth_rate_ = 2.0, diff --git a/src/IO/MySQLPacketPayloadWriteBuffer.h b/src/IO/MySQLPacketPayloadWriteBuffer.h index f54bec06dfb..d4ce8a8955e 100644 --- a/src/IO/MySQLPacketPayloadWriteBuffer.h +++ b/src/IO/MySQLPacketPayloadWriteBuffer.h @@ -13,7 +13,7 @@ class MySQLPacketPayloadWriteBuffer : public WriteBuffer public: MySQLPacketPayloadWriteBuffer(WriteBuffer & out_, size_t payload_length_, uint8_t & sequence_id_); - bool remainingPayloadSize() { return total_left; } + bool remainingPayloadSize() const { return total_left; } protected: void nextImpl() override; diff --git a/src/IO/NullWriteBuffer.h b/src/IO/NullWriteBuffer.h index 233268474d3..615a9bf5cef 100644 --- a/src/IO/NullWriteBuffer.h +++ b/src/IO/NullWriteBuffer.h @@ -11,7 +11,7 @@ namespace DB class NullWriteBuffer : public BufferWithOwnMemory, boost::noncopyable { public: - NullWriteBuffer(size_t buf_size = 16<<10, char * existing_memory = nullptr, size_t alignment = false); + explicit NullWriteBuffer(size_t buf_size = 16<<10, char * existing_memory = nullptr, size_t alignment = false); void nextImpl() override; }; diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp new file mode 100644 index 00000000000..7fa10c160ad --- /dev/null +++ b/src/IO/ParallelReadBuffer.cpp @@ -0,0 +1,290 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int SEEK_POSITION_OUT_OF_BOUND; + +} + +ParallelReadBuffer::ParallelReadBuffer( + std::unique_ptr reader_factory_, + ThreadPool * pool_, + size_t max_working_readers_, + WorkerSetup worker_setup_, + WorkerCleanup worker_cleanup_) + : SeekableReadBufferWithSize(nullptr, 0) + , pool(pool_) + , max_working_readers(max_working_readers_) + , reader_factory(std::move(reader_factory_)) + , worker_setup(std::move(worker_setup_)) + , worker_cleanup(std::move(worker_cleanup_)) +{ + std::unique_lock lock{mutex}; + addReaders(lock); +} + +bool ParallelReadBuffer::addReaderToPool(std::unique_lock & /*buffer_lock*/) +{ + auto reader = reader_factory->getReader(); + if (!reader) + { + return false; + } + + auto worker = read_workers.emplace_back(std::make_shared(std::move(reader))); + + pool->scheduleOrThrow( + [&, this, worker = std::move(worker)]() mutable + { + ThreadStatus thread_status; + + { + std::lock_guard lock{mutex}; + ++active_working_reader; + } + + SCOPE_EXIT({ + worker_cleanup(thread_status); + + std::lock_guard lock{mutex}; + --active_working_reader; + if (active_working_reader == 0) + { + readers_done.notify_all(); + } + }); + worker_setup(thread_status); + + readerThreadFunction(std::move(worker)); + }); + return true; +} + +void ParallelReadBuffer::addReaders(std::unique_lock & buffer_lock) +{ + while (read_workers.size() < max_working_readers && addReaderToPool(buffer_lock)) + ; +} + +off_t ParallelReadBuffer::seek(off_t offset, int whence) +{ + if (whence != SEEK_SET) + throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + if (offset < 0) + throw Exception("Seek position is out of bounds. Offset: " + std::to_string(offset), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + + if (!working_buffer.empty() && static_cast(offset) >= current_position - working_buffer.size() && offset < current_position) + { + pos = working_buffer.end() - (current_position - offset); + assert(pos >= working_buffer.begin()); + assert(pos <= working_buffer.end()); + + return offset; + } + + std::unique_lock lock{mutex}; + const auto offset_is_in_range + = [&](const auto & range) { return static_cast(offset) >= range.left && static_cast(offset) <= *range.right; }; + + while (!read_workers.empty() && (offset < current_position || !offset_is_in_range(read_workers.front()->range))) + { + read_workers.front()->cancel = true; + read_workers.pop_front(); + } + + if (!read_workers.empty()) + { + auto & front_worker = read_workers.front(); + auto & segments = front_worker->segments; + current_position = front_worker->range.left; + while (true) + { + next_condvar.wait(lock, [&] { return emergency_stop || !segments.empty(); }); + + if (emergency_stop) + handleEmergencyStop(); + + auto next_segment = front_worker->nextSegment(); + if (static_cast(offset) < current_position + next_segment.size()) + { + current_segment = std::move(next_segment); + working_buffer = internal_buffer = Buffer(current_segment.data(), current_segment.data() + current_segment.size()); + current_position += current_segment.size(); + pos = working_buffer.end() - (current_position - offset); + addReaders(lock); + return offset; + } + + current_position += next_segment.size(); + } + } + + lock.unlock(); + finishAndWait(); + + reader_factory->seek(offset, whence); + all_completed = false; + read_workers.clear(); + + current_position = offset; + resetWorkingBuffer(); + + emergency_stop = false; + + lock.lock(); + addReaders(lock); + return offset; +} + +std::optional ParallelReadBuffer::getTotalSize() +{ + std::lock_guard lock{mutex}; + return reader_factory->getTotalSize(); +} + +off_t ParallelReadBuffer::getPosition() +{ + return current_position - available(); +} + +bool ParallelReadBuffer::currentWorkerReady() const +{ + assert(!read_workers.empty()); + return read_workers.front()->finished || !read_workers.front()->segments.empty(); +} + +bool ParallelReadBuffer::currentWorkerCompleted() const +{ + assert(!read_workers.empty()); + return read_workers.front()->finished && read_workers.front()->segments.empty(); +} + +void ParallelReadBuffer::handleEmergencyStop() +{ + // this can only be called from the main thread when there is an exception + assert(background_exception); + if (background_exception) + std::rethrow_exception(background_exception); +} + +bool ParallelReadBuffer::nextImpl() +{ + if (all_completed) + return false; + + while (true) + { + std::unique_lock lock(mutex); + next_condvar.wait( + lock, + [this]() + { + /// Check if no more readers left or current reader can be processed + return emergency_stop || currentWorkerReady(); + }); + + bool worker_removed = false; + /// Remove completed units + while (!read_workers.empty() && currentWorkerCompleted() && !emergency_stop) + { + read_workers.pop_front(); + worker_removed = true; + } + + if (emergency_stop) + handleEmergencyStop(); + + if (worker_removed) + addReaders(lock); + + /// All readers processed, stop + if (read_workers.empty()) + { + all_completed = true; + return false; + } + + auto & front_worker = read_workers.front(); + /// Read data from first segment of the first reader + if (!front_worker->segments.empty()) + { + current_segment = front_worker->nextSegment(); + if (currentWorkerCompleted()) + { + read_workers.pop_front(); + all_completed = !addReaderToPool(lock) && read_workers.empty(); + } + break; + } + } + working_buffer = internal_buffer = Buffer(current_segment.data(), current_segment.data() + current_segment.size()); + current_position += working_buffer.size(); + return true; +} + +void ParallelReadBuffer::readerThreadFunction(ReadWorkerPtr read_worker) +{ + try + { + while (!emergency_stop && !read_worker->cancel) + { + if (!read_worker->reader->next()) + throw Exception("Failed to read all the data from the reader", ErrorCodes::LOGICAL_ERROR); + + if (emergency_stop || read_worker->cancel) + break; + + Buffer buffer = read_worker->reader->buffer(); + size_t bytes_to_copy = std::min(buffer.size(), read_worker->bytes_left); + Segment new_segment(bytes_to_copy, &arena); + memcpy(new_segment.data(), buffer.begin(), bytes_to_copy); + read_worker->reader->ignore(bytes_to_copy); + read_worker->bytes_left -= bytes_to_copy; + { + /// New data ready to be read + std::lock_guard lock(mutex); + read_worker->segments.emplace_back(std::move(new_segment)); + read_worker->finished = read_worker->bytes_left == 0; + next_condvar.notify_all(); + } + + if (read_worker->finished) + { + break; + } + } + } + catch (...) + { + onBackgroundException(); + } +} + +void ParallelReadBuffer::onBackgroundException() +{ + std::lock_guard lock(mutex); + if (!background_exception) + { + background_exception = std::current_exception(); + } + emergency_stop = true; + next_condvar.notify_all(); +} + +void ParallelReadBuffer::finishAndWait() +{ + emergency_stop = true; + + std::unique_lock lock{mutex}; + readers_done.wait(lock, [&] { return active_working_reader == 0; }); +} + +} diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h new file mode 100644 index 00000000000..7b364205e8e --- /dev/null +++ b/src/IO/ParallelReadBuffer.h @@ -0,0 +1,174 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +/** + * Reads from multiple ReadBuffers in parallel. + * Preserves order of readers obtained from ReadBufferFactory. + * + * It consumes multiple readers and yields data from them in order as it passed. + * Each working reader save segments of data to internal queue. + * + * ParallelReadBuffer in nextImpl method take first available segment from first reader in deque and fed it to user. + * When first reader finish reading, they will be removed from worker deque and data from next reader consumed. + * + * Number of working readers limited by max_working_readers. + */ +class ParallelReadBuffer : public SeekableReadBufferWithSize +{ +private: + /// Blocks until data occurred in the first reader or this reader indicate finishing + /// Finished readers removed from queue and data from next readers processed + bool nextImpl() override; + + class Segment : private boost::noncopyable + { + public: + Segment(size_t size_, SynchronizedArenaWithFreeLists * arena_) : arena(arena_), m_data(arena->alloc(size_)), m_size(size_) { } + + Segment() = default; + + Segment(Segment && other) noexcept : arena(other.arena) + { + std::swap(m_data, other.m_data); + std::swap(m_size, other.m_size); + } + + Segment & operator=(Segment && other) noexcept + { + arena = other.arena; + std::swap(m_data, other.m_data); + std::swap(m_size, other.m_size); + return *this; + } + + ~Segment() + { + if (m_data) + { + arena->free(m_data, m_size); + } + } + + auto data() const noexcept { return m_data; } + auto size() const noexcept { return m_size; } + + private: + SynchronizedArenaWithFreeLists * arena{nullptr}; + char * m_data{nullptr}; + size_t m_size{0}; + }; + +public: + class ReadBufferFactory + { + public: + virtual SeekableReadBufferPtr getReader() = 0; + virtual ~ReadBufferFactory() = default; + virtual off_t seek(off_t off, int whence) = 0; + virtual std::optional getTotalSize() = 0; + }; + + using WorkerSetup = std::function; + using WorkerCleanup = std::function; + explicit ParallelReadBuffer( + std::unique_ptr reader_factory_, + ThreadPool * pool, + size_t max_working_readers, + WorkerSetup worker_setup = {}, + WorkerCleanup worker_cleanup = {}); + + ~ParallelReadBuffer() override { finishAndWait(); } + + off_t seek(off_t off, int whence) override; + std::optional getTotalSize() override; + off_t getPosition() override; + +private: + /// Reader in progress with a list of read segments + struct ReadWorker + { + explicit ReadWorker(SeekableReadBufferPtr reader_) : reader(std::move(reader_)), range(reader->getRemainingReadRange()) + { + assert(range.right); + bytes_left = *range.right - range.left + 1; + } + + Segment nextSegment() + { + assert(!segments.empty()); + auto next_segment = std::move(segments.front()); + segments.pop_front(); + range.left += next_segment.size(); + return next_segment; + } + + SeekableReadBufferPtr reader; + std::deque segments; + bool finished{false}; + SeekableReadBuffer::Range range; + size_t bytes_left{0}; + std::atomic_bool cancel{false}; + }; + + using ReadWorkerPtr = std::shared_ptr; + + /// First worker in deque have new data or processed all available amount + bool currentWorkerReady() const; + /// First worker in deque processed and flushed all data + bool currentWorkerCompleted() const; + + void handleEmergencyStop(); + + void addReaders(std::unique_lock & buffer_lock); + bool addReaderToPool(std::unique_lock & buffer_lock); + + /// Process read_worker, read data and save into internal segments queue + void readerThreadFunction(ReadWorkerPtr read_worker); + + void onBackgroundException(); + void finishAndWait(); + + SynchronizedArenaWithFreeLists arena; + + Segment current_segment; + + ThreadPool * pool; + size_t max_working_readers; + size_t active_working_reader{0}; + // Triggered when all reader workers are done + std::condition_variable readers_done; + + std::unique_ptr reader_factory; + + WorkerSetup worker_setup; + WorkerCleanup worker_cleanup; + + /** + * FIFO queue of readers. + * Each worker contains reader itself and downloaded segments. + * When reader read all available data it will be removed from + * deque and data from next reader will be consumed to user. + */ + std::deque read_workers; + + std::mutex mutex; + /// Triggered when new data available + std::condition_variable next_condvar; + + std::exception_ptr background_exception = nullptr; + std::atomic_bool emergency_stop{false}; + + off_t current_position{0}; + + bool all_completed{false}; +}; + +} diff --git a/src/IO/Progress.cpp b/src/IO/Progress.cpp index ebc7d04e86e..1d16f54de7b 100644 --- a/src/IO/Progress.cpp +++ b/src/IO/Progress.cpp @@ -126,7 +126,7 @@ ProgressValues Progress::fetchAndResetPiecewiseAtomically() return res; } -Progress & Progress::operator=(Progress && other) +Progress & Progress::operator=(Progress && other) noexcept { read_rows = other.read_rows.load(std::memory_order_relaxed); read_bytes = other.read_bytes.load(std::memory_order_relaxed); diff --git a/src/IO/Progress.h b/src/IO/Progress.h index c00eea98ff4..4f1a3df0ffd 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -56,7 +56,7 @@ struct FileProgress size_t read_bytes; size_t total_bytes_to_read; - FileProgress(size_t read_bytes_, size_t total_bytes_to_read_ = 0) : read_bytes(read_bytes_), total_bytes_to_read(total_bytes_to_read_) {} + explicit FileProgress(size_t read_bytes_, size_t total_bytes_to_read_ = 0) : read_bytes(read_bytes_), total_bytes_to_read(total_bytes_to_read_) {} }; @@ -111,9 +111,9 @@ struct Progress ProgressValues fetchAndResetPiecewiseAtomically(); - Progress & operator=(Progress && other); + Progress & operator=(Progress && other) noexcept; - Progress(Progress && other) + Progress(Progress && other) noexcept { *this = std::move(other); } diff --git a/src/IO/ReadBuffer.h b/src/IO/ReadBuffer.h index b6927ffcf0e..b620f0c49c6 100644 --- a/src/IO/ReadBuffer.h +++ b/src/IO/ReadBuffer.h @@ -50,6 +50,29 @@ public: // FIXME: behavior differs greately from `BufferBase::set()` and it's very confusing. void set(Position ptr, size_t size) { BufferBase::set(ptr, size, 0); working_buffer.resize(0); } + /// Set buffer to given piece of memory but with certain bytes ignored from beginning. + /// + /// internal_buffer: |__________________| + /// working_buffer: |xxxxx|____________| + /// ^ ^ + /// bytes_to_ignore + /// + /// It's used for lazy seek. We also have another lazy seek mechanism that uses + /// `nextimpl_working_buffer_offset` to set offset in `next` method. It's important that we + /// don't do double lazy seek, which means `nextimpl_working_buffer_offset` should be zero. It's + /// useful to keep internal_buffer points to the real span of the underlying memory, because its + /// size might be used to allocate other buffers. It's also important to have pos starts at + /// working_buffer.begin(), because some buffers assume this condition to be true and uses + /// offset() to check read bytes. + void setWithBytesToIgnore(Position ptr, size_t size, size_t bytes_to_ignore) + { + assert(bytes_to_ignore < size); + assert(nextimpl_working_buffer_offset == 0); + internal_buffer = Buffer(ptr, ptr + size); + working_buffer = Buffer(ptr + bytes_to_ignore, ptr + size); + pos = ptr + bytes_to_ignore; + } + /** read next data and fill a buffer with it; set position to the beginning; * return `false` in case of end, `true` otherwise; throw an exception, if something is wrong */ @@ -206,9 +229,11 @@ public: virtual void prefetch() {} /** - * For reading from remote filesystem, when it matters how much we read. + * Set upper bound for read range [..., position). + * Required for reading from remote filesystem, when it matters how much we read. */ virtual void setReadUntilPosition(size_t /* position */) {} + virtual void setReadUntilEnd() {} protected: diff --git a/src/IO/ReadBufferFromAzureBlobStorage.h b/src/IO/ReadBufferFromAzureBlobStorage.h index 53749ad3199..78d973747ba 100644 --- a/src/IO/ReadBufferFromAzureBlobStorage.h +++ b/src/IO/ReadBufferFromAzureBlobStorage.h @@ -33,6 +33,8 @@ public: bool nextImpl() override; + size_t getFileOffsetOfBufferEnd() const override { return offset; } + private: void initialize(); diff --git a/src/IO/ReadBufferFromEmptyFile.h b/src/IO/ReadBufferFromEmptyFile.h index 311aee1559b..0a14c07dd5c 100644 --- a/src/IO/ReadBufferFromEmptyFile.h +++ b/src/IO/ReadBufferFromEmptyFile.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB { diff --git a/src/IO/ReadBufferFromFile.h b/src/IO/ReadBufferFromFile.h index ff19fa40fdf..52b18b94616 100644 --- a/src/IO/ReadBufferFromFile.h +++ b/src/IO/ReadBufferFromFile.h @@ -49,6 +49,10 @@ public: { return file_name; } + + Range getRemainingReadRange() const override { return Range{ .left = file_offset_of_buffer_end, .right = std::nullopt }; } + + size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; } }; @@ -57,7 +61,7 @@ public: class ReadBufferFromFilePRead : public ReadBufferFromFile { public: - ReadBufferFromFilePRead( + explicit ReadBufferFromFilePRead( const std::string & file_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, int flags = -1, @@ -80,7 +84,7 @@ private: OpenedFileCache::OpenedFilePtr file; public: - ReadBufferFromFilePReadWithDescriptorsCache( + explicit ReadBufferFromFilePReadWithDescriptorsCache( const std::string & file_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, int flags = -1, diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index 188cdd709b5..ba1502fb9aa 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -27,7 +27,7 @@ protected: std::string getFileName() const override; public: - ReadBufferFromFileDescriptor( + explicit ReadBufferFromFileDescriptor( int fd_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, @@ -70,7 +70,7 @@ private: class ReadBufferFromFileDescriptorPRead : public ReadBufferFromFileDescriptor { public: - ReadBufferFromFileDescriptorPRead( + explicit ReadBufferFromFileDescriptorPRead( int fd_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/ReadBufferFromMemory.h b/src/IO/ReadBufferFromMemory.h index 28d78f75f1c..dc5c464604b 100644 --- a/src/IO/ReadBufferFromMemory.h +++ b/src/IO/ReadBufferFromMemory.h @@ -12,7 +12,8 @@ namespace DB class ReadBufferFromMemory : public SeekableReadBuffer { public: - template > + template + requires (sizeof(CharT) == 1) ReadBufferFromMemory(const CharT * buf, size_t size) : SeekableReadBuffer(const_cast(reinterpret_cast(buf)), size, 0) {} diff --git a/src/IO/ReadBufferFromPocoSocket.cpp b/src/IO/ReadBufferFromPocoSocket.cpp index 527b68e623a..988ad75cdf4 100644 --- a/src/IO/ReadBufferFromPocoSocket.cpp +++ b/src/IO/ReadBufferFromPocoSocket.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -35,6 +37,12 @@ bool ReadBufferFromPocoSocket::nextImpl() ssize_t bytes_read = 0; Stopwatch watch; + SCOPE_EXIT({ + // / NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one + ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::NetworkReceiveBytes, bytes_read); + }); + /// Add more details to exceptions. try { @@ -66,10 +74,6 @@ bool ReadBufferFromPocoSocket::nextImpl() if (bytes_read < 0) throw NetException("Cannot read from socket (" + peer_address.toString() + ")", ErrorCodes::CANNOT_READ_FROM_SOCKET); - /// NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one - ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::NetworkReceiveBytes, bytes_read); - if (bytes_read) working_buffer.resize(bytes_read); else diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 869432b9484..93bbe02c9cd 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -42,7 +42,8 @@ ReadBufferFromS3::ReadBufferFromS3( UInt64 max_single_read_retries_, const ReadSettings & settings_, bool use_external_buffer_, - size_t read_until_position_) + size_t read_until_position_, + bool restricted_seek_) : SeekableReadBufferWithSize(nullptr, 0) , client_ptr(std::move(client_ptr_)) , bucket(bucket_) @@ -51,6 +52,7 @@ ReadBufferFromS3::ReadBufferFromS3( , read_settings(settings_) , use_external_buffer(use_external_buffer_) , read_until_position(read_until_position_) + , restricted_seek(restricted_seek_) { } @@ -152,10 +154,14 @@ bool ReadBufferFromS3::nextImpl() off_t ReadBufferFromS3::seek(off_t offset_, int whence) { - bool restricted_seek = read_type == SeekableReadBufferWithSize::ReadType::DISK_READ; + if (offset_ == offset && whence == SEEK_SET) + return offset; if (impl && restricted_seek) - throw Exception("Seek is allowed only before first read attempt from the buffer.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + throw Exception( + ErrorCodes::CANNOT_SEEK_THROUGH_FILE, + "Seek is allowed only before first read attempt from the buffer (current offset: {}, new offset: {}, reading until position: {}, available: {})", + offset, offset_, read_until_position, available()); if (whence != SEEK_SET) throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); @@ -219,6 +225,15 @@ off_t ReadBufferFromS3::getPosition() return offset - available(); } +void ReadBufferFromS3::setReadUntilPosition(size_t position) +{ + if (position != static_cast(read_until_position)) + { + read_until_position = position; + impl.reset(); + } +} + std::unique_ptr ReadBufferFromS3::initialize() { Aws::S3::Model::GetObjectRequest req; @@ -249,7 +264,9 @@ std::unique_ptr ReadBufferFromS3::initialize() if (outcome.IsSuccess()) { read_result = outcome.GetResultWithOwnership(); - return std::make_unique(read_result.GetBody(), read_settings.remote_fs_buffer_size); + + size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; + return std::make_unique(read_result.GetBody(), buffer_size); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index e903ba11118..157b6d46b6d 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -31,6 +31,7 @@ private: String key; UInt64 max_single_read_retries; off_t offset = 0; + Aws::S3::Model::GetObjectResult read_result; std::unique_ptr impl; @@ -44,7 +45,8 @@ public: UInt64 max_single_read_retries_, const ReadSettings & settings_, bool use_external_buffer = false, - size_t read_until_position_ = 0); + size_t read_until_position_ = 0, + bool restricted_seek_ = false); bool nextImpl() override; @@ -54,6 +56,12 @@ public: std::optional getTotalSize() override; + void setReadUntilPosition(size_t position) override; + + Range getRemainingReadRange() const override { return Range{ .left = static_cast(offset), .right = read_until_position }; } + + size_t getFileOffsetOfBufferEnd() const override { return offset; } + private: std::unique_ptr initialize(); @@ -62,6 +70,10 @@ private: bool use_external_buffer; off_t read_until_position = 0; + + /// There is different seek policy for disk seek and for non-disk seek + /// (non-disk seek is applied for seekable input formats: orc, arrow, parquet). + bool restricted_seek; }; } diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index a6125818155..e086f16be54 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -248,7 +248,7 @@ void readString(String & s, ReadBuffer & buf) } template void readStringInto>(PaddedPODArray & s, ReadBuffer & buf); - +template void readStringInto(String & s, ReadBuffer & buf); template void readStringUntilEOFInto(Vector & s, ReadBuffer & buf) @@ -580,6 +580,7 @@ void readQuotedStringWithSQLStyle(String & s, ReadBuffer & buf) template void readQuotedStringInto(PaddedPODArray & s, ReadBuffer & buf); +template void readQuotedStringInto(String & s, ReadBuffer & buf); template void readDoubleQuotedStringInto(NullOutput & s, ReadBuffer & buf); void readDoubleQuotedString(String & s, ReadBuffer & buf) @@ -782,6 +783,68 @@ template bool readJSONStringInto, bool>(PaddedPODArray(NullOutput & s, ReadBuffer & buf); template void readJSONStringInto(String & s, ReadBuffer & buf); +template +ReturnType readJSONObjectPossiblyInvalid(Vector & s, ReadBuffer & buf) +{ + static constexpr bool throw_exception = std::is_same_v; + + auto error = [](const char * message [[maybe_unused]], int code [[maybe_unused]]) + { + if constexpr (throw_exception) + throw ParsingException(message, code); + return ReturnType(false); + }; + + if (buf.eof() || *buf.position() != '{') + return error("JSON should start from opening curly bracket", ErrorCodes::INCORRECT_DATA); + + s.push_back(*buf.position()); + ++buf.position(); + + Int64 balance = 1; + bool quotes = false; + + while (!buf.eof()) + { + char * next_pos = find_first_symbols<'\\', '{', '}', '"'>(buf.position(), buf.buffer().end()); + appendToStringOrVector(s, buf, next_pos); + buf.position() = next_pos; + + if (!buf.hasPendingData()) + continue; + + s.push_back(*buf.position()); + + if (*buf.position() == '\\') + { + ++buf.position(); + if (!buf.eof()) + { + s.push_back(*buf.position()); + ++buf.position(); + } + + continue; + } + + if (*buf.position() == '"') + quotes = !quotes; + else if (!quotes) // can be only '{' or '}' + balance += *buf.position() == '{' ? 1 : -1; + + ++buf.position(); + + if (balance == 0) + return ReturnType(true); + + if (balance < 0) + break; + } + + return error("JSON should have equal number of opening and closing brackets", ErrorCodes::INCORRECT_DATA); +} + +template void readJSONObjectPossiblyInvalid(String & s, ReadBuffer & buf); template ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 5d580f6b130..9396e1d32f7 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -106,7 +106,7 @@ inline void readChar(char & x, ReadBuffer & buf) template inline void readPODBinary(T & x, ReadBuffer & buf) { - buf.readStrict(reinterpret_cast(&x), sizeof(x)); + buf.readStrict(reinterpret_cast(&x), sizeof(x)); /// NOLINT } template @@ -601,6 +601,12 @@ bool tryReadJSONStringInto(Vector & s, ReadBuffer & buf) return readJSONStringInto(s, buf); } +/// Reads chunk of data between {} in that way, +/// that it has balanced parentheses sequence of {}. +/// So, it may form a JSON object, but it can be incorrenct. +template +ReturnType readJSONObjectPossiblyInvalid(Vector & s, ReadBuffer & buf); + template void readStringUntilWhitespaceInto(Vector & s, ReadBuffer & buf); @@ -611,7 +617,7 @@ void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf); struct NullOutput { void append(const char *, size_t) {} - void push_back(char) {} + void push_back(char) {} /// NOLINT }; void parseUUID(const UInt8 * src36, UInt8 * dst16); @@ -686,6 +692,16 @@ inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf) return readDateTextFallback(date, buf); } +inline void convertToDayNum(DayNum & date, ExtendedDayNum & from) +{ + if (unlikely(from < 0)) + date = 0; + else if (unlikely(from > 0xFFFF)) + date = 0xFFFF; + else + date = from; +} + template inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf) { @@ -698,7 +714,8 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf) else if (!readDateTextImpl(local_date, buf)) return false; - date = DateLUT::instance().makeDayNum(local_date.year(), local_date.month(), local_date.day()); + ExtendedDayNum ret = DateLUT::instance().makeDayNum(local_date.year(), local_date.month(), local_date.day()); + convertToDayNum(date,ret); return ReturnType(true); } @@ -955,8 +972,8 @@ inline void readDateTimeText(LocalDateTime & datetime, ReadBuffer & buf) /// Generic methods to read value in native binary format. template -inline std::enable_if_t, void> -readBinary(T & x, ReadBuffer & buf) { readPODBinary(x, buf); } +requires is_arithmetic_v +inline void readBinary(T & x, ReadBuffer & buf) { readPODBinary(x, buf); } inline void readBinary(String & x, ReadBuffer & buf) { readStringBinary(x, buf); } inline void readBinary(Int128 & x, ReadBuffer & buf) { readPODBinary(x, buf); } @@ -971,8 +988,8 @@ inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); template -inline std::enable_if_t && (sizeof(T) <= 8), void> -readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. +requires is_arithmetic_v && (sizeof(T) <= 8) +inline void readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. { readPODBinary(x, buf); @@ -987,8 +1004,8 @@ readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian archi } template -inline std::enable_if_t, void> -readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. +requires is_big_int_v +inline void readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. { for (size_t i = 0; i != std::size(x.items); ++i) { @@ -1023,8 +1040,8 @@ inline void readText(UUID & x, ReadBuffer & buf) { readUUIDText(x, buf); } /// Generic methods to read value in text format, /// possibly in single quotes (only for data types that use quotes in VALUES format of INSERT statement in SQL). template -inline std::enable_if_t, void> -readQuoted(T & x, ReadBuffer & buf) { readText(x, buf); } +requires is_arithmetic_v +inline void readQuoted(T & x, ReadBuffer & buf) { readText(x, buf); } inline void readQuoted(String & x, ReadBuffer & buf) { readQuotedString(x, buf); } @@ -1052,8 +1069,8 @@ inline void readQuoted(UUID & x, ReadBuffer & buf) /// Same as above, but in double quotes. template -inline std::enable_if_t, void> -readDoubleQuoted(T & x, ReadBuffer & buf) { readText(x, buf); } +requires is_arithmetic_v +inline void readDoubleQuoted(T & x, ReadBuffer & buf) { readText(x, buf); } inline void readDoubleQuoted(String & x, ReadBuffer & buf) { readDoubleQuotedString(x, buf); } @@ -1090,7 +1107,8 @@ inline void readCSVSimple(T & x, ReadBuffer & buf) } template -inline std::enable_if_t, void> readCSV(T & x, ReadBuffer & buf) +requires is_arithmetic_v +inline void readCSV(T & x, ReadBuffer & buf) { readCSVSimple(x, buf); } @@ -1267,7 +1285,6 @@ inline void readTextWithSizeSuffix(T & x, ReadBuffer & buf) default: return; } - return; } /// Read something from text format and trying to parse the suffix. diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index e290cbab36b..e321eecf104 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -76,9 +77,13 @@ struct ReadSettings size_t remote_fs_read_max_backoff_ms = 10000; size_t remote_fs_read_backoff_max_tries = 4; + bool remote_fs_enable_cache = true; + size_t remote_fs_cache_max_wait_sec = 1; size_t remote_read_min_bytes_for_seek = DBMS_DEFAULT_BUFFER_SIZE; + FileCachePtr remote_fs_cache; + size_t http_max_tries = 1; size_t http_retry_initial_backoff_ms = 100; size_t http_retry_max_backoff_ms = 1600; diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 4e08a595484..061dd772212 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -1,32 +1,33 @@ #pragma once #include -#include -#include #include #include +#include #include #include #include #include +#include +#include +#include #include #include #include #include #include #include +#include #include #include #include #include #include -#include -#include namespace ProfileEvents { - extern const Event ReadBufferSeekCancelConnection; +extern const Event ReadBufferSeekCancelConnection; } namespace DB @@ -48,7 +49,7 @@ class UpdatableSessionBase { protected: SessionPtr session; - UInt64 redirects { 0 }; + UInt64 redirects{0}; Poco::URI initial_uri; ConnectionTimeouts timeouts; UInt64 max_redirects; @@ -56,19 +57,12 @@ protected: public: virtual void buildNewSession(const Poco::URI & uri) = 0; - explicit UpdatableSessionBase(const Poco::URI uri, - const ConnectionTimeouts & timeouts_, - UInt64 max_redirects_) - : initial_uri { uri } - , timeouts { timeouts_ } - , max_redirects { max_redirects_ } + explicit UpdatableSessionBase(const Poco::URI uri, const ConnectionTimeouts & timeouts_, UInt64 max_redirects_) + : initial_uri{uri}, timeouts{timeouts_}, max_redirects{max_redirects_} { } - SessionPtr getSession() - { - return session; - } + SessionPtr getSession() { return session; } void updateSession(const Poco::URI & uri) { @@ -99,7 +93,7 @@ namespace detail /// HTTP range, including right bound [begin, end]. struct Range { - size_t begin = 0; + std::optional begin; std::optional end; }; @@ -115,7 +109,7 @@ namespace detail const Poco::Net::HTTPBasicCredentials & credentials; std::vector cookies; HTTPHeaderEntries http_header_entries; - RemoteHostFilter remote_host_filter; + const RemoteHostFilter * remote_host_filter = nullptr; std::function next_callback; size_t buffer_size; @@ -144,10 +138,9 @@ namespace detail return read_range.begin || read_range.end || retry_with_range_header; } - size_t getOffset() const - { - return read_range.begin + offset_from_begin_pos; - } + size_t getRangeBegin() const { return read_range.begin.value_or(0); } + + size_t getOffset() const { return getRangeBegin() + offset_from_begin_pos; } std::istream * callImpl(Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_) { @@ -161,7 +154,7 @@ namespace detail if (out_stream_callback) request.setChunkedTransferEncoding(true); - for (auto & http_header_entry: http_header_entries) + for (auto & http_header_entry : http_header_entries) request.set(std::get<0>(http_header_entry), std::get<1>(http_header_entry)); if (withPartialContent()) @@ -207,25 +200,14 @@ namespace detail std::optional getTotalSize() override { if (read_range.end) - return *read_range.end - read_range.begin; + return *read_range.end - getRangeBegin(); Poco::Net::HTTPResponse response; for (size_t i = 0; i < 10; ++i) { try { - call(response, Poco::Net::HTTPRequest::HTTP_HEAD); - - while (isRedirect(response.getStatus())) - { - Poco::URI uri_redirect(response.get("Location")); - remote_host_filter.checkURL(uri_redirect); - - session->updateSession(uri_redirect); - - istr = callImpl(uri_redirect, response, method); - } - + callWithRedirects(response, Poco::Net::HTTPRequest::HTTP_HEAD); break; } catch (const Poco::Exception & e) @@ -235,7 +217,7 @@ namespace detail } if (response.hasContentLength()) - read_range.end = read_range.begin + response.getContentLength(); + read_range.end = getRangeBegin() + response.getContentLength(); return read_range.end; } @@ -251,6 +233,21 @@ namespace detail InitializeError initialization_error = InitializeError::NONE; + private: + void setupExternalBuffer() + { + /** + * use_external_buffer -- means we read into the buffer which + * was passed to us from somewhere else. We do not check whether + * previously returned buffer was read or not (no hasPendingData() check is needed), + * because this branch means we are prefetching data, + * each nextImpl() call we can fill a different buffer. + */ + impl->set(internal_buffer.begin(), internal_buffer.size()); + assert(working_buffer.begin() != nullptr); + assert(!internal_buffer.empty()); + } + public: using NextCallback = std::function; using OutStreamCallback = std::function; @@ -265,7 +262,7 @@ namespace detail const ReadSettings & settings_ = {}, HTTPHeaderEntries http_header_entries_ = {}, Range read_range_ = {}, - const RemoteHostFilter & remote_host_filter_ = {}, + const RemoteHostFilter * remote_host_filter_ = nullptr, bool delay_initialization = false, bool use_external_buffer_ = false, bool http_skip_not_found_url_ = false) @@ -275,7 +272,7 @@ namespace detail , session {session_} , out_stream_callback {out_stream_callback_} , credentials {credentials_} - , http_header_entries {http_header_entries_} + , http_header_entries {std::move(http_header_entries_)} , remote_host_filter {remote_host_filter_} , buffer_size {buffer_size_} , use_external_buffer {use_external_buffer_} @@ -286,18 +283,21 @@ namespace detail { if (settings.http_max_tries <= 0 || settings.http_retry_initial_backoff_ms <= 0 || settings.http_retry_initial_backoff_ms >= settings.http_retry_max_backoff_ms) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Invalid setting for http backoff, " - "must be http_max_tries >= 1 (current is {}) and " - "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})", - settings.http_max_tries, settings.http_retry_initial_backoff_ms, settings.http_retry_max_backoff_ms); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Invalid setting for http backoff, " + "must be http_max_tries >= 1 (current is {}) and " + "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})", + settings.http_max_tries, + settings.http_retry_initial_backoff_ms, + settings.http_retry_max_backoff_ms); // Configure User-Agent if it not already set. const std::string user_agent = "User-Agent"; - auto iter = std::find_if(http_header_entries.begin(), http_header_entries.end(), [&user_agent](const HTTPHeaderEntry & entry) - { - return std::get<0>(entry) == user_agent; - }); + auto iter = std::find_if( + http_header_entries.begin(), + http_header_entries.end(), + [&user_agent](const HTTPHeaderEntry & entry) { return std::get<0>(entry) == user_agent; }); if (iter == http_header_entries.end()) { @@ -312,7 +312,36 @@ namespace detail } } - void call(Poco::Net::HTTPResponse & response, const String & method_) + static bool isRetriableError(const Poco::Net::HTTPResponse::HTTPStatus http_status) noexcept + { + constexpr std::array non_retriable_errors{ + Poco::Net::HTTPResponse::HTTPStatus::HTTP_BAD_REQUEST, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN, + Poco::Net::HTTPResponse::HTTPStatus::HTTP_METHOD_NOT_ALLOWED}; + + return std::all_of( + non_retriable_errors.begin(), non_retriable_errors.end(), [&](const auto status) { return http_status != status; }); + } + + void callWithRedirects(Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false) + { + call(response, method_, throw_on_all_errors); + + while (isRedirect(response.getStatus())) + { + Poco::URI uri_redirect(response.get("Location")); + if (remote_host_filter) + remote_host_filter->checkURL(uri_redirect); + + session->updateSession(uri_redirect); + + istr = callImpl(uri_redirect, response, method); + } + } + + void call(Poco::Net::HTTPResponse & response, const String & method_, bool throw_on_all_errors = false) { try { @@ -320,11 +349,22 @@ namespace detail } catch (...) { - if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND - && http_skip_not_found_url) + if (throw_on_all_errors) + { + throw; + } + + auto http_status = response.getStatus(); + + if (http_status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND && http_skip_not_found_url) { initialization_error = InitializeError::SKIP_NOT_FOUND_URL; } + else if (!isRetriableError(http_status)) + { + initialization_error = InitializeError::NON_RETRIABLE_ERROR; + exception = std::current_exception(); + } else { throw; @@ -348,7 +388,8 @@ namespace detail while (isRedirect(response.getStatus())) { Poco::URI uri_redirect(response.get("Location")); - remote_host_filter.checkURL(uri_redirect); + if (remote_host_filter) + remote_host_filter->checkURL(uri_redirect); session->updateSession(uri_redirect); @@ -359,12 +400,14 @@ namespace detail if (withPartialContent() && response.getStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT) { /// Having `200 OK` instead of `206 Partial Content` is acceptable in case we retried with range.begin == 0. - if (read_range.begin) + if (read_range.begin && *read_range.begin != 0) { if (!exception) - exception = std::make_exception_ptr( - Exception(ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, - "Cannot read with range: [{}, {}]", read_range.begin, read_range.end ? *read_range.end : '-')); + exception = std::make_exception_ptr(Exception( + ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, + "Cannot read with range: [{}, {}]", + *read_range.begin, + read_range.end ? *read_range.end : '-')); initialization_error = InitializeError::NON_RETRIABLE_ERROR; return; @@ -373,12 +416,12 @@ namespace detail { /// We could have range.begin == 0 and range.end != 0 in case of DiskWeb and failing to read with partial content /// will affect only performance, so a warning is enough. - LOG_WARNING(log, "Unable to read with range header: [{}, {}]", read_range.begin, *read_range.end); + LOG_WARNING(log, "Unable to read with range header: [{}, {}]", getRangeBegin(), *read_range.end); } } if (!offset_from_begin_pos && !read_range.end && response.hasContentLength()) - read_range.end = read_range.begin + response.getContentLength(); + read_range.end = getRangeBegin() + response.getContentLength(); try { @@ -386,12 +429,7 @@ namespace detail if (use_external_buffer) { - /** - * See comment 30 lines below. - */ - impl->set(internal_buffer.begin(), internal_buffer.size()); - assert(working_buffer.begin() != nullptr); - assert(!internal_buffer.empty()); + setupExternalBuffer(); } } catch (const Poco::Exception & e) @@ -413,23 +451,17 @@ namespace detail if (next_callback) next_callback(count()); - if (read_range.end && getOffset() == read_range.end.value()) + if (read_range.end && getOffset() > read_range.end.value()) + { + assert(getOffset() == read_range.end.value() + 1); return false; + } if (impl) { if (use_external_buffer) { - /** - * use_external_buffer -- means we read into the buffer which - * was passed to us from somewhere else. We do not check whether - * previously returned buffer was read or not (no hasPendingData() check is needed), - * because this branch means we are prefetching data, - * each nextImpl() call we can fill a different buffer. - */ - impl->set(internal_buffer.begin(), internal_buffer.size()); - assert(working_buffer.begin() != nullptr); - assert(!internal_buffer.empty()); + setupExternalBuffer(); } else { @@ -464,10 +496,7 @@ namespace detail if (use_external_buffer) { - /// See comment 40 lines above. - impl->set(internal_buffer.begin(), internal_buffer.size()); - assert(working_buffer.begin() != nullptr); - assert(!internal_buffer.empty()); + setupExternalBuffer(); } } @@ -485,13 +514,18 @@ namespace detail if (!can_retry_request) throw; - LOG_ERROR(log, - "HTTP request to `{}` failed at try {}/{} with bytes read: {}/{}. " - "Error: {}. (Current backoff wait is {}/{} ms)", - uri.toString(), i + 1, settings.http_max_tries, - getOffset(), read_range.end ? toString(*read_range.end) : "unknown", - e.displayText(), - milliseconds_to_wait, settings.http_retry_max_backoff_ms); + LOG_ERROR( + log, + "HTTP request to `{}` failed at try {}/{} with bytes read: {}/{}. " + "Error: {}. (Current backoff wait is {}/{} ms)", + uri.toString(), + i + 1, + settings.http_max_tries, + getOffset(), + read_range.end ? toString(*read_range.end) : "unknown", + e.displayText(), + milliseconds_to_wait, + settings.http_retry_max_backoff_ms); retry_with_range_header = true; exception = std::current_exception(); @@ -516,10 +550,7 @@ namespace detail return true; } - off_t getPosition() override - { - return getOffset() - available(); - } + off_t getPosition() override { return getOffset() - available(); } off_t seek(off_t offset_, int whence) override { @@ -527,12 +558,11 @@ namespace detail throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); if (offset_ < 0) - throw Exception("Seek position is out of bounds. Offset: " + std::to_string(offset_), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + throw Exception( + "Seek position is out of bounds. Offset: " + std::to_string(offset_), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); off_t current_offset = getOffset(); - if (!working_buffer.empty() - && size_t(offset_) >= current_offset - working_buffer.size() - && offset_ < current_offset) + if (!working_buffer.empty() && size_t(offset_) >= current_offset - working_buffer.size() && offset_ < current_offset) { pos = working_buffer.end() - (current_offset - offset_); assert(pos >= working_buffer.begin()); @@ -554,7 +584,6 @@ namespace detail if (impl) { - ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection); impl.reset(); } @@ -567,6 +596,8 @@ namespace detail return offset_; } + SeekableReadBuffer::Range getRemainingReadRange() const override { return {getOffset(), read_range.end}; } + std::string getResponseCookie(const std::string & name, const std::string & def) const { for (const auto & cookie : cookies) @@ -586,10 +617,7 @@ namespace detail next_callback(count()); } - const std::string & getCompressionMethod() const - { - return content_encoding; - } + const std::string & getCompressionMethod() const { return content_encoding; } }; } @@ -598,19 +626,50 @@ class UpdatableSession : public UpdatableSessionBase using Parent = UpdatableSessionBase; public: - UpdatableSession( - const Poco::URI uri, - const ConnectionTimeouts & timeouts_, - const UInt64 max_redirects_) + UpdatableSession(const Poco::URI uri, const ConnectionTimeouts & timeouts_, const UInt64 max_redirects_) : Parent(uri, timeouts_, max_redirects_) { session = makeHTTPSession(initial_uri, timeouts); } - void buildNewSession(const Poco::URI & uri) override + void buildNewSession(const Poco::URI & uri) override { session = makeHTTPSession(uri, timeouts); } +}; + +class RangeGenerator +{ +public: + explicit RangeGenerator(size_t total_size_, size_t range_step_, size_t range_start = 0) + : from(range_start), range_step(range_step_), total_size(total_size_) { - session = makeHTTPSession(uri, timeouts); } + + size_t totalRanges() const { return static_cast(round(static_cast(total_size - from) / range_step)); } + + using Range = std::pair; + + // return upper exclusive range of values, i.e. [from_range, to_range> + std::optional nextRange() + { + if (from >= total_size) + { + return std::nullopt; + } + + auto to = from + range_step; + if (to >= total_size) + { + to = total_size; + } + + Range range{from, to}; + from = to; + return std::move(range); + } + +private: + size_t from; + size_t range_step; + size_t total_size; }; class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase> @@ -618,7 +677,7 @@ class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase>; public: - ReadWriteBufferFromHTTP( + ReadWriteBufferFromHTTP( Poco::URI uri_, const std::string & method_, OutStreamCallback out_stream_callback_, @@ -629,18 +688,121 @@ public: const ReadSettings & settings_ = {}, const HTTPHeaderEntries & http_header_entries_ = {}, Range read_range_ = {}, - const RemoteHostFilter & remote_host_filter_ = {}, + const RemoteHostFilter * remote_host_filter_ = nullptr, bool delay_initialization_ = true, bool use_external_buffer_ = false, bool skip_not_found_url_ = false) - : Parent(std::make_shared(uri_, timeouts, max_redirects), - uri_, credentials_, method_, out_stream_callback_, buffer_size_, - settings_, http_header_entries_, read_range_, remote_host_filter_, - delay_initialization_, use_external_buffer_, skip_not_found_url_) + : Parent( + std::make_shared(uri_, timeouts, max_redirects), + uri_, + credentials_, + method_, + out_stream_callback_, + buffer_size_, + settings_, + http_header_entries_, + read_range_, + remote_host_filter_, + delay_initialization_, + use_external_buffer_, + skip_not_found_url_) { } }; +class RangedReadWriteBufferFromHTTPFactory : public ParallelReadBuffer::ReadBufferFactory +{ + using OutStreamCallback = ReadWriteBufferFromHTTP::OutStreamCallback; + +public: + RangedReadWriteBufferFromHTTPFactory( + size_t total_object_size_, + size_t range_step_, + Poco::URI uri_, + std::string method_, + OutStreamCallback out_stream_callback_, + ConnectionTimeouts timeouts_, + const Poco::Net::HTTPBasicCredentials & credentials_, + UInt64 max_redirects_ = 0, + size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, + ReadSettings settings_ = {}, + ReadWriteBufferFromHTTP::HTTPHeaderEntries http_header_entries_ = {}, + const RemoteHostFilter * remote_host_filter_ = nullptr, + bool delay_initialization_ = true, + bool use_external_buffer_ = false, + bool skip_not_found_url_ = false) + : range_generator(total_object_size_, range_step_) + , total_object_size(total_object_size_) + , range_step(range_step_) + , uri(uri_) + , method(std::move(method_)) + , out_stream_callback(out_stream_callback_) + , timeouts(std::move(timeouts_)) + , credentials(credentials_) + , max_redirects(max_redirects_) + , buffer_size(buffer_size_) + , settings(std::move(settings_)) + , http_header_entries(std::move(http_header_entries_)) + , remote_host_filter(remote_host_filter_) + , delay_initialization(delay_initialization_) + , use_external_buffer(use_external_buffer_) + , skip_not_found_url(skip_not_found_url_) + { + } + + SeekableReadBufferPtr getReader() override + { + const auto next_range = range_generator.nextRange(); + if (!next_range) + { + return nullptr; + } + + return std::make_shared( + uri, + method, + out_stream_callback, + timeouts, + credentials, + max_redirects, + buffer_size, + settings, + http_header_entries, + // HTTP Range has inclusive bounds, i.e. [from, to] + ReadWriteBufferFromHTTP::Range{next_range->first, next_range->second - 1}, + remote_host_filter, + delay_initialization, + use_external_buffer, + skip_not_found_url); + } + + off_t seek(off_t off, [[maybe_unused]] int whence) override + { + range_generator = RangeGenerator{total_object_size, range_step, static_cast(off)}; + return off; + } + + std::optional getTotalSize() override { return total_object_size; } + +private: + RangeGenerator range_generator; + size_t total_object_size; + size_t range_step; + Poco::URI uri; + std::string method; + OutStreamCallback out_stream_callback; + ConnectionTimeouts timeouts; + const Poco::Net::HTTPBasicCredentials & credentials; + UInt64 max_redirects; + size_t buffer_size; + ReadSettings settings; + ReadWriteBufferFromHTTP::HTTPHeaderEntries http_header_entries; + const RemoteHostFilter * remote_host_filter; + bool delay_initialization; + bool use_external_buffer; + bool skip_not_found_url; +}; + class UpdatablePooledSession : public UpdatableSessionBase { using Parent = UpdatableSessionBase; @@ -649,20 +811,14 @@ private: size_t per_endpoint_pool_size; public: - explicit UpdatablePooledSession(const Poco::URI uri, - const ConnectionTimeouts & timeouts_, - const UInt64 max_redirects_, - size_t per_endpoint_pool_size_) - : Parent(uri, timeouts_, max_redirects_) - , per_endpoint_pool_size { per_endpoint_pool_size_ } + explicit UpdatablePooledSession( + const Poco::URI uri, const ConnectionTimeouts & timeouts_, const UInt64 max_redirects_, size_t per_endpoint_pool_size_) + : Parent(uri, timeouts_, max_redirects_), per_endpoint_pool_size{per_endpoint_pool_size_} { session = makePooledHTTPSession(initial_uri, timeouts, per_endpoint_pool_size); } - void buildNewSession(const Poco::URI & uri) override - { - session = makePooledHTTPSession(uri, timeouts, per_endpoint_pool_size); - } + void buildNewSession(const Poco::URI & uri) override { session = makePooledHTTPSession(uri, timeouts, per_endpoint_pool_size); } }; class PooledReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase> @@ -670,7 +826,8 @@ class PooledReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase using Parent = detail::ReadWriteBufferFromHTTPBase>; public: - explicit PooledReadWriteBufferFromHTTP(Poco::URI uri_, + explicit PooledReadWriteBufferFromHTTP( + Poco::URI uri_, const std::string & method_ = {}, OutStreamCallback out_stream_callback_ = {}, const ConnectionTimeouts & timeouts_ = {}, @@ -678,12 +835,13 @@ public: size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, const UInt64 max_redirects = 0, size_t max_connections_per_endpoint = DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT) - : Parent(std::make_shared(uri_, timeouts_, max_redirects, max_connections_per_endpoint), - uri_, - credentials_, - method_, - out_stream_callback_, - buffer_size_) + : Parent( + std::make_shared(uri_, timeouts_, max_redirects, max_connections_per_endpoint), + uri_, + credentials_, + method_, + out_stream_callback_, + buffer_size_) { } }; diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 2647e254626..defd029f05a 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -49,13 +49,13 @@ class PocoHTTPResponse : public Aws::Http::Standard::StandardHttpResponse public: using SessionPtr = HTTPSessionPtr; - PocoHTTPResponse(const std::shared_ptr request) + explicit PocoHTTPResponse(const std::shared_ptr request) : Aws::Http::Standard::StandardHttpResponse(request) , body_stream(request->GetResponseStreamFactory()) { } - void SetResponseBody(Aws::IStream & incoming_stream, SessionPtr & session_) + void SetResponseBody(Aws::IStream & incoming_stream, SessionPtr & session_) /// NOLINT { body_stream = Aws::Utils::Stream::ResponseStream( Aws::New>("http result streambuf", session_, incoming_stream.rdbuf()) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 59a4dab837b..e63e6fde1f4 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -517,7 +517,7 @@ private: class S3CredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain { public: - explicit S3CredentialsProviderChain(const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, bool use_environment_credentials, bool use_insecure_imds_request) + S3CredentialsProviderChain(const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, bool use_environment_credentials, bool use_insecure_imds_request) { auto * logger = &Poco::Logger::get("S3CredentialsProviderChain"); @@ -529,17 +529,18 @@ public: static const char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED"; /// The only difference from DefaultAWSCredentialsProviderChain::DefaultAWSCredentialsProviderChain() - /// is that this chain uses custom ClientConfiguration. - - AddProvider(std::make_shared()); - AddProvider(std::make_shared()); - AddProvider(std::make_shared()); - + /// is that this chain uses custom ClientConfiguration. Also we removed process provider because it's useless in our case. + /// + /// AWS API tries credentials providers one by one. Some of providers (like ProfileConfigFileAWSCredentialsProvider) can be + /// quite verbose even if nobody configured them. So we use our provider first and only after it use default providers. { DB::S3::PocoHTTPClientConfiguration aws_client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration(configuration.region, configuration.remote_host_filter, configuration.s3_max_redirects); AddProvider(std::make_shared(aws_client_configuration)); } + AddProvider(std::make_shared()); + + /// ECS TaskRole Credentials only available when ENVIRONMENT VARIABLE is set. const auto relative_uri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI); LOG_DEBUG(logger, "The environment variable value {} is {}", AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI, @@ -601,6 +602,9 @@ public: } AddProvider(std::make_shared(credentials)); + /// Quite verbose provider (argues if file with credentials doesn't exist) so iut's the last one + /// in chain. + AddProvider(std::make_shared()); } }; diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 72774499445..97cb4f74f90 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -49,7 +49,6 @@ public: private: ClientFactory(); -private: Aws::SDKOptions aws_options; }; diff --git a/src/IO/SeekableReadBuffer.h b/src/IO/SeekableReadBuffer.h index 2dc901ccfd9..3a46630350a 100644 --- a/src/IO/SeekableReadBuffer.h +++ b/src/IO/SeekableReadBuffer.h @@ -6,6 +6,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + + class SeekableReadBuffer : public ReadBuffer { public: @@ -32,6 +38,26 @@ public: * @return Offset from the begin of the underlying buffer / file corresponds to the buffer current position. */ virtual off_t getPosition() = 0; + + struct Range + { + size_t left; + std::optional right; + }; + + /** + * Returns a struct, where `left` is current read position in file and `right` is the + * last included offset for reading according to setReadUntilPosition() or setReadUntilEnd(). + * E.g. next nextImpl() call will read within range [left, right]. + */ + virtual Range getRemainingReadRange() const + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRemainingReadRange() not implemented"); + } + + virtual String getInfoForLog() { return ""; } + + virtual size_t getFileOffsetOfBufferEnd() const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getFileOffsetOfBufferEnd() not implemented"); } }; using SeekableReadBufferPtr = std::shared_ptr; @@ -48,22 +74,7 @@ public: /// set std::nullopt in case it is impossible to find out total size. virtual std::optional getTotalSize() = 0; - /** - * Some buffers might have different seek restrictions according to where it is used. - * For example, ReadBufferFromS3 and ReadBufferFromWebServer, when used for reading - * from remote disks, require some additional invariants and restrictions, which - * are not needed in other cases. - */ - enum class ReadType - { - DEFAULT, - DISK_READ - }; - - void setReadType(ReadType type) { read_type = type; } - protected: - ReadType read_type = ReadType::DEFAULT; std::optional file_size; }; diff --git a/src/IO/SynchronousReader.cpp b/src/IO/SynchronousReader.cpp index 4414da28d28..7cef3bd8963 100644 --- a/src/IO/SynchronousReader.cpp +++ b/src/IO/SynchronousReader.cpp @@ -82,8 +82,7 @@ std::future SynchronousReader::submit(Request reque watch.stop(); ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); - return Result{ .size = bytes_read, .offset = 0}; - + return Result{ .size = bytes_read, .offset = request.ignore }; }); } diff --git a/src/IO/UncompressedCache.h b/src/IO/UncompressedCache.h index 5826b7f020a..93ca1235a42 100644 --- a/src/IO/UncompressedCache.h +++ b/src/IO/UncompressedCache.h @@ -42,7 +42,7 @@ private: using Base = LRUCache; public: - UncompressedCache(size_t max_size_in_bytes) + explicit UncompressedCache(size_t max_size_in_bytes) : Base(max_size_in_bytes) {} /// Calculate key from path to file and offset. diff --git a/src/IO/VarInt.h b/src/IO/VarInt.h index 50fc158ba76..3161ca6d8a8 100644 --- a/src/IO/VarInt.h +++ b/src/IO/VarInt.h @@ -108,8 +108,8 @@ inline void readVarInt(Int16 & x, ReadBuffer & istr) } template -inline std::enable_if_t, void> -readVarUInt(T & x, ReadBuffer & istr) +requires (!std::is_same_v) +inline void readVarUInt(T & x, ReadBuffer & istr) { UInt64 tmp; readVarUInt(tmp, istr); @@ -132,7 +132,7 @@ inline void readVarUIntImpl(UInt64 & x, ReadBuffer & istr) if (istr.eof()) throwReadAfterEOF(); - UInt64 byte = *istr.position(); + UInt64 byte = *istr.position(); /// NOLINT ++istr.position(); x |= (byte & 0x7F) << (7 * i); @@ -172,7 +172,7 @@ inline const char * readVarUInt(UInt64 & x, const char * istr, size_t size) if (istr == end) throwReadAfterEOF(); - UInt64 byte = *istr; + UInt64 byte = *istr; /// NOLINT ++istr; x |= (byte & 0x7F) << (7 * i); diff --git a/src/IO/WriteBufferFromFile.h b/src/IO/WriteBufferFromFile.h index 988b0be7d00..3363a568bac 100644 --- a/src/IO/WriteBufferFromFile.h +++ b/src/IO/WriteBufferFromFile.h @@ -28,7 +28,7 @@ protected: CurrentMetrics::Increment metric_increment{CurrentMetrics::OpenFileForWrite}; public: - WriteBufferFromFile( + explicit WriteBufferFromFile( const std::string & file_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, int flags = -1, @@ -37,7 +37,7 @@ public: size_t alignment = 0); /// Use pre-opened file descriptor. - WriteBufferFromFile( + explicit WriteBufferFromFile( int & fd, /// Will be set to -1 if constructor didn't throw and ownership of file descriptor is passed to the object. const std::string & original_file_name = {}, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/IO/WriteBufferFromFileDescriptor.h b/src/IO/WriteBufferFromFileDescriptor.h index b065e22cf95..cc69567932f 100644 --- a/src/IO/WriteBufferFromFileDescriptor.h +++ b/src/IO/WriteBufferFromFileDescriptor.h @@ -11,7 +11,7 @@ namespace DB class WriteBufferFromFileDescriptor : public WriteBufferFromFileBase { public: - WriteBufferFromFileDescriptor( + explicit WriteBufferFromFileDescriptor( int fd_ = -1, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/WriteBufferFromFileDescriptorDiscardOnFailure.h b/src/IO/WriteBufferFromFileDescriptorDiscardOnFailure.h index 53e01c3cb26..2803dd4e8bf 100644 --- a/src/IO/WriteBufferFromFileDescriptorDiscardOnFailure.h +++ b/src/IO/WriteBufferFromFileDescriptorDiscardOnFailure.h @@ -17,7 +17,7 @@ protected: public: using WriteBufferFromFileDescriptor::WriteBufferFromFileDescriptor; - ~WriteBufferFromFileDescriptorDiscardOnFailure() override {} + ~WriteBufferFromFileDescriptorDiscardOnFailure() override = default; }; } diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index 5ddc28d2db1..622fab91fcc 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -10,6 +10,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( const Poco::URI & uri, const std::string & method, const std::string & content_type, + const std::string & content_encoding, const ConnectionTimeouts & timeouts, size_t buffer_size_) : WriteBufferFromOStream(buffer_size_) @@ -24,6 +25,9 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( request.set("Content-Type", content_type); } + if (!content_encoding.empty()) + request.set("Content-Encoding", content_encoding); + LOG_TRACE((&Poco::Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); ostr = &session->sendRequest(request); @@ -31,6 +35,10 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( void WriteBufferFromHTTP::finalizeImpl() { + // for compressed body, the data is stored in buffered first + // here, make sure the content in the buffer has been flushed + this->nextImpl(); + receiveResponse(*session, request, response, false); /// TODO: Response body is ignored. } diff --git a/src/IO/WriteBufferFromHTTP.h b/src/IO/WriteBufferFromHTTP.h index 31b2a921889..6966bc8a5c5 100644 --- a/src/IO/WriteBufferFromHTTP.h +++ b/src/IO/WriteBufferFromHTTP.h @@ -21,6 +21,7 @@ public: explicit WriteBufferFromHTTP(const Poco::URI & uri, const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only const std::string & content_type = "", + const std::string & content_encoding = "", const ConnectionTimeouts & timeouts = {}, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); diff --git a/src/IO/WriteBufferFromOStream.h b/src/IO/WriteBufferFromOStream.h index ea3301fef18..f8b45c2fa59 100644 --- a/src/IO/WriteBufferFromOStream.h +++ b/src/IO/WriteBufferFromOStream.h @@ -12,7 +12,7 @@ namespace DB class WriteBufferFromOStream : public BufferWithOwnMemory { public: - WriteBufferFromOStream( + explicit WriteBufferFromOStream( std::ostream & ostr_, size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, @@ -21,7 +21,7 @@ public: ~WriteBufferFromOStream() override; protected: - WriteBufferFromOStream(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0); + explicit WriteBufferFromOStream(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0); void nextImpl() override; diff --git a/src/IO/WriteBufferFromPocoSocket.cpp b/src/IO/WriteBufferFromPocoSocket.cpp index 4dc703c34eb..fb4e5df9b59 100644 --- a/src/IO/WriteBufferFromPocoSocket.cpp +++ b/src/IO/WriteBufferFromPocoSocket.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include @@ -38,8 +40,13 @@ void WriteBufferFromPocoSocket::nextImpl() return; Stopwatch watch; - size_t bytes_written = 0; + + SCOPE_EXIT({ + ProfileEvents::increment(ProfileEvents::NetworkSendElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::NetworkSendBytes, bytes_written); + }); + while (bytes_written < offset()) { ssize_t res = 0; @@ -70,9 +77,6 @@ void WriteBufferFromPocoSocket::nextImpl() bytes_written += res; } - - ProfileEvents::increment(ProfileEvents::NetworkSendElapsedMicroseconds, watch.elapsedMicroseconds()); - ProfileEvents::increment(ProfileEvents::NetworkSendBytes, bytes_written); } WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size) diff --git a/src/IO/WriteBufferFromPocoSocket.h b/src/IO/WriteBufferFromPocoSocket.h index 2fb203189f3..295ca16ecaf 100644 --- a/src/IO/WriteBufferFromPocoSocket.h +++ b/src/IO/WriteBufferFromPocoSocket.h @@ -14,7 +14,7 @@ namespace DB class WriteBufferFromPocoSocket : public BufferWithOwnMemory { public: - WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + explicit WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); ~WriteBufferFromPocoSocket() override; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 275fb3957bb..eda7bb6f8ae 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -79,6 +79,10 @@ void WriteBufferFromS3::nextImpl() if (!offset()) return; + /// Buffer in a bad state after exception + if (temporary_buffer->tellp() == -1) + allocateBuffer(); + temporary_buffer->write(working_buffer.begin(), offset()); ProfileEvents::increment(ProfileEvents::S3WriteBytes, offset()); @@ -91,6 +95,7 @@ void WriteBufferFromS3::nextImpl() if (!multipart_upload_id.empty() && last_part_size > upload_part_size) { + writePart(); allocateBuffer(); @@ -111,7 +116,14 @@ void WriteBufferFromS3::allocateBuffer() WriteBufferFromS3::~WriteBufferFromS3() { - finalize(); + try + { + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } void WriteBufferFromS3::preFinalize() @@ -147,6 +159,10 @@ void WriteBufferFromS3::createMultipartUpload() Aws::S3::Model::CreateMultipartUploadRequest req; req.SetBucket(bucket); req.SetKey(key); + + /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 + req.SetContentType("binary/octet-stream"); + if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); @@ -155,7 +171,7 @@ void WriteBufferFromS3::createMultipartUpload() if (outcome.IsSuccess()) { multipart_upload_id = outcome.GetResult().GetUploadId(); - LOG_DEBUG(log, "Multipart upload has created. Bucket: {}, Key: {}, Upload id: {}", bucket, key, multipart_upload_id); + LOG_TRACE(log, "Multipart upload has created. Bucket: {}, Key: {}, Upload id: {}", bucket, key, multipart_upload_id); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -165,14 +181,17 @@ void WriteBufferFromS3::writePart() { auto size = temporary_buffer->tellp(); - LOG_DEBUG(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Size: {}", bucket, key, multipart_upload_id, size); + LOG_TRACE(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Size: {}", bucket, key, multipart_upload_id, size); if (size < 0) - throw Exception("Failed to write part. Buffer in invalid state.", ErrorCodes::S3_ERROR); + { + LOG_WARNING(log, "Skipping part upload. Buffer is in bad state, it means that we have tried to upload something, but got an exception."); + return; + } if (size == 0) { - LOG_DEBUG(log, "Skipping writing part. Buffer is empty."); + LOG_TRACE(log, "Skipping writing part. Buffer is empty."); return; } @@ -234,6 +253,9 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re req.SetUploadId(multipart_upload_id); req.SetContentLength(temporary_buffer->tellp()); req.SetBody(temporary_buffer); + + /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 + req.SetContentType("binary/octet-stream"); } void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) @@ -243,7 +265,7 @@ void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) if (outcome.IsSuccess()) { task.tag = outcome.GetResult().GetETag(); - LOG_DEBUG(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size()); + LOG_TRACE(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size()); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -253,7 +275,7 @@ void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) void WriteBufferFromS3::completeMultipartUpload() { - LOG_DEBUG(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); + LOG_TRACE(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); if (part_tags.empty()) throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); @@ -275,7 +297,7 @@ void WriteBufferFromS3::completeMultipartUpload() auto outcome = client_ptr->CompleteMultipartUpload(req); if (outcome.IsSuccess()) - LOG_DEBUG(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); + LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); else { throw Exception(ErrorCodes::S3_ERROR, "{} Tags:{}", @@ -289,14 +311,17 @@ void WriteBufferFromS3::makeSinglepartUpload() auto size = temporary_buffer->tellp(); bool with_pool = bool(schedule); - LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); + LOG_TRACE(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); if (size < 0) - throw Exception("Failed to make single part upload. Buffer in invalid state", ErrorCodes::S3_ERROR); + { + LOG_WARNING(log, "Skipping single part upload. Buffer is in bad state, it mean that we have tried to upload something, but got an exception."); + return; + } if (size == 0) { - LOG_DEBUG(log, "Skipping single part upload. Buffer is empty."); + LOG_TRACE(log, "Skipping single part upload. Buffer is empty."); return; } @@ -343,6 +368,9 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) req.SetBody(temporary_buffer); if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); + + /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 + req.SetContentType("binary/octet-stream"); } void WriteBufferFromS3::processPutRequest(PutObjectTask & task) @@ -351,7 +379,7 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task) bool with_pool = bool(schedule); if (outcome.IsSuccess()) - LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); + LOG_TRACE(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } @@ -365,7 +393,7 @@ void WriteBufferFromS3::waitForReadyBackGroundTasks() while (!upload_object_tasks.empty() && upload_object_tasks.front().is_finised) { auto & task = upload_object_tasks.front(); - auto exception = std::move(task.exception); + auto exception = task.exception; auto tag = std::move(task.tag); upload_object_tasks.pop_front(); @@ -392,7 +420,7 @@ void WriteBufferFromS3::waitForAllBackGroundTasks() { auto & task = upload_object_tasks.front(); if (task.exception) - std::rethrow_exception(std::move(task.exception)); + std::rethrow_exception(task.exception); part_tags.push_back(task.tag); @@ -403,7 +431,7 @@ void WriteBufferFromS3::waitForAllBackGroundTasks() { bg_tasks_condvar.wait(lock, [this]() { return put_object_task->is_finised; }); if (put_object_task->exception) - std::rethrow_exception(std::move(put_object_task->exception)); + std::rethrow_exception(put_object_task->exception); } } } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 8b89626ee18..a4fbcbcdeeb 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -6,6 +6,7 @@ # include # include +# include # include # include @@ -14,8 +15,6 @@ # include -# include - namespace Aws::S3 { class S3Client; diff --git a/src/IO/WriteBufferFromTemporaryFile.h b/src/IO/WriteBufferFromTemporaryFile.h index 642c36b9be6..06e2911db26 100644 --- a/src/IO/WriteBufferFromTemporaryFile.h +++ b/src/IO/WriteBufferFromTemporaryFile.h @@ -20,7 +20,7 @@ public: ~WriteBufferFromTemporaryFile() override; private: - WriteBufferFromTemporaryFile(std::unique_ptr && tmp_file); + explicit WriteBufferFromTemporaryFile(std::unique_ptr && tmp_file); std::shared_ptr getReadBufferImpl() override; diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index 23ae3a70ef3..d74b366b8e2 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -67,7 +67,7 @@ private: void finalizeImpl() override final { vector.resize( - ((position() - reinterpret_cast(vector.data())) + ((position() - reinterpret_cast(vector.data())) /// NOLINT + sizeof(typename VectorType::value_type) - 1) /// Align up. / sizeof(typename VectorType::value_type)); diff --git a/src/IO/WriteBufferValidUTF8.h b/src/IO/WriteBufferValidUTF8.h index 8b33593c930..daaf0427f88 100644 --- a/src/IO/WriteBufferValidUTF8.h +++ b/src/IO/WriteBufferValidUTF8.h @@ -16,7 +16,7 @@ class WriteBufferValidUTF8 final : public BufferWithOwnMemory public: static const size_t DEFAULT_SIZE; - WriteBufferValidUTF8( + explicit WriteBufferValidUTF8( WriteBuffer & output_buffer_, bool group_replacements_ = true, const char * replacement_ = "\xEF\xBF\xBD", diff --git a/src/IO/WriteHelpers.cpp b/src/IO/WriteHelpers.cpp index b41f621e0b9..9433d31027c 100644 --- a/src/IO/WriteHelpers.cpp +++ b/src/IO/WriteHelpers.cpp @@ -7,7 +7,7 @@ namespace DB { template -void formatHex(IteratorSrc src, IteratorDst dst, const size_t num_bytes) +void formatHex(IteratorSrc src, IteratorDst dst, size_t num_bytes) { size_t src_pos = 0; size_t dst_pos = 0; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index ca2c202014c..7c6abf2aec7 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -80,7 +80,7 @@ inline void writeChar(char c, size_t n, WriteBuffer & buf) template inline void writePODBinary(const T & x, WriteBuffer & buf) { - buf.write(reinterpret_cast(&x), sizeof(x)); + buf.write(reinterpret_cast(&x), sizeof(x)); /// NOLINT } template @@ -663,7 +663,7 @@ inline void writeXMLStringForTextElement(const StringRef & s, WriteBuffer & buf) } template -void formatHex(IteratorSrc src, IteratorDst dst, const size_t num_bytes); +void formatHex(IteratorSrc src, IteratorDst dst, size_t num_bytes); void formatUUID(const UInt8 * src16, UInt8 * dst36); void formatUUID(std::reverse_iterator src16, UInt8 * dst36); @@ -869,8 +869,8 @@ inline void writeDateTimeUnixTimestamp(DateTime64 datetime64, UInt32 scale, Writ /// Methods for output in binary format. template -inline std::enable_if_t, void> -writeBinary(const T & x, WriteBuffer & buf) { writePODBinary(x, buf); } +requires is_arithmetic_v +inline void writeBinary(const T & x, WriteBuffer & buf) { writePODBinary(x, buf); } inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary(x, buf); } inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); } @@ -988,8 +988,8 @@ void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr, bool trailing_zer /// String, date, datetime are in single quotes with C-style escaping. Numbers - without. template -inline std::enable_if_t, void> -writeQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } +requires is_arithmetic_v +inline void writeQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } inline void writeQuoted(const String & x, WriteBuffer & buf) { writeQuotedString(x, buf); } @@ -1021,8 +1021,8 @@ inline void writeQuoted(const UUID & x, WriteBuffer & buf) /// String, date, datetime are in double quotes with C-style escaping. Numbers - without. template -inline std::enable_if_t, void> -writeDoubleQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } +requires is_arithmetic_v +inline void writeDoubleQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } inline void writeDoubleQuoted(const String & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } @@ -1054,8 +1054,8 @@ inline void writeDoubleQuoted(const UUID & x, WriteBuffer & buf) /// String - in double quotes and with CSV-escaping; date, datetime - in double quotes. Numbers - without. template -inline std::enable_if_t, void> -writeCSV(const T & x, WriteBuffer & buf) { writeText(x, buf); } +requires is_arithmetic_v +inline void writeCSV(const T & x, WriteBuffer & buf) { writeText(x, buf); } inline void writeCSV(const String & x, WriteBuffer & buf) { writeCSVString<>(x, buf); } inline void writeCSV(const LocalDate & x, WriteBuffer & buf) { writeDoubleQuoted(x, buf); } @@ -1124,8 +1124,8 @@ inline void writeNullTerminatedString(const String & s, WriteBuffer & buffer) } template -inline std::enable_if_t && (sizeof(T) <= 8), void> -writeBinaryBigEndian(T x, WriteBuffer & buf) /// Assuming little endian architecture. +requires is_arithmetic_v && (sizeof(T) <= 8) +inline void writeBinaryBigEndian(T x, WriteBuffer & buf) /// Assuming little endian architecture. { if constexpr (sizeof(x) == 2) x = __builtin_bswap16(x); @@ -1138,8 +1138,8 @@ writeBinaryBigEndian(T x, WriteBuffer & buf) /// Assuming little endian archi } template -inline std::enable_if_t, void> -writeBinaryBigEndian(const T & x, WriteBuffer & buf) /// Assuming little endian architecture. +requires is_big_int_v +inline void writeBinaryBigEndian(const T & x, WriteBuffer & buf) /// Assuming little endian architecture. { for (size_t i = 0; i != std::size(x.items); ++i) { diff --git a/src/IO/WriteIntText.h b/src/IO/WriteIntText.h index b8d2acc7d5d..c9a4cb0241a 100644 --- a/src/IO/WriteIntText.h +++ b/src/IO/WriteIntText.h @@ -5,22 +5,19 @@ #include -namespace -{ - template constexpr size_t max_int_width = 20; - template <> inline constexpr size_t max_int_width = 3; /// 255 - template <> inline constexpr size_t max_int_width = 4; /// -128 - template <> inline constexpr size_t max_int_width = 5; /// 65535 - template <> inline constexpr size_t max_int_width = 6; /// -32768 - template <> inline constexpr size_t max_int_width = 10; /// 4294967295 - template <> inline constexpr size_t max_int_width = 11; /// -2147483648 - template <> inline constexpr size_t max_int_width = 20; /// 18446744073709551615 - template <> inline constexpr size_t max_int_width = 20; /// -9223372036854775808 - template <> inline constexpr size_t max_int_width = 39; /// 340282366920938463463374607431768211455 - template <> inline constexpr size_t max_int_width = 40; /// -170141183460469231731687303715884105728 - template <> inline constexpr size_t max_int_width = 78; /// 115792089237316195423570985008687907853269984665640564039457584007913129639935 - template <> inline constexpr size_t max_int_width = 78; /// -57896044618658097711785492504343953926634992332820282019728792003956564819968 -} +template constexpr size_t max_int_width = 20; +template <> inline constexpr size_t max_int_width = 3; /// 255 +template <> inline constexpr size_t max_int_width = 4; /// -128 +template <> inline constexpr size_t max_int_width = 5; /// 65535 +template <> inline constexpr size_t max_int_width = 6; /// -32768 +template <> inline constexpr size_t max_int_width = 10; /// 4294967295 +template <> inline constexpr size_t max_int_width = 11; /// -2147483648 +template <> inline constexpr size_t max_int_width = 20; /// 18446744073709551615 +template <> inline constexpr size_t max_int_width = 20; /// -9223372036854775808 +template <> inline constexpr size_t max_int_width = 39; /// 340282366920938463463374607431768211455 +template <> inline constexpr size_t max_int_width = 40; /// -170141183460469231731687303715884105728 +template <> inline constexpr size_t max_int_width = 78; /// 115792089237316195423570985008687907853269984665640564039457584007913129639935 +template <> inline constexpr size_t max_int_width = 78; /// -57896044618658097711785492504343953926634992332820282019728792003956564819968 namespace DB diff --git a/src/IO/ZstdInflatingReadBuffer.h b/src/IO/ZstdInflatingReadBuffer.h index ec80b860e0e..7f246b02127 100644 --- a/src/IO/ZstdInflatingReadBuffer.h +++ b/src/IO/ZstdInflatingReadBuffer.h @@ -16,7 +16,7 @@ namespace ErrorCodes class ZstdInflatingReadBuffer : public BufferWithOwnMemory { public: - ZstdInflatingReadBuffer( + explicit ZstdInflatingReadBuffer( std::unique_ptr in_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, diff --git a/src/IO/examples/hadoop_snappy_read_buffer.cpp b/src/IO/examples/hadoop_snappy_read_buffer.cpp index 9cb01e6d697..eeac3db40a7 100644 --- a/src/IO/examples/hadoop_snappy_read_buffer.cpp +++ b/src/IO/examples/hadoop_snappy_read_buffer.cpp @@ -38,6 +38,11 @@ int main() return 1; } } + if (uncompress(256) != output) + { + std::cout << "test hadoop snappy read buffer failed, buf_size:" << 256 << std::endl; + return 1; + } std::cout << "test hadoop snappy read buffer success" << std::endl; return 0; } diff --git a/src/IO/readFloatText.h b/src/IO/readFloatText.h index 1c5b1cdb0c9..b6be7adbbee 100644 --- a/src/IO/readFloatText.h +++ b/src/IO/readFloatText.h @@ -154,7 +154,7 @@ ReturnType readFloatTextPreciseImpl(T & x, ReadBuffer & buf) if (likely(!buf.eof() && buf.position() + MAX_LENGTH <= buf.buffer().end())) { - auto initial_position = buf.position(); + auto * initial_position = buf.position(); auto res = fast_float::from_chars(initial_position, buf.buffer().end(), x); if (unlikely(res.ec != std::errc())) diff --git a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp new file mode 100644 index 00000000000..f681e8e61e1 --- /dev/null +++ b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp @@ -0,0 +1,66 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace DB; +TEST(HadoopSnappyDecoder, repeatNeedMoreInput) +{ + String snappy_base64_content = "AAAl6gAAB67qSzQyMDIxLTA2LTAxAXh4eGIEACQzMTkuNzQyNDMKnjEAHDQyLjgyMTcynjEAIDI5Ni4yODQwNqIxA" + "BgyNy43MjYzqpMAGDMuNTIyMzSiYgAcNjUuNDk1OTeiMQAYOTQuNTg1NaYxABg4OC40NzgyojEAHDMyMS4zOTE1os" + "QAHDM0Ni4xNTI3qjEAGDEuMjA3MTWm9QAQMi41MjamYQAcMjIuNTEyNDieYQAcMzMwLjI5MTKiGgIcMzIzLjAzNDi" + "iwwAcMzE1LjA1MDmiYgAcNDM1Ljc2ODaqxAAUMS45NDA5nvQACDAuMP4rAEorABwzMDMuMjAyNaYZARgwOC4xOTEy" + "pugAGDQ2LjQ0MjKilQMcMjc4Ljk3MTiiMQAcMzUwLjc3NTeirAGqSwEcMzI5LjkyMzGiXAAcMzMxLjc2NzamwAMUM" + "TMuNjM4pjEAGDI3NC4yMzK2MQAINDg0qrMBFDExLjgzNqbbBRgyNDkuNTI5qtsFGDUwLjE4ODmi5AGlSAgwNjWmiA" + "EUMjIuNjU4pqcCBDUzYcCqdgIYMDEuMzcxNbbPBgQ5Na5TBBA0Ljc1OaIiBMGdDDM0OTGeJwQcMjg3LjIyNTmm/AM" + "hVAAyopAAGDMxOC4wMjGmMAAB8AQ0OKpGAhgyMC42MTM4poMBFDg3LjEzOKoxABA5My4xNaZSARQ5NS41ODemTgVh" + "OQwwODg2osIAGDMyNi45NTSmMQAcMjc3LjgxNDmqjQcMNS42MqpqA0F3DDg2MDamzAPhKwQ4OKJWARgzMDYuMTc1q" + "i0EGDgwLjIwNTSihAUYMjk3LjY5NaYiBRAyOTAuM6aNBBgyMzkuMzI5pkIJwdOi7wcYMzcxLjIyNqpiBxQ0NS44Nz" + "Gq9woEODAOZAoANqJ+BRgyNzYuMjExpnYCIYIMMjIyOKKnAmVrBDc0psQAEDMwOS4xqtEJGDMwNC45MzSq8wAMNC4" + "0OKomCyG3DDE4MTGi/AMhJAQxMKqjBhgyNjEuNDQ4rqMGFDIuOTEwN6I5AwQzN7JMCQw2LjcwqqoMGDI2MC44NzOm" + "dwIOTAkMNDgzMqLSBhQyNTkuMjGmYweBiwg3MzOmyQMYNDM3Ljg1N6ZyBq5QARQzMy43MjSqKw4UMTIuNzkxpkkFD" + "mgNDDc4MzCmUAEUOTUuOTQypnoFDiQIDDI2ODmmBQMUNTEuMjc2qikEDtkJBDA1qgUDFDA3LjE1N6ZiAOGUCDMwOa" + "oxABA3NC42NqqmAhA5Ni45N6rIAxwzMDcuMjkzMaL+ChQyNzUuODau/QoANOExpugBGDI0Ny4xODSm5wEYOTEuNDE" + "3MZ7MChQzMzUuNjWquQQUNTMuODg1psMHDu8SCDIyOaYJDoFbCDk4M6aWDhwzNDEuNTcyMKK1AUF4ADSqCwoQMzg1" + "LjSujBIB9Aw0MDUwotoJDi4PCDc0N6aHARgyMjMuODMxpgYRwmcRGDIxMi4xNjWqSgIQMDkuODmuzgMYMTkuNTg0M" + "aK7CMFFADmuZQcQMDYuMzKqXwAIOS4zrl8ADu4PBDQ0qtQUGDQ3LjAzODGmFwIYMTAuOTIwMKLDAKG0DDM3MDOiYg" + "CqNgcORgkEMzeuGwWqXQAhqwg2MDWmSQUYMjY0LjE2N6aZFBIgFgQyM6aiCRQwNi41NTSm9AcYMjczLjczNqqSABg" + "0NS45OTIzpugPFDIxLjc3MqZ4EBwyODYuMDkyNKZAAhg0OS4yMjQzom8GDu0LCDEwNKaTBwAzDiUIADimGQkUMzM4" + "Ljc2qlITADcOmBUAOaYNBhwyNzAuODA4N6qrCQw3LjAwppkYwT4IMjYzrg0GDDMuOTmq/xEQMjIuODOqRgkEMjQOX" + "xKmQA0IMzAwDggVqjwREDY1LjYxsh8aCDQuOKrCBxgyNTQuNjQ2phMUISQENzmqsAwOLgsENTWqeAIQOTEuNTiuzR" + "EANw55CQAwpp8GEDI2My44rgsRFDI0LjMxNqZuBhIrFgAxqswDGDI4OS4zMzCqXwQANoHyADCmbAMUMzI4LjM2pps" + "DDDY1LjKBj57+Cg5PFwQ1NaoVBmFrADaqwgccMjk5LjgxMTCqdwYQMy4wODKmZwcEMzIOqBQAMaaCBRgyMjUuMTE2" + "qtkJADEOLw8AMKYwBBgyMzAuMTQyprwPGDMwMi4wMjemiAEOzQ4MODA0M6YaAhA1NC4yNKYkBWEMDsELqmEAFDIuN" + "jE4N6LNBxgyODMuNTM1qqUfFDk5Ljc4NKaaGQ5UEAgyNjSuqw2usgkYNDMuMDY0MZ5rAyHkCDMzOa6sHg6+CwAwpn" + "YGDnseCDk1MqaoAsHYDDgzNjeiLgsYMjg1LjkzMqZ1EQ67IQgyNTmmMQBB2Qg0OTamuhMUMjcxLjkzqpMWBDMyDoo" + "hADSmYgChhAg2NjimeAIQMzkxLjiqyw4IOTkuDt8bpoYBDDk0LjamaQMO4hAIOTI3qqQYFDQyLjk1M6oxAAQ4NA7G" + "HaZKIg6YCwwxNzYzpiQXFDkwLjk0OKqqAhQ5Ny4yNzSmvwQANg54GKq/CA4AIQg1MzOm/wMUNTYuNzQ2phcCHDM0N" + "S4wOTEyoswHDoAQCDA5M6rOGRA5MS42N6ZPGyQyNzUuNzExMTIK"; + String snappy_content; + Poco::MemoryInputStream istr(snappy_base64_content.data(), snappy_base64_content.size()); + Poco::Base64Decoder decoder(istr); + Poco::StreamCopier::copyToString(decoder, snappy_content); + auto file_writer = std::make_unique("./test.snappy"); + file_writer->write(snappy_content.c_str(), snappy_content.size()); + file_writer->close(); + std::unique_ptr in = std::make_unique("./test.snappy", 128); + HadoopSnappyReadBuffer read_buffer(std::move(in)); + String output; + WriteBufferFromString out(output); + copyData(read_buffer, out); + UInt128 hashcode = sipHash128(output.c_str(), output.size()); + String hashcode_str = getHexUIntLowercase(hashcode); + ASSERT_EQ(hashcode_str, "593afe14f61866915cc00b8c7bd86046"); +} diff --git a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp index c88e9c299a8..72b4b149bd7 100644 --- a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -45,22 +46,24 @@ namespace BlockIO InterpreterCreateRowPolicyQuery::execute() { auto & query = query_ptr->as(); - auto & access_control = getContext()->getAccessControl(); - getContext()->checkAccess(query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY); + auto required_access = getRequiredAccess(); if (!query.cluster.empty()) { query.replaceCurrentUserTag(getContext()->getUserName()); - return executeDDLQueryOnCluster(query_ptr, getContext()); + return executeDDLQueryOnCluster(query_ptr, getContext(), required_access); } assert(query.names->cluster.empty()); + auto & access_control = getContext()->getAccessControl(); + getContext()->checkAccess(required_access); + + query.replaceEmptyDatabase(getContext()->getCurrentDatabase()); + std::optional roles_from_query; if (query.roles) roles_from_query = RolesOrUsersSet{*query.roles, access_control, getContext()->getUserID()}; - query.replaceEmptyDatabase(getContext()->getCurrentDatabase()); - if (query.alter) { auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr @@ -105,4 +108,15 @@ void InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(RowPolicy & polic updateRowPolicyFromQueryImpl(policy, query, {}, {}); } + +AccessRightsElements InterpreterCreateRowPolicyQuery::getRequiredAccess() const +{ + const auto & query = query_ptr->as(); + AccessRightsElements res; + auto access_type = (query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY); + for (const auto & row_policy_name : query.names->full_names) + res.emplace_back(access_type, row_policy_name.database, row_policy_name.table_name); + return res; +} + } diff --git a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h index 8adfe6b0855..e76cc1c165d 100644 --- a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h +++ b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h @@ -6,8 +6,8 @@ namespace DB { - class ASTCreateRowPolicyQuery; +class AccessRightsElements; struct RowPolicy; class InterpreterCreateRowPolicyQuery : public IInterpreter, WithMutableContext @@ -20,6 +20,8 @@ public: static void updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query); private: + AccessRightsElements getRequiredAccess() const; + ASTPtr query_ptr; }; diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 33d85afb7c3..627b4dbac17 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -14,6 +14,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + +} namespace { void updateUserFromQueryImpl( @@ -22,7 +27,9 @@ namespace const std::shared_ptr & override_name, const std::optional & override_default_roles, const std::optional & override_settings, - const std::optional & override_grantees) + const std::optional & override_grantees, + bool allow_no_password, + bool allow_plaintext_password) { if (override_name) user.setName(override_name->toString()); @@ -34,6 +41,19 @@ namespace if (query.auth_data) user.auth_data = *query.auth_data; + if (query.auth_data || !query.alter) + { + auto auth_type = user.auth_data.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", + toString(auth_type), + AuthenticationTypeInfo::get(auth_type).name); + } + } + if (override_name && !override_name->host_pattern.empty()) { user.allowed_client_hosts = AllowedClientHosts{}; @@ -75,13 +95,14 @@ namespace } } - BlockIO InterpreterCreateUserQuery::execute() { const auto & query = query_ptr->as(); auto & access_control = getContext()->getAccessControl(); auto access = getContext()->getAccess(); access->checkAccess(query.alter ? AccessType::ALTER_USER : AccessType::CREATE_USER); + bool no_password_allowed = access_control.isNoPasswordAllowed(); + bool plaintext_password_allowed = access_control.isPlaintextPasswordAllowed(); std::optional default_roles_from_query; if (query.default_roles) @@ -93,10 +114,8 @@ BlockIO InterpreterCreateUserQuery::execute() access->checkAdminOption(role); } } - if (!query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, getContext()); - std::optional settings_from_query; if (query.settings) settings_from_query = SettingsProfileElements{*query.settings, access_control}; @@ -110,7 +129,7 @@ BlockIO InterpreterCreateUserQuery::execute() auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); - updateUserFromQueryImpl(*updated_user, query, {}, default_roles_from_query, settings_from_query, grantees_from_query); + updateUserFromQueryImpl(*updated_user, query, {}, default_roles_from_query, settings_from_query, grantees_from_query, no_password_allowed, plaintext_password_allowed); return updated_user; }; @@ -129,7 +148,7 @@ BlockIO InterpreterCreateUserQuery::execute() for (const auto & name : *query.names) { auto new_user = std::make_shared(); - updateUserFromQueryImpl(*new_user, query, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}); + updateUserFromQueryImpl(*new_user, query, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, no_password_allowed, plaintext_password_allowed); new_users.emplace_back(std::move(new_user)); } @@ -157,9 +176,9 @@ BlockIO InterpreterCreateUserQuery::execute() } -void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query) +void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password) { - updateUserFromQueryImpl(user, query, {}, {}, {}, {}); + updateUserFromQueryImpl(user, query, {}, {}, {}, {}, allow_no_password, allow_plaintext_password); } } diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.h b/src/Interpreters/Access/InterpreterCreateUserQuery.h index 7d357924d35..372066cfd5e 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.h +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.h @@ -17,7 +17,7 @@ public: BlockIO execute() override; - static void updateUserFromQuery(User & user, const ASTCreateUserQuery & query); + static void updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password); private: ASTPtr query_ptr; diff --git a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp index 4d2e880561e..3437e7fe0f4 100644 --- a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp @@ -49,12 +49,37 @@ AccessRightsElements InterpreterDropAccessEntityQuery::getRequiredAccess() const AccessRightsElements res; switch (query.type) { - case AccessEntityType::USER: res.emplace_back(AccessType::DROP_USER); return res; - case AccessEntityType::ROLE: res.emplace_back(AccessType::DROP_ROLE); return res; - case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); return res; - case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::DROP_ROW_POLICY); return res; - case AccessEntityType::QUOTA: res.emplace_back(AccessType::DROP_QUOTA); return res; - case AccessEntityType::MAX: break; + case AccessEntityType::USER: + { + res.emplace_back(AccessType::DROP_USER); + return res; + } + case AccessEntityType::ROLE: + { + res.emplace_back(AccessType::DROP_ROLE); + return res; + } + case AccessEntityType::SETTINGS_PROFILE: + { + res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); + return res; + } + case AccessEntityType::ROW_POLICY: + { + if (query.row_policy_names) + { + for (const auto & row_policy_name : query.row_policy_names->full_names) + res.emplace_back(AccessType::DROP_ROW_POLICY, row_policy_name.database, row_policy_name.table_name); + } + return res; + } + case AccessEntityType::QUOTA: + { + res.emplace_back(AccessType::DROP_QUOTA); + return res; + } + case AccessEntityType::MAX: + break; } throw Exception( toString(query.type) + ": type is not supported by DROP query", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h index 0ee478e904e..ea2d127913f 100644 --- a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h +++ b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h @@ -6,7 +6,6 @@ namespace DB { - class AccessRightsElements; class InterpreterDropAccessEntityQuery : public IInterpreter, WithMutableContext diff --git a/src/Interpreters/Access/InterpreterShowAccessQuery.cpp b/src/Interpreters/Access/InterpreterShowAccessQuery.cpp index e16ee03c711..d1d8ee63b8e 100644 --- a/src/Interpreters/Access/InterpreterShowAccessQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowAccessQuery.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace DB @@ -76,11 +76,11 @@ ASTs InterpreterShowAccessQuery::getCreateAndGrantQueries() const { create_queries.push_back(InterpreterShowCreateAccessEntityQuery::getCreateQuery(*entity, access_control)); if (entity->isTypeOf(AccessEntityType::USER) || entity->isTypeOf(AccessEntityType::ROLE)) - boost::range::push_back(grant_queries, InterpreterShowGrantsQuery::getGrantQueries(*entity, access_control)); + insertAtEnd(grant_queries, InterpreterShowGrantsQuery::getGrantQueries(*entity, access_control)); } ASTs result = std::move(create_queries); - boost::range::push_back(result, std::move(grant_queries)); + insertAtEnd(result, std::move(grant_queries)); return result; } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 163cb57cab5..27345218e07 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -377,12 +377,48 @@ AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess() AccessRightsElements res; switch (show_query.type) { - case AccessEntityType::USER: res.emplace_back(AccessType::SHOW_USERS); return res; - case AccessEntityType::ROLE: res.emplace_back(AccessType::SHOW_ROLES); return res; - case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); return res; - case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); return res; - case AccessEntityType::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); return res; - case AccessEntityType::MAX: break; + case AccessEntityType::USER: + { + res.emplace_back(AccessType::SHOW_USERS); + return res; + } + case AccessEntityType::ROLE: + { + res.emplace_back(AccessType::SHOW_ROLES); + return res; + } + case AccessEntityType::SETTINGS_PROFILE: + { + res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); + return res; + } + case AccessEntityType::ROW_POLICY: + { + if (show_query.row_policy_names) + { + for (const auto & row_policy_name : show_query.row_policy_names->full_names) + res.emplace_back(AccessType::SHOW_ROW_POLICIES, row_policy_name.database, row_policy_name.table_name); + } + else if (show_query.database_and_table_name) + { + if (show_query.database_and_table_name->second.empty()) + res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first); + else + res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first, show_query.database_and_table_name->second); + } + else + { + res.emplace_back(AccessType::SHOW_ROW_POLICIES); + } + return res; + } + case AccessEntityType::QUOTA: + { + res.emplace_back(AccessType::SHOW_QUOTAS); + return res; + } + case AccessEntityType::MAX: + break; } throw Exception(toString(show_query.type) + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 6ed35210251..25116f5145a 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -517,7 +517,7 @@ Block ActionsDAG::updateHeader(Block header) const { auto & list = it->second; pos_to_remove.insert(pos); - node_to_column[inputs[list.front()]] = std::move(col); + node_to_column[inputs[list.front()]] = col; list.pop_front(); } } @@ -590,7 +590,7 @@ Block ActionsDAG::updateHeader(Block header) const for (auto & col : result_columns) res.insert(std::move(col)); - for (const auto & item : header) + for (auto && item : header) res.insert(std::move(item)); return res; @@ -651,8 +651,8 @@ NameSet ActionsDAG::foldActionsByProjection( { /// Projection folding. node->type = ActionsDAG::ActionType::INPUT; - node->result_type = std::move(column_with_type_name->type); - node->result_name = std::move(column_with_type_name->name); + node->result_type = column_with_type_name->type; + node->result_name = column_with_type_name->name; node->children.clear(); inputs.push_back(node); } @@ -724,7 +724,7 @@ void ActionsDAG::addAliases(const NamesWithAliases & aliases) Node node; node.type = ActionType::ALIAS; node.result_type = child->result_type; - node.result_name = std::move(item.second); + node.result_name = item.second; node.column = child->column; node.children.emplace_back(child); @@ -771,7 +771,7 @@ void ActionsDAG::project(const NamesWithAliases & projection) Node node; node.type = ActionType::ALIAS; node.result_type = child->result_type; - node.result_name = std::move(item.second); + node.result_name = item.second; node.column = child->column; node.children.emplace_back(child); @@ -1061,7 +1061,6 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( throw Exception(ErrorCodes::THERE_IS_NO_COLUMN, "Cannot find column `{}` in source stream, there are only columns: [{}]", res_elem.name, Block(source).dumpNames()); - } else { @@ -1080,14 +1079,16 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( if (ignore_constant_values) dst_node = &actions_dag->addColumn(res_elem); else if (res_const->getField() != src_const->getField()) - throw Exception("Cannot convert column " + backQuote(res_elem.name) + " because " - "it is constant but values of constants are different in source and result", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot convert column `{}` because it is constant but values of constants are different in source and result", + res_elem.name); } else - throw Exception("Cannot convert column " + backQuote(res_elem.name) + " because " - "it is non constant in source stream but must be constant in result", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Cannot convert column `{}` because it is non constant in source stream but must be constant in result", + res_elem.name); } /// Add CAST function to convert into result type if needed. @@ -1119,10 +1120,8 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions( if (add_casted_columns) { if (inputs.contains(dst_node->result_name)) - throw Exception("Cannot convert column " + backQuote(res_elem.name) + - " to "+ backQuote(dst_node->result_name) + - " because other column have same name", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot convert column `{}` to `{}` because other column have same name", + res_elem.name, dst_node->result_name); if (new_names) new_names->emplace(res_elem.name, dst_node->result_name); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index bc937755618..181ac9aed7e 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -81,7 +81,10 @@ static Block createBlockFromCollection(const Collection & collection, const Data size_t columns_num = types.size(); MutableColumns columns(columns_num); for (size_t i = 0; i < columns_num; ++i) + { columns[i] = types[i]->createColumn(); + columns[i]->reserve(collection.size()); + } Row tuple_values; for (const auto & value : collection) @@ -120,7 +123,7 @@ static Block createBlockFromCollection(const Collection & collection, const Data if (i == tuple_size) for (i = 0; i < tuple_size; ++i) - columns[i]->insert(std::move(tuple_values[i])); + columns[i]->insert(tuple_values[i]); } } @@ -369,8 +372,8 @@ SetPtr makeExplicitSet( element_type = low_cardinality_type->getDictionaryType(); auto set_key = PreparedSetKey::forLiteral(*right_arg, set_element_types); - if (prepared_sets.count(set_key)) - return prepared_sets.at(set_key); /// Already prepared. + if (auto it = prepared_sets.find(set_key); it != prepared_sets.end()) + return it->second; /// Already prepared. Block block; const auto & right_arg_func = std::dynamic_pointer_cast(right_arg); @@ -385,13 +388,13 @@ SetPtr makeExplicitSet( set->insertFromBlock(block.getColumnsWithTypeAndName()); set->finishInsert(); - prepared_sets[set_key] = set; + prepared_sets.emplace(set_key, set); return set; } ScopeStack::Level::~Level() = default; ScopeStack::Level::Level() = default; -ScopeStack::Level::Level(Level &&) = default; +ScopeStack::Level::Level(Level &&) noexcept = default; class ScopeStack::Index { @@ -704,7 +707,7 @@ ASTs ActionsMatcher::doUntuple(const ASTFunction * function, ActionsMatcher::Dat if (tid != 0) tuple_ast = tuple_ast->clone(); - auto literal = std::make_shared(UInt64(++tid)); + auto literal = std::make_shared(UInt64{++tid}); visit(*literal, literal, data); auto func = makeASTFunction("tupleElement", tuple_ast, literal); @@ -811,14 +814,13 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (!data.only_consts) { /// We are in the part of the tree that we are not going to compute. You just need to define types. - /// Do not subquery and create sets. We replace "in*" function to "in*IgnoreSet". + /// Do not evaluate subquery and create sets. We replace "in*" function to "in*IgnoreSet". auto argument_name = node.arguments->children.at(0)->getColumnName(); - data.addFunction( - FunctionFactory::instance().get(node.name + "IgnoreSet", data.getContext()), - { argument_name, argument_name }, - column_name); + FunctionFactory::instance().get(node.name + "IgnoreSet", data.getContext()), + {argument_name, argument_name}, + column_name); } return; } @@ -1142,8 +1144,8 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su if (no_subqueries) return {}; auto set_key = PreparedSetKey::forSubquery(*right_in_operand); - if (data.prepared_sets.count(set_key)) - return data.prepared_sets.at(set_key); + if (auto it = data.prepared_sets.find(set_key); it != data.prepared_sets.end()) + return it->second; /// A special case is if the name of the table is specified on the right side of the IN statement, /// and the table has the type Set (a previously prepared set). @@ -1157,7 +1159,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su StorageSet * storage_set = dynamic_cast(table.get()); if (storage_set) { - data.prepared_sets[set_key] = storage_set->getSet(); + data.prepared_sets.emplace(set_key, storage_set->getSet()); return storage_set->getSet(); } } @@ -1171,7 +1173,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su /// If you already created a Set with the same subquery / table. if (subquery_for_set.set) { - data.prepared_sets[set_key] = subquery_for_set.set; + data.prepared_sets.emplace(set_key, subquery_for_set.set); return subquery_for_set.set; } @@ -1193,7 +1195,7 @@ SetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool no_su } subquery_for_set.set = set; - data.prepared_sets[set_key] = set; + data.prepared_sets.emplace(set_key, set); return set; } else diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 1d7d64f739a..342cc9eef9d 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -10,6 +10,7 @@ namespace DB { +class ASTExpressionList; class ASTFunction; class ExpressionActions; @@ -72,7 +73,7 @@ struct ScopeStack : WithContext NameSet inputs; Level(); - Level(Level &&); + Level(Level &&) noexcept; ~Level(); }; @@ -89,10 +90,7 @@ struct ScopeStack : WithContext void addColumn(ColumnWithTypeAndName column); void addAlias(const std::string & name, std::string alias); void addArrayJoin(const std::string & source_name, std::string result_name); - void addFunction( - const FunctionOverloadResolverPtr & function, - const Names & argument_names, - std::string result_name); + void addFunction(const FunctionOverloadResolverPtr & function, const Names & argument_names, std::string result_name); ActionsDAGPtr popLevel(); diff --git a/src/Interpreters/AddIndexConstraintsOptimizer.h b/src/Interpreters/AddIndexConstraintsOptimizer.h index 228d8d8ad1a..9ed4a8978c8 100644 --- a/src/Interpreters/AddIndexConstraintsOptimizer.h +++ b/src/Interpreters/AddIndexConstraintsOptimizer.h @@ -23,8 +23,7 @@ using StorageMetadataPtr = std::shared_ptr; class AddIndexConstraintsOptimizer final { public: - AddIndexConstraintsOptimizer( - const StorageMetadataPtr & metadata_snapshot); + explicit AddIndexConstraintsOptimizer(const StorageMetadataPtr & metadata_snapshot); void perform(CNFQuery & cnf_query); diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h index 00d2f76f043..5904cc48084 100644 --- a/src/Interpreters/AggregationCommon.h +++ b/src/Interpreters/AggregationCommon.h @@ -42,9 +42,6 @@ using Sizes = std::vector; /// 2,1,1 /// -namespace -{ - template constexpr auto getBitmapSize() { @@ -62,8 +59,6 @@ constexpr auto getBitmapSize() 0))); } -} - template void fillFixedBatch(size_t num_rows, const T * source, T * dest) { @@ -255,7 +250,7 @@ static inline T ALWAYS_INLINE packFixed( /// Hash a set of keys into a UInt128 value. -static inline UInt128 ALWAYS_INLINE hash128( +static inline UInt128 ALWAYS_INLINE hash128( /// NOLINT size_t i, size_t keys_size, const ColumnRawPtrs & key_columns) { UInt128 key; @@ -269,29 +264,9 @@ static inline UInt128 ALWAYS_INLINE hash128( return key; } - -/// Copy keys to the pool. Then put into pool StringRefs to them and return the pointer to the first. -static inline StringRef * ALWAYS_INLINE placeKeysInPool( - size_t keys_size, StringRefs & keys, Arena & pool) -{ - for (size_t j = 0; j < keys_size; ++j) - { - char * place = pool.alloc(keys[j].size); - memcpySmallAllowReadWriteOverflow15(place, keys[j].data, keys[j].size); - keys[j].data = place; - } - - /// Place the StringRefs on the newly copied keys in the pool. - char * res = pool.alignedAlloc(keys_size * sizeof(StringRef), alignof(StringRef)); - memcpySmallAllowReadWriteOverflow15(res, keys.data(), keys_size * sizeof(StringRef)); - - return reinterpret_cast(res); -} - - /** Serialize keys into a continuous chunk of memory. */ -static inline StringRef ALWAYS_INLINE serializeKeysToPoolContiguous( +static inline StringRef ALWAYS_INLINE serializeKeysToPoolContiguous( /// NOLINT size_t i, size_t keys_size, const ColumnRawPtrs & key_columns, Arena & pool) { const char * begin = nullptr; diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 05c9133cb35..f03bf45fbc6 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -184,7 +184,9 @@ struct AggregationMethodOneNumber AggregationMethodOneNumber() = default; template - AggregationMethodOneNumber(const Other & other) : data(other.data) {} + explicit AggregationMethodOneNumber(const Other & other) : data(other.data) + { + } /// To use one `Method` in different threads, use different `State`. using State = ColumnsHashing::HashMethodOneNumber - AggregationMethodString(const Other & other) : data(other.data) {} + explicit AggregationMethodString(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodString; @@ -247,7 +251,9 @@ struct AggregationMethodStringNoCache AggregationMethodStringNoCache() = default; template - AggregationMethodStringNoCache(const Other & other) : data(other.data) {} + explicit AggregationMethodStringNoCache(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodString; @@ -275,7 +281,9 @@ struct AggregationMethodFixedString AggregationMethodFixedString() = default; template - AggregationMethodFixedString(const Other & other) : data(other.data) {} + explicit AggregationMethodFixedString(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodFixedString; @@ -302,7 +310,9 @@ struct AggregationMethodFixedStringNoCache AggregationMethodFixedStringNoCache() = default; template - AggregationMethodFixedStringNoCache(const Other & other) : data(other.data) {} + explicit AggregationMethodFixedStringNoCache(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodFixedString; @@ -373,7 +383,9 @@ struct AggregationMethodKeysFixed AggregationMethodKeysFixed() = default; template - AggregationMethodKeysFixed(const Other & other) : data(other.data) {} + explicit AggregationMethodKeysFixed(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodKeysFixed< typename Data::value_type, @@ -462,7 +474,9 @@ struct AggregationMethodSerialized AggregationMethodSerialized() = default; template - AggregationMethodSerialized(const Other & other) : data(other.data) {} + explicit AggregationMethodSerialized(const Other & other) : data(other.data) + { + } using State = ColumnsHashing::HashMethodSerialized; @@ -646,7 +660,7 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::without_key: break; #define M(NAME, IS_TWO_LEVEL) \ - case Type::NAME: NAME = std::make_unique(); break; + case Type::NAME: (NAME) = std::make_unique(); break; APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M } @@ -663,7 +677,7 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::without_key: return 1; #define M(NAME, IS_TWO_LEVEL) \ - case Type::NAME: return NAME->data.size() + (without_key != nullptr); + case Type::NAME: return (NAME)->data.size() + (without_key != nullptr); APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M } @@ -680,7 +694,7 @@ struct AggregatedDataVariants : private boost::noncopyable case Type::without_key: return 1; #define M(NAME, IS_TWO_LEVEL) \ - case Type::NAME: return NAME->data.size(); + case Type::NAME: return (NAME)->data.size(); APPLY_FOR_AGGREGATED_VARIANTS(M) #undef M } @@ -739,6 +753,7 @@ struct AggregatedDataVariants : private boost::noncopyable M(low_cardinality_key_string) \ M(low_cardinality_key_fixed_string) \ + /// NOLINTNEXTLINE #define APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) \ M(key8) \ M(key16) \ @@ -752,6 +767,7 @@ struct AggregatedDataVariants : private boost::noncopyable M(low_cardinality_key8) \ M(low_cardinality_key16) \ + /// NOLINTNEXTLINE #define APPLY_FOR_VARIANTS_SINGLE_LEVEL(M) \ APPLY_FOR_VARIANTS_NOT_CONVERTIBLE_TO_TWO_LEVEL(M) \ APPLY_FOR_VARIANTS_CONVERTIBLE_TO_TWO_LEVEL(M) \ @@ -773,6 +789,7 @@ struct AggregatedDataVariants : private boost::noncopyable void convertToTwoLevel(); + /// NOLINTNEXTLINE #define APPLY_FOR_VARIANTS_TWO_LEVEL(M) \ M(key32_two_level) \ M(key64_two_level) \ @@ -1327,7 +1344,7 @@ private: template Method & getDataVariant(AggregatedDataVariants & variants); #define M(NAME, IS_TWO_LEVEL) \ - template <> inline decltype(AggregatedDataVariants::NAME)::element_type & getDataVariant(AggregatedDataVariants & variants) { return *variants.NAME; } + template <> inline decltype(AggregatedDataVariants::NAME)::element_type & getDataVariant(AggregatedDataVariants & variants) { return *variants.NAME; } /// NOLINT APPLY_FOR_AGGREGATED_VARIANTS(M) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 5321d5f6fd3..c60ab0f6510 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -165,9 +165,9 @@ void AsynchronousInsertQueue::scheduleDataProcessingJob(const InsertQuery & key, { /// Wrap 'unique_ptr' with 'shared_ptr' to make this /// lambda copyable and allow to save it to the thread pool. - pool.scheduleOrThrowOnError([=, data = std::make_shared(std::move(data))] + pool.scheduleOrThrowOnError([key, global_context, data = std::make_shared(std::move(data))]() mutable { - processData(std::move(key), std::move(*data), std::move(global_context)); + processData(key, std::move(*data), std::move(global_context)); }); } @@ -184,7 +184,10 @@ void AsynchronousInsertQueue::push(ASTPtr query, ContextPtr query_context) if (!FormatFactory::instance().isInputFormat(insert_query.format)) throw Exception(ErrorCodes::UNKNOWN_FORMAT, "Unknown input format {}", insert_query.format); - query_context->checkAccess(AccessType::INSERT, insert_query.table_id, sample_block.getNames()); + /// For table functions we check access while executing + /// InterpreterInsertQuery::getTable() -> ITableFunction::execute(). + if (insert_query.table_id) + query_context->checkAccess(AccessType::INSERT, insert_query.table_id, sample_block.getNames()); String bytes; { @@ -411,7 +414,7 @@ try }; std::shared_ptr adding_defaults_transform; - if (insert_context->getSettingsRef().input_format_defaults_for_omitted_fields) + if (insert_context->getSettingsRef().input_format_defaults_for_omitted_fields && insert_query.table_id) { StoragePtr storage = DatabaseCatalog::instance().getTable(insert_query.table_id, insert_context); auto metadata_snapshot = storage->getInMemoryMetadataPtr(); diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 1dd2ad216aa..db3cb3049fd 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -138,10 +138,10 @@ private: static void finishWithException(const ASTPtr & query, const std::list & entries, const E & exception); public: - Queue getQueue() const + auto getQueueLocked() const { std::shared_lock lock(rwlock); - return queue; + return std::make_pair(std::ref(queue), std::move(lock)); } }; diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 72e49505b54..c87ce12c2fa 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -373,7 +373,7 @@ static void calculateMaxAndSum(Max & max, Sum & sum, T x) max = x; } -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 +#if USE_JEMALLOC uint64_t updateJemallocEpoch() { uint64_t value = 0; @@ -620,13 +620,15 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti new_values["Uptime"] = getContext()->getUptimeSeconds(); /// Process process memory usage according to OS -#if defined(OS_LINUX) +#if defined(OS_LINUX) || defined(OS_FREEBSD) { MemoryStatisticsOS::Data data = memory_stat.get(); new_values["MemoryVirtual"] = data.virt; new_values["MemoryResident"] = data.resident; +#if !defined(OS_FREEBSD) new_values["MemoryShared"] = data.shared; +#endif new_values["MemoryCode"] = data.code; new_values["MemoryDataAndStack"] = data.data_and_stack; @@ -653,7 +655,9 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti CurrentMetrics::set(CurrentMetrics::MemoryTracking, new_amount); } } +#endif +#if defined(OS_LINUX) if (loadavg) { try @@ -1429,7 +1433,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti } } -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 +#if USE_JEMALLOC // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 3c7581ce1a3..e4bcb2890f3 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -76,9 +76,11 @@ private: bool first_run = true; std::chrono::system_clock::time_point previous_update_time; -#if defined(OS_LINUX) +#if defined(OS_LINUX) || defined(OS_FREEBSD) MemoryStatisticsOS memory_stat; +#endif +#if defined(OS_LINUX) std::optional meminfo; std::optional loadavg; std::optional proc_stat; diff --git a/src/Interpreters/BloomFilter.h b/src/Interpreters/BloomFilter.h index 279ab6947ec..1fb9895cc27 100644 --- a/src/Interpreters/BloomFilter.h +++ b/src/Interpreters/BloomFilter.h @@ -31,7 +31,7 @@ public: using UnderType = UInt64; using Container = std::vector; - BloomFilter(const BloomFilterParameters & params); + explicit BloomFilter(const BloomFilterParameters & params); /// size -- size of filter in bytes. /// hashes -- number of used hash functions. /// seed -- random seed for hash functions generation. diff --git a/src/Interpreters/CatBoostModel.cpp b/src/Interpreters/CatBoostModel.cpp index 1b6e30a0959..cffaa81c4f0 100644 --- a/src/Interpreters/CatBoostModel.cpp +++ b/src/Interpreters/CatBoostModel.cpp @@ -26,10 +26,10 @@ extern const int CANNOT_LOAD_CATBOOST_MODEL; extern const int CANNOT_APPLY_CATBOOST_MODEL; } - /// CatBoost wrapper interface functions. -struct CatBoostWrapperAPI +class CatBoostWrapperAPI { +public: using ModelCalcerHandle = void; ModelCalcerHandle * (* ModelCalcerCreate)(); // NOLINT @@ -68,9 +68,6 @@ struct CatBoostWrapperAPI }; -namespace -{ - class CatBoostModelHolder { private: @@ -84,7 +81,61 @@ public: }; -class CatBoostModelImpl : public ICatBoostModel +/// Holds CatBoost wrapper library and provides wrapper interface. +class CatBoostLibHolder +{ +public: + explicit CatBoostLibHolder(std::string lib_path_) : lib_path(std::move(lib_path_)), lib(lib_path) { initAPI(); } + + const CatBoostWrapperAPI & getAPI() const { return api; } + const std::string & getCurrentPath() const { return lib_path; } + +private: + CatBoostWrapperAPI api; + std::string lib_path; + SharedLibrary lib; + + void initAPI() + { + load(api.ModelCalcerCreate, "ModelCalcerCreate"); + load(api.ModelCalcerDelete, "ModelCalcerDelete"); + load(api.GetErrorString, "GetErrorString"); + load(api.LoadFullModelFromFile, "LoadFullModelFromFile"); + load(api.CalcModelPredictionFlat, "CalcModelPredictionFlat"); + load(api.CalcModelPrediction, "CalcModelPrediction"); + load(api.CalcModelPredictionWithHashedCatFeatures, "CalcModelPredictionWithHashedCatFeatures"); + load(api.GetStringCatFeatureHash, "GetStringCatFeatureHash"); + load(api.GetIntegerCatFeatureHash, "GetIntegerCatFeatureHash"); + load(api.GetFloatFeaturesCount, "GetFloatFeaturesCount"); + load(api.GetCatFeaturesCount, "GetCatFeaturesCount"); + tryLoad(api.CheckModelMetadataHasKey, "CheckModelMetadataHasKey"); + tryLoad(api.GetModelInfoValueSize, "GetModelInfoValueSize"); + tryLoad(api.GetModelInfoValue, "GetModelInfoValue"); + tryLoad(api.GetTreeCount, "GetTreeCount"); + tryLoad(api.GetDimensionsCount, "GetDimensionsCount"); + } + + template + void load(T& func, const std::string & name) { func = lib.get(name); } + + template + void tryLoad(T& func, const std::string & name) { func = lib.tryGet(name); } +}; + +std::shared_ptr getCatBoostWrapperHolder(const std::string & lib_path) +{ + static std::shared_ptr ptr; + static std::mutex mutex; + + std::lock_guard lock(mutex); + + if (!ptr || ptr->getCurrentPath() != lib_path) + ptr = std::make_shared(lib_path); + + return ptr; +} + +class CatBoostModelImpl { public: CatBoostModelImpl(const CatBoostWrapperAPI * api_, const std::string & model_path) : api(api_) @@ -92,13 +143,15 @@ public: handle = std::make_unique(api); if (!handle) { - std::string msg = "Cannot create CatBoost model: "; - throw Exception(msg + api->GetErrorString(), ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL); + throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, + "Cannot create CatBoost model: {}", + api->GetErrorString()); } if (!api->LoadFullModelFromFile(handle->get(), model_path.c_str())) { - std::string msg = "Cannot load CatBoost model: "; - throw Exception(msg + api->GetErrorString(), ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL); + throw Exception(ErrorCodes::CANNOT_LOAD_CATBOOST_MODEL, + "Cannot load CatBoost model: {}", + api->GetErrorString()); } float_features_count = api->GetFloatFeaturesCount(handle->get()); @@ -108,32 +161,22 @@ public: tree_count = api->GetDimensionsCount(handle->get()); } - ColumnPtr evaluate(const ColumnRawPtrs & columns) const override + ColumnPtr evaluate(const ColumnRawPtrs & columns) const { if (columns.empty()) - throw Exception("Got empty columns list for CatBoost model.", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Got empty columns list for CatBoost model."); if (columns.size() != float_features_count + cat_features_count) - { - std::string msg; - { - WriteBufferFromString buffer(msg); - buffer << "Number of columns is different with number of features: "; - buffer << columns.size() << " vs " << float_features_count << " + " << cat_features_count; - } - throw Exception(msg, ErrorCodes::BAD_ARGUMENTS); - } + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Number of columns is different with number of features: columns size {} float features size {} + cat features size {}", + float_features_count, + cat_features_count); for (size_t i = 0; i < float_features_count; ++i) { if (!columns[i]->isNumeric()) { - std::string msg; - { - WriteBufferFromString buffer(msg); - buffer << "Column " << i << " should be numeric to make float feature."; - } - throw Exception(msg, ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric to make float feature.", i); } } @@ -142,16 +185,13 @@ public: { const auto * column = columns[i]; if (column->isNumeric()) + { cat_features_are_strings = false; + } else if (!(typeid_cast(column) || typeid_cast(column))) { - std::string msg; - { - WriteBufferFromString buffer(msg); - buffer << "Column " << i << " should be numeric or string."; - } - throw Exception(msg, ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} should be numeric or string.", i); } } @@ -187,9 +227,9 @@ public: return ColumnTuple::create(std::move(mutable_columns)); } - size_t getFloatFeaturesCount() const override { return float_features_count; } - size_t getCatFeaturesCount() const override { return cat_features_count; } - size_t getTreeCount() const override { return tree_count; } + size_t getFloatFeaturesCount() const { return float_features_count; } + size_t getCatFeaturesCount() const { return cat_features_count; } + size_t getTreeCount() const { return tree_count; } private: std::unique_ptr handle; @@ -435,66 +475,6 @@ private: } }; - -/// Holds CatBoost wrapper library and provides wrapper interface. -class CatBoostLibHolder: public CatBoostWrapperAPIProvider -{ -public: - explicit CatBoostLibHolder(std::string lib_path_) : lib_path(std::move(lib_path_)), lib(lib_path) { initAPI(); } - - const CatBoostWrapperAPI & getAPI() const override { return api; } - const std::string & getCurrentPath() const { return lib_path; } - -private: - CatBoostWrapperAPI api; - std::string lib_path; - SharedLibrary lib; - - void initAPI(); - - template - void load(T& func, const std::string & name) { func = lib.get(name); } - - template - void tryLoad(T& func, const std::string & name) { func = lib.tryGet(name); } -}; - -void CatBoostLibHolder::initAPI() -{ - load(api.ModelCalcerCreate, "ModelCalcerCreate"); - load(api.ModelCalcerDelete, "ModelCalcerDelete"); - load(api.GetErrorString, "GetErrorString"); - load(api.LoadFullModelFromFile, "LoadFullModelFromFile"); - load(api.CalcModelPredictionFlat, "CalcModelPredictionFlat"); - load(api.CalcModelPrediction, "CalcModelPrediction"); - load(api.CalcModelPredictionWithHashedCatFeatures, "CalcModelPredictionWithHashedCatFeatures"); - load(api.GetStringCatFeatureHash, "GetStringCatFeatureHash"); - load(api.GetIntegerCatFeatureHash, "GetIntegerCatFeatureHash"); - load(api.GetFloatFeaturesCount, "GetFloatFeaturesCount"); - load(api.GetCatFeaturesCount, "GetCatFeaturesCount"); - tryLoad(api.CheckModelMetadataHasKey, "CheckModelMetadataHasKey"); - tryLoad(api.GetModelInfoValueSize, "GetModelInfoValueSize"); - tryLoad(api.GetModelInfoValue, "GetModelInfoValue"); - tryLoad(api.GetTreeCount, "GetTreeCount"); - tryLoad(api.GetDimensionsCount, "GetDimensionsCount"); -} - -std::shared_ptr getCatBoostWrapperHolder(const std::string & lib_path) -{ - static std::shared_ptr ptr; - static std::mutex mutex; - - std::lock_guard lock(mutex); - - if (!ptr || ptr->getCurrentPath() != lib_path) - ptr = std::make_shared(lib_path); - - return ptr; -} - -} - - CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::string lib_path_, const ExternalLoadableLifetime & lifetime_) : name(std::move(name_)), model_path(std::move(model_path_)), lib_path(std::move(lib_path_)), lifetime(lifetime_) @@ -502,43 +482,28 @@ CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::st api_provider = getCatBoostWrapperHolder(lib_path); api = &api_provider->getAPI(); model = std::make_unique(api, model_path); - float_features_count = model->getFloatFeaturesCount(); - cat_features_count = model->getCatFeaturesCount(); - tree_count = model->getTreeCount(); } -const ExternalLoadableLifetime & CatBoostModel::getLifetime() const -{ - return lifetime; -} - -bool CatBoostModel::isModified() const -{ - return true; -} - -std::shared_ptr CatBoostModel::clone() const -{ - return std::make_shared(name, model_path, lib_path, lifetime); -} +CatBoostModel::~CatBoostModel() = default; size_t CatBoostModel::getFloatFeaturesCount() const { - return float_features_count; + return model->getFloatFeaturesCount(); } size_t CatBoostModel::getCatFeaturesCount() const { - return cat_features_count; + return model->getCatFeaturesCount(); } size_t CatBoostModel::getTreeCount() const { - return tree_count; + return model->getTreeCount(); } DataTypePtr CatBoostModel::getReturnType() const { + size_t tree_count = getTreeCount(); auto type = std::make_shared(); if (tree_count == 1) return type; @@ -552,6 +517,7 @@ ColumnPtr CatBoostModel::evaluate(const ColumnRawPtrs & columns) const { if (!model) throw Exception("CatBoost model was not loaded.", ErrorCodes::LOGICAL_ERROR); + return model->evaluate(columns); } diff --git a/src/Interpreters/CatBoostModel.h b/src/Interpreters/CatBoostModel.h index 51bf0ba94f5..7bb1df92b67 100644 --- a/src/Interpreters/CatBoostModel.h +++ b/src/Interpreters/CatBoostModel.h @@ -8,47 +8,32 @@ namespace DB { -/// CatBoost wrapper interface functions. -struct CatBoostWrapperAPI; -class CatBoostWrapperAPIProvider -{ -public: - virtual ~CatBoostWrapperAPIProvider() = default; - virtual const CatBoostWrapperAPI & getAPI() const = 0; -}; - -/// CatBoost model interface. -class ICatBoostModel -{ -public: - virtual ~ICatBoostModel() = default; - /// Evaluate model. Use first `float_features_count` columns as float features, - /// the others `cat_features_count` as categorical features. - virtual ColumnPtr evaluate(const ColumnRawPtrs & columns) const = 0; - - virtual size_t getFloatFeaturesCount() const = 0; - virtual size_t getCatFeaturesCount() const = 0; - virtual size_t getTreeCount() const = 0; -}; +class CatBoostLibHolder; +class CatBoostWrapperAPI; +class CatBoostModelImpl; class IDataType; using DataTypePtr = std::shared_ptr; /// General ML model evaluator interface. -class IModel : public IExternalLoadable +class IMLModel : public IExternalLoadable { public: + IMLModel() = default; virtual ColumnPtr evaluate(const ColumnRawPtrs & columns) const = 0; virtual std::string getTypeName() const = 0; virtual DataTypePtr getReturnType() const = 0; + virtual ~IMLModel() override = default; }; -class CatBoostModel : public IModel +class CatBoostModel : public IMLModel { public: CatBoostModel(std::string name, std::string model_path, std::string lib_path, const ExternalLoadableLifetime & lifetime); + ~CatBoostModel() override; + ColumnPtr evaluate(const ColumnRawPtrs & columns) const override; std::string getTypeName() const override { return "catboost"; } @@ -59,29 +44,28 @@ public: /// IExternalLoadable interface. - const ExternalLoadableLifetime & getLifetime() const override; + const ExternalLoadableLifetime & getLifetime() const override { return lifetime; } std::string getLoadableName() const override { return name; } bool supportUpdates() const override { return true; } - bool isModified() const override; + bool isModified() const override { return true; } - std::shared_ptr clone() const override; + std::shared_ptr clone() const override + { + return std::make_shared(name, model_path, lib_path, lifetime); + } private: const std::string name; std::string model_path; std::string lib_path; ExternalLoadableLifetime lifetime; - std::shared_ptr api_provider; + std::shared_ptr api_provider; const CatBoostWrapperAPI * api; - std::unique_ptr model; - - size_t float_features_count; - size_t cat_features_count; - size_t tree_count; + std::unique_ptr model; void init(); }; diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 827e7d27409..75af25e842e 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -19,7 +19,7 @@ namespace ErrorCodes } -void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) const +void ClientInfo::write(WriteBuffer & out, UInt64 server_protocol_revision) const { if (server_protocol_revision < DBMS_MIN_REVISION_WITH_CLIENT_INFO) throw Exception("Logical error: method ClientInfo::write is called for unsupported server revision", ErrorCodes::LOGICAL_ERROR); @@ -99,7 +99,7 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) } -void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) +void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision) { if (client_protocol_revision < DBMS_MIN_REVISION_WITH_CLIENT_INFO) throw Exception("Logical error: method ClientInfo::read is called for unsupported client revision", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 3ce740c6436..0b40c78becc 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -119,8 +119,8 @@ public: * Only values that are not calculated automatically or passed separately are serialized. * Revisions are passed to use format that server will understand or client was used. */ - void write(WriteBuffer & out, const UInt64 server_protocol_revision) const; - void read(ReadBuffer & in, const UInt64 client_protocol_revision); + void write(WriteBuffer & out, UInt64 server_protocol_revision) const; + void read(ReadBuffer & in, UInt64 client_protocol_revision); /// Initialize parameters on client initiating query. void setInitialQuery(); diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index c01b19d81de..d558d1cfd67 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int LOGICAL_ERROR; extern const int SHARD_HAS_NO_CONNECTIONS; + extern const int NO_ELEMENTS_IN_CONFIG; extern const int SYNTAX_ERROR; } @@ -97,7 +98,6 @@ Cluster::Address::Address( , replica_index(replica_index_) { host_name = config.getString(config_prefix + ".host"); - port = static_cast(config.getInt(config_prefix + ".port")); if (config.has(config_prefix + ".user")) user_specified = true; @@ -106,7 +106,14 @@ Cluster::Address::Address( default_database = config.getString(config_prefix + ".default_database", ""); secure = ConfigHelper::getBool(config, config_prefix + ".secure", false, /* empty_as */true) ? Protocol::Secure::Enable : Protocol::Secure::Disable; priority = config.getInt(config_prefix + ".priority", 1); + const char * port_type = secure == Protocol::Secure::Enable ? "tcp_port_secure" : "tcp_port"; + auto default_port = config.getInt(port_type, 0); + + port = static_cast(config.getInt(config_prefix + ".port", default_port)); + if (!port) + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Port is not specified in cluster configuration: {}", config_prefix + ".port"); + is_local = isLocal(config.getInt(port_type, 0)); /// By default compression is disabled if address looks like localhost. diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index 3773dadaf13..e9f26c21089 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -63,7 +63,6 @@ public: /// is used to set a limit on the size of the timeout static Poco::Timespan saturate(Poco::Timespan v, Poco::Timespan limit); -public: using SlotToShard = std::vector; struct Address @@ -192,7 +191,6 @@ public: /// Name of directory for asynchronous write to StorageDistributed if has_internal_replication const std::string & insertPathForInternalReplication(bool prefer_localhost_replica, bool use_compact_format) const; - public: ShardInfoInsertPathForInternalReplication insert_path_for_internal_replication; /// Number of the shard, the indexation begins with 1 UInt32 shard_num = 0; @@ -207,7 +205,6 @@ public: using ShardsInfo = std::vector; - String getHashOfAddresses() const { return hash_of_addresses; } const ShardsInfo & getShardsInfo() const { return shards_info; } const AddressesWithFailover & getShardsAddresses() const { return addresses_with_failover; } @@ -263,7 +260,6 @@ private: /// Inter-server secret String secret; - String hash_of_addresses; /// Description of the cluster shards. ShardsInfo shards_info; /// Any remote shard. diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 5d35525aee9..2b92fab15de 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -35,9 +36,13 @@ namespace ClusterProxy SelectStreamFactory::SelectStreamFactory( const Block & header_, + const ColumnsDescriptionByShardNum & objects_by_shard_, + const StorageSnapshotPtr & storage_snapshot_, QueryProcessingStage::Enum processed_stage_) - : header(header_) - , processed_stage{processed_stage_} + : header(header_), + objects_by_shard(objects_by_shard_), + storage_snapshot(storage_snapshot_), + processed_stage(processed_stage_) { } @@ -100,6 +105,10 @@ void SelectStreamFactory::createForShard( Shards & remote_shards, UInt32 shard_count) { + auto it = objects_by_shard.find(shard_info.shard_num); + if (it != objects_by_shard.end()) + replaceMissedSubcolumnsByConstants(storage_snapshot->object_columns, it->second, query_ast); + auto emplace_local_stream = [&]() { local_plans.emplace_back(createLocalPlan(query_ast, header, context, processed_stage, shard_info.shard_num, shard_count)); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index 55e81feee33..731bf3acd10 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -11,11 +12,15 @@ namespace DB namespace ClusterProxy { +using ColumnsDescriptionByShardNum = std::unordered_map; + class SelectStreamFactory final : public IStreamFactory { public: SelectStreamFactory( const Block & header_, + const ColumnsDescriptionByShardNum & objects_by_shard_, + const StorageSnapshotPtr & storage_snapshot_, QueryProcessingStage::Enum processed_stage_); void createForShard( @@ -30,6 +35,8 @@ public: private: const Block header; + const ColumnsDescriptionByShardNum objects_by_shard; + const StorageSnapshotPtr storage_snapshot; QueryProcessingStage::Enum processed_stage; }; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 0db07267231..3f1823fb171 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -116,7 +116,7 @@ void executeQuery( const Settings & settings = context->getSettingsRef(); - if (settings.max_distributed_depth && context->getClientInfo().distributed_depth > settings.max_distributed_depth) + if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth) throw Exception("Maximum distributed depth exceeded", ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH); std::vector plans; diff --git a/src/Interpreters/ComparisonGraph.cpp b/src/Interpreters/ComparisonGraph.cpp index e236de67fdc..37d603b4923 100644 --- a/src/Interpreters/ComparisonGraph.cpp +++ b/src/Interpreters/ComparisonGraph.cpp @@ -159,7 +159,7 @@ ComparisonGraph::CompareResult ComparisonGraph::pathToCompareResult(Path path, b __builtin_unreachable(); } -std::optional ComparisonGraph::findPath(const size_t start, const size_t finish) const +std::optional ComparisonGraph::findPath(size_t start, size_t finish) const { const auto it = dists.find(std::make_pair(start, finish)); if (it == std::end(dists)) @@ -232,7 +232,7 @@ ComparisonGraph::CompareResult ComparisonGraph::compare(const ASTPtr & left, con return CompareResult::UNKNOWN; } -bool ComparisonGraph::isPossibleCompare(const CompareResult expected, const ASTPtr & left, const ASTPtr & right) const +bool ComparisonGraph::isPossibleCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const { const auto result = compare(left, right); @@ -267,7 +267,7 @@ bool ComparisonGraph::isPossibleCompare(const CompareResult expected, const ASTP return possible_pairs.contains({expected, result}); } -bool ComparisonGraph::isAlwaysCompare(const CompareResult expected, const ASTPtr & left, const ASTPtr & right) const +bool ComparisonGraph::isAlwaysCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const { const auto result = compare(left, right); @@ -324,12 +324,12 @@ std::optional ComparisonGraph::getComponentId(const ASTPtr & ast) const } } -bool ComparisonGraph::hasPath(const size_t left, const size_t right) const +bool ComparisonGraph::hasPath(size_t left, size_t right) const { return findPath(left, right) || findPath(right, left); } -ASTs ComparisonGraph::getComponent(const size_t id) const +ASTs ComparisonGraph::getComponent(size_t id) const { return graph.vertices[id].asts; } @@ -387,7 +387,7 @@ ComparisonGraph::CompareResult ComparisonGraph::functionNameToCompareResult(cons return it == std::end(relation_to_compare) ? CompareResult::UNKNOWN : it->second; } -ComparisonGraph::CompareResult ComparisonGraph::inverseCompareResult(const CompareResult result) +ComparisonGraph::CompareResult ComparisonGraph::inverseCompareResult(CompareResult result) { static const std::unordered_map inverse_relations = { @@ -486,7 +486,7 @@ std::vector ComparisonGraph::getVertices() const void ComparisonGraph::dfsComponents( const Graph & reversed_graph, size_t v, - OptionalIndices & components, const size_t component) + OptionalIndices & components, size_t component) { components[v] = component; for (const auto & edge : reversed_graph.edges[v]) diff --git a/src/Interpreters/ComparisonGraph.h b/src/Interpreters/ComparisonGraph.h index 20d6f135a0d..3891fbf51cf 100644 --- a/src/Interpreters/ComparisonGraph.h +++ b/src/Interpreters/ComparisonGraph.h @@ -17,7 +17,7 @@ class ComparisonGraph { public: /// atomic_formulas are extracted from constraints. - ComparisonGraph(const std::vector & atomic_formulas); + explicit ComparisonGraph(const std::vector & atomic_formulas); enum class CompareResult { @@ -32,15 +32,15 @@ public: static CompareResult atomToCompareResult(const CNFQuery::AtomicFormula & atom); static CompareResult functionNameToCompareResult(const std::string & name); - static CompareResult inverseCompareResult(const CompareResult result); + static CompareResult inverseCompareResult(CompareResult result); CompareResult compare(const ASTPtr & left, const ASTPtr & right) const; /// It's possible that left right - bool isPossibleCompare(const CompareResult expected, const ASTPtr & left, const ASTPtr & right) const; + bool isPossibleCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const; /// It's always true that left right - bool isAlwaysCompare(const CompareResult expected, const ASTPtr & left, const ASTPtr & right) const; + bool isAlwaysCompare(CompareResult expected, const ASTPtr & left, const ASTPtr & right) const; /// Returns all expressions from component to which @ast belongs if any. std::vector getEqual(const ASTPtr & ast) const; @@ -52,11 +52,11 @@ public: std::optional getComponentId(const ASTPtr & ast) const; /// Returns all expressions from component. - std::vector getComponent(const std::size_t id) const; + std::vector getComponent(size_t id) const; size_t getNumOfComponents() const { return graph.vertices.size(); } - bool hasPath(const size_t left, const size_t right) const; + bool hasPath(size_t left, size_t right) const; /// Find constants lessOrEqual and greaterOrEqual. /// For int and double linear programming can be applied here. @@ -131,7 +131,7 @@ private: /// Assigns index of component for each vertex. static void dfsComponents( const Graph & reversed_graph, size_t v, - OptionalIndices & components, const size_t component); + OptionalIndices & components, size_t component); enum class Path { @@ -140,7 +140,7 @@ private: }; static CompareResult pathToCompareResult(Path path, bool inverse); - std::optional findPath(const size_t start, const size_t finish) const; + std::optional findPath(size_t start, size_t finish) const; /// Calculate @dists. static std::map, Path> buildDistsFromGraph(const Graph & g); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 822f1dcb534..ac1bfc620b0 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -179,10 +180,10 @@ struct ContextSharedPart mutable VolumePtr backups_volume; /// Volume for all the backups. - mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. - mutable std::optional external_dictionaries_loader; - mutable std::optional external_user_defined_executable_functions_loader; - mutable std::optional external_models_loader; + mutable std::unique_ptr embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. + mutable std::unique_ptr external_dictionaries_loader; + mutable std::unique_ptr external_user_defined_executable_functions_loader; + mutable std::unique_ptr external_models_loader; ExternalLoaderXMLConfigRepository * external_models_config_repository = nullptr; scope_guard models_repository_guard; @@ -208,15 +209,16 @@ struct ContextSharedPart mutable MarkCachePtr index_mark_cache; /// Cache of marks in compressed files of MergeTree indices. mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads. ProcessList process_list; /// Executing queries at the moment. + GlobalOvercommitTracker global_overcommit_tracker; MergeList merge_list; /// The list of executable merge (for (Replicated)?MergeTree) ReplicatedFetchList replicated_fetch_list; ConfigurationPtr users_config; /// Config with the users, profiles and quotas sections. InterserverIOHandler interserver_io_handler; /// Handler for interserver communication. - mutable std::optional buffer_flush_schedule_pool; /// A thread pool that can do background flush for Buffer tables. - mutable std::optional schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables) - mutable std::optional distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends) - mutable std::optional message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used for message brokers, like RabbitMQ and Kafka) + mutable std::unique_ptr buffer_flush_schedule_pool; /// A thread pool that can do background flush for Buffer tables. + mutable std::unique_ptr schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables) + mutable std::unique_ptr distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends) + mutable std::unique_ptr message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used for message brokers, like RabbitMQ and Kafka) mutable ThrottlerPtr replicated_fetches_throttler; /// A server-wide throttler for replicated fetches mutable ThrottlerPtr replicated_sends_throttler; /// A server-wide throttler for replicated sends @@ -275,7 +277,9 @@ struct ContextSharedPart Context::ConfigReloadCallback config_reload_callback; ContextSharedPart() - : access_control(std::make_unique()), macros(std::make_unique()) + : access_control(std::make_unique()) + , global_overcommit_tracker(&process_list) + , macros(std::make_unique()) { /// TODO: make it singleton (?) static std::atomic num_calls{0}; @@ -290,6 +294,17 @@ struct ContextSharedPart ~ContextSharedPart() { + /// Wait for thread pool for background writes, + /// since it may use per-user MemoryTracker which will be destroyed here. + try + { + IDiskRemote::getThreadPoolWriter().wait(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + try { shutdown(); @@ -341,12 +356,23 @@ struct ContextSharedPart common_executor->wait(); std::unique_ptr delete_system_logs; + std::unique_ptr delete_embedded_dictionaries; + std::unique_ptr delete_external_dictionaries_loader; + std::unique_ptr delete_external_user_defined_executable_functions_loader; + std::unique_ptr delete_external_models_loader; + std::unique_ptr delete_buffer_flush_schedule_pool; + std::unique_ptr delete_schedule_pool; + std::unique_ptr delete_distributed_schedule_pool; + std::unique_ptr delete_message_broker_schedule_pool; + std::unique_ptr delete_ddl_worker; + std::unique_ptr delete_access_control; + { auto lock = std::lock_guard(mutex); - /** Compiled expressions stored in cache need to be destroyed before destruction of static objects. - * Because CHJIT instance can be static object. - */ + /** Compiled expressions stored in cache need to be destroyed before destruction of static objects. + * Because CHJIT instance can be static object. + */ #if USE_EMBEDDED_COMPILER if (auto * cache = CompiledExpressionCacheFactory::instance().tryGetCache()) cache->reset(); @@ -366,19 +392,19 @@ struct ContextSharedPart /// but at least they can be preserved for storage termination. dictionaries_xmls.reset(); user_defined_executable_functions_xmls.reset(); + models_repository_guard.reset(); delete_system_logs = std::move(system_logs); - embedded_dictionaries.reset(); - external_dictionaries_loader.reset(); - external_user_defined_executable_functions_loader.reset(); - models_repository_guard.reset(); - external_models_loader.reset(); - buffer_flush_schedule_pool.reset(); - schedule_pool.reset(); - distributed_schedule_pool.reset(); - message_broker_schedule_pool.reset(); - ddl_worker.reset(); - access_control.reset(); + delete_embedded_dictionaries = std::move(embedded_dictionaries); + delete_external_dictionaries_loader = std::move(external_dictionaries_loader); + delete_external_user_defined_executable_functions_loader = std::move(external_user_defined_executable_functions_loader); + delete_external_models_loader = std::move(external_models_loader); + delete_buffer_flush_schedule_pool = std::move(buffer_flush_schedule_pool); + delete_schedule_pool = std::move(schedule_pool); + delete_distributed_schedule_pool = std::move(distributed_schedule_pool); + delete_message_broker_schedule_pool = std::move(message_broker_schedule_pool); + delete_ddl_worker = std::move(ddl_worker); + delete_access_control = std::move(access_control); /// Stop trace collector if any trace_collector.reset(); @@ -388,6 +414,17 @@ struct ContextSharedPart /// Can be removed w/o context lock delete_system_logs.reset(); + delete_embedded_dictionaries.reset(); + delete_external_dictionaries_loader.reset(); + delete_external_user_defined_executable_functions_loader.reset(); + delete_external_models_loader.reset(); + delete_ddl_worker.reset(); + delete_buffer_flush_schedule_pool.reset(); + delete_schedule_pool.reset(); + delete_distributed_schedule_pool.reset(); + delete_message_broker_schedule_pool.reset(); + delete_ddl_worker.reset(); + delete_access_control.reset(); } bool hasTraceCollector() const @@ -419,7 +456,7 @@ Context::Context(const Context &) = default; Context & Context::operator=(const Context &) = default; SharedContextHolder::SharedContextHolder(SharedContextHolder &&) noexcept = default; -SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) = default; +SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) noexcept = default; SharedContextHolder::SharedContextHolder() = default; SharedContextHolder::~SharedContextHolder() = default; SharedContextHolder::SharedContextHolder(std::unique_ptr shared_context) @@ -474,6 +511,7 @@ std::unique_lock Context::getLock() const ProcessList & Context::getProcessList() { return shared->process_list; } const ProcessList & Context::getProcessList() const { return shared->process_list; } +OvercommitTracker * Context::getGlobalOvercommitTracker() const { return &shared->global_overcommit_tracker; } MergeList & Context::getMergeList() { return shared->merge_list; } const MergeList & Context::getMergeList() const { return shared->merge_list; } ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; } @@ -1361,7 +1399,8 @@ ExternalDictionariesLoader & Context::getExternalDictionariesLoader() ExternalDictionariesLoader & Context::getExternalDictionariesLoaderUnlocked() { if (!shared->external_dictionaries_loader) - shared->external_dictionaries_loader.emplace(getGlobalContext()); + shared->external_dictionaries_loader = + std::make_unique(getGlobalContext()); return *shared->external_dictionaries_loader; } @@ -1379,7 +1418,8 @@ ExternalUserDefinedExecutableFunctionsLoader & Context::getExternalUserDefinedEx ExternalUserDefinedExecutableFunctionsLoader & Context::getExternalUserDefinedExecutableFunctionsLoaderUnlocked() { if (!shared->external_user_defined_executable_functions_loader) - shared->external_user_defined_executable_functions_loader.emplace(getGlobalContext()); + shared->external_user_defined_executable_functions_loader = + std::make_unique(getGlobalContext()); return *shared->external_user_defined_executable_functions_loader; } @@ -1397,7 +1437,8 @@ ExternalModelsLoader & Context::getExternalModelsLoader() ExternalModelsLoader & Context::getExternalModelsLoaderUnlocked() { if (!shared->external_models_loader) - shared->external_models_loader.emplace(getGlobalContext()); + shared->external_models_loader = + std::make_unique(getGlobalContext()); return *shared->external_models_loader; } @@ -1432,7 +1473,7 @@ EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool throw_on_ { auto geo_dictionaries_loader = std::make_unique(); - shared->embedded_dictionaries.emplace( + shared->embedded_dictionaries = std::make_unique( std::move(geo_dictionaries_loader), getGlobalContext(), throw_on_error); @@ -1691,7 +1732,7 @@ BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const { auto lock = getLock(); if (!shared->buffer_flush_schedule_pool) - shared->buffer_flush_schedule_pool.emplace( + shared->buffer_flush_schedule_pool = std::make_unique( settings.background_buffer_flush_schedule_pool_size, CurrentMetrics::BackgroundBufferFlushSchedulePoolTask, "BgBufSchPool"); @@ -1733,7 +1774,7 @@ BackgroundSchedulePool & Context::getSchedulePool() const { auto lock = getLock(); if (!shared->schedule_pool) - shared->schedule_pool.emplace( + shared->schedule_pool = std::make_unique( settings.background_schedule_pool_size, CurrentMetrics::BackgroundSchedulePoolTask, "BgSchPool"); @@ -1744,7 +1785,7 @@ BackgroundSchedulePool & Context::getDistributedSchedulePool() const { auto lock = getLock(); if (!shared->distributed_schedule_pool) - shared->distributed_schedule_pool.emplace( + shared->distributed_schedule_pool = std::make_unique( settings.background_distributed_schedule_pool_size, CurrentMetrics::BackgroundDistributedSchedulePoolTask, "BgDistSchPool"); @@ -1755,7 +1796,7 @@ BackgroundSchedulePool & Context::getMessageBrokerSchedulePool() const { auto lock = getLock(); if (!shared->message_broker_schedule_pool) - shared->message_broker_schedule_pool.emplace( + shared->message_broker_schedule_pool = std::make_unique( settings.background_message_broker_schedule_pool_size, CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask, "BgMBSchPool"); @@ -3137,6 +3178,8 @@ ReadSettings Context::getReadSettings() const res.remote_fs_read_max_backoff_ms = settings.remote_fs_read_max_backoff_ms; res.remote_fs_read_backoff_max_tries = settings.remote_fs_read_backoff_max_tries; + res.remote_fs_enable_cache = settings.remote_fs_enable_cache; + res.remote_fs_cache_max_wait_sec = settings.remote_fs_cache_max_wait_sec; res.remote_read_min_bytes_for_seek = settings.remote_read_min_bytes_for_seek; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6b0a4671efb..c3615db9068 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -29,6 +29,7 @@ namespace Poco::Net { class IPAddress; } namespace zkutil { class ZooKeeper; } +struct OvercommitTracker; namespace DB { @@ -169,7 +170,7 @@ struct SharedContextHolder explicit SharedContextHolder(std::unique_ptr shared_context); SharedContextHolder(SharedContextHolder &&) noexcept; - SharedContextHolder & operator=(SharedContextHolder &&); + SharedContextHolder & operator=(SharedContextHolder &&) noexcept; ContextSharedPart * get() const { return shared.get(); } void reset(); @@ -657,6 +658,8 @@ public: ProcessList & getProcessList(); const ProcessList & getProcessList() const; + OvercommitTracker * getGlobalOvercommitTracker() const; + MergeList & getMergeList(); const MergeList & getMergeList() const; diff --git a/src/Interpreters/CrashLog.h b/src/Interpreters/CrashLog.h index 61503d95cee..930515c43ea 100644 --- a/src/Interpreters/CrashLog.h +++ b/src/Interpreters/CrashLog.h @@ -41,7 +41,7 @@ class CrashLog : public SystemLog public: static void initialize(std::shared_ptr crash_log_) { - crash_log = std::move(crash_log_); + crash_log = crash_log_; } }; diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 64b9bf88ae9..a490d7bed43 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -259,13 +259,17 @@ bool DDLTask::tryFindHostInCluster() * */ is_circular_replicated = true; auto * query_with_table = dynamic_cast(query.get()); - if (!query_with_table || !query_with_table->database) + + /// For other DDLs like CREATE USER, there is no database name and should be executed successfully. + if (query_with_table) { - throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, - "For a distributed DDL on circular replicated cluster its table name must be qualified by database name."); + if (!query_with_table->database) + throw Exception(ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "For a distributed DDL on circular replicated cluster its table name must be qualified by database name."); + + if (default_database == query_with_table->getDatabase()) + return true; } - if (default_database == query_with_table->getDatabase()) - return true; } } found_exact_match = true; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 117119a3ee8..360a5d430e0 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -103,13 +103,13 @@ TemporaryTableHolder::TemporaryTableHolder( { } -TemporaryTableHolder::TemporaryTableHolder(TemporaryTableHolder && rhs) +TemporaryTableHolder::TemporaryTableHolder(TemporaryTableHolder && rhs) noexcept : WithContext(rhs.context), temporary_tables(rhs.temporary_tables), id(rhs.id) { rhs.id = UUIDHelpers::Nil; } -TemporaryTableHolder & TemporaryTableHolder::operator = (TemporaryTableHolder && rhs) +TemporaryTableHolder & TemporaryTableHolder::operator=(TemporaryTableHolder && rhs) noexcept { id = rhs.id; rhs.id = UUIDHelpers::Nil; diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index a32995658f1..34b42a3397c 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -98,8 +98,8 @@ struct TemporaryTableHolder : boost::noncopyable, WithContext const ASTPtr & query = {}, bool create_for_global_subquery = false); - TemporaryTableHolder(TemporaryTableHolder && rhs); - TemporaryTableHolder & operator = (TemporaryTableHolder && rhs); + TemporaryTableHolder(TemporaryTableHolder && rhs) noexcept; + TemporaryTableHolder & operator=(TemporaryTableHolder && rhs) noexcept; ~TemporaryTableHolder(); @@ -107,7 +107,7 @@ struct TemporaryTableHolder : boost::noncopyable, WithContext StoragePtr getTable() const; - operator bool () const { return id != UUIDHelpers::Nil; } + operator bool () const { return id != UUIDHelpers::Nil; } /// NOLINT IDatabase * temporary_tables = nullptr; UUID id = UUIDHelpers::Nil; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 30c832e4917..83f8de78fa6 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -748,7 +748,7 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run) if (execution_context.columns[pos].column) res.insert(execution_context.columns[pos]); - for (const auto & item : block) + for (auto && item : block) res.insert(std::move(item)); block.swap(res); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 4a5f18a408f..841d7bc567f 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -259,7 +259,7 @@ NamesAndTypesList ExpressionAnalyzer::getColumnsAfterArrayJoin(ActionsDAGPtr & a if (!array_join_expression_list) return src_columns; - getRootActionsNoMakeSet(array_join_expression_list, true, actions, false); + getRootActionsNoMakeSet(array_join_expression_list, actions, false); auto array_join = addMultipleArrayJoinAction(actions, is_array_join_left); auto sample_columns = actions->getResultColumns(); @@ -294,14 +294,14 @@ NamesAndTypesList ExpressionAnalyzer::analyzeJoin(ActionsDAGPtr & actions, const const ASTTablesInSelectQueryElement * join = select_query->join(); if (join) { - getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), true, actions, false); + getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), actions, false); auto sample_columns = actions->getNamesAndTypesList(); syntax->analyzed_join->addJoinedColumnsAndCorrectTypes(sample_columns, true); actions = std::make_shared(sample_columns); } NamesAndTypesList result_columns = src_columns; - syntax->analyzed_join->addJoinedColumnsAndCorrectTypes(result_columns,false); + syntax->analyzed_join->addJoinedColumnsAndCorrectTypes(result_columns, false); return result_columns; } @@ -332,14 +332,14 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) { NameSet unique_keys; ASTs & group_asts = group_by_ast->children; - for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i) + for (ssize_t i = 0; i < static_cast(group_asts.size()); ++i) { ssize_t size = group_asts.size(); if (getContext()->getSettingsRef().enable_positional_arguments) replaceForPositionalArguments(group_asts[i], select_query, ASTSelectQuery::Expression::GROUP_BY); - getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); + getRootActionsNoMakeSet(group_asts[i], temp_actions, false); const auto & column_name = group_asts[i]->getColumnName(); @@ -405,8 +405,8 @@ void ExpressionAnalyzer::initGlobalSubqueriesAndExternalTables(bool do_global) { if (do_global) { - GlobalSubqueriesVisitor::Data subqueries_data(getContext(), subquery_depth, isRemoteStorage(), - external_tables, subqueries_for_sets, has_global_subqueries); + GlobalSubqueriesVisitor::Data subqueries_data( + getContext(), subquery_depth, isRemoteStorage(), external_tables, subqueries_for_sets, has_global_subqueries); GlobalSubqueriesVisitor(subqueries_data).visit(query); } } @@ -416,7 +416,7 @@ void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_ { auto set_key = PreparedSetKey::forSubquery(*subquery_or_table_name); - if (prepared_sets.count(set_key)) + if (prepared_sets.contains(set_key)) return; /// Already prepared. if (auto set_ptr_from_storage_set = isPlainStorageSetInSubquery(subquery_or_table_name)) @@ -509,33 +509,62 @@ void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node) } -void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_subqueries, ActionsDAGPtr & actions, bool only_consts) +void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts) { LogAST log; - ActionsVisitor::Data visitor_data(getContext(), settings.size_limits_for_set, subquery_depth, - sourceColumns(), std::move(actions), prepared_sets, subqueries_for_sets, - no_subqueries, false, only_consts, !isRemoteStorage()); + ActionsVisitor::Data visitor_data( + getContext(), + settings.size_limits_for_set, + subquery_depth, + sourceColumns(), + std::move(actions), + prepared_sets, + subqueries_for_sets, + no_makeset_for_subqueries, + false /* no_makeset */, + only_consts, + !isRemoteStorage() /* create_source_for_in */); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } -void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, bool no_subqueries, ActionsDAGPtr & actions, bool only_consts) +void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGPtr & actions, bool only_consts) { LogAST log; - ActionsVisitor::Data visitor_data(getContext(), settings.size_limits_for_set, subquery_depth, - sourceColumns(), std::move(actions), prepared_sets, subqueries_for_sets, - no_subqueries, true, only_consts, !isRemoteStorage()); + ActionsVisitor::Data visitor_data( + getContext(), + settings.size_limits_for_set, + subquery_depth, + sourceColumns(), + std::move(actions), + prepared_sets, + subqueries_for_sets, + true /* no_makeset_for_subqueries, no_makeset implies no_makeset_for_subqueries */, + true /* no_makeset */, + only_consts, + !isRemoteStorage() /* create_source_for_in */); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } -void ExpressionAnalyzer::getRootActionsForHaving(const ASTPtr & ast, bool no_subqueries, ActionsDAGPtr & actions, bool only_consts) + +void ExpressionAnalyzer::getRootActionsForHaving( + const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts) { LogAST log; - ActionsVisitor::Data visitor_data(getContext(), settings.size_limits_for_set, subquery_depth, - sourceColumns(), std::move(actions), prepared_sets, subqueries_for_sets, - no_subqueries, false, only_consts, true); + ActionsVisitor::Data visitor_data( + getContext(), + settings.size_limits_for_set, + subquery_depth, + sourceColumns(), + std::move(actions), + prepared_sets, + subqueries_for_sets, + no_makeset_for_subqueries, + false /* no_makeset */, + only_consts, + true /* create_source_for_in */); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -547,7 +576,7 @@ void ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions, Aggr { AggregateDescription aggregate; if (node->arguments) - getRootActionsNoMakeSet(node->arguments, true, actions); + getRootActionsNoMakeSet(node->arguments, actions); aggregate.column_name = node->getColumnName(); @@ -746,8 +775,7 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAGPtr actions) // Requiring a constant reference to a shared pointer to non-const AST // doesn't really look sane, but the visitor does indeed require it. // Hence we clone the node (not very sane either, I know). - getRootActionsNoMakeSet(window_function.function_node->clone(), - true, actions); + getRootActionsNoMakeSet(window_function.function_node->clone(), actions); const ASTs & arguments = window_function.function_node->arguments->children; @@ -867,8 +895,7 @@ ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActi auto array_join = addMultipleArrayJoinAction(step.actions(), is_array_join_left); before_array_join = chain.getLastActions(); - chain.steps.push_back(std::make_unique( - array_join, step.getResultColumns())); + chain.steps.push_back(std::make_unique(array_join, step.getResultColumns())); chain.addStep(); @@ -991,7 +1018,8 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( if (auto storage = analyzed_join->getStorageJoin()) { - std::tie(left_convert_actions, right_convert_actions) = analyzed_join->createConvertingActions(left_columns, {}); + auto right_columns = storage->getRightSampleBlock().getColumnsWithTypeAndName(); + std::tie(left_convert_actions, right_convert_actions) = analyzed_join->createConvertingActions(left_columns, right_columns); return storage->getJoinLocked(analyzed_join, getContext()); } @@ -1098,8 +1126,8 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere( } } - chain.steps.emplace_back(std::make_unique( - std::make_shared(std::move(columns)))); + chain.steps.emplace_back( + std::make_unique(std::make_shared(std::move(columns)))); chain.steps.back()->additional_input = std::move(unused_source_columns); chain.getLastActions(); chain.addStep(); @@ -1209,8 +1237,7 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( // recursively together with (1b) as ASTFunction::window_definition. if (getSelectQuery()->window()) { - getRootActionsNoMakeSet(getSelectQuery()->window(), - true /* no_subqueries */, step.actions()); + getRootActionsNoMakeSet(getSelectQuery()->window(), step.actions()); } for (const auto & [_, w] : window_descriptions) @@ -1221,8 +1248,7 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments( // definitions (1a). // Requiring a constant reference to a shared pointer to non-const AST // doesn't really look sane, but the visitor does indeed require it. - getRootActionsNoMakeSet(f.function_node->clone(), - true /* no_subqueries */, step.actions()); + getRootActionsNoMakeSet(f.function_node->clone(), step.actions()); // (2b) Required function argument columns. for (const auto & a : f.function_node->arguments->children) @@ -1366,7 +1392,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain auto child_name = child->getColumnName(); if (!aggregated_names.count(child_name)) - step.addRequiredOutput(std::move(child_name)); + step.addRequiredOutput(child_name); } return true; @@ -1455,7 +1481,7 @@ ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool project_r alias = name; result_columns.emplace_back(name, alias); result_names.push_back(alias); - getRootActions(ast, false, actions_dag); + getRootActions(ast, false /* no_makeset_for_subqueries */, actions_dag); } if (add_aliases) @@ -1495,7 +1521,7 @@ ExpressionActionsPtr ExpressionAnalyzer::getConstActions(const ColumnsWithTypeAn { auto actions = std::make_shared(constant_inputs); - getRootActions(query, true, actions, true); + getRootActions(query, true /* no_makeset_for_subqueries */, actions, true /* only_consts */); return std::make_shared(actions, ExpressionActionsSettings::fromContext(getContext())); } @@ -1512,13 +1538,13 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::simpleSelectActions() } ExpressionAnalysisResult::ExpressionAnalysisResult( - SelectQueryExpressionAnalyzer & query_analyzer, - const StorageMetadataPtr & metadata_snapshot, - bool first_stage_, - bool second_stage_, - bool only_types, - const FilterDAGInfoPtr & filter_info_, - const Block & source_header) + SelectQueryExpressionAnalyzer & query_analyzer, + const StorageMetadataPtr & metadata_snapshot, + bool first_stage_, + bool second_stage_, + bool only_types, + const FilterDAGInfoPtr & filter_info_, + const Block & source_header) : first_stage(first_stage_) , second_stage(second_stage_) , need_aggregate(query_analyzer.hasAggregation()) diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index b6bb3c5fad5..5dcbdc2486b 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -172,15 +172,15 @@ protected: ArrayJoinActionPtr addMultipleArrayJoinAction(ActionsDAGPtr & actions, bool is_left) const; - void getRootActions(const ASTPtr & ast, bool no_subqueries, ActionsDAGPtr & actions, bool only_consts = false); + void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false); /** Similar to getRootActions but do not make sets when analyzing IN functions. It's used in * analyzeAggregation which happens earlier than analyzing PREWHERE and WHERE. If we did, the * prepared sets would not be applicable for MergeTree index optimization. */ - void getRootActionsNoMakeSet(const ASTPtr & ast, bool no_subqueries, ActionsDAGPtr & actions, bool only_consts = false); + void getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGPtr & actions, bool only_consts = false); - void getRootActionsForHaving(const ASTPtr & ast, bool no_subqueries, ActionsDAGPtr & actions, bool only_consts = false); + void getRootActionsForHaving(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false); /** Add aggregation keys to aggregation_keys, aggregate functions to aggregate_descriptions, * Create a set of columns aggregated_columns resulting after the aggregation, if any, diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index aab3a9e7437..a75cdce820c 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -56,7 +56,7 @@ namespace static_assert(std::is_same_v); ExternalLoader::Loadables objects; objects.reserve(results.size()); - for (const auto & result : results) + for (auto && result : results) { if (auto object = std::move(result.object)) objects.push_back(std::move(object)); diff --git a/src/Interpreters/ExternalModelsLoader.h b/src/Interpreters/ExternalModelsLoader.h index 18e1f1123f6..042906bee9e 100644 --- a/src/Interpreters/ExternalModelsLoader.h +++ b/src/Interpreters/ExternalModelsLoader.h @@ -15,14 +15,14 @@ namespace DB class ExternalModelsLoader : public ExternalLoader, WithContext { public: - using ModelPtr = std::shared_ptr; + using ModelPtr = std::shared_ptr; /// Models will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds. explicit ExternalModelsLoader(ContextPtr context_); ModelPtr getModel(const std::string & model_name) const { - return std::static_pointer_cast(load(model_name)); + return std::static_pointer_cast(load(model_name)); } void reloadModel(const std::string & model_name) const diff --git a/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp index b266746642f..e3d40033cff 100644 --- a/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Interpreters/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -83,6 +83,10 @@ ExternalLoader::LoadablePtr ExternalUserDefinedExecutableFunctionsLoader::create String format = config.getString(key_in_config + ".format"); DataTypePtr result_type = DataTypeFactory::instance().get(config.getString(key_in_config + ".return_type")); + String result_name = "result"; + if (config.has(key_in_config + ".return_name")) + result_name = config.getString(key_in_config + ".return_name"); + bool send_chunk_header = config.getBool(key_in_config + ".send_chunk_header", false); size_t command_termination_timeout_seconds = config.getUInt64(key_in_config + ".command_termination_timeout", 10); size_t command_read_timeout_milliseconds = config.getUInt64(key_in_config + ".command_read_timeout", 10000); @@ -106,33 +110,46 @@ ExternalLoader::LoadablePtr ExternalUserDefinedExecutableFunctionsLoader::create if (config.has(key_in_config + ".lifetime")) lifetime = ExternalLoadableLifetime(config, key_in_config + ".lifetime"); - std::vector argument_types; + std::vector arguments; Poco::Util::AbstractConfiguration::Keys config_elems; config.keys(key_in_config, config_elems); + size_t argument_number = 1; + for (const auto & config_elem : config_elems) { if (!startsWith(config_elem, "argument")) continue; + UserDefinedExecutableFunctionArgument argument; + const auto argument_prefix = key_in_config + '.' + config_elem + '.'; - auto argument_type = DataTypeFactory::instance().get(config.getString(argument_prefix + "type")); - argument_types.emplace_back(std::move(argument_type)); + + argument.type = DataTypeFactory::instance().get(config.getString(argument_prefix + "type")); + + if (config.has(argument_prefix + "name")) + argument.name = config.getString(argument_prefix + "name"); + else + argument.name = "c" + std::to_string(argument_number); + + ++argument_number; + arguments.emplace_back(std::move(argument)); } UserDefinedExecutableFunctionConfiguration function_configuration { - .name = std::move(name), //-V1030 - .command = std::move(command_value), //-V1030 - .command_arguments = std::move(command_arguments), //-V1030 - .argument_types = std::move(argument_types), //-V1030 - .result_type = std::move(result_type), //-V1030 + .name = name, + .command = std::move(command_value), + .command_arguments = std::move(command_arguments), + .arguments = std::move(arguments), + .result_type = std::move(result_type), + .result_name = std::move(result_name), }; ShellCommandSourceCoordinator::Configuration shell_command_coordinator_configration { - .format = std::move(format), //-V1030 + .format = std::move(format), .command_termination_timeout_seconds = command_termination_timeout_seconds, .command_read_timeout_milliseconds = command_read_timeout_milliseconds, .command_write_timeout_milliseconds = command_write_timeout_milliseconds, diff --git a/src/Interpreters/GetAggregatesVisitor.h b/src/Interpreters/GetAggregatesVisitor.h index 0eeab8348fd..3966653235a 100644 --- a/src/Interpreters/GetAggregatesVisitor.h +++ b/src/Interpreters/GetAggregatesVisitor.h @@ -2,6 +2,9 @@ #include #include +#include +#include +#include namespace DB { diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 5d2df583b9e..50ce7977534 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,7 +18,11 @@ #include #include #include +#include #include +#include +#include +#include namespace DB { @@ -34,7 +39,6 @@ public: { size_t subquery_depth; bool is_remote; - size_t external_table_id; TemporaryTablesMapping & external_tables; SubqueriesForSets & subqueries_for_sets; bool & has_global_subqueries; @@ -49,7 +53,6 @@ public: : WithContext(context_) , subquery_depth(subquery_depth_) , is_remote(is_remote_) - , external_table_id(1) , external_tables(tables) , subqueries_for_sets(subqueries_for_sets_) , has_global_subqueries(has_global_subqueries_) @@ -92,48 +95,33 @@ public: { /// If this is already an external table, you do not need to add anything. Just remember its presence. auto temporary_table_name = getIdentifierName(subquery_or_table_name); - bool exists_in_local_map = external_tables.end() != external_tables.find(temporary_table_name); + bool exists_in_local_map = external_tables.contains(temporary_table_name); bool exists_in_context = static_cast(getContext()->tryResolveStorageID( StorageID("", temporary_table_name), Context::ResolveExternal)); if (exists_in_local_map || exists_in_context) return; } - String external_table_name = subquery_or_table_name->tryGetAlias(); - if (external_table_name.empty()) + String alias = subquery_or_table_name->tryGetAlias(); + String external_table_name; + if (alias.empty()) { - /// Generate the name for the external table. - external_table_name = "_data" + toString(external_table_id); - while (external_tables.count(external_table_name)) - { - ++external_table_id; - external_table_name = "_data" + toString(external_table_id); - } + auto hash = subquery_or_table_name->getTreeHash(); + external_table_name = fmt::format("_data_{}_{}", hash.first, hash.second); } - - auto interpreter = interpretSubquery(subquery_or_table_name, getContext(), subquery_depth, {}); - - Block sample = interpreter->getSampleBlock(); - NamesAndTypesList columns = sample.getNamesAndTypesList(); - - auto external_storage_holder = std::make_shared( - getContext(), - ColumnsDescription{columns}, - ConstraintsDescription{}, - nullptr, - /*create_for_global_subquery*/ true); - StoragePtr external_storage = external_storage_holder->getTable(); + else + external_table_name = alias; /** We replace the subquery with the name of the temporary table. * It is in this form, the request will go to the remote server. * This temporary table will go to the remote server, and on its side, * instead of doing a subquery, you just need to read it. + * TODO We can do better than using alias to name external tables */ auto database_and_table_name = std::make_shared(external_table_name); if (set_alias) { - String alias = subquery_or_table_name->tryGetAlias(); if (auto * table_name = subquery_or_table_name->as()) if (alias.empty()) alias = table_name->shortName(); @@ -151,8 +139,27 @@ public: else ast = database_and_table_name; - external_tables[external_table_name] = external_storage_holder; + if (external_tables.contains(external_table_name)) + return; + auto interpreter = interpretSubquery(subquery_or_table_name, getContext(), subquery_depth, {}); + + Block sample = interpreter->getSampleBlock(); + NamesAndTypesList columns = sample.getNamesAndTypesList(); + + auto external_storage_holder = std::make_shared( + getContext(), + ColumnsDescription{columns}, + ConstraintsDescription{}, + nullptr, + /*create_for_global_subquery*/ true); + StoragePtr external_storage = external_storage_holder->getTable(); + + external_tables.emplace(external_table_name, external_storage_holder); + + /// We need to materialize external tables immediately because reading from distributed + /// tables might generate local plans which can refer to external tables during index + /// analysis. It's too late to populate the external table via CreatingSetsTransform. if (getContext()->getSettingsRef().use_index_for_in_with_subqueries) { auto external_table = external_storage_holder->getTable(); diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 82d8356b7c7..e81db1427ef 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -113,6 +113,23 @@ namespace JoinStuff } } + template + void JoinUsedFlags::setUsed(const Block * block, size_t row_num, size_t offset) + { + if constexpr (!use_flags) + return; + + /// Could be set simultaneously from different threads. + if constexpr (multiple_disjuncts) + { + flags[block][row_num].store(true, std::memory_order_relaxed); + } + else + { + flags[nullptr][offset].store(true, std::memory_order_relaxed); + } + } + template bool JoinUsedFlags::getUsed(const FindResult & f) { @@ -232,8 +249,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s : table_join(table_join_) , kind(table_join->kind()) , strictness(table_join->strictness()) - , nullable_right_side(table_join->forceNullableRight()) - , nullable_left_side(table_join->forceNullableLeft()) , any_take_last_row(any_take_last_row_) , asof_inequality(table_join->getAsofInequality()) , data(std::make_shared()) @@ -273,9 +288,6 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s JoinCommon::createMissedColumns(sample_block_with_columns_to_add); - if (nullable_right_side) - JoinCommon::convertColumnsToNullable(sample_block_with_columns_to_add); - size_t disjuncts_num = table_join->getClauses().size(); data->maps.resize(disjuncts_num); key_sizes.reserve(disjuncts_num); @@ -306,11 +318,8 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s if (key_columns.size() <= 1) throw Exception("ASOF join needs at least one equi-join column", ErrorCodes::SYNTAX_ERROR); - if (right_table_keys.getByName(key_names_right.back()).type->isNullable()) - throw Exception("ASOF join over right table Nullable column is not implemented", ErrorCodes::NOT_IMPLEMENTED); - size_t asof_size; - asof_type = AsofRowRefs::getTypeSize(*key_columns.back(), asof_size); + asof_type = SortedLookupVectorBase::getTypeSize(*key_columns.back(), asof_size); key_columns.pop_back(); /// this is going to set up the appropriate hash table for the direct lookup part of the join @@ -619,8 +628,8 @@ namespace TypeIndex asof_type = *join.getAsofType(); if (emplace_result.isInserted()) - time_series_map = new (time_series_map) typename Map::mapped_type(asof_type); - time_series_map->insert(asof_type, asof_column, stored_block, i); + time_series_map = new (time_series_map) typename Map::mapped_type(createAsofRowRef(asof_type, join.getAsofInequality())); + (*time_series_map)->insert(asof_column, stored_block, i); } }; @@ -719,12 +728,6 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample) saved_block_sample.insert(column); } } - - if (nullable_right_side) - { - JoinCommon::convertColumnsToNullable(saved_block_sample, (isFull(kind) && !multiple_disjuncts ? right_table_keys.columns() : 0)); - } - } Block HashJoin::structureRightBlock(const Block & block) const @@ -909,8 +912,6 @@ public: bool is_join_get_) : join_on_keys(join_on_keys_) , rows_to_add(block.rows()) - , asof_type(join.getAsofType()) - , asof_inequality(join.getAsofInequality()) , is_join_get(is_join_get_) { size_t num_columns_to_add = block_with_columns_to_add.columns(); @@ -992,8 +993,6 @@ public: } } - TypeIndex asofType() const { return *asof_type; } - ASOF::Inequality asofInequality() const { return asof_inequality; } const IColumn & leftAsofKey() const { return *left_asof_key; } std::vector join_on_keys; @@ -1008,8 +1007,6 @@ private: std::vector right_indexes; size_t lazy_defaults_count = 0; /// for ASOF - std::optional asof_type; - ASOF::Inequality asof_inequality; const IColumn * left_asof_key = nullptr; bool is_join_get; @@ -1238,19 +1235,18 @@ NO_INLINE IColumn::Filter joinRightColumns( auto & mapped = find_result.getMapped(); if constexpr (jf.is_asof_join) { - TypeIndex asof_type = added_columns.asofType(); - ASOF::Inequality asof_inequality = added_columns.asofInequality(); const IColumn & left_asof_key = added_columns.leftAsofKey(); - if (const RowRef * found = mapped.findAsof(asof_type, asof_inequality, left_asof_key, i)) + auto row_ref = mapped->findAsof(left_asof_key, i); + if (row_ref.block) { setUsed(filter, i); if constexpr (multiple_disjuncts) - used_flags.template setUsed(FindResultImpl(found, true, 0)); + used_flags.template setUsed(row_ref.block, row_ref.row_num, 0); else used_flags.template setUsed(find_result); - added_columns.appendFromBlock(*found->block, found->row_num); + added_columns.appendFromBlock(*row_ref.block, row_ref.row_num); } else addNotFoundRow(added_columns, current_offset); @@ -1476,9 +1472,6 @@ void HashJoin::joinBlockImpl( if constexpr (jf.right || jf.full) { materializeBlockInplace(block); - - if (nullable_left_side) - JoinCommon::convertColumnsToNullable(block); } /** For LEFT/INNER JOIN, the saved blocks do not contain keys. @@ -1527,13 +1520,13 @@ void HashJoin::joinBlockImpl( continue; const auto & col = block.getByName(left_name); - bool is_nullable = nullable_right_side || right_key.type->isNullable(); + bool is_nullable = JoinCommon::isNullable(right_key.type); auto right_col_name = getTableJoin().renamedRightColumnName(right_key.name); ColumnWithTypeAndName right_col(col.column, col.type, right_col_name); if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) JoinCommon::changeLowCardinalityInplace(right_col); right_col = correctNullability(std::move(right_col), is_nullable); - block.insert(right_col); + block.insert(std::move(right_col)); } } } @@ -1559,7 +1552,7 @@ void HashJoin::joinBlockImpl( continue; const auto & col = block.getByName(left_name); - bool is_nullable = nullable_right_side || right_key.type->isNullable(); + bool is_nullable = JoinCommon::isNullable(right_key.type); ColumnPtr thin_column = filterWithBlanks(col.column, filter); @@ -1567,7 +1560,7 @@ void HashJoin::joinBlockImpl( if (right_col.type->lowCardinality() != right_key.type->lowCardinality()) JoinCommon::changeLowCardinalityInplace(right_col); right_col = correctNullability(std::move(right_col), is_nullable, null_map_filter); - block.insert(right_col); + block.insert(std::move(right_col)); if constexpr (jf.need_replication) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); @@ -1754,8 +1747,6 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) if (kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full) { materializeBlockInplace(block); - if (nullable_left_side) - JoinCommon::convertColumnsToNullable(block); } if (overDictionary()) @@ -2091,7 +2082,7 @@ void HashJoin::reuseJoinedData(const HashJoin & join) const ColumnWithTypeAndName & HashJoin::rightAsofKeyColumn() const { - /// It should be nullable if nullable_right_side is true + /// It should be nullable when right side is nullable return savedBlockSample().getByName(table_join->getOnlyClause().key_names_right.back()); } diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 50c8b2c55ee..b2a7aedaa5a 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -62,6 +62,9 @@ public: template void setUsed(const T & f); + template + void setUsed(const Block * block, size_t row_num, size_t offset); + template bool getUsed(const T & f); @@ -360,8 +363,6 @@ private: /// This join was created from StorageJoin and it is already filled. bool from_storage_join = false; - bool nullable_right_side; /// In case of LEFT and FULL joins, if use_nulls, convert right-side columns to Nullable. - bool nullable_left_side; /// In case of RIGHT and FULL joins, if use_nulls, convert left-side columns to Nullable. bool any_take_last_row; /// Overwrite existing values when encountering the same key again std::optional asof_type; ASOF::Inequality asof_inequality; diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h index 1f59dd36354..7906ab189fc 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.h +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -47,7 +47,7 @@ public: /// then we can cache the scalars forever (for any query that doesn't use the virtual storage either), but if it does use the virtual /// storage then we can only keep the scalar result around while we are working with that source block /// You can find more details about this under ExecuteScalarSubqueriesMatcher::visit - bool usesViewSource() { return uses_view_source; } + bool usesViewSource() const { return uses_view_source; } protected: ASTPtr query_ptr; diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index ba8367b57e3..2a3171adccd 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -25,7 +25,7 @@ public: /// Add block of data from right hand of JOIN. /// @returns false, if some limit was exceeded and you should not insert more data. - virtual bool addJoinedBlock(const Block & block, bool check_limits = true) = 0; + virtual bool addJoinedBlock(const Block & block, bool check_limits = true) = 0; /// NOLINT virtual void checkTypesOfKeys(const Block & block) const = 0; diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.h b/src/Interpreters/InJoinSubqueriesPreprocessor.h index d2166185d2b..e4ec3c81ed2 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.h +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.h @@ -43,7 +43,7 @@ public: /// These methods could be overridden for the need of the unit test. virtual bool hasAtLeastTwoShards(const IStorage & table) const; virtual std::pair getRemoteDatabaseAndTableName(const IStorage & table) const; - virtual ~CheckShardsAndTables() {} + virtual ~CheckShardsAndTables() = default; }; InJoinSubqueriesPreprocessor( diff --git a/src/Interpreters/InterpreterBackupQuery.cpp b/src/Interpreters/InterpreterBackupQuery.cpp index 45eb8e48599..01970bc5cc2 100644 --- a/src/Interpreters/InterpreterBackupQuery.cpp +++ b/src/Interpreters/InterpreterBackupQuery.cpp @@ -1,10 +1,12 @@ #include +#include +#include +#include #include #include #include -#include -#include -#include +#include +#include #include @@ -12,40 +14,43 @@ namespace DB { namespace { - BackupMutablePtr createBackup(const ASTBackupQuery & query, const ContextPtr & context) + BackupMutablePtr createBackup(const BackupInfo & backup_info, const BackupSettings & backup_settings, const ContextPtr & context) { BackupFactory::CreateParams params; - params.open_mode = (query.kind == ASTBackupQuery::BACKUP) ? IBackup::OpenMode::WRITE : IBackup::OpenMode::READ; + params.open_mode = IBackup::OpenMode::WRITE; params.context = context; - - params.backup_info = BackupInfo::fromAST(*query.backup_name); - if (query.base_backup_name) - params.base_backup_info = BackupInfo::fromAST(*query.base_backup_name); - + params.backup_info = backup_info; + params.base_backup_info = backup_settings.base_backup_info; + params.compression_method = backup_settings.compression_method; + params.compression_level = backup_settings.compression_level; + params.password = backup_settings.password; return BackupFactory::instance().createBackup(params); } -#if 0 - void getBackupSettings(const ASTBackupQuery & query, BackupSettings & settings, std::optional & base_backup) + BackupMutablePtr openBackup(const BackupInfo & backup_info, const RestoreSettings & restore_settings, const ContextPtr & context) { - settings = {}; - if (query.settings) - settings.applyChanges(query.settings->as().changes); - return settings; + BackupFactory::CreateParams params; + params.open_mode = IBackup::OpenMode::READ; + params.context = context; + params.backup_info = backup_info; + params.base_backup_info = restore_settings.base_backup_info; + params.password = restore_settings.password; + return BackupFactory::instance().createBackup(params); } -#endif - void executeBackup(const ASTBackupQuery & query, const ContextPtr & context) + void executeBackup(const ContextPtr & context, const ASTBackupQuery & query) { - BackupMutablePtr backup = createBackup(query, context); - auto backup_entries = makeBackupEntries(query.elements, context); + auto backup_settings = BackupSettings::fromBackupQuery(query); + BackupMutablePtr backup = createBackup(BackupInfo::fromAST(*query.backup_name), backup_settings, context); + auto backup_entries = makeBackupEntries(context, query.elements, backup_settings); writeBackupEntries(backup, std::move(backup_entries), context->getSettingsRef().max_backup_threads); } - void executeRestore(const ASTBackupQuery & query, ContextMutablePtr context) + void executeRestore(ContextMutablePtr context, const ASTBackupQuery & query) { - BackupPtr backup = createBackup(query, context); - auto restore_tasks = makeRestoreTasks(query.elements, context, backup); + auto restore_settings = RestoreSettings::fromRestoreQuery(query); + BackupPtr backup = openBackup(BackupInfo::fromAST(*query.backup_name), restore_settings, context); + auto restore_tasks = makeRestoreTasks(context, backup, query.elements, restore_settings); executeRestoreTasks(std::move(restore_tasks), context->getSettingsRef().max_backup_threads); } } @@ -54,9 +59,9 @@ BlockIO InterpreterBackupQuery::execute() { const auto & query = query_ptr->as(); if (query.kind == ASTBackupQuery::BACKUP) - executeBackup(query, context); + executeBackup(context, query); else if (query.kind == ASTBackupQuery::RESTORE) - executeRestore(query, context); + executeRestore(context, query); return {}; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 4e0561ad750..f7dbd1c8b65 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include @@ -52,6 +53,7 @@ #include #include #include +#include #include #include @@ -94,6 +96,7 @@ namespace ErrorCodes extern const int PATH_ACCESS_DENIED; extern const int NOT_IMPLEMENTED; extern const int ENGINE_REQUIRED; + extern const int UNKNOWN_STORAGE; } namespace fs = std::filesystem; @@ -642,6 +645,11 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti properties.indices = as_storage_metadata->getSecondaryIndices(); properties.projections = as_storage_metadata->getProjections().clone(); } + else + { + /// Only MergeTree support TTL + properties.columns.resetColumnTTLs(); + } properties.constraints = as_storage_metadata->getConstraints(); } @@ -726,11 +734,26 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat { String message = "Cannot create table with column '" + name_and_type_pair.name + "' which type is '" + type + "' because experimental geo types are not allowed. " - + "Set setting allow_experimental_geo_types = 1 in order to allow it."; + + "Set setting allow_experimental_geo_types = 1 in order to allow it"; throw Exception(message, ErrorCodes::ILLEGAL_COLUMN); } } } + + if (!create.attach && !settings.allow_experimental_object_type) + { + for (const auto & [name, type] : properties.columns.getAllPhysical()) + { + if (isObject(type)) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Cannot create table with column '{}' which type is '{}' " + "because experimental Object type is not allowed. " + "Set setting allow_experimental_object_type = 1 in order to allow it", + name, type->getName()); + } + } + } } String InterpreterCreateQuery::getTableEngineName(DefaultTableEngine default_table_engine) @@ -1057,7 +1080,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) QualifiedTableName qualified_name{database_name, create.getTable()}; TableNamesSet loading_dependencies = getDependenciesSetFromCreateQuery(getContext()->getGlobalContext(), qualified_name, query_ptr); if (!loading_dependencies.empty()) - DatabaseCatalog::instance().addLoadingDependencies(std::move(qualified_name), std::move(loading_dependencies)); + DatabaseCatalog::instance().addLoadingDependencies(qualified_name, std::move(loading_dependencies)); return fillTableIfNeeded(create); } @@ -1106,6 +1129,20 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, throw Exception(storage_already_exists_error_code, "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); } + else if (!create.attach) + { + /// Checking that table may exists in detached/detached permanently state + try + { + database->checkMetadataFilenameAvailability(create.getTable()); + } + catch (const Exception &) + { + if (create.if_not_exists) + return false; + throw; + } + } data_path = database->getTableDataPath(create); @@ -1171,6 +1208,17 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, properties.columns, properties.constraints, false); + + /// If schema wes inferred while storage creation, add columns description to create query. + addColumnsDescriptionToCreateQueryIfNecessary(query_ptr->as(), res); + } + + if (!create.attach && getContext()->getSettingsRef().database_replicated_allow_only_replicated_engine) + { + bool is_replicated_storage = typeid_cast(res.get()) != nullptr; + if (!is_replicated_storage && res->storesDataOnDisk() && database && database->getEngineName() == "Replicated") + throw Exception(ErrorCodes::UNKNOWN_STORAGE, + "Only table with Replicated engine or tables which does not store data on disk are allowed in Replicated database"); } if (from_path && !res->storesDataOnDisk()) @@ -1198,6 +1246,14 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, /// we can safely destroy the object without a call to "shutdown", because there is guarantee /// that no background threads/similar resources remain after exception from "startup". + if (!res->supportsDynamicSubcolumns() && hasObjectColumns(res->getInMemoryMetadataPtr()->getColumns())) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Cannot create table with column of type Object, " + "because storage {} doesn't support dynamic subcolumns", + res->getName()); + } + res->startup(); return true; } @@ -1470,4 +1526,26 @@ void InterpreterCreateQuery::extendQueryLogElemImpl(QueryLogElement & elem, cons } } +void InterpreterCreateQuery::addColumnsDescriptionToCreateQueryIfNecessary(ASTCreateQuery & create, const StoragePtr & storage) +{ + if (create.is_dictionary || (create.columns_list && create.columns_list->columns && !create.columns_list->columns->children.empty())) + return; + + auto ast_storage = std::make_shared(); + auto query_from_storage = DB::getCreateQueryFromStorage(storage, ast_storage, false, + getContext()->getSettingsRef().max_parser_depth, true); + auto & create_query_from_storage = query_from_storage->as(); + + if (!create.columns_list) + { + ASTPtr columns_list = std::make_shared(*create_query_from_storage.columns_list); + create.set(create.columns_list, columns_list); + } + else + { + ASTPtr columns = std::make_shared(*create_query_from_storage.columns_list->columns); + create.columns_list->set(create.columns_list->columns, columns); + } +} + } diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 5804d817fe2..b6c8e10668a 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -96,6 +96,10 @@ private: void assertOrSetUUID(ASTCreateQuery & create, const DatabasePtr & database) const; + /// Update create query with columns description from storage if query doesn't have it. + /// It's used to prevent automatic schema inference while table creation on each server startup. + void addColumnsDescriptionToCreateQueryIfNecessary(ASTCreateQuery & create, const StoragePtr & storage); + ASTPtr query_ptr; /// Skip safety threshold when loading tables. diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 36ea2949b6a..da5fcedd469 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -64,9 +64,12 @@ Block InterpreterDescribeQuery::getSampleBlock(bool include_subcolumns) BlockIO InterpreterDescribeQuery::execute() { ColumnsDescription columns; + StorageSnapshotPtr storage_snapshot; const auto & ast = query_ptr->as(); const auto & table_expression = ast.table_expression->as(); + const auto & settings = getContext()->getSettingsRef(); + if (table_expression.subquery) { auto names_and_types = InterpreterSelectWithUnionQuery::getSampleBlock( @@ -83,19 +86,27 @@ BlockIO InterpreterDescribeQuery::execute() auto table_id = getContext()->resolveStorageID(table_expression.database_and_table_name); getContext()->checkAccess(AccessType::SHOW_COLUMNS, table_id); auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout); + auto metadata_snapshot = table->getInMemoryMetadataPtr(); + storage_snapshot = table->getStorageSnapshot(metadata_snapshot); columns = metadata_snapshot->getColumns(); } - bool include_subcolumns = getContext()->getSettingsRef().describe_include_subcolumns; + bool extend_object_types = settings.describe_extend_object_types && storage_snapshot; + bool include_subcolumns = settings.describe_include_subcolumns; + Block sample_block = getSampleBlock(include_subcolumns); MutableColumns res_columns = sample_block.cloneEmptyColumns(); for (const auto & column : columns) { res_columns[0]->insert(column.name); - res_columns[1]->insert(column.type->getName()); + + if (extend_object_types) + res_columns[1]->insert(storage_snapshot->getConcreteType(column.name)->getName()); + else + res_columns[1]->insert(column.type->getName()); if (column.default_desc.expression) { @@ -128,6 +139,8 @@ BlockIO InterpreterDescribeQuery::execute() { for (const auto & column : columns) { + auto type = extend_object_types ? storage_snapshot->getConcreteType(column.name) : column.type; + IDataType::forEachSubcolumn([&](const auto & path, const auto & name, const auto & data) { res_columns[0]->insert(Nested::concatenateName(column.name, name)); @@ -150,7 +163,7 @@ BlockIO InterpreterDescribeQuery::execute() res_columns[6]->insertDefault(); res_columns[7]->insert(1u); - }, {column.type->getDefaultSerialization(), column.type, nullptr, nullptr}); + }, { type->getDefaultSerialization(), type, nullptr, nullptr }); } } diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 37b944d72d6..edca48d3600 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -141,6 +141,18 @@ namespace /// Settings. Different for each explain type. +struct QueryASTSettings +{ + bool graph = false; + + constexpr static char name[] = "AST"; + + std::unordered_map> boolean_settings = + { + {"graph", graph}, + }; +}; + struct QueryPlanSettings { QueryPlan::ExplainPlanOptions query_plan_options; @@ -260,10 +272,11 @@ QueryPipeline InterpreterExplainQuery::executeImpl() { case ASTExplainQuery::ParsedAST: { - if (ast.getSettings()) - throw Exception("Settings are not supported for EXPLAIN AST query.", ErrorCodes::UNKNOWN_SETTING); - - dumpAST(*ast.getExplainedQuery(), buf); + auto settings = checkAndGetSettings(ast.getSettings()); + if (settings.graph) + dumpASTInDotFormat(*ast.getExplainedQuery(), buf); + else + dumpAST(*ast.getExplainedQuery(), buf); break; } case ASTExplainQuery::AnalyzedSyntax: diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index d340308122f..df44814a96e 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -60,6 +60,18 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) { const auto & factory = TableFunctionFactory::instance(); TableFunctionPtr table_function_ptr = factory.get(query.table_function, getContext()); + + /// If table function needs structure hint from select query + /// we can create a temporary pipeline and get the header. + if (query.select && table_function_ptr->needStructureHint()) + { + InterpreterSelectWithUnionQuery interpreter_select{ + query.select, getContext(), SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + QueryPipelineBuilder tmp_pipeline = interpreter_select.buildQueryPipeline(); + ColumnsDescription structure_hint{tmp_pipeline.getHeader().getNamesAndTypesList()}; + table_function_ptr->setStructureHint(structure_hint); + } + return table_function_ptr->execute(query.table_function, getContext(), table_function_ptr->getName()); } @@ -109,22 +121,31 @@ Block InterpreterInsertQuery::getSampleBlock( const StoragePtr & table, const StorageMetadataPtr & metadata_snapshot) const { - Block table_sample = metadata_snapshot->getSampleBlock(); - Block table_sample_non_materialized = metadata_snapshot->getSampleBlockNonMaterialized(); + Block table_sample_physical = metadata_snapshot->getSampleBlock(); + Block table_sample_insertable = metadata_snapshot->getSampleBlockInsertable(); Block res; for (const auto & current_name : names) { - /// The table does not have a column with that name - if (!table_sample.has(current_name)) - throw Exception("No such column " + current_name + " in table " + table->getStorageID().getNameForLogs(), - ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); - - if (!allow_materialized && !table_sample_non_materialized.has(current_name)) - throw Exception("Cannot insert column " + current_name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); if (res.has(current_name)) throw Exception("Column " + current_name + " specified more than once", ErrorCodes::DUPLICATE_COLUMN); - res.insert(ColumnWithTypeAndName(table_sample.getByName(current_name).type, current_name)); + /// Column is not ordinary or ephemeral + if (!table_sample_insertable.has(current_name)) + { + /// Column is materialized + if (table_sample_physical.has(current_name)) + { + if (!allow_materialized) + throw Exception("Cannot insert column " + current_name + ", because it is MATERIALIZED column.", + ErrorCodes::ILLEGAL_COLUMN); + res.insert(ColumnWithTypeAndName(table_sample_physical.getByName(current_name).type, current_name)); + } + else /// The table does not have a column with that name + throw Exception("No such column " + current_name + " in table " + table->getStorageID().getNameForLogs(), + ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + } + else + res.insert(ColumnWithTypeAndName(table_sample_insertable.getByName(current_name).type, current_name)); } return res; } @@ -176,7 +197,7 @@ Chain InterpreterInsertQuery::buildChain( std::atomic_uint64_t * elapsed_counter_ms) { auto sample = getSampleBlock(columns, table, metadata_snapshot); - return buildChainImpl(table, metadata_snapshot, std::move(sample) , thread_status, elapsed_counter_ms); + return buildChainImpl(table, metadata_snapshot, sample, thread_status, elapsed_counter_ms); } Chain InterpreterInsertQuery::buildChainImpl( @@ -274,6 +295,9 @@ BlockIO InterpreterInsertQuery::execute() auto metadata_snapshot = table->getInMemoryMetadataPtr(); auto query_sample_block = getSampleBlock(query, table, metadata_snapshot); + + /// For table functions we check access while executing + /// getTable() -> ITableFunction::execute(). if (!query.table_function) getContext()->checkAccess(AccessType::INSERT, query.table_id, query_sample_block.getNames()); diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 2b949266c17..5ec6abb08a7 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -133,7 +133,7 @@ public: , process_list(process_list_) , processes_to_stop(std::move(processes_to_stop_)) , processes_block(std::move(processes_block_)) - , res_sample_block(std::move(res_sample_block_)) + , res_sample_block(res_sample_block_) { addTotalRowsApprox(processes_to_stop.size()); } diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index f9a701a0a77..d4fe7604ced 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -32,6 +32,7 @@ BlockIO InterpreterOptimizeQuery::execute() auto table_id = getContext()->resolveStorageID(ast, Context::ResolveOrdinary); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); auto metadata_snapshot = table->getInMemoryMetadataPtr(); + auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot); // Empty list of names means we deduplicate by all columns, but user can explicitly state which columns to use. Names column_names; @@ -46,7 +47,7 @@ BlockIO InterpreterOptimizeQuery::execute() column_names.emplace_back(col->getColumnName()); } - metadata_snapshot->check(column_names, NamesAndTypesList{}, table_id); + storage_snapshot->check(column_names); Names required_columns; { required_columns = metadata_snapshot->getColumnsRequiredForSortingKey(); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index ac807f4f782..27ed8438fc8 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -138,7 +138,7 @@ String InterpreterSelectQuery::generateFilterActions(ActionsDAGPtr & actions, co table_expr->children.push_back(table_expr->database_and_table_name); /// Using separate expression analyzer to prevent any possible alias injection - auto syntax_result = TreeRewriter(context).analyzeSelect(query_ast, TreeRewriterResult({}, storage, metadata_snapshot)); + auto syntax_result = TreeRewriter(context).analyzeSelect(query_ast, TreeRewriterResult({}, storage, storage_snapshot)); SelectQueryExpressionAnalyzer analyzer(query_ast, syntax_result, context, metadata_snapshot); actions = analyzer.simpleSelectActions(); @@ -286,7 +286,6 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.ignore_projections = options.ignore_projections; query_info.is_projection_query = options.is_projection_query; - query_info.original_query = query_ptr->clone(); initSettings(); const Settings & settings = context->getSettingsRef(); @@ -310,6 +309,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( ApplyWithSubqueryVisitor().visit(query_ptr); } + query_info.original_query = query_ptr->clone(); + JoinedTables joined_tables(getSubqueryContext(context), getSelectQuery(), options.with_all_cols); bool got_storage_from_query = false; @@ -327,6 +328,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( table_id = storage->getStorageID(); if (!metadata_snapshot) metadata_snapshot = storage->getInMemoryMetadataPtr(); + + storage_snapshot = storage->getStorageSnapshotForQuery(metadata_snapshot, query_ptr); } if (has_input || !joined_tables.resolveTables()) @@ -356,6 +359,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( table_lock.reset(); table_id = StorageID::createEmpty(); metadata_snapshot = nullptr; + storage_snapshot = nullptr; } } @@ -394,7 +398,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( syntax_analyzer_result = TreeRewriter(context).analyzeSelect( query_ptr, - TreeRewriterResult(source_header.getNamesAndTypesList(), storage, metadata_snapshot), + TreeRewriterResult(source_header.getNamesAndTypesList(), storage, storage_snapshot), options, joined_tables.tablesWithColumns(), required_result_column_names, table_join); query_info.syntax_analyzer_result = syntax_analyzer_result; @@ -515,7 +519,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( } } - source_header = metadata_snapshot->getSampleBlockForColumns(required_columns, storage->getVirtuals(), storage->getStorageID()); + source_header = storage_snapshot->getSampleBlockForColumns(required_columns); } /// Calculate structure of the result. @@ -548,7 +552,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Reuse already built sets for multiple passes of analysis subquery_for_sets = std::move(query_analyzer->getSubqueriesForSets()); - prepared_sets = query_info.sets.empty() ? std::move(query_analyzer->getPreparedSets()) : std::move(query_info.sets); + prepared_sets = query_info.sets.empty() ? query_analyzer->getPreparedSets() : query_info.sets; /// Do not try move conditions to PREWHERE for the second time. /// Otherwise, we won't be able to fallback from inefficient PREWHERE to WHERE later. @@ -581,6 +585,9 @@ InterpreterSelectQuery::InterpreterSelectQuery( analysis_result.required_columns = required_columns; } + if (query_info.projection) + storage_snapshot->addProjection(query_info.projection->desc); + /// Blocks used in expression analysis contains size 1 const columns for constant folding and /// null non-const columns to avoid useless memory allocations. However, a valid block sample /// requires all columns to be of size 0, thus we need to sanitize the block here. @@ -622,8 +629,6 @@ BlockIO InterpreterSelectQuery::execute() Block InterpreterSelectQuery::getSampleBlockImpl() { - OpenTelemetrySpanHolder span(__PRETTY_FUNCTION__); - query_info.query = query_ptr; query_info.has_window = query_analyzer->hasWindow(); if (storage && !options.only_analyze) @@ -632,10 +637,9 @@ Block InterpreterSelectQuery::getSampleBlockImpl() query_analyzer->makeSetsForIndex(query.where()); query_analyzer->makeSetsForIndex(query.prewhere()); query_info.sets = query_analyzer->getPreparedSets(); - } - if (storage && !options.only_analyze) - from_stage = storage->getQueryProcessingStage(context, options.to_stage, metadata_snapshot, query_info); + from_stage = storage->getQueryProcessingStage(context, options.to_stage, storage_snapshot, query_info); + } /// Do I need to perform the first part of the pipeline? /// Running on remote servers during distributed processing or if query is not distributed. @@ -1238,10 +1242,6 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

hasGlobalSubqueries() && !subqueries_for_sets.empty()) - executeSubqueriesInSetsAndJoins(query_plan, subqueries_for_sets); } if (expressions.second_stage || from_aggregation_stage) @@ -1424,7 +1424,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

hasGlobalSubqueries())) + if (!subqueries_for_sets.empty()) executeSubqueriesInSetsAndJoins(query_plan, subqueries_for_sets); } @@ -1725,7 +1725,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() } auto syntax_result - = TreeRewriter(context).analyze(required_columns_all_expr, required_columns_after_prewhere, storage, metadata_snapshot); + = TreeRewriter(context).analyze(required_columns_all_expr, required_columns_after_prewhere, storage, storage_snapshot); alias_actions = ExpressionAnalyzer(required_columns_all_expr, syntax_result, context).getActionsDAG(true); /// The set of required columns could be added as a result of adding an action to calculate ALIAS. @@ -1776,6 +1776,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc bool optimize_trivial_count = syntax_analyzer_result->optimize_trivial_count && (settings.max_parallel_replicas <= 1) + && !settings.allow_experimental_query_deduplication && storage && storage->getName() != "MaterializedMySQL" && !row_policy_filter @@ -1887,7 +1888,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc && limit_length <= std::numeric_limits::max() - limit_offset && limit_length + limit_offset < max_block_size) { - max_block_size = std::max(UInt64(1), limit_length + limit_offset); + max_block_size = std::max(UInt64{1}, limit_length + limit_offset); max_threads_execute_query = max_streams = 1; } @@ -1902,20 +1903,16 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc else if (interpreter_subquery) { /// Subquery. - /// If we need less number of columns that subquery have - update the interpreter. - if (required_columns.size() < source_header.columns()) - { - ASTPtr subquery = extractTableExpression(query, 0); - if (!subquery) - throw Exception("Subquery expected", ErrorCodes::LOGICAL_ERROR); + ASTPtr subquery = extractTableExpression(query, 0); + if (!subquery) + throw Exception("Subquery expected", ErrorCodes::LOGICAL_ERROR); - interpreter_subquery = std::make_unique( - subquery, getSubqueryContext(context), - options.copy().subquery().noModify(), required_columns); + interpreter_subquery = std::make_unique( + subquery, getSubqueryContext(context), + options.copy().subquery().noModify(), required_columns); - if (query_analyzer->hasAggregation()) - interpreter_subquery->ignoreWithTotals(); - } + if (query_analyzer->hasAggregation()) + interpreter_subquery->ignoreWithTotals(); interpreter_subquery->buildQueryPlan(query_plan); query_plan.addInterpreterContext(context); @@ -2004,7 +2001,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc quota = context->getQuota(); query_info.settings_limit_offset_done = options.settings_limit_offset_done; - storage->read(query_plan, required_columns, metadata_snapshot, query_info, context, processing_stage, max_block_size, max_streams); + storage->read(query_plan, required_columns, storage_snapshot, query_info, context, processing_stage, max_block_size, max_streams); if (context->hasQueryContext() && !options.is_internal) { @@ -2021,11 +2018,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc /// Create step which reads from empty source if storage has no data. if (!query_plan.isInitialized()) { - auto header = query_info.projection - ? query_info.projection->desc->metadata->getSampleBlockForColumns( - query_info.projection->required_columns, storage->getVirtuals(), storage->getStorageID()) - : metadata_snapshot->getSampleBlockForColumns(required_columns, storage->getVirtuals(), storage->getStorageID()); - + auto header = storage_snapshot->getSampleBlockForColumns(required_columns); addEmptySourceToQueryPlan(query_plan, header, query_info, context); } @@ -2581,11 +2574,11 @@ void InterpreterSelectQuery::executeExtremes(QueryPlan & query_plan) void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPlan & query_plan, SubqueriesForSets & subqueries_for_sets) { - const auto & input_order_info = query_info.input_order_info - ? query_info.input_order_info - : (query_info.projection ? query_info.projection->input_order_info : nullptr); - if (input_order_info) - executeMergeSorted(query_plan, input_order_info->order_key_prefix_descr, 0, "before creating sets for subqueries and joins"); + // const auto & input_order_info = query_info.input_order_info + // ? query_info.input_order_info + // : (query_info.projection ? query_info.projection->input_order_info : nullptr); + // if (input_order_info) + // executeMergeSorted(query_plan, input_order_info->order_key_prefix_descr, 0, "before creating sets for subqueries and joins"); const Settings & settings = context->getSettingsRef(); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 4298cbbb794..6bb12caff7d 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -202,6 +202,7 @@ private: Poco::Logger * log; StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; /// Reuse already built sets for multiple passes of analysis, possibly across interpreters. PreparedSets prepared_sets; diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 723db59f04b..130b3aae58d 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -208,8 +208,10 @@ Block InterpreterSelectWithUnionQuery::getCurrentChildResultHeader(const ASTPtr if (ast_ptr_->as()) return InterpreterSelectWithUnionQuery(ast_ptr_, context, options.copy().analyze().noModify(), required_result_column_names) .getSampleBlock(); - else + else if (ast_ptr_->as()) return InterpreterSelectQuery(ast_ptr_, context, options.copy().analyze().noModify()).getSampleBlock(); + else + return InterpreterSelectIntersectExceptQuery(ast_ptr_, context, options.copy().analyze().noModify()).getSampleBlock(); } std::unique_ptr diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index ed20b1b2048..d6a00ba89b4 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -576,7 +576,7 @@ std::shared_ptr subqueryExpressionList( needed_columns[table_pos].fillExpressionList(*expression_list); for (const auto & expr : alias_pushdown[table_pos]) - expression_list->children.emplace_back(std::move(expr)); + expression_list->children.emplace_back(expr); return expression_list; } diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 3aae3982758..482a813bfef 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -183,7 +183,9 @@ std::unique_ptr JoinedTables::makeLeftTableSubq { if (!isLeftTableSubquery()) return {}; - return std::make_unique(left_table_expression, context, select_options); + + /// Only build dry_run interpreter during analysis. We will reconstruct the subquery interpreter during plan building. + return std::make_unique(left_table_expression, context, select_options.copy().analyze()); } StoragePtr JoinedTables::getLeftTableStorage() diff --git a/src/Interpreters/LogicalExpressionsOptimizer.h b/src/Interpreters/LogicalExpressionsOptimizer.h index 1a04b199a13..4991d31f8b1 100644 --- a/src/Interpreters/LogicalExpressionsOptimizer.h +++ b/src/Interpreters/LogicalExpressionsOptimizer.h @@ -29,7 +29,7 @@ class LogicalExpressionsOptimizer final { const UInt64 optimize_min_equality_disjunction_chain_length; - ExtractedSettings(UInt64 optimize_min_equality_disjunction_chain_length_) + explicit ExtractedSettings(UInt64 optimize_min_equality_disjunction_chain_length_) : optimize_min_equality_disjunction_chain_length(optimize_min_equality_disjunction_chain_length_) {} }; @@ -68,7 +68,6 @@ private: using DisjunctiveEqualityChainsMap = std::map; using DisjunctiveEqualityChain = DisjunctiveEqualityChainsMap::value_type; -private: /** Collect information about all the equations in the OR chains (not necessarily homogeneous). * This information is grouped by the expression that is on the left side of the equation. */ @@ -92,12 +91,10 @@ private: /// Restore the original column order after optimization. void reorderColumns(); -private: using ParentNodes = std::vector; using FunctionParentMap = std::unordered_map; using ColumnToPosition = std::unordered_map; -private: ASTSelectQuery * select_query; const ExtractedSettings settings; /// Information about the OR-chains inside the query. diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 7f22386f54b..e1e03e53014 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -465,8 +465,6 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right : table_join(table_join_) , size_limits(table_join->sizeLimits()) , right_sample_block(right_sample_block_) - , nullable_right_side(table_join->forceNullableRight()) - , nullable_left_side(table_join->forceNullableLeft()) , is_any_join(table_join->strictness() == ASTTableJoin::Strictness::Any) , is_all_join(table_join->strictness() == ASTTableJoin::Strictness::All) , is_semi_join(table_join->strictness() == ASTTableJoin::Strictness::Semi) @@ -545,9 +543,6 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right JoinCommon::createMissedColumns(right_columns_to_add); - if (nullable_right_side) - JoinCommon::convertColumnsToNullable(right_columns_to_add); - makeSortAndMerge(key_names_left, left_sort_description, left_merge_description); makeSortAndMerge(key_names_right, right_sort_description, right_merge_description); @@ -710,9 +705,6 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) JoinCommon::convertToFullColumnsInplace(block, key_names_left, false); sortBlock(block, left_sort_description); - - if (nullable_left_side) - JoinCommon::convertColumnsToNullable(block); } if (!not_processed && left_blocks_buffer) @@ -889,6 +881,7 @@ bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block { right_cursor.nextN(range.right_length); right_block_info.skip = right_cursor.position(); + left_cursor.nextN(range.left_length); return false; } } diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 2cf287fd2fd..89570015d0f 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -102,8 +102,6 @@ private: SortedBlocksWriter::SortedFiles flushed_right_blocks; Block totals; std::atomic is_in_memory{true}; - const bool nullable_right_side; - const bool nullable_left_side; const bool is_any_join; const bool is_all_join; const bool is_semi_join; diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 1c7b970e731..5e795c5760a 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -802,7 +802,9 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & /// e.g. ALTER referencing the same table in scalar subquery bool execute_scalar_subqueries = !dry_run; auto syntax_result = TreeRewriter(context).analyze( - all_asts, all_columns, storage, metadata_snapshot, false, true, execute_scalar_subqueries); + all_asts, all_columns, storage, storage->getStorageSnapshot(metadata_snapshot), + false, true, execute_scalar_subqueries); + if (execute_scalar_subqueries && context->hasQueryContext()) for (const auto & it : syntax_result->getScalars()) context->getQueryContext()->addScalar(it.first, it.second); diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index df74a94ee57..515ef6c3058 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -130,6 +130,9 @@ static NamesAndTypesList getColumnsList(const ASTExpressionList * columns_defini child = new_child; } } + + if (type_name_upper == "DATE") + data_type_function->name = "Date32"; } if (is_nullable) data_type = makeASTFunction("Nullable", data_type); @@ -335,7 +338,7 @@ static ASTPtr getPartitionPolicy(const NamesAndTypesList & primary_keys) if (which.isNullable()) throw Exception("LOGICAL ERROR: MySQL primary key must be not null, it is a bug.", ErrorCodes::LOGICAL_ERROR); - if (which.isDate() || which.isDateTime() || which.isDateTime64()) + if (which.isDate() || which.isDate32() || which.isDateTime() || which.isDateTime64()) { /// In any case, date or datetime is always the best partitioning key return makeASTFunction("toYYYYMM", std::make_shared(primary_key.name)); diff --git a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp index 680b9bd5606..71578bd5db7 100644 --- a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp +++ b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp @@ -39,7 +39,7 @@ TEST(MySQLCreateRewritten, ColumnsDataType) { {"TINYINT", "Int8"}, {"SMALLINT", "Int16"}, {"MEDIUMINT", "Int32"}, {"INT", "Int32"}, {"INTEGER", "Int32"}, {"BIGINT", "Int64"}, {"FLOAT", "Float32"}, {"DOUBLE", "Float64"}, - {"VARCHAR(10)", "String"}, {"CHAR(10)", "String"}, {"Date", "Date"}, {"DateTime", "DateTime"}, + {"VARCHAR(10)", "String"}, {"CHAR(10)", "String"}, {"Date", "Date32"}, {"DateTime", "DateTime"}, {"TIMESTAMP", "DateTime"}, {"BOOLEAN", "Bool"}, {"BIT", "UInt64"}, {"SET", "UInt64"}, {"YEAR", "UInt16"}, {"TIME", "Int64"}, {"GEOMETRY", "String"} }; @@ -104,7 +104,7 @@ TEST(MySQLCreateRewritten, PartitionPolicy) {"MEDIUMINT", "Int32", " PARTITION BY intDiv(key, 4294967)"}, {"INT", "Int32", " PARTITION BY intDiv(key, 4294967)"}, {"INTEGER", "Int32", " PARTITION BY intDiv(key, 4294967)"}, {"BIGINT", "Int64", " PARTITION BY intDiv(key, 18446744073709551)"}, {"FLOAT", "Float32", ""}, {"DOUBLE", "Float64", ""}, {"VARCHAR(10)", "String", ""}, {"CHAR(10)", "String", ""}, - {"Date", "Date", " PARTITION BY toYYYYMM(key)"}, {"DateTime", "DateTime", " PARTITION BY toYYYYMM(key)"}, + {"Date", "Date32", " PARTITION BY toYYYYMM(key)"}, {"DateTime", "DateTime", " PARTITION BY toYYYYMM(key)"}, {"TIMESTAMP", "DateTime", " PARTITION BY toYYYYMM(key)"}, {"BOOLEAN", "Bool", " PARTITION BY key"} }; @@ -135,7 +135,7 @@ TEST(MySQLCreateRewritten, OrderbyPolicy) {"MEDIUMINT", "Int32", " PARTITION BY intDiv(key, 4294967)"}, {"INT", "Int32", " PARTITION BY intDiv(key, 4294967)"}, {"INTEGER", "Int32", " PARTITION BY intDiv(key, 4294967)"}, {"BIGINT", "Int64", " PARTITION BY intDiv(key, 18446744073709551)"}, {"FLOAT", "Float32", ""}, {"DOUBLE", "Float64", ""}, {"VARCHAR(10)", "String", ""}, {"CHAR(10)", "String", ""}, - {"Date", "Date", " PARTITION BY toYYYYMM(key)"}, {"DateTime", "DateTime", " PARTITION BY toYYYYMM(key)"}, + {"Date", "Date32", " PARTITION BY toYYYYMM(key)"}, {"DateTime", "DateTime", " PARTITION BY toYYYYMM(key)"}, {"TIMESTAMP", "DateTime", " PARTITION BY toYYYYMM(key)"}, {"BOOLEAN", "Bool", " PARTITION BY key"} }; diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 40f31e4976c..36ffd617cd6 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -150,6 +150,42 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() } } +void OpenTelemetrySpanHolder::addAttribute(const std::string& name, UInt64 value) +{ + if (trace_id == UUID()) + return; + + this->attribute_names.push_back(name); + this->attribute_values.push_back(std::to_string(value)); +} + +void OpenTelemetrySpanHolder::addAttribute(const std::string& name, const std::string& value) +{ + if (trace_id == UUID()) + return; + + this->attribute_names.push_back(name); + this->attribute_values.push_back(value); +} + +void OpenTelemetrySpanHolder::addAttribute(const Exception & e) +{ + if (trace_id == UUID()) + return; + + this->attribute_names.push_back("clickhouse.exception"); + this->attribute_values.push_back(getExceptionMessage(e, false)); +} + +void OpenTelemetrySpanHolder::addAttribute(std::exception_ptr e) +{ + if (trace_id == UUID() || e == nullptr) + return; + + this->attribute_names.push_back("clickhouse.exception"); + this->attribute_values.push_back(getExceptionMessage(e, false)); +} + bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & traceparent, std::string & error) { diff --git a/src/Interpreters/OpenTelemetrySpanLog.h b/src/Interpreters/OpenTelemetrySpanLog.h index 8dfc2eccc00..aa99a9f8e4b 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.h +++ b/src/Interpreters/OpenTelemetrySpanLog.h @@ -25,7 +25,7 @@ struct OpenTelemetrySpan struct OpenTelemetrySpanLogElement : public OpenTelemetrySpan { OpenTelemetrySpanLogElement() = default; - OpenTelemetrySpanLogElement(const OpenTelemetrySpan & span) + explicit OpenTelemetrySpanLogElement(const OpenTelemetrySpan & span) : OpenTelemetrySpan(span) {} static std::string name() { return "OpenTelemetrySpanLog"; } @@ -44,7 +44,12 @@ public: struct OpenTelemetrySpanHolder : public OpenTelemetrySpan { - OpenTelemetrySpanHolder(const std::string & _operation_name); + explicit OpenTelemetrySpanHolder(const std::string & _operation_name); + void addAttribute(const std::string& name, UInt64 value); + void addAttribute(const std::string& name, const std::string& value); + void addAttribute(const Exception & e); + void addAttribute(std::exception_ptr e); + ~OpenTelemetrySpanHolder(); }; diff --git a/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h b/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h index 05d0330196b..ad98f92bafd 100644 --- a/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h +++ b/src/Interpreters/OptimizeIfWithConstantConditionVisitor.h @@ -10,7 +10,7 @@ namespace DB class OptimizeIfWithConstantConditionVisitor { public: - OptimizeIfWithConstantConditionVisitor(Aliases & aliases_) + explicit OptimizeIfWithConstantConditionVisitor(Aliases & aliases_) : aliases(aliases_) {} diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index ecfda4cd0c1..991b449196d 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -55,7 +55,12 @@ bool shardContains( data.sharding_key_column_name); /// The value from IN can be non-numeric, /// but in this case it should be convertible to numeric type, let's try. - sharding_value = convertFieldToType(sharding_value, DataTypeUInt64()); + /// + /// NOTE: that conversion should not be done for signed types, + /// since it uses accurate cast, that will return Null, + /// but we need static_cast<> (as createBlockSelector()). + if (!isInt64OrUInt64FieldType(sharding_value.getType())) + sharding_value = convertFieldToType(sharding_value, DataTypeUInt64()); /// In case of conversion is not possible (NULL), shard cannot contain the value anyway. if (sharding_value.isNull()) return false; diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index f89f836871a..4947b50513c 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -46,6 +46,7 @@ NamesAndTypesList PartLogElement::getNamesAndTypes() {"table", std::make_shared()}, {"part_name", std::make_shared()}, {"partition_id", std::make_shared()}, + {"disk_name", std::make_shared()}, {"path_on_disk", std::make_shared()}, {"rows", std::make_shared()}, @@ -79,6 +80,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(table_name); columns[i++]->insert(part_name); columns[i++]->insert(partition_id); + columns[i++]->insert(disk_name); columns[i++]->insert(path_on_disk); columns[i++]->insert(rows); @@ -155,6 +157,7 @@ bool PartLog::addNewParts( elem.table_name = table_id.table_name; elem.partition_id = part->info.partition_id; elem.part_name = part->name; + elem.disk_name = part->volume->getDisk()->getName(); elem.path_on_disk = part->getFullPath(); elem.bytes_compressed_on_disk = part->getBytesOnDisk(); diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index bdd1db4334a..5f502edb339 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -32,6 +32,7 @@ struct PartLogElement String table_name; String part_name; String partition_id; + String disk_name; String path_on_disk; /// Size of the part diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 3c91f00ebf4..26146781327 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -195,33 +195,24 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING); } - auto process_it = processes.emplace(processes.end(), - query_context, query_, client_info, priorities.insert(settings.priority), query_kind); - - increaseQueryKindAmount(query_kind); - - res = std::make_shared(*this, process_it); - - ProcessListForUser & user_process_list = user_to_queries[client_info.current_user]; - user_process_list.queries.emplace(client_info.current_query_id, &res->get()); - - process_it->setUserProcessList(&user_process_list); - - /// Track memory usage for all simultaneously running queries from single user. - user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); - user_process_list.user_memory_tracker.setDescription("(for user)"); + auto user_process_list_it = user_to_queries.find(client_info.current_user); + if (user_process_list_it == user_to_queries.end()) + user_process_list_it = user_to_queries.emplace(client_info.current_user, this).first; + ProcessListForUser & user_process_list = user_process_list_it->second; /// Actualize thread group info - if (auto thread_group = CurrentThread::getGroup()) + auto thread_group = CurrentThread::getGroup(); + if (thread_group) { std::lock_guard lock_thread_group(thread_group->mutex); thread_group->performance_counters.setParent(&user_process_list.user_performance_counters); thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker); - thread_group->query = process_it->query; - thread_group->normalized_query_hash = normalizedQueryHash(process_it->query); + thread_group->query = query_; + thread_group->normalized_query_hash = normalizedQueryHash(query_); /// Set query-level memory trackers thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage); + thread_group->memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage); if (query_context->hasTraceCollector()) { @@ -236,10 +227,25 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values /// since allocation and deallocation could happen in different threads - - process_it->thread_group = std::move(thread_group); } + auto process_it = processes.emplace(processes.end(), + query_context, query_, client_info, priorities.insert(settings.priority), std::move(thread_group), query_kind); + + increaseQueryKindAmount(query_kind); + + res = std::make_shared(*this, process_it); + + process_it->setUserProcessList(&user_process_list); + + user_process_list.queries.emplace(client_info.current_query_id, &res->get()); + + /// Track memory usage for all simultaneously running queries from single user. + user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); + user_process_list.user_memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage_for_user); + user_process_list.user_memory_tracker.setDescription("(for user)"); + user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds); + if (!user_process_list.user_throttler) { if (settings.max_network_bandwidth_for_user) @@ -268,9 +274,6 @@ ProcessListEntry::~ProcessListEntry() const QueryStatus * process_list_element_ptr = &*it; - /// This removes the memory_tracker of one request. - parent.processes.erase(it); - auto user_process_list_it = parent.user_to_queries.find(user); if (user_process_list_it == parent.user_to_queries.end()) { @@ -291,6 +294,9 @@ ProcessListEntry::~ProcessListEntry() } } + /// This removes the memory_tracker of one request. + parent.processes.erase(it); + if (!found) { LOG_ERROR(&Poco::Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"); @@ -312,10 +318,16 @@ ProcessListEntry::~ProcessListEntry() QueryStatus::QueryStatus( - ContextPtr context_, const String & query_, const ClientInfo & client_info_, QueryPriorities::Handle && priority_handle_, IAST::QueryKind query_kind_) + ContextPtr context_, + const String & query_, + const ClientInfo & client_info_, + QueryPriorities::Handle && priority_handle_, + ThreadGroupStatusPtr && thread_group_, + IAST::QueryKind query_kind_) : WithContext(context_) , query(query_) , client_info(client_info_) + , thread_group(std::move(thread_group_)) , priority_handle(std::move(priority_handle_)) , query_kind(query_kind_) , num_queries_increment(CurrentMetrics::Query) @@ -328,6 +340,14 @@ QueryStatus::QueryStatus( QueryStatus::~QueryStatus() { assert(executors.empty()); + + if (auto * memory_tracker = getMemoryTracker()) + { + if (user_process_list) + user_process_list->user_overcommit_tracker.unsubscribe(memory_tracker); + if (auto shared_context = getContext()) + shared_context->getGlobalOvercommitTracker()->unsubscribe(memory_tracker); + } } CancellationCode QueryStatus::cancelQuery(bool) @@ -481,7 +501,11 @@ ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_ev } -ProcessListForUser::ProcessListForUser() = default; +ProcessListForUser::ProcessListForUser(ProcessList * global_process_list) + : user_overcommit_tracker(global_process_list, this) +{ + user_memory_tracker.setOvercommitTracker(&user_overcommit_tracker); +} ProcessListForUserInfo ProcessListForUser::getInfo(bool get_profile_events) const diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 2ba0b0814ee..c90c271679c 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -76,6 +77,7 @@ protected: friend class ThreadStatus; friend class CurrentThread; friend class ProcessListEntry; + friend struct ::GlobalOvercommitTracker; String query; ClientInfo client_info; @@ -132,6 +134,7 @@ public: const String & query_, const ClientInfo & client_info_, QueryPriorities::Handle && priority_handle_, + ThreadGroupStatusPtr && thread_group_, IAST::QueryKind query_kind_ ); @@ -154,6 +157,13 @@ public: ThrottlerPtr getUserNetworkThrottler(); + MemoryTracker * getMemoryTracker() const + { + if (!thread_group) + return nullptr; + return &thread_group->memory_tracker; + } + bool updateProgressIn(const Progress & value) { CurrentThread::updateProgressIn(value); @@ -206,7 +216,7 @@ struct ProcessListForUserInfo /// Data about queries for one user. struct ProcessListForUser { - ProcessListForUser(); + explicit ProcessListForUser(ProcessList * global_process_list); /// query_id -> ProcessListElement(s). There can be multiple queries with the same query_id as long as all queries except one are cancelled. using QueryToElement = std::unordered_map; @@ -216,6 +226,8 @@ struct ProcessListForUser /// Limit and counter for memory of all simultaneously running queries of single user. MemoryTracker user_memory_tracker{VariableContext::User}; + UserOvercommitTracker user_overcommit_tracker; + /// Count network usage for all simultaneously running queries of single user. ThrottlerPtr user_throttler; @@ -279,6 +291,8 @@ public: protected: friend class ProcessListEntry; + friend struct ::UserOvercommitTracker; + friend struct ::GlobalOvercommitTracker; mutable std::mutex mutex; mutable std::condition_variable have_space; /// Number of currently running queries has become less than maximum. @@ -337,6 +351,15 @@ public: max_size = max_size_; } + // Before calling this method you should be sure + // that lock is acquired. + template + void processEachQueryStatus(F && func) const + { + for (auto && query : processes) + func(query); + } + void setMaxInsertQueriesAmount(size_t max_insert_queries_amount_) { std::lock_guard lock(mutex); diff --git a/src/Interpreters/ProfileEventsExt.cpp b/src/Interpreters/ProfileEventsExt.cpp index 472efc109fb..ea87d565854 100644 --- a/src/Interpreters/ProfileEventsExt.cpp +++ b/src/Interpreters/ProfileEventsExt.cpp @@ -1,5 +1,7 @@ #include "ProfileEventsExt.h" #include +#include +#include #include #include #include @@ -36,7 +38,7 @@ void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, if (nonzero_only && 0 == value) continue; - const char * desc = ProfileEvents::getName(event); + const char * desc = getName(event); key_column.insertData(desc, strlen(desc)); value_column.insert(value); size++; @@ -45,4 +47,133 @@ void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, offsets.push_back(offsets.back() + size); } +/// Add records about provided non-zero ProfileEvents::Counters. +static void dumpProfileEvents(ProfileEventsSnapshot const & snapshot, DB::MutableColumns & columns, String const & host_name) +{ + size_t rows = 0; + auto & name_column = columns[NAME_COLUMN_INDEX]; + auto & value_column = columns[VALUE_COLUMN_INDEX]; + for (Event event = 0; event < Counters::num_counters; ++event) + { + Int64 value = snapshot.counters[event]; + + if (value == 0) + continue; + + const char * desc = getName(event); + name_column->insertData(desc, strlen(desc)); + value_column->insert(value); + rows++; + } + + // Fill the rest of the columns with data + for (size_t row = 0; row < rows; ++row) + { + size_t i = 0; + columns[i++]->insertData(host_name.data(), host_name.size()); + columns[i++]->insert(UInt64(snapshot.current_time)); + columns[i++]->insert(UInt64{snapshot.thread_id}); + columns[i++]->insert(Type::INCREMENT); + } +} + +static void dumpMemoryTracker(ProfileEventsSnapshot const & snapshot, DB::MutableColumns & columns, String const & host_name) +{ + size_t i = 0; + columns[i++]->insertData(host_name.data(), host_name.size()); + columns[i++]->insert(UInt64(snapshot.current_time)); + columns[i++]->insert(UInt64{snapshot.thread_id}); + columns[i++]->insert(Type::GAUGE); + + columns[i++]->insertData(MemoryTracker::USAGE_EVENT_NAME, strlen(MemoryTracker::USAGE_EVENT_NAME)); + columns[i++]->insert(snapshot.memory_usage); +} + +void getProfileEvents( + const String & server_display_name, + DB::InternalProfileEventsQueuePtr profile_queue, + DB::Block & block, + ThreadIdToCountersSnapshot & last_sent_snapshots) +{ + using namespace DB; + static const NamesAndTypesList column_names_and_types = { + {"host_name", std::make_shared()}, + {"current_time", std::make_shared()}, + {"thread_id", std::make_shared()}, + {"type", TypeEnum}, + {"name", std::make_shared()}, + {"value", std::make_shared()}, + }; + + ColumnsWithTypeAndName temp_columns; + for (auto const & name_and_type : column_names_and_types) + temp_columns.emplace_back(name_and_type.type, name_and_type.name); + + block = std::move(temp_columns); + MutableColumns columns = block.mutateColumns(); + auto thread_group = CurrentThread::getGroup(); + auto const current_thread_id = CurrentThread::get().thread_id; + std::vector snapshots; + ThreadIdToCountersSnapshot new_snapshots; + ProfileEventsSnapshot group_snapshot; + { + auto stats = thread_group->getProfileEventsCountersAndMemoryForThreads(); + snapshots.reserve(stats.size()); + + for (auto & stat : stats) + { + auto const thread_id = stat.thread_id; + if (thread_id == current_thread_id) + continue; + auto current_time = time(nullptr); + auto previous_snapshot = last_sent_snapshots.find(thread_id); + auto increment = + previous_snapshot != last_sent_snapshots.end() + ? CountersIncrement(stat.counters, previous_snapshot->second) + : CountersIncrement(stat.counters); + snapshots.push_back(ProfileEventsSnapshot{ + thread_id, + std::move(increment), + stat.memory_usage, + current_time + }); + new_snapshots[thread_id] = std::move(stat.counters); + } + + group_snapshot.thread_id = 0; + group_snapshot.current_time = time(nullptr); + group_snapshot.memory_usage = thread_group->memory_tracker.get(); + auto group_counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); + auto prev_group_snapshot = last_sent_snapshots.find(0); + group_snapshot.counters = + prev_group_snapshot != last_sent_snapshots.end() + ? CountersIncrement(group_counters, prev_group_snapshot->second) + : CountersIncrement(group_counters); + new_snapshots[0] = std::move(group_counters); + } + last_sent_snapshots = std::move(new_snapshots); + + for (auto & snapshot : snapshots) + { + dumpProfileEvents(snapshot, columns, server_display_name); + dumpMemoryTracker(snapshot, columns, server_display_name); + } + dumpProfileEvents(group_snapshot, columns, server_display_name); + dumpMemoryTracker(group_snapshot, columns, server_display_name); + + Block curr_block; + size_t rows = 0; + + for (; profile_queue->tryPop(curr_block); ++rows) + { + auto curr_columns = curr_block.getColumns(); + for (size_t j = 0; j < curr_columns.size(); ++j) + columns[j]->insertRangeFrom(*curr_columns[j], 0, curr_columns[j]->size()); + } + + bool empty = columns[0]->empty(); + if (!empty) + block.setColumns(std::move(columns)); +} + } diff --git a/src/Interpreters/ProfileEventsExt.h b/src/Interpreters/ProfileEventsExt.h index 8a92eadec79..7d9fc512d15 100644 --- a/src/Interpreters/ProfileEventsExt.h +++ b/src/Interpreters/ProfileEventsExt.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include @@ -7,9 +8,28 @@ namespace ProfileEvents { +constexpr size_t NAME_COLUMN_INDEX = 4; +constexpr size_t VALUE_COLUMN_INDEX = 5; + +struct ProfileEventsSnapshot +{ + UInt64 thread_id; + CountersIncrement counters; + Int64 memory_usage; + time_t current_time; +}; + +using ThreadIdToCountersSnapshot = std::unordered_map; + /// Dumps profile events to columns Map(String, UInt64) void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only = true); +void getProfileEvents( + const String & server_display_name, + DB::InternalProfileEventsQueuePtr profile_queue, + DB::Block & block, + ThreadIdToCountersSnapshot & last_sent_snapshots); + /// This is for ProfileEvents packets. enum Type : int8_t { diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 2cbb9634446..b464d9c1ca5 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -98,6 +98,7 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes() {"http_referer", std::make_shared()}, {"forwarded_for", std::make_shared()}, {"quota_key", std::make_shared()}, + {"distributed_depth", std::make_shared()}, {"revision", std::make_shared()}, @@ -289,5 +290,6 @@ void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableCo columns[i++]->insert(client_info.forwarded_for); columns[i++]->insert(client_info.quota_key); + columns[i++]->insert(client_info.distributed_depth); } } diff --git a/src/Interpreters/QueryNormalizer.h b/src/Interpreters/QueryNormalizer.h index eebcff62cde..f532d869789 100644 --- a/src/Interpreters/QueryNormalizer.h +++ b/src/Interpreters/QueryNormalizer.h @@ -25,7 +25,7 @@ class QueryNormalizer bool prefer_column_name_to_alias; template - ExtractedSettings(const T & settings) + ExtractedSettings(const T & settings) /// NOLINT : max_ast_depth(settings.max_ast_depth) , max_expanded_ast_elements(settings.max_expanded_ast_elements) , prefer_column_name_to_alias(settings.prefer_column_name_to_alias) diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 7ca3c10045e..d9feaf0a0c3 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -68,6 +68,7 @@ NamesAndTypesList QueryThreadLogElement::getNamesAndTypes() {"http_referer", std::make_shared()}, {"forwarded_for", std::make_shared()}, {"quota_key", std::make_shared()}, + {"distributed_depth", std::make_shared()}, {"revision", std::make_shared()}, diff --git a/src/Interpreters/RedundantFunctionsInOrderByVisitor.h b/src/Interpreters/RedundantFunctionsInOrderByVisitor.h index 09362ea6be2..60c9fcf2a24 100644 --- a/src/Interpreters/RedundantFunctionsInOrderByVisitor.h +++ b/src/Interpreters/RedundantFunctionsInOrderByVisitor.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace DB { diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.h b/src/Interpreters/ReplaceQueryParameterVisitor.h index cb3d0f668d8..dd785cd768e 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.h +++ b/src/Interpreters/ReplaceQueryParameterVisitor.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace DB diff --git a/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp b/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp index 842f61cbdd2..a34c81d52e2 100644 --- a/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp +++ b/src/Interpreters/RewriteFunctionToSubcolumnVisitor.cpp @@ -83,6 +83,7 @@ void RewriteFunctionToSubcolumnData::visit(ASTFunction & function, ASTPtr & ast) const auto & column_type = columns.get(name_in_storage).type; TypeIndex column_type_id = column_type->getTypeId(); + const auto & alias = function.tryGetAlias(); if (arguments.size() == 1) { @@ -91,7 +92,10 @@ void RewriteFunctionToSubcolumnData::visit(ASTFunction & function, ASTPtr & ast) { const auto & [type_id, subcolumn_name, transformer] = it->second; if (column_type_id == type_id) + { ast = transformer(name_in_storage, subcolumn_name); + ast->setAlias(alias); + } } } else @@ -116,6 +120,7 @@ void RewriteFunctionToSubcolumnData::visit(ASTFunction & function, ASTPtr & ast) return; ast = transformToSubcolumn(name_in_storage, subcolumn_name); + ast->setAlias(alias); } else { @@ -124,7 +129,10 @@ void RewriteFunctionToSubcolumnData::visit(ASTFunction & function, ASTPtr & ast) { const auto & [type_id, subcolumn_name, transformer] = it->second; if (column_type_id == type_id) + { ast = transformer(name_in_storage, subcolumn_name, arguments[1]); + ast->setAlias(alias); + } } } } diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index e67e16b9b64..2b791f5a189 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -1,12 +1,10 @@ #include -#include -#include -#include -#include +#include +#include #include -#include -#include +#include +#include namespace DB @@ -15,6 +13,7 @@ namespace DB namespace ErrorCodes { extern const int BAD_TYPE_OF_FIELD; + extern const int LOGICAL_ERROR; } namespace @@ -22,145 +21,250 @@ namespace /// maps enum values to types template -void callWithType(TypeIndex which, F && f) +void callWithType(TypeIndex type, F && f) { - switch (which) - { - case TypeIndex::UInt8: return f(UInt8()); - case TypeIndex::UInt16: return f(UInt16()); - case TypeIndex::UInt32: return f(UInt32()); - case TypeIndex::UInt64: return f(UInt64()); - case TypeIndex::Int8: return f(Int8()); - case TypeIndex::Int16: return f(Int16()); - case TypeIndex::Int32: return f(Int32()); - case TypeIndex::Int64: return f(Int64()); - case TypeIndex::Float32: return f(Float32()); - case TypeIndex::Float64: return f(Float64()); - case TypeIndex::Decimal32: return f(Decimal32()); - case TypeIndex::Decimal64: return f(Decimal64()); - case TypeIndex::Decimal128: return f(Decimal128()); - case TypeIndex::DateTime64: return f(DateTime64()); - default: - break; - } + WhichDataType which(type); + +#define DISPATCH(TYPE) \ + if (which.idx == TypeIndex::TYPE) \ + return f(TYPE()); + + FOR_NUMERIC_TYPES(DISPATCH) + DISPATCH(Decimal32) + DISPATCH(Decimal64) + DISPATCH(Decimal128) + DISPATCH(Decimal256) + DISPATCH(DateTime64) +#undef DISPATCH __builtin_unreachable(); } -} - - -AsofRowRefs::AsofRowRefs(TypeIndex type) +template +class SortedLookupVector : public SortedLookupVectorBase { - auto call = [&](const auto & t) + struct Entry { - using T = std::decay_t; - using LookupType = typename Entry::LookupType; - lookups = std::make_unique(); + TKey value; + uint32_t row_ref_index; + + Entry() = delete; + Entry(TKey value_, uint32_t row_ref_index_) + : value(value_) + , row_ref_index(row_ref_index_) + { } + }; - callWithType(type, call); -} - -void AsofRowRefs::insert(TypeIndex type, const IColumn & asof_column, const Block * block, size_t row_num) -{ - auto call = [&](const auto & t) + struct LessEntryOperator { - using T = std::decay_t; - using LookupPtr = typename Entry::LookupPtr; - - using ColumnType = ColumnVectorOrDecimal; - const auto & column = typeid_cast(asof_column); - - T key = column.getElement(row_num); - auto entry = Entry(key, RowRef(block, row_num)); - std::get(lookups)->insert(entry); + ALWAYS_INLINE bool operator()(const Entry & lhs, const Entry & rhs) const + { + return lhs.value < rhs.value; + } }; - callWithType(type, call); -} - -const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn & asof_column, size_t row_num) const -{ - const RowRef * out = nullptr; - - bool ascending = (inequality == ASOF::Inequality::Less) || (inequality == ASOF::Inequality::LessOrEquals); - bool is_strict = (inequality == ASOF::Inequality::Less) || (inequality == ASOF::Inequality::Greater); - - auto call = [&](const auto & t) + struct GreaterEntryOperator { - using T = std::decay_t; - using EntryType = Entry; - using LookupPtr = typename EntryType::LookupPtr; - - using ColumnType = ColumnVectorOrDecimal; - const auto & column = typeid_cast(asof_column); - T key = column.getElement(row_num); - auto & typed_lookup = std::get(lookups); - - if (is_strict) - out = typed_lookup->upperBound(EntryType(key), ascending); - else - out = typed_lookup->lowerBound(EntryType(key), ascending); + ALWAYS_INLINE bool operator()(const Entry & lhs, const Entry & rhs) const + { + return lhs.value > rhs.value; + } }; - callWithType(type, call); - return out; -} -std::optional AsofRowRefs::getTypeSize(const IColumn & asof_column, size_t & size) -{ - TypeIndex idx = asof_column.getDataType(); +public: + using Keys = std::vector; + using Entries = PaddedPODArray; + using RowRefs = PaddedPODArray; - switch (idx) + static constexpr bool is_descending = (inequality == ASOF::Inequality::Greater || inequality == ASOF::Inequality::GreaterOrEquals); + static constexpr bool is_strict = (inequality == ASOF::Inequality::Less) || (inequality == ASOF::Inequality::Greater); + + void insert(const IColumn & asof_column, const Block * block, size_t row_num) override { - case TypeIndex::UInt8: - size = sizeof(UInt8); - return idx; - case TypeIndex::UInt16: - size = sizeof(UInt16); - return idx; - case TypeIndex::UInt32: - size = sizeof(UInt32); - return idx; - case TypeIndex::UInt64: - size = sizeof(UInt64); - return idx; - case TypeIndex::Int8: - size = sizeof(Int8); - return idx; - case TypeIndex::Int16: - size = sizeof(Int16); - return idx; - case TypeIndex::Int32: - size = sizeof(Int32); - return idx; - case TypeIndex::Int64: - size = sizeof(Int64); - return idx; - //case TypeIndex::Int128: - case TypeIndex::Float32: - size = sizeof(Float32); - return idx; - case TypeIndex::Float64: - size = sizeof(Float64); - return idx; - case TypeIndex::Decimal32: - size = sizeof(Decimal32); - return idx; - case TypeIndex::Decimal64: - size = sizeof(Decimal64); - return idx; - case TypeIndex::Decimal128: - size = sizeof(Decimal128); - return idx; - case TypeIndex::DateTime64: - size = sizeof(DateTime64); - return idx; - default: - break; + using ColumnType = ColumnVectorOrDecimal; + const auto & column = assert_cast(asof_column); + TKey key = column.getElement(row_num); + + assert(!sorted.load(std::memory_order_acquire)); + + entries.emplace_back(key, row_refs.size()); + row_refs.emplace_back(RowRef(block, row_num)); } + /// Unrolled version of upper_bound and lower_bound + /// Loosely based on https://academy.realm.io/posts/how-we-beat-cpp-stl-binary-search/ + /// In the future it'd interesting to replace it with a B+Tree Layout as described + /// at https://en.algorithmica.org/hpc/data-structures/s-tree/ + size_t boundSearch(TKey value) + { + size_t size = entries.size(); + size_t low = 0; + + /// This is a single binary search iteration as a macro to unroll. Takes into account the inequality: + /// is_strict -> Equal values are not requested + /// is_descending -> The vector is sorted in reverse (for greater or greaterOrEquals) +#define BOUND_ITERATION \ + { \ + size_t half = size / 2; \ + size_t other_half = size - half; \ + size_t probe = low + half; \ + size_t other_low = low + other_half; \ + TKey & v = entries[probe].value; \ + size = half; \ + if constexpr (is_descending) \ + { \ + if constexpr (is_strict) \ + low = value <= v ? other_low : low; \ + else \ + low = value < v ? other_low : low; \ + } \ + else \ + { \ + if constexpr (is_strict) \ + low = value >= v ? other_low : low; \ + else \ + low = value > v ? other_low : low; \ + } \ + } + + while (size >= 8) + { + BOUND_ITERATION + BOUND_ITERATION + BOUND_ITERATION + } + + while (size > 0) + { + BOUND_ITERATION + } + +#undef BOUND_ITERATION + return low; + } + + RowRef findAsof(const IColumn & asof_column, size_t row_num) override + { + sort(); + + using ColumnType = ColumnVectorOrDecimal; + const auto & column = assert_cast(asof_column); + TKey k = column.getElement(row_num); + + size_t pos = boundSearch(k); + if (pos != entries.size()) + { + size_t row_ref_index = entries[pos].row_ref_index; + return row_refs[row_ref_index]; + } + + return {nullptr, 0}; + } + +private: + std::atomic sorted = false; + mutable std::mutex lock; + Entries entries; + RowRefs row_refs; + + // Double checked locking with SC atomics works in C++ + // https://preshing.com/20130930/double-checked-locking-is-fixed-in-cpp11/ + // The first thread that calls one of the lookup methods sorts the data + // After calling the first lookup method it is no longer allowed to insert any data + // the array becomes immutable + void sort() + { + if (!sorted.load(std::memory_order_acquire)) + { + std::lock_guard l(lock); + + if (!sorted.load(std::memory_order_relaxed)) + { + if constexpr (std::is_arithmetic_v && !std::is_floating_point_v) + { + if (likely(entries.size() > 256)) + { + struct RadixSortTraits : RadixSortNumTraits + { + using Element = Entry; + using Result = Element; + + static TKey & extractKey(Element & elem) { return elem.value; } + static Element extractResult(Element & elem) { return elem; } + }; + + if constexpr (is_descending) + RadixSort::executeLSD(entries.data(), entries.size(), true); + else + RadixSort::executeLSD(entries.data(), entries.size(), false); + + sorted.store(true, std::memory_order_release); + return; + } + } + + if constexpr (is_descending) + ::sort(entries.begin(), entries.end(), GreaterEntryOperator()); + else + ::sort(entries.begin(), entries.end(), LessEntryOperator()); + + sorted.store(true, std::memory_order_release); + } + } + } +}; +} + +AsofRowRefs createAsofRowRef(TypeIndex type, ASOF::Inequality inequality) +{ + AsofRowRefs result; + auto call = [&](const auto & t) + { + using T = std::decay_t; + switch (inequality) + { + case ASOF::Inequality::LessOrEquals: + result = std::make_unique>(); + break; + case ASOF::Inequality::Less: + result = std::make_unique>(); + break; + case ASOF::Inequality::GreaterOrEquals: + result = std::make_unique>(); + break; + case ASOF::Inequality::Greater: + result = std::make_unique>(); + break; + default: + throw Exception("Invalid ASOF Join order", ErrorCodes::LOGICAL_ERROR); + } + }; + + callWithType(type, call); + return result; +} + +std::optional SortedLookupVectorBase::getTypeSize(const IColumn & asof_column, size_t & size) +{ + WhichDataType which(asof_column.getDataType()); +#define DISPATCH(TYPE) \ + if (which.idx == TypeIndex::TYPE) \ + { \ + size = sizeof(TYPE); \ + return asof_column.getDataType(); \ + } + + + FOR_NUMERIC_TYPES(DISPATCH) + DISPATCH(Decimal32) + DISPATCH(Decimal64) + DISPATCH(Decimal128) + DISPATCH(Decimal256) + DISPATCH(DateTime64) +#undef DISPATCH + throw Exception("ASOF join not supported for type: " + std::string(asof_column.getFamilyName()), ErrorCodes::BAD_TYPE_OF_FIELD); } diff --git a/src/Interpreters/RowRefs.h b/src/Interpreters/RowRefs.h index cae20b98caf..fa5ce867613 100644 --- a/src/Interpreters/RowRefs.h +++ b/src/Interpreters/RowRefs.h @@ -1,16 +1,18 @@ #pragma once -#include -#include +#include +#include #include #include -#include +#include +#include -#include - -#include +#include +#include #include #include +#include +#include namespace DB @@ -26,7 +28,7 @@ struct RowRef const Block * block = nullptr; SizeT row_num = 0; - RowRef() {} + RowRef() = default; RowRef(const Block * block_, size_t row_num_) : block(block_), row_num(row_num_) {} }; @@ -42,7 +44,7 @@ struct RowRefList : RowRef Batch * next; RowRef row_refs[MAX_SIZE]; - Batch(Batch * parent) + explicit Batch(Batch * parent) : next(parent) {} @@ -52,7 +54,7 @@ struct RowRefList : RowRef { if (full()) { - auto batch = pool.alloc(); + auto * batch = pool.alloc(); *batch = Batch(this); batch->insert(std::move(row_ref), pool); return batch; @@ -66,7 +68,7 @@ struct RowRefList : RowRef class ForwardIterator { public: - ForwardIterator(const RowRefList * begin) + explicit ForwardIterator(const RowRefList * begin) : root(begin) , first(true) , batch(root->next) @@ -115,7 +117,7 @@ struct RowRefList : RowRef size_t position; }; - RowRefList() {} + RowRefList() {} /// NOLINT RowRefList(const Block * block_, size_t row_num_) : RowRef(block_, row_num_) {} ForwardIterator begin() const { return ForwardIterator(this); } @@ -141,123 +143,23 @@ private: * After calling any of the lookup methods, it is no longer allowed to insert more data as this would invalidate the * references that can be returned by the lookup methods */ - -template -class SortedLookupVector +struct SortedLookupVectorBase { -public: - using Base = std::vector; - - // First stage, insertions into the vector - template - void insert(U && x, TAllocatorParams &&... allocator_params) - { - assert(!sorted.load(std::memory_order_acquire)); - array.push_back(std::forward(x), std::forward(allocator_params)...); - } - - const RowRef * upperBound(const TEntry & k, bool ascending) - { - sort(ascending); - auto it = std::upper_bound(array.cbegin(), array.cend(), k, (ascending ? less : greater)); - if (it != array.cend()) - return &(it->row_ref); - return nullptr; - } - - const RowRef * lowerBound(const TEntry & k, bool ascending) - { - sort(ascending); - auto it = std::lower_bound(array.cbegin(), array.cend(), k, (ascending ? less : greater)); - if (it != array.cend()) - return &(it->row_ref); - return nullptr; - } - -private: - std::atomic sorted = false; - Base array; - mutable std::mutex lock; - - static bool less(const TEntry & a, const TEntry & b) - { - return a.asof_value < b.asof_value; - } - - static bool greater(const TEntry & a, const TEntry & b) - { - return a.asof_value > b.asof_value; - } - - // Double checked locking with SC atomics works in C++ - // https://preshing.com/20130930/double-checked-locking-is-fixed-in-cpp11/ - // The first thread that calls one of the lookup methods sorts the data - // After calling the first lookup method it is no longer allowed to insert any data - // the array becomes immutable - void sort(bool ascending) - { - if (!sorted.load(std::memory_order_acquire)) - { - std::lock_guard l(lock); - if (!sorted.load(std::memory_order_relaxed)) - { - if (!array.empty()) - ::sort(array.begin(), array.end(), (ascending ? less : greater)); - - sorted.store(true, std::memory_order_release); - } - } - } -}; - -class AsofRowRefs -{ -public: - template - struct Entry - { - using LookupType = SortedLookupVector, T>; - using LookupPtr = std::unique_ptr; - T asof_value; - RowRef row_ref; - - Entry(T v) : asof_value(v) {} - Entry(T v, RowRef rr) : asof_value(v), row_ref(rr) {} - }; - - using Lookups = std::variant< - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr, - Entry::LookupPtr>; - - AsofRowRefs() {} - AsofRowRefs(TypeIndex t); + SortedLookupVectorBase() = default; + virtual ~SortedLookupVectorBase() = default; static std::optional getTypeSize(const IColumn & asof_column, size_t & type_size); // This will be synchronized by the rwlock mutex in Join.h - void insert(TypeIndex type, const IColumn & asof_column, const Block * block, size_t row_num); + virtual void insert(const IColumn &, const Block *, size_t) = 0; - // This will internally synchronize - const RowRef * findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn & asof_column, size_t row_num) const; - -private: - // Lookups can be stored in a HashTable because it is memmovable - // A std::variant contains a currently active type id (memmovable), together with a union of the types - // The types are all std::unique_ptr, which contains a single pointer, which is memmovable. - // Source: https://github.com/ClickHouse/ClickHouse/issues/4906 - Lookups lookups; + // This needs to be synchronized internally + virtual RowRef findAsof(const IColumn &, size_t) = 0; }; + +// It only contains a std::unique_ptr which is memmovable. +// Source: https://github.com/ClickHouse/ClickHouse/issues/4906 +using AsofRowRefs = std::unique_ptr; +AsofRowRefs createAsofRowRef(TypeIndex type, ASOF::Inequality inequality); } diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index d9698be1a9b..a0c29c07d38 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -77,7 +77,7 @@ SessionLogElement::SessionLogElement(const UUID & auth_id_, Type type_) NamesAndTypesList SessionLogElement::getNamesAndTypes() { - const auto event_type = std::make_shared( + auto event_type = std::make_shared( DataTypeEnum8::Values { {"LoginFailure", static_cast(SESSION_LOGIN_FAILURE)}, @@ -86,7 +86,7 @@ NamesAndTypesList SessionLogElement::getNamesAndTypes() }); #define AUTH_TYPE_NAME_AND_VALUE(v) std::make_pair(AuthenticationTypeInfo::get(v).raw_name, static_cast(v)) - const auto identified_with_column = std::make_shared( + auto identified_with_column = std::make_shared( DataTypeEnum8::Values { AUTH_TYPE_NAME_AND_VALUE(AuthType::NO_PASSWORD), @@ -98,7 +98,7 @@ NamesAndTypesList SessionLogElement::getNamesAndTypes() }); #undef AUTH_TYPE_NAME_AND_VALUE - const auto interface_type_column = std::make_shared( + auto interface_type_column = std::make_shared( DataTypeEnum8::Values { {"TCP", static_cast(Interface::TCP)}, @@ -108,9 +108,9 @@ NamesAndTypesList SessionLogElement::getNamesAndTypes() {"PostgreSQL", static_cast(Interface::POSTGRESQL)} }); - const auto lc_string_datatype = std::make_shared(std::make_shared()); + auto lc_string_datatype = std::make_shared(std::make_shared()); - const auto settings_type_column = std::make_shared( + auto settings_type_column = std::make_shared( std::make_shared( DataTypes({ // setting name diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 32dac7f9e9b..224b13d2c45 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -165,7 +165,7 @@ void Set::setHeader(const ColumnsWithTypeAndName & header) bool Set::insertFromBlock(const ColumnsWithTypeAndName & columns) { - std::unique_lock lock(rwlock); + std::lock_guard lock(rwlock); if (data.empty()) throw Exception("Method Set::setHeader must be called before Set::insertFromBlock", ErrorCodes::LOGICAL_ERROR); @@ -445,7 +445,7 @@ MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vector & key_ranges, const DataTypes & data_types) const +BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, const DataTypes & data_types, bool single_point) const { size_t tuple_size = indexes_mapping.size(); @@ -468,7 +468,8 @@ BoolMask MergeTreeSetIndex::checkInRange(const std::vector & key_ranges, std::optional new_range = KeyCondition::applyMonotonicFunctionsChainToRange( key_ranges[indexes_mapping[i].key_index], indexes_mapping[i].functions, - data_types[indexes_mapping[i].key_index]); + data_types[indexes_mapping[i].key_index], + single_point); if (!new_range) return {true, true}; diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index 3146b6af03f..2eecb0211a4 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -214,7 +214,7 @@ public: bool hasMonotonicFunctionsChain() const; - BoolMask checkInRange(const std::vector & key_ranges, const DataTypes & data_types) const; + BoolMask checkInRange(const std::vector & key_ranges, const DataTypes & data_types, bool single_point = false) const; private: // If all arguments in tuple are key columns, we can optimize NOT IN when there is only one element. diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h index f1fcfde25c0..29ba24c2e4c 100644 --- a/src/Interpreters/StorageID.h +++ b/src/Interpreters/StorageID.h @@ -41,9 +41,9 @@ struct StorageID assertNotEmpty(); } - StorageID(const ASTQueryWithTableAndOutput & query); - StorageID(const ASTTableIdentifier & table_identifier_node); - StorageID(const ASTPtr & node); + StorageID(const ASTQueryWithTableAndOutput & query); /// NOLINT + StorageID(const ASTTableIdentifier & table_identifier_node); /// NOLINT + StorageID(const ASTPtr & node); /// NOLINT String getDatabaseName() const; diff --git a/src/Interpreters/SubqueryForSet.cpp b/src/Interpreters/SubqueryForSet.cpp index 08fc07c71e1..d669e091131 100644 --- a/src/Interpreters/SubqueryForSet.cpp +++ b/src/Interpreters/SubqueryForSet.cpp @@ -7,7 +7,7 @@ namespace DB SubqueryForSet::SubqueryForSet() = default; SubqueryForSet::~SubqueryForSet() = default; -SubqueryForSet::SubqueryForSet(SubqueryForSet &&) = default; -SubqueryForSet & SubqueryForSet::operator= (SubqueryForSet &&) = default; +SubqueryForSet::SubqueryForSet(SubqueryForSet &&) noexcept = default; +SubqueryForSet & SubqueryForSet::operator= (SubqueryForSet &&) noexcept = default; } diff --git a/src/Interpreters/SubqueryForSet.h b/src/Interpreters/SubqueryForSet.h index 974f5bd3e58..f737ec4582b 100644 --- a/src/Interpreters/SubqueryForSet.h +++ b/src/Interpreters/SubqueryForSet.h @@ -17,8 +17,8 @@ struct SubqueryForSet { SubqueryForSet(); ~SubqueryForSet(); - SubqueryForSet(SubqueryForSet &&); - SubqueryForSet & operator= (SubqueryForSet &&); + SubqueryForSet(SubqueryForSet &&) noexcept; + SubqueryForSet & operator=(SubqueryForSet &&) noexcept; /// The source is obtained using the InterpreterSelectQuery subquery. std::unique_ptr source; diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index ec6fd98010d..3b4d665e41b 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -22,7 +22,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -39,6 +41,57 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + +namespace +{ + class StorageWithComment : public IAST + { + public: + ASTPtr storage; + ASTPtr comment; + + String getID(char) const override { return "Storage with comment definition"; } + + ASTPtr clone() const override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method clone is not supported"); + } + + void formatImpl(const FormatSettings &, FormatState &, FormatStateStacked) const override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method formatImpl is not supported"); + } + }; + + class ParserStorageWithComment : public IParserBase + { + protected: + const char * getName() const override { return "storage definition with comment"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override + { + ParserStorage storage_p; + ASTPtr storage; + + if (!storage_p.parse(pos, storage, expected)) + return false; + + ParserKeyword s_comment("COMMENT"); + ParserStringLiteral string_literal_parser; + ASTPtr comment; + + if (s_comment.ignore(pos, expected)) + string_literal_parser.parse(pos, comment, expected); + + auto storage_with_comment = std::make_shared(); + storage_with_comment->storage = std::move(storage); + storage_with_comment->comment = std::move(comment); + + node = storage_with_comment; + return true; + } + }; } namespace @@ -100,8 +153,9 @@ std::shared_ptr createSystemLog( engine += " TTL " + ttl; engine += " ORDER BY (event_date, event_time)"; } + // Validate engine definition grammatically to prevent some configuration errors - ParserStorage storage_parser; + ParserStorageWithComment storage_parser; parseQuery(storage_parser, engine.data(), engine.data() + engine.size(), "Storage to create table for " + config_prefix, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); @@ -112,9 +166,7 @@ std::shared_ptr createSystemLog( } -/// returns CREATE TABLE query, but with removed: -/// - UUID -/// - SETTINGS (for MergeTree) +/// returns CREATE TABLE query, but with removed UUID /// That way it can be used to compare with the SystemLog::getCreateTableQuery() ASTPtr getCreateTableQueryClean(const StorageID & table_id, ContextPtr context) { @@ -123,11 +175,6 @@ ASTPtr getCreateTableQueryClean(const StorageID & table_id, ContextPtr context) auto & old_create_query_ast = old_ast->as(); /// Reset UUID old_create_query_ast.uuid = UUIDHelpers::Nil; - /// Existing table has default settings (i.e. `index_granularity = 8192`), reset them. - if (ASTStorage * storage = old_create_query_ast.storage) - { - storage->reset(storage->settings); - } return old_ast; } @@ -455,7 +502,6 @@ void SystemLog::prepareTable() is_prepared = true; } - template ASTPtr SystemLog::getCreateTableQuery() { @@ -470,11 +516,26 @@ ASTPtr SystemLog::getCreateTableQuery() new_columns_list->set(new_columns_list->columns, InterpreterCreateQuery::formatColumns(ordinary_columns, alias_columns)); create->set(create->columns_list, new_columns_list); - ParserStorage storage_parser; - ASTPtr storage_ast = parseQuery( + ParserStorageWithComment storage_parser; + + ASTPtr storage_with_comment_ast = parseQuery( storage_parser, storage_def.data(), storage_def.data() + storage_def.size(), "Storage to create table for " + LogElement::name(), 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - create->set(create->storage, storage_ast); + + StorageWithComment & storage_with_comment = storage_with_comment_ast->as(); + + create->set(create->storage, storage_with_comment.storage); + create->set(create->comment, storage_with_comment.comment); + + /// Write additional (default) settings for MergeTree engine to make it make it possible to compare ASTs + /// and recreate tables on settings changes. + const auto & engine = create->storage->engine->as(); + if (endsWith(engine.name, "MergeTree")) + { + auto storage_settings = std::make_unique(getContext()->getMergeTreeSettings()); + storage_settings->loadFromQuery(*create->storage); + } + return create; } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 42424b81192..7b7ccb689c3 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ #include #include +#include +#include namespace DB @@ -468,8 +471,33 @@ TableJoin::createConvertingActions(const ColumnsWithTypeAndName & left_sample_co NameToNameMap left_key_column_rename; NameToNameMap right_key_column_rename; - auto left_converting_actions = applyKeyConvertToTable(left_sample_columns, left_type_map, left_key_column_rename); - auto right_converting_actions = applyKeyConvertToTable(right_sample_columns, right_type_map, right_key_column_rename); + auto left_converting_actions = applyKeyConvertToTable(left_sample_columns, left_type_map, left_key_column_rename, forceNullableLeft()); + auto right_converting_actions = applyKeyConvertToTable(right_sample_columns, right_type_map, right_key_column_rename, forceNullableRight()); + + { + auto log_actions = [](const String & side, const ActionsDAGPtr & dag) + { + if (dag) + { + std::vector input_cols; + for (const auto & col : dag->getRequiredColumns()) + input_cols.push_back(col.name + ": " + col.type->getName()); + + std::vector output_cols; + for (const auto & col : dag->getResultColumns()) + output_cols.push_back(col.name + ": " + col.type->getName()); + + LOG_DEBUG(&Poco::Logger::get("TableJoin"), "{} JOIN converting actions: [{}] -> [{}]", + side, fmt::join(input_cols, ", "), fmt::join(output_cols, ", ")); + } + else + { + LOG_DEBUG(&Poco::Logger::get("TableJoin"), "{} JOIN converting actions: empty", side); + } + }; + log_actions("Left", left_converting_actions); + log_actions("Right", right_converting_actions); + } forAllKeys(clauses, [&](auto & left_key, auto & right_key) { @@ -482,10 +510,18 @@ TableJoin::createConvertingActions(const ColumnsWithTypeAndName & left_sample_co } template -bool TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right) +void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right) { + if (strictness() == ASTTableJoin::Strictness::Asof) + { + if (clauses.size() != 1) + throw DB::Exception("ASOF join over multiple keys is not supported", ErrorCodes::NOT_IMPLEMENTED); + if (right.back().type->isNullable()) + throw DB::Exception("ASOF join over right table Nullable column is not implemented", ErrorCodes::NOT_IMPLEMENTED); + } + if (!left_type_map.empty() || !right_type_map.empty()) - return true; + return; NameToTypeMap left_types; for (const auto & col : left) @@ -515,7 +551,7 @@ bool TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig try { /// TODO(vdimir): use getMostSubtype if possible - common_type = DB::getLeastSupertype({ltype->second, rtype->second}); + common_type = DB::getLeastSupertype(DataTypes{ltype->second, rtype->second}); } catch (DB::Exception & ex) { @@ -544,33 +580,74 @@ bool TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig formatTypeMap(left_type_map, left_types), formatTypeMap(right_type_map, right_types)); } - - return !left_type_map.empty(); } -ActionsDAGPtr TableJoin::applyKeyConvertToTable( - const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename) const +static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src, + const TableJoin::NameToTypeMap & type_mapping, + bool add_new_cols, + NameToNameMap & key_column_rename) { - bool has_some_to_do = false; - ColumnsWithTypeAndName cols_dst = cols_src; + bool has_some_to_do = false; for (auto & col : cols_dst) { if (auto it = type_mapping.find(col.name); it != type_mapping.end()) { - has_some_to_do = true; col.type = it->second; col.column = nullptr; + has_some_to_do = true; } } if (!has_some_to_do) return nullptr; + return ActionsDAG::makeConvertingActions(cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, add_new_cols, &key_column_rename); +} - /// Returns converting actions for tables that need to be performed before join - auto dag = ActionsDAG::makeConvertingActions( - cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, !hasUsing(), &key_column_rename); +static ActionsDAGPtr changeTypesToNullable(const ColumnsWithTypeAndName & cols_src, const NameSet & exception_cols) +{ + ColumnsWithTypeAndName cols_dst = cols_src; + bool has_some_to_do = false; + for (auto & col : cols_dst) + { + if (exception_cols.contains(col.name)) + continue; + col.type = JoinCommon::convertTypeToNullable(col.type); + col.column = nullptr; + has_some_to_do = true; + } - return dag; + if (!has_some_to_do) + return nullptr; + return ActionsDAG::makeConvertingActions(cols_src, cols_dst, ActionsDAG::MatchColumnsMode::Name, true, false, nullptr); +} + +ActionsDAGPtr TableJoin::applyKeyConvertToTable( + const ColumnsWithTypeAndName & cols_src, + const NameToTypeMap & type_mapping, + NameToNameMap & key_column_rename, + bool make_nullable) const +{ + /// Create DAG to convert key columns + ActionsDAGPtr dag_stage1 = changeKeyTypes(cols_src, type_mapping, !hasUsing(), key_column_rename); + + /// Create DAG to make columns nullable if needed + if (make_nullable) + { + /// Do not need to make nullable temporary columns that would be used only as join keys, but now shown to user + NameSet cols_not_nullable; + for (const auto & t : key_column_rename) + cols_not_nullable.insert(t.second); + + ColumnsWithTypeAndName input_cols = dag_stage1 ? dag_stage1->getResultColumns() : cols_src; + ActionsDAGPtr dag_stage2 = changeTypesToNullable(input_cols, cols_not_nullable); + + /// Merge dags if we got two ones + if (dag_stage1) + return ActionsDAG::merge(std::move(*dag_stage1), std::move(*dag_stage2)); + else + return dag_stage2; + } + return dag_stage1; } void TableJoin::setStorageJoin(std::shared_ptr storage) diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index b3e5748fb2f..f7c03ac6e1a 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -103,7 +103,7 @@ private: friend class TreeRewriter; - const SizeLimits size_limits; + SizeLimits size_limits; const size_t default_max_bytes = 0; const bool join_use_nulls = false; const size_t max_joined_block_rows = 0; @@ -114,7 +114,7 @@ private: const String temporary_files_codec = "LZ4"; /// the limit has no technical reasons, it supposed to improve safety - const size_t MAX_DISJUNCTS = 16; + const size_t MAX_DISJUNCTS = 16; /// NOLINT ASTs key_asts_left; ASTs key_asts_right; @@ -152,7 +152,8 @@ private: /// Create converting actions and change key column names if required ActionsDAGPtr applyKeyConvertToTable( - const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename) const; + const ColumnsWithTypeAndName & cols_src, const NameToTypeMap & type_mapping, NameToNameMap & key_column_rename, + bool make_nullable) const; void addKey(const String & left_name, const String & right_name, const ASTPtr & left_ast, const ASTPtr & right_ast = nullptr); @@ -160,7 +161,7 @@ private: /// Calculates common supertypes for corresponding join key columns. template - bool inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right); + void inferJoinKeyCommonType(const LeftNamesAndTypes & left, const RightNamesAndTypes & right, bool allow_right); NamesAndTypesList correctedColumnsAddedByJoin() const; diff --git a/src/Interpreters/TableOverrideUtils.cpp b/src/Interpreters/TableOverrideUtils.cpp index 922dd6af25b..58e885380bf 100644 --- a/src/Interpreters/TableOverrideUtils.cpp +++ b/src/Interpreters/TableOverrideUtils.cpp @@ -96,7 +96,7 @@ void TableOverrideAnalyzer::analyze(const StorageInMemoryMetadata & metadata, Re { auto * override_column = column_ast->as(); auto override_type = DataTypeFactory::instance().get(override_column->type); - auto found = metadata.columns.tryGetColumnOrSubcolumn(ColumnsDescription::GetFlags::All, override_column->name); + auto found = metadata.columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, override_column->name); std::optional override_default_kind; if (!override_column->default_specifier.empty()) override_default_kind = columnDefaultKindFromString(override_column->default_specifier); diff --git a/src/Interpreters/TraceCollector.h b/src/Interpreters/TraceCollector.h index 3a9edf676be..b3f11ca5756 100644 --- a/src/Interpreters/TraceCollector.h +++ b/src/Interpreters/TraceCollector.h @@ -18,7 +18,7 @@ class TraceLog; class TraceCollector { public: - TraceCollector(std::shared_ptr trace_log_); + explicit TraceCollector(std::shared_ptr trace_log_); ~TraceCollector(); static inline void collect(TraceType trace_type, const StackTrace & stack_trace, Int64 size) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 0d7d56058b9..6016d54c7dc 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -303,7 +303,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt } /// 'select * from a join b using id' should result one 'id' column -void TranslateQualifiedNamesMatcher::extractJoinUsingColumns(const ASTPtr ast, Data & data) +void TranslateQualifiedNamesMatcher::extractJoinUsingColumns(ASTPtr ast, Data & data) { const auto & table_join = ast->as(); diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.h b/src/Interpreters/TranslateQualifiedNamesVisitor.h index 0f35d052ed2..9c46d926eca 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -52,7 +52,7 @@ private: static void visit(ASTExpressionList &, const ASTPtr &, Data &); static void visit(ASTFunction &, const ASTPtr &, Data &); - static void extractJoinUsingColumns(const ASTPtr ast, Data & data); + static void extractJoinUsingColumns(ASTPtr ast, Data & data); }; /// Visits AST for names qualification. diff --git a/src/Interpreters/TreeCNFConverter.h b/src/Interpreters/TreeCNFConverter.h index 0c090c8d56b..a5d42e6b989 100644 --- a/src/Interpreters/TreeCNFConverter.h +++ b/src/Interpreters/TreeCNFConverter.h @@ -36,7 +36,7 @@ public: using OrGroup = std::set; using AndGroup = std::set; - CNFQuery(AndGroup && statements_) : statements(std::move(statements_)) { } + CNFQuery(AndGroup && statements_) : statements(std::move(statements_)) { } /// NOLINT template CNFQuery & filterAlwaysTrueGroups(P predicate_is_unknown) /// delete always true groups @@ -91,7 +91,7 @@ public: CNFQuery & appendGroup(AndGroup&& and_group) { for (auto && or_group : and_group) - statements.emplace(std::move(or_group)); + statements.emplace(or_group); return *this; } diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 64b25ca9777..8885db1ad78 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -445,7 +445,7 @@ void optimizeMonotonousFunctionsInOrderBy(ASTSelectQuery * select_query, Context } } - auto sorting_key_columns = result.metadata_snapshot ? result.metadata_snapshot->getSortingKeyColumns() : Names{}; + auto sorting_key_columns = result.storage_snapshot ? result.storage_snapshot->metadata->getSortingKeyColumns() : Names{}; bool is_sorting_key_prefix = true; for (size_t i = 0; i < order_by->children.size(); ++i) @@ -740,9 +740,8 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, if (!select_query) throw Exception("Select analyze for not select asts.", ErrorCodes::LOGICAL_ERROR); - if (settings.optimize_functions_to_subcolumns && result.storage - && result.storage->supportsSubcolumns() && result.metadata_snapshot) - optimizeFunctionsToSubcolumns(query, result.metadata_snapshot); + if (settings.optimize_functions_to_subcolumns && result.storage_snapshot && result.storage->supportsSubcolumns()) + optimizeFunctionsToSubcolumns(query, result.storage_snapshot->metadata); /// Move arithmetic operations out of aggregation functions if (settings.optimize_arithmetic_operations_in_aggregate_functions) @@ -752,14 +751,14 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, if (settings.convert_query_to_cnf) converted_to_cnf = convertQueryToCNF(select_query); - if (converted_to_cnf && settings.optimize_using_constraints) + if (converted_to_cnf && settings.optimize_using_constraints && result.storage_snapshot) { optimizeWithConstraints(select_query, result.aliases, result.source_columns_set, - tables_with_columns, result.metadata_snapshot, settings.optimize_append_index); + tables_with_columns, result.storage_snapshot->metadata, settings.optimize_append_index); if (settings.optimize_substitute_columns) optimizeSubstituteColumn(select_query, result.aliases, result.source_columns_set, - tables_with_columns, result.metadata_snapshot, result.storage); + tables_with_columns, result.storage_snapshot->metadata, result.storage); } /// GROUP BY injective function elimination. diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index fc3ef681c2c..78e7ed33f8f 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -518,7 +518,7 @@ void getArrayJoinedColumns(ASTPtr & query, TreeRewriterResult & result, const AS bool found = false; for (const auto & column : source_columns) { - auto split = Nested::splitName(column.name); + auto split = Nested::splitName(column.name, /*reverse=*/ true); if (split.first == source_name && !split.second.empty()) { result.array_join_result_to_source[Nested::concatenateName(result_name, split.second)] = column.name; @@ -831,10 +831,10 @@ using RewriteShardNumVisitor = InDepthNodeVisitor; TreeRewriterResult::TreeRewriterResult( const NamesAndTypesList & source_columns_, ConstStoragePtr storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, bool add_special) : storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , source_columns(source_columns_) { collectSourceColumns(add_special); @@ -847,13 +847,12 @@ void TreeRewriterResult::collectSourceColumns(bool add_special) { if (storage) { - const ColumnsDescription & columns = metadata_snapshot->getColumns(); - - NamesAndTypesList columns_from_storage; + auto options = GetColumnsOptions(add_special ? GetColumnsOptions::All : GetColumnsOptions::AllPhysical); + options.withExtendedObjects(); if (storage->supportsSubcolumns()) - columns_from_storage = add_special ? columns.getAllWithSubcolumns() : columns.getAllPhysicalWithSubcolumns(); - else - columns_from_storage = add_special ? columns.getAll() : columns.getAllPhysical(); + options.withSubcolumns(); + + auto columns_from_storage = storage_snapshot->getColumns(options); if (source_columns.empty()) source_columns.swap(columns_from_storage); @@ -960,9 +959,9 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select /// If we have no information about columns sizes, choose a column of minimum size of its data type. required.insert(ExpressionActions::getSmallestColumn(source_columns)); } - else if (is_select && metadata_snapshot && !columns_context.has_array_join) + else if (is_select && storage_snapshot && !columns_context.has_array_join) { - const auto & partition_desc = metadata_snapshot->getPartitionKey(); + const auto & partition_desc = storage_snapshot->metadata->getPartitionKey(); if (partition_desc.expression) { auto partition_source_columns = partition_desc.expression->getRequiredColumns(); @@ -1018,7 +1017,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select { for (const auto & name_type : storage_virtuals) { - if (name_type.name == "_shard_num" && storage->isVirtualColumn("_shard_num", metadata_snapshot)) + if (name_type.name == "_shard_num" && storage->isVirtualColumn("_shard_num", storage_snapshot->getMetadataForQuery())) { has_virtual_shard_num = true; break; @@ -1190,7 +1189,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// rewrite filters for select query, must go after getArrayJoinedColumns bool is_initiator = getContext()->getClientInfo().distributed_depth == 0; - if (settings.optimize_respect_aliases && result.metadata_snapshot && is_initiator) + if (settings.optimize_respect_aliases && result.storage_snapshot && is_initiator) { std::unordered_set excluded_nodes; { @@ -1201,7 +1200,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( excluded_nodes.insert(table_join_ast->using_expression_list.get()); } - bool is_changed = replaceAliasColumnsInQuery(query, result.metadata_snapshot->getColumns(), + bool is_changed = replaceAliasColumnsInQuery(query, result.storage_snapshot->metadata->getColumns(), result.array_join_result_to_source, getContext(), excluded_nodes); /// If query is changed, we need to redo some work to correct name resolution. if (is_changed) @@ -1234,7 +1233,7 @@ TreeRewriterResultPtr TreeRewriter::analyze( ASTPtr & query, const NamesAndTypesList & source_columns, ConstStoragePtr storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, bool allow_aggregations, bool allow_self_aliases, bool execute_scalar_subqueries) const @@ -1244,7 +1243,7 @@ TreeRewriterResultPtr TreeRewriter::analyze( const auto & settings = getContext()->getSettingsRef(); - TreeRewriterResult result(source_columns, storage, metadata_snapshot, false); + TreeRewriterResult result(source_columns, storage, storage_snapshot, false); normalize(query, result.aliases, result.source_columns_set, false, settings, allow_self_aliases); diff --git a/src/Interpreters/TreeRewriter.h b/src/Interpreters/TreeRewriter.h index 45b3a5a00e3..7fbe4e45fb3 100644 --- a/src/Interpreters/TreeRewriter.h +++ b/src/Interpreters/TreeRewriter.h @@ -19,11 +19,13 @@ struct SelectQueryOptions; using Scalars = std::map; struct StorageInMemoryMetadata; using StorageMetadataPtr = std::shared_ptr; +struct StorageSnapshot; +using StorageSnapshotPtr = std::shared_ptr; struct TreeRewriterResult { ConstStoragePtr storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; std::shared_ptr analyzed_join; const ASTTablesInSelectQueryElement * ast_join = nullptr; @@ -80,7 +82,7 @@ struct TreeRewriterResult explicit TreeRewriterResult( const NamesAndTypesList & source_columns_, ConstStoragePtr storage_ = {}, - const StorageMetadataPtr & metadata_snapshot_ = {}, + const StorageSnapshotPtr & storage_snapshot_ = {}, bool add_special = true); void collectSourceColumns(bool add_special); @@ -112,7 +114,7 @@ public: ASTPtr & query, const NamesAndTypesList & source_columns_, ConstStoragePtr storage = {}, - const StorageMetadataPtr & metadata_snapshot = {}, + const StorageSnapshotPtr & storage_snapshot = {}, bool allow_aggregations = false, bool allow_self_aliases = true, bool execute_scalar_subqueries = true) const; diff --git a/src/Interpreters/UserDefinedExecutableFunction.h b/src/Interpreters/UserDefinedExecutableFunction.h index 80d6b85ad90..434c77e9236 100644 --- a/src/Interpreters/UserDefinedExecutableFunction.h +++ b/src/Interpreters/UserDefinedExecutableFunction.h @@ -10,13 +10,20 @@ namespace DB { +struct UserDefinedExecutableFunctionArgument +{ + DataTypePtr type; + String name; +}; + struct UserDefinedExecutableFunctionConfiguration { std::string name; std::string command; std::vector command_arguments; - std::vector argument_types; + std::vector arguments; DataTypePtr result_type; + String result_name; }; class UserDefinedExecutableFunction final : public IExternalLoadable diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp index 10cb806028e..6d7dee7a4c7 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp @@ -42,11 +42,12 @@ public: bool isVariadic() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - size_t getNumberOfArguments() const override { return executable_function->getConfiguration().argument_types.size(); } + size_t getNumberOfArguments() const override { return executable_function->getConfiguration().arguments.size(); } bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForNulls() const override { return true; } bool isDeterministic() const override { return false; } + bool isDeterministicInScopeOfQuery() const override { return false; } DataTypePtr getReturnTypeImpl(const DataTypes &) const override { @@ -90,7 +91,11 @@ public: auto & column_with_type = arguments_copy[i]; column_with_type.column = column_with_type.column->convertToFullColumnIfConst(); - const auto & argument_type = configuration.argument_types[i]; + const auto & argument = configuration.arguments[i]; + column_with_type.name = argument.name; + + const auto & argument_type = argument.type; + if (areTypesEqual(arguments_copy[i].type, argument_type)) continue; @@ -101,7 +106,7 @@ public: column_with_type = std::move(column_to_cast); } - ColumnWithTypeAndName result(result_type, "result"); + ColumnWithTypeAndName result(result_type, configuration.result_name); Block result_block({result}); Block arguments_block(arguments_copy); diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index f6d5f83fce3..d043fd16bb5 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -63,7 +63,7 @@ ActionsDAGPtr addMissingDefaults( { const auto & nested_type = array_type->getNestedType(); ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(0); - const auto & constant = actions->addColumn({std::move(nested_column), nested_type, column.name}); + const auto & constant = actions->addColumn({nested_column, nested_type, column.name}); auto & group = nested_groups[offsets_name]; group[0] = &constant; @@ -76,17 +76,17 @@ ActionsDAGPtr addMissingDefaults( * it can be full (or the interpreter may decide that it is constant everywhere). */ auto new_column = column.type->createColumnConstWithDefaultValue(0); - const auto * col = &actions->addColumn({std::move(new_column), column.type, column.name}); + const auto * col = &actions->addColumn({new_column, column.type, column.name}); index.push_back(&actions->materializeNode(*col)); } /// Computes explicitly specified values by default and materialized columns. if (auto dag = evaluateMissingDefaults(actions->getResultColumns(), required_columns, columns, context, true, null_as_default)) actions = ActionsDAG::merge(std::move(*actions), std::move(*dag)); - else - /// Removes unused columns and reorders result. - /// The same is done in evaluateMissingDefaults if not empty dag is returned. - actions->removeUnusedActions(required_columns.getNames()); + + /// Removes unused columns and reorders result. + actions->removeUnusedActions(required_columns.getNames(), false); + actions->addMaterializingOutputActions(); return actions; } diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 5813b8c3926..7abe8342100 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #include #include @@ -246,6 +248,8 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID } return src; } + + return applyVisitor(FieldVisitorToString(), src); } else if (const DataTypeArray * type_array = typeid_cast(&type)) { @@ -363,6 +367,46 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID return src; } + else if (isObject(type)) + { + const auto * from_type_tuple = typeid_cast(from_type_hint); + if (src.getType() == Field::Types::Tuple && from_type_tuple && from_type_tuple->haveExplicitNames()) + { + const auto & names = from_type_tuple->getElementNames(); + const auto & tuple = src.get(); + + if (names.size() != tuple.size()) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Bad size of tuple in IN or VALUES section (while converting to Object). Expected size: {}, actual size: {}", + names.size(), tuple.size()); + + Object object; + for (size_t i = 0; i < names.size(); ++i) + object[names[i]] = tuple[i]; + + return object; + } + + if (src.getType() == Field::Types::Map) + { + Object object; + const auto & map = src.get(); + for (size_t i = 0; i < map.size(); ++i) + { + const auto & map_entry = map[i].get(); + const auto & key = map_entry[0]; + const auto & value = map_entry[1]; + + if (key.getType() != Field::Types::String) + throw Exception(ErrorCodes::TYPE_MISMATCH, + "Cannot convert from Map with key of type {} to Object", key.getTypeName()); + + object[key.get()] = value; + } + + return object; + } + } /// Conversion from string by parsing. if (src.getType() == Field::Types::String) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index ae304906476..e7c5095b7fb 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,9 @@ namespace ErrorCodes std::pair> evaluateConstantExpression(const ASTPtr & node, ContextPtr context) { + if (ASTLiteral * literal = node->as()) + return std::make_pair(literal->value, applyVisitor(FieldToDataType(), literal->value)); + NamesAndTypesList source_columns = {{ "_dummy", std::make_shared() }}; auto ast = node->clone(); ReplaceQueryParameterVisitor param_visitor(context->getQueryParameters()); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index f40d35e970b..c1606700540 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -413,9 +413,10 @@ static std::tuple executeQueryImpl( ASTPtr ast; const char * query_end; - /// Don't limit the size of internal queries. - size_t max_query_size = 0; - if (!internal) max_query_size = settings.max_query_size; + size_t max_query_size = settings.max_query_size; + /// Don't limit the size of internal queries or distributed subquery. + if (internal || client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) + max_query_size = 0; String query_database; String query_table; @@ -780,8 +781,8 @@ static std::tuple executeQueryImpl( element.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0; - element.thread_ids = std::move(info.thread_ids); - element.profile_counters = std::move(info.profile_counters); + element.thread_ids = info.thread_ids; + element.profile_counters = info.profile_counters; /// We need to refresh the access info since dependent views might have added extra information, either during /// creation of the view (PushingToViewsBlockOutputStream) or while executing its internal SELECT diff --git a/src/Interpreters/getColumnFromBlock.cpp b/src/Interpreters/getColumnFromBlock.cpp new file mode 100644 index 00000000000..ce6fa2904db --- /dev/null +++ b/src/Interpreters/getColumnFromBlock.cpp @@ -0,0 +1,50 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_FOUND_COLUMN_IN_BLOCK; +} + +ColumnPtr tryGetColumnFromBlock(const Block & block, const NameAndTypePair & requested_column) +{ + const auto * elem = block.findByName(requested_column.getNameInStorage()); + if (!elem) + return nullptr; + + DataTypePtr elem_type; + ColumnPtr elem_column; + + if (requested_column.isSubcolumn()) + { + auto subcolumn_name = requested_column.getSubcolumnName(); + elem_type = elem->type->tryGetSubcolumnType(subcolumn_name); + elem_column = elem->type->tryGetSubcolumn(subcolumn_name, elem->column); + + if (!elem_type || !elem_column) + return nullptr; + } + else + { + elem_type = elem->type; + elem_column = elem->column; + } + + return castColumn({elem_column, elem_type, ""}, requested_column.type); +} + +ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & requested_column) +{ + auto result_column = tryGetColumnFromBlock(block, requested_column); + if (!result_column) + throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, + "Not found column or subcolumn {} in block. There are only columns: {}", + requested_column.name, block.dumpNames()); + + return result_column; +} + +} diff --git a/src/Interpreters/getColumnFromBlock.h b/src/Interpreters/getColumnFromBlock.h new file mode 100644 index 00000000000..26500cfdd17 --- /dev/null +++ b/src/Interpreters/getColumnFromBlock.h @@ -0,0 +1,13 @@ +#pragma once +#include + +namespace DB +{ + +/// Helps in-memory storages to extract columns from block. +/// Properly handles cases, when column is a subcolumn and when it is compressed. +ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & requested_column); + +ColumnPtr tryGetColumnFromBlock(const Block & block, const NameAndTypePair & requested_column); + +} diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index 69b7b7d833f..6f4d7d5c525 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -57,7 +57,7 @@ bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, C const size_t left_table_pos = 0; /// Test each argument of `and` function and select ones related to only left table std::shared_ptr new_conj = makeASTFunction("and"); - for (const auto & node : collectConjunctions(where)) + for (auto && node : collectConjunctions(where)) { if (membership_collector.getIdentsMembership(node) == left_table_pos) new_conj->arguments->children.push_back(std::move(node)); @@ -82,9 +82,8 @@ bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, C } Block getHeaderForProcessingStage( - const IStorage & storage, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage) @@ -93,7 +92,8 @@ Block getHeaderForProcessingStage( { case QueryProcessingStage::FetchColumns: { - Block header = metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID()); + Block header = storage_snapshot->getSampleBlockForColumns(column_names); + if (query_info.prewhere_info) { auto & prewhere_info = *query_info.prewhere_info; @@ -123,7 +123,7 @@ Block getHeaderForProcessingStage( removeJoin(*query->as(), new_rewriter_result, context); auto pipe = Pipe(std::make_shared( - metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID()))); + storage_snapshot->getSampleBlockForColumns(column_names))); return InterpreterSelectQuery(query, context, std::move(pipe), SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); } } diff --git a/src/Interpreters/getHeaderForProcessingStage.h b/src/Interpreters/getHeaderForProcessingStage.h index 54a1126a3df..6ada136030e 100644 --- a/src/Interpreters/getHeaderForProcessingStage.h +++ b/src/Interpreters/getHeaderForProcessingStage.h @@ -10,8 +10,8 @@ namespace DB { class IStorage; -struct StorageInMemoryMetadata; -using StorageMetadataPtr = std::shared_ptr; +struct StorageSnapshot; +using StorageSnapshotPtr = std::shared_ptr; struct SelectQueryInfo; struct TreeRewriterResult; class ASTSelectQuery; @@ -20,9 +20,8 @@ bool hasJoin(const ASTSelectQuery & select); bool removeJoin(ASTSelectQuery & select, TreeRewriterResult & rewriter_result, ContextPtr context); Block getHeaderForProcessingStage( - const IStorage & storage, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage); diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 2841abe757e..15dd9229194 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -15,11 +15,20 @@ #include #include #include +#include +#include +#include +#include namespace DB { +namespace ErrorCode +{ + extern const int LOGICAL_ERROR; +} + namespace { @@ -134,7 +143,6 @@ ActionsDAGPtr createExpressions( const Block & header, ASTPtr expr_list, bool save_unneeded_columns, - const NamesAndTypesList & required_columns, ContextPtr context) { if (!expr_list) @@ -146,12 +154,6 @@ ActionsDAGPtr createExpressions( auto actions = expression_analyzer.getActionsDAG(true, !save_unneeded_columns); dag = ActionsDAG::merge(std::move(*dag), std::move(*actions)); - if (save_unneeded_columns) - { - dag->removeUnusedActions(required_columns.getNames()); - dag->addMaterializingOutputActions(); - } - return dag; } @@ -163,7 +165,7 @@ void performRequiredConversions(Block & block, const NamesAndTypesList & require if (conversion_expr_list->children.empty()) return; - if (auto dag = createExpressions(block, conversion_expr_list, true, required_columns, context)) + if (auto dag = createExpressions(block, conversion_expr_list, true, context)) { auto expression = std::make_shared(std::move(dag), ExpressionActionsSettings::fromContext(context)); expression->execute(block); @@ -182,7 +184,100 @@ ActionsDAGPtr evaluateMissingDefaults( return nullptr; ASTPtr expr_list = defaultRequiredExpressions(header, required_columns, columns, null_as_default); - return createExpressions(header, expr_list, save_unneeded_columns, required_columns, context); + return createExpressions(header, expr_list, save_unneeded_columns, context); +} + +static bool arrayHasNoElementsRead(const IColumn & column) +{ + const auto * column_array = typeid_cast(&column); + + if (!column_array) + return false; + + size_t size = column_array->size(); + if (!size) + return false; + + size_t data_size = column_array->getData().size(); + if (data_size) + return false; + + size_t last_offset = column_array->getOffsets()[size - 1]; + return last_offset != 0; +} + +void fillMissingColumns( + Columns & res_columns, + size_t num_rows, + const NamesAndTypesList & requested_columns, + StorageMetadataPtr metadata_snapshot) +{ + size_t num_columns = requested_columns.size(); + if (num_columns != res_columns.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Invalid number of columns passed to fillMissingColumns. Expected {}, got {}", + num_columns, res_columns.size()); + + /// For a missing column of a nested data structure we must create not a column of empty + /// arrays, but a column of arrays of correct length. + + /// First, collect offset columns for all arrays in the block. + + std::unordered_map offset_columns; + auto requested_column = requested_columns.begin(); + for (size_t i = 0; i < num_columns; ++i, ++requested_column) + { + if (res_columns[i] == nullptr) + continue; + + if (const auto * array = typeid_cast(res_columns[i].get())) + { + String offsets_name = Nested::extractTableName(requested_column->name); + auto & offsets_column = offset_columns[offsets_name]; + + /// If for some reason multiple offsets columns are present for the same nested data structure, + /// choose the one that is not empty. + if (!offsets_column || offsets_column->empty()) + offsets_column = array->getOffsetsPtr(); + } + } + + /// insert default values only for columns without default expressions + requested_column = requested_columns.begin(); + for (size_t i = 0; i < num_columns; ++i, ++requested_column) + { + const auto & [name, type] = *requested_column; + + if (res_columns[i] && arrayHasNoElementsRead(*res_columns[i])) + res_columns[i] = nullptr; + + if (res_columns[i] == nullptr) + { + if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(name)) + continue; + + String offsets_name = Nested::extractTableName(name); + auto offset_it = offset_columns.find(offsets_name); + const auto * array_type = typeid_cast(type.get()); + if (offset_it != offset_columns.end() && array_type) + { + const auto & nested_type = array_type->getNestedType(); + ColumnPtr offsets_column = offset_it->second; + size_t nested_rows = typeid_cast(*offsets_column).getData().back(); + + ColumnPtr nested_column = + nested_type->createColumnConstWithDefaultValue(nested_rows)->convertToFullColumnIfConst(); + + res_columns[i] = ColumnArray::create(nested_column, offsets_column); + } + else + { + /// We must turn a constant column into a full column because the interpreter could infer + /// that it is constant everywhere but in some blocks (from other parts) it can be a full column. + res_columns[i] = type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst(); + } + } + } } } diff --git a/src/Interpreters/inplaceBlockConversions.h b/src/Interpreters/inplaceBlockConversions.h index cc8261693f9..b3113ddfa5c 100644 --- a/src/Interpreters/inplaceBlockConversions.h +++ b/src/Interpreters/inplaceBlockConversions.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -14,6 +15,13 @@ class Block; class NamesAndTypesList; class ColumnsDescription; +class IColumn; +using ColumnPtr = COW::Ptr; +using Columns = std::vector; + +struct StorageInMemoryMetadata; +using StorageMetadataPtr = std::shared_ptr; + class ActionsDAG; using ActionsDAGPtr = std::shared_ptr; @@ -31,4 +39,10 @@ ActionsDAGPtr evaluateMissingDefaults( /// Tries to convert columns in block to required_columns void performRequiredConversions(Block & block, const NamesAndTypesList & required_columns, ContextPtr context); +void fillMissingColumns( + Columns & res_columns, + size_t num_rows, + const NamesAndTypesList & requested_columns, + StorageMetadataPtr metadata_snapshot); + } diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index ca55fde0740..478df653f3b 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -115,10 +115,21 @@ bool canBecomeNullable(const DataTypePtr & type) return can_be_inside; } +bool isNullable(const DataTypePtr & type) +{ + bool is_nullable = type->isNullable(); + if (const auto * low_cardinality_type = typeid_cast(type.get())) + is_nullable |= low_cardinality_type->getDictionaryType()->isNullable(); + return is_nullable; +} + /// Add nullability to type. /// Note: LowCardinality(T) transformed to LowCardinality(Nullable(T)) DataTypePtr convertTypeToNullable(const DataTypePtr & type) { + if (isNullable(type)) + return type; + if (const auto * low_cardinality_type = typeid_cast(type.get())) { const auto & dict_type = low_cardinality_type->getDictionaryType(); @@ -425,10 +436,13 @@ void checkTypesOfKeys(const Block & block_left, const Names & key_names_left, DataTypePtr right_type = removeNullable(recursiveRemoveLowCardinality(block_right.getByName(key_names_right[i]).type)); if (!left_type->equals(*right_type)) - throw Exception("Type mismatch of columns to JOIN by: " - + key_names_left[i] + " " + left_type->getName() + " at left, " - + key_names_right[i] + " " + right_type->getName() + " at right", - ErrorCodes::TYPE_MISMATCH); + { + throw DB::Exception( + ErrorCodes::TYPE_MISMATCH, + "Type mismatch of columns to JOIN by: {} {} at left, {} {} at right", + key_names_left[i], left_type->getName(), + key_names_right[i], right_type->getName()); + } } } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 3e5a22f33bf..38b431db3e0 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -59,6 +59,7 @@ private: }; +bool isNullable(const DataTypePtr & type); bool canBecomeNullable(const DataTypePtr & type); DataTypePtr convertTypeToNullable(const DataTypePtr & type); void convertColumnToNullable(ColumnWithTypeAndName & column); diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index c8a2d0903f2..3281445022e 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -41,15 +41,14 @@ struct PartialSortingLessImpl explicit PartialSortingLessImpl(const ColumnsWithSortDescriptions & columns_) : columns(columns_) { } - inline bool operator()(size_t a, size_t b) const + ALWAYS_INLINE int compare(size_t lhs, size_t rhs) const { + int res = 0; + for (const auto & elem : columns) { - int res; - if (elem.column_const) { - res = 0; continue; } @@ -57,52 +56,37 @@ struct PartialSortingLessImpl { if (isCollationRequired(elem.description)) { - res = elem.column->compareAtWithCollation(a, b, *elem.column, elem.description.nulls_direction, *elem.description.collator); + res = elem.column->compareAtWithCollation(lhs, rhs, *elem.column, elem.description.nulls_direction, *elem.description.collator); } else { - res = elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); + res = elem.column->compareAt(lhs, rhs, *elem.column, elem.description.nulls_direction); } } else { - res = elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); + res = elem.column->compareAt(lhs, rhs, *elem.column, elem.description.nulls_direction); } res *= elem.description.direction; - if (res < 0) - return true; - else if (res > 0) - return false; + + if (res != 0) + break; } - return false; + + return res; + } + + ALWAYS_INLINE bool operator()(size_t lhs, size_t rhs) const + { + int res = compare(lhs, rhs); + return res < 0; } }; using PartialSortingLess = PartialSortingLessImpl; using PartialSortingLessWithCollation = PartialSortingLessImpl; -} - -void convertTupleColumnIntoSortDescriptions( - const ColumnTuple * tuple, const SortColumnDescription & description, ColumnsWithSortDescriptions & result) -{ - for (const auto & column : tuple->getColumns()) - { - if (const auto * subtuple = typeid_cast(column.get())) - { - convertTupleColumnIntoSortDescriptions(subtuple, description, result); - } - else - { - result.emplace_back(ColumnWithSortDescription{column.get(), description, isColumnConst(*column)}); - - if (isCollationRequired(description) && !result.back().column->isCollationSupported()) - result.back().description.collator = nullptr; - } - } -} - ColumnsWithSortDescriptions getColumnsWithSortDescription(const Block & block, const SortDescription & description) { size_t size = description.size(); @@ -127,16 +111,13 @@ ColumnsWithSortDescriptions getColumnsWithSortDescription(const Block & block, c ErrorCodes::BAD_COLLATION); } - if (const auto * tuple = typeid_cast(column)) - convertTupleColumnIntoSortDescriptions(tuple, sort_column_description, result); - else - result.emplace_back(ColumnWithSortDescription{column, sort_column_description, isColumnConst(*column)}); + result.emplace_back(ColumnWithSortDescription{column, sort_column_description, isColumnConst(*column)}); } return result; } -void sortBlock(Block & block, const SortDescription & description, UInt64 limit) +void getBlockSortPermutationImpl(const Block & block, const SortDescription & description, IColumn::PermutationSortStability stability, UInt64 limit, IColumn::Permutation & permutation) { if (!block) return; @@ -152,25 +133,24 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) break; } } - if (all_const) - return; - IColumn::Permutation permutation; + if (unlikely(all_const)) + return; /// If only one column to sort by if (columns_with_sort_descriptions.size() == 1) { auto & column_with_sort_description = columns_with_sort_descriptions[0]; - bool reverse = column_with_sort_description.description.direction == -1; + IColumn::PermutationSortDirection direction = column_with_sort_description.description.direction == -1 ? IColumn::PermutationSortDirection::Descending : IColumn::PermutationSortDirection::Ascending; int nan_direction_hint = column_with_sort_description.description.nulls_direction; const auto & column = column_with_sort_description.column; if (isCollationRequired(column_with_sort_description.description)) column->getPermutationWithCollation( - *column_with_sort_description.description.collator, reverse, limit, nan_direction_hint, permutation); + *column_with_sort_description.description.collator, direction, stability, limit, nan_direction_hint, permutation); else - column->getPermutation(reverse, limit, nan_direction_hint, permutation); + column->getPermutation(direction, stability, limit, nan_direction_hint, permutation); } else { @@ -197,21 +177,32 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) continue; bool is_collation_required = isCollationRequired(column_with_sort_description.description); - bool reverse = column_with_sort_description.description.direction < 0; + IColumn::PermutationSortDirection direction = column_with_sort_description.description.direction == -1 ? IColumn::PermutationSortDirection::Descending : IColumn::PermutationSortDirection::Ascending; int nan_direction_hint = column_with_sort_description.description.nulls_direction; const auto & column = column_with_sort_description.column; if (is_collation_required) { column->updatePermutationWithCollation( - *column_with_sort_description.description.collator, reverse, limit, nan_direction_hint, permutation, ranges); + *column_with_sort_description.description.collator, direction, stability, limit, nan_direction_hint, permutation, ranges); } else { - column->updatePermutation(reverse, limit, nan_direction_hint, permutation, ranges); + column->updatePermutation(direction, stability, limit, nan_direction_hint, permutation, ranges); } } } +} + +} + +void sortBlock(Block & block, const SortDescription & description, UInt64 limit) +{ + IColumn::Permutation permutation; + getBlockSortPermutationImpl(block, description, IColumn::PermutationSortStability::Unstable, limit, permutation); + + if (permutation.empty()) + return; size_t columns = block.columns(); for (size_t i = 0; i < columns; ++i) @@ -221,19 +212,31 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit) } } +void stableSortBlock(Block & block, const SortDescription & description) +{ + if (!block) + return; + + IColumn::Permutation permutation; + getBlockSortPermutationImpl(block, description, IColumn::PermutationSortStability::Stable, 0, permutation); + + if (permutation.empty()) + return; + + size_t columns = block.columns(); + for (size_t i = 0; i < columns; ++i) + { + auto & column_to_sort = block.getByPosition(i).column; + column_to_sort = column_to_sort->permute(permutation, 0); + } +} + void stableGetPermutation(const Block & block, const SortDescription & description, IColumn::Permutation & out_permutation) { if (!block) return; - size_t size = block.rows(); - out_permutation.resize(size); - for (size_t i = 0; i < size; ++i) - out_permutation[i] = i; - - ColumnsWithSortDescriptions columns_with_sort_desc = getColumnsWithSortDescription(block, description); - - std::stable_sort(out_permutation.begin(), out_permutation.end(), PartialSortingLess(columns_with_sort_desc)); + getBlockSortPermutationImpl(block, description, IColumn::PermutationSortStability::Stable, 0, out_permutation); } bool isAlreadySorted(const Block & block, const SortDescription & description) @@ -270,21 +273,4 @@ bool isAlreadySorted(const Block & block, const SortDescription & description) return true; } - -void stableSortBlock(Block & block, const SortDescription & description) -{ - if (!block) - return; - - IColumn::Permutation permutation; - stableGetPermutation(block, description, permutation); - - size_t columns = block.columns(); - for (size_t i = 0; i < columns; ++i) - { - auto & column_to_sort = block.safeGetByPosition(i).column; - column_to_sort = column_to_sort->permute(permutation, 0); - } -} - } diff --git a/src/Parsers/ASTAssignment.h b/src/Parsers/ASTAssignment.h index 88d4bb96c15..a37a31ae38e 100644 --- a/src/Parsers/ASTAssignment.h +++ b/src/Parsers/ASTAssignment.h @@ -28,6 +28,7 @@ public: protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override { + settings.ostr << (settings.hilite ? hilite_identifier : ""); settings.writeIdentifier(column_name); settings.ostr << (settings.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTBackupQuery.cpp b/src/Parsers/ASTBackupQuery.cpp index adc6bb97985..f8fcbd98872 100644 --- a/src/Parsers/ASTBackupQuery.cpp +++ b/src/Parsers/ASTBackupQuery.cpp @@ -11,31 +11,50 @@ namespace using Element = ASTBackupQuery::Element; using ElementType = ASTBackupQuery::ElementType; - void formatName(const DatabaseAndTableName & name, ElementType type, const IAST::FormatSettings & format) + void formatTypeWithName(const DatabaseAndTableName & name, bool name_is_in_temp_db, ElementType type, bool show_type, const IAST::FormatSettings & format) { switch (type) { - case ElementType::TABLE: [[fallthrough]]; - case ElementType::DICTIONARY: + case ElementType::TABLE: { + if (show_type) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : ""); + if (name_is_in_temp_db) + format.ostr << " TEMPORARY TABLE"; + else + format.ostr << " TABLE"; + format.ostr << (format.hilite ? IAST::hilite_none : ""); + } + format.ostr << " "; - if (!name.first.empty()) + if (!name_is_in_temp_db && !name.first.empty()) format.ostr << backQuoteIfNeed(name.first) << "."; format.ostr << backQuoteIfNeed(name.second); break; } case ElementType::DATABASE: { - format.ostr << " " << backQuoteIfNeed(name.first); + if (show_type) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : ""); + if (name_is_in_temp_db) + format.ostr << " ALL TEMPORARY TABLES"; + else + format.ostr << " DATABASE"; + format.ostr << (format.hilite ? IAST::hilite_none : ""); + } + + if (!name_is_in_temp_db) + format.ostr << " " << backQuoteIfNeed(name.first); break; } - case ElementType::TEMPORARY_TABLE: + case ElementType::ALL_DATABASES: { - format.ostr << " " << backQuoteIfNeed(name.second); + if (show_type) + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " ALL DATABASES" << (format.hilite ? IAST::hilite_none : ""); break; } - default: - break; } } @@ -55,32 +74,36 @@ namespace } } + void formatExceptList(const std::set & except_list, const IAST::FormatSettings & format) + { + if (except_list.empty()) + return; + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " EXCEPT " + << (format.hilite ? IAST::hilite_none : ""); + bool need_comma = false; + for (const auto & item : except_list) + { + if (std::exchange(need_comma, true)) + format.ostr << ","; + format.ostr << " " << backQuoteIfNeed(item); + } + } + void formatElement(const Element & element, Kind kind, const IAST::FormatSettings & format) { - format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " "; - switch (element.type) - { - case ElementType::TABLE: format.ostr << "TABLE"; break; - case ElementType::DICTIONARY: format.ostr << "DICTIONARY"; break; - case ElementType::DATABASE: format.ostr << "DATABASE"; break; - case ElementType::ALL_DATABASES: format.ostr << "ALL DATABASES"; break; - case ElementType::TEMPORARY_TABLE: format.ostr << "TEMPORARY TABLE"; break; - case ElementType::ALL_TEMPORARY_TABLES: format.ostr << "ALL TEMPORARY TABLES"; break; - case ElementType::EVERYTHING: format.ostr << "EVERYTHING"; break; - } - format.ostr << (format.hilite ? IAST::hilite_none : ""); + formatTypeWithName(element.name, element.name_is_in_temp_db, element.type, true, format); - formatName(element.name, element.type, format); + formatPartitions(element.partitions, format); + formatExceptList(element.except_list, format); - bool under_another_name = !element.new_name.first.empty() || !element.new_name.second.empty(); - if (under_another_name) + bool new_name_is_different = (element.new_name != element.name) || (element.new_name_is_in_temp_db != element.name_is_in_temp_db); + if (new_name_is_different) { format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " " << ((kind == Kind::BACKUP) ? "AS" : "INTO") << (format.hilite ? IAST::hilite_none : ""); - formatName(element.new_name, element.type, format); + bool show_type = (element.new_name_is_in_temp_db != element.name_is_in_temp_db); + formatTypeWithName(element.new_name, element.new_name_is_in_temp_db, element.type, show_type, format); } - - formatPartitions(element.partitions, format); } void formatElements(const std::vector & elements, Kind kind, const IAST::FormatSettings & format) diff --git a/src/Parsers/ASTBackupQuery.h b/src/Parsers/ASTBackupQuery.h index 0042fca558f..648bcf27bce 100644 --- a/src/Parsers/ASTBackupQuery.h +++ b/src/Parsers/ASTBackupQuery.h @@ -11,22 +11,20 @@ using DatabaseAndTableName = std::pair; /** BACKUP { TABLE [db.]table_name [AS [db.]table_name_in_backup] [PARTITION[S] partition_expr [,...]] | * DICTIONARY [db.]dictionary_name [AS [db.]dictionary_name_in_backup] | - * DATABASE database_name [AS database_name_in_backup] | - * ALL DATABASES | - * TEMPORARY TABLE table_name [AS table_name_in_backup] - * ALL TEMPORARY TABLES | - * EVERYTHING } [,...] + * TEMPORARY TABLE table_name [AS table_name_in_backup] | + * ALL TEMPORARY TABLES [EXCEPT ...] | + * DATABASE database_name [EXCEPT ...] [AS database_name_in_backup] | + * ALL DATABASES [EXCEPT ...] } [,...] * TO { File('path/') | * Disk('disk_name', 'path/') * [SETTINGS base_backup = {File(...) | Disk(...)}] * * RESTORE { TABLE [db.]table_name_in_backup [INTO [db.]table_name] [PARTITION[S] partition_expr [,...]] | * DICTIONARY [db.]dictionary_name_in_backup [INTO [db.]dictionary_name] | - * DATABASE database_name_in_backup [INTO database_name] | - * ALL DATABASES | * TEMPORARY TABLE table_name_in_backup [INTO table_name] | - * ALL TEMPORARY TABLES | - * EVERYTHING } [,...] + * ALL TEMPORARY TABLES [EXCEPT ...] | + * DATABASE database_name_in_backup [EXCEPT ...] [INTO database_name] | + * ALL DATABASES [EXCEPT ...] } [,...] * FROM {File(...) | Disk(...)} * * Notes: @@ -57,12 +55,8 @@ public: enum ElementType { TABLE, - DICTIONARY, DATABASE, ALL_DATABASES, - TEMPORARY_TABLE, - ALL_TEMPORARY_TABLES, - EVERYTHING, }; struct Element @@ -70,6 +64,8 @@ public: ElementType type; DatabaseAndTableName name; DatabaseAndTableName new_name; + bool name_is_in_temp_db = false; + bool new_name_is_in_temp_db = false; ASTs partitions; std::set except_list; }; diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index e61a0f55142..23881cd3fbb 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -198,8 +198,6 @@ ASTPtr ASTCreateQuery::clone() const res->set(res->storage, storage->clone()); if (select) res->set(res->select, select->clone()); - if (tables) - res->set(res->tables, tables->clone()); if (table_overrides) res->set(res->table_overrides, table_overrides->clone()); @@ -434,12 +432,6 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (comment ? ")" : ""); } - if (tables) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH " << (settings.hilite ? hilite_none : ""); - tables->formatImpl(settings, state, frame); - } - if (comment) { settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "COMMENT " << (settings.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index fcc4107bb5f..04755a02399 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -51,7 +51,7 @@ public: void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; - bool empty() + bool empty() const { return (!columns || columns->children.empty()) && (!indices || indices->children.empty()) && (!constraints || constraints->children.empty()) && (!projections || projections->children.empty()); @@ -73,7 +73,6 @@ public: bool replace_view{false}; /// CREATE OR REPLACE VIEW ASTColumns * columns_list = nullptr; - ASTExpressionList * tables = nullptr; StorageID to_table_id = StorageID::createEmpty(); /// For CREATE MATERIALIZED VIEW mv TO table. UUID to_inner_uuid = UUIDHelpers::Nil; /// For materialized view with inner table diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.h b/src/Parsers/ASTFunctionWithKeyValueArguments.h index 5820e8564ac..4b745e2c1a2 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.h +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.h @@ -19,7 +19,6 @@ public: /// Value is closed in brackets (HOST '127.0.0.1') bool second_with_brackets; -public: explicit ASTPair(bool second_with_brackets_) : second_with_brackets(second_with_brackets_) { @@ -54,7 +53,6 @@ public: { } -public: String getID(char delim) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTHelpers.h b/src/Parsers/ASTHelpers.h index 086b361bf85..0b3db8e02d5 100644 --- a/src/Parsers/ASTHelpers.h +++ b/src/Parsers/ASTHelpers.h @@ -6,7 +6,7 @@ namespace DB { -static inline bool isFunctionCast(const ASTFunction * function) +static inline bool isFunctionCast(const ASTFunction * function) /// NOLINT { if (function) return function->name == "CAST" || function->name == "_CAST"; diff --git a/src/Parsers/ASTProjectionSelectQuery.h b/src/Parsers/ASTProjectionSelectQuery.h index 71334c50868..d93c10b6e39 100644 --- a/src/Parsers/ASTProjectionSelectQuery.h +++ b/src/Parsers/ASTProjectionSelectQuery.h @@ -26,10 +26,10 @@ public: ASTPtr & refSelect() { return getExpression(Expression::SELECT); } - const ASTPtr with() const { return getExpression(Expression::WITH); } - const ASTPtr select() const { return getExpression(Expression::SELECT); } - const ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); } - const ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); } + ASTPtr with() const { return getExpression(Expression::WITH); } + ASTPtr select() const { return getExpression(Expression::SELECT); } + ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); } + ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); } /// Set/Reset/Remove expression. void setExpression(Expression expr, ASTPtr && ast); diff --git a/src/Parsers/ASTQueryWithOnCluster.h b/src/Parsers/ASTQueryWithOnCluster.h index b309ae5e847..c5daaa6ce37 100644 --- a/src/Parsers/ASTQueryWithOnCluster.h +++ b/src/Parsers/ASTQueryWithOnCluster.h @@ -17,7 +17,7 @@ public: /// new_database should be used by queries that refer to default db /// and default_database is specified for remote server - virtual ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database = {}) const = 0; + virtual ASTPtr getRewrittenASTWithoutOnCluster(const std::string & new_database = {}) const = 0; /// NOLINT /// Returns a query prepared for execution on remote server std::string getRewrittenQueryWithoutOnCluster(const std::string & new_database = {}) const; diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index 1c631783fdb..9a8f1dbd2e7 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -91,21 +91,21 @@ public: ASTPtr & refWhere() { return getExpression(Expression::WHERE); } ASTPtr & refHaving() { return getExpression(Expression::HAVING); } - const ASTPtr with() const { return getExpression(Expression::WITH); } - const ASTPtr select() const { return getExpression(Expression::SELECT); } - const ASTPtr tables() const { return getExpression(Expression::TABLES); } - const ASTPtr prewhere() const { return getExpression(Expression::PREWHERE); } - const ASTPtr where() const { return getExpression(Expression::WHERE); } - const ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); } - const ASTPtr having() const { return getExpression(Expression::HAVING); } - const ASTPtr window() const { return getExpression(Expression::WINDOW); } - const ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); } - const ASTPtr limitByOffset() const { return getExpression(Expression::LIMIT_BY_OFFSET); } - const ASTPtr limitByLength() const { return getExpression(Expression::LIMIT_BY_LENGTH); } - const ASTPtr limitBy() const { return getExpression(Expression::LIMIT_BY); } - const ASTPtr limitOffset() const { return getExpression(Expression::LIMIT_OFFSET); } - const ASTPtr limitLength() const { return getExpression(Expression::LIMIT_LENGTH); } - const ASTPtr settings() const { return getExpression(Expression::SETTINGS); } + ASTPtr with() const { return getExpression(Expression::WITH); } + ASTPtr select() const { return getExpression(Expression::SELECT); } + ASTPtr tables() const { return getExpression(Expression::TABLES); } + ASTPtr prewhere() const { return getExpression(Expression::PREWHERE); } + ASTPtr where() const { return getExpression(Expression::WHERE); } + ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); } + ASTPtr having() const { return getExpression(Expression::HAVING); } + ASTPtr window() const { return getExpression(Expression::WINDOW); } + ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); } + ASTPtr limitByOffset() const { return getExpression(Expression::LIMIT_BY_OFFSET); } + ASTPtr limitByLength() const { return getExpression(Expression::LIMIT_BY_LENGTH); } + ASTPtr limitBy() const { return getExpression(Expression::LIMIT_BY); } + ASTPtr limitOffset() const { return getExpression(Expression::LIMIT_OFFSET); } + ASTPtr limitLength() const { return getExpression(Expression::LIMIT_LENGTH); } + ASTPtr settings() const { return getExpression(Expression::SETTINGS); } bool hasFiltration() const { return where() || prewhere() || having(); } diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index b8056862bfc..a4b0a69faaa 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -134,6 +134,12 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, << (settings.hilite ? hilite_none : ""); }; + auto print_identifier = [&](const String & identifier) + { + settings.ostr << " " << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(identifier) + << (settings.hilite ? hilite_none : ""); + }; + if (!cluster.empty()) formatOnCluster(settings); @@ -161,9 +167,19 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, || type == Type::RESTORE_REPLICA || type == Type::SYNC_REPLICA || type == Type::FLUSH_DISTRIBUTED - || type == Type::RELOAD_DICTIONARY) + || type == Type::RELOAD_DICTIONARY + || type == Type::RELOAD_MODEL + || type == Type::RELOAD_FUNCTION + || type == Type::RESTART_DISK) { - print_database_table(); + if (table) + print_database_table(); + else if (!target_model.empty()) + print_identifier(target_model); + else if (!target_function.empty()) + print_identifier(target_function); + else if (!disk.empty()) + print_identifier(disk); } else if (type == Type::DROP_REPLICA) { diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp index 2d22c1b4307..90278e27c0c 100644 --- a/src/Parsers/ASTTTLElement.cpp +++ b/src/Parsers/ASTTTLElement.cpp @@ -1,13 +1,17 @@ - #include #include #include #include - +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + ASTPtr ASTTTLElement::clone() const { auto clone = std::make_shared(*this); @@ -29,13 +33,21 @@ ASTPtr ASTTTLElement::clone() const void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { ttl()->formatImpl(settings, state, frame); - if (mode == TTLMode::MOVE && destination_type == DataDestinationType::DISK) + if (mode == TTLMode::MOVE) { - settings.ostr << " TO DISK " << quoteString(destination_name); - } - else if (mode == TTLMode::MOVE && destination_type == DataDestinationType::VOLUME) - { - settings.ostr << " TO VOLUME " << quoteString(destination_name); + if (destination_type == DataDestinationType::DISK) + settings.ostr << " TO DISK "; + else if (destination_type == DataDestinationType::VOLUME) + settings.ostr << " TO VOLUME "; + else + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unsupported destination type {} for TTL MOVE", + magic_enum::enum_name(destination_type)); + + if (if_exists) + settings.ostr << "IF EXISTS "; + + settings.ostr << quoteString(destination_name); } else if (mode == TTLMode::GROUP_BY) { diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h index ce011d76c7b..9705cafbce3 100644 --- a/src/Parsers/ASTTTLElement.h +++ b/src/Parsers/ASTTTLElement.h @@ -16,16 +16,18 @@ public: TTLMode mode; DataDestinationType destination_type; String destination_name; + bool if_exists = false; ASTs group_by_key; ASTs group_by_assignments; ASTPtr recompression_codec; - ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_) + ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_, bool if_exists_) : mode(mode_) , destination_type(destination_type_) , destination_name(destination_name_) + , if_exists(if_exists_) , ttl_expr_pos(-1) , where_expr_pos(-1) { @@ -35,8 +37,8 @@ public: ASTPtr clone() const override; - const ASTPtr ttl() const { return getExpression(ttl_expr_pos); } - const ASTPtr where() const { return getExpression(where_expr_pos); } + ASTPtr ttl() const { return getExpression(ttl_expr_pos); } + ASTPtr where() const { return getExpression(where_expr_pos); } void setTTL(ASTPtr && ast) { setExpression(ttl_expr_pos, std::forward(ast)); } void setWhere(ASTPtr && ast) { setExpression(where_expr_pos, std::forward(ast)); } @@ -48,7 +50,6 @@ private: int ttl_expr_pos; int where_expr_pos; -private: void setExpression(int & pos, ASTPtr && ast); ASTPtr getExpression(int pos, bool clone = false) const; }; diff --git a/src/Parsers/ASTTableOverrides.cpp b/src/Parsers/ASTTableOverrides.cpp index 8fc21db218f..0f34a9fb247 100644 --- a/src/Parsers/ASTTableOverrides.cpp +++ b/src/Parsers/ASTTableOverrides.cpp @@ -93,7 +93,7 @@ ASTPtr ASTTableOverrideList::tryGetTableOverride(const String & name) const return children[it->second]; } -void ASTTableOverrideList::setTableOverride(const String & name, const ASTPtr ast) +void ASTTableOverrideList::setTableOverride(const String & name, ASTPtr ast) { auto it = positions.find(name); if (it == positions.end()) diff --git a/src/Parsers/ASTTableOverrides.h b/src/Parsers/ASTTableOverrides.h index c0603f7a8e0..c47260789d8 100644 --- a/src/Parsers/ASTTableOverrides.h +++ b/src/Parsers/ASTTableOverrides.h @@ -40,7 +40,7 @@ public: String getID(char) const override { return "TableOverrideList"; } ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; - void setTableOverride(const String & name, const ASTPtr ast); + void setTableOverride(const String & name, ASTPtr ast); void removeTableOverride(const String & name); ASTPtr tryGetTableOverride(const String & name) const; bool hasOverride(const String & name) const; diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp index 4680acc4c64..7435b22f7b7 100644 --- a/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/src/Parsers/ASTTablesInSelectQuery.cpp @@ -247,10 +247,12 @@ void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & sta void ASTArrayJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { + std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); frame.expression_list_prepend_whitespace = true; settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws + << indent_str << (kind == Kind::Left ? "LEFT " : "") << "ARRAY JOIN" << (settings.hilite ? hilite_none : ""); settings.one_line diff --git a/src/Parsers/ASTUseQuery.h b/src/Parsers/ASTUseQuery.h index 4e4a13c2a7f..16d449f905f 100644 --- a/src/Parsers/ASTUseQuery.h +++ b/src/Parsers/ASTUseQuery.h @@ -25,7 +25,6 @@ protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override { settings.ostr << (settings.hilite ? hilite_keyword : "") << "USE " << (settings.hilite ? hilite_none : "") << backQuoteIfNeed(database); - return; } }; diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 18030a5ed80..f8e1109886e 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -34,46 +34,58 @@ namespace } String auth_type_name = AuthenticationTypeInfo::get(auth_type).name; - String by_keyword = "BY"; - std::optional by_value; + String value_prefix; + std::optional value; + const boost::container::flat_set * values = nullptr; - if ( - show_password || + if (show_password || auth_type == AuthenticationType::LDAP || - auth_type == AuthenticationType::KERBEROS - ) + auth_type == AuthenticationType::KERBEROS || + auth_type == AuthenticationType::SSL_CERTIFICATE) { switch (auth_type) { case AuthenticationType::PLAINTEXT_PASSWORD: { - by_value = auth_data.getPassword(); + value_prefix = "BY"; + value = auth_data.getPassword(); break; } case AuthenticationType::SHA256_PASSWORD: { auth_type_name = "sha256_hash"; - by_value = auth_data.getPasswordHashHex(); + value_prefix = "BY"; + value = auth_data.getPasswordHashHex(); break; } case AuthenticationType::DOUBLE_SHA1_PASSWORD: { auth_type_name = "double_sha1_hash"; - by_value = auth_data.getPasswordHashHex(); + value_prefix = "BY"; + value = auth_data.getPasswordHashHex(); break; } case AuthenticationType::LDAP: { - by_keyword = "SERVER"; - by_value = auth_data.getLDAPServerName(); + value_prefix = "SERVER"; + value = auth_data.getLDAPServerName(); break; } case AuthenticationType::KERBEROS: { - by_keyword = "REALM"; const auto & realm = auth_data.getKerberosRealm(); if (!realm.empty()) - by_value = realm; + { + value_prefix = "REALM"; + value = realm; + } + break; + } + + case AuthenticationType::SSL_CERTIFICATE: + { + value_prefix = "CN"; + values = &auth_data.getSSLCertificateCommonNames(); break; } @@ -86,10 +98,26 @@ namespace settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << auth_type_name << (settings.hilite ? IAST::hilite_none : ""); - if (by_value) + if (!value_prefix.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << by_keyword << " " - << (settings.hilite ? IAST::hilite_none : "") << quoteString(*by_value); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << value_prefix + << (settings.hilite ? IAST::hilite_none : ""); + } + + if (value) + { + settings.ostr << " " << quoteString(*value); + } + else if (values) + { + settings.ostr << " "; + bool need_comma = false; + for (const auto & item : *values) + { + if (std::exchange(need_comma, true)) + settings.ostr << ", "; + settings.ostr << quoteString(item); + } } } diff --git a/src/Parsers/Access/ASTUserNameWithHost.h b/src/Parsers/Access/ASTUserNameWithHost.h index ada9bfb0673..bd28b42b48a 100644 --- a/src/Parsers/Access/ASTUserNameWithHost.h +++ b/src/Parsers/Access/ASTUserNameWithHost.h @@ -23,7 +23,7 @@ public: void concatParts(); ASTUserNameWithHost() = default; - ASTUserNameWithHost(const String & name_) : base_name(name_) {} + explicit ASTUserNameWithHost(const String & name_) : base_name(name_) {} String getID(char) const override { return "UserNameWithHost"; } ASTPtr clone() const override { return std::make_shared(*this); } void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; @@ -39,7 +39,7 @@ public: auto begin() const { return names.begin(); } auto end() const { return names.end(); } auto front() const { return *begin(); } - void push_back(const String & name_) { names.push_back(std::make_shared(name_)); } + void push_back(const String & name_) { names.push_back(std::make_shared(name_)); } /// NOLINT Strings toStrings() const; void concatParts(); diff --git a/src/Parsers/Access/ParserCreateRoleQuery.cpp b/src/Parsers/Access/ParserCreateRoleQuery.cpp index 314075cb7c0..da9749958ee 100644 --- a/src/Parsers/Access/ParserCreateRoleQuery.cpp +++ b/src/Parsers/Access/ParserCreateRoleQuery.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace DB @@ -37,7 +37,7 @@ namespace if (!elements_p.parse(pos, new_settings_ast, expected)) return false; - settings = std::move(new_settings_ast->as().elements); + settings = std::move(new_settings_ast->as().elements); return true; }); } @@ -102,7 +102,8 @@ bool ParserCreateRoleQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { if (!settings) settings = std::make_shared(); - boost::range::push_back(settings->elements, std::move(new_settings)); + + insertAtEnd(settings->elements, std::move(new_settings)); continue; } diff --git a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp index 731564a14c7..83156c6a8e1 100644 --- a/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/Access/ParserCreateRowPolicyQuery.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace DB @@ -264,7 +264,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & std::vector> new_filters; if (parseForClauses(pos, expected, alter, new_filters)) { - boost::range::push_back(filters, std::move(new_filters)); + insertAtEnd(filters, std::move(new_filters)); continue; } diff --git a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp index 8b5f2df2dd2..c58a3035dc6 100644 --- a/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/Access/ParserCreateSettingsProfileQuery.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace DB @@ -39,7 +39,7 @@ namespace if (!elements_p.parse(pos, new_settings_ast, expected)) return false; - settings = std::move(new_settings_ast->as().elements); + settings = std::move(new_settings_ast->as().elements); return true; }); } @@ -122,7 +122,8 @@ bool ParserCreateSettingsProfileQuery::parseImpl(Pos & pos, ASTPtr & node, Expec { if (!settings) settings = std::make_shared(); - boost::range::push_back(settings->elements, std::move(new_settings)); + + insertAtEnd(settings->elements, std::move(new_settings)); continue; } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index c5b8c9e37b3..da8e212fe2f 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include namespace DB @@ -52,6 +52,7 @@ namespace bool expect_hash = false; bool expect_ldap_server_name = false; bool expect_kerberos_realm = false; + bool expect_common_names = false; if (ParserKeyword{"WITH"}.ignore(pos, expected)) { @@ -65,6 +66,8 @@ namespace expect_ldap_server_name = true; else if (check_type == AuthenticationType::KERBEROS) expect_kerberos_realm = true; + else if (check_type == AuthenticationType::SSL_CERTIFICATE) + expect_common_names = true; else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; @@ -96,6 +99,7 @@ namespace } String value; + boost::container::flat_set common_names; if (expect_password || expect_hash) { ASTPtr ast; @@ -123,6 +127,18 @@ namespace value = ast->as().value.safeGet(); } } + else if (expect_common_names) + { + if (!ParserKeyword{"CN"}.ignore(pos, expected)) + return false; + + ASTPtr ast; + if (!ParserList{std::make_unique(), std::make_unique(TokenType::Comma), false}.parse(pos, ast, expected)) + return false; + + for (const auto & ast_child : ast->children) + common_names.insert(ast_child->as().value.safeGet()); + } auth_data = AuthenticationData{*type}; if (expect_password) @@ -133,6 +149,8 @@ namespace auth_data.setLDAPServerName(value); else if (expect_kerberos_realm) auth_data.setKerberosRealm(value); + else if (expect_common_names) + auth_data.setSSLCertificateCommonNames(std::move(common_names)); return true; }); @@ -232,7 +250,7 @@ namespace if (!parseHostsWithoutPrefix(pos, expected, res_hosts)) return false; - hosts.add(std::move(res_hosts)); + hosts.add(res_hosts); return true; }); } @@ -271,7 +289,7 @@ namespace if (!elements_p.parse(pos, new_settings_ast, expected)) return false; - settings = std::move(new_settings_ast->as().elements); + settings = std::move(new_settings_ast->as().elements); return true; }); } @@ -396,7 +414,8 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { if (!settings) settings = std::make_shared(); - boost::range::push_back(settings->elements, std::move(new_settings)); + + insertAtEnd(settings->elements, std::move(new_settings)); continue; } diff --git a/src/Parsers/Access/ParserGrantQuery.cpp b/src/Parsers/Access/ParserGrantQuery.cpp index 9f7e8535a14..43e1cedd34d 100644 --- a/src/Parsers/Access/ParserGrantQuery.cpp +++ b/src/Parsers/Access/ParserGrantQuery.cpp @@ -156,7 +156,7 @@ namespace } - void eraseNonGrantable(AccessRightsElements & elements) + void throwIfNotGrantable(AccessRightsElements & elements) { boost::range::remove_erase_if(elements, [](AccessRightsElement & element) { @@ -303,7 +303,12 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } if (!is_revoke) - eraseNonGrantable(elements); + { + if (attach_mode) + elements.eraseNonGrantable(); + else + throwIfNotGrantable(elements); + } auto query = std::make_shared(); node = query; diff --git a/src/Parsers/Access/ParserRowPolicyName.cpp b/src/Parsers/Access/ParserRowPolicyName.cpp index 7df4e5a36dc..cf5d2ab21b6 100644 --- a/src/Parsers/Access/ParserRowPolicyName.cpp +++ b/src/Parsers/Access/ParserRowPolicyName.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace DB @@ -179,7 +179,7 @@ bool ParserRowPolicyNames::parseImpl(Pos & pos, ASTPtr & node, Expected & expect return false; num_added_names_last_time = new_full_names.size(); - boost::range::push_back(full_names, std::move(new_full_names)); + insertAtEnd(full_names, std::move(new_full_names)); return true; }; diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 58fac2341cf..d2911754b24 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -33,7 +33,8 @@ class ParserToken : public IParserBase private: TokenType token_type; public: - ParserToken(TokenType token_type_) : token_type(token_type_) {} + ParserToken(TokenType token_type_) : token_type(token_type_) {} /// NOLINT + protected: const char * getName() const override { return "token"; } diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index e8efeb4b59c..5b6d8798fc1 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -86,6 +86,75 @@ inline void dumpAST(const IAST & ast, WriteBuffer & ostr, DumpASTNode * parent = dumpAST(*child, ostr, &dump); } +class DumpASTNodeInDotFormat +{ +public: + DumpASTNodeInDotFormat(const IAST & ast_, WriteBuffer * ostr_, bool root_ = true, const char * label_ = nullptr) + : ast(ast_), ostr(ostr_), root(root_), label(label_) + { + if (!ostr) + return; + + if (root) + (*ostr) << "digraph " << (label ? String(label) : "") << "{\n rankdir=\"UD\";\n"; + + printNode(); + } + + ~DumpASTNodeInDotFormat() + { + if (!ostr) + return; + + for (const auto & child : ast.children) + printEdge(ast, *child); + + if (root) + (*ostr) << "}\n"; + } + +private: + const IAST & ast; + WriteBuffer * ostr; + bool root; + const char * label; + + String getASTId() const { return ast.getID(' '); } + static String getNodeId(const IAST & a) { return "n" + std::to_string(reinterpret_cast(&a)); } + + void printNode() const + { + (*ostr) << " " << getNodeId(ast) << "[label=\""; + (*ostr) << getASTId(); + + String alias = ast.tryGetAlias(); + if (!alias.empty()) + (*ostr) << " (" + << "alias" + << " " << alias << ")"; + + if (!ast.children.empty()) + (*ostr) << " (children" + << " " << ast.children.size() << ")"; + (*ostr) << "\"];\n"; + } + + void printEdge(const IAST & parent, const IAST & child) const + { + (*ostr) << " " << getNodeId(parent) << " -> " << getNodeId(child) << ";\n"; + } +}; + + +/// Print AST in "dot" format for GraphViz +/// You can render it with: dot -Tpng ast.dot ast.png +inline void dumpASTInDotFormat(const IAST & ast, WriteBuffer & ostr, bool root = true) +{ + DumpASTNodeInDotFormat dump(ast, &ostr, root); + for (const auto & child : ast.children) + dumpASTInDotFormat(*child, ostr, false); +} + /// String stream dumped in dtor template diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 9c8f8b4e46b..c51201750c5 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -442,9 +442,9 @@ namespace pattern_list_args->children = { std::make_shared("^["), to_remove, - std::make_shared("]*|["), + std::make_shared("]+|["), to_remove, - std::make_shared("]*$") + std::make_shared("]+$") }; func_name = "replaceRegexpAll"; } @@ -455,7 +455,7 @@ namespace pattern_list_args->children = { std::make_shared("^["), to_remove, - std::make_shared("]*") + std::make_shared("]+") }; } else @@ -464,7 +464,7 @@ namespace pattern_list_args->children = { std::make_shared("["), to_remove, - std::make_shared("]*$") + std::make_shared("]+$") }; } func_name = "replaceRegexpOne"; @@ -2360,6 +2360,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_to_disk("TO DISK"); ParserKeyword s_to_volume("TO VOLUME"); + ParserKeyword s_if_exists("IF EXISTS"); ParserKeyword s_delete("DELETE"); ParserKeyword s_where("WHERE"); ParserKeyword s_group_by("GROUP BY"); @@ -2414,9 +2415,13 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr group_by_key; ASTPtr recompression_codec; ASTPtr group_by_assignments; + bool if_exists = false; if (mode == TTLMode::MOVE) { + if (s_if_exists.ignore(pos)) + if_exists = true; + ASTPtr ast_space_name; if (!parser_string_literal.parse(pos, ast_space_name, expected)) return false; @@ -2448,7 +2453,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } - auto ttl_element = std::make_shared(mode, destination_type, destination_name); + auto ttl_element = std::make_shared(mode, destination_type, destination_name, if_exists); ttl_element->setTTL(std::move(ttl_expr)); if (where_expr) ttl_element->setWhere(std::move(where_expr)); diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 358fe778f91..86d0fd0f861 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -207,7 +207,7 @@ private: ParserPtr elem_parser; public: - ParserCastExpression(ParserPtr && elem_parser_) + explicit ParserCastExpression(ParserPtr && elem_parser_) : elem_parser(std::move(elem_parser_)) { } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index fdf821c4a0b..bd8167c64fe 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -69,7 +69,7 @@ public: } /** Get the text that identifies this element. */ - virtual String getID(char delimiter = '_') const = 0; + virtual String getID(char delimiter = '_') const = 0; /// NOLINT ASTPtr ptr() { return shared_from_this(); } diff --git a/src/Parsers/ParserBackupQuery.cpp b/src/Parsers/ParserBackupQuery.cpp index 666600b58de..844a91fa515 100644 --- a/src/Parsers/ParserBackupQuery.cpp +++ b/src/Parsers/ParserBackupQuery.cpp @@ -18,38 +18,109 @@ namespace using Element = ASTBackupQuery::Element; using ElementType = ASTBackupQuery::ElementType; - bool parseName(IParser::Pos & pos, Expected & expected, ElementType type, DatabaseAndTableName & name) + bool parseType(IParser::Pos & pos, Expected & expected, ElementType & type, bool & name_is_in_temp_db) { + name_is_in_temp_db = false; + if (ParserKeyword{"TABLE"}.ignore(pos, expected) || ParserKeyword{"DICTIONARY"}.ignore(pos, expected)) + { + type = ElementType::TABLE; + return true; + } + if (ParserKeyword{"TEMPORARY TABLE"}.ignore(pos, expected)) + { + type = ElementType::TABLE; + name_is_in_temp_db = true; + return true; + } + if (ParserKeyword{"DATABASE"}.ignore(pos, expected)) + { + type = ElementType::DATABASE; + return true; + } + if (ParserKeyword{"ALL TEMPORARY TABLES"}.ignore(pos, expected)) + { + type = ElementType::DATABASE; + name_is_in_temp_db = true; + return true; + } + if (ParserKeyword{"ALL DATABASES"}.ignore(pos, expected)) + { + type = ElementType::ALL_DATABASES; + return true; + } + return false; + } + + bool parseTempDBFlag(IParser::Pos & pos, Expected & expected, ElementType type, bool & temp_db_flag) + { + temp_db_flag = false; switch (type) { - case ElementType::TABLE: [[fallthrough]]; - case ElementType::DICTIONARY: + case ElementType::TABLE: { + if (ParserKeyword{"TABLE"}.ignore(pos, expected) || ParserKeyword{"DICTIONARY"}.ignore(pos, expected)) + { + return true; + } + if (ParserKeyword{"TEMPORARY TABLE"}.ignore(pos, expected)) + { + temp_db_flag = true; + return true; + } + return false; + } + + case ElementType::DATABASE: + { + if (ParserKeyword{"DATABASE"}.ignore(pos, expected)) + { + return true; + } + if (ParserKeyword{"ALL TEMPORARY TABLES"}.ignore(pos, expected)) + { + temp_db_flag = true; + return true; + } + return false; + } + + default: + return false; + } + } + + bool parseName(IParser::Pos & pos, Expected & expected, ElementType type, bool name_is_in_temp_db, DatabaseAndTableName & name) + { + name.first.clear(); + name.second.clear(); + switch (type) + { + case ElementType::TABLE: + { + if (name_is_in_temp_db) + { + ASTPtr ast; + if (!ParserIdentifier{}.parse(pos, ast, expected)) + return false; + name.second = getIdentifierName(ast); + return true; + } return parseDatabaseAndTableName(pos, expected, name.first, name.second); } case ElementType::DATABASE: { + if (name_is_in_temp_db) + return false; ASTPtr ast; if (!ParserIdentifier{}.parse(pos, ast, expected)) return false; name.first = getIdentifierName(ast); - name.second.clear(); - return true; - } - - case ElementType::TEMPORARY_TABLE: - { - ASTPtr ast; - if (!ParserIdentifier{}.parse(pos, ast, expected)) - return false; - name.second = getIdentifierName(ast); - name.first.clear(); return true; } default: - return true; + return false; } } @@ -64,7 +135,7 @@ namespace ASTPtr ast; if (!ParserPartition{}.parse(pos, ast, expected)) return false; - result.emplace_back(ast); + result.push_back(ast); return true; }; if (!ParserList::parseUtil(pos, expected, parse_list_element, false)) @@ -74,50 +145,72 @@ namespace return true; } + bool parseExceptList(IParser::Pos & pos, Expected & expected, std::set & except_list) + { + if (!ParserKeyword{"EXCEPT"}.ignore(pos, expected)) + return false; + + std::set result; + auto parse_list_element = [&] + { + ASTPtr ast; + if (!ParserIdentifier{}.parse(pos, ast, expected)) + return false; + result.insert(getIdentifierName(ast)); + return true; + }; + if (!ParserList::parseUtil(pos, expected, parse_list_element, false)) + return false; + + except_list = std::move(result); + return true; + } + bool parseElement(IParser::Pos & pos, Expected & expected, Element & entry) { return IParserBase::wrapParseImpl(pos, [&] { ElementType type; - if (ParserKeyword{"TABLE"}.ignore(pos, expected)) - type = ElementType::TABLE; - else if (ParserKeyword{"DICTIONARY"}.ignore(pos, expected)) - type = ElementType::DICTIONARY; - else if (ParserKeyword{"DATABASE"}.ignore(pos, expected)) - type = ElementType::DATABASE; - else if (ParserKeyword{"ALL DATABASES"}.ignore(pos, expected)) - type = ElementType::ALL_DATABASES; - else if (ParserKeyword{"TEMPORARY TABLE"}.ignore(pos, expected)) - type = ElementType::TEMPORARY_TABLE; - else if (ParserKeyword{"ALL TEMPORARY TABLES"}.ignore(pos, expected)) - type = ElementType::ALL_TEMPORARY_TABLES; - else if (ParserKeyword{"EVERYTHING"}.ignore(pos, expected)) - type = ElementType::EVERYTHING; - else + bool name_is_in_temp_db = false; + if (!parseType(pos, expected, type, name_is_in_temp_db)) return false; DatabaseAndTableName name; - if (!parseName(pos, expected, type, name)) - return false; + if ((type == ElementType::TABLE) || (type == ElementType::DATABASE && !name_is_in_temp_db)) + { + if (!parseName(pos, expected, type, name_is_in_temp_db, name)) + return false; + } + + bool new_name_is_in_temp_db = name_is_in_temp_db; + DatabaseAndTableName new_name = name; + if (ParserKeyword{"AS"}.ignore(pos, expected) || ParserKeyword{"INTO"}.ignore(pos, expected)) + { + if (!parseTempDBFlag(pos, expected, type, new_name_is_in_temp_db)) + new_name_is_in_temp_db = name_is_in_temp_db; + + if ((type == ElementType::TABLE) || (type == ElementType::DATABASE && !new_name_is_in_temp_db)) + { + if (!parseName(pos, expected, type, new_name_is_in_temp_db, new_name)) + new_name = name; + } + } ASTs partitions; if (type == ElementType::TABLE) parsePartitions(pos, expected, partitions); - DatabaseAndTableName new_name; - if (ParserKeyword{"AS"}.ignore(pos, expected) || ParserKeyword{"INTO"}.ignore(pos, expected)) - { - if (!parseName(pos, expected, type, new_name)) - return false; - } - - if ((type == ElementType::TABLE) && partitions.empty()) - parsePartitions(pos, expected, partitions); + std::set except_list; + if (type != ElementType::TABLE) + parseExceptList(pos, expected, except_list); entry.type = type; entry.name = std::move(name); entry.new_name = std::move(new_name); + entry.name_is_in_temp_db = name_is_in_temp_db; + entry.new_name_is_in_temp_db = new_name_is_in_temp_db; entry.partitions = std::move(partitions); + entry.except_list = std::move(except_list); return true; }); } diff --git a/src/Parsers/ParserBackupQuery.h b/src/Parsers/ParserBackupQuery.h index e42326c2590..b01c149601c 100644 --- a/src/Parsers/ParserBackupQuery.h +++ b/src/Parsers/ParserBackupQuery.h @@ -8,22 +8,20 @@ namespace DB /** Parses queries like * BACKUP { TABLE [db.]table_name [AS [db.]table_name_in_backup] [PARTITION[S] partition_expr [,...]] | * DICTIONARY [db.]dictionary_name [AS [db.]dictionary_name_in_backup] | - * DATABASE database_name [AS database_name_in_backup] | - * ALL DATABASES | * TEMPORARY TABLE table_name [AS table_name_in_backup] - * ALL TEMPORARY TABLES | - * EVERYTHING } [,...] + * ALL TEMPORARY TABLES [EXCEPT ...] | + * DATABASE database_name [AS database_name_in_backup] [EXCEPT ...] | + * ALL DATABASES [EXCEPT ...] } [,...] * TO { File('path/') | * Disk('disk_name', 'path/') * [SETTINGS base_backup = {FILE(...) | DISK(...)}] * * RESTORE { TABLE [db.]table_name_in_backup [INTO [db.]table_name] [PARTITION[S] partition_expr [,...]] | * DICTIONARY [db.]dictionary_name_in_backup [INTO [db.]dictionary_name] | - * DATABASE database_name_in_backup [INTO database_name] | - * ALL DATABASES | * TEMPORARY TABLE table_name_in_backup [INTO table_name] | - * ALL TEMPORARY TABLES | - * EVERYTHING } [,...] + * ALL TEMPORARY TABLES [EXCEPT ...] | + * DATABASE database_name_in_backup [EXCEPT ...] [INTO database_name] | + * ALL DATABASES [EXCEPT ...] } [,...] * FROM {File(...) | Disk(...)} */ class ParserBackupQuery : public IParserBase diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index c48cea9c480..a4dbe635664 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -124,6 +124,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E ParserKeyword s_null{"NULL"}; ParserKeyword s_not{"NOT"}; ParserKeyword s_materialized{"MATERIALIZED"}; + ParserKeyword s_ephemeral{"EPHEMERAL"}; ParserKeyword s_alias{"ALIAS"}; ParserKeyword s_comment{"COMMENT"}; ParserKeyword s_codec{"CODEC"}; @@ -171,6 +172,7 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E if (!s_default.checkWithoutMoving(pos, expected) && !s_materialized.checkWithoutMoving(pos, expected) + && !s_ephemeral.checkWithoutMoving(pos, expected) && !s_alias.checkWithoutMoving(pos, expected) && (require_type || (!s_comment.checkWithoutMoving(pos, expected) @@ -183,7 +185,8 @@ bool IParserColumnDeclaration::parseImpl(Pos & pos, ASTPtr & node, E } Pos pos_before_specifier = pos; - if (s_default.ignore(pos, expected) || s_materialized.ignore(pos, expected) || s_alias.ignore(pos, expected)) + if (s_default.ignore(pos, expected) || s_materialized.ignore(pos, expected) || + s_ephemeral.ignore(pos, expected) || s_alias.ignore(pos, expected)) { default_specifier = Poco::toUpper(std::string{pos_before_specifier->begin, pos_before_specifier->end}); diff --git a/src/Parsers/ParserExplainQuery.h b/src/Parsers/ParserExplainQuery.h index a1865e30239..ba30e97a58f 100644 --- a/src/Parsers/ParserExplainQuery.h +++ b/src/Parsers/ParserExplainQuery.h @@ -14,7 +14,7 @@ protected: const char * getName() const override { return "EXPLAIN"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; public: - ParserExplainQuery(const char* end_) : end(end_) {} + explicit ParserExplainQuery(const char* end_) : end(end_) {} }; } diff --git a/src/Parsers/ParserQueryWithOutput.h b/src/Parsers/ParserQueryWithOutput.h index 854d5a74ffd..1fd7bec1eea 100644 --- a/src/Parsers/ParserQueryWithOutput.h +++ b/src/Parsers/ParserQueryWithOutput.h @@ -15,7 +15,7 @@ protected: const char * getName() const override { return "Query with output"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; public: - ParserQueryWithOutput(const char * end_) : end(end_) {} + explicit ParserQueryWithOutput(const char * end_) : end(end_) {} }; } diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 915a01cc063..61e96b9c1de 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -64,6 +64,78 @@ static bool parseQueryWithOnClusterAndMaybeTable(std::shared_ptr return true; } +enum class SystemQueryTargetType +{ + Model, + Function, + Disk +}; + +static bool parseQueryWithOnClusterAndTarget(std::shared_ptr & res, IParser::Pos & pos, Expected & expected, SystemQueryTargetType target_type) +{ + /// Better form for user: SYSTEM target_name ON CLUSTER cluster + /// Query rewritten form + form while executing on cluster: SYSTEM ON CLUSTER cluster target_name + /// Need to support both + + String cluster; + bool parsed_on_cluster = false; + + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) + return false; + parsed_on_cluster = true; + } + + String target; + ASTPtr temporary_string_literal; + + if (ParserStringLiteral{}.parse(pos, temporary_string_literal, expected)) + { + target = temporary_string_literal->as().value.safeGet(); + } + else + { + ParserIdentifier identifier_parser; + ASTPtr identifier; + + if (!identifier_parser.parse(pos, identifier, expected)) + return false; + + if (!tryGetIdentifierNameInto(identifier, target)) + return false; + } + + if (!parsed_on_cluster && ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) + return false; + } + + res->cluster = cluster; + + switch (target_type) + { + case SystemQueryTargetType::Model: + { + res->target_model = std::move(target); + break; + } + case SystemQueryTargetType::Function: + { + res->target_function = std::move(target); + break; + } + case SystemQueryTargetType::Disk: + { + res->disk = std::move(target); + break; + } + } + + return true; +} + static bool parseQueryWithOnCluster(std::shared_ptr & res, IParser::Pos & pos, Expected & expected) { @@ -112,50 +184,14 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::RELOAD_MODEL: { - parseQueryWithOnCluster(res, pos, expected); - - ASTPtr ast; - if (ParserStringLiteral{}.parse(pos, ast, expected)) - { - res->target_model = ast->as().value.safeGet(); - } - else - { - ParserIdentifier model_parser; - ASTPtr model; - String target_model; - - if (!model_parser.parse(pos, model, expected)) - return false; - - if (!tryGetIdentifierNameInto(model, res->target_model)) - return false; - } - + if (!parseQueryWithOnClusterAndTarget(res, pos, expected, SystemQueryTargetType::Model)) + return false; break; } case Type::RELOAD_FUNCTION: { - parseQueryWithOnCluster(res, pos, expected); - - ASTPtr ast; - if (ParserStringLiteral{}.parse(pos, ast, expected)) - { - res->target_function = ast->as().value.safeGet(); - } - else - { - ParserIdentifier function_parser; - ASTPtr function; - String target_function; - - if (!function_parser.parse(pos, function, expected)) - return false; - - if (!tryGetIdentifierNameInto(function, res->target_function)) - return false; - } - + if (!parseQueryWithOnClusterAndTarget(res, pos, expected, SystemQueryTargetType::Function)) + return false; break; } case Type::DROP_REPLICA: @@ -211,14 +247,8 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & case Type::RESTART_DISK: { - parseQueryWithOnCluster(res, pos, expected); - - ASTPtr ast; - if (ParserIdentifier{}.parse(pos, ast, expected)) - res->disk = ast->as().name(); - else + if (!parseQueryWithOnClusterAndTarget(res, pos, expected, SystemQueryTargetType::Disk)) return false; - break; } diff --git a/src/Parsers/ParserTablesInSelectQuery.h b/src/Parsers/ParserTablesInSelectQuery.h index 9e5b591ccbe..772f1992f4d 100644 --- a/src/Parsers/ParserTablesInSelectQuery.h +++ b/src/Parsers/ParserTablesInSelectQuery.h @@ -21,7 +21,7 @@ protected: class ParserTablesInSelectQueryElement : public IParserBase { public: - ParserTablesInSelectQueryElement(bool is_first_) : is_first(is_first_) {} + explicit ParserTablesInSelectQueryElement(bool is_first_) : is_first(is_first_) {} protected: const char * getName() const override { return "table, table function, subquery or list of joined tables"; } diff --git a/src/Parsers/fuzzers/codegen_fuzzer/gen.py b/src/Parsers/fuzzers/codegen_fuzzer/gen.py index 95936247489..84ee09916c4 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/gen.py +++ b/src/Parsers/fuzzers/codegen_fuzzer/gen.py @@ -7,16 +7,14 @@ import string TOKEN_TEXT = 1 TOKEN_VAR = 2 -TOKEN_COLON = ':' -TOKEN_SEMI = ';' -TOKEN_OR = '|' -TOKEN_QUESTIONMARK = '?' -TOKEN_ROUND_BRACKET_OPEN = '(' -TOKEN_ROUND_BRACKET_CLOSE = ')' -TOKEN_ASTERISK = '*' -TOKEN_SLASH = '/' - - +TOKEN_COLON = ":" +TOKEN_SEMI = ";" +TOKEN_OR = "|" +TOKEN_QUESTIONMARK = "?" +TOKEN_ROUND_BRACKET_OPEN = "(" +TOKEN_ROUND_BRACKET_CLOSE = ")" +TOKEN_ASTERISK = "*" +TOKEN_SLASH = "/" class TextValue: @@ -27,9 +25,9 @@ class TextValue: def get_slug(self): if self.slug is not None: return self.slug - slug = '' + slug = "" for c in self.t: - slug += c if c in string.ascii_letters else '_' + slug += c if c in string.ascii_letters else "_" self.slug = slug return slug @@ -37,12 +35,12 @@ class TextValue: return f"TextValue_{self.get_slug()}" def __repr__(self): - return f"TextValue(\"{self.t}\")" + return f'TextValue("{self.t}")' class Var: def __init__(self, id_): - self.id_ = id_ + self.id_ = id_ def __repr__(self): return f"Var({self.id_})" @@ -59,8 +57,8 @@ class Parser: self.cur_tok = None self.includes = [] - self.proto = '' - self.cpp = '' + self.proto = "" + self.cpp = "" def parse_file(self, filename): with open(filename) as f: @@ -81,7 +79,7 @@ class Parser: if self.text[0] == '"': return self.parse_txt_value() - if self.text[0] == '$': + if self.text[0] == "$": return self.parse_var_value() c, self.text = self.text[0], self.text[1:] @@ -89,9 +87,9 @@ class Parser: return c def parse_var_value(self): - i = self.text.find(' ') + i = self.text.find(" ") - id_, self.text = self.text[1:i], self.text[i+1:] + id_, self.text = self.text[1:i], self.text[i + 1 :] self.var_id = int(id_) self.cur_tok = TOKEN_VAR return TOKEN_VAR @@ -100,12 +98,12 @@ class Parser: if self.text[0] != '"': raise Exception("parse_txt_value: expected quote at the start") - self.t = '' + self.t = "" self.text = self.text[1:] while self.text[0] != '"': - if self.text[0] == '\\': - if self.text[1] == 'x': + if self.text[0] == "\\": + if self.text[1] == "x": self.t += self.text[:4] self.text = self.text[4:] elif self.text[1] in 'nt\\"': @@ -123,7 +121,7 @@ class Parser: def skip_ws(self): while self.text and self.text[0] in string.whitespace: - if self.text[0] == '\n': + if self.text[0] == "\n": self.line += 1 self.col = 0 self.text = self.text[1:] @@ -134,10 +132,9 @@ class Parser: def skip_line(self): self.line += 1 - index = self.text.find('\n') + index = self.text.find("\n") self.text = self.text[index:] - def parse_statement(self): if self.skip_ws() is None: return None @@ -164,52 +161,54 @@ class Parser: def generate(self): self.proto = 'syntax = "proto3";\n\n' - self.cpp = '#include \n#include \n#include \n\n#include \n\n' + self.cpp = "#include \n#include \n#include \n\n#include \n\n" for incl_file in self.includes: self.cpp += f'#include "{incl_file}"\n' - self.cpp += '\n' + self.cpp += "\n" - self.proto += 'message Word {\n' - self.proto += '\tenum Value {\n' + self.proto += "message Word {\n" + self.proto += "\tenum Value {\n" - self.cpp += 'void GenerateWord(const Word&, std::string&, int);\n\n' + self.cpp += "void GenerateWord(const Word&, std::string&, int);\n\n" - self.cpp += 'void GenerateSentence(const Sentence& stc, std::string &s, int depth) {\n' - self.cpp += '\tfor (int i = 0; i < stc.words_size(); i++ ) {\n' - self.cpp += '\t\tGenerateWord(stc.words(i), s, ++depth);\n' - self.cpp += '\t}\n' - self.cpp += '}\n' + self.cpp += ( + "void GenerateSentence(const Sentence& stc, std::string &s, int depth) {\n" + ) + self.cpp += "\tfor (int i = 0; i < stc.words_size(); i++ ) {\n" + self.cpp += "\t\tGenerateWord(stc.words(i), s, ++depth);\n" + self.cpp += "\t}\n" + self.cpp += "}\n" - self.cpp += 'void GenerateWord(const Word& word, std::string &s, int depth) {\n' + self.cpp += "void GenerateWord(const Word& word, std::string &s, int depth) {\n" - self.cpp += '\tif (depth > 5) return;\n\n' - self.cpp += '\tswitch (word.value()) {\n' + self.cpp += "\tif (depth > 5) return;\n\n" + self.cpp += "\tswitch (word.value()) {\n" for idx, chain in enumerate(self.chains): - self.proto += f'\t\tvalue_{idx} = {idx};\n' + self.proto += f"\t\tvalue_{idx} = {idx};\n" - self.cpp += f'\t\tcase {idx}: {{\n' + self.cpp += f"\t\tcase {idx}: {{\n" num_var = 0 for item in chain: if isinstance(item, TextValue): self.cpp += f'\t\t\ts += "{item.t}";\n' elif isinstance(item, Var): - self.cpp += f'\t\t\tif (word.inner().words_size() > {num_var})\t\t\t\tGenerateWord(word.inner().words({num_var}), s, ++depth);\n' + self.cpp += f"\t\t\tif (word.inner().words_size() > {num_var})\t\t\t\tGenerateWord(word.inner().words({num_var}), s, ++depth);\n" num_var += 1 else: raise Exception("unknown token met during generation") - self.cpp += '\t\t\tbreak;\n\t\t}\n' - self.cpp += '\t\tdefault: break;\n' + self.cpp += "\t\t\tbreak;\n\t\t}\n" + self.cpp += "\t\tdefault: break;\n" - self.cpp += '\t}\n' + self.cpp += "\t}\n" - self.proto += '\t}\n' - self.proto += '\tValue value = 1;\n' - self.proto += '\tSentence inner = 2;\n' - self.proto += '}\nmessage Sentence {\n\trepeated Word words = 1;\n}' + self.proto += "\t}\n" + self.proto += "\tValue value = 1;\n" + self.proto += "\tSentence inner = 2;\n" + self.proto += "}\nmessage Sentence {\n\trepeated Word words = 1;\n}" - self.cpp += '}\n' + self.cpp += "}\n" return self.cpp, self.proto def fatal_parsing_error(self, msg): @@ -220,7 +219,7 @@ class Parser: def main(args): input_file, outfile_cpp, outfile_proto = args - if not outfile_proto.endswith('.proto'): + if not outfile_proto.endswith(".proto"): raise Exception("outfile_proto (argv[3]) should end with `.proto`") include_filename = outfile_proto[:-6] + ".pb.h" @@ -231,17 +230,17 @@ def main(args): cpp, proto = p.generate() - proto = proto.replace('\t', ' ' * 4) - cpp = cpp.replace('\t', ' ' * 4) + proto = proto.replace("\t", " " * 4) + cpp = cpp.replace("\t", " " * 4) - with open(outfile_cpp, 'w') as f: + with open(outfile_cpp, "w") as f: f.write(cpp) - with open(outfile_proto, 'w') as f: + with open(outfile_proto, "w") as f: f.write(proto) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) < 3: print(f"Usage {sys.argv[0]} ") sys.exit(1) diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index c0b57d9b1f5..a4eb5404552 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -26,20 +26,20 @@ namespace const std::unordered_set keywords { - "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", - "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", - "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", - "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", - "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", - "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", - "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", - "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", - "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", - "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", - "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", - "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", - "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "DICTIONARY", "OFFSET", - "TRIM", "LTRIM", "RTRIM", "BOTH", "LEADING", "TRAILING" + "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "EPHEMERAL", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", + "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", + "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", + "INTO", "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", + "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", + "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", + "INNER", "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", + "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", + "ASC", "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", + "ROLE", "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", + "REPLACE", "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", + "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "DICTIONARY", "OFFSET", "TRIM", + "LTRIM", "RTRIM", "BOTH", "LEADING", "TRAILING" }; const std::unordered_set keep_words diff --git a/src/Parsers/parseIntervalKind.cpp b/src/Parsers/parseIntervalKind.cpp index 7d36133e81c..0704aa107ca 100644 --- a/src/Parsers/parseIntervalKind.cpp +++ b/src/Parsers/parseIntervalKind.cpp @@ -7,6 +7,27 @@ namespace DB { bool parseIntervalKind(IParser::Pos & pos, Expected & expected, IntervalKind & result) { + if (ParserKeyword("NANOSECOND").ignore(pos, expected) || ParserKeyword("SQL_TSI_NANOSECOND").ignore(pos, expected) + || ParserKeyword("NS").ignore(pos, expected)) + { + result = IntervalKind::Nanosecond; + return true; + } + + if (ParserKeyword("MICROSECOND").ignore(pos, expected) || ParserKeyword("SQL_TSI_MICROSECOND").ignore(pos, expected) + || ParserKeyword("MCS").ignore(pos, expected)) + { + result = IntervalKind::Microsecond; + return true; + } + + if (ParserKeyword("MILLISECOND").ignore(pos, expected) || ParserKeyword("SQL_TSI_MILLISECOND").ignore(pos, expected) + || ParserKeyword("MS").ignore(pos, expected)) + { + result = IntervalKind::Millisecond; + return true; + } + if (ParserKeyword("SECOND").ignore(pos, expected) || ParserKeyword("SQL_TSI_SECOND").ignore(pos, expected) || ParserKeyword("SS").ignore(pos, expected) || ParserKeyword("S").ignore(pos, expected)) { diff --git a/src/Processors/Chunk.h b/src/Processors/Chunk.h index e70ba57a267..1c9240ba114 100644 --- a/src/Processors/Chunk.h +++ b/src/Processors/Chunk.h @@ -90,7 +90,7 @@ public: bool hasRows() const { return num_rows > 0; } bool hasColumns() const { return !columns.empty(); } bool empty() const { return !hasRows() && !hasColumns(); } - operator bool() const { return !empty(); } + operator bool() const { return !empty(); } /// NOLINT void addColumn(ColumnPtr column); void addColumn(size_t position, ColumnPtr column); diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp index 198d5ce5d8d..0f091e73743 100644 --- a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp @@ -35,7 +35,7 @@ struct PullingAsyncPipelineExecutor::Data if (has_exception) { has_exception = false; - std::rethrow_exception(std::move(exception)); + std::rethrow_exception(exception); } } }; diff --git a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp index 6c2e62b77dc..07cdb554aba 100644 --- a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp @@ -90,7 +90,7 @@ struct PushingAsyncPipelineExecutor::Data if (has_exception) { has_exception = false; - std::rethrow_exception(std::move(exception)); + std::rethrow_exception(exception); } } }; diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index c4857326e6e..645100dad19 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -1,4 +1,5 @@ #include +#include #include // toString #include @@ -199,6 +200,7 @@ Chunk IRowInputFormat::generate() return {}; } + finalizeObjectColumns(columns); Chunk chunk(std::move(columns), num_rows); //chunk.setChunkInfo(std::move(chunk_missing_values)); return chunk; diff --git a/src/Processors/Formats/ISchemaReader.h b/src/Processors/Formats/ISchemaReader.h index 67a8eb88d61..2d35809e26a 100644 --- a/src/Processors/Formats/ISchemaReader.h +++ b/src/Processors/Formats/ISchemaReader.h @@ -14,7 +14,7 @@ namespace DB class ISchemaReader { public: - ISchemaReader(ReadBuffer & in_) : in(in_) {} + explicit ISchemaReader(ReadBuffer & in_) : in(in_) {} virtual NamesAndTypesList readSchema() = 0; diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index 558ba9bdd65..cf5cfa681a1 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -114,7 +114,7 @@ static std::shared_ptr createFileReader(ReadB if (is_stopped) return nullptr; - auto file_reader_status = arrow::ipc::RecordBatchFileReader::Open(std::move(arrow_file)); + auto file_reader_status = arrow::ipc::RecordBatchFileReader::Open(arrow_file); if (!file_reader_status.ok()) throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, "Error while opening a table: {}", file_reader_status.status().ToString()); diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.h b/src/Processors/Formats/Impl/ArrowBufferedStreams.h index d649c52557f..e06eab04f1b 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.h +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.h @@ -44,7 +44,7 @@ class RandomAccessFileFromSeekableReadBuffer : public arrow::io::RandomAccessFil public: RandomAccessFileFromSeekableReadBuffer(SeekableReadBuffer & in_, off_t file_size_); - RandomAccessFileFromSeekableReadBuffer(SeekableReadBufferWithSize & in_); + explicit RandomAccessFileFromSeekableReadBuffer(SeekableReadBufferWithSize & in_); arrow::Result GetSize() override; diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index c09181ed74f..14c81a0d90d 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,6 @@ #include #include - /// UINT16 and UINT32 are processed separately, see comments in readColumnFromArrowColumn. #define FOR_ARROW_NUMERIC_TYPES(M) \ M(arrow::Type::UINT8, DB::UInt8) \ @@ -65,9 +65,9 @@ namespace ErrorCodes extern const int DUPLICATE_COLUMN; extern const int THERE_IS_NO_COLUMN; extern const int UNKNOWN_EXCEPTION; + extern const int INCORRECT_NUMBER_OF_COLUMNS; } - /// Inserts numeric data right into internal column data to reduce an overhead template > static ColumnWithTypeAndName readColumnWithNumericData(std::shared_ptr & arrow_column, const String & column_name) @@ -206,38 +206,19 @@ static ColumnWithTypeAndName readColumnWithDate64Data(std::shared_ptr & arrow_column, const String & column_name) { - auto internal_type = std::make_shared(); + const auto & arrow_type = static_cast(*(arrow_column->type())); + const UInt8 scale = arrow_type.unit() * 3; + auto internal_type = std::make_shared(scale, arrow_type.timezone()); auto internal_column = internal_type->createColumn(); - auto & column_data = assert_cast &>(*internal_column).getData(); + auto & column_data = assert_cast &>(*internal_column).getData(); column_data.reserve(arrow_column->length()); for (size_t chunk_i = 0, num_chunks = static_cast(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i) { - auto & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); - const auto & type = static_cast(*chunk.type()); - - UInt32 divide = 1; - const auto unit = type.unit(); - switch (unit) - { - case arrow::TimeUnit::SECOND: - divide = 1; - break; - case arrow::TimeUnit::MILLI: - divide = 1000; - break; - case arrow::TimeUnit::MICRO: - divide = 1000000; - break; - case arrow::TimeUnit::NANO: - divide = 1000000000; - break; - } - + const auto & chunk = dynamic_cast(*(arrow_column->chunk(chunk_i))); for (size_t value_i = 0, length = static_cast(chunk.length()); value_i < length; ++value_i) { - auto timestamp = static_cast(chunk.Value(value_i) / divide); // ms! TODO: check other 's' 'ns' ... - column_data.emplace_back(timestamp); + column_data.emplace_back(chunk.Value(value_i)); } } return {std::move(internal_column), std::move(internal_type), column_name}; @@ -259,7 +240,7 @@ static ColumnWithTypeAndName readColumnWithDecimalDataImpl(std::shared_ptr(chunk.Value(value_i))); // TODO: copy column } } - return {std::move(internal_column), std::move(internal_type), column_name}; + return {std::move(internal_column), internal_type, column_name}; } template @@ -355,7 +336,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( auto nested_column = readColumnFromArrowColumn(arrow_column, column_name, format_name, true, dictionary_values, read_ints_as_dates); auto nullmap_column = readByteMapFromArrowColumn(arrow_column); auto nullable_type = std::make_shared(std::move(nested_column.type)); - auto nullable_column = ColumnNullable::create(std::move(nested_column.column), std::move(nullmap_column)); + auto nullable_column = ColumnNullable::create(nested_column.column, nullmap_column); return {std::move(nullable_column), std::move(nullable_type), column_name}; } @@ -402,7 +383,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( const auto * tuple_column = assert_cast(nested_column.column.get()); const auto * tuple_type = assert_cast(nested_column.type.get()); - auto map_column = ColumnMap::create(std::move(tuple_column->getColumnPtr(0)), std::move(tuple_column->getColumnPtr(1)), std::move(offsets_column)); + auto map_column = ColumnMap::create(tuple_column->getColumnPtr(0), tuple_column->getColumnPtr(1), offsets_column); auto map_type = std::make_shared(tuple_type->getElements()[0], tuple_type->getElements()[1]); return {std::move(map_column), std::move(map_type), column_name}; } @@ -411,7 +392,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( auto arrow_nested_column = getNestedArrowColumn(arrow_column); auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_values, read_ints_as_dates); auto offsets_column = readOffsetsFromArrowListColumn(arrow_column); - auto array_column = ColumnArray::create(std::move(nested_column.column), std::move(offsets_column)); + auto array_column = ColumnArray::create(nested_column.column, offsets_column); auto array_type = std::make_shared(nested_column.type); return {std::move(array_column), std::move(array_type), column_name}; } @@ -476,7 +457,7 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( auto arrow_indexes_column = std::make_shared(indexes_array); auto indexes_column = readColumnWithIndexesData(arrow_indexes_column); - auto lc_column = ColumnLowCardinality::create(dict_values->column, std::move(indexes_column)); + auto lc_column = ColumnLowCardinality::create(dict_values->column, indexes_column); auto lc_type = std::make_shared(dict_values->type); return {std::move(lc_column), std::move(lc_type), column_name}; } @@ -485,7 +466,6 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( return readColumnWithNumericData(arrow_column, column_name); FOR_ARROW_NUMERIC_TYPES(DISPATCH) # undef DISPATCH - // TODO: support TIMESTAMP_MICROS and TIMESTAMP_MILLIS with truncated micro- and milliseconds? // TODO: read JSON as a string? // TODO: read UUID as a string? default: @@ -504,11 +484,17 @@ static void checkStatus(const arrow::Status & status, const String & column_name throw Exception{ErrorCodes::UNKNOWN_EXCEPTION, "Error with a {} column '{}': {}.", format_name, column_name, status.ToString()}; } -Block ArrowColumnToCHColumn::arrowSchemaToCHHeader(const arrow::Schema & schema, const std::string & format_name) +Block ArrowColumnToCHColumn::arrowSchemaToCHHeader(const arrow::Schema & schema, const std::string & format_name, const Block * hint_header) { ColumnsWithTypeAndName sample_columns; + std::unordered_set nested_table_names; + if (hint_header) + nested_table_names = Nested::getAllTableNames(*hint_header); for (const auto & field : schema.fields()) { + if (hint_header && !hint_header->has(field->name()) && !nested_table_names.contains(field->name())) + continue; + /// Create empty arrow column by it's type and convert it to ClickHouse column. arrow::MemoryPool* pool = arrow::default_memory_pool(); std::unique_ptr array_builder; @@ -551,6 +537,9 @@ void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptrsecond->length(); columns_list.reserve(header.rows()); @@ -620,7 +609,7 @@ void ArrowColumnToCHColumn::arrowColumnsToCHChunk(Chunk & res, NameToColumnPtr & std::vector ArrowColumnToCHColumn::getMissingColumns(const arrow::Schema & schema) const { std::vector missing_columns; - auto block_from_arrow = arrowSchemaToCHHeader(schema, format_name); + auto block_from_arrow = arrowSchemaToCHHeader(schema, format_name, &header); auto flatten_block_from_arrow = Nested::flatten(block_from_arrow); for (size_t i = 0, columns = header.columns(); i < columns; ++i) { diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index 07e7fb36404..cf4f6bb3ff3 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -34,7 +34,9 @@ public: /// Get missing columns that exists in header but not in arrow::Schema std::vector getMissingColumns(const arrow::Schema & schema) const; - static Block arrowSchemaToCHHeader(const arrow::Schema & schema, const std::string & format_name); + /// Transform arrow schema to ClickHouse header. If hint_header is provided, + /// we will skip columns in schema that are not in hint_header. + static Block arrowSchemaToCHHeader(const arrow::Schema & schema, const std::string & format_name, const Block * hint_header = nullptr); private: const Block & header; diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index 1e8ee4aebb9..7a598de1f6a 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -61,7 +61,7 @@ private: , target_column_idx(target_column_idx_) , deserialize_fn(deserialize_fn_) {} - Action(SkipFn skip_fn_) + explicit Action(SkipFn skip_fn_) : type(Skip) , skip_fn(skip_fn_) {} diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp index bb202a3e177..6918220feb4 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -15,9 +15,9 @@ namespace ErrorCodes BinaryRowInputFormat::BinaryRowInputFormat(ReadBuffer & in_, Block header, Params params_, bool with_names_, bool with_types_, const FormatSettings & format_settings_) : RowInputFormatWithNamesAndTypes( - std::move(header), + header, in_, - std::move(params_), + params_, with_names_, with_types_, format_settings_, diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 014715ba9ee..043e4f1e724 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -2,6 +2,7 @@ #if USE_ARROW || USE_PARQUET +// #include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +57,7 @@ namespace DB extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_TYPE; extern const int LOGICAL_ERROR; + extern const int DECIMAL_OVERFLOW; } static const std::initializer_list>> internal_type_to_arrow_type = @@ -120,6 +123,42 @@ namespace DB checkStatus(status, write_column->getName(), format_name); } + static void fillArrowArrayWithDateTime64ColumnData( + const DataTypeDateTime64 * type, + ColumnPtr write_column, + const PaddedPODArray * null_bytemap, + const String & format_name, + arrow::ArrayBuilder* array_builder, + size_t start, + size_t end) + { + const auto & column = assert_cast &>(*write_column); + arrow::TimestampBuilder & builder = assert_cast(*array_builder); + arrow::Status status; + + auto scale = type->getScale(); + bool need_rescale = scale % 3; + auto rescale_multiplier = DecimalUtils::scaleMultiplier(3 - scale % 3); + for (size_t value_i = start; value_i < end; ++value_i) + { + if (null_bytemap && (*null_bytemap)[value_i]) + { + status = builder.AppendNull(); + } + else + { + auto value = static_cast(column[value_i].get>().getValue()); + if (need_rescale) + { + if (common::mulOverflow(value, rescale_multiplier, value)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + } + status = builder.Append(value); + } + checkStatus(status, write_column->getName(), format_name); + } + } + static void fillArrowArray( const String & column_name, ColumnPtr & column, @@ -480,6 +519,11 @@ namespace DB if (!callOnIndexAndDataType(column_type->getTypeId(), fill_decimal)) throw Exception{ErrorCodes::LOGICAL_ERROR, "Cannot fill arrow array with decimal data with type {}", column_type_name}; } + else if (isDateTime64(column_type)) + { + const auto * datetime64_type = assert_cast(column_type.get()); + fillArrowArrayWithDateTime64ColumnData(datetime64_type, column, null_bytemap, format_name, array_builder, start, end); + } #define DISPATCH(CPP_NUMERIC_TYPE, ARROW_BUILDER_TYPE) \ else if (#CPP_NUMERIC_TYPE == column_type_name) \ { \ @@ -546,6 +590,18 @@ namespace DB } } + static arrow::TimeUnit::type getArrowTimeUnit(const DataTypeDateTime64 * type) + { + UInt32 scale = type->getScale(); + if (scale == 0) + return arrow::TimeUnit::SECOND; + if (scale > 0 && scale <= 3) + return arrow::TimeUnit::MILLI; + if (scale > 3 && scale <= 6) + return arrow::TimeUnit::MICRO; + return arrow::TimeUnit::NANO; + } + static std::shared_ptr getArrowType( DataTypePtr column_type, ColumnPtr column, const std::string & column_name, const std::string & format_name, bool * out_is_column_nullable) { @@ -602,7 +658,7 @@ namespace DB auto nested_arrow_type = getArrowType(nested_types[i], tuple_column->getColumnPtr(i), name, format_name, out_is_column_nullable); nested_fields.push_back(std::make_shared(name, nested_arrow_type, *out_is_column_nullable)); } - return arrow::struct_(std::move(nested_fields)); + return arrow::struct_(nested_fields); } if (column_type->lowCardinality()) @@ -638,6 +694,12 @@ namespace DB return arrow_type_it->second; } + if (isDateTime64(column_type)) + { + const auto * datetime64_type = assert_cast(column_type.get()); + return arrow::timestamp(getArrowTimeUnit(datetime64_type), datetime64_type->getTimeZone().getTimeZone()); + } + throw Exception(ErrorCodes::UNKNOWN_TYPE, "The type '{}' of a column '{}' is not supported for conversion into {} data format.", column_type->getName(), column_name, format_name); diff --git a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp index 58f88c5c7cf..fd33abfb587 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp @@ -169,7 +169,7 @@ static std::optional convertToDynamicValue( auto value_builder = initStructFieldBuilder(nested_column, row_num, struct_builder, value_field); auto value = convertToDynamicValue(nested_column, nullable_type->getNestedType(), row_num, value_builder, enum_comparing_mode, temporary_text_data_storage); if (value) - struct_builder.set(value_field, std::move(*value)); + struct_builder.set(value_field, *value); } } else @@ -184,7 +184,7 @@ static std::optional convertToDynamicValue( = initStructFieldBuilder(nested_columns[pos], row_num, struct_builder, nested_struct_schema.getFieldByName(name)); auto value = convertToDynamicValue(nested_columns[pos], nested_types[pos], row_num, field_builder, enum_comparing_mode, temporary_text_data_storage); if (value) - struct_builder.set(name, std::move(*value)); + struct_builder.set(name, *value); } } return std::nullopt; @@ -215,7 +215,7 @@ static std::optional convertToDynamicValue( auto value = convertToDynamicValue(nested_column, nested_type, offset + i, value_builder, enum_comparing_mode, temporary_text_data_storage); if (value) - list_builder.set(i, std::move(*value)); + list_builder.set(i, *value); } return std::nullopt; } diff --git a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h index 288b36508ce..12dc5eda2b3 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h @@ -15,7 +15,7 @@ namespace DB class CapnProtoOutputStream : public kj::OutputStream { public: - CapnProtoOutputStream(WriteBuffer & out_); + explicit CapnProtoOutputStream(WriteBuffer & out_); void write(const void * buffer, size_t size) override; diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.h b/src/Processors/Formats/Impl/ConstantExpressionTemplate.h index 6659243df63..c5d4f033258 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.h +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace DB { diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp index 56ba975dea1..914ec27fc46 100644 --- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp @@ -14,30 +14,25 @@ namespace ErrorCodes extern const int INCORRECT_DATA; } -JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) - : JSONAsStringRowInputFormat(header_, std::make_unique(in_), params_) {} +JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) + : JSONAsRowInputFormat(header_, std::make_unique(in_), params_) {} -JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_) : +JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_) : IRowInputFormat(header_, *buf_, std::move(params_)), buf(std::move(buf_)) { if (header_.columns() > 1) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "This input format is only suitable for tables with a single column of type String but the number of columns is {}", + "This input format is only suitable for tables with a single column of type String or Object, but the number of columns is {}", header_.columns()); - - if (!isString(removeNullable(removeLowCardinality(header_.getByPosition(0).type)))) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "This input format is only suitable for tables with a single column of type String but the column type is {}", - header_.getByPosition(0).type->getName()); } -void JSONAsStringRowInputFormat::resetParser() +void JSONAsRowInputFormat::resetParser() { IRowInputFormat::resetParser(); buf->reset(); } -void JSONAsStringRowInputFormat::readPrefix() +void JSONAsRowInputFormat::readPrefix() { /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. skipBOMIfExists(*buf); @@ -50,7 +45,7 @@ void JSONAsStringRowInputFormat::readPrefix() } } -void JSONAsStringRowInputFormat::readSuffix() +void JSONAsRowInputFormat::readSuffix() { skipWhitespaceIfAny(*buf); if (data_in_square_brackets) @@ -66,6 +61,57 @@ void JSONAsStringRowInputFormat::readSuffix() assertEOF(*buf); } +bool JSONAsRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + assert(columns.size() == 1); + assert(serializations.size() == 1); + + if (!allow_new_rows) + return false; + + skipWhitespaceIfAny(*buf); + if (!buf->eof()) + { + if (!data_in_square_brackets && *buf->position() == ';') + { + /// ';' means the end of query, but it cannot be before ']'. + return allow_new_rows = false; + } + else if (data_in_square_brackets && *buf->position() == ']') + { + /// ']' means the end of query. + return allow_new_rows = false; + } + } + + if (!buf->eof()) + readJSONObject(*columns[0]); + + skipWhitespaceIfAny(*buf); + if (!buf->eof() && *buf->position() == ',') + ++buf->position(); + skipWhitespaceIfAny(*buf); + + return !buf->eof(); +} + +void JSONAsRowInputFormat::setReadBuffer(ReadBuffer & in_) +{ + buf = std::make_unique(in_); + IInputFormat::setReadBuffer(*buf); +} + + +JSONAsStringRowInputFormat::JSONAsStringRowInputFormat( + const Block & header_, ReadBuffer & in_, Params params_) + : JSONAsRowInputFormat(header_, in_, params_) +{ + if (!isString(removeNullable(removeLowCardinality(header_.getByPosition(0).type)))) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "This input format is only suitable for tables with a single column of type String but the column type is {}", + header_.getByPosition(0).type->getName()); +} + void JSONAsStringRowInputFormat::readJSONObject(IColumn & column) { PeekableReadBufferCheckpoint checkpoint{*buf}; @@ -143,41 +189,21 @@ void JSONAsStringRowInputFormat::readJSONObject(IColumn & column) buf->position() = end; } -bool JSONAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) + +JSONAsObjectRowInputFormat::JSONAsObjectRowInputFormat( + const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) + : JSONAsRowInputFormat(header_, in_, params_) + , format_settings(format_settings_) { - if (!allow_new_rows) - return false; - - skipWhitespaceIfAny(*buf); - if (!buf->eof()) - { - if (!data_in_square_brackets && *buf->position() == ';') - { - /// ';' means the end of query, but it cannot be before ']'. - return allow_new_rows = false; - } - else if (data_in_square_brackets && *buf->position() == ']') - { - /// ']' means the end of query. - return allow_new_rows = false; - } - } - - if (!buf->eof()) - readJSONObject(*columns[0]); - - skipWhitespaceIfAny(*buf); - if (!buf->eof() && *buf->position() == ',') - ++buf->position(); - skipWhitespaceIfAny(*buf); - - return !buf->eof(); + if (!isObject(header_.getByPosition(0).type)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Input format JSONAsObject is only suitable for tables with a single column of type Object but the column type is {}", + header_.getByPosition(0).type->getName()); } -void JSONAsStringRowInputFormat::setReadBuffer(ReadBuffer & in_) +void JSONAsObjectRowInputFormat::readJSONObject(IColumn & column) { - buf = std::make_unique(in_); - IInputFormat::setReadBuffer(*buf); + serializations[0]->deserializeTextJSON(column, *buf, format_settings); } void registerInputFormatJSONAsString(FormatFactory & factory) @@ -202,6 +228,23 @@ void registerNonTrivialPrefixAndSuffixCheckerJSONAsString(FormatFactory & factor factory.registerNonTrivialPrefixAndSuffixChecker("JSONAsString", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl); } +void registerInputFormatJSONAsObject(FormatFactory & factory) +{ + factory.registerInputFormat("JSONAsObject", []( + ReadBuffer & buf, + const Block & sample, + IRowInputFormat::Params params, + const FormatSettings & settings) + { + return std::make_shared(sample, buf, std::move(params), settings); + }); +} + +void registerNonTrivialPrefixAndSuffixCheckerJSONAsObject(FormatFactory & factory) +{ + factory.registerNonTrivialPrefixAndSuffixChecker("JSONAsObject", nonTrivialPrefixAndSuffixCheckerJSONEachRowImpl); +} + void registerJSONAsStringSchemaReader(FormatFactory & factory) { factory.registerExternalSchemaReader("JSONAsString", [](const FormatSettings &) diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h index 9979a5d1474..f7880eac867 100644 --- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h @@ -12,35 +12,58 @@ namespace DB class ReadBuffer; /// This format parses a sequence of JSON objects separated by newlines, spaces and/or comma. -/// Each JSON object is parsed as a whole to string. -/// This format can only parse a table with single field of type String. - -class JSONAsStringRowInputFormat final : public IRowInputFormat +class JSONAsRowInputFormat : public IRowInputFormat { public: - JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); + JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); - String getName() const override { return "JSONAsStringRowInputFormat"; } void resetParser() override; void setReadBuffer(ReadBuffer & in_) override; private: - JSONAsStringRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_); + JSONAsRowInputFormat(const Block & header_, std::unique_ptr buf_, Params params_); bool readRow(MutableColumns & columns, RowReadExtension & ext) override; void readPrefix() override; void readSuffix() override; - void readJSONObject(IColumn & column); - +protected: + virtual void readJSONObject(IColumn & column) = 0; std::unique_ptr buf; +private: /// This flag is needed to know if data is in square brackets. bool data_in_square_brackets = false; bool allow_new_rows = true; }; +/// Each JSON object is parsed as a whole to string. +/// This format can only parse a table with single field of type String. +class JSONAsStringRowInputFormat final : public JSONAsRowInputFormat +{ +public: + JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); + String getName() const override { return "JSONAsStringRowInputFormat"; } + +private: + void readJSONObject(IColumn & column) override; +}; + + +/// Each JSON object is parsed as a whole to object. +/// This format can only parse a table with single field of type Object. +class JSONAsObjectRowInputFormat final : public JSONAsRowInputFormat +{ +public: + JSONAsObjectRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_); + String getName() const override { return "JSONAsObjectRowInputFormat"; } + +private: + void readJSONObject(IColumn & column) override; + const FormatSettings format_settings; +}; + class JSONAsStringExternalSchemaReader : public IExternalSchemaReader { public: diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index dcab55743cb..c087749d8d8 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -27,7 +27,7 @@ JSONCompactEachRowRowInputFormat::JSONCompactEachRowRowInputFormat( : RowInputFormatWithNamesAndTypes( header_, in_, - std::move(params_), + params_, with_names_, with_types_, format_settings_, diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp index c4645e0d63d..4a2c4209acf 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp @@ -55,9 +55,9 @@ void JSONCompactEachRowRowOutputFormat::writeRowEndDelimiter() void JSONCompactEachRowRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) { writeChar('\n', out); - size_t num_columns = columns.size(); + size_t columns_size = columns.size(); writeRowStartDelimiter(); - for (size_t i = 0; i < num_columns; ++i) + for (size_t i = 0; i < columns_size; ++i) { if (i != 0) writeFieldDelimiter(); diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index 8130b2b4cb1..61ac25ca441 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -154,9 +154,9 @@ void JSONRowOutputFormat::writeBeforeTotals() void JSONRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) { - size_t num_columns = columns.size(); + size_t columns_size = columns.size(); - for (size_t i = 0; i < num_columns; ++i) + for (size_t i = 0; i < columns_size; ++i) { if (i != 0) writeTotalsFieldDelimiter(); diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 56fc5d7857b..607e6f36767 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -353,7 +353,7 @@ bool MsgPackVisitor::visit_nil() bool MsgPackVisitor::visit_ext(const char * value, uint32_t size) { int8_t type = *value; - if (*value == int8_t(MsgPackExtensionTypes::UUID)) + if (*value == int8_t(MsgPackExtensionTypes::UUIDType)) { insertUUID(info_stack.top().column, info_stack.top().type, value + 1, size - 1); return true; @@ -496,11 +496,12 @@ DataTypePtr MsgPackSchemaReader::getDataType(const msgpack::object & object) case msgpack::type::object_type::EXT: { msgpack::object_ext object_ext = object.via.ext; - if (object_ext.type() == int8_t(MsgPackExtensionTypes::UUID)) + if (object_ext.type() == int8_t(MsgPackExtensionTypes::UUIDType)) return std::make_shared(); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Msgpack extension type {%x} is not supported", object_ext.type()); } } + __builtin_unreachable(); } DataTypes MsgPackSchemaReader::readRowAndGetDataTypes() diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp index edec9774b5f..e53aafb4e56 100644 --- a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp @@ -199,7 +199,7 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr writeBinaryBigEndian(value.toUnderType().items[0], buf); writeBinaryBigEndian(value.toUnderType().items[1], buf); StringRef uuid_ext = buf.stringRef(); - packer.pack_ext(sizeof(UUID), int8_t(MsgPackExtensionTypes::UUID)); + packer.pack_ext(sizeof(UUID), int8_t(MsgPackExtensionTypes::UUIDType)); packer.pack_ext_body(uuid_ext.data, uuid_ext.size); return; } @@ -213,8 +213,8 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr void MsgPackRowOutputFormat::write(const Columns & columns, size_t row_num) { - size_t num_columns = columns.size(); - for (size_t i = 0; i < num_columns; ++i) + size_t columns_size = columns.size(); + for (size_t i = 0; i < columns_size; ++i) { serializeField(*columns[i], types[i], row_num); } diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 61511d634d3..aa9f7874ae8 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -1,4 +1,5 @@ #include "ORCBlockInputFormat.h" +#include #if USE_ORC #include @@ -52,6 +53,9 @@ Chunk ORCBlockInputFormat::generate() if (!table || !table->num_rows()) return res; + if (format_settings.use_lowercase_column_name) + table = *table->RenameColumns(include_column_names); + arrow_column_to_ch_column->arrowTableToCHChunk(res, table); /// If defaults_for_omitted_fields is true, calculate the default values from default expression for omitted fields. /// Otherwise fill the missing columns with zero values of its type. @@ -69,6 +73,7 @@ void ORCBlockInputFormat::resetParser() file_reader.reset(); include_indices.clear(); + include_column_names.clear(); block_missing_values.clear(); } @@ -111,7 +116,7 @@ static void getFileReaderAndSchema( if (is_stopped) return; - auto result = arrow::adapters::orc::ORCFileReader::Open(std::move(arrow_file), arrow::default_memory_pool()); + auto result = arrow::adapters::orc::ORCFileReader::Open(arrow_file, arrow::default_memory_pool()); if (!result.ok()) throw Exception(result.status().ToString(), ErrorCodes::BAD_ARGUMENTS); file_reader = std::move(result).ValueOrDie(); @@ -120,6 +125,20 @@ static void getFileReaderAndSchema( if (!read_schema_result.ok()) throw Exception(read_schema_result.status().ToString(), ErrorCodes::BAD_ARGUMENTS); schema = std::move(read_schema_result).ValueOrDie(); + + if (format_settings.use_lowercase_column_name) + { + std::vector> fields; + fields.reserve(schema->num_fields()); + for (int i = 0; i < schema->num_fields(); ++i) + { + const auto& field = schema->field(i); + auto name = field->name(); + boost::to_lower(name); + fields.push_back(field->WithName(name)); + } + schema = arrow::schema(fields, schema->metadata()); + } } void ORCBlockInputFormat::prepareReader() @@ -148,9 +167,11 @@ void ORCBlockInputFormat::prepareReader() const auto & name = schema->field(i)->name(); if (getPort().getHeader().has(name) || nested_table_names.contains(name)) { - column_names.push_back(name); for (int j = 0; j != indexes_count; ++j) + { include_indices.push_back(index + j); + include_column_names.push_back(name); + } } index += indexes_count; } diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index bb136d02d6e..bd2151d78ff 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -45,10 +45,9 @@ private: std::unique_ptr arrow_column_to_ch_column; - std::vector column_names; - // indices of columns to read from ORC file std::vector include_indices; + std::vector include_column_names; std::vector missing_columns; BlockMissingValues block_missing_values; diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index 2ffee597e8f..f69fd1c0aab 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -17,7 +17,7 @@ class WriteBuffer; class ORCOutputStream : public orc::OutputStream { public: - ORCOutputStream(WriteBuffer & out_); + explicit ORCOutputStream(WriteBuffer & out_); uint64_t getLength() const override; uint64_t getNaturalWriteSize() const override; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 3f0d9980573..548bf0138f5 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -1,4 +1,6 @@ #include "ParquetBlockInputFormat.h" +#include + #if USE_PARQUET #include @@ -13,9 +15,6 @@ #include "ArrowColumnToCHColumn.h" #include -#include - - namespace DB { @@ -57,6 +56,9 @@ Chunk ParquetBlockInputFormat::generate() throw ParsingException{"Error while reading Parquet data: " + read_status.ToString(), ErrorCodes::CANNOT_READ_ALL_DATA}; + if (format_settings.use_lowercase_column_name) + table = *table->RenameColumns(column_names); + ++row_group_current; arrow_column_to_ch_column->arrowTableToCHChunk(res, table); @@ -76,6 +78,7 @@ void ParquetBlockInputFormat::resetParser() file_reader.reset(); column_indices.clear(); + column_names.clear(); row_group_current = 0; block_missing_values.clear(); } @@ -120,6 +123,20 @@ static void getFileReaderAndSchema( return; THROW_ARROW_NOT_OK(parquet::arrow::OpenFile(std::move(arrow_file), arrow::default_memory_pool(), &file_reader)); THROW_ARROW_NOT_OK(file_reader->GetSchema(&schema)); + + if (format_settings.use_lowercase_column_name) + { + std::vector> fields; + fields.reserve(schema->num_fields()); + for (int i = 0; i < schema->num_fields(); ++i) + { + const auto& field = schema->field(i); + auto name = field->name(); + boost::to_lower(name); + fields.push_back(field->WithName(name)); + } + schema = arrow::schema(fields, schema->metadata()); + } } void ParquetBlockInputFormat::prepareReader() @@ -150,7 +167,10 @@ void ParquetBlockInputFormat::prepareReader() if (getPort().getHeader().has(name) || nested_table_names.contains(name)) { for (int j = 0; j != indexes_count; ++j) + { column_indices.push_back(index + j); + column_names.push_back(name); + } } index += indexes_count; } diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index 1faadaa3d21..eba9aac29f2 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -40,6 +40,7 @@ private: int row_group_total = 0; // indices of columns to read from Parquet file std::vector column_indices; + std::vector column_names; std::unique_ptr arrow_column_to_ch_column; int row_group_current = 0; std::vector missing_columns; diff --git a/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp b/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp new file mode 100644 index 00000000000..42b329fd879 --- /dev/null +++ b/src/Processors/Formats/Impl/ProtobufListInputFormat.cpp @@ -0,0 +1,97 @@ +#include "ProtobufListInputFormat.h" + +#if USE_PROTOBUF +# include +# include +# include +# include +# include + +namespace DB +{ + +ProtobufListInputFormat::ProtobufListInputFormat(ReadBuffer & in_, const Block & header_, const Params & params_, const FormatSchemaInfo & schema_info_) + : IRowInputFormat(header_, in_, params_) + , reader(std::make_unique(in_)) + , serializer(ProtobufSerializer::create( + header_.getNames(), + header_.getDataTypes(), + missing_column_indices, + *ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info_, ProtobufSchemas::WithEnvelope::Yes), + /* with_length_delimiter = */ true, + /* with_envelope = */ true, + *reader)) +{ +} + +bool ProtobufListInputFormat::readRow(MutableColumns & columns, RowReadExtension & row_read_extension) +{ + if (reader->eof()) + { + reader->endMessage(/*ignore_errors =*/ false); + return false; + } + + size_t row_num = columns.empty() ? 0 : columns[0]->size(); + if (!row_num) + serializer->setColumns(columns.data(), columns.size()); + + serializer->readRow(row_num); + + row_read_extension.read_columns.clear(); + row_read_extension.read_columns.resize(columns.size(), true); + for (size_t column_idx : missing_column_indices) + row_read_extension.read_columns[column_idx] = false; + return true; +} + +ProtobufListSchemaReader::ProtobufListSchemaReader(const FormatSettings & format_settings) + : schema_info( + format_settings.schema.format_schema, + "Protobuf", + true, + format_settings.schema.is_server, + format_settings.schema.format_schema_path) +{ +} + +NamesAndTypesList ProtobufListSchemaReader::readSchema() +{ + const auto * message_descriptor = ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info, ProtobufSchemas::WithEnvelope::Yes); + return protobufSchemaToCHSchema(message_descriptor); +} + +void registerInputFormatProtobufList(FormatFactory & factory) +{ + factory.registerInputFormat( + "ProtobufList", + [](ReadBuffer &buf, + const Block & sample, + RowInputFormatParams params, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, std::move(params), FormatSchemaInfo(settings, "Protobuf", true)); + }); + factory.markFormatAsColumnOriented("ProtobufList"); +} + +void registerProtobufListSchemaReader(FormatFactory & factory) +{ + factory.registerExternalSchemaReader("ProtobufList", [](const FormatSettings & settings) + { + return std::make_shared(settings); + }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerInputFormatProtobufList(FormatFactory &) {} +void registerProtobufListSchemaReader(FormatFactory &) {} +} + +#endif diff --git a/src/Processors/Formats/Impl/ProtobufListInputFormat.h b/src/Processors/Formats/Impl/ProtobufListInputFormat.h new file mode 100644 index 00000000000..cecda3acda0 --- /dev/null +++ b/src/Processors/Formats/Impl/ProtobufListInputFormat.h @@ -0,0 +1,52 @@ +#pragma once + +#include "config_formats.h" + +#if USE_PROTOBUF +# include +# include +# include + +namespace DB +{ +class Block; +class ProtobufReader; +class ProtobufSerializer; +class ReadBuffer; + +/** Stream designed to deserialize data from the google protobuf format. + * One nested Protobuf message is parsed as one row of data. + * + * Parsing of the protobuf format requires the 'format_schema' setting to be set, e.g. + * INSERT INTO table FORMAT Protobuf SETTINGS format_schema = 'schema:Message' + * where schema is the name of "schema.proto" file specifying protobuf schema. + */ +class ProtobufListInputFormat final : public IRowInputFormat +{ +public: + ProtobufListInputFormat(ReadBuffer & in_, const Block & header_, const Params & params_, const FormatSchemaInfo & schema_info_); + + String getName() const override { return "ProtobufListInputFormat"; } + +private: + bool readRow(MutableColumns & columns, RowReadExtension & row_read_extension) override; + + std::unique_ptr reader; + std::vector missing_column_indices; + std::unique_ptr serializer; +}; + +class ProtobufListSchemaReader : public IExternalSchemaReader +{ +public: + explicit ProtobufListSchemaReader(const FormatSettings & format_settings); + + NamesAndTypesList readSchema() override; + +private: + const FormatSchemaInfo schema_info; +}; + +} + +#endif diff --git a/src/Processors/Formats/Impl/ProtobufListOutputFormat.cpp b/src/Processors/Formats/Impl/ProtobufListOutputFormat.cpp new file mode 100644 index 00000000000..e78c9e3b13b --- /dev/null +++ b/src/Processors/Formats/Impl/ProtobufListOutputFormat.cpp @@ -0,0 +1,68 @@ +#include "ProtobufListOutputFormat.h" + +#if USE_PROTOBUF +# include +# include +# include +# include +# include + +namespace DB +{ + +ProtobufListOutputFormat::ProtobufListOutputFormat( + WriteBuffer & out_, + const Block & header_, + const RowOutputFormatParams & params_, + const FormatSchemaInfo & schema_info_) + : IRowOutputFormat(header_, out_, params_) + , writer(std::make_unique(out)) + , serializer(ProtobufSerializer::create( + header_.getNames(), + header_.getDataTypes(), + *ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info_, ProtobufSchemas::WithEnvelope::Yes), + /* with_length_delimiter = */ true, + /* with_envelope = */ true, + *writer)) +{ +} + +void ProtobufListOutputFormat::write(const Columns & columns, size_t row_num) +{ + if (row_num == 0) + serializer->setColumns(columns.data(), columns.size()); + + serializer->writeRow(row_num); +} + +void ProtobufListOutputFormat::finalizeImpl() +{ + serializer->finalizeWrite(); +} + +void registerOutputFormatProtobufList(FormatFactory & factory) +{ + factory.registerOutputFormat( + "ProtobufList", + [](WriteBuffer & buf, + const Block & header, + const RowOutputFormatParams & params, + const FormatSettings & settings) + { + return std::make_shared( + buf, header, params, + FormatSchemaInfo(settings, "Protobuf", true)); + }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerOutputFormatProtobufList(FormatFactory &) {} +} + +#endif diff --git a/src/Processors/Formats/Impl/ProtobufListOutputFormat.h b/src/Processors/Formats/Impl/ProtobufListOutputFormat.h new file mode 100644 index 00000000000..47fe618d2e1 --- /dev/null +++ b/src/Processors/Formats/Impl/ProtobufListOutputFormat.h @@ -0,0 +1,48 @@ +#pragma once + +#include "config_formats.h" + +#if USE_PROTOBUF +# include + +namespace DB +{ +class FormatSchemaInfo; +class ProtobufWriter; +class ProtobufSerializer; + +/** Stream designed to serialize data in the google protobuf format. + * Each row is written as a separated nested message, and all rows are enclosed by a single + * top-level, envelope message + * + * Serializing in the protobuf format requires the 'format_schema' setting to be set, e.g. + * SELECT * from table FORMAT Protobuf SETTINGS format_schema = 'schema:Message' + * where schema is the name of "schema.proto" file specifying protobuf schema. + */ +// class ProtobufListOutputFormat final : public IOutputFormat +class ProtobufListOutputFormat final : public IRowOutputFormat +{ +public: + ProtobufListOutputFormat( + WriteBuffer & out_, + const Block & header_, + const RowOutputFormatParams & params_, + const FormatSchemaInfo & schema_info_); + + String getName() const override { return "ProtobufListOutputFormat"; } + + String getContentType() const override { return "application/octet-stream"; } + +private: + void write(const Columns & columns, size_t row_num) override; + void writeField(const IColumn &, const ISerialization &, size_t) override {} + + void finalizeImpl() override; + + std::unique_ptr writer; + std::unique_ptr serializer; +}; + +} + +#endif diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp b/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp index 395d8294aa5..38c18aa967c 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp @@ -3,16 +3,13 @@ #if USE_PROTOBUF # include # include -# include # include # include # include -# include -# include - namespace DB { + ProtobufRowInputFormat::ProtobufRowInputFormat(ReadBuffer & in_, const Block & header_, const Params & params_, const FormatSchemaInfo & schema_info_, bool with_length_delimiter_) : IRowInputFormat(header_, in_, params_) , reader(std::make_unique(in_)) @@ -20,14 +17,13 @@ ProtobufRowInputFormat::ProtobufRowInputFormat(ReadBuffer & in_, const Block & h header_.getNames(), header_.getDataTypes(), missing_column_indices, - *ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info_), + *ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info_, ProtobufSchemas::WithEnvelope::No), with_length_delimiter_, + /* with_envelope = */ false, *reader)) { } -ProtobufRowInputFormat::~ProtobufRowInputFormat() = default; - bool ProtobufRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & row_read_extension) { if (reader->eof()) @@ -85,7 +81,7 @@ ProtobufSchemaReader::ProtobufSchemaReader(const FormatSettings & format_setting NamesAndTypesList ProtobufSchemaReader::readSchema() { - const auto * message_descriptor = ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info); + const auto * message_descriptor = ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info, ProtobufSchemas::WithEnvelope::No); return protobufSchemaToCHSchema(message_descriptor); } @@ -111,7 +107,6 @@ namespace DB { class FormatFactory; void registerInputFormatProtobuf(FormatFactory &) {} - void registerProtobufSchemaReader(FormatFactory &) {} } diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h index 9566cb45106..1c276356664 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h @@ -3,17 +3,16 @@ #include "config_formats.h" #if USE_PROTOBUF -# include -# include -# include +# include +# include +# include namespace DB { class Block; -class FormatSchemaInfo; class ProtobufReader; class ProtobufSerializer; - +class ReadBuffer; /** Stream designed to deserialize data from the google protobuf format. * One Protobuf message is parsed as one row of data. @@ -30,12 +29,11 @@ class ProtobufRowInputFormat final : public IRowInputFormat { public: ProtobufRowInputFormat(ReadBuffer & in_, const Block & header_, const Params & params_, const FormatSchemaInfo & schema_info_, bool with_length_delimiter_); - ~ProtobufRowInputFormat() override; String getName() const override { return "ProtobufRowInputFormat"; } private: - bool readRow(MutableColumns & columns, RowReadExtension &) override; + bool readRow(MutableColumns & columns, RowReadExtension & row_read_extension) override; bool allowSyncAfterError() const override; void syncAfterError() override; @@ -52,7 +50,7 @@ public: NamesAndTypesList readSchema() override; private: - FormatSchemaInfo schema_info; + const FormatSchemaInfo schema_info; }; } diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp index 29cd9be79bc..fa730f8ec2a 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp @@ -4,12 +4,12 @@ # include # include # include +# include # include # include # include # include - namespace DB { namespace ErrorCodes @@ -17,7 +17,6 @@ namespace ErrorCodes extern const int NO_ROW_DELIMITER; } - ProtobufRowOutputFormat::ProtobufRowOutputFormat( WriteBuffer & out_, const Block & header_, @@ -30,8 +29,9 @@ ProtobufRowOutputFormat::ProtobufRowOutputFormat( , serializer(ProtobufSerializer::create( header_.getNames(), header_.getDataTypes(), - *ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info_), + *ProtobufSchemas::instance().getMessageTypeForFormatSchema(schema_info_, ProtobufSchemas::WithEnvelope::No), with_length_delimiter_, + /* with_envelope = */ false, *writer)) , allow_multiple_rows(with_length_delimiter_ || settings_.protobuf.allow_multiple_rows_without_delimiter) { @@ -44,13 +44,12 @@ void ProtobufRowOutputFormat::write(const Columns & columns, size_t row_num) "The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter.", ErrorCodes::NO_ROW_DELIMITER); - if (!row_num) + if (row_num == 0) serializer->setColumns(columns.data(), columns.size()); serializer->writeRow(row_num); } - void registerOutputFormatProtobuf(FormatFactory & factory) { for (bool with_length_delimiter : {false, true}) diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index 43d79b4d091..9f7f0b96923 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -3,17 +3,15 @@ #include "config_formats.h" #if USE_PROTOBUF -# include -# include -# include # include - namespace DB { -class ProtobufWriter; -class ProtobufSerializer; +class DB; class FormatSchemaInfo; +class ProtobufSerializer; +class ProtobufWriter; +class WriteBuffer; struct FormatSettings; /** Stream designed to serialize data in the google protobuf format. diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.h b/src/Processors/Formats/Impl/RegexpRowInputFormat.h index 75c630d0607..04f24bbb3e4 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.h +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.h @@ -22,7 +22,7 @@ class ReadBuffer; class RegexpFieldExtractor { public: - RegexpFieldExtractor(const FormatSettings & format_settings); + explicit RegexpFieldExtractor(const FormatSettings & format_settings); /// Return true if row was successfully parsed and row fields were extracted. bool parseRow(PeekableReadBuffer & buf); diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index f63d6fa9c46..87ba1b18fa7 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -4,7 +4,6 @@ #include #include #include -#include namespace DB @@ -242,15 +241,16 @@ std::unordered_map TSKVSchemaReader::readRowAndGetNamesAndD std::unordered_map names_and_types; StringRef name_ref; - String name_tmp; + String name_buf; String value; do { - bool has_value = readName(in, name_ref, name_tmp); + bool has_value = readName(in, name_ref, name_buf); + String name = String(name_ref); if (has_value) { readEscapedString(value, in); - names_and_types[String(name_ref)] = determineDataTypeByEscapingRule(value, format_settings, FormatSettings::EscapingRule::Escaped); + names_and_types[std::move(name)] = determineDataTypeByEscapingRule(value, format_settings, FormatSettings::EscapingRule::Escaped); } else { diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h index ed67a8256bc..abab5b02c96 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -53,7 +53,7 @@ public: bool parseFieldDelimiterWithDiagnosticInfo(WriteBuffer & out) override; bool parseRowEndWithDiagnosticInfo(WriteBuffer & out) override; - FormatSettings::EscapingRule getEscapingRule() + FormatSettings::EscapingRule getEscapingRule() const { return is_raw ? FormatSettings::EscapingRule::Raw : FormatSettings::EscapingRule::Escaped; } diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index c9337929adc..bf8feb077ed 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -105,6 +106,7 @@ Chunk ValuesBlockInputFormat::generate() return {}; } + finalizeObjectColumns(columns); size_t rows_in_block = columns[0]->size(); return Chunk{std::move(columns), rows_in_block}; } diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp b/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp index 0905e4243cd..468770e2515 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp @@ -141,7 +141,7 @@ void VerticalRowOutputFormat::writeSpecialRow(const Columns & columns, size_t ro row_number = 0; field_number = 0; - size_t num_columns = columns.size(); + size_t columns_size = columns.size(); writeCString(title, out); writeCString(":\n", out); @@ -151,7 +151,7 @@ void VerticalRowOutputFormat::writeSpecialRow(const Columns & columns, size_t ro writeCString("─", out); writeChar('\n', out); - for (size_t i = 0; i < num_columns; ++i) + for (size_t i = 0; i < columns_size; ++i) writeField(*columns[i], *serializations[i], row_num); } diff --git a/src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h b/src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h index 35cfded4214..ff8f113d9a6 100644 --- a/src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h +++ b/src/Processors/Merges/Algorithms/FixedSizeDequeWithGaps.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace DB { diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 2c6d08ed287..c0f595fa539 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -89,7 +89,7 @@ inline static const Patterns & selectPatternsForMetricType(const Graphite::Param Graphite::RollupRule selectPatternForPath( const Graphite::Params & params, - const StringRef path) + StringRef path) { const Graphite::Pattern * first_match = &undef_pattern; diff --git a/src/Processors/Merges/Algorithms/Graphite.h b/src/Processors/Merges/Algorithms/Graphite.h index dc39cb46386..05306ebe30f 100644 --- a/src/Processors/Merges/Algorithms/Graphite.h +++ b/src/Processors/Merges/Algorithms/Graphite.h @@ -147,7 +147,7 @@ struct Params using RollupRule = std::pair; -Graphite::RollupRule selectPatternForPath(const Graphite::Params & params, const StringRef path); +Graphite::RollupRule selectPatternForPath(const Graphite::Params & params, StringRef path); void setGraphitePatternsFromConfig(ContextPtr context, const String & config_element, Graphite::Params & params); diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h index 9bf33d72f31..89da346980d 100644 --- a/src/Processors/Merges/Algorithms/MergedData.h +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -1,5 +1,11 @@ #pragma once +#include +#include +#include +#include + + namespace DB { diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 72ad4616174..0247b8677af 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -457,7 +457,7 @@ static void postprocessChunk( { const auto & from_type = desc.nested_type; const auto & to_type = desc.real_type; - res_columns[desc.column_numbers[0]] = recursiveTypeConversion(std::move(column), from_type, to_type); + res_columns[desc.column_numbers[0]] = recursiveTypeConversion(column, from_type, to_type); } else res_columns[desc.column_numbers[0]] = std::move(column); diff --git a/src/Processors/Port.h b/src/Processors/Port.h index 9f27b440be5..7cb25f3930e 100644 --- a/src/Processors/Port.h +++ b/src/Processors/Port.h @@ -214,7 +214,7 @@ protected: public: using Data = State::Data; - Port(Block header_) : header(std::move(header_)) {} + Port(Block header_) : header(std::move(header_)) {} /// NOLINT Port(Block header_, IProcessor * processor_) : header(std::move(header_)), processor(processor_) {} void setUpdateInfo(UpdateInfo * info) { update_info = info; } @@ -303,12 +303,12 @@ public: Chunk ALWAYS_INLINE pull(bool set_not_needed = false) { - auto data_ = pullData(set_not_needed); + auto pull_data = pullData(set_not_needed); - if (data_.exception) - std::rethrow_exception(data_.exception); + if (pull_data.exception) + std::rethrow_exception(pull_data.exception); - return std::move(data_.chunk); + return std::move(pull_data.chunk); } bool ALWAYS_INLINE isFinished() const @@ -396,7 +396,7 @@ public: void ALWAYS_INLINE pushException(std::exception_ptr exception) { - pushData({.chunk = {}, .exception = std::move(exception)}); + pushData({.chunk = {}, .exception = exception}); } void ALWAYS_INLINE pushData(Data data_) diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 45c3719ebca..6b6f9d361ef 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -138,7 +138,7 @@ void addCreatingSetsStep( auto creating_set = std::make_unique( plan->getCurrentDataStream(), - std::move(description), + description, std::move(set), limits, context); diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index a271ef78dfa..d948c16a78d 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -22,8 +22,8 @@ namespace ErrorCodes QueryPlan::QueryPlan() = default; QueryPlan::~QueryPlan() = default; -QueryPlan::QueryPlan(QueryPlan &&) = default; -QueryPlan & QueryPlan::operator=(QueryPlan &&) = default; +QueryPlan::QueryPlan(QueryPlan &&) noexcept = default; +QueryPlan & QueryPlan::operator=(QueryPlan &&) noexcept = default; void QueryPlan::checkInitialized() const { diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 4e342d746d1..5e064713abd 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -44,8 +44,8 @@ class QueryPlan public: QueryPlan(); ~QueryPlan(); - QueryPlan(QueryPlan &&); - QueryPlan & operator=(QueryPlan &&); + QueryPlan(QueryPlan &&) noexcept; + QueryPlan & operator=(QueryPlan &&) noexcept; void unitePlans(QueryPlanStepPtr step, std::vector plans); void addStep(QueryPlanStepPtr step); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 9a9a71f9688..1bfc1ec7306 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -68,8 +68,7 @@ ReadFromMergeTree::ReadFromMergeTree( Names virt_column_names_, const MergeTreeData & data_, const SelectQueryInfo & query_info_, - StorageMetadataPtr metadata_snapshot_, - StorageMetadataPtr metadata_snapshot_base_, + StorageSnapshotPtr storage_snapshot_, ContextPtr context_, size_t max_block_size_, size_t num_streams_, @@ -79,7 +78,7 @@ ReadFromMergeTree::ReadFromMergeTree( MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_, bool enable_parallel_reading) : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( - metadata_snapshot_->getSampleBlockForColumns(real_column_names_, data_.getVirtuals(), data_.getStorageID()), + storage_snapshot_->getSampleBlockForColumns(real_column_names_), getPrewhereInfo(query_info_), data_.getPartitionValueType(), virt_column_names_)}) @@ -91,8 +90,8 @@ ReadFromMergeTree::ReadFromMergeTree( , query_info(query_info_) , prewhere_info(getPrewhereInfo(query_info)) , actions_settings(ExpressionActionsSettings::fromContext(context_)) - , metadata_snapshot(std::move(metadata_snapshot_)) - , metadata_snapshot_base(std::move(metadata_snapshot_base_)) + , storage_snapshot(std::move(storage_snapshot_)) + , metadata_for_reading(storage_snapshot->getMetadataForQuery()) , context(std::move(context_)) , max_block_size(max_block_size_) , requested_num_streams(num_streams_) @@ -142,7 +141,7 @@ Pipe ReadFromMergeTree::readFromPool( min_marks_for_concurrent_read, std::move(parts_with_range), data, - metadata_snapshot, + storage_snapshot, prewhere_info, required_columns, backoff_settings, @@ -169,7 +168,7 @@ Pipe ReadFromMergeTree::readFromPool( auto source = std::make_shared( i, pool, min_marks_for_concurrent_read, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, - data, metadata_snapshot, use_uncompressed_cache, + data, storage_snapshot, use_uncompressed_cache, prewhere_info, actions_settings, reader_settings, virt_column_names, std::move(extension)); /// Set the approximate number of rows for the first source only @@ -205,7 +204,7 @@ ProcessorPtr ReadFromMergeTree::createSource( }; } return std::make_shared( - data, metadata_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, + data, storage_snapshot, part.data_part, max_block_size, preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, required_columns, part.ranges, use_uncompressed_cache, prewhere_info, actions_settings, reader_settings, virt_column_names, part.part_index_in_query, has_limit_below_one_block, std::move(extension)); } @@ -511,12 +510,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( size_t fixed_prefix_size = input_order_info->order_key_fixed_prefix_descr.size(); size_t prefix_size = fixed_prefix_size + input_order_info->order_key_prefix_descr.size(); - auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); + auto order_key_prefix_ast = metadata_for_reading->getSortingKey().expression_list_ast->clone(); order_key_prefix_ast->children.resize(prefix_size); - auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); + auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_for_reading->getColumns().getAllPhysical()); auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); - const auto & sorting_columns = metadata_snapshot->getSortingKey().column_names; + const auto & sorting_columns = metadata_for_reading->getSortingKey().column_names; SortDescription sort_description; for (size_t j = 0; j < prefix_size; ++j) @@ -745,7 +744,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( } auto sorting_expr = std::make_shared( - metadata_snapshot->getSortingKey().expression->getActionsDAG().clone()); + metadata_for_reading->getSortingKey().expression->getActionsDAG().clone()); pipe.addSimpleTransform([sorting_expr](const Block & header) { @@ -762,12 +761,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( continue; } - Names sort_columns = metadata_snapshot->getSortingKeyColumns(); + Names sort_columns = metadata_for_reading->getSortingKeyColumns(); SortDescription sort_description; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); - Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names; + Names partition_key_columns = metadata_for_reading->getPartitionKey().column_names; const auto & header = pipe.getHeader(); for (size_t i = 0; i < sort_columns_size; ++i) @@ -807,7 +806,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( out_projection = createProjection(pipe.getHeader()); auto sorting_expr = std::make_shared( - metadata_snapshot->getSortingKey().expression->getActionsDAG().clone()); + metadata_for_reading->getSortingKey().expression->getActionsDAG().clone()); pipe.addSimpleTransform([sorting_expr](const Block & header) { @@ -824,8 +823,8 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(Merge { return selectRangesToRead( std::move(parts), - metadata_snapshot_base, - metadata_snapshot, + storage_snapshot->metadata, + storage_snapshot->getMetadataForQuery(), query_info, context, requested_num_streams, @@ -867,7 +866,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( result.column_names_to_read.push_back(ExpressionActions::getSmallestColumn(available_real_columns)); } - metadata_snapshot->check(result.column_names_to_read, data.getVirtuals(), data.getStorageID()); + // storage_snapshot->check(result.column_names_to_read); // Build and check if primary key is used when necessary const auto & primary_key = metadata_snapshot->getPrimaryKey(); @@ -982,7 +981,7 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const { auto result_ptr = analyzed_result_ptr ? analyzed_result_ptr : selectRangesToRead(prepared_parts); if (std::holds_alternative(result_ptr->result)) - std::rethrow_exception(std::move(std::get(result_ptr->result))); + std::rethrow_exception(std::get(result_ptr->result)); return std::get(result_ptr->result); } @@ -1045,7 +1044,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons if (select.final()) { /// Add columns needed to calculate the sorting expression and the sign. - std::vector add_columns = metadata_snapshot->getColumnsRequiredForSortingKey(); + std::vector add_columns = metadata_for_reading->getColumnsRequiredForSortingKey(); column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); if (!data.merging_params.sign_column.empty()) @@ -1326,7 +1325,7 @@ bool MergeTreeDataSelectAnalysisResult::error() const size_t MergeTreeDataSelectAnalysisResult::marks() const { if (std::holds_alternative(result)) - std::rethrow_exception(std::move(std::get(result))); + std::rethrow_exception(std::get(result)); const auto & index_stats = std::get(result).index_stats; if (index_stats.empty()) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 0d07a3e2ea2..685b99a7bdc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -89,8 +89,7 @@ public: Names virt_column_names_, const MergeTreeData & data_, const SelectQueryInfo & query_info_, - StorageMetadataPtr metadata_snapshot_, - StorageMetadataPtr metadata_snapshot_base_, + StorageSnapshotPtr storage_snapshot, ContextPtr context_, size_t max_block_size_, size_t num_streams_, @@ -141,8 +140,8 @@ private: PrewhereInfoPtr prewhere_info; ExpressionActionsSettings actions_settings; - StorageMetadataPtr metadata_snapshot; - StorageMetadataPtr metadata_snapshot_base; + StorageSnapshotPtr storage_snapshot; + StorageMetadataPtr metadata_for_reading; ContextPtr context; diff --git a/src/Processors/QueueBuffer.h b/src/Processors/QueueBuffer.h index 826f4a22b8b..6856e214823 100644 --- a/src/Processors/QueueBuffer.h +++ b/src/Processors/QueueBuffer.h @@ -17,7 +17,7 @@ private: public: String getName() const override { return "QueueBuffer"; } - QueueBuffer(Block header) + explicit QueueBuffer(Block header) : IAccumulatingTransform(header, header) { } diff --git a/src/Processors/Sources/DelayedSource.cpp b/src/Processors/Sources/DelayedSource.cpp index 205ea6e2253..6cfdeeeeec5 100644 --- a/src/Processors/Sources/DelayedSource.cpp +++ b/src/Processors/Sources/DelayedSource.cpp @@ -64,7 +64,7 @@ IProcessor::Status DelayedSource::prepare() continue; } - if (!output->isNeeded()) + if (!output->canPush()) return Status::PortFull; if (input->isFinished()) diff --git a/src/Processors/Sources/MySQLSource.cpp b/src/Processors/Sources/MySQLSource.cpp index 538aba9d1f3..a9b408064d9 100644 --- a/src/Processors/Sources/MySQLSource.cpp +++ b/src/Processors/Sources/MySQLSource.cpp @@ -225,6 +225,10 @@ namespace assert_cast(column).insertValue(UInt16(value.getDate().getDayNum())); read_bytes_size += 2; break; + case ValueType::vtDate32: + assert_cast(column).insertValue(Int32(value.getDate().getExtenedDayNum())); + read_bytes_size += 4; + break; case ValueType::vtDateTime: { ReadBufferFromString in(value); diff --git a/src/Processors/Transforms/AddingDefaultsTransform.cpp b/src/Processors/Transforms/AddingDefaultsTransform.cpp index 01048354b82..82c235d9034 100644 --- a/src/Processors/Transforms/AddingDefaultsTransform.cpp +++ b/src/Processors/Transforms/AddingDefaultsTransform.cpp @@ -187,7 +187,7 @@ void AddingDefaultsTransform::transform(Chunk & chunk) { const String & column_name = column_def.name; - if (column_defaults.count(column_name) == 0) + if (column_defaults.count(column_name) == 0 || !res.has(column_name)) continue; size_t block_column_position = res.getPositionByName(column_name); diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.h b/src/Processors/Transforms/AggregatingInOrderTransform.h index 929ab98d6e6..e4c217a8f81 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.h +++ b/src/Processors/Transforms/AggregatingInOrderTransform.h @@ -11,7 +11,7 @@ namespace DB struct ChunkInfoWithAllocatedBytes : public ChunkInfo { - ChunkInfoWithAllocatedBytes(Int64 allocated_bytes_) + explicit ChunkInfoWithAllocatedBytes(Int64 allocated_bytes_) : allocated_bytes(allocated_bytes_) {} Int64 allocated_bytes; }; diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 01df264005b..d7917fc95a7 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -12,7 +12,7 @@ class AggregatedArenasChunkInfo : public ChunkInfo { public: Arenas arenas; - AggregatedArenasChunkInfo(Arenas arenas_) + explicit AggregatedArenasChunkInfo(Arenas arenas_) : arenas(std::move(arenas_)) {} }; diff --git a/src/Processors/Transforms/ColumnGathererTransform.h b/src/Processors/Transforms/ColumnGathererTransform.h index 2d013e596ce..da6dc877abf 100644 --- a/src/Processors/Transforms/ColumnGathererTransform.h +++ b/src/Processors/Transforms/ColumnGathererTransform.h @@ -20,7 +20,7 @@ struct RowSourcePart RowSourcePart() = default; - RowSourcePart(size_t source_num, bool skip_flag = false) + explicit RowSourcePart(size_t source_num, bool skip_flag = false) { static_assert(sizeof(*this) == 1, "Size of RowSourcePart is too big due to compiler settings"); setSourceNum(source_num); diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 01cef654388..5600476fd77 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -24,7 +24,7 @@ void DistinctSortedTransform::transform(Chunk & chunk) if (column_ptrs.empty()) return; - const ColumnRawPtrs clearing_hint_columns(getClearingColumns(chunk, column_ptrs)); + ColumnRawPtrs clearing_hint_columns(getClearingColumns(chunk, column_ptrs)); if (data.type == ClearableSetVariants::Type::EMPTY) data.init(ClearableSetVariants::chooseMethod(column_ptrs, key_sizes)); diff --git a/src/Processors/Transforms/ExceptionKeepingTransform.cpp b/src/Processors/Transforms/ExceptionKeepingTransform.cpp index f2b29a45f84..266407f21a5 100644 --- a/src/Processors/Transforms/ExceptionKeepingTransform.cpp +++ b/src/Processors/Transforms/ExceptionKeepingTransform.cpp @@ -138,7 +138,7 @@ void ExceptionKeepingTransform::work() { stage = Stage::Exception; ready_output = true; - data.exception = std::move(exception); + data.exception = exception; onException(); } } @@ -152,7 +152,7 @@ void ExceptionKeepingTransform::work() { stage = Stage::Exception; ready_output = true; - data.exception = std::move(exception); + data.exception = exception; onException(); } else @@ -166,7 +166,7 @@ void ExceptionKeepingTransform::work() { stage = Stage::Exception; ready_output = true; - data.exception = std::move(exception); + data.exception = exception; onException(); } else @@ -188,7 +188,7 @@ void ExceptionKeepingTransform::work() { stage = Stage::Exception; ready_output = true; - data.exception = std::move(exception); + data.exception = exception; onException(); } } diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 1276157cc91..ae97a769b23 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -34,16 +34,16 @@ Block FillingTransform::transformHeader(Block header, const SortDescription & so template static FillColumnDescription::StepFunction getStepFunction( - IntervalKind kind, Int64 step, const DateLUTImpl & date_lut) + IntervalKind kind, Int64 step, const DateLUTImpl & date_lut, UInt16 scale = DataTypeDateTime64::default_scale) { switch (kind) { - #define DECLARE_CASE(NAME) \ +#define DECLARE_CASE(NAME) \ case IntervalKind::NAME: \ - return [step, &date_lut](Field & field) { field = Add##NAME##sImpl::execute(get(field), step, date_lut); }; + return [step, scale, &date_lut](Field & field) { field = Add##NAME##sImpl::execute(get(field), step, date_lut, scale); }; FOR_EACH_INTERVAL_KIND(DECLARE_CASE) - #undef DECLARE_CASE +#undef DECLARE_CASE } __builtin_unreachable(); } @@ -92,7 +92,7 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & Int64 avg_seconds = get(descr.fill_step) * descr.step_kind->toAvgSeconds(); if (avg_seconds < 86400) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, - "Value of step is to low ({} seconds). Must be >= 1 day", avg_seconds); + "Value of step is to low ({} seconds). Must be >= 1 day", avg_seconds); } if (which.isDate()) @@ -108,25 +108,23 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & switch (*descr.step_kind) { - #define DECLARE_CASE(NAME) \ +#define DECLARE_CASE(NAME) \ case IntervalKind::NAME: \ descr.step_func = [step, &time_zone = date_time64->getTimeZone()](Field & field) \ { \ auto field_decimal = get>(field); \ - auto components = DecimalUtils::splitWithScaleMultiplier(field_decimal.getValue(), field_decimal.getScaleMultiplier()); \ - auto res = Add##NAME##sImpl::execute(components, step, time_zone); \ - auto res_decimal = decimalFromComponentsWithMultiplier(res, field_decimal.getScaleMultiplier()); \ - field = DecimalField(res_decimal, field_decimal.getScale()); \ + auto res = Add##NAME##sImpl::execute(field_decimal.getValue(), step, time_zone, field_decimal.getScale()); \ + field = DecimalField(res, field_decimal.getScale()); \ }; \ break; FOR_EACH_INTERVAL_KIND(DECLARE_CASE) - #undef DECLARE_CASE +#undef DECLARE_CASE } } else throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, - "STEP of Interval type can be used only with Date/DateTime types, but got {}", type->getName()); + "STEP of Interval type can be used only with Date/DateTime types, but got {}", type->getName()); } else { @@ -140,12 +138,12 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & } FillingTransform::FillingTransform( - const Block & header_, const SortDescription & sort_description_, bool on_totals_) - : ISimpleTransform(header_, transformHeader(header_, sort_description_), true) - , sort_description(sort_description_) - , on_totals(on_totals_) - , filling_row(sort_description_) - , next_row(sort_description_) + const Block & header_, const SortDescription & sort_description_, bool on_totals_) + : ISimpleTransform(header_, transformHeader(header_, sort_description_), true) + , sort_description(sort_description_) + , on_totals(on_totals_) + , filling_row(sort_description_) + , next_row(sort_description_) { if (on_totals) return; @@ -162,14 +160,14 @@ FillingTransform::FillingTransform( if (!tryConvertFields(descr, type)) throw Exception("Incompatible types of WITH FILL expression values with column type " - + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); + + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); if (type->isValueRepresentedByUnsignedInteger() && ((!descr.fill_from.isNull() && less(descr.fill_from, Field{0}, 1)) || - (!descr.fill_to.isNull() && less(descr.fill_to, Field{0}, 1)))) + (!descr.fill_to.isNull() && less(descr.fill_to, Field{0}, 1)))) { throw Exception("WITH FILL bound values cannot be negative for unsigned type " - + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); + + type->getName(), ErrorCodes::INVALID_WITH_FILL_EXPRESSION); } } @@ -214,7 +212,7 @@ void FillingTransform::transform(Chunk & chunk) MutableColumns res_other_columns; auto init_columns_by_positions = [](const Columns & old_columns, Columns & new_columns, - MutableColumns & new_mutable_columns, const Positions & positions) + MutableColumns & new_mutable_columns, const Positions & positions) { for (size_t pos : positions) { diff --git a/src/Processors/Transforms/FilterTransform.cpp b/src/Processors/Transforms/FilterTransform.cpp index 364fb8e1958..9164599f3b1 100644 --- a/src/Processors/Transforms/FilterTransform.cpp +++ b/src/Processors/Transforms/FilterTransform.cpp @@ -138,8 +138,6 @@ void FilterTransform::transform(Chunk & chunk) return; } - FilterDescription filter_and_holder(*filter_column); - /** Let's find out how many rows will be in result. * To do this, we filter out the first non-constant column * or calculate number of set bytes in the filter. @@ -154,14 +152,20 @@ void FilterTransform::transform(Chunk & chunk) } } + std::unique_ptr filter_description; + if (filter_column->isSparse()) + filter_description = std::make_unique(*filter_column); + else + filter_description = std::make_unique(*filter_column); + size_t num_filtered_rows = 0; if (first_non_constant_column != num_columns) { - columns[first_non_constant_column] = columns[first_non_constant_column]->filter(*filter_and_holder.data, -1); + columns[first_non_constant_column] = filter_description->filter(*columns[first_non_constant_column], -1); num_filtered_rows = columns[first_non_constant_column]->size(); } else - num_filtered_rows = countBytesInFilter(*filter_and_holder.data); + num_filtered_rows = filter_description->countBytesInFilter(); /// If the current block is completely filtered out, let's move on to the next one. if (num_filtered_rows == 0) @@ -207,7 +211,7 @@ void FilterTransform::transform(Chunk & chunk) if (isColumnConst(*current_column)) current_column = current_column->cut(0, num_filtered_rows); else - current_column = current_column->filter(*filter_and_holder.data, num_filtered_rows); + current_column = filter_description->filter(*current_column, num_filtered_rows); } chunk.setColumns(std::move(columns), num_filtered_rows); diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index d01a809e666..905620d39f9 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -246,6 +246,9 @@ IProcessor::Status GroupingAggregatedTransform::prepare() void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input) { + if (!chunk.hasRows()) + return; + const auto & info = chunk.getChunkInfo(); if (!info) throw Exception("Chunk info was not set for chunk in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); @@ -266,7 +269,7 @@ void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input) last_bucket_number[input] = bucket; } } - else if (const auto * in_order_info = typeid_cast(info.get())) + else if (typeid_cast(info.get())) { single_level_chunks.emplace_back(std::move(chunk)); } @@ -334,7 +337,7 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) blocks_list.emplace_back(std::move(block)); } - else if (const auto * in_order_info = typeid_cast(cur_info.get())) + else if (typeid_cast(cur_info.get())) { Block block = header.cloneWithColumns(cur_chunk.detachColumns()); block.info.is_overflows = false; diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 37419f55aae..11d32278caf 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -34,21 +35,30 @@ void MergingAggregatedTransform::consume(Chunk chunk) if (!info) throw Exception("Chunk info was not set for chunk in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - const auto * agg_info = typeid_cast(info.get()); - if (!agg_info) + if (const auto * agg_info = typeid_cast(info.get())) + { + /** If the remote servers used a two-level aggregation method, + * then blocks will contain information about the number of the bucket. + * Then the calculations can be parallelized by buckets. + * We decompose the blocks to the bucket numbers indicated in them. + */ + + auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); + block.info.is_overflows = agg_info->is_overflows; + block.info.bucket_num = agg_info->bucket_num; + + bucket_to_blocks[agg_info->bucket_num].emplace_back(std::move(block)); + } + else if (typeid_cast(info.get())) + { + auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); + block.info.is_overflows = false; + block.info.bucket_num = -1; + + bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); + } + else throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - - /** If the remote servers used a two-level aggregation method, - * then blocks will contain information about the number of the bucket. - * Then the calculations can be parallelized by buckets. - * We decompose the blocks to the bucket numbers indicated in them. - */ - - auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); - block.info.is_overflows = agg_info->is_overflows; - block.info.bucket_num = agg_info->bucket_num; - - bucket_to_blocks[agg_info->bucket_num].emplace_back(std::move(block)); } Chunk MergingAggregatedTransform::generate() diff --git a/src/Processors/Transforms/PostgreSQLSource.cpp b/src/Processors/Transforms/PostgreSQLSource.cpp index 88f092a2533..a31cd879257 100644 --- a/src/Processors/Transforms/PostgreSQLSource.cpp +++ b/src/Processors/Transforms/PostgreSQLSource.cpp @@ -28,7 +28,7 @@ PostgreSQLSource::PostgreSQLSource( postgres::ConnectionHolderPtr connection_holder_, const std::string & query_str_, const Block & sample_block, - const UInt64 max_block_size_) + UInt64 max_block_size_) : SourceWithProgress(sample_block.cloneEmpty()) , query_str(query_str_) , max_block_size(max_block_size_) @@ -43,7 +43,7 @@ PostgreSQLSource::PostgreSQLSource( std::shared_ptr tx_, const std::string & query_str_, const Block & sample_block, - const UInt64 max_block_size_, + UInt64 max_block_size_, bool auto_commit_) : SourceWithProgress(sample_block.cloneEmpty()) , query_str(query_str_) diff --git a/src/Processors/Transforms/PostgreSQLSource.h b/src/Processors/Transforms/PostgreSQLSource.h index c7e55c09c32..bd6203042bb 100644 --- a/src/Processors/Transforms/PostgreSQLSource.h +++ b/src/Processors/Transforms/PostgreSQLSource.h @@ -24,7 +24,7 @@ public: postgres::ConnectionHolderPtr connection_holder_, const String & query_str_, const Block & sample_block, - const UInt64 max_block_size_); + UInt64 max_block_size_); String getName() const override { return "PostgreSQL"; } @@ -33,7 +33,7 @@ protected: std::shared_ptr tx_, const std::string & query_str_, const Block & sample_block, - const UInt64 max_block_size_, + UInt64 max_block_size_, bool auto_commit_); String query_str; diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index 0b7797da24f..45e972afa3f 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -138,7 +138,7 @@ IProcessor::Status TotalsHavingTransform::prepare() if (!totals_output.canPush()) return Status::PortFull; - if (!totals) + if (!total_prepared) return Status::Ready; totals_output.push(std::move(totals)); @@ -312,6 +312,8 @@ void TotalsHavingTransform::prepareTotals() /// Note: after expression totals may have several rows if `arrayJoin` was used in expression. totals = Chunk(block.getColumns(), num_rows); } + + total_prepared = true; } } diff --git a/src/Processors/Transforms/TotalsHavingTransform.h b/src/Processors/Transforms/TotalsHavingTransform.h index 03635054c65..6b4afb2fa8b 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.h +++ b/src/Processors/Transforms/TotalsHavingTransform.h @@ -46,6 +46,7 @@ protected: void transform(Chunk & chunk) override; bool finished_transform = false; + bool total_prepared = false; Chunk totals; private: diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 0da7541556b..b81ed099915 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; + extern const int ILLEGAL_COLUMN; } // Interface for true window functions. It's not much of an interface, they just @@ -206,7 +208,7 @@ WindowTransform::WindowTransform(const Block & input_header_, { column = std::move(column)->convertToFullColumnIfConst(); } - input_header.setColumns(std::move(input_columns)); + input_header.setColumns(input_columns); // Initialize window function workspaces. workspaces.reserve(functions.size()); @@ -986,7 +988,23 @@ void WindowTransform::writeOutCurrentRow() auto * buf = ws.aggregate_function_state.data(); // FIXME does it also allocate the result on the arena? // We'll have to pass it out with blocks then... - a->insertResultInto(buf, *result_column, arena.get()); + + if (a->isState()) + { + /// AggregateFunction's states should be inserted into column using specific way + auto * res_col_aggregate_function = typeid_cast(result_column); + if (!res_col_aggregate_function) + { + throw Exception("State function " + a->getName() + " inserts results into non-state column ", + ErrorCodes::ILLEGAL_COLUMN); + } + res_col_aggregate_function->insertFrom(buf); + } + else + { + a->insertResultInto(buf, *result_column, arena.get()); + } + } } @@ -1981,7 +1999,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction return; } - const auto supertype = getLeastSupertype({argument_types[0], argument_types[2]}); + const auto supertype = getLeastSupertype(DataTypes{argument_types[0], argument_types[2]}); if (!supertype) { throw Exception(ErrorCodes::BAD_ARGUMENTS, diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h index 077979e83b9..d536c8780d2 100644 --- a/src/Processors/Transforms/WindowTransform.h +++ b/src/Processors/Transforms/WindowTransform.h @@ -245,7 +245,6 @@ public: return RowNumber{first_block_number, 0}; } -public: /* * Data (formerly) inherited from ISimpleTransform, needed for the * implementation of the IProcessor interface. @@ -349,10 +348,10 @@ public: template <> struct fmt::formatter { - constexpr auto parse(format_parse_context & ctx) + static constexpr auto parse(format_parse_context & ctx) { - auto it = ctx.begin(); - auto end = ctx.end(); + const auto * it = ctx.begin(); + const auto * end = ctx.end(); /// Only support {}. if (it != end && *it != '}') diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 19302afb5c9..a993b8acd7d 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -695,7 +695,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() return Status::Ready; if (any_exception) - output.pushException(std::move(any_exception)); + output.pushException(any_exception); output.finish(); return Status::Finished; @@ -708,7 +708,7 @@ static std::exception_ptr addStorageToException(std::exception_ptr ptr, const St { try { - std::rethrow_exception(std::move(ptr)); + std::rethrow_exception(ptr); } catch (DB::Exception & exception) { @@ -736,7 +736,7 @@ void FinalizingViewsTransform::work() if (!any_exception) any_exception = status.exception; - view.setException(addStorageToException(std::move(status.exception), view.table_id)); + view.setException(addStorageToException(status.exception, view.table_id)); } else { diff --git a/src/QueryPipeline/BlockIO.cpp b/src/QueryPipeline/BlockIO.cpp index 671ba6e4c39..84cf3829a13 100644 --- a/src/QueryPipeline/BlockIO.cpp +++ b/src/QueryPipeline/BlockIO.cpp @@ -23,7 +23,7 @@ void BlockIO::reset() /// TODO Do we need also reset callbacks? In which order? } -BlockIO & BlockIO::operator= (BlockIO && rhs) +BlockIO & BlockIO::operator= (BlockIO && rhs) noexcept { if (this == &rhs) return *this; diff --git a/src/QueryPipeline/BlockIO.h b/src/QueryPipeline/BlockIO.h index 748e46c3a1e..94c6fbc83cb 100644 --- a/src/QueryPipeline/BlockIO.h +++ b/src/QueryPipeline/BlockIO.h @@ -14,7 +14,7 @@ struct BlockIO BlockIO() = default; BlockIO(BlockIO &&) = default; - BlockIO & operator= (BlockIO && rhs); + BlockIO & operator= (BlockIO && rhs) noexcept; ~BlockIO(); BlockIO(const BlockIO &) = delete; diff --git a/src/QueryPipeline/PipelineResourcesHolder.cpp b/src/QueryPipeline/PipelineResourcesHolder.cpp index a4b85ed662b..2f6b6a9de32 100644 --- a/src/QueryPipeline/PipelineResourcesHolder.cpp +++ b/src/QueryPipeline/PipelineResourcesHolder.cpp @@ -5,10 +5,10 @@ namespace DB { PipelineResourcesHolder::PipelineResourcesHolder() = default; -PipelineResourcesHolder::PipelineResourcesHolder(PipelineResourcesHolder &&) = default; +PipelineResourcesHolder::PipelineResourcesHolder(PipelineResourcesHolder &&) noexcept = default; PipelineResourcesHolder::~PipelineResourcesHolder() = default; -PipelineResourcesHolder & PipelineResourcesHolder::operator=(PipelineResourcesHolder && rhs) +PipelineResourcesHolder & PipelineResourcesHolder::operator=(PipelineResourcesHolder && rhs) noexcept { table_locks.insert(table_locks.end(), rhs.table_locks.begin(), rhs.table_locks.end()); storage_holders.insert(storage_holders.end(), rhs.storage_holders.begin(), rhs.storage_holders.end()); diff --git a/src/QueryPipeline/PipelineResourcesHolder.h b/src/QueryPipeline/PipelineResourcesHolder.h index 9fb1438424a..7853fa3ae4c 100644 --- a/src/QueryPipeline/PipelineResourcesHolder.h +++ b/src/QueryPipeline/PipelineResourcesHolder.h @@ -16,10 +16,10 @@ class Context; struct PipelineResourcesHolder { PipelineResourcesHolder(); - PipelineResourcesHolder(PipelineResourcesHolder &&); + PipelineResourcesHolder(PipelineResourcesHolder &&) noexcept; ~PipelineResourcesHolder(); /// Custom mode assignment does not destroy data from lhs. It appends data from rhs to lhs. - PipelineResourcesHolder& operator=(PipelineResourcesHolder &&); + PipelineResourcesHolder& operator=(PipelineResourcesHolder &&) noexcept; /// Some processors may implicitly use Context or temporary Storage created by Interpreter. /// But lifetime of Streams is not nested in lifetime of Interpreters, so we have to store it here, diff --git a/src/QueryPipeline/QueryPipeline.cpp b/src/QueryPipeline/QueryPipeline.cpp index ce1c9473f60..0412049bd58 100644 --- a/src/QueryPipeline/QueryPipeline.cpp +++ b/src/QueryPipeline/QueryPipeline.cpp @@ -24,8 +24,8 @@ namespace ErrorCodes } QueryPipeline::QueryPipeline() = default; -QueryPipeline::QueryPipeline(QueryPipeline &&) = default; -QueryPipeline & QueryPipeline::operator=(QueryPipeline &&) = default; +QueryPipeline::QueryPipeline(QueryPipeline &&) noexcept = default; +QueryPipeline & QueryPipeline::operator=(QueryPipeline &&) noexcept = default; QueryPipeline::~QueryPipeline() = default; static void checkInput(const InputPort & input, const ProcessorPtr & processor) diff --git a/src/QueryPipeline/QueryPipeline.h b/src/QueryPipeline/QueryPipeline.h index beb46361f95..29b5dd76017 100644 --- a/src/QueryPipeline/QueryPipeline.h +++ b/src/QueryPipeline/QueryPipeline.h @@ -32,10 +32,10 @@ class QueryPipeline { public: QueryPipeline(); - QueryPipeline(QueryPipeline &&); + QueryPipeline(QueryPipeline &&) noexcept; QueryPipeline(const QueryPipeline &) = delete; - QueryPipeline & operator=(QueryPipeline &&); + QueryPipeline & operator=(QueryPipeline &&) noexcept; QueryPipeline & operator=(const QueryPipeline &) = delete; ~QueryPipeline(); diff --git a/src/QueryPipeline/RemoteInserter.cpp b/src/QueryPipeline/RemoteInserter.cpp index 13d087f0db9..6acdf19090d 100644 --- a/src/QueryPipeline/RemoteInserter.cpp +++ b/src/QueryPipeline/RemoteInserter.cpp @@ -32,8 +32,19 @@ RemoteInserter::RemoteInserter( modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; if (CurrentThread::isInitialized()) { - modified_client_info.client_trace_context - = CurrentThread::get().thread_trace_context; + auto& thread_trace_context = CurrentThread::get().thread_trace_context; + + if (thread_trace_context.trace_id != UUID()) + { + // overwrite the trace context only if current thread trace context is available + modified_client_info.client_trace_context = thread_trace_context; + } + else + { + // if the trace on the thread local is not enabled(for example running in a background thread) + // we should not clear the trace context on the client info because the client info may hold trace context + // and this trace context should be propagated to the remote server so that the tracing of distributed table insert is complete. + } } /** Send query and receive "header", that describes table structure. diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 142e56ceb25..110d4308236 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -210,7 +210,7 @@ static Block adaptBlockStructure(const Block & block, const Block & header) return res; } -void RemoteQueryExecutor::sendQuery() +void RemoteQueryExecutor::sendQuery(ClientInfo::QueryKind query_kind) { if (sent_query) return; @@ -237,13 +237,7 @@ void RemoteQueryExecutor::sendQuery() auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings); ClientInfo modified_client_info = context->getClientInfo(); - modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; - /// Set initial_query_id to query_id for the clickhouse-benchmark. - /// - /// (since first query of clickhouse-benchmark will be issued as SECONDARY_QUERY, - /// due to it executes queries via RemoteBlockInputStream) - if (modified_client_info.initial_query_id.empty()) - modified_client_info.initial_query_id = query_id; + modified_client_info.query_kind = query_kind; if (CurrentThread::isInitialized()) { modified_client_info.client_trace_context = CurrentThread::get().thread_trace_context; @@ -569,12 +563,13 @@ void RemoteQueryExecutor::sendExternalTables() { SelectQueryInfo query_info; auto metadata_snapshot = cur->getInMemoryMetadataPtr(); + auto storage_snapshot = cur->getStorageSnapshot(metadata_snapshot); QueryProcessingStage::Enum read_from_table_stage = cur->getQueryProcessingStage( - context, QueryProcessingStage::Complete, metadata_snapshot, query_info); + context, QueryProcessingStage::Complete, storage_snapshot, query_info); Pipe pipe = cur->read( metadata_snapshot->getColumns().getNamesOfPhysical(), - metadata_snapshot, query_info, context, + storage_snapshot, query_info, context, read_from_table_stage, DEFAULT_BLOCK_SIZE, 1); if (pipe.empty()) diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index 655bd5603de..78bc9f611ab 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -83,7 +83,13 @@ public: ~RemoteQueryExecutor(); /// Create connection and send query, external tables and scalars. - void sendQuery(); + /// + /// @param query_kind - kind of query, usually it is SECONDARY_QUERY, + /// since this is the queries between servers + /// (for which this code was written in general). + /// But clickhouse-benchmark uses the same code, + /// and it should pass INITIAL_QUERY. + void sendQuery(ClientInfo::QueryKind query_kind = ClientInfo::QueryKind::SECONDARY_QUERY); /// Query is resent to a replica, the query itself can be modified. std::atomic resent_query { false }; diff --git a/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp b/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp index 4064643f1f8..575cdb95431 100644 --- a/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp +++ b/src/QueryPipeline/RemoteQueryExecutorReadContext.cpp @@ -174,7 +174,7 @@ bool RemoteQueryExecutorReadContext::resumeRoutine() fiber = std::move(fiber).resume(); if (exception) - std::rethrow_exception(std::move(exception)); + std::rethrow_exception(exception); } return true; diff --git a/src/QueryPipeline/SizeLimits.h b/src/QueryPipeline/SizeLimits.h index ce7e1795475..fc052714b0c 100644 --- a/src/QueryPipeline/SizeLimits.h +++ b/src/QueryPipeline/SizeLimits.h @@ -26,7 +26,7 @@ struct SizeLimits UInt64 max_bytes = 0; OverflowMode overflow_mode = OverflowMode::THROW; - SizeLimits() {} + SizeLimits() = default; SizeLimits(UInt64 max_rows_, UInt64 max_bytes_, OverflowMode overflow_mode_) : max_rows(max_rows_), max_bytes(max_bytes_), overflow_mode(overflow_mode_) {} diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index f3f366876da..aaffd08365c 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -37,7 +37,7 @@ int CertificateReloader::setCertificate(SSL * ssl) return -1; SSL_use_certificate(ssl, const_cast(current->cert.certificate())); - SSL_use_RSAPrivateKey(ssl, current->key.impl()->getRSA()); + SSL_use_PrivateKey(ssl, const_cast(static_cast(current->key))); int err = SSL_check_private_key(ssl); if (err != 1) diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 7f93b006875..88c732c2db6 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -74,7 +74,7 @@ private: struct Data { Poco::Crypto::X509Certificate cert; - Poco::Crypto::RSAKey key; + Poco::Crypto::EVPPKey key; Data(std::string cert_path, std::string key_path); }; diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 10bbce24913..eeaf5b32a92 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -642,6 +642,9 @@ namespace void throwIfFailedToReadQueryInfo(); bool isQueryCancelled(); + void addQueryDetailsToResult(); + void addOutputFormatToResult(); + void addOutputColumnsNamesAndTypesToResult(const Block & headers); void addProgressToResult(); void addTotalsToResult(const Block & totals); void addExtremesToResult(const Block & extremes); @@ -667,6 +670,7 @@ namespace CompressionMethod input_compression_method = CompressionMethod::None; PODArray output; String output_format; + bool send_output_columns_names_and_types = false; CompressionMethod output_compression_method = CompressionMethod::None; int output_compression_level = 0; @@ -888,6 +892,8 @@ namespace if (output_format.empty()) output_format = query_context->getDefaultFormat(); + send_output_columns_names_and_types = query_info.send_output_columns(); + /// Choose compression. String input_compression_method_str = query_info.input_compression_type(); if (input_compression_method_str.empty()) @@ -1150,6 +1156,9 @@ namespace void Call::generateOutput() { + /// We add query_id and time_zone to the first result anyway. + addQueryDetailsToResult(); + if (!io.pipeline.initialized() || io.pipeline.pushing()) return; @@ -1189,6 +1198,9 @@ namespace return true; }; + addOutputFormatToResult(); + addOutputColumnsNamesAndTypesToResult(header); + Block block; while (check_for_cancel()) { @@ -1439,6 +1451,29 @@ namespace return false; } + void Call::addQueryDetailsToResult() + { + *result.mutable_query_id() = query_context->getClientInfo().current_query_id; + *result.mutable_time_zone() = DateLUT::instance().getTimeZone(); + } + + void Call::addOutputFormatToResult() + { + *result.mutable_output_format() = output_format; + } + + void Call::addOutputColumnsNamesAndTypesToResult(const Block & header) + { + if (!send_output_columns_names_and_types) + return; + for (const auto & column : header) + { + auto & name_and_type = *result.add_output_columns(); + *name_and_type.mutable_name() = column.name; + *name_and_type.mutable_type() = column.type->getName(); + } + } + void Call::addProgressToResult() { auto values = progress.fetchAndResetPiecewiseAtomically(); diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index f9ef14765c9..bb72c2a4010 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -13,6 +13,12 @@ #include #include +#if USE_SSL +#include +#include +#include +#endif + namespace DB { HTTPServerRequest::HTTPServerRequest(ContextPtr context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session) @@ -69,6 +75,31 @@ bool HTTPServerRequest::checkPeerConnected() const return true; } +#if USE_SSL +bool HTTPServerRequest::havePeerCertificate() const +{ + if (!secure) + return false; + + const Poco::Net::SecureStreamSocketImpl * secure_socket = dynamic_cast(socket); + if (!secure_socket) + return false; + + return secure_socket->havePeerCertificate(); +} + +Poco::Net::X509Certificate HTTPServerRequest::peerCertificate() const +{ + if (secure) + { + const Poco::Net::SecureStreamSocketImpl * secure_socket = dynamic_cast(socket); + if (secure_socket) + return secure_socket->peerCertificate(); + } + throw Poco::Net::SSLException("No certificate available"); +} +#endif + void HTTPServerRequest::readRequest(ReadBuffer & in) { char ch; diff --git a/src/Server/HTTP/HTTPServerRequest.h b/src/Server/HTTP/HTTPServerRequest.h index 33463177743..cfaeb108095 100644 --- a/src/Server/HTTP/HTTPServerRequest.h +++ b/src/Server/HTTP/HTTPServerRequest.h @@ -3,9 +3,12 @@ #include #include #include +#include #include +namespace Poco::Net { class X509Certificate; } + namespace DB { @@ -38,6 +41,11 @@ public: /// Returns the server's address. const Poco::Net::SocketAddress & serverAddress() const { return server_address; } +#if USE_SSL + bool havePeerCertificate() const; + Poco::Net::X509Certificate peerCertificate() const; +#endif + private: /// Limits for basic sanity checks when reading a header enum Limits diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index a42df54aed7..9218c75c390 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -44,6 +44,10 @@ #include #include +#if USE_SSL +#include +#endif + namespace DB { @@ -98,6 +102,7 @@ namespace ErrorCodes extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; + extern const int SUPPORT_IS_DISABLED; } namespace @@ -315,68 +320,93 @@ bool HTTPHandler::authenticateUser( std::string password = request.get("X-ClickHouse-Key", ""); std::string quota_key = request.get("X-ClickHouse-Quota", ""); + /// The header 'X-ClickHouse-SSL-Certificate-Auth: on' enables checking the common name + /// extracted from the SSL certificate used for this connection instead of checking password. + bool has_ssl_certificate_auth = (request.get("X-ClickHouse-SSL-Certificate-Auth", "") == "on"); + bool has_auth_headers = !user.empty() || !password.empty() || !quota_key.empty() || has_ssl_certificate_auth; + + /// User name and password can be passed using HTTP Basic auth or query parameters + /// (both methods are insecure). + bool has_http_credentials = request.hasCredentials(); + bool has_credentials_in_query_params = params.has("user") || params.has("password") || params.has("quota_key"); + std::string spnego_challenge; + std::string certificate_common_name; - if (user.empty() && password.empty() && quota_key.empty()) + if (has_auth_headers) { - /// User name and password can be passed using query parameters - /// or using HTTP Basic auth (both methods are insecure). - if (request.hasCredentials()) + /// It is prohibited to mix different authorization schemes. + if (has_http_credentials) + throw Exception("Invalid authentication: it is not allowed to use SSL certificate authentication and Authorization HTTP header simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + if (has_credentials_in_query_params) + throw Exception("Invalid authentication: it is not allowed to use SSL certificate authentication and authentication via parameters simultaneously simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + + if (has_ssl_certificate_auth) { - /// It is prohibited to mix different authorization schemes. - if (params.has("user") || params.has("password")) - throw Exception("Invalid authentication: it is not allowed to use Authorization HTTP header and authentication via parameters simultaneously", ErrorCodes::AUTHENTICATION_FAILED); +#if USE_SSL + if (!password.empty()) + throw Exception("Invalid authentication: it is not allowed to use SSL certificate authentication and authentication via password simultaneously", ErrorCodes::AUTHENTICATION_FAILED); - std::string scheme; - std::string auth_info; - request.getCredentials(scheme, auth_info); + if (request.havePeerCertificate()) + certificate_common_name = request.peerCertificate().commonName(); - if (Poco::icompare(scheme, "Basic") == 0) - { - HTTPBasicCredentials credentials(auth_info); - user = credentials.getUsername(); - password = credentials.getPassword(); - } - else if (Poco::icompare(scheme, "Negotiate") == 0) - { - spnego_challenge = auth_info; + if (certificate_common_name.empty()) + throw Exception("Invalid authentication: SSL certificate authentication requires nonempty certificate's Common Name", ErrorCodes::AUTHENTICATION_FAILED); +#else + throw Exception( + "SSL certificate authentication disabled because ClickHouse was built without SSL library", + ErrorCodes::SUPPORT_IS_DISABLED); +#endif + } + } + else if (has_http_credentials) + { + /// It is prohibited to mix different authorization schemes. + if (has_credentials_in_query_params) + throw Exception("Invalid authentication: it is not allowed to use Authorization HTTP header and authentication via parameters simultaneously", ErrorCodes::AUTHENTICATION_FAILED); - if (spnego_challenge.empty()) - throw Exception("Invalid authentication: SPNEGO challenge is empty", ErrorCodes::AUTHENTICATION_FAILED); - } - else - { - throw Exception("Invalid authentication: '" + scheme + "' HTTP Authorization scheme is not supported", ErrorCodes::AUTHENTICATION_FAILED); - } + std::string scheme; + std::string auth_info; + request.getCredentials(scheme, auth_info); + + if (Poco::icompare(scheme, "Basic") == 0) + { + HTTPBasicCredentials credentials(auth_info); + user = credentials.getUsername(); + password = credentials.getPassword(); + } + else if (Poco::icompare(scheme, "Negotiate") == 0) + { + spnego_challenge = auth_info; + + if (spnego_challenge.empty()) + throw Exception("Invalid authentication: SPNEGO challenge is empty", ErrorCodes::AUTHENTICATION_FAILED); } else { - user = params.get("user", "default"); - password = params.get("password", ""); + throw Exception("Invalid authentication: '" + scheme + "' HTTP Authorization scheme is not supported", ErrorCodes::AUTHENTICATION_FAILED); } quota_key = params.get("quota_key", ""); } else { - /// It is prohibited to mix different authorization schemes. - if (request.hasCredentials() || params.has("user") || params.has("password") || params.has("quota_key")) - throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + /// If the user name is not set we assume it's the 'default' user. + user = params.get("user", "default"); + password = params.get("password", ""); + quota_key = params.get("quota_key", ""); } - if (spnego_challenge.empty()) // I.e., now using user name and password strings ("Basic"). + if (!certificate_common_name.empty()) { if (!request_credentials) - request_credentials = std::make_unique(); + request_credentials = std::make_unique(user, certificate_common_name); - auto * basic_credentials = dynamic_cast(request_credentials.get()); - if (!basic_credentials) - throw Exception("Invalid authentication: unexpected 'Basic' HTTP Authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); - - basic_credentials->setUserName(user); - basic_credentials->setPassword(password); + auto * certificate_credentials = dynamic_cast(request_credentials.get()); + if (!certificate_credentials) + throw Exception("Invalid authentication: expected SSL certificate authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); } - else + else if (!spnego_challenge.empty()) { if (!request_credentials) request_credentials = server.context()->makeGSSAcceptorContext(); @@ -403,6 +433,18 @@ bool HTTPHandler::authenticateUser( return false; } } + else // I.e., now using user name and password strings ("Basic"). + { + if (!request_credentials) + request_credentials = std::make_unique(); + + auto * basic_credentials = dynamic_cast(request_credentials.get()); + if (!basic_credentials) + throw Exception("Invalid authentication: expected 'Basic' HTTP Authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); + + basic_credentials->setUserName(user); + basic_credentials->setPassword(password); + } /// Set client info. It will be used for quota accounting parameters in 'setUser' method. ClientInfo & client_info = session->getClientInfo(); @@ -1080,7 +1122,7 @@ std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLFo HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server, const std::string & config_prefix) { - const auto & query_param_name = server.config().getString(config_prefix + ".handler.query_param_name", "query"); + auto query_param_name = server.config().getString(config_prefix + ".handler.query_param_name", "query"); auto factory = std::make_shared>(server, std::move(query_param_name)); factory->addFiltersFromConfig(server.config(), config_prefix); diff --git a/src/Server/HTTPHandlerRequestFilter.h b/src/Server/HTTPHandlerRequestFilter.h index 078dcb04595..3236b35d5ae 100644 --- a/src/Server/HTTPHandlerRequestFilter.h +++ b/src/Server/HTTPHandlerRequestFilter.h @@ -40,7 +40,7 @@ static inline bool checkExpression(const StringRef & match_str, const std::pair< return match_str == expression.first; } -static inline auto methodsFilter(Poco::Util::AbstractConfiguration & config, const std::string & config_path) +static inline auto methodsFilter(Poco::Util::AbstractConfiguration & config, const std::string & config_path) /// NOLINT { std::vector methods; Poco::StringTokenizer tokenizer(config.getString(config_path), ","); @@ -64,7 +64,7 @@ static inline auto getExpression(const std::string & expression) return std::make_pair(expression, compiled_regex); } -static inline auto urlFilter(Poco::Util::AbstractConfiguration & config, const std::string & config_path) +static inline auto urlFilter(Poco::Util::AbstractConfiguration & config, const std::string & config_path) /// NOLINT { return [expression = getExpression(config.getString(config_path))](const HTTPServerRequest & request) { @@ -75,7 +75,7 @@ static inline auto urlFilter(Poco::Util::AbstractConfiguration & config, const s }; } -static inline auto headersFilter(Poco::Util::AbstractConfiguration & config, const std::string & prefix) +static inline auto headersFilter(Poco::Util::AbstractConfiguration & config, const std::string & prefix) /// NOLINT { std::unordered_map> headers_expression; Poco::Util::AbstractConfiguration::Keys headers_name; diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 07964c29577..655d17e61fa 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -202,25 +202,30 @@ struct SocketInterruptablePollWrapper #endif }; -KeeperTCPHandler::KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket_) +KeeperTCPHandler::KeeperTCPHandler( + const Poco::Util::AbstractConfiguration & config_ref, + std::shared_ptr keeper_dispatcher_, + Poco::Timespan receive_timeout_, + Poco::Timespan send_timeout_, + const Poco::Net::StreamSocket & socket_) : Poco::Net::TCPServerConnection(socket_) - , server(server_) , log(&Poco::Logger::get("KeeperTCPHandler")) - , global_context(Context::createCopy(server.context())) - , keeper_dispatcher(global_context->getKeeperDispatcher()) + , keeper_dispatcher(keeper_dispatcher_) , operation_timeout( 0, - global_context->getConfigRef().getUInt( + config_ref.getUInt( "keeper_server.coordination_settings.operation_timeout_ms", Coordination::DEFAULT_OPERATION_TIMEOUT_MS) * 1000) , min_session_timeout( 0, - global_context->getConfigRef().getUInt( + config_ref.getUInt( "keeper_server.coordination_settings.min_session_timeout_ms", Coordination::DEFAULT_MIN_SESSION_TIMEOUT_MS) * 1000) , max_session_timeout( 0, - global_context->getConfigRef().getUInt( + config_ref.getUInt( "keeper_server.coordination_settings.session_timeout_ms", Coordination::DEFAULT_MAX_SESSION_TIMEOUT_MS) * 1000) , poll_wrapper(std::make_unique(socket_)) + , send_timeout(send_timeout_) + , receive_timeout(receive_timeout_) , responses(std::make_unique(std::numeric_limits::max())) , last_op(std::make_unique(EMPTY_LAST_OP)) { @@ -289,11 +294,9 @@ void KeeperTCPHandler::runImpl() { setThreadName("KeeperHandler"); ThreadStatus thread_status; - auto global_receive_timeout = global_context->getSettingsRef().receive_timeout; - auto global_send_timeout = global_context->getSettingsRef().send_timeout; - socket().setReceiveTimeout(global_receive_timeout); - socket().setSendTimeout(global_send_timeout); + socket().setReceiveTimeout(receive_timeout); + socket().setSendTimeout(send_timeout); socket().setNoDelay(true); in = std::make_shared(socket()); @@ -544,19 +547,13 @@ std::pair KeeperTCPHandler::receiveReque void KeeperTCPHandler::packageSent() { - { - std::lock_guard lock(conn_stats_mutex); - conn_stats.incrementPacketsSent(); - } + conn_stats.incrementPacketsSent(); keeper_dispatcher->incrementPacketsSent(); } void KeeperTCPHandler::packageReceived() { - { - std::lock_guard lock(conn_stats_mutex); - conn_stats.incrementPacketsReceived(); - } + conn_stats.incrementPacketsReceived(); keeper_dispatcher->incrementPacketsReceived(); } @@ -566,10 +563,7 @@ void KeeperTCPHandler::updateStats(Coordination::ZooKeeperResponsePtr & response if (response->xid != Coordination::WATCH_XID && response->getOpNum() != Coordination::OpNum::Heartbeat) { Int64 elapsed = (Poco::Timestamp() - operations[response->xid]) / 1000; - { - std::lock_guard lock(conn_stats_mutex); - conn_stats.updateLatency(elapsed); - } + conn_stats.updateLatency(elapsed); operations.erase(response->xid); keeper_dispatcher->updateKeeperStatLatency(elapsed); @@ -584,15 +578,14 @@ void KeeperTCPHandler::updateStats(Coordination::ZooKeeperResponsePtr & response } -KeeperConnectionStats KeeperTCPHandler::getConnectionStats() const +KeeperConnectionStats & KeeperTCPHandler::getConnectionStats() { - std::lock_guard lock(conn_stats_mutex); return conn_stats; } void KeeperTCPHandler::dumpStats(WriteBufferFromOwnString & buf, bool brief) { - KeeperConnectionStats stats = getConnectionStats(); + auto & stats = getConnectionStats(); writeText(' ', buf); writeText(socket().peerAddress().toString(), buf); @@ -641,10 +634,7 @@ void KeeperTCPHandler::dumpStats(WriteBufferFromOwnString & buf, bool brief) void KeeperTCPHandler::resetStats() { - { - std::lock_guard lock(conn_stats_mutex); - conn_stats.reset(); - } + conn_stats.reset(); last_op.set(std::make_unique(EMPTY_LAST_OP)); } diff --git a/src/Server/KeeperTCPHandler.h b/src/Server/KeeperTCPHandler.h index 7953dfd2cbe..9895c335c96 100644 --- a/src/Server/KeeperTCPHandler.h +++ b/src/Server/KeeperTCPHandler.h @@ -48,19 +48,22 @@ private: static std::unordered_set connections; public: - KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket_); + KeeperTCPHandler( + const Poco::Util::AbstractConfiguration & config_ref, + std::shared_ptr keeper_dispatcher_, + Poco::Timespan receive_timeout_, + Poco::Timespan send_timeout_, + const Poco::Net::StreamSocket & socket_); void run() override; - KeeperConnectionStats getConnectionStats() const; + KeeperConnectionStats & getConnectionStats(); void dumpStats(WriteBufferFromOwnString & buf, bool brief); void resetStats(); ~KeeperTCPHandler() override; private: - IServer & server; Poco::Logger * log; - ContextPtr global_context; std::shared_ptr keeper_dispatcher; Poco::Timespan operation_timeout; Poco::Timespan min_session_timeout; @@ -69,6 +72,8 @@ private: int64_t session_id{-1}; Stopwatch session_stopwatch; SocketInterruptablePollWrapperPtr poll_wrapper; + Poco::Timespan send_timeout; + Poco::Timespan receive_timeout; ThreadSafeResponseQueuePtr responses; @@ -100,7 +105,6 @@ private: LastOpMultiVersion last_op; - mutable std::mutex conn_stats_mutex; KeeperConnectionStats conn_stats; }; diff --git a/src/Server/KeeperTCPHandlerFactory.h b/src/Server/KeeperTCPHandlerFactory.h index 58dc73d7c27..eb9f92bdd25 100644 --- a/src/Server/KeeperTCPHandlerFactory.h +++ b/src/Server/KeeperTCPHandlerFactory.h @@ -10,11 +10,17 @@ namespace DB { +using ConfigGetter = std::function; + class KeeperTCPHandlerFactory : public TCPServerConnectionFactory { private: - IServer & server; + ConfigGetter config_getter; + std::shared_ptr keeper_dispatcher; Poco::Logger * log; + Poco::Timespan receive_timeout; + Poco::Timespan send_timeout; + class DummyTCPHandler : public Poco::Net::TCPServerConnection { public: @@ -23,9 +29,17 @@ private: }; public: - KeeperTCPHandlerFactory(IServer & server_, bool secure) - : server(server_) + KeeperTCPHandlerFactory( + ConfigGetter config_getter_, + std::shared_ptr keeper_dispatcher_, + uint64_t receive_timeout_seconds, + uint64_t send_timeout_seconds, + bool secure) + : config_getter(config_getter_) + , keeper_dispatcher(keeper_dispatcher_) , log(&Poco::Logger::get(std::string{"KeeperTCP"} + (secure ? "S" : "") + "HandlerFactory")) + , receive_timeout(/* seconds = */ receive_timeout_seconds, /* microseconds = */ 0) + , send_timeout(/* seconds = */ send_timeout_seconds, /* microseconds = */ 0) { } @@ -34,7 +48,7 @@ public: try { LOG_TRACE(log, "Keeper request. Address: {}", socket.peerAddress().toString()); - return new KeeperTCPHandler(server, socket); + return new KeeperTCPHandler(config_getter(), keeper_dispatcher, receive_timeout, send_timeout, socket); } catch (const Poco::Net::NetException &) { diff --git a/src/Server/ProtocolServerAdapter.cpp b/src/Server/ProtocolServerAdapter.cpp index b41ad2376f1..dbc676432f5 100644 --- a/src/Server/ProtocolServerAdapter.cpp +++ b/src/Server/ProtocolServerAdapter.cpp @@ -1,7 +1,7 @@ #include #include -#if USE_GRPC +#if USE_GRPC && !defined(KEEPER_STANDALONE_BUILD) #include #endif @@ -37,7 +37,7 @@ ProtocolServerAdapter::ProtocolServerAdapter( { } -#if USE_GRPC +#if USE_GRPC && !defined(KEEPER_STANDALONE_BUILD) class ProtocolServerAdapter::GRPCServerAdapterImpl : public Impl { public: diff --git a/src/Server/ProtocolServerAdapter.h b/src/Server/ProtocolServerAdapter.h index 9b3b1af0301..90aec7471ee 100644 --- a/src/Server/ProtocolServerAdapter.h +++ b/src/Server/ProtocolServerAdapter.h @@ -21,7 +21,7 @@ public: ProtocolServerAdapter & operator =(ProtocolServerAdapter && src) = default; ProtocolServerAdapter(const std::string & listen_host_, const char * port_name_, const std::string & description_, std::unique_ptr tcp_server_); -#if USE_GRPC +#if USE_GRPC && !defined(KEEPER_STANDALONE_BUILD) ProtocolServerAdapter(const std::string & listen_host_, const char * port_name_, const std::string & description_, std::unique_ptr grpc_server_); #endif @@ -52,7 +52,7 @@ private: class Impl { public: - virtual ~Impl() {} + virtual ~Impl() = default; virtual void start() = 0; virtual void stop() = 0; virtual bool isStopping() const = 0; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 99523ff09e3..f4592a8b2c9 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -853,163 +852,15 @@ void TCPHandler::sendExtremes(const Block & extremes) } } - -namespace -{ - using namespace ProfileEvents; - - constexpr size_t NAME_COLUMN_INDEX = 4; - constexpr size_t VALUE_COLUMN_INDEX = 5; - - struct ProfileEventsSnapshot - { - UInt64 thread_id; - ProfileEvents::CountersIncrement counters; - Int64 memory_usage; - time_t current_time; - }; - - /* - * Add records about provided non-zero ProfileEvents::Counters. - */ - void dumpProfileEvents( - ProfileEventsSnapshot const & snapshot, - MutableColumns & columns, - String const & host_name) - { - size_t rows = 0; - auto & name_column = columns[NAME_COLUMN_INDEX]; - auto & value_column = columns[VALUE_COLUMN_INDEX]; - for (ProfileEvents::Event event = 0; event < ProfileEvents::Counters::num_counters; ++event) - { - Int64 value = snapshot.counters[event]; - - if (value == 0) - continue; - - const char * desc = ProfileEvents::getName(event); - name_column->insertData(desc, strlen(desc)); - value_column->insert(value); - rows++; - } - - // Fill the rest of the columns with data - for (size_t row = 0; row < rows; ++row) - { - size_t i = 0; - columns[i++]->insertData(host_name.data(), host_name.size()); - columns[i++]->insert(UInt64(snapshot.current_time)); - columns[i++]->insert(UInt64{snapshot.thread_id}); - columns[i++]->insert(ProfileEvents::Type::INCREMENT); - } - } - - void dumpMemoryTracker( - ProfileEventsSnapshot const & snapshot, - MutableColumns & columns, - String const & host_name) - { - { - size_t i = 0; - columns[i++]->insertData(host_name.data(), host_name.size()); - columns[i++]->insert(UInt64(snapshot.current_time)); - columns[i++]->insert(UInt64{snapshot.thread_id}); - columns[i++]->insert(ProfileEvents::Type::GAUGE); - - columns[i++]->insertData(MemoryTracker::USAGE_EVENT_NAME, strlen(MemoryTracker::USAGE_EVENT_NAME)); - columns[i++]->insert(snapshot.memory_usage); - } - } -} - - void TCPHandler::sendProfileEvents() { if (client_tcp_protocol_version < DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS) return; - NamesAndTypesList column_names_and_types = { - { "host_name", std::make_shared() }, - { "current_time", std::make_shared() }, - { "thread_id", std::make_shared() }, - { "type", ProfileEvents::TypeEnum }, - { "name", std::make_shared() }, - { "value", std::make_shared() }, - }; - - ColumnsWithTypeAndName temp_columns; - for (auto const & name_and_type : column_names_and_types) - temp_columns.emplace_back(name_and_type.type, name_and_type.name); - - Block block(std::move(temp_columns)); - - MutableColumns columns = block.mutateColumns(); - auto thread_group = CurrentThread::getGroup(); - auto const current_thread_id = CurrentThread::get().thread_id; - std::vector snapshots; - ThreadIdToCountersSnapshot new_snapshots; - ProfileEventsSnapshot group_snapshot; + Block block; + ProfileEvents::getProfileEvents(server_display_name, state.profile_queue, block, last_sent_snapshots); + if (block.rows() != 0) { - auto stats = thread_group->getProfileEventsCountersAndMemoryForThreads(); - snapshots.reserve(stats.size()); - - for (auto & stat : stats) - { - auto const thread_id = stat.thread_id; - if (thread_id == current_thread_id) - continue; - auto current_time = time(nullptr); - auto previous_snapshot = last_sent_snapshots.find(thread_id); - auto increment = - previous_snapshot != last_sent_snapshots.end() - ? CountersIncrement(stat.counters, previous_snapshot->second) - : CountersIncrement(stat.counters); - snapshots.push_back(ProfileEventsSnapshot{ - thread_id, - std::move(increment), - stat.memory_usage, - current_time - }); - new_snapshots[thread_id] = std::move(stat.counters); - } - - group_snapshot.thread_id = 0; - group_snapshot.current_time = time(nullptr); - group_snapshot.memory_usage = thread_group->memory_tracker.get(); - auto group_counters = thread_group->performance_counters.getPartiallyAtomicSnapshot(); - auto prev_group_snapshot = last_sent_snapshots.find(0); - group_snapshot.counters = - prev_group_snapshot != last_sent_snapshots.end() - ? CountersIncrement(group_counters, prev_group_snapshot->second) - : CountersIncrement(group_counters); - new_snapshots[0] = std::move(group_counters); - } - last_sent_snapshots = std::move(new_snapshots); - - for (auto & snapshot : snapshots) - { - dumpProfileEvents(snapshot, columns, server_display_name); - dumpMemoryTracker(snapshot, columns, server_display_name); - } - dumpProfileEvents(group_snapshot, columns, server_display_name); - dumpMemoryTracker(group_snapshot, columns, server_display_name); - - MutableColumns logs_columns; - Block curr_block; - size_t rows = 0; - - for (; state.profile_queue->tryPop(curr_block); ++rows) - { - auto curr_columns = curr_block.getColumns(); - for (size_t j = 0; j < curr_columns.size(); ++j) - columns[j]->insertRangeFrom(*curr_columns[j], 0, curr_columns[j]->size()); - } - - bool empty = columns[0]->empty(); - if (!empty) - { - block.setColumns(std::move(columns)); - initProfileEventsBlockOutput(block); writeVarUInt(Protocol::Server::ProfileEvents, *out); diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 6afda654e6a..153b8c35ea4 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -3,9 +3,10 @@ #include #include -#include "Common/ProfileEvents.h" +#include #include #include +#include #include #include #include @@ -13,7 +14,9 @@ #include #include #include +#include #include +#include #include @@ -36,6 +39,8 @@ struct Settings; class ColumnsDescription; struct ProfileInfo; class TCPServer; +class NativeWriter; +class NativeReader; /// State of query processing. struct QueryState @@ -189,9 +194,7 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection}; - using ThreadIdToCountersSnapshot = std::unordered_map; - - ThreadIdToCountersSnapshot last_sent_snapshots; + ProfileEvents::ThreadIdToCountersSnapshot last_sent_snapshots; /// It is the name of the server that will be sent to the client. String server_display_name; diff --git a/src/Server/TCPHandlerFactory.h b/src/Server/TCPHandlerFactory.h index 03b2592198d..6e27dfc93bd 100644 --- a/src/Server/TCPHandlerFactory.h +++ b/src/Server/TCPHandlerFactory.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include diff --git a/src/Server/grpc_protos/clickhouse_grpc.proto b/src/Server/grpc_protos/clickhouse_grpc.proto index f596c3b7d6d..4593cfff096 100644 --- a/src/Server/grpc_protos/clickhouse_grpc.proto +++ b/src/Server/grpc_protos/clickhouse_grpc.proto @@ -82,6 +82,9 @@ message QueryInfo { // Default output format. If not specified, 'TabSeparated' is used. string output_format = 7; + // Set it if you want the names and the types of output columns to be sent to the client. + bool send_output_columns = 24; + repeated ExternalTable external_tables = 8; string user_name = 9; @@ -187,7 +190,17 @@ message Exception { // Result of execution of a query which is sent back by the ClickHouse server to the client. message Result { - // Output of the query, represented in the `output_format` or in a format specified in `query`. + string query_id = 9; + string time_zone = 10; + + // The format in which `output`, `totals` and `extremes` are written. + // It's either the same as `output_format` specified in `QueryInfo` or the format specified in the query itself. + string output_format = 11; + + // The names and types of columns of the result written in `output`. + repeated NameAndType output_columns = 12; + + // Output of the query, represented in the `output_format`. bytes output = 1; bytes totals = 2; bytes extremes = 3; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 90e10abfa92..44f208adacc 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -516,8 +516,13 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) }); if (insert_it == metadata.secondary_indices.end()) - throw Exception("Wrong index name. Cannot find index " + backQuote(after_index_name) + " to insert after.", - ErrorCodes::BAD_ARGUMENTS); + { + auto hints = metadata.secondary_indices.getHints(after_index_name); + auto hints_string = !hints.empty() ? ", may be you meant: " + toString(hints) : ""; + throw Exception( + "Wrong index name. Cannot find index " + backQuote(after_index_name) + " to insert after" + hints_string, + ErrorCodes::BAD_ARGUMENTS); + } ++insert_it; } @@ -540,7 +545,10 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) { if (if_exists) return; - throw Exception("Wrong index name. Cannot find index " + backQuote(index_name) + " to drop.", ErrorCodes::BAD_ARGUMENTS); + auto hints = metadata.secondary_indices.getHints(index_name); + auto hints_string = !hints.empty() ? ", may be you meant: " + toString(hints) : ""; + throw Exception( + "Wrong index name. Cannot find index " + backQuote(index_name) + " to drop" + hints_string, ErrorCodes::BAD_ARGUMENTS); } metadata.secondary_indices.erase(erase_it); @@ -582,7 +590,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) { if (if_exists) return; - throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop.", + throw Exception("Wrong constraint name. Cannot find constraint `" + constraint_name + "` to drop", ErrorCodes::BAD_ARGUMENTS); } constraints.erase(erase_it); @@ -763,9 +771,13 @@ bool AlterCommand::isRequireMutationStage(const StorageInMemoryMetadata & metada if (isRemovingProperty() || type == REMOVE_TTL || type == REMOVE_SAMPLE_BY) return false; - if (type == DROP_COLUMN || type == DROP_INDEX || type == DROP_PROJECTION || type == RENAME_COLUMN) + if (type == DROP_INDEX || type == DROP_PROJECTION || type == RENAME_COLUMN) return true; + /// Drop alias is metadata alter, in other case mutation is required. + if (type == DROP_COLUMN) + return metadata.columns.hasPhysical(column_name); + if (type != MODIFY_COLUMN || data_type == nullptr) return false; diff --git a/src/Storages/Cache/ExternalDataSourceCache.cpp b/src/Storages/Cache/ExternalDataSourceCache.cpp index 711dfeebcae..18607c16ffa 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.cpp +++ b/src/Storages/Cache/ExternalDataSourceCache.cpp @@ -27,7 +27,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -LocalFileHolder::LocalFileHolder(RemoteFileCacheType::MappedHolderPtr cache_controller) : file_cache_controller(std::move(cache_controller)) +LocalFileHolder::LocalFileHolder(RemoteFileCacheType::MappedHolderPtr cache_controller) + : file_cache_controller(std::move(cache_controller)), original_readbuffer(nullptr), thread_pool(nullptr) { file_buffer = file_cache_controller->value().allocFile(); if (!file_buffer) @@ -35,18 +36,43 @@ LocalFileHolder::LocalFileHolder(RemoteFileCacheType::MappedHolderPtr cache_cont ErrorCodes::LOGICAL_ERROR, "Create file readbuffer failed. {}", file_cache_controller->value().getLocalPath().string()); } +LocalFileHolder::LocalFileHolder( + RemoteFileCacheType::MappedHolderPtr cache_controller, + std::unique_ptr original_readbuffer_, + BackgroundSchedulePool * thread_pool_) + : file_cache_controller(std::move(cache_controller)) + , file_buffer(nullptr) + , original_readbuffer(std::move(original_readbuffer_)) + , thread_pool(thread_pool_) +{ +} + +LocalFileHolder::~LocalFileHolder() +{ + if (original_readbuffer) + { + dynamic_cast(original_readbuffer.get())->seek(0, SEEK_SET); + file_cache_controller->value().startBackgroundDownload(std::move(original_readbuffer), *thread_pool); + } +} + RemoteReadBuffer::RemoteReadBuffer(size_t buff_size) : BufferWithOwnMemory(buff_size) { } std::unique_ptr RemoteReadBuffer::create( - ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr read_buffer, size_t buff_size) + ContextPtr context, + IRemoteFileMetadataPtr remote_file_metadata, + std::unique_ptr read_buffer, + size_t buff_size, + bool is_random_accessed) + { auto remote_path = remote_file_metadata->remote_path; auto remote_read_buffer = std::make_unique(buff_size); std::tie(remote_read_buffer->local_file_holder, read_buffer) - = ExternalDataSourceCache::instance().createReader(context, remote_file_metadata, read_buffer); + = ExternalDataSourceCache::instance().createReader(context, remote_file_metadata, read_buffer, is_random_accessed); if (remote_read_buffer->local_file_holder == nullptr) return read_buffer; remote_read_buffer->remote_file_size = remote_file_metadata->file_size; @@ -55,6 +81,19 @@ std::unique_ptr RemoteReadBuffer::create( bool RemoteReadBuffer::nextImpl() { + if (local_file_holder->original_readbuffer) + { + auto status = local_file_holder->original_readbuffer->next(); + if (status) + { + BufferBase::set( + local_file_holder->original_readbuffer->buffer().begin(), + local_file_holder->original_readbuffer->buffer().size(), + local_file_holder->original_readbuffer->offset()); + } + return status; + } + auto start_offset = local_file_holder->file_buffer->getPosition(); auto end_offset = start_offset + local_file_holder->file_buffer->internalBuffer().size(); local_file_holder->file_cache_controller->value().waitMoreData(start_offset, end_offset); @@ -73,6 +112,16 @@ bool RemoteReadBuffer::nextImpl() off_t RemoteReadBuffer::seek(off_t offset, int whence) { + if (local_file_holder->original_readbuffer) + { + auto ret = dynamic_cast(local_file_holder->original_readbuffer.get())->seek(offset, whence); + BufferBase::set( + local_file_holder->original_readbuffer->buffer().begin(), + local_file_holder->original_readbuffer->buffer().size(), + local_file_holder->original_readbuffer->offset()); + return ret; + } + if (!local_file_holder->file_buffer) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot call seek() in this buffer. It's a bug!"); /* @@ -88,6 +137,10 @@ off_t RemoteReadBuffer::seek(off_t offset, int whence) off_t RemoteReadBuffer::getPosition() { + if (local_file_holder->original_readbuffer) + { + return dynamic_cast(local_file_holder->original_readbuffer.get())->getPosition(); + } return local_file_holder->file_buffer->getPosition(); } @@ -144,7 +197,7 @@ void ExternalDataSourceCache::initOnce(ContextPtr context, const String & root_d local_cache_bytes_read_before_flush = bytes_read_before_flush_; lru_caches = std::make_unique(limit_size_); - /// create if root_dir not exists + /// Create if root_dir not exists. if (!fs::exists(fs::path(root_dir))) { fs::create_directories(fs::path(root_dir)); @@ -156,7 +209,7 @@ void ExternalDataSourceCache::initOnce(ContextPtr context, const String & root_d String ExternalDataSourceCache::calculateLocalPath(IRemoteFileMetadataPtr metadata) const { - // add version into the full_path, and not block to read the new version + // Add version into the full_path, and not block to read the new version. String full_path = metadata->getName() + ":" + metadata->remote_path + ":" + metadata->getVersion(); UInt128 hashcode = sipHash128(full_path.c_str(), full_path.size()); String hashcode_str = getHexUIntLowercase(hashcode); @@ -164,9 +217,9 @@ String ExternalDataSourceCache::calculateLocalPath(IRemoteFileMetadataPtr metada } std::pair, std::unique_ptr> ExternalDataSourceCache::createReader( - ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr & read_buffer) + ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr & read_buffer, bool is_random_accessed) { - // If something is wrong on startup, rollback to read from the original ReadBuffer + // If something is wrong on startup, rollback to read from the original ReadBuffer. if (!isInitialized()) { LOG_ERROR(log, "ExternalDataSourceCache has not been initialized"); @@ -180,7 +233,12 @@ std::pair, std::unique_ptr> Externa auto cache = lru_caches->get(local_path); if (cache) { - // the remote file has been updated, need to redownload + if (!cache->value().isEnable()) + { + return {nullptr, std::move(read_buffer)}; + } + + // The remote file has been updated, need to redownload. if (!cache->value().isValid() || cache->value().isModified(remote_file_metadata)) { LOG_TRACE( @@ -201,7 +259,7 @@ std::pair, std::unique_ptr> Externa if (!fs::exists(local_path)) fs::create_directories(local_path); - // cache is not found or is invalid, try to remove it at first + // Cache is not found or is invalid, try to remove it at first. lru_caches->tryRemove(local_path); auto new_cache_controller @@ -216,6 +274,17 @@ std::pair, std::unique_ptr> Externa lru_caches->weight()); return {nullptr, std::move(read_buffer)}; } + /* + If read_buffer is seekable, use read_buffer directly inside LocalFileHolder. And once LocalFileHolder is released, + start the download process in background. + The cache is marked disable until the download process finish. + For reading parquet files from hdfs, with this optimization, the speedup can reach 3x. + */ + if (dynamic_cast(read_buffer.get()) && is_random_accessed) + { + new_cache->value().disable(); + return {std::make_unique(std::move(new_cache), std::move(read_buffer), &context->getSchedulePool()), nullptr}; + } new_cache->value().startBackgroundDownload(std::move(read_buffer), context->getSchedulePool()); return {std::make_unique(std::move(new_cache)), nullptr}; } diff --git a/src/Storages/Cache/ExternalDataSourceCache.h b/src/Storages/Cache/ExternalDataSourceCache.h index 3e2bbb05104..5ffb2b20fc7 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.h +++ b/src/Storages/Cache/ExternalDataSourceCache.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -34,10 +34,13 @@ class LocalFileHolder { public: explicit LocalFileHolder(RemoteFileCacheType::MappedHolderPtr cache_controller); - ~LocalFileHolder() = default; + explicit LocalFileHolder(RemoteFileCacheType::MappedHolderPtr cache_controller, std::unique_ptr original_readbuffer_, BackgroundSchedulePool * thread_pool_); + ~LocalFileHolder(); RemoteFileCacheType::MappedHolderPtr file_cache_controller; std::unique_ptr file_buffer; + std::unique_ptr original_readbuffer; + BackgroundSchedulePool * thread_pool; }; class RemoteReadBuffer : public BufferWithOwnMemory @@ -45,7 +48,7 @@ class RemoteReadBuffer : public BufferWithOwnMemory public: explicit RemoteReadBuffer(size_t buff_size); ~RemoteReadBuffer() override = default; - static std::unique_ptr create(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr read_buffer, size_t buff_size); + static std::unique_ptr create(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr read_buffer, size_t buff_size, bool is_random_accessed = false); bool nextImpl() override; off_t seek(off_t off, int whence) override; @@ -70,7 +73,8 @@ public: inline bool isInitialized() const { return initialized; } std::pair, std::unique_ptr> - createReader(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr & read_buffer); + createReader(ContextPtr context, IRemoteFileMetadataPtr remote_file_metadata, std::unique_ptr & read_buffer, bool is_random_accessed); + void updateTotalSize(size_t size) { total_size += size; } @@ -78,7 +82,7 @@ protected: ExternalDataSourceCache(); private: - // root directory of local cache for remote filesystem + // Root directory of local cache for remote filesystem. String root_dir; size_t local_cache_bytes_read_before_flush = 0; diff --git a/src/Storages/Cache/IRemoteFileMetadata.h b/src/Storages/Cache/IRemoteFileMetadata.h index 0b07103b786..1d152fcdd15 100644 --- a/src/Storages/Cache/IRemoteFileMetadata.h +++ b/src/Storages/Cache/IRemoteFileMetadata.h @@ -18,7 +18,7 @@ public: // serialize virtual String toString() const = 0; - // used for comparing two file metadatas are the same or not. + // Used for comparing two file metadatas are the same or not. virtual String getVersion() const = 0; String remote_path; diff --git a/src/Storages/Cache/RemoteCacheController.cpp b/src/Storages/Cache/RemoteCacheController.cpp index b0bb31c09e7..b72f5336ea4 100644 --- a/src/Storages/Cache/RemoteCacheController.cpp +++ b/src/Storages/Cache/RemoteCacheController.cpp @@ -31,9 +31,9 @@ std::shared_ptr RemoteCacheController::recover(const std: auto cache_controller = std::make_shared(nullptr, local_path_, 0); if (cache_controller->file_status != DOWNLOADED) { - // do not load this invalid cached file and clear it. the clear action is in + // Do not load this invalid cached file and clear it. the clear action is in // ExternalDataSourceCache::recoverTask(), because deleting directories during iteration will - // cause unexpected behaviors + // cause unexpected behaviors. LOG_INFO(log, "Recover cached file failed. local path:{}", local_path_.string()); return nullptr; } @@ -76,7 +76,7 @@ RemoteCacheController::RemoteCacheController( , local_cache_bytes_read_before_flush(cache_bytes_before_flush_) , current_offset(0) { - // on recover, file_metadata_ptr is null, but it will be allocated after loading from metadata.txt + // On recover, file_metadata_ptr is null, but it will be allocated after loading from metadata.txt // when we allocate a whole new file cache，file_metadata_ptr must not be null. if (file_metadata_ptr) { @@ -106,14 +106,14 @@ void RemoteCacheController::waitMoreData(size_t start_offset_, size_t end_offset std::unique_lock lock{mutex}; if (file_status == DOWNLOADED) { - // finish reading + // Finish reading. if (start_offset_ >= current_offset) { lock.unlock(); return; } } - else // block until more data is ready + else // Block until more data is ready. { if (current_offset >= end_offset_) { @@ -169,6 +169,7 @@ void RemoteCacheController::backgroundDownload(ReadBufferPtr remote_read_buffer) file_status = DOWNLOADED; flush(true); data_file_writer.reset(); + is_enable = true; lock.unlock(); more_data_signal.notify_all(); ExternalDataSourceCache::instance().updateTotalSize(file_metadata_ptr->file_size); diff --git a/src/Storages/Cache/RemoteCacheController.h b/src/Storages/Cache/RemoteCacheController.h index 6047dbd5eb4..5f9d92c1349 100644 --- a/src/Storages/Cache/RemoteCacheController.h +++ b/src/Storages/Cache/RemoteCacheController.h @@ -29,7 +29,7 @@ public: IRemoteFileMetadataPtr file_metadata_, const std::filesystem::path & local_path_, size_t cache_bytes_before_flush_); ~RemoteCacheController(); - // recover from local disk + // Recover from local disk. static std::shared_ptr recover(const std::filesystem::path & local_path); /** @@ -40,9 +40,9 @@ public: void close(); /** - * called in LocalCachedFileReader read(), the reading process would be blocked until + * Called in LocalCachedFileReader read(), the reading process would be blocked until * enough data be downloaded. - * If the file has finished download, the process would unblocked + * If the file has finished download, the process would unblocked. */ void waitMoreData(size_t start_offset_, size_t end_offset_); @@ -63,13 +63,29 @@ public: std::lock_guard lock(mutex); return valid; } + inline bool isEnable() + { + std::lock_guard lock(mutex); + return is_enable; + + } + inline void disable() + { + std::lock_guard lock(mutex); + is_enable = false; + } + inline void enable() + { + std::lock_guard lock(mutex); + is_enable = true; + } IRemoteFileMetadataPtr getFileMetadata() { return file_metadata_ptr; } inline size_t getFileSize() const { return file_metadata_ptr->file_size; } void startBackgroundDownload(std::unique_ptr in_readbuffer_, BackgroundSchedulePool & thread_pool); private: - // flush file and status information + // Flush file and status information. void flush(bool need_flush_status = false); BackgroundSchedulePool::TaskHolder download_task_holder; @@ -79,10 +95,21 @@ private: std::condition_variable more_data_signal; String metadata_class; - LocalFileStatus file_status = TO_DOWNLOAD; // for tracking download process + LocalFileStatus file_status = TO_DOWNLOAD; // For tracking download process. IRemoteFileMetadataPtr file_metadata_ptr; std::filesystem::path local_path; + /** + * is_enable = true, only when the remotereadbuffer has been cached at local disk. + * + * The first time to access a remotebuffer which is not cached at local disk, we use the original remotebuffer directly and mark RemoteCacheController::is_enable = false. + * When the first time access is finished, LocalFileHolder will start a background download process by reusing the same remotebuffer object. After the download process + * finish, is_enable is set true. + * + * So when is_enable=false, if there is anther thread trying to access the same remote file, it would fail to use the local file buffer and use the original remotebuffer + * instead. Avoid multi threads trying to save the same file in to disk at the same time. + */ + bool is_enable = true; bool valid = true; size_t local_cache_bytes_read_before_flush; size_t current_offset; diff --git a/src/Storages/Cache/RemoteFileCachePolicy.h b/src/Storages/Cache/RemoteFileCachePolicy.h index 7d742d6ea14..5c212264bd2 100644 --- a/src/Storages/Cache/RemoteFileCachePolicy.h +++ b/src/Storages/Cache/RemoteFileCachePolicy.h @@ -1,6 +1,10 @@ #pragma once + +#include + namespace DB { + struct RemoteFileCacheWeightFunction { size_t operator()(const RemoteCacheController & cache) const { return cache.getFileSize(); } @@ -14,4 +18,5 @@ struct RemoteFileCacheReleaseFunction controller->close(); } }; + } diff --git a/src/Storages/ColumnDefault.cpp b/src/Storages/ColumnDefault.cpp index d35d3c31470..3cf49ea69fc 100644 --- a/src/Storages/ColumnDefault.cpp +++ b/src/Storages/ColumnDefault.cpp @@ -9,6 +9,7 @@ struct AliasNames static constexpr const char * DEFAULT = "DEFAULT"; static constexpr const char * MATERIALIZED = "MATERIALIZED"; static constexpr const char * ALIAS = "ALIAS"; + static constexpr const char * EPHEMERAL = "EPHEMERAL"; }; } @@ -27,7 +28,8 @@ ColumnDefaultKind columnDefaultKindFromString(const std::string & str) static const std::unordered_map map{ { AliasNames::DEFAULT, ColumnDefaultKind::Default }, { AliasNames::MATERIALIZED, ColumnDefaultKind::Materialized }, - { AliasNames::ALIAS, ColumnDefaultKind::Alias } + { AliasNames::ALIAS, ColumnDefaultKind::Alias }, + { AliasNames::EPHEMERAL, ColumnDefaultKind::Ephemeral } }; const auto it = map.find(str); @@ -43,7 +45,8 @@ std::string toString(const ColumnDefaultKind kind) static const std::unordered_map map{ { ColumnDefaultKind::Default, AliasNames::DEFAULT }, { ColumnDefaultKind::Materialized, AliasNames::MATERIALIZED }, - { ColumnDefaultKind::Alias, AliasNames::ALIAS } + { ColumnDefaultKind::Alias, AliasNames::ALIAS }, + { ColumnDefaultKind::Ephemeral, AliasNames::EPHEMERAL } }; const auto it = map.find(kind); diff --git a/src/Storages/ColumnDefault.h b/src/Storages/ColumnDefault.h index 38b61415a9a..096a1f177ab 100644 --- a/src/Storages/ColumnDefault.h +++ b/src/Storages/ColumnDefault.h @@ -13,7 +13,8 @@ enum class ColumnDefaultKind { Default, Materialized, - Alias + Alias, + Ephemeral }; diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 568257b4fd7..88f38b54f2b 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -117,7 +117,7 @@ void ColumnDescription::readText(ReadBuffer & buf) ParserColumnDeclaration column_parser(/* require type */ true); ASTPtr ast = parseQuery(column_parser, "x T " + modifiers, "column parser", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - if (const auto * col_ast = ast->as()) + if (auto * col_ast = ast->as()) { if (col_ast->default_expression) { @@ -309,7 +309,7 @@ void ColumnsDescription::flattenNested() continue; } - ColumnDescription column = std::move(*it); + ColumnDescription column = *it; removeSubcolumns(column.name); it = columns.get<0>().erase(it); @@ -340,6 +340,15 @@ NamesAndTypesList ColumnsDescription::getOrdinary() const return ret; } +NamesAndTypesList ColumnsDescription::getInsertable() const +{ + NamesAndTypesList ret; + for (const auto & col : columns) + if (col.default_desc.kind == ColumnDefaultKind::Default || col.default_desc.kind == ColumnDefaultKind::Ephemeral) + ret.emplace_back(col.name, col.type); + return ret; +} + NamesAndTypesList ColumnsDescription::getMaterialized() const { NamesAndTypesList ret; @@ -358,6 +367,15 @@ NamesAndTypesList ColumnsDescription::getAliases() const return ret; } +NamesAndTypesList ColumnsDescription::getEphemeral() const +{ + NamesAndTypesList ret; + for (const auto & col : columns) + if (col.default_desc.kind == ColumnDefaultKind::Ephemeral) + ret.emplace_back(col.name, col.type); + return ret; +} + NamesAndTypesList ColumnsDescription::getAll() const { NamesAndTypesList ret; @@ -366,6 +384,56 @@ NamesAndTypesList ColumnsDescription::getAll() const return ret; } +NamesAndTypesList ColumnsDescription::getSubcolumns(const String & name_in_storage) const +{ + auto range = subcolumns.get<1>().equal_range(name_in_storage); + return NamesAndTypesList(range.first, range.second); +} + +void ColumnsDescription::addSubcolumnsToList(NamesAndTypesList & source_list) const +{ + NamesAndTypesList subcolumns_list; + for (const auto & col : source_list) + { + auto range = subcolumns.get<1>().equal_range(col.name); + if (range.first != range.second) + subcolumns_list.insert(subcolumns_list.end(), range.first, range.second); + } + + source_list.splice(source_list.end(), std::move(subcolumns_list)); +} + +NamesAndTypesList ColumnsDescription::get(const GetColumnsOptions & options) const +{ + NamesAndTypesList res; + switch (options.kind) + { + case GetColumnsOptions::All: + res = getAll(); + break; + case GetColumnsOptions::AllPhysical: + res = getAllPhysical(); + break; + case GetColumnsOptions::Ordinary: + res = getOrdinary(); + break; + case GetColumnsOptions::Materialized: + res = getMaterialized(); + break; + case GetColumnsOptions::Aliases: + res = getAliases(); + break; + case GetColumnsOptions::Ephemeral: + res = getEphemeral(); + break; + } + + if (options.with_subcolumns) + addSubcolumnsToList(res); + + return res; +} + bool ColumnsDescription::has(const String & column_name) const { return columns.get<1>().find(column_name) != columns.get<1>().end(); @@ -392,35 +460,37 @@ const ColumnDescription & ColumnsDescription::get(const String & column_name) co return *it; } -static ColumnsDescription::GetFlags defaultKindToGetFlag(ColumnDefaultKind kind) +static GetColumnsOptions::Kind defaultKindToGetKind(ColumnDefaultKind kind) { switch (kind) { case ColumnDefaultKind::Default: - return ColumnsDescription::Ordinary; + return GetColumnsOptions::Ordinary; case ColumnDefaultKind::Materialized: - return ColumnsDescription::Materialized; + return GetColumnsOptions::Materialized; case ColumnDefaultKind::Alias: - return ColumnsDescription::Aliases; + return GetColumnsOptions::Aliases; + case ColumnDefaultKind::Ephemeral: + return GetColumnsOptions::Ephemeral; } __builtin_unreachable(); } -NamesAndTypesList ColumnsDescription::getByNames(GetFlags flags, const Names & names, bool with_subcolumns) const +NamesAndTypesList ColumnsDescription::getByNames(const GetColumnsOptions & options, const Names & names) const { NamesAndTypesList res; for (const auto & name : names) { if (auto it = columns.get<1>().find(name); it != columns.get<1>().end()) { - auto kind = defaultKindToGetFlag(it->default_desc.kind); - if (flags & kind) + auto kind = defaultKindToGetKind(it->default_desc.kind); + if (options.kind & kind) { res.emplace_back(name, it->type); continue; } } - else if (with_subcolumns) + else if (options.with_subcolumns) { auto jt = subcolumns.get<0>().find(name); if (jt != subcolumns.get<0>().end()) @@ -441,7 +511,7 @@ NamesAndTypesList ColumnsDescription::getAllPhysical() const { NamesAndTypesList ret; for (const auto & col : columns) - if (col.default_desc.kind != ColumnDefaultKind::Alias) + if (col.default_desc.kind != ColumnDefaultKind::Alias && col.default_desc.kind != ColumnDefaultKind::Ephemeral) ret.emplace_back(col.name, col.type); return ret; } @@ -450,27 +520,45 @@ Names ColumnsDescription::getNamesOfPhysical() const { Names ret; for (const auto & col : columns) - if (col.default_desc.kind != ColumnDefaultKind::Alias) + if (col.default_desc.kind != ColumnDefaultKind::Alias && col.default_desc.kind != ColumnDefaultKind::Ephemeral) ret.emplace_back(col.name); return ret; } -std::optional ColumnsDescription::tryGetColumnOrSubcolumn(GetFlags flags, const String & column_name) const +std::optional ColumnsDescription::tryGetColumn(const GetColumnsOptions & options, const String & column_name) const { auto it = columns.get<1>().find(column_name); - if (it != columns.get<1>().end() && (defaultKindToGetFlag(it->default_desc.kind) & flags)) + if (it != columns.get<1>().end() && (defaultKindToGetKind(it->default_desc.kind) & options.kind)) return NameAndTypePair(it->name, it->type); - auto jt = subcolumns.get<0>().find(column_name); - if (jt != subcolumns.get<0>().end()) - return *jt; + if (options.with_subcolumns) + { + auto jt = subcolumns.get<0>().find(column_name); + if (jt != subcolumns.get<0>().end()) + return *jt; + } return {}; } -NameAndTypePair ColumnsDescription::getColumnOrSubcolumn(GetFlags flags, const String & column_name) const +NameAndTypePair ColumnsDescription::getColumn(const GetColumnsOptions & options, const String & column_name) const { - auto column = tryGetColumnOrSubcolumn(flags, column_name); + auto column = tryGetColumn(options, column_name); + if (!column) + throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, + "There is no column {} in table.", column_name); + + return *column; +} + +std::optional ColumnsDescription::tryGetColumnOrSubcolumn(GetColumnsOptions::Kind kind, const String & column_name) const +{ + return tryGetColumn(GetColumnsOptions(kind).withSubcolumns(), column_name); +} + +NameAndTypePair ColumnsDescription::getColumnOrSubcolumn(GetColumnsOptions::Kind kind, const String & column_name) const +{ + auto column = tryGetColumnOrSubcolumn(kind, column_name); if (!column) throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no column or subcolumn {} in table.", column_name); @@ -480,11 +568,7 @@ NameAndTypePair ColumnsDescription::getColumnOrSubcolumn(GetFlags flags, const S std::optional ColumnsDescription::tryGetPhysical(const String & column_name) const { - auto it = columns.get<1>().find(column_name); - if (it == columns.get<1>().end() || it->default_desc.kind == ColumnDefaultKind::Alias) - return {}; - - return NameAndTypePair(it->name, it->type); + return tryGetColumn(GetColumnsOptions::AllPhysical, column_name); } NameAndTypePair ColumnsDescription::getPhysical(const String & column_name) const @@ -500,44 +584,18 @@ NameAndTypePair ColumnsDescription::getPhysical(const String & column_name) cons bool ColumnsDescription::hasPhysical(const String & column_name) const { auto it = columns.get<1>().find(column_name); - return it != columns.get<1>().end() && it->default_desc.kind != ColumnDefaultKind::Alias; + return it != columns.get<1>().end() && + it->default_desc.kind != ColumnDefaultKind::Alias && it->default_desc.kind != ColumnDefaultKind::Ephemeral; } -bool ColumnsDescription::hasColumnOrSubcolumn(GetFlags flags, const String & column_name) const +bool ColumnsDescription::hasColumnOrSubcolumn(GetColumnsOptions::Kind kind, const String & column_name) const { auto it = columns.get<1>().find(column_name); return (it != columns.get<1>().end() - && (defaultKindToGetFlag(it->default_desc.kind) & flags)) + && (defaultKindToGetKind(it->default_desc.kind) & kind)) || hasSubcolumn(column_name); } -void ColumnsDescription::addSubcolumnsToList(NamesAndTypesList & source_list) const -{ - NamesAndTypesList subcolumns_list; - for (const auto & col : source_list) - { - auto range = subcolumns.get<1>().equal_range(col.name); - if (range.first != range.second) - subcolumns_list.insert(subcolumns_list.end(), range.first, range.second); - } - - source_list.splice(source_list.end(), std::move(subcolumns_list)); -} - -NamesAndTypesList ColumnsDescription::getAllWithSubcolumns() const -{ - auto columns_list = getAll(); - addSubcolumnsToList(columns_list); - return columns_list; -} - -NamesAndTypesList ColumnsDescription::getAllPhysicalWithSubcolumns() const -{ - auto columns_list = getAllPhysical(); - addSubcolumnsToList(columns_list); - return columns_list; -} - bool ColumnsDescription::hasDefaults() const { for (const auto & column : columns) @@ -613,6 +671,22 @@ ColumnsDescription::ColumnTTLs ColumnsDescription::getColumnTTLs() const return ret; } +void ColumnsDescription::resetColumnTTLs() +{ + std::vector old_columns; + old_columns.reserve(columns.size()); + for (const auto & col : columns) + old_columns.emplace_back(col); + + columns.clear(); + + for (auto & col : old_columns) + { + col.ttl.reset(); + add(col); + } +} + String ColumnsDescription::toString() const { @@ -652,17 +726,16 @@ ColumnsDescription ColumnsDescription::parse(const String & str) void ColumnsDescription::addSubcolumns(const String & name_in_storage, const DataTypePtr & type_in_storage) { - for (const auto & subcolumn_name : type_in_storage->getSubcolumnNames()) + IDataType::forEachSubcolumn([&](const auto &, const auto & subname, const auto & subdata) { - auto subcolumn = NameAndTypePair(name_in_storage, subcolumn_name, - type_in_storage, type_in_storage->getSubcolumnType(subcolumn_name)); + auto subcolumn = NameAndTypePair(name_in_storage, subname, type_in_storage, subdata.type); if (has(subcolumn.name)) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot add subcolumn {}: column with this name already exists", subcolumn.name); subcolumns.get<0>().insert(std::move(subcolumn)); - } + }, {type_in_storage->getDefaultSerialization(), type_in_storage, nullptr, nullptr}); } void ColumnsDescription::removeSubcolumns(const String & name_in_storage) diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 44f895c89ce..4ae1dcfc2cd 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -28,6 +28,44 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +struct GetColumnsOptions +{ + enum Kind : UInt8 + { + Ordinary = 1, + Materialized = 2, + Aliases = 4, + Ephemeral = 8, + + AllPhysical = Ordinary | Materialized, + All = AllPhysical | Aliases | Ephemeral, + }; + + GetColumnsOptions(Kind kind_) : kind(kind_) {} + + GetColumnsOptions & withSubcolumns(bool value = true) + { + with_subcolumns = value; + return *this; + } + + GetColumnsOptions & withVirtuals(bool value = true) + { + with_virtuals = value; + return *this; + } + + GetColumnsOptions & withExtendedObjects(bool value = true) + { + with_extended_objects = value; + return *this; + } + + Kind kind; + bool with_subcolumns = false; + bool with_virtuals = false; + bool with_extended_objects = false; +}; /// Description of a single table column (in CREATE TABLE for example). struct ColumnDescription @@ -79,28 +117,21 @@ public: auto begin() const { return columns.begin(); } auto end() const { return columns.end(); } - enum GetFlags : UInt8 - { - Ordinary = 1, - Materialized = 2, - Aliases = 4, - - AllPhysical = Ordinary | Materialized, - All = AllPhysical | Aliases, - }; - - NamesAndTypesList getByNames(GetFlags flags, const Names & names, bool with_subcolumns) const; + NamesAndTypesList get(const GetColumnsOptions & options) const; + NamesAndTypesList getByNames(const GetColumnsOptions & options, const Names & names) const; NamesAndTypesList getOrdinary() const; NamesAndTypesList getMaterialized() const; + NamesAndTypesList getInsertable() const; /// ordinary + ephemeral NamesAndTypesList getAliases() const; + NamesAndTypesList getEphemeral() const; NamesAndTypesList getAllPhysical() const; /// ordinary + materialized. - NamesAndTypesList getAll() const; /// ordinary + materialized + aliases - NamesAndTypesList getAllWithSubcolumns() const; - NamesAndTypesList getAllPhysicalWithSubcolumns() const; + NamesAndTypesList getAll() const; /// ordinary + materialized + aliases + ephemeral + NamesAndTypesList getSubcolumns(const String & name_in_storage) const; using ColumnTTLs = std::unordered_map; ColumnTTLs getColumnTTLs() const; + void resetColumnTTLs(); bool has(const String & column_name) const; bool hasNested(const String & column_name) const; @@ -119,22 +150,27 @@ public: auto it = columns.get<1>().find(column_name); if (it == columns.get<1>().end()) throw Exception("Cannot find column " + column_name + " in ColumnsDescription", ErrorCodes::LOGICAL_ERROR); + + removeSubcolumns(it->name); if (!columns.get<1>().modify(it, std::forward(f))) throw Exception("Cannot modify ColumnDescription for column " + column_name + ": column name cannot be changed", ErrorCodes::LOGICAL_ERROR); + addSubcolumns(it->name, it->type); modifyColumnOrder(column_name, after_column, first); } Names getNamesOfPhysical() const; bool hasPhysical(const String & column_name) const; - bool hasColumnOrSubcolumn(GetFlags flags, const String & column_name) const; + bool hasColumnOrSubcolumn(GetColumnsOptions::Kind kind, const String & column_name) const; NameAndTypePair getPhysical(const String & column_name) const; - NameAndTypePair getColumnOrSubcolumn(GetFlags flags, const String & column_name) const; + NameAndTypePair getColumnOrSubcolumn(GetColumnsOptions::Kind kind, const String & column_name) const; + NameAndTypePair getColumn(const GetColumnsOptions & options, const String & column_name) const; std::optional tryGetPhysical(const String & column_name) const; - std::optional tryGetColumnOrSubcolumn(GetFlags flags, const String & column_name) const; + std::optional tryGetColumnOrSubcolumn(GetColumnsOptions::Kind kind, const String & column_name) const; + std::optional tryGetColumn(const GetColumnsOptions & options, const String & column_name) const; ColumnDefaults getDefaults() const; /// TODO: remove bool hasDefault(const String & column_name) const; diff --git a/src/Storages/CompressionCodecSelector.h b/src/Storages/CompressionCodecSelector.h index 746b3ce37ee..4c088924cdb 100644 --- a/src/Storages/CompressionCodecSelector.h +++ b/src/Storages/CompressionCodecSelector.h @@ -69,7 +69,7 @@ private: std::vector elements; public: - CompressionCodecSelector() {} /// Always returns the default method. + CompressionCodecSelector() = default; /// Always returns the default method. CompressionCodecSelector(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) { @@ -78,7 +78,7 @@ public: for (const auto & name : keys) { - if (!startsWith(name.data(), "case")) + if (!startsWith(name, "case")) throw Exception("Unknown element in config: " + config_prefix + "." + name + ", must be 'case'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); elements.emplace_back(config, config_prefix + "." + name); diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index 60202e2055e..7085c6e14c8 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -198,6 +198,20 @@ ConstraintsDescription & ConstraintsDescription::operator=(const ConstraintsDesc return *this; } +ConstraintsDescription::ConstraintsDescription(ConstraintsDescription && other) noexcept + : constraints(std::move(other.constraints)) +{ + update(); +} + +ConstraintsDescription & ConstraintsDescription::operator=(ConstraintsDescription && other) noexcept +{ + constraints = std::move(other.constraints); + update(); + + return *this; +} + void ConstraintsDescription::update() { if (constraints.empty()) diff --git a/src/Storages/ConstraintsDescription.h b/src/Storages/ConstraintsDescription.h index ad8bd371f38..eb1eb95d33d 100644 --- a/src/Storages/ConstraintsDescription.h +++ b/src/Storages/ConstraintsDescription.h @@ -14,10 +14,13 @@ struct ConstraintsDescription { public: ConstraintsDescription() { update(); } - ConstraintsDescription(const ASTs & constraints_); + explicit ConstraintsDescription(const ASTs & constraints_); ConstraintsDescription(const ConstraintsDescription & other); ConstraintsDescription & operator=(const ConstraintsDescription & other); + ConstraintsDescription(ConstraintsDescription && other) noexcept; + ConstraintsDescription & operator=(ConstraintsDescription && other) noexcept; + bool empty() const { return constraints.empty(); } String toString() const; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index d7422b1ddbc..d833371a742 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -219,7 +219,7 @@ namespace return distributed_header; } - /// remote_error argument is used to decide whether some errors should be + /// 'remote_error' argument is used to decide whether some errors should be /// ignored or not, in particular: /// /// - ATTEMPT_TO_READ_AFTER_EOF should not be ignored @@ -399,7 +399,7 @@ void StorageDistributedDirectoryMonitor::flushAllData() { processFiles(files); - /// Update counters + /// Update counters. getFiles(); } } @@ -475,7 +475,7 @@ void StorageDistributedDirectoryMonitor::run() break; } - /// Update counters + /// Update counters. getFiles(); if (!quit && do_sleep) @@ -491,8 +491,8 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri const auto & shards_info = cluster->getShardsInfo(); const auto & shards_addresses = cluster->getShardsAddresses(); - /// check new format shard{shard_index}_replica{replica_index} - /// (shard_index and replica_index starts from 1) + /// Check new format shard{shard_index}_replica{replica_index} + /// (shard_index and replica_index starts from 1). if (address.shard_index != 0) { if (!address.replica_index) @@ -511,7 +511,7 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri return shard_info.per_replica_pools[address.replica_index - 1]; } - /// existing connections pool have a higher priority + /// Existing connections pool have a higher priority. for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) { const Cluster::Addresses & replicas_addresses = shards_addresses[shard_index]; @@ -1152,7 +1152,7 @@ void StorageDistributedDirectoryMonitor::markAsSend(const std::string & file_pat bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) { - /// mark file as broken if necessary + /// Mark file as broken if necessary. if (isFileBrokenErrorCode(e.code(), e.isRemoteException())) { markAsBroken(file_path); diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index 307b57a5668..3c0f3b29dde 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -52,7 +52,7 @@ public: static std::shared_ptr createSourceFromFile(const String & file_name); - /// For scheduling via DistributedBlockOutputStream + /// For scheduling via DistributedBlockOutputStream. bool addAndSchedule(size_t file_size, size_t ms); struct InternalStatus diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index be0d2ea90db..aa703bcbb89 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -125,7 +126,7 @@ DistributedSink::DistributedSink( , log(&Poco::Logger::get("DistributedBlockOutputStream")) { const auto & settings = context->getSettingsRef(); - if (settings.max_distributed_depth && context->getClientInfo().distributed_depth > settings.max_distributed_depth) + if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth) throw Exception("Maximum distributed depth exceeded", ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH); context->getClientInfo().distributed_depth += 1; random_shard_insert = settings.insert_distributed_one_random_shard && !storage.has_sharding_key; @@ -331,9 +332,14 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si const Settings & settings = context->getSettingsRef(); /// Do not initiate INSERT for empty block. - if (shard_block.rows() == 0) + size_t rows = shard_block.rows(); + if (rows == 0) return; + OpenTelemetrySpanHolder span(__PRETTY_FUNCTION__); + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); + span.addAttribute("clickhouse.written_rows", rows); + if (!job.is_local_job || !settings.prefer_localhost_replica) { if (!job.executor) @@ -406,13 +412,15 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si } job.blocks_written += 1; - job.rows_written += shard_block.rows(); + job.rows_written += rows; }; } void DistributedSink::writeSync(const Block & block) { + OpenTelemetrySpanHolder span(__PRETTY_FUNCTION__); + const Settings & settings = context->getSettingsRef(); const auto & shards_info = cluster->getShardsInfo(); Block block_to_send = removeSuperfluousColumns(block); @@ -456,6 +464,10 @@ void DistributedSink::writeSync(const Block & block) size_t num_shards = end - start; + span.addAttribute("clickhouse.start_shard", start); + span.addAttribute("clickhouse.end_shard", end); + span.addAttribute("db.statement", this->query_string); + if (num_shards > 1) { auto current_selector = createSelector(block); @@ -489,6 +501,7 @@ void DistributedSink::writeSync(const Block & block) catch (Exception & exception) { exception.addMessage(getCurrentStateDescription()); + span.addAttribute(exception); throw; } @@ -597,10 +610,15 @@ void DistributedSink::writeSplitAsync(const Block & block) void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) { + OpenTelemetrySpanHolder span("DistributedBlockOutputStream::writeAsyncImpl()"); + const auto & shard_info = cluster->getShardsInfo()[shard_id]; const auto & settings = context->getSettingsRef(); Block block_to_send = removeSuperfluousColumns(block); + span.addAttribute("clickhouse.shard_num", shard_info.shard_num); + span.addAttribute("clickhouse.written_rows", block.rows()); + if (shard_info.hasInternalReplication()) { if (shard_info.isLocal() && settings.prefer_localhost_replica) @@ -634,6 +652,9 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) void DistributedSink::writeToLocal(const Block & block, size_t repeats) { + OpenTelemetrySpanHolder span(__PRETTY_FUNCTION__); + span.addAttribute("db.statement", this->query_string); + InterpreterInsertQuery interp(query_ast, context, allow_materialized); auto block_io = interp.execute(); @@ -647,6 +668,8 @@ void DistributedSink::writeToLocal(const Block & block, size_t repeats) void DistributedSink::writeToShard(const Block & block, const std::vector & dir_names) { + OpenTelemetrySpanHolder span(__PRETTY_FUNCTION__); + const auto & settings = context->getSettingsRef(); const auto & distributed_settings = storage.getDistributedSettingsRef(); @@ -713,7 +736,19 @@ void DistributedSink::writeToShard(const Block & block, const std::vectorgetSettingsRef().write(header_buf); - context->getClientInfo().write(header_buf, DBMS_TCP_PROTOCOL_VERSION); + + if (context->getClientInfo().client_trace_context.trace_id != UUID() && CurrentThread::isInitialized()) + { + // if the distributed tracing is enabled, use the trace context in current thread as parent of next span + auto client_info = context->getClientInfo(); + client_info.client_trace_context = CurrentThread::get().thread_trace_context; + client_info.write(header_buf, DBMS_TCP_PROTOCOL_VERSION); + } + else + { + context->getClientInfo().write(header_buf, DBMS_TCP_PROTOCOL_VERSION); + } + writeVarUInt(block.rows(), header_buf); writeVarUInt(block.bytes(), header_buf); writeStringBinary(block.cloneEmpty().dumpStructure(), header_buf); /// obsolete diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index 2d4b05c51b5..5549a816a06 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -34,7 +34,7 @@ IMPLEMENT_SETTINGS_TRAITS(EmptySettingsTraits, EMPTY_SETTINGS) static const std::unordered_set dictionary_allowed_keys = { "host", "port", "user", "password", "db", "database", "table", "schema", "replica", - "update_field", "update_tag", "invalidate_query", "query", + "update_field", "update_lag", "invalidate_query", "query", "where", "name", "secure", "uri", "collection"}; diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index 1e08b088b1d..cc3e136ba50 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -16,7 +16,7 @@ struct ExternalDataSourceConfiguration { String host; UInt16 port = 0; - String username; + String username = "default"; String password; String database; String table; diff --git a/src/Storages/FileLog/FileLogDirectoryWatcher.h b/src/Storages/FileLog/FileLogDirectoryWatcher.h index 0b0c86397aa..bc855a1c2fa 100644 --- a/src/Storages/FileLog/FileLogDirectoryWatcher.h +++ b/src/Storages/FileLog/FileLogDirectoryWatcher.h @@ -45,7 +45,7 @@ public: private: friend class DirectoryWatcherBase; - /// Here must pass by value, otherwise will lead to stack-use-of-scope + /// Here must pass by value, otherwise will lead to stack-use-of-scope. void onItemAdded(DirectoryWatcherBase::DirectoryEvent ev); void onItemRemoved(DirectoryWatcherBase::DirectoryEvent ev); void onItemModified(DirectoryWatcherBase::DirectoryEvent ev); diff --git a/src/Storages/FileLog/FileLogSource.cpp b/src/Storages/FileLog/FileLogSource.cpp index 7d4b5ac6fec..be818b93a4c 100644 --- a/src/Storages/FileLog/FileLogSource.cpp +++ b/src/Storages/FileLog/FileLogSource.cpp @@ -12,25 +12,24 @@ static constexpr auto MAX_FAILED_POLL_ATTEMPTS = 10; FileLogSource::FileLogSource( StorageFileLog & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const ContextPtr & context_, const Names & columns, size_t max_block_size_, size_t poll_time_out_, size_t stream_number_, size_t max_streams_number_) - : SourceWithProgress(metadata_snapshot_->getSampleBlockForColumns(columns, storage_.getVirtuals(), storage_.getStorageID())) + : SourceWithProgress(storage_snapshot_->getSampleBlockForColumns(columns)) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , context(context_) , column_names(columns) , max_block_size(max_block_size_) , poll_time_out(poll_time_out_) , stream_number(stream_number_) , max_streams_number(max_streams_number_) - , non_virtual_header(metadata_snapshot_->getSampleBlockNonMaterialized()) - , virtual_header( - metadata_snapshot->getSampleBlockForColumns(storage.getVirtualColumnNames(), storage.getVirtuals(), storage.getStorageID())) + , non_virtual_header(storage_snapshot->metadata->getSampleBlockNonMaterialized()) + , virtual_header(storage_snapshot->getSampleBlockForColumns(storage.getVirtualColumnNames())) { buffer = std::make_unique(storage, max_block_size, poll_time_out, context, stream_number_, max_streams_number_); diff --git a/src/Storages/FileLog/FileLogSource.h b/src/Storages/FileLog/FileLogSource.h index c9fd1cc5a79..831f4c907a5 100644 --- a/src/Storages/FileLog/FileLogSource.h +++ b/src/Storages/FileLog/FileLogSource.h @@ -15,7 +15,7 @@ class FileLogSource : public SourceWithProgress public: FileLogSource( StorageFileLog & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const ContextPtr & context_, const Names & columns, size_t max_block_size_, @@ -36,7 +36,7 @@ protected: private: StorageFileLog & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; ContextPtr context; Names column_names; UInt64 max_block_size; @@ -52,7 +52,7 @@ private: Block virtual_header; /// The start pos and end pos of files responsible by this stream, - /// does not include end + /// does not include end. size_t start; size_t end; }; diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index f89caaec685..32ca936f039 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -53,6 +53,7 @@ StorageFileLog::StorageFileLog( ContextPtr context_, const ColumnsDescription & columns_, const String & path_, + const String & metadata_base_path_, const String & format_name_, std::unique_ptr settings, const String & comment, @@ -61,6 +62,7 @@ StorageFileLog::StorageFileLog( , WithContext(context_->getGlobalContext()) , filelog_settings(std::move(settings)) , path(path_) + , metadata_base_path(std::filesystem::path(metadata_base_path_) / "metadata") , format_name(format_name_) , log(&Poco::Logger::get("StorageFileLog (" + table_id_.table_name + ")")) , milliseconds_to_wait(filelog_settings->poll_directory_watch_events_backoff_init.totalMilliseconds()) @@ -94,17 +96,24 @@ StorageFileLog::StorageFileLog( void StorageFileLog::loadMetaFiles(bool attach) { - const auto & storage = getStorageID(); - root_meta_path - = std::filesystem::path(getContext()->getPath()) / ".filelog_storage_metadata" / storage.getDatabaseName() / storage.getTableName(); - /// Attach table if (attach) { - /// Meta file may lost, log and create directory - if (!std::filesystem::exists(root_meta_path)) + const auto & storage = getStorageID(); + + auto metadata_path_exist = std::filesystem::exists(metadata_base_path); + auto previous_path = std::filesystem::path(getContext()->getPath()) / ".filelog_storage_metadata" / storage.getDatabaseName() / storage.getTableName(); + + /// For compatibility with the previous path version. + if (std::filesystem::exists(previous_path) && !metadata_path_exist) { - /// Create root_meta_path directory when store meta data + std::filesystem::copy(previous_path, metadata_base_path, std::filesystem::copy_options::recursive); + std::filesystem::remove_all(previous_path); + } + /// Meta file may lost, log and create directory + else if (!metadata_path_exist) + { + /// Create metadata_base_path directory when store meta data LOG_ERROR(log, "Metadata files of table {} are lost.", getStorageID().getTableName()); } /// Load all meta info to file_infos; @@ -113,14 +122,14 @@ void StorageFileLog::loadMetaFiles(bool attach) /// Create table, just create meta data directory else { - if (std::filesystem::exists(root_meta_path)) + if (std::filesystem::exists(metadata_base_path)) { throw Exception( ErrorCodes::TABLE_METADATA_ALREADY_EXISTS, "Metadata files already exist by path: {}, remove them manually if it is intended", - root_meta_path); + metadata_base_path); } - /// We do not create the root_meta_path directory at creation time, create it at the moment of serializing + /// We do not create the metadata_base_path directory at creation time, create it at the moment of serializing /// meta files, such that can avoid unnecessarily create this directory if create table failed. } } @@ -211,9 +220,9 @@ void StorageFileLog::loadFiles() void StorageFileLog::serialize() const { - if (!std::filesystem::exists(root_meta_path)) + if (!std::filesystem::exists(metadata_base_path)) { - std::filesystem::create_directories(root_meta_path); + std::filesystem::create_directories(metadata_base_path); } for (const auto & [inode, meta] : file_infos.meta_by_inode) { @@ -235,9 +244,9 @@ void StorageFileLog::serialize() const void StorageFileLog::serialize(UInt64 inode, const FileMeta & file_meta) const { - if (!std::filesystem::exists(root_meta_path)) + if (!std::filesystem::exists(metadata_base_path)) { - std::filesystem::create_directories(root_meta_path); + std::filesystem::create_directories(metadata_base_path); } auto full_name = getFullMetaPath(file_meta.file_name); if (!std::filesystem::exists(full_name)) @@ -256,11 +265,11 @@ void StorageFileLog::serialize(UInt64 inode, const FileMeta & file_meta) const void StorageFileLog::deserialize() { - if (!std::filesystem::exists(root_meta_path)) + if (!std::filesystem::exists(metadata_base_path)) return; /// In case of single file (not a watched directory), /// iterated directory always has one file inside. - for (const auto & dir_entry : std::filesystem::directory_iterator{root_meta_path}) + for (const auto & dir_entry : std::filesystem::directory_iterator{metadata_base_path}) { if (!dir_entry.is_regular_file()) { @@ -268,7 +277,7 @@ void StorageFileLog::deserialize() ErrorCodes::BAD_FILE_TYPE, "The file {} under {} is not a regular file when deserializing meta files", dir_entry.path().c_str(), - root_meta_path); + metadata_base_path); } ReadBufferFromFile in(dir_entry.path().c_str()); @@ -304,7 +313,7 @@ UInt64 StorageFileLog::getInode(const String & file_name) Pipe StorageFileLog::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /* query_info */, ContextPtr local_context, QueryProcessingStage::Enum /* processed_stage */, @@ -346,7 +355,7 @@ Pipe StorageFileLog::read( { pipes.emplace_back(std::make_shared( *this, - metadata_snapshot, + storage_snapshot, modified_context, column_names, getMaxBlockSize(), @@ -372,8 +381,8 @@ void StorageFileLog::drop() { try { - if (std::filesystem::exists(root_meta_path)) - std::filesystem::remove_all(root_meta_path); + if (std::filesystem::exists(metadata_base_path)) + std::filesystem::remove_all(metadata_base_path); } catch (...) { @@ -668,7 +677,9 @@ bool StorageFileLog::streamToViews() auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); if (!table) throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist", ErrorCodes::LOGICAL_ERROR); + auto metadata_snapshot = getInMemoryMetadataPtr(); + auto storage_snapshot = getStorageSnapshot(metadata_snapshot); auto max_streams_number = std::min(filelog_settings->max_threads.value, file_infos.file_names.size()); /// No files to parse @@ -696,7 +707,7 @@ bool StorageFileLog::streamToViews() { pipes.emplace_back(std::make_shared( *this, - metadata_snapshot, + storage_snapshot, new_context, block_io.pipeline.getHeader().getNames(), getPollMaxBatchSize(), @@ -801,6 +812,7 @@ void registerStorageFileLog(StorageFactory & factory) args.getContext(), args.columns, path, + args.relative_data_path, format, std::move(filelog_settings), args.comment, @@ -817,6 +829,9 @@ void registerStorageFileLog(StorageFactory & factory) bool StorageFileLog::updateFileInfos() { + if (file_infos.file_names.empty()) + return false; + if (!directory_watch) { /// For table just watch one file, we can not use directory monitor to watch it diff --git a/src/Storages/FileLog/StorageFileLog.h b/src/Storages/FileLog/StorageFileLog.h index db6ca386d84..ded97ecbd8c 100644 --- a/src/Storages/FileLog/StorageFileLog.h +++ b/src/Storages/FileLog/StorageFileLog.h @@ -43,7 +43,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -52,17 +52,11 @@ public: void drop() override; - /// We need to call drop() immediately to remove meta data directory, - /// otherwise, if another filelog table with same name created before - /// the table be dropped finally, then its meta data directory will - /// be deleted by this table drop finally - bool dropTableImmediately() override { return true; } - const auto & getFormatName() const { return format_name; } enum class FileStatus { - OPEN, /// first time open file after table start up + OPEN, /// First time open file after table start up. NO_CHANGE, UPDATED, REMOVED, @@ -89,13 +83,13 @@ public: { InodeToFileMeta meta_by_inode; FileNameToContext context_by_name; - /// file names without path + /// File names without path. Names file_names; }; auto & getFileInfos() { return file_infos; } - String getFullMetaPath(const String & file_name) const { return std::filesystem::path(root_meta_path) / file_name; } + String getFullMetaPath(const String & file_name) const { return std::filesystem::path(metadata_base_path) / file_name; } String getFullDataPath(const String & file_name) const { return std::filesystem::path(root_data_path) / file_name; } NamesAndTypesList getVirtuals() const override; @@ -137,6 +131,7 @@ protected: ContextPtr context_, const ColumnsDescription & columns_, const String & path_, + const String & metadata_base_path_, const String & format_name_, std::unique_ptr settings, const String & comment, @@ -151,7 +146,7 @@ private: /// If path argument of the table is a regular file, it equals to user_files_path /// otherwise, it equals to user_files_path/ + path_argument/, e.g. path String root_data_path; - String root_meta_path; + String metadata_base_path; FileInfos file_infos; @@ -205,7 +200,7 @@ private: /// Used in shutdown() void serialize() const; - /// Used in FileSource closeFileAndStoreMeta(file_name); + /// Used in FileSource closeFileAndStoreMeta(file_name). void serialize(UInt64 inode, const FileMeta & file_meta) const; void deserialize(); diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/HDFS/ReadBufferFromHDFS.cpp index 0ad55162fb2..902307fc828 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/HDFS/ReadBufferFromHDFS.cpp @@ -184,6 +184,11 @@ off_t ReadBufferFromHDFS::getPosition() return impl->getPosition() - available(); } +size_t ReadBufferFromHDFS::getFileOffsetOfBufferEnd() const +{ + return impl->getPosition(); +} + } #endif diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.h b/src/Storages/HDFS/ReadBufferFromHDFS.h index aa20e20fa48..e8cdcb27360 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.h +++ b/src/Storages/HDFS/ReadBufferFromHDFS.h @@ -39,6 +39,8 @@ public: std::optional getTotalSize() override; + size_t getFileOffsetOfBufferEnd() const override; + private: std::unique_ptr impl; }; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 7b07e929c76..74f6937dbae 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -172,23 +172,43 @@ ColumnsDescription StorageHDFS::getTableStructureFromData( const String & compression_method, ContextPtr ctx) { - auto read_buffer_creator = [&]() + const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(uri); + auto paths = getPathsList(path_from_uri, uri, ctx); + + std::string exception_messages; + bool read_buffer_creator_was_used = false; + for (const auto & path : paths) { - const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(uri); - auto paths = getPathsList(path_from_uri, uri, ctx); - if (paths.empty()) - throw Exception( - ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, - "Cannot extract table structure from {} format file, because there are no files in HDFS with provided path. You must " - "specify table structure manually", - format); + auto read_buffer_creator = [&, uri_without_path = uri_without_path]() + { + read_buffer_creator_was_used = true; - auto compression = chooseCompressionMethod(paths[0], compression_method); - return wrapReadBufferWithCompressionMethod( - std::make_unique(uri_without_path, paths[0], ctx->getGlobalContext()->getConfigRef()), compression); - }; + if (paths.empty()) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot extract table structure from {} format file, because there are no files in HDFS with provided path. You must " + "specify table structure manually", + format); - return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, ctx); + auto compression = chooseCompressionMethod(path, compression_method); + return wrapReadBufferWithCompressionMethod( + std::make_unique(uri_without_path, path, ctx->getGlobalContext()->getConfigRef()), compression); + }; + + try + { + return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, ctx); + } + catch (...) + { + if (paths.size() == 1 || !read_buffer_creator_was_used) + throw; + + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } + } + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from hdfs files failed. Errors:\n{}", exception_messages); } class HDFSSource::DisclosedGlobIterator::Impl @@ -272,16 +292,15 @@ Block HDFSSource::getHeader(const StorageMetadataPtr & metadata_snapshot, bool n Block HDFSSource::getBlockForSource( const StorageHDFSPtr & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const ColumnsDescription & columns_description, bool need_path_column, bool need_file_column) { if (storage->isColumnOriented()) - return metadata_snapshot->getSampleBlockForColumns( - columns_description.getNamesOfPhysical(), storage->getVirtuals(), storage->getStorageID()); + return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); else - return getHeader(metadata_snapshot, need_path_column, need_file_column); + return getHeader(storage_snapshot->metadata, need_path_column, need_file_column); } HDFSSource::DisclosedGlobIterator::DisclosedGlobIterator(ContextPtr context_, const String & uri) @@ -304,17 +323,17 @@ String HDFSSource::URISIterator::next() HDFSSource::HDFSSource( StorageHDFSPtr storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, ContextPtr context_, UInt64 max_block_size_, bool need_path_column_, bool need_file_column_, std::shared_ptr file_iterator_, ColumnsDescription columns_description_) - : SourceWithProgress(getBlockForSource(storage_, metadata_snapshot_, columns_description_, need_path_column_, need_file_column_)) + : SourceWithProgress(getBlockForSource(storage_, storage_snapshot_, columns_description_, need_path_column_, need_file_column_)) , WithContext(context_) , storage(std::move(storage_)) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , max_block_size(max_block_size_) , need_path_column(need_path_column_) , need_file_column(need_file_column_) @@ -345,8 +364,8 @@ bool HDFSSource::initialize() auto get_block_for_format = [&]() -> Block { if (storage->isColumnOriented()) - return metadata_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); - return metadata_snapshot->getSampleBlock(); + return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); + return storage_snapshot->metadata->getSampleBlock(); }; auto input_format = getContext()->getInputFormat(storage->format_name, *read_buf, get_block_for_format(), max_block_size); @@ -500,7 +519,7 @@ bool StorageHDFS::isColumnOriented() const Pipe StorageHDFS::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context_, QueryProcessingStage::Enum /*processed_stage*/, @@ -551,15 +570,14 @@ Pipe StorageHDFS::read( const auto get_columns_for_format = [&]() -> ColumnsDescription { if (isColumnOriented()) - return ColumnsDescription{ - metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()).getNamesAndTypesList()}; + return ColumnsDescription{storage_snapshot->getSampleBlockForColumns(column_names).getNamesAndTypesList()}; else - return metadata_snapshot->getColumns(); + return storage_snapshot->metadata->getColumns(); }; pipes.emplace_back(std::make_shared( this_ptr, - metadata_snapshot, + storage_snapshot, context_, max_block_size, need_path_column, diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 0e23e45f4c5..e87564aef32 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -24,7 +24,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -47,7 +47,7 @@ public: /// Is is useful because column oriented formats could effectively skip unknown columns /// So we can create a header of only required columns in read method and ask /// format to read only them. Note: this hack cannot be done with ordinary formats like TSV. - bool isColumnOriented() const; + bool isColumnOriented() const override; static ColumnsDescription getTableStructureFromData( const String & format, @@ -117,14 +117,14 @@ public: static Block getBlockForSource( const StorageHDFSPtr & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot_, const ColumnsDescription & columns_description, bool need_path_column, bool need_file_column); HDFSSource( StorageHDFSPtr storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, ContextPtr context_, UInt64 max_block_size_, bool need_path_column_, @@ -140,7 +140,7 @@ public: private: StorageHDFSPtr storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; UInt64 max_block_size; bool need_path_column; bool need_file_column; @@ -150,7 +150,7 @@ private: std::unique_ptr read_buf; std::unique_ptr pipeline; std::unique_ptr reader; - /// onCancel and generate can be called concurrently + /// onCancel and generate can be called concurrently. std::mutex reader_mutex; String current_path; diff --git a/src/Storages/HDFS/StorageHDFSCluster.cpp b/src/Storages/HDFS/StorageHDFSCluster.cpp index 7b370b7e63f..b039caa4330 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.cpp +++ b/src/Storages/HDFS/StorageHDFSCluster.cpp @@ -2,17 +2,9 @@ #if USE_HDFS -#include -#include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include @@ -21,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -29,16 +20,13 @@ #include #include #include -#include -#include #include -#include -#include -#include + namespace DB { + StorageHDFSCluster::StorageHDFSCluster( String cluster_name_, const String & uri_, @@ -62,7 +50,7 @@ StorageHDFSCluster::StorageHDFSCluster( /// The code executes on initiator Pipe StorageHDFSCluster::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -118,12 +106,12 @@ Pipe StorageHDFSCluster::read( } } - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); return Pipe::unitePipes(std::move(pipes)); } QueryProcessingStage::Enum StorageHDFSCluster::getQueryProcessingStage( - ContextPtr context, QueryProcessingStage::Enum to_stage, const StorageMetadataPtr &, SelectQueryInfo &) const + ContextPtr context, QueryProcessingStage::Enum to_stage, const StorageSnapshotPtr &, SelectQueryInfo &) const { /// Initiator executes query on remote node. if (context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) diff --git a/src/Storages/HDFS/StorageHDFSCluster.h b/src/Storages/HDFS/StorageHDFSCluster.h index 0e568a9faf8..953311de056 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.h +++ b/src/Storages/HDFS/StorageHDFSCluster.h @@ -24,11 +24,11 @@ class StorageHDFSCluster : public shared_ptr_helper, public public: std::string getName() const override { return "HDFSCluster"; } - Pipe read(const Names &, const StorageMetadataPtr &, SelectQueryInfo &, + Pipe read(const Names &, const StorageSnapshotPtr &, SelectQueryInfo &, ContextPtr, QueryProcessingStage::Enum, size_t /*max_block_size*/, unsigned /*num_streams*/) override; QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override; + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override; NamesAndTypesList getVirtuals() const override; diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.cpp b/src/Storages/HDFS/WriteBufferFromHDFS.cpp index 2addfc0069f..42ec3962beb 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.cpp +++ b/src/Storages/HDFS/WriteBufferFromHDFS.cpp @@ -25,7 +25,7 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl HDFSBuilderWrapper builder; HDFSFSPtr fs; - explicit WriteBufferFromHDFSImpl( + WriteBufferFromHDFSImpl( const std::string & hdfs_uri_, const Poco::Util::AbstractConfiguration & config_, int replication_, diff --git a/src/Storages/Hive/HiveCommon.cpp b/src/Storages/Hive/HiveCommon.cpp index aa19ff042e2..a9d0c22d6a5 100644 --- a/src/Storages/Hive/HiveCommon.cpp +++ b/src/Storages/Hive/HiveCommon.cpp @@ -1,3 +1,4 @@ +#include #include #if USE_HIVE @@ -5,6 +6,7 @@ #include #include #include +#include namespace DB @@ -15,6 +17,18 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +static const unsigned max_hive_metastore_client_connections = 16; +static const int max_hive_metastore_client_retry = 3; +static const UInt64 get_hive_metastore_client_timeout = 1000000; +static const int hive_metastore_client_conn_timeout_ms = 10000; +static const int hive_metastore_client_recv_timeout_ms = 10000; +static const int hive_metastore_client_send_timeout_ms = 10000; + +ThriftHiveMetastoreClientPool::ThriftHiveMetastoreClientPool(ThriftHiveMetastoreClientBuilder builder_) + : PoolBase(max_hive_metastore_client_connections, &Poco::Logger::get("ThriftHiveMetastoreClientPool")), builder(builder_) +{ +} + bool HiveMetastoreClient::shouldUpdateTableMetadata( const String & db_name, const String & table_name, const std::vector & partitions) { @@ -40,25 +54,42 @@ bool HiveMetastoreClient::shouldUpdateTableMetadata( return false; } +void HiveMetastoreClient::tryCallHiveClient(std::function func) +{ + int i = 0; + String err_msg; + for (; i < max_hive_metastore_client_retry; ++i) + { + auto client = client_pool.get(get_hive_metastore_client_timeout); + try + { + func(client); + } + catch (apache::thrift::transport::TTransportException & e) + { + client.expire(); + err_msg = e.what(); + continue; + } + break; + } + if (i >= max_hive_metastore_client_retry) + throw Exception(ErrorCodes::NO_HIVEMETASTORE, "Hive Metastore expired because {}", err_msg); +} + HiveMetastoreClient::HiveTableMetadataPtr HiveMetastoreClient::getTableMetadata(const String & db_name, const String & table_name) { LOG_TRACE(log, "Get table metadata for {}.{}", db_name, table_name); - std::lock_guard lock{mutex}; auto table = std::make_shared(); std::vector partitions; - try + auto client_call = [&](ThriftHiveMetastoreClientPool::Entry & client) { client->get_table(*table, db_name, table_name); - /// Query the latest partition info to check new change. client->get_partitions(partitions, db_name, table_name, -1); - } - catch (apache::thrift::transport::TTransportException & e) - { - setExpired(); - throw Exception(ErrorCodes::NO_HIVEMETASTORE, "Hive Metastore expired because {}", String(e.what())); - } + }; + tryCallHiveClient(client_call); bool update_cache = shouldUpdateTableMetadata(db_name, table_name, partitions); String cache_key = getCacheKey(db_name, table_name); @@ -103,23 +134,26 @@ HiveMetastoreClient::HiveTableMetadataPtr HiveMetastoreClient::getTableMetadata( return metadata; } +std::shared_ptr HiveMetastoreClient::getHiveTable(const String & db_name, const String & table_name) +{ + auto table = std::make_shared(); + auto client_call = [&](ThriftHiveMetastoreClientPool::Entry & client) + { + client->get_table(*table, db_name, table_name); + }; + tryCallHiveClient(client_call); + return table; +} + void HiveMetastoreClient::clearTableMetadata(const String & db_name, const String & table_name) { String cache_key = getCacheKey(db_name, table_name); - std::lock_guard lock{mutex}; HiveTableMetadataPtr metadata = table_metadata_cache.get(cache_key); if (metadata) table_metadata_cache.remove(cache_key); } -void HiveMetastoreClient::setClient(std::shared_ptr client_) -{ - std::lock_guard lock{mutex}; - client = client_; - clearExpired(); -} - bool HiveMetastoreClient::PartitionInfo::haveSameParameters(const Apache::Hadoop::Hive::Partition & other) const { /// Parameters include keys:numRows,numFiles,rawDataSize,totalSize,transient_lastDdlTime @@ -192,53 +226,52 @@ HiveMetastoreClientFactory & HiveMetastoreClientFactory::instance() return factory; } +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace Apache::Hadoop::Hive; + HiveMetastoreClientPtr HiveMetastoreClientFactory::getOrCreate(const String & name, ContextPtr context) { - using namespace apache::thrift; - using namespace apache::thrift::protocol; - using namespace apache::thrift::transport; - using namespace Apache::Hadoop::Hive; std::lock_guard lock(mutex); auto it = clients.find(name); - if (it == clients.end() || it->second->isExpired()) + if (it == clients.end()) { - /// Connect to hive metastore - Poco::URI hive_metastore_url(name); - const auto & host = hive_metastore_url.getHost(); - auto port = hive_metastore_url.getPort(); - - std::shared_ptr socket = std::make_shared(host, port); - socket->setKeepAlive(true); - socket->setConnTimeout(conn_timeout_ms); - socket->setRecvTimeout(recv_timeout_ms); - socket->setSendTimeout(send_timeout_ms); - std::shared_ptr transport(new TBufferedTransport(socket)); - std::shared_ptr protocol(new TBinaryProtocol(transport)); - std::shared_ptr thrift_client = std::make_shared(protocol); - try + auto builder = [name]() { - transport->open(); - } - catch (TException & tx) - { - throw Exception("connect to hive metastore:" + name + " failed." + tx.what(), ErrorCodes::BAD_ARGUMENTS); - } - - if (it == clients.end()) - { - HiveMetastoreClientPtr client = std::make_shared(std::move(thrift_client), context); - clients[name] = client; - return client; - } - else - { - it->second->setClient(std::move(thrift_client)); - return it->second; - } + return createThriftHiveMetastoreClient(name); + }; + auto client = std::make_shared(builder, context->getGlobalContext()); + clients[name] = client; + return client; } return it->second; } +std::shared_ptr HiveMetastoreClientFactory::createThriftHiveMetastoreClient(const String &name) +{ + Poco::URI hive_metastore_url(name); + const auto & host = hive_metastore_url.getHost(); + auto port = hive_metastore_url.getPort(); + + std::shared_ptr socket = std::make_shared(host, port); + socket->setKeepAlive(true); + socket->setConnTimeout(hive_metastore_client_conn_timeout_ms); + socket->setRecvTimeout(hive_metastore_client_recv_timeout_ms); + socket->setSendTimeout(hive_metastore_client_send_timeout_ms); + std::shared_ptr transport = std::make_shared(socket); + std::shared_ptr protocol = std::make_shared(transport); + std::shared_ptr thrift_client = std::make_shared(protocol); + try + { + transport->open(); + } + catch (TException & tx) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "connect to hive metastore: {} failed. {}", name, tx.what()); + } + return thrift_client; +} } #endif diff --git a/src/Storages/Hive/HiveCommon.h b/src/Storages/Hive/HiveCommon.h index e88e67b0257..b8075457a02 100644 --- a/src/Storages/Hive/HiveCommon.h +++ b/src/Storages/Hive/HiveCommon.h @@ -1,5 +1,6 @@ #pragma once +#include #include #if USE_HIVE @@ -10,12 +11,32 @@ #include #include +#include #include namespace DB { +using ThriftHiveMetastoreClientBuilder = std::function()>; + +class ThriftHiveMetastoreClientPool : public PoolBase +{ +public: + using Object = Apache::Hadoop::Hive::ThriftHiveMetastoreClient; + using ObjectPtr = std::shared_ptr; + using Entry = PoolBase::Entry; + explicit ThriftHiveMetastoreClientPool(ThriftHiveMetastoreClientBuilder builder_); + +protected: + ObjectPtr allocObject() override + { + return builder(); + } + +private: + ThriftHiveMetastoreClientBuilder builder; +}; class HiveMetastoreClient : public WithContext { public: @@ -26,7 +47,9 @@ public: UInt64 last_modify_time; /// In ms size_t size; - FileInfo() = default; + explicit FileInfo() = default; + FileInfo & operator = (const FileInfo &) = default; + FileInfo(const FileInfo &) = default; FileInfo(const String & path_, UInt64 last_modify_time_, size_t size_) : path(path_), last_modify_time(last_modify_time_), size(size_) { @@ -94,17 +117,18 @@ public: using HiveTableMetadataPtr = std::shared_ptr; - explicit HiveMetastoreClient(std::shared_ptr client_, ContextPtr context_) - : WithContext(context_), client(client_), table_metadata_cache(1000) + explicit HiveMetastoreClient(ThriftHiveMetastoreClientBuilder builder_, ContextPtr context_) + : WithContext(context_) + , table_metadata_cache(1000) + , client_pool(builder_) { } + HiveTableMetadataPtr getTableMetadata(const String & db_name, const String & table_name); + // Access hive table information by hive client + std::shared_ptr getHiveTable(const String & db_name, const String & table_name); void clearTableMetadata(const String & db_name, const String & table_name); - void setClient(std::shared_ptr client_); - bool isExpired() const { return expired; } - void setExpired() { expired = true; } - void clearExpired() { expired = false; } private: static String getCacheKey(const String & db_name, const String & table_name) { return db_name + "." + table_name; } @@ -112,10 +136,10 @@ private: bool shouldUpdateTableMetadata( const String & db_name, const String & table_name, const std::vector & partitions); - std::shared_ptr client; + void tryCallHiveClient(std::function func); + LRUCache table_metadata_cache; - mutable std::mutex mutex; - std::atomic expired{false}; + ThriftHiveMetastoreClientPool client_pool; Poco::Logger * log = &Poco::Logger::get("HiveMetastoreClient"); }; @@ -128,13 +152,11 @@ public: HiveMetastoreClientPtr getOrCreate(const String & name, ContextPtr context); + static std::shared_ptr createThriftHiveMetastoreClient(const String & name); + private: std::mutex mutex; std::map clients; - - const int conn_timeout_ms = 10000; - const int recv_timeout_ms = 10000; - const int send_timeout_ms = 10000; }; } diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index b0cfa9809e1..dffcca61a9c 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -8,8 +8,10 @@ #include #include #include +#include #include #include +#include #include #include #include diff --git a/src/Storages/Hive/HiveFile.h b/src/Storages/Hive/HiveFile.h index 63cca2562eb..6d2ba29ba0f 100644 --- a/src/Storages/Hive/HiveFile.h +++ b/src/Storages/Hive/HiveFile.h @@ -7,8 +7,6 @@ #include #include -#include -#include #include #include @@ -18,6 +16,8 @@ namespace orc { class Reader; +class Statistics; +class ColumnStatistics; } namespace parquet @@ -36,6 +36,11 @@ namespace io class RandomAccessFile; } +namespace fs +{ + class FileSystem; +} + class Buffer; } diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 3040ad23283..b95f38d4886 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -116,16 +116,16 @@ public: , compression_method(compression_method_) , max_block_size(max_block_size_) , sample_block(std::move(sample_block_)) - , to_read_block(sample_block) , columns_description(getColumnsDescription(sample_block, source_info)) , text_input_field_names(text_input_field_names_) , format_settings(getFormatSettings(getContext())) { - /// Initialize to_read_block, which is used to read data from HDFS. to_read_block = sample_block; + /// Initialize to_read_block, which is used to read data from HDFS. for (const auto & name_type : source_info->partition_name_types) { - to_read_block.erase(name_type.name); + if (to_read_block.has(name_type.name)) + to_read_block.erase(name_type.name); } /// Initialize format settings @@ -171,9 +171,13 @@ public: size_t buff_size = raw_read_buf->internalBuffer().size(); if (buff_size == 0) buff_size = DBMS_DEFAULT_BUFFER_SIZE; - remote_read_buf = RemoteReadBuffer::create(getContext(), - std::make_shared("Hive", getNameNodeCluster(hdfs_namenode_url), uri_with_path, curr_file->getSize(), curr_file->getLastModTs()), - std::move(raw_read_buf), buff_size); + remote_read_buf = RemoteReadBuffer::create( + getContext(), + std::make_shared( + "Hive", getNameNodeCluster(hdfs_namenode_url), uri_with_path, curr_file->getSize(), curr_file->getLastModTs()), + std::move(raw_read_buf), + buff_size, + format == "Parquet" || format == "ORC"); } else remote_read_buf = std::move(raw_read_buf); @@ -207,11 +211,17 @@ public: /// Enrich with partition columns. auto types = source_info->partition_name_types.getTypes(); + auto names = source_info->partition_name_types.getNames(); + auto fields = source_info->hive_files[current_idx]->getPartitionValues(); for (size_t i = 0; i < types.size(); ++i) { - auto column = types[i]->createColumnConst(num_rows, source_info->hive_files[current_idx]->getPartitionValues()[i]); - auto previous_idx = sample_block.getPositionByName(source_info->partition_name_types.getNames()[i]); - columns.insert(columns.begin() + previous_idx, column->convertToFullColumnIfConst()); + // Only add the required partition columns. partition columns are not read from readbuffer + // the column must be in sample_block, otherwise sample_block.getPositionByName(names[i]) will throw an exception + if (!sample_block.has(names[i])) + continue; + auto column = types[i]->createColumnConst(num_rows, fields[i]); + auto previous_idx = sample_block.getPositionByName(names[i]); + columns.insert(columns.begin() + previous_idx, column); } /// Enrich with virtual columns. @@ -286,14 +296,22 @@ StorageHive::StorageHive( storage_metadata.setConstraints(constraints_); storage_metadata.setComment(comment_); setInMemoryMetadata(storage_metadata); +} + +void StorageHive::lazyInitialize() +{ + std::lock_guard lock{init_mutex}; + if (has_initialized) + return; + auto hive_metastore_client = HiveMetastoreClientFactory::instance().getOrCreate(hive_metastore_url, getContext()); - auto hive_table_metadata = hive_metastore_client->getTableMetadata(hive_database, hive_table); + auto hive_table_metadata = hive_metastore_client->getHiveTable(hive_database, hive_table); - hdfs_namenode_url = getNameNodeUrl(hive_table_metadata->getTable()->sd.location); - table_schema = hive_table_metadata->getTable()->sd.cols; + hdfs_namenode_url = getNameNodeUrl(hive_table_metadata->sd.location); + table_schema = hive_table_metadata->sd.cols; - FileFormat hdfs_file_format = IHiveFile::toFileFormat(hive_table_metadata->getTable()->sd.inputFormat); + FileFormat hdfs_file_format = IHiveFile::toFileFormat(hive_table_metadata->sd.inputFormat); switch (hdfs_file_format) { case FileFormat::TEXT: @@ -331,6 +349,7 @@ StorageHive::StorageHive( } initMinMaxIndexExpression(); + has_initialized = true; } void StorageHive::initMinMaxIndexExpression() @@ -542,16 +561,45 @@ HiveFilePtr StorageHive::createHiveFileIfNeeded( } return hive_file; } +bool StorageHive::isColumnOriented() const +{ + return format_name == "Parquet" || format_name == "ORC"; +} +void StorageHive::getActualColumnsToRead(Block & sample_block, const Block & header_block, const NameSet & partition_columns) const +{ + if (!isColumnOriented()) + sample_block = header_block; + UInt32 erased_columns = 0; + for (const auto & column : partition_columns) + { + if (sample_block.has(column)) + erased_columns++; + } + if (erased_columns == sample_block.columns()) + { + for (size_t i = 0; i < header_block.columns(); ++i) + { + const auto & col = header_block.getByPosition(i); + if (!partition_columns.count(col.name)) + { + sample_block.insert(col); + break; + } + } + } +} Pipe StorageHive::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context_, QueryProcessingStage::Enum /* processed_stage */, size_t max_block_size, unsigned num_streams) { + lazyInitialize(); + HDFSBuilderWrapper builder = createHDFSBuilder(hdfs_namenode_url, context_->getGlobalContext()->getConfigRef()); HDFSFSPtr fs = createHDFSFS(builder.get()); auto hive_metastore_client = HiveMetastoreClientFactory::instance().getOrCreate(hive_metastore_url, getContext()); @@ -606,14 +654,20 @@ Pipe StorageHive::read( sources_info->table_name = hive_table; sources_info->hive_metastore_client = hive_metastore_client; sources_info->partition_name_types = partition_name_types; + + const auto & header_block = storage_snapshot->metadata->getSampleBlock(); + Block sample_block; for (const auto & column : column_names) { + sample_block.insert(header_block.getByName(column)); if (column == "_path") sources_info->need_path_column = true; if (column == "_file") sources_info->need_file_column = true; } + getActualColumnsToRead(sample_block, header_block, NameSet{partition_names.begin(), partition_names.end()}); + if (num_streams > sources_info->hive_files.size()) num_streams = sources_info->hive_files.size(); @@ -625,7 +679,7 @@ Pipe StorageHive::read( hdfs_namenode_url, format_name, compression_method, - metadata_snapshot->getSampleBlock(), + sample_block, context_, max_block_size, text_input_field_names)); diff --git a/src/Storages/Hive/StorageHive.h b/src/Storages/Hive/StorageHive.h index 9629629e057..376aab311d0 100644 --- a/src/Storages/Hive/StorageHive.h +++ b/src/Storages/Hive/StorageHive.h @@ -36,13 +36,13 @@ public: ContextPtr /* query_context */, const StorageMetadataPtr & /* metadata_snapshot */) const override { - return false; + return true; } Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -53,6 +53,8 @@ public: NamesAndTypesList getVirtuals() const override; + bool isColumnOriented() const override; + protected: friend class StorageHiveSource; StorageHive( @@ -88,12 +90,17 @@ private: HiveFilePtr createHiveFileIfNeeded(const FileInfo & file_info, const FieldVector & fields, SelectQueryInfo & query_info, ContextPtr context_); + void getActualColumnsToRead(Block & sample_block, const Block & header_block, const NameSet & partition_columns) const; + String hive_metastore_url; /// Hive database and table String hive_database; String hive_table; + std::mutex init_mutex; + bool has_initialized = false; + /// Hive table meta std::vector table_schema; Names text_input_field_names; /// Defines schema of hive file, only used when text input format is TEXT @@ -116,6 +123,8 @@ private: std::shared_ptr storage_settings; Poco::Logger * log = &Poco::Logger::get("StorageHive"); + + void lazyInitialize(); }; } diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index a923258b111..88ddde32d83 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -90,7 +90,7 @@ TableExclusiveLockHolder IStorage::lockExclusively(const String & query_id, cons Pipe IStorage::read( const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, @@ -103,18 +103,17 @@ Pipe IStorage::read( void IStorage::read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { - auto pipe = read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + auto pipe = read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); if (pipe.empty()) { - auto header = (query_info.projection ? query_info.projection->desc->metadata : metadata_snapshot) - ->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + auto header = storage_snapshot->getSampleBlockForColumns(column_names); InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info, context); } else @@ -217,14 +216,14 @@ bool IStorage::isStaticStorage() const return false; } -BackupEntries IStorage::backup(const ASTs &, ContextPtr) +BackupEntries IStorage::backupData(ContextPtr, const ASTs &) { throw Exception("Table engine " + getName() + " doesn't support backups", ErrorCodes::NOT_IMPLEMENTED); } -RestoreDataTasks IStorage::restoreFromBackup(const BackupPtr &, const String &, const ASTs &, ContextMutablePtr) +RestoreTaskPtr IStorage::restoreData(ContextMutablePtr, const ASTs &, const BackupPtr &, const String &, const StorageRestoreSettings &) { - throw Exception("Table engine " + getName() + " doesn't support restoring", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Table engine " + getName() + " doesn't support backups", ErrorCodes::NOT_IMPLEMENTED); } std::string PrewhereInfo::dump() const diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index fc79c7d174f..17e9e55455c 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -71,7 +72,9 @@ class IBackup; using BackupPtr = std::shared_ptr; class IBackupEntry; using BackupEntries = std::vector>>; -using RestoreDataTasks = std::vector>; +class IRestoreTask; +using RestoreTaskPtr = std::unique_ptr; +struct StorageRestoreSettings; struct ColumnSize { @@ -158,6 +161,10 @@ public: /// Returns true if the storage supports reading of subcolumns of complex types. virtual bool supportsSubcolumns() const { return false; } + /// Returns true if the storage supports storing of dynamic subcolumns. + /// For now it makes sense only for data type Object. + virtual bool supportsDynamicSubcolumns() const { return false; } + /// Requires squashing small blocks to large for optimal storage. /// This is true for most storages that store data on disk. virtual bool prefersLargeBlocks() const { return true; } @@ -211,11 +218,14 @@ public: NameDependencies getDependentViewsByColumn(ContextPtr context) const; + /// Returns true if the backup is hollow, which means it doesn't contain any data. + virtual bool hasDataToBackup() const { return false; } + /// Prepares entries to backup data of the storage. - virtual BackupEntries backup(const ASTs & partitions, ContextPtr context); + virtual BackupEntries backupData(ContextPtr context, const ASTs & partitions); /// Extract data from the backup and put it to the storage. - virtual RestoreDataTasks restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context); + virtual RestoreTaskPtr restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings & restore_settings); /// Returns whether the column is virtual - by default all columns are real. /// Initially reserved virtual column name may be shadowed by real column. @@ -269,8 +279,7 @@ public: * QueryProcessingStage::Enum required for Distributed over Distributed, * since it cannot return Complete for intermediate queries never. */ - virtual QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const + virtual QueryProcessingStage::Enum getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const { return QueryProcessingStage::FetchColumns; } @@ -331,7 +340,7 @@ public: */ virtual Pipe read( const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, @@ -343,7 +352,7 @@ public: virtual void read( QueryPlan & query_plan, const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, @@ -562,6 +571,8 @@ public: /// Returns true if all disks of storage are read-only. virtual bool isStaticStorage() const; + virtual bool isColumnOriented() const { return false; } + /// If it is possible to quickly determine exact number of rows in the table at this moment of time, then return it. /// Used for: /// - Simple count() optimization @@ -598,9 +609,17 @@ public: /// Does not takes underlying Storage (if any) into account. virtual std::optional lifetimeBytes() const { return {}; } - /// Should table->drop be called at once or with delay (in case of atomic database engine). - /// Needed for integration engines, when there must be no delay for calling drop() method. - virtual bool dropTableImmediately() { return false; } + /// Creates a storage snapshot from given metadata. + virtual StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const + { + return std::make_shared(*this, metadata_snapshot); + } + + /// Creates a storage snapshot from given metadata and columns, which are used in query. + virtual StorageSnapshotPtr getStorageSnapshotForQuery(const StorageMetadataPtr & metadata_snapshot, const ASTPtr & /*query*/) const + { + return getStorageSnapshot(metadata_snapshot); + } private: /// Lock required for alter queries (lockForAlter). diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 9cdf9695172..a0a1bcbce2d 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -172,4 +172,13 @@ ExpressionActionsPtr IndicesDescription::getSingleExpressionForIndices(const Col return ExpressionAnalyzer(combined_expr_list, syntax_result, context).getActions(false); } +Names IndicesDescription::getAllRegisteredNames() const +{ + Names result; + for (const auto & index : *this) + { + result.emplace_back(index.name); + } + return result; +} } diff --git a/src/Storages/IndicesDescription.h b/src/Storages/IndicesDescription.h index 7071ec89890..72e0748778f 100644 --- a/src/Storages/IndicesDescription.h +++ b/src/Storages/IndicesDescription.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB { @@ -61,7 +62,7 @@ struct IndexDescription }; /// All secondary indices in storage -struct IndicesDescription : public std::vector +struct IndicesDescription : public std::vector, IHints<1, IndicesDescription> { /// Index with name exists bool has(const String & name) const; @@ -72,6 +73,9 @@ struct IndicesDescription : public std::vector /// Return common expression for all stored indices ExpressionActionsPtr getSingleExpressionForIndices(const ColumnsDescription & columns, ContextPtr context) const; + +public: + Names getAllRegisteredNames() const override; }; } diff --git a/src/Storages/Kafka/KafkaSource.cpp b/src/Storages/Kafka/KafkaSource.cpp index 3e24608a180..99130f615f5 100644 --- a/src/Storages/Kafka/KafkaSource.cpp +++ b/src/Storages/Kafka/KafkaSource.cpp @@ -19,22 +19,22 @@ const auto MAX_FAILED_POLL_ATTEMPTS = 10; KafkaSource::KafkaSource( StorageKafka & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const ContextPtr & context_, const Names & columns, Poco::Logger * log_, size_t max_block_size_, bool commit_in_suffix_) - : SourceWithProgress(metadata_snapshot_->getSampleBlockForColumns(columns, storage_.getVirtuals(), storage_.getStorageID())) + : SourceWithProgress(storage_snapshot_->getSampleBlockForColumns(columns)) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , context(context_) , column_names(columns) , log(log_) , max_block_size(max_block_size_) , commit_in_suffix(commit_in_suffix_) - , non_virtual_header(metadata_snapshot->getSampleBlockNonMaterialized()) - , virtual_header(metadata_snapshot->getSampleBlockForColumns(storage.getVirtualColumnNames(), storage.getVirtuals(), storage.getStorageID())) + , non_virtual_header(storage_snapshot->metadata->getSampleBlockNonMaterialized()) + , virtual_header(storage_snapshot->getSampleBlockForColumns(storage.getVirtualColumnNames())) , handle_error_mode(storage.getHandleKafkaErrorMode()) { } diff --git a/src/Storages/Kafka/KafkaSource.h b/src/Storages/Kafka/KafkaSource.h index e80edfb9ef4..59b6d370b71 100644 --- a/src/Storages/Kafka/KafkaSource.h +++ b/src/Storages/Kafka/KafkaSource.h @@ -18,7 +18,7 @@ class KafkaSource : public SourceWithProgress public: KafkaSource( StorageKafka & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const ContextPtr & context_, const Names & columns, Poco::Logger * log_, @@ -35,7 +35,7 @@ public: private: StorageKafka & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; ContextPtr context; Names column_names; Poco::Logger * log; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 30acbcdf62b..4c7465d587d 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -263,7 +263,7 @@ String StorageKafka::getDefaultClientId(const StorageID & table_id_) Pipe StorageKafka::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /* query_info */, ContextPtr local_context, QueryProcessingStage::Enum /* processed_stage */, @@ -291,7 +291,7 @@ Pipe StorageKafka::read( /// Use block size of 1, otherwise LIMIT won't work properly as it will buffer excess messages in the last block /// TODO: probably that leads to awful performance. /// FIXME: seems that doesn't help with extra reading and committing unprocessed messages. - pipes.emplace_back(std::make_shared(*this, metadata_snapshot, modified_context, column_names, log, 1, kafka_settings->kafka_commit_on_select)); + pipes.emplace_back(std::make_shared(*this, storage_snapshot, modified_context, column_names, log, 1, kafka_settings->kafka_commit_on_select)); } LOG_DEBUG(log, "Starting reading {} streams", pipes.size()); @@ -405,7 +405,7 @@ ProducerBufferPtr StorageKafka::createWriteBuffer(const Block & header) } -ConsumerBufferPtr StorageKafka::createReadBuffer(const size_t consumer_number) +ConsumerBufferPtr StorageKafka::createReadBuffer(size_t consumer_number) { cppkafka::Configuration conf; @@ -614,7 +614,8 @@ bool StorageKafka::streamToViews() auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); if (!table) throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); - auto metadata_snapshot = getInMemoryMetadataPtr(); + + auto storage_snapshot = getStorageSnapshot(getInMemoryMetadataPtr()); // Create an INSERT query for streaming data auto insert = std::make_shared(); @@ -640,7 +641,7 @@ bool StorageKafka::streamToViews() pipes.reserve(stream_count); for (size_t i = 0; i < stream_count; ++i) { - auto source = std::make_shared(*this, metadata_snapshot, kafka_context, block_io.pipeline.getHeader().getNames(), log, block_size, false); + auto source = std::make_shared(*this, storage_snapshot, kafka_context, block_io.pipeline.getHeader().getNames(), log, block_size, false); sources.emplace_back(source); pipes.emplace_back(source); diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h index 62de3e5183d..707db7a798e 100644 --- a/src/Storages/Kafka/StorageKafka.h +++ b/src/Storages/Kafka/StorageKafka.h @@ -43,7 +43,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -120,7 +120,7 @@ private: HandleKafkaErrorMode handle_error_mode; SettingsChanges createSettingsAdjustments(); - ConsumerBufferPtr createReadBuffer(const size_t consumer_number); + ConsumerBufferPtr createReadBuffer(size_t consumer_number); /// If named_collection is specified. String collection_name; diff --git a/src/Storages/LiveView/LiveViewEventsSource.h b/src/Storages/LiveView/LiveViewEventsSource.h index 77ee06c702c..1f9f8bfb785 100644 --- a/src/Storages/LiveView/LiveViewEventsSource.h +++ b/src/Storages/LiveView/LiveViewEventsSource.h @@ -44,7 +44,7 @@ public: : SourceWithProgress({ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "version")}), storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), - active_ptr(std::move(active_ptr_)), has_limit(has_limit_), + active_ptr(active_ptr_), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000) { diff --git a/src/Storages/LiveView/LiveViewSource.h b/src/Storages/LiveView/LiveViewSource.h index ec726359581..8d63890f603 100644 --- a/src/Storages/LiveView/LiveViewSource.h +++ b/src/Storages/LiveView/LiveViewSource.h @@ -26,7 +26,7 @@ public: : SourceWithProgress(storage_->getHeader()) , storage(std::move(storage_)), blocks_ptr(std::move(blocks_ptr_)), blocks_metadata_ptr(std::move(blocks_metadata_ptr_)), - active_ptr(std::move(active_ptr_)), + active_ptr(active_ptr_), has_limit(has_limit_), limit(limit_), heartbeat_interval_usec(heartbeat_interval_sec_ * 1000000) { diff --git a/src/Storages/LiveView/StorageBlocks.h b/src/Storages/LiveView/StorageBlocks.h index b87d3f051d0..bc860a1fa3c 100644 --- a/src/Storages/LiveView/StorageBlocks.h +++ b/src/Storages/LiveView/StorageBlocks.h @@ -18,9 +18,9 @@ public: QueryProcessingStage::Enum to_stage_) : IStorage(table_id_), pipes(std::move(pipes_)), to_stage(to_stage_) { - StorageInMemoryMetadata metadata_; - metadata_.setColumns(columns_); - setInMemoryMetadata(metadata_); + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns_); + setInMemoryMetadata(storage_metadata); } static StoragePtr createStorage(const StorageID & table_id, const ColumnsDescription & columns, Pipes pipes, QueryProcessingStage::Enum to_stage) @@ -34,14 +34,14 @@ public: bool supportsFinal() const override { return true; } QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override { return to_stage; } Pipe read( const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 83578e3b5b9..8f80f8632cc 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -540,7 +540,7 @@ void StorageLiveView::refresh(bool grab_lock) Pipe StorageLiveView::read( const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h index 17e2f50e7ec..2fb2ec509fa 100644 --- a/src/Storages/LiveView/StorageLiveView.h +++ b/src/Storages/LiveView/StorageLiveView.h @@ -146,7 +146,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.h b/src/Storages/LiveView/TemporaryLiveViewCleaner.h index 3fe0079a46f..9cc5933eb89 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.h +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.h @@ -31,7 +31,7 @@ public: private: friend std::unique_ptr::deleter_type; - TemporaryLiveViewCleaner(ContextMutablePtr global_context_); + explicit TemporaryLiveViewCleaner(ContextMutablePtr global_context_); ~TemporaryLiveViewCleaner(); void backgroundThreadFunc(); diff --git a/src/Storages/MarkCache.h b/src/Storages/MarkCache.h index 06143e954f8..a3f92650426 100644 --- a/src/Storages/MarkCache.h +++ b/src/Storages/MarkCache.h @@ -40,7 +40,7 @@ private: using Base = LRUCache; public: - MarkCache(size_t max_size_in_bytes) + explicit MarkCache(size_t max_size_in_bytes) : Base(max_size_in_bytes) {} /// Calculate key from path to file and offset. diff --git a/src/Storages/MergeTree/ActiveDataPartSet.h b/src/Storages/MergeTree/ActiveDataPartSet.h index 0b747ab83b9..8ab03625d5c 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.h +++ b/src/Storages/MergeTree/ActiveDataPartSet.h @@ -22,15 +22,14 @@ using Strings = std::vector; class ActiveDataPartSet { public: - ActiveDataPartSet(MergeTreeDataFormatVersion format_version_) : format_version(format_version_) {} + explicit ActiveDataPartSet(MergeTreeDataFormatVersion format_version_) : format_version(format_version_) {} ActiveDataPartSet(MergeTreeDataFormatVersion format_version_, const Strings & names); - ActiveDataPartSet(const ActiveDataPartSet & other) - : format_version(other.format_version) - , part_info_to_name(other.part_info_to_name) - {} + ActiveDataPartSet(const ActiveDataPartSet & other) = default; - ActiveDataPartSet(ActiveDataPartSet && other) noexcept { swap(other); } + ActiveDataPartSet & operator=(const ActiveDataPartSet & other) = default; + + ActiveDataPartSet(ActiveDataPartSet && other) noexcept = default; void swap(ActiveDataPartSet & other) noexcept { @@ -38,16 +37,6 @@ public: std::swap(part_info_to_name, other.part_info_to_name); } - ActiveDataPartSet & operator=(const ActiveDataPartSet & other) - { - if (&other != this) - { - ActiveDataPartSet tmp(other); - swap(tmp); - } - return *this; - } - /// Returns true if the part was actually added. If out_replaced_parts != nullptr, it will contain /// parts that were replaced from the set by the newly added part. bool add(const String & name, Strings * out_replaced_parts = nullptr); diff --git a/src/Storages/MergeTree/AllMergeSelector.cpp b/src/Storages/MergeTree/AllMergeSelector.cpp index 79080df1570..5e406c6e4f7 100644 --- a/src/Storages/MergeTree/AllMergeSelector.cpp +++ b/src/Storages/MergeTree/AllMergeSelector.cpp @@ -8,7 +8,7 @@ namespace DB AllMergeSelector::PartsRange AllMergeSelector::select( const PartsRanges & parts_ranges, - const size_t /*max_total_size_to_merge*/) + size_t /*max_total_size_to_merge*/) { size_t min_partition_size = 0; PartsRanges::const_iterator best_partition; diff --git a/src/Storages/MergeTree/AllMergeSelector.h b/src/Storages/MergeTree/AllMergeSelector.h index d3b399b2fc5..6cd3bb6f3fa 100644 --- a/src/Storages/MergeTree/AllMergeSelector.h +++ b/src/Storages/MergeTree/AllMergeSelector.h @@ -13,7 +13,7 @@ public: /// Parameter max_total_size_to_merge is ignored. PartsRange select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) override; + size_t max_total_size_to_merge) override; }; } diff --git a/src/Storages/MergeTree/BackgroundProcessList.h b/src/Storages/MergeTree/BackgroundProcessList.h index 81aded5e45c..baf3e281257 100644 --- a/src/Storages/MergeTree/BackgroundProcessList.h +++ b/src/Storages/MergeTree/BackgroundProcessList.h @@ -26,7 +26,7 @@ public: BackgroundProcessListEntry(const BackgroundProcessListEntry &) = delete; BackgroundProcessListEntry & operator=(const BackgroundProcessListEntry &) = delete; - BackgroundProcessListEntry(BackgroundProcessListEntry &&) = default; + BackgroundProcessListEntry(BackgroundProcessListEntry &&) noexcept = default; BackgroundProcessListEntry(BackgroundProcessList & list_, const typename container_t::iterator it_, const CurrentMetrics::Metric & metric) : list(list_), it{it_}, metric_increment{metric} diff --git a/src/Storages/MergeTree/BoolMask.h b/src/Storages/MergeTree/BoolMask.h index c26a0ed6c58..11f9238aa28 100644 --- a/src/Storages/MergeTree/BoolMask.h +++ b/src/Storages/MergeTree/BoolMask.h @@ -6,7 +6,7 @@ struct BoolMask bool can_be_true = false; bool can_be_false = false; - BoolMask() {} + BoolMask() = default; BoolMask(bool can_be_true_, bool can_be_false_) : can_be_true(can_be_true_), can_be_false(can_be_false_) {} BoolMask operator &(const BoolMask & m) const diff --git a/src/Storages/MergeTree/ColumnSizeEstimator.h b/src/Storages/MergeTree/ColumnSizeEstimator.h index 61c0ac64dbd..597dc80e525 100644 --- a/src/Storages/MergeTree/ColumnSizeEstimator.h +++ b/src/Storages/MergeTree/ColumnSizeEstimator.h @@ -1,6 +1,7 @@ #pragma once -#include "Storages/MergeTree/IMergeTreeDataPart.h" +#include +#include namespace DB diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index d2dcf8c1abd..4e7dcc60696 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -314,6 +314,10 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part, WriteBuffer & out) { + auto disk = part->volume->getDisk(); + if (!disk->supportZeroCopyReplication()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Disk '{}' doesn't support zero-copy replication", disk->getName()); + /// We'll take a list of files from the list of checksums. MergeTreeData::DataPart::Checksums checksums = part->checksums; /// Add files that are not in the checksum list. @@ -321,11 +325,13 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part for (const auto & file_name : file_names_without_checksums) checksums.files[file_name] = {}; - auto disk = part->volume->getDisk(); - if (!disk->supportZeroCopyReplication()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "disk {} doesn't support zero-copy replication", disk->getName()); + std::vector paths; + paths.reserve(checksums.files.size()); + for (const auto & it : checksums.files) + paths.push_back(fs::path(part->getFullRelativePath()) / it.first); - part->storage.lockSharedData(*part); + /// Serialized metadatadatas with zero ref counts. + auto metadatas = disk->getSerializedMetadata(paths); String part_id = part->getUniqueId(); writeStringBinary(part_id, out); @@ -333,29 +339,32 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part writeBinary(checksums.files.size(), out); for (const auto & it : checksums.files) { - String file_name = it.first; - - String metadata_file = fs::path(disk->getPath()) / part->getFullRelativePath() / file_name; - - fs::path metadata(metadata_file); + const String & file_name = it.first; + String file_path_prefix = fs::path(part->getFullRelativePath()) / file_name; + /// Just some additional checks + String metadata_file_path = fs::path(disk->getPath()) / file_path_prefix; + fs::path metadata(metadata_file_path); if (!fs::exists(metadata)) throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not exists", file_name); if (!fs::is_regular_file(metadata)) throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not a file", file_name); - UInt64 file_size = fs::file_size(metadata); + + /// Actual metadata send + auto metadata_str = metadatas[file_path_prefix]; + UInt64 file_size = metadata_str.size(); + ReadBufferFromString buf(metadata_str); writeStringBinary(it.first, out); writeBinary(file_size, out); - auto file_in = createReadBufferFromFileBase(metadata_file, /* settings= */ {}); HashingWriteBuffer hashing_out(out); - copyDataWithThrottler(*file_in, hashing_out, blocker.getCounter(), data.getSendsThrottler()); + copyDataWithThrottler(buf, hashing_out, blocker.getCounter(), data.getSendsThrottler()); if (blocker.isCancelled()) throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (hashing_out.count() != file_size) - throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", metadata_file); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", metadata_file_path); writePODBinary(hashing_out.getHash(), out); } @@ -767,9 +776,12 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( { throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()); } + LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.", part_name, part_id, disk->getName()); + data.lockSharedDataTemporary(part_name, part_id, disk); + static const String TMP_PREFIX = "tmp-fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; @@ -834,7 +846,10 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - new_data_part->storage.lockSharedData(*new_data_part); + data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true); + + LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", + part_name, part_id, disk->getName()); return new_data_part; } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 0c12cc51cc7..8dfcfef9a8b 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -63,7 +63,7 @@ private: class Fetcher final : private boost::noncopyable { public: - explicit Fetcher(MergeTreeData & data_) : data(data_), log(&Poco::Logger::get("Fetcher")) {} + explicit Fetcher(StorageReplicatedMergeTree & data_) : data(data_), log(&Poco::Logger::get("Fetcher")) {} /// Downloads a part to tmp_directory. If to_detached - downloads to the `detached` directory. MergeTreeData::MutableDataPartPtr fetchPart( @@ -129,7 +129,7 @@ private: PooledReadWriteBufferFromHTTP & in, ThrottlerPtr throttler); - MergeTreeData & data; + StorageReplicatedMergeTree & data; Poco::Logger * log; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index f1b66057306..9028023dc80 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1187,16 +1187,7 @@ std::optional IMergeTreeDataPart::keepSharedDataInDecoupledStorage() const if (force_keep_shared_data) return true; - /// TODO Unlocking in try-catch and ignoring exception look ugly - try - { - return !storage.unlockSharedData(*this); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__, "There is a problem with deleting part " + name + " from filesystem"); - } - return {}; + return !storage.unlockSharedData(*this); } void IMergeTreeDataPart::remove() const @@ -1642,18 +1633,10 @@ String IMergeTreeDataPart::getUniqueId() const if (!disk->supportZeroCopyReplication()) throw Exception(fmt::format("Disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); - String id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); - return id; + return disk->getUniqueId(fs::path(getFullRelativePath()) / FILE_FOR_REFERENCES_CHECK); } - -UInt32 IMergeTreeDataPart::getNumberOfRefereneces() const -{ - return volume->getDisk()->getRefCount(fs::path(getFullRelativePath()) / "checksums.txt"); -} - - -String IMergeTreeDataPart::getZeroLevelPartBlockID(const std::string_view token) const +String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const { if (info.level != 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get block id for non zero level part {}", name); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index b6e3b1c0e9d..85088b10f70 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -408,6 +408,18 @@ public: /// (number of rows, number of rows with default values, etc). static inline constexpr auto SERIALIZATION_FILE_NAME = "serialization.json"; + /// One of part files which is used to check how many references (I'd like + /// to say hardlinks, but it will confuse even more) we have for the part + /// for zero copy replication. Sadly it's very complex. + /// + /// NOTE: it's not a random "metadata" file for part like 'columns.txt'. If + /// two relative parts (for example all_1_1_0 and all_1_1_0_100) has equal + /// checksums.txt it means that one part was obtained by FREEZE operation or + /// it was mutation without any change for source part. In this case we + /// really don't need to remove data from remote FS and need only decrement + /// reference counter locally. + static inline constexpr auto FILE_FOR_REFERENCES_CHECK = "checksums.txt"; + /// Checks that all TTLs (table min/max, column ttls, so on) for part /// calculated. Part without calculated TTL may exist if TTL was added after /// part creation (using alter query with materialize_ttl setting). @@ -417,10 +429,6 @@ public: /// Required for distinguish different copies of the same part on remote FS. String getUniqueId() const; - /// Return hardlink count for part. - /// Required for keep data on remote FS when part has shadow copies. - UInt32 getNumberOfRefereneces() const; - protected: /// Total size of all columns, calculated once in calcuateColumnSizesOnDisk diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 79186402027..3a823345dda 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -62,101 +62,13 @@ const IMergeTreeReader::ValueSizeMap & IMergeTreeReader::getAvgValueSizeHints() return avg_value_size_hints; } - -static bool arrayHasNoElementsRead(const IColumn & column) -{ - const auto * column_array = typeid_cast(&column); - - if (!column_array) - return false; - - size_t size = column_array->size(); - if (!size) - return false; - - size_t data_size = column_array->getData().size(); - if (data_size) - return false; - - size_t last_offset = column_array->getOffsets()[size - 1]; - return last_offset != 0; -} - -void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) +void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) const { try { - size_t num_columns = columns.size(); - - if (res_columns.size() != num_columns) - throw Exception("invalid number of columns passed to MergeTreeReader::fillMissingColumns. " - "Expected " + toString(num_columns) + ", " - "got " + toString(res_columns.size()), ErrorCodes::LOGICAL_ERROR); - - /// For a missing column of a nested data structure we must create not a column of empty - /// arrays, but a column of arrays of correct length. - - /// First, collect offset columns for all arrays in the block. - OffsetColumns offset_columns; - auto requested_column = columns.begin(); - for (size_t i = 0; i < num_columns; ++i, ++requested_column) - { - if (res_columns[i] == nullptr) - continue; - - if (const auto * array = typeid_cast(res_columns[i].get())) - { - String offsets_name = Nested::extractTableName(requested_column->name); - auto & offsets_column = offset_columns[offsets_name]; - - /// If for some reason multiple offsets columns are present for the same nested data structure, - /// choose the one that is not empty. - if (!offsets_column || offsets_column->empty()) - offsets_column = array->getOffsetsPtr(); - } - } - - should_evaluate_missing_defaults = false; - - /// insert default values only for columns without default expressions - requested_column = columns.begin(); - for (size_t i = 0; i < num_columns; ++i, ++requested_column) - { - auto & [name, type] = *requested_column; - - if (res_columns[i] && arrayHasNoElementsRead(*res_columns[i])) - res_columns[i] = nullptr; - - if (res_columns[i] == nullptr) - { - if (metadata_snapshot->getColumns().hasDefault(name)) - { - should_evaluate_missing_defaults = true; - continue; - } - - String offsets_name = Nested::extractTableName(name); - auto offset_it = offset_columns.find(offsets_name); - const auto * array_type = typeid_cast(type.get()); - if (offset_it != offset_columns.end() && array_type) - { - const auto & nested_type = array_type->getNestedType(); - ColumnPtr offsets_column = offset_it->second; - size_t nested_rows = typeid_cast(*offsets_column).getData().back(); - - ColumnPtr nested_column = - nested_type->createColumnConstWithDefaultValue(nested_rows)->convertToFullColumnIfConst(); - - res_columns[i] = ColumnArray::create(nested_column, offsets_column); - } - else - { - /// We must turn a constant column into a full column because the interpreter could infer - /// that it is constant everywhere but in some blocks (from other parts) it can be a full column. - res_columns[i] = type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst(); - } - } - } + DB::fillMissingColumns(res_columns, num_rows, columns, metadata_snapshot); + should_evaluate_missing_defaults = std::any_of( + res_columns.begin(), res_columns.end(), [](const auto & column) { return column == nullptr; }); } catch (Exception & e) { @@ -166,7 +78,7 @@ void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_e } } -void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns & res_columns) +void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns & res_columns) const { try { @@ -192,6 +104,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns additional_columns, columns, metadata_snapshot->getColumns(), storage.getContext()); if (dag) { + dag->addMaterializingOutputActions(); auto actions = std::make_shared< ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromSettings(storage.getContext()->getSettingsRef())); @@ -244,7 +157,7 @@ NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & requ return {String(it->getKey()), type}; } -void IMergeTreeReader::performRequiredConversions(Columns & res_columns) +void IMergeTreeReader::performRequiredConversions(Columns & res_columns) const { try { diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index 28334b9a8bb..7c5977b5cb2 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -44,13 +44,13 @@ public: /// Add columns from ordered_names that are not present in the block. /// Missing columns are added in the order specified by ordered_names. /// num_rows is needed in case if all res_columns are nullptr. - void fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows); + void fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) const; /// Evaluate defaulted columns if necessary. - void evaluateMissingDefaults(Block additional_columns, Columns & res_columns); + void evaluateMissingDefaults(Block additional_columns, Columns & res_columns) const; /// If part metadata is not equal to storage metadata, than /// try to perform conversions of columns. - void performRequiredConversions(Columns & res_columns); + void performRequiredConversions(Columns & res_columns) const; const NamesAndTypesList & getColumns() const { return columns; } size_t numColumnsInResult() const { return columns.size(); } diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 5393d71ff86..b4a902499db 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -79,10 +79,7 @@ NameSet IMergedBlockOutputStream::removeEmptyColumnsFromPart( for (const String & removed_file : remove_files) { if (checksums.files.count(removed_file)) - { - data_part->volume->getDisk()->removeFile(data_part->getFullRelativePath() + removed_file); checksums.files.erase(removed_file); - } } /// Remove columns from columns array diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 323b59e2902..eeff7e4c875 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -448,7 +448,7 @@ KeyCondition::KeyCondition( { for (size_t i = 0, size = key_column_names.size(); i < size; ++i) { - std::string name = key_column_names[i]; + const auto & name = key_column_names[i]; if (!key_columns.count(name)) key_columns[name] = i; } @@ -715,18 +715,12 @@ bool KeyCondition::transformConstantWithValidFunctions( if (is_valid_chain) { - /// Here we cast constant to the input type. - /// It is not clear, why this works in general. - /// I can imagine the case when expression like `column < const` is legal, - /// but `type(column)` and `type(const)` are of different types, - /// and const cannot be casted to column type. - /// (There could be `superType(type(column), type(const))` which is used for comparison). - /// - /// However, looks like this case newer happenes (I could not find such). - /// Let's assume that any two comparable types are castable to each other. auto const_type = cur_node->result_type; auto const_column = out_type->createColumnConst(1, out_value); - auto const_value = (*castColumn({const_column, out_type, ""}, const_type))[0]; + auto const_value = (*castColumnAccurateOrNull({const_column, out_type, ""}, const_type))[0]; + + if (const_value.isNull()) + return false; while (!chain.empty()) { @@ -1999,7 +1993,7 @@ BoolMask KeyCondition::checkInHyperrectangle( if (!element.set_index) throw Exception("Set for IN is not created yet", ErrorCodes::LOGICAL_ERROR); - rpn_stack.emplace_back(element.set_index->checkInRange(hyperrectangle, data_types)); + rpn_stack.emplace_back(element.set_index->checkInRange(hyperrectangle, data_types, single_point)); if (element.function == RPNElement::FUNCTION_NOT_IN_SET) rpn_stack.back() = !rpn_stack.back(); } diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index dee46ae52ce..afe4a9f3e20 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -31,7 +31,7 @@ struct FieldRef : public Field /// Create as explicit field without block. template - FieldRef(T && value) : Field(std::forward(value)) {} + FieldRef(T && value) : Field(std::forward(value)) {} /// NOLINT /// Create as reference to field in block. FieldRef(ColumnsWithTypeAndName * columns_, size_t row_idx_, size_t column_idx_) @@ -60,10 +60,10 @@ public: bool right_included = false; /// includes the right border /// The whole universe (not null). - Range() {} + Range() {} /// NOLINT /// One point. - Range(const FieldRef & point) + Range(const FieldRef & point) /// NOLINT : left(point), right(point), left_included(true), right_included(true) {} /// A bounded two-sided range. @@ -313,8 +313,8 @@ private: ALWAYS_TRUE, }; - RPNElement() {} - RPNElement(Function function_) : function(function_) {} + RPNElement() = default; + RPNElement(Function function_) : function(function_) {} /// NOLINT RPNElement(Function function_, size_t key_column_) : function(function_), key_column(key_column_) {} RPNElement(Function function_, size_t key_column_, const Range & range_) : function(function_), range(range_), key_column(key_column_) {} diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index b05026d52f9..6d3281c8c61 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -1,11 +1,13 @@ #pragma once +#include #include #include #include #include #include +namespace fs = std::filesystem; namespace zkutil { diff --git a/src/Storages/MergeTree/LevelMergeSelector.cpp b/src/Storages/MergeTree/LevelMergeSelector.cpp index 7bcfbf6160a..16947277463 100644 --- a/src/Storages/MergeTree/LevelMergeSelector.cpp +++ b/src/Storages/MergeTree/LevelMergeSelector.cpp @@ -105,7 +105,7 @@ void selectWithinPartition( LevelMergeSelector::PartsRange LevelMergeSelector::select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) + size_t max_total_size_to_merge) { Estimator estimator; diff --git a/src/Storages/MergeTree/LevelMergeSelector.h b/src/Storages/MergeTree/LevelMergeSelector.h index 5849b34e320..f4080c379c4 100644 --- a/src/Storages/MergeTree/LevelMergeSelector.h +++ b/src/Storages/MergeTree/LevelMergeSelector.h @@ -21,7 +21,7 @@ public: PartsRange select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) override; + size_t max_total_size_to_merge) override; private: const Settings settings; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index b0b8aad2841..66356fd005b 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes } -std::pair MergeFromLogEntryTask::prepare() +ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() { LOG_TRACE(log, "Executing log entry to merge parts {} to {}", fmt::join(entry.source_parts, ", "), entry.new_part_name); @@ -30,7 +30,11 @@ std::pair MergeFromLogEntryT if (storage_settings_ptr->always_fetch_merged_part) { LOG_INFO(log, "Will fetch part {} because setting 'always_fetch_merged_part' is true", entry.new_part_name); - return {false, {}}; + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; } if (entry.merge_type == MergeType::TTL_RECOMPRESS && @@ -40,27 +44,32 @@ std::pair MergeFromLogEntryT LOG_INFO(log, "Will try to fetch part {} until '{}' because this part assigned to recompression merge. " "Source replica {} will try to merge this part first", entry.new_part_name, DateLUT::instance().timeToString(entry.create_time + storage_settings_ptr->try_fetch_recompressed_part_timeout.totalSeconds()), entry.source_replica); - return {false, {}}; + /// Waiting other replica to recompress part. No need to check it. + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = false, + .part_log_writer = {} + }; } /// In some use cases merging can be more expensive than fetching /// and it may be better to spread merges tasks across the replicas /// instead of doing exactly the same merge cluster-wise - std::optional replica_to_execute_merge; - bool replica_to_execute_merge_picked = false; if (storage.merge_strategy_picker.shouldMergeOnSingleReplica(entry)) { - replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); - replica_to_execute_merge_picked = true; - + std::optional replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); if (replica_to_execute_merge) { LOG_DEBUG(log, "Prefer fetching part {} from replica {} due to execute_merges_on_single_replica_time_threshold", entry.new_part_name, replica_to_execute_merge.value()); - return {false, {}}; + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; } } @@ -73,7 +82,11 @@ std::pair MergeFromLogEntryT { /// We do not have one of source parts locally, try to take some already merged part from someone. LOG_DEBUG(log, "Don't have all parts for merge {}; will try to fetch it instead", entry.new_part_name); - return {false, {}}; + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; } if (source_part_or_covering->name != source_part_name) @@ -87,7 +100,12 @@ std::pair MergeFromLogEntryT LOG_WARNING(log, fmt::runtime(message), source_part_name, source_part_or_covering->name, entry.new_part_name); if (!source_part_or_covering->info.contains(MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version))) throw Exception(ErrorCodes::LOGICAL_ERROR, message, source_part_name, source_part_or_covering->name, entry.new_part_name); - return {false, {}}; + + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; } parts.push_back(source_part_or_covering); @@ -110,7 +128,12 @@ std::pair MergeFromLogEntryT if (!replica.empty()) { LOG_DEBUG(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); - return {false, {}}; + /// We found covering part, no checks for missing part. + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = false, + .part_log_writer = {} + }; } } } @@ -158,22 +181,35 @@ std::pair MergeFromLogEntryT future_merged_part->updatePath(storage, reserved_space.get()); future_merged_part->merge_type = entry.merge_type; + if (storage_settings_ptr->allow_remote_fs_zero_copy_replication) { if (auto disk = reserved_space->getDisk(); disk->getType() == DB::DiskType::S3) { - if (storage.merge_strategy_picker.shouldMergeOnSingleReplicaShared(entry)) + String dummy; + if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, true, dummy).empty()) { - if (!replica_to_execute_merge_picked) - replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); + LOG_DEBUG(log, "Merge of part {} finished by some other replica, will fetch merged part", entry.new_part_name); + /// We found covering part, no checks for missing part. + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = false, + .part_log_writer = {} + }; + } - if (replica_to_execute_merge) - { - LOG_DEBUG(log, - "Prefer fetching part {} from replica {} due s3_execute_merges_on_single_replica_time_threshold", - entry.new_part_name, replica_to_execute_merge.value()); - return {false, {}}; - } + zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); + + if (!zero_copy_lock) + { + LOG_DEBUG(log, "Merge of part {} started by some other replica, will wait it and fetch merged part", entry.new_part_name); + /// Don't check for missing part -- it's missing because other replica still not + /// finished merge. + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = false, + .part_log_writer = {} + }; } } } @@ -189,9 +225,7 @@ std::pair MergeFromLogEntryT merge_mutate_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_merged_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); transaction_ptr = std::make_unique(storage); stopwatch_ptr = std::make_unique(); @@ -214,7 +248,7 @@ std::pair MergeFromLogEntryT for (auto & item : future_merged_part->parts) priority += item->getBytesOnDisk(); - return {true, [this, stopwatch = *stopwatch_ptr] (const ExecutionStatus & execution_status) + return {true, true, [this, stopwatch = *stopwatch_ptr] (const ExecutionStatus & execution_status) { storage.writePartLog( PartLogElement::MERGE_PARTS, execution_status, stopwatch.elapsed(), @@ -271,6 +305,9 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite throw; } + if (zero_copy_lock) + zero_copy_lock->lock->unlock(); + /** Removing old parts from ZK and from the disk is delayed - see ReplicatedMergeTreeCleanupThread, clearOldParts. */ diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.h b/src/Storages/MergeTree/MergeFromLogEntryTask.h index da286b27b20..250086e0f7d 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.h +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -24,7 +25,7 @@ public: protected: /// Both return false if we can't execute merge. - std::pair prepare() override; + ReplicatedMergeMutateTaskBase::PrepareResult prepare() override; bool finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) override; bool executeInnerTask() override @@ -37,6 +38,7 @@ private: MergeTreeData::DataPartsVector parts; MergeTreeData::TransactionUniquePtr transaction_ptr{nullptr}; + std::optional zero_copy_lock; StopwatchUniquePtr stopwatch_ptr{nullptr}; MergeTreeData::MutableDataPartPtr part; diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index a03cb053e1f..11d0bc8c565 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -16,23 +16,8 @@ MemoryTrackerThreadSwitcher::MemoryTrackerThreadSwitcher(MergeListEntry & merge_ { // Each merge is executed into separate background processing pool thread background_thread_memory_tracker = CurrentThread::getMemoryTracker(); - if (background_thread_memory_tracker) - { - /// From the query context it will be ("for thread") memory tracker with VariableContext::Thread level, - /// which does not have any limits and sampling settings configured. - /// And parent for this memory tracker should be ("(for query)") with VariableContext::Process level, - /// that has limits and sampling configured. - MemoryTracker * parent; - if (background_thread_memory_tracker->level == VariableContext::Thread && - (parent = background_thread_memory_tracker->getParent()) && - parent != &total_memory_tracker) - { - background_thread_memory_tracker = parent; - } - - background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); - background_thread_memory_tracker->setParent(&merge_list_entry->memory_tracker); - } + background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); + background_thread_memory_tracker->setParent(&merge_list_entry->memory_tracker); prev_untracked_memory_limit = current_thread->untracked_memory_limit; current_thread->untracked_memory_limit = merge_list_entry->max_untracked_memory; @@ -50,9 +35,7 @@ MemoryTrackerThreadSwitcher::MemoryTrackerThreadSwitcher(MergeListEntry & merge_ MemoryTrackerThreadSwitcher::~MemoryTrackerThreadSwitcher() { // Unplug memory_tracker from current background processing pool thread - - if (background_thread_memory_tracker) - background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); + background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); current_thread->untracked_memory_limit = prev_untracked_memory_limit; @@ -65,16 +48,14 @@ MemoryTrackerThreadSwitcher::~MemoryTrackerThreadSwitcher() MergeListElement::MergeListElement( const StorageID & table_id_, FutureMergedMutatedPartPtr future_part, - UInt64 memory_profiler_step, - UInt64 memory_profiler_sample_probability, - UInt64 max_untracked_memory_) + const Settings & settings) : table_id{table_id_} , partition_id{future_part->part_info.partition_id} , result_part_name{future_part->name} , result_part_path{future_part->path} , result_part_info{future_part->part_info} , num_parts{future_part->parts.size()} - , max_untracked_memory(max_untracked_memory_) + , max_untracked_memory(settings.max_untracked_memory) , query_id(table_id.getShortName() + "::" + result_part_name) , thread_id{getThreadId()} , merge_type{future_part->merge_type} @@ -97,8 +78,33 @@ MergeListElement::MergeListElement( } memory_tracker.setDescription("Mutate/Merge"); - memory_tracker.setProfilerStep(memory_profiler_step); - memory_tracker.setSampleProbability(memory_profiler_sample_probability); + /// MemoryTracker settings should be set here, because + /// later (see MemoryTrackerThreadSwitcher) + /// parent memory tracker will be changed, and if merge executed from the + /// query (OPTIMIZE TABLE), all settings will be lost (since + /// current_thread::memory_tracker will have Thread level MemoryTracker, + /// which does not have any settings itself, it relies on the settings of the + /// thread_group::memory_tracker, but MemoryTrackerThreadSwitcher will reset parent). + memory_tracker.setProfilerStep(settings.memory_profiler_step); + memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); + memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage); + if (settings.memory_tracker_fault_probability) + memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); + + /// Let's try to copy memory related settings from the query, + /// since settings that we have here is not from query, but global, from the table. + /// + /// NOTE: Remember, that Thread level MemoryTracker does not have any settings, + /// so it's parent is required. + MemoryTracker * query_memory_tracker = CurrentThread::getMemoryTracker(); + MemoryTracker * parent_query_memory_tracker; + if (query_memory_tracker->level == VariableContext::Thread && + (parent_query_memory_tracker = query_memory_tracker->getParent()) && + parent_query_memory_tracker != &total_memory_tracker) + { + memory_tracker.setOrRaiseHardLimit(parent_query_memory_tracker->getHardLimit()); + } + } MergeInfo MergeListElement::getInfo() const diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index 2df32df7f92..a944779ad44 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -58,6 +58,8 @@ using FutureMergedMutatedPartPtr = std::shared_ptr; struct MergeListElement; using MergeListEntry = BackgroundProcessListEntry; +struct Settings; + /** * Since merge is executed with multiple threads, this class @@ -127,9 +129,7 @@ struct MergeListElement : boost::noncopyable MergeListElement( const StorageID & table_id_, FutureMergedMutatedPartPtr future_part, - UInt64 memory_profiler_step, - UInt64 memory_profiler_sample_probability, - UInt64 max_untracked_memory_); + const Settings & settings); MergeInfo getInfo() const; diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index 1cbf928047c..14e43b2897e 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -81,9 +81,7 @@ void MergePlainMergeTreeTask::prepare() merge_list_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); write_part_log = [this] (const ExecutionStatus & execution_status) { diff --git a/src/Storages/MergeTree/MergeSelector.h b/src/Storages/MergeTree/MergeSelector.h index aac805823a9..c55f738f879 100644 --- a/src/Storages/MergeTree/MergeSelector.h +++ b/src/Storages/MergeTree/MergeSelector.h @@ -63,7 +63,7 @@ public: */ virtual PartsRange select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) = 0; + size_t max_total_size_to_merge) = 0; virtual ~IMergeSelector() = default; }; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5d5035d62ca..935a11ec5fa 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -126,18 +127,18 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() if (ctx->disk->exists(local_new_part_tmp_path)) throw Exception("Directory " + fullPath(ctx->disk, local_new_part_tmp_path) + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - { - std::lock_guard lock(global_ctx->mutator->tmp_parts_lock); - global_ctx->mutator->tmp_parts.emplace(local_tmp_part_basename); - } + global_ctx->data->temporary_parts.add(local_tmp_part_basename); SCOPE_EXIT( - std::lock_guard lock(global_ctx->mutator->tmp_parts_lock); - global_ctx->mutator->tmp_parts.erase(local_tmp_part_basename); + global_ctx->data->temporary_parts.remove(local_tmp_part_basename); ); global_ctx->all_column_names = global_ctx->metadata_snapshot->getColumns().getNamesOfPhysical(); global_ctx->storage_columns = global_ctx->metadata_snapshot->getColumns().getAllPhysical(); + auto object_columns = MergeTreeData::getObjectColumns(global_ctx->future_part->parts, global_ctx->metadata_snapshot->getColumns()); + global_ctx->storage_snapshot = std::make_shared(*global_ctx->data, global_ctx->metadata_snapshot, object_columns); + extendObjectColumns(global_ctx->storage_columns, object_columns, false); + extractMergingAndGatheringColumns( global_ctx->storage_columns, global_ctx->metadata_snapshot->getSortingKey().expression, @@ -241,9 +242,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() global_ctx->merging_column_names, global_ctx->gathering_column_names); - if (global_ctx->data->getSettings()->fsync_part_directory) - global_ctx->sync_guard = ctx->disk->getDirectorySyncGuard(local_new_part_tmp_path); - break; } default : @@ -421,7 +419,7 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const for (size_t part_num = 0; part_num < global_ctx->future_part->parts.size(); ++part_num) { auto column_part_source = std::make_shared( - *global_ctx->data, global_ctx->metadata_snapshot, global_ctx->future_part->parts[part_num], column_names, ctx->read_with_direct_io, true); + *global_ctx->data, global_ctx->storage_snapshot, global_ctx->future_part->parts[part_num], column_names, ctx->read_with_direct_io, true); /// Dereference unique_ptr column_part_source->setProgressCallback( @@ -585,12 +583,7 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c projection_future_part, projection.metadata, global_ctx->merge_entry, - std::make_unique( - (*global_ctx->merge_entry)->table_id, - projection_future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory), + std::make_unique((*global_ctx->merge_entry)->table_id, projection_future_part, settings), global_ctx->time_of_merge, global_ctx->context, global_ctx->space_reservation, @@ -760,7 +753,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() for (const auto & part : global_ctx->future_part->parts) { auto input = std::make_unique( - *global_ctx->data, global_ctx->metadata_snapshot, part, global_ctx->merging_column_names, ctx->read_with_direct_io, true); + *global_ctx->data, global_ctx->storage_snapshot, part, global_ctx->merging_column_names, ctx->read_with_direct_io, true); /// Dereference unique_ptr and pass horizontal_stage_progress by reference input->setProgressCallback( diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index aa64c4c2265..04da9ad77c4 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -127,6 +127,7 @@ private: MergeTreeDataMergerMutator * mutator{nullptr}; ActionBlocker * merges_blocker{nullptr}; ActionBlocker * ttl_merges_blocker{nullptr}; + StorageSnapshotPtr storage_snapshot{nullptr}; StorageMetadataPtr metadata_snapshot{nullptr}; FutureMergedMutatedPartPtr future_part{nullptr}; /// This will be either nullptr or new_data_part, so raw pointer is ok. @@ -155,7 +156,6 @@ private: QueryPipeline merged_pipeline; std::unique_ptr merging_executor; - SyncGuardPtr sync_guard{nullptr}; MergeTreeData::MutableDataPartPtr new_data_part{nullptr}; size_t rows_written{0}; diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 5b69a4e68b6..c656de61bfd 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -29,7 +29,7 @@ namespace ErrorCodes MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( Block header, const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, ExpressionActionsSettings actions_settings, UInt64 max_block_size_rows_, @@ -41,7 +41,7 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( std::optional extension_) : SourceWithProgress(transformHeader(std::move(header), prewhere_info_, storage_.getPartitionValueType(), virt_column_names_)) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , prewhere_info(prewhere_info_) , max_block_size_rows(max_block_size_rows_) , preferred_block_size_bytes(preferred_block_size_bytes_) diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h index c462c34aa83..4b933175ba0 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.h @@ -35,7 +35,7 @@ public: MergeTreeBaseSelectProcessor( Block header, const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, ExpressionActionsSettings actions_settings, UInt64 max_block_size_rows_, @@ -86,9 +86,8 @@ protected: void initializeRangeReaders(MergeTreeReadTask & task); -protected: const MergeTreeData & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; PrewhereInfoPtr prewhere_info; std::unique_ptr prewhere_actions; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 07d51d25700..6e72b843f10 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -35,7 +36,7 @@ bool injectRequiredColumnsRecursively( /// stages. checkStackSize(); - auto column_in_storage = storage_columns.tryGetColumnOrSubcolumn(ColumnsDescription::AllPhysical, column_name); + auto column_in_storage = storage_columns.tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column_name); if (column_in_storage) { auto column_name_in_part = column_in_storage->getNameInStorage(); @@ -92,8 +93,15 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetada alter_conversions = storage.getAlterConversionsForPart(part); for (size_t i = 0; i < columns.size(); ++i) { + auto name_in_storage = Nested::extractTableName(columns[i]); + if (storage_columns.has(name_in_storage) && isObject(storage_columns.get(name_in_storage).type)) + { + have_at_least_one_physical_column = true; + continue; + } + /// We are going to fetch only physical columns - if (!storage_columns.hasColumnOrSubcolumn(ColumnsDescription::AllPhysical, columns[i])) + if (!storage_columns.hasColumnOrSubcolumn(GetColumnsOptions::AllPhysical, columns[i])) throw Exception("There is no physical column or subcolumn " + columns[i] + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); have_at_least_one_physical_column |= injectRequiredColumnsRecursively( @@ -118,9 +126,9 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetada MergeTreeReadTask::MergeTreeReadTask( - const MergeTreeData::DataPartPtr & data_part_, const MarkRanges & mark_ranges_, const size_t part_index_in_query_, + const MergeTreeData::DataPartPtr & data_part_, const MarkRanges & mark_ranges_, size_t part_index_in_query_, const Names & ordered_names_, const NameSet & column_name_set_, const NamesAndTypesList & columns_, - const NamesAndTypesList & pre_columns_, const bool remove_prewhere_column_, const bool should_reorder_, + const NamesAndTypesList & pre_columns_, bool remove_prewhere_column_, bool should_reorder_, MergeTreeBlockSizePredictorPtr && size_predictor_) : data_part{data_part_}, mark_ranges{mark_ranges_}, part_index_in_query{part_index_in_query_}, ordered_names{ordered_names_}, column_name_set{column_name_set_}, columns{columns_}, pre_columns{pre_columns_}, @@ -254,7 +262,7 @@ void MergeTreeBlockSizePredictor::update(const Block & sample_block, const Colum MergeTreeReadTaskColumns getReadTaskColumns( const MergeTreeData & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const MergeTreeData::DataPartPtr & data_part, const Names & required_columns, const PrewhereInfoPtr & prewhere_info) @@ -263,7 +271,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( Names pre_column_names; /// inject columns required for defaults evaluation - bool should_reorder = !injectRequiredColumns(storage, metadata_snapshot, data_part, column_names).empty(); + bool should_reorder = !injectRequiredColumns(storage, storage_snapshot->getMetadataForQuery(), data_part, column_names).empty(); if (prewhere_info) { @@ -288,7 +296,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( if (pre_column_names.empty()) pre_column_names.push_back(column_names[0]); - const auto injected_pre_columns = injectRequiredColumns(storage, metadata_snapshot, data_part, pre_column_names); + const auto injected_pre_columns = injectRequiredColumns(storage, storage_snapshot->getMetadataForQuery(), data_part, pre_column_names); if (!injected_pre_columns.empty()) should_reorder = true; @@ -303,12 +311,12 @@ MergeTreeReadTaskColumns getReadTaskColumns( } MergeTreeReadTaskColumns result; + NamesAndTypesList all_columns; - auto columns = metadata_snapshot->getColumns(); - result.pre_columns = columns.getByNames(ColumnsDescription::All, pre_column_names, true); - result.columns = columns.getByNames(ColumnsDescription::All, column_names, true); + auto options = GetColumnsOptions(GetColumnsOptions::All).withSubcolumns().withExtendedObjects(); + result.pre_columns = storage_snapshot->getColumnsByNames(options, pre_column_names); + result.columns = storage_snapshot->getColumnsByNames(options, column_names); result.should_reorder = should_reorder; - return result; } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index b931a13c027..2373881f954 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -55,9 +55,9 @@ struct MergeTreeReadTask bool isFinished() const { return mark_ranges.empty() && range_reader.isCurrentRangeFinished(); } MergeTreeReadTask( - const MergeTreeData::DataPartPtr & data_part_, const MarkRanges & mark_ranges_, const size_t part_index_in_query_, + const MergeTreeData::DataPartPtr & data_part_, const MarkRanges & mark_ranges_, size_t part_index_in_query_, const Names & ordered_names_, const NameSet & column_name_set_, const NamesAndTypesList & columns_, - const NamesAndTypesList & pre_columns_, const bool remove_prewhere_column_, const bool should_reorder_, + const NamesAndTypesList & pre_columns_, bool remove_prewhere_column_, bool should_reorder_, MergeTreeBlockSizePredictorPtr && size_predictor_); }; @@ -73,7 +73,7 @@ struct MergeTreeReadTaskColumns MergeTreeReadTaskColumns getReadTaskColumns( const MergeTreeData & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const MergeTreeData::DataPartPtr & data_part, const Names & required_columns, const PrewhereInfoPtr & prewhere_info); @@ -86,7 +86,7 @@ struct MergeTreeBlockSizePredictor void startBlock(); /// Updates statistic for more accurate prediction - void update(const Block & sample_block, const Columns & columns, size_t num_rows, double decay = DECAY()); + void update(const Block & sample_block, const Columns & columns, size_t num_rows, double decay = calculateDecay()); /// Return current block size (after update()) inline size_t getBlockSize() const @@ -112,7 +112,7 @@ struct MergeTreeBlockSizePredictor : 0; } - inline void updateFilteredRowsRation(size_t rows_was_read, size_t rows_was_filtered, double decay = DECAY()) + inline void updateFilteredRowsRation(size_t rows_was_read, size_t rows_was_filtered, double decay = calculateDecay()) { double alpha = std::pow(1. - decay, rows_was_read); double current_ration = rows_was_filtered / std::max(1.0, static_cast(rows_was_read)); @@ -125,7 +125,7 @@ struct MergeTreeBlockSizePredictor /// After n=NUM_UPDATES_TO_TARGET_WEIGHT updates v_{n} = (1 - TARGET_WEIGHT) * v_{0} + TARGET_WEIGHT * v_{target} static constexpr double TARGET_WEIGHT = 0.5; static constexpr size_t NUM_UPDATES_TO_TARGET_WEIGHT = 8192; - static double DECAY() { return 1. - std::pow(TARGET_WEIGHT, 1. / NUM_UPDATES_TO_TARGET_WEIGHT); } + static double calculateDecay() { return 1. - std::pow(TARGET_WEIGHT, 1. / NUM_UPDATES_TO_TARGET_WEIGHT); } protected: diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 68fa81e1df9..f66586b121a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -265,6 +269,14 @@ MergeTreeData::MergeTreeData( /// Creating directories, if not exist. for (const auto & disk : getDisks()) { + /// TODO: implement it the main issue in DataPartsExchange (not able to send directories metadata) + if (supportsReplication() && settings->allow_remote_fs_zero_copy_replication + && disk->supportZeroCopyReplication() && metadata_.hasProjections()) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections are not supported when zero-copy replication is enabled for table. " + "Currently disk '{}' supports zero copy replication", disk->getName()); + } + if (disk->isBroken()) continue; @@ -275,10 +287,7 @@ MergeTreeData::MergeTreeData( if (disk->exists(current_version_file_path)) { if (!version_file.first.empty()) - { - LOG_ERROR(log, "Duplication of version file {} and {}", fullPath(version_file.second, version_file.first), current_version_file_path); - throw Exception("Multiple format_version.txt file", ErrorCodes::CORRUPTED_DATA); - } + throw Exception(ErrorCodes::CORRUPTED_DATA, "Duplication of version file {} and {}", fullPath(version_file.second, version_file.first), current_version_file_path); version_file = {current_version_file_path, disk}; } } @@ -650,13 +659,14 @@ void MergeTreeData::checkTTLExpressions(const StorageInMemoryMetadata & new_meta { for (const auto & move_ttl : new_table_ttl.move_ttl) { - if (!getDestinationForMoveTTL(move_ttl)) + if (!move_ttl.if_exists && !getDestinationForMoveTTL(move_ttl)) { String message; if (move_ttl.destination_type == DataDestinationType::DISK) - message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy."; + message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy"; else - message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy."; + message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy"; + throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION); } } @@ -1283,6 +1293,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (num_parts == 0 && parts_from_wal.empty()) { + resetObjectColumnsFromActiveParts(part_lock); LOG_DEBUG(log, "There are no data parts"); return; } @@ -1355,6 +1366,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) } } + resetObjectColumnsFromActiveParts(part_lock); calculateColumnAndSecondaryIndexSizesImpl(); LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); @@ -1377,7 +1389,7 @@ static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_pa } -size_t MergeTreeData::clearOldTemporaryDirectories(const MergeTreeDataMergerMutator & merger_mutator, size_t custom_directories_lifetime_seconds) +size_t MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds) { /// If the method is already called from another thread, then we don't need to do anything. std::unique_lock lock(clear_old_temporary_directories_mutex, std::defer_lock); @@ -1409,9 +1421,9 @@ size_t MergeTreeData::clearOldTemporaryDirectories(const MergeTreeDataMergerMuta { if (disk->isDirectory(it->path()) && isOldPartDirectory(disk, it->path(), deadline)) { - if (merger_mutator.hasTemporaryPart(basename)) + if (temporary_parts.contains(basename)) { - LOG_WARNING(log, "{} is an active destination for one of merge/mutation (consider increasing temporary_directories_lifetime setting)", full_path); + LOG_WARNING(log, "{} is in use (by merge/mutation/INSERT) (consider increasing temporary_directories_lifetime setting)", full_path); continue; } else @@ -2032,11 +2044,26 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context "ALTER ADD INDEX is not supported for tables with the old syntax", ErrorCodes::BAD_ARGUMENTS); } - if (command.type == AlterCommand::ADD_PROJECTION && !is_custom_partitioned) + if (command.type == AlterCommand::ADD_PROJECTION) + { - throw Exception( - "ALTER ADD PROJECTION is not supported for tables with the old syntax", - ErrorCodes::BAD_ARGUMENTS); + if (!is_custom_partitioned) + throw Exception( + "ALTER ADD PROJECTION is not supported for tables with the old syntax", + ErrorCodes::BAD_ARGUMENTS); + + /// TODO: implement it the main issue in DataPartsExchange (not able to send directories metadata) + if (supportsReplication() && getSettings()->allow_remote_fs_zero_copy_replication) + { + auto storage_policy = getStoragePolicy(); + auto disks = storage_policy->getDisks(); + for (const auto & disk : disks) + { + if (disk->supportZeroCopyReplication()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ALTER ADD PROJECTION is not supported when zero-copy replication is enabled for table. " + "Currently disk '{}' supports zero copy replication", disk->getName()); + } + } } if (command.type == AlterCommand::RENAME_COLUMN) { @@ -2600,6 +2627,11 @@ bool MergeTreeData::renameTempPartAndReplace( modifyPartState(part_it, DataPartState::Active); addPartContributionToColumnAndSecondaryIndexSizes(part); + if (covered_parts.empty()) + updateObjectColumns(*part_it, lock); + else + resetObjectColumnsFromActiveParts(lock); + ssize_t diff_bytes = part->getBytesOnDisk() - reduce_bytes; ssize_t diff_rows = part->rows_count - reduce_rows; ssize_t diff_parts = 1 - reduce_parts; @@ -2640,9 +2672,10 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( return covered_parts; } -void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVector & remove, bool clear_without_timeout, DataPartsLock & /*acquired_lock*/) +void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVector & remove, bool clear_without_timeout, DataPartsLock & acquired_lock) { auto remove_time = clear_without_timeout ? 0 : time(nullptr); + bool removed_active_part = false; for (const DataPartPtr & part : remove) { @@ -2650,6 +2683,7 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect { removePartContributionToColumnAndSecondaryIndexSizes(part); removePartContributionToDataVolume(part); + removed_active_part = true; } if (part->getState() == IMergeTreeDataPart::State::Active || clear_without_timeout) @@ -2661,11 +2695,15 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect if (isInMemoryPart(part) && getSettings()->in_memory_parts_enable_wal) getWriteAheadLog()->dropPart(part->name); } + + if (removed_active_part) + resetObjectColumnsFromActiveParts(acquired_lock); } void MergeTreeData::removePartsFromWorkingSetImmediatelyAndSetTemporaryState(const DataPartsVector & remove) { auto lock = lockParts(); + bool removed_active_part = false; for (const auto & part : remove) { @@ -2673,10 +2711,16 @@ void MergeTreeData::removePartsFromWorkingSetImmediatelyAndSetTemporaryState(con if (it_part == data_parts_by_info.end()) throw Exception("Part " + part->getNameWithState() + " not found in data_parts", ErrorCodes::LOGICAL_ERROR); + if (part->getState() == IMergeTreeDataPart::State::Active) + removed_active_part = true; + modifyPartState(part, IMergeTreeDataPart::State::Temporary); /// Erase immediately data_parts_indexes.erase(it_part); } + + if (removed_active_part) + resetObjectColumnsFromActiveParts(lock); } void MergeTreeData::removePartsFromWorkingSet(const DataPartsVector & remove, bool clear_without_timeout, DataPartsLock * acquired_lock) @@ -2769,8 +2813,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c return parts_to_remove; } -void MergeTreeData::forgetPartAndMoveToDetached(const MergeTreeData::DataPartPtr & part_to_detach, const String & prefix, bool -restore_covered) +void MergeTreeData::forgetPartAndMoveToDetached(const MergeTreeData::DataPartPtr & part_to_detach, const String & prefix, bool restore_covered) { if (prefix.empty()) LOG_INFO(log, "Renaming {} to {} and forgetting it.", part_to_detach->relative_path, part_to_detach->name); @@ -2778,6 +2821,8 @@ restore_covered) LOG_INFO(log, "Renaming {} to {}_{} and forgetting it.", part_to_detach->relative_path, prefix, part_to_detach->name); auto lock = lockParts(); + bool removed_active_part = false; + bool restored_active_part = false; auto it_part = data_parts_by_info.find(part_to_detach->info); if (it_part == data_parts_by_info.end()) @@ -2790,7 +2835,9 @@ restore_covered) { removePartContributionToDataVolume(part); removePartContributionToColumnAndSecondaryIndexSizes(part); + removed_active_part = true; } + modifyPartState(it_part, DataPartState::Deleting); part->renameToDetached(prefix); @@ -2840,6 +2887,7 @@ restore_covered) addPartContributionToColumnAndSecondaryIndexSizes(*it); addPartContributionToDataVolume(*it); modifyPartState(it, DataPartState::Active); // iterator is not invalidated here + restored_active_part = true; } pos = (*it)->info.max_block + 1; @@ -2871,6 +2919,7 @@ restore_covered) addPartContributionToColumnAndSecondaryIndexSizes(*it); addPartContributionToDataVolume(*it); modifyPartState(it, DataPartState::Active); + restored_active_part = true; } pos = (*it)->info.max_block + 1; @@ -2890,6 +2939,9 @@ restore_covered) LOG_ERROR(log, "The set of parts restored in place of {} looks incomplete. There might or might not be a data loss.{}", part->name, (error_parts.empty() ? "" : " Suspicious parts: " + error_parts)); } } + + if (removed_active_part || restored_active_part) + resetObjectColumnsFromActiveParts(lock); } @@ -3363,9 +3415,6 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & parts = getDataPartsVectorInPartition(MergeTreeDataPartState::Active, partition_id); auto disk = getStoragePolicy()->getDiskByName(name); - if (!disk) - throw Exception("Disk " + name + " does not exists on policy " + getStoragePolicy()->getName(), ErrorCodes::UNKNOWN_DISK); - parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) { return part_ptr->volume->getDisk()->getName() == disk->getName(); @@ -3581,7 +3630,7 @@ Pipe MergeTreeData::alterPartition( } -BackupEntries MergeTreeData::backup(const ASTs & partitions, ContextPtr local_context) +BackupEntries MergeTreeData::backupData(ContextPtr local_context, const ASTs & partitions) { DataPartsVector data_parts; if (partitions.empty()) @@ -3633,40 +3682,78 @@ BackupEntries MergeTreeData::backupDataParts(const DataPartsVector & data_parts) } -RestoreDataTasks MergeTreeData::restoreDataPartsFromBackup(const BackupPtr & backup, const String & data_path_in_backup, - const std::unordered_set & partition_ids, - SimpleIncrement * increment) +class MergeTreeDataRestoreTask : public IRestoreTask { - RestoreDataTasks restore_tasks; - - Strings part_names = backup->listFiles(data_path_in_backup); - for (const String & part_name : part_names) +public: + MergeTreeDataRestoreTask( + const std::shared_ptr & storage_, + const BackupPtr & backup_, + const String & data_path_in_backup_, + const std::unordered_set & partition_ids_, + SimpleIncrement * increment_) + : storage(storage_) + , backup(backup_) + , data_path_in_backup(data_path_in_backup_) + , partition_ids(partition_ids_) + , increment(increment_) { - const auto part_info = MergeTreePartInfo::tryParsePartName(part_name, format_version); + } - if (!part_info) - continue; - - if (!partition_ids.empty() && !partition_ids.contains(part_info->partition_id)) - continue; - - UInt64 total_size_of_part = 0; - Strings filenames = backup->listFiles(data_path_in_backup + part_name + "/", ""); - for (const String & filename : filenames) - total_size_of_part += backup->getFileSize(data_path_in_backup + part_name + "/" + filename); - - std::shared_ptr reservation = getStoragePolicy()->reserveAndCheck(total_size_of_part); - - auto restore_task = [this, - backup, - data_path_in_backup, - part_name, - part_info = std::move(part_info), - filenames = std::move(filenames), - reservation, - increment]() + RestoreTasks run() override + { + RestoreTasks restore_part_tasks; + Strings part_names = backup->listFiles(data_path_in_backup); + for (const String & part_name : part_names) { + const auto part_info = MergeTreePartInfo::tryParsePartName(part_name, storage->format_version); + if (!part_info) + continue; + + if (!partition_ids.empty() && !partition_ids.contains(part_info->partition_id)) + continue; + + restore_part_tasks.push_back( + std::make_unique(storage, backup, data_path_in_backup, part_name, *part_info, increment)); + } + return restore_part_tasks; + } + +private: + std::shared_ptr storage; + BackupPtr backup; + String data_path_in_backup; + std::unordered_set partition_ids; + SimpleIncrement * increment; + + class RestorePartTask : public IRestoreTask + { + public: + RestorePartTask( + const std::shared_ptr & storage_, + const BackupPtr & backup_, + const String & data_path_in_backup_, + const String & part_name_, + const MergeTreePartInfo & part_info_, + SimpleIncrement * increment_) + : storage(storage_) + , backup(backup_) + , data_path_in_backup(data_path_in_backup_) + , part_name(part_name_) + , part_info(part_info_) + , increment(increment_) + { + } + + RestoreTasks run() override + { + UInt64 total_size_of_part = 0; + Strings filenames = backup->listFiles(data_path_in_backup + part_name + "/", ""); + for (const String & filename : filenames) + total_size_of_part += backup->getFileSize(data_path_in_backup + part_name + "/" + filename); + + std::shared_ptr reservation = storage->getStoragePolicy()->reserveAndCheck(total_size_of_part); auto disk = reservation->getDisk(); + String relative_data_path = storage->getRelativeDataPath(); auto temp_part_dir_owner = std::make_shared(disk, relative_data_path + "restoring_" + part_name + "_"); String temp_part_dir = temp_part_dir_owner->getPath(); @@ -3681,18 +3768,33 @@ RestoreDataTasks MergeTreeData::restoreDataPartsFromBackup(const BackupPtr & bac auto read_buffer = backup_entry->getReadBuffer(); auto write_buffer = disk->writeFile(temp_part_dir + "/" + filename); copyData(*read_buffer, *write_buffer); + reservation->update(reservation->getSize() - backup_entry->getSize()); } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto part = createPart(part_name, *part_info, single_disk_volume, relative_temp_part_dir); + auto part = storage->createPart(part_name, part_info, single_disk_volume, relative_temp_part_dir); part->loadColumnsChecksumsIndexes(false, true); - renameTempPartAndAdd(part, increment); - }; + storage->renameTempPartAndAdd(part, increment); + return {}; + } - restore_tasks.emplace_back(std::move(restore_task)); - } + private: + std::shared_ptr storage; + BackupPtr backup; + String data_path_in_backup; + String part_name; + MergeTreePartInfo part_info; + SimpleIncrement * increment; + }; +}; - return restore_tasks; + +RestoreTaskPtr MergeTreeData::restoreDataParts(const std::unordered_set & partition_ids, + const BackupPtr & backup, const String & data_path_in_backup, + SimpleIncrement * increment) +{ + return std::make_unique( + std::static_pointer_cast(shared_from_this()), backup, data_path_in_backup, partition_ids, increment); } @@ -3820,53 +3922,62 @@ std::set MergeTreeData::getPartitionIdsAffectedByCommands( } -MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVector( - const DataPartStates & affordable_states, DataPartStateVector * out_states, bool require_projection_parts) const +MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVectorUnlocked( + const DataPartStates & affordable_states, + const DataPartsLock & /*lock*/, + DataPartStateVector * out_states, + bool require_projection_parts) const { DataPartsVector res; DataPartsVector buf; + + for (auto state : affordable_states) { - auto lock = lockParts(); + auto range = getDataPartsStateRange(state); - for (auto state : affordable_states) + if (require_projection_parts) { - auto range = getDataPartsStateRange(state); - - if (require_projection_parts) + for (const auto & part : range) { - for (const auto & part : range) - { - for (const auto & [_, projection_part] : part->getProjectionParts()) - res.push_back(projection_part); - } - } - else - { - std::swap(buf, res); - res.clear(); - std::merge(range.begin(), range.end(), buf.begin(), buf.end(), std::back_inserter(res), LessDataPart()); //-V783 + for (const auto & [_, projection_part] : part->getProjectionParts()) + res.push_back(projection_part); } } - - if (out_states != nullptr) + else { - out_states->resize(res.size()); - if (require_projection_parts) - { - for (size_t i = 0; i < res.size(); ++i) - (*out_states)[i] = res[i]->getParentPart()->getState(); - } - else - { - for (size_t i = 0; i < res.size(); ++i) - (*out_states)[i] = res[i]->getState(); - } + std::swap(buf, res); + res.clear(); + std::merge(range.begin(), range.end(), buf.begin(), buf.end(), std::back_inserter(res), LessDataPart()); //-V783 + } + } + + if (out_states != nullptr) + { + out_states->resize(res.size()); + if (require_projection_parts) + { + for (size_t i = 0; i < res.size(); ++i) + (*out_states)[i] = res[i]->getParentPart()->getState(); + } + else + { + for (size_t i = 0; i < res.size(); ++i) + (*out_states)[i] = res[i]->getState(); } } return res; } +MergeTreeData::DataPartsVector MergeTreeData::getDataPartsVector( + const DataPartStates & affordable_states, + DataPartStateVector * out_states, + bool require_projection_parts) const +{ + auto lock = lockParts(); + return getDataPartsVectorUnlocked(affordable_states, lock, out_states, require_projection_parts); +} + MergeTreeData::DataPartsVector MergeTreeData::getAllDataPartsVector(MergeTreeData::DataPartStateVector * out_states, bool require_projection_parts) const { @@ -4112,10 +4223,10 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules( SpacePtr destination_ptr = getDestinationForMoveTTL(*move_ttl_entry, is_insert); if (!destination_ptr) { - if (move_ttl_entry->destination_type == DataDestinationType::VOLUME) + if (move_ttl_entry->destination_type == DataDestinationType::VOLUME && !move_ttl_entry->if_exists) LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found or rule is not applicable at the moment", move_ttl_entry->destination_name, log_name); - else if (move_ttl_entry->destination_type == DataDestinationType::DISK) + else if (move_ttl_entry->destination_type == DataDestinationType::DISK && !move_ttl_entry->if_exists) LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found or rule is not applicable at the moment", move_ttl_entry->destination_name, log_name); } @@ -4149,7 +4260,7 @@ SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl auto policy = getStoragePolicy(); if (move_ttl.destination_type == DataDestinationType::VOLUME) { - auto volume = policy->getVolumeByName(move_ttl.destination_name); + auto volume = policy->tryGetVolumeByName(move_ttl.destination_name); if (!volume) return {}; @@ -4161,7 +4272,8 @@ SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl } else if (move_ttl.destination_type == DataDestinationType::DISK) { - auto disk = policy->getDiskByName(move_ttl.destination_name); + auto disk = policy->tryGetDiskByName(move_ttl.destination_name); + if (!disk) return {}; @@ -4315,7 +4427,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: else { total_covered_parts.insert(total_covered_parts.end(), covered_parts.begin(), covered_parts.end()); - for (const DataPartPtr & covered_part : covered_parts) + for (const auto & covered_part : covered_parts) { covered_part->remove_time.store(current_time, std::memory_order_relaxed); @@ -4325,6 +4437,7 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: data.modifyPartState(covered_part, DataPartState::Outdated); data.removePartContributionToColumnAndSecondaryIndexSizes(covered_part); } + reduce_parts += covered_parts.size(); add_bytes += part->getBytesOnDisk(); @@ -4336,6 +4449,14 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: } } + if (reduce_parts == 0) + { + for (const auto & part : precommitted_parts) + data.updateObjectColumns(part, parts_lock); + } + else + data.resetObjectColumnsFromActiveParts(parts_lock); + ssize_t diff_bytes = add_bytes - reduce_bytes; ssize_t diff_rows = add_rows - reduce_rows; ssize_t diff_parts = add_parts - reduce_parts; @@ -4419,7 +4540,7 @@ using PartitionIdToMaxBlock = std::unordered_map; static void selectBestProjection( const MergeTreeDataSelectExecutor & reader, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, const Names & required_columns, ProjectionCandidate & candidate, @@ -4448,7 +4569,7 @@ static void selectBestProjection( auto projection_result_ptr = reader.estimateNumMarksToRead( projection_parts, candidate.required_columns, - metadata_snapshot, + storage_snapshot->metadata, candidate.desc->metadata, query_info, query_context, @@ -4470,9 +4591,9 @@ static void selectBestProjection( auto normal_result_ptr = reader.estimateNumMarksToRead( normal_parts, required_columns, - metadata_snapshot, - metadata_snapshot, - query_info, + storage_snapshot->metadata, + storage_snapshot->metadata, + query_info, // TODO syntax_analysis_result set in index query_context, settings.max_threads, max_added_blocks); @@ -4675,8 +4796,9 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( std::optional MergeTreeData::getQueryProcessingStageWithAggregateProjection( - ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info) const + ContextPtr query_context, const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info) const { + const auto & metadata_snapshot = storage_snapshot->metadata; const auto & settings = query_context->getSettingsRef(); if (!settings.allow_experimental_projection_optimization || query_info.ignore_projections || query_info.is_projection_query) return std::nullopt; @@ -4698,6 +4820,10 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (select_query->sampleSize()) return std::nullopt; + // Currently projection don't support deduplication when moving parts between shards. + if (settings.allow_experimental_query_deduplication) + return std::nullopt; + // Currently projections don't support ARRAY JOIN yet. if (select_query->arrayJoinExpressionList().first) return std::nullopt; @@ -4708,13 +4834,18 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (select_query->join()) return std::nullopt; + auto query_options = SelectQueryOptions( + QueryProcessingStage::WithMergeableState, + /* depth */ 1, + /* is_subquery_= */ true + ).ignoreProjections().ignoreAlias(); InterpreterSelectQuery select( query_ptr, query_context, - SelectQueryOptions{QueryProcessingStage::WithMergeableState}.ignoreProjections().ignoreAlias(), - query_info.sets /* prepared_sets */); + query_options, + /* prepared_sets_= */ query_info.sets); const auto & analysis_result = select.getAnalysisResult(); - query_info.sets = std::move(select.getQueryAnalyzer()->getPreparedSets()); + query_info.sets = select.getQueryAnalyzer()->getPreparedSets(); bool can_use_aggregate_projection = true; /// If the first stage of the query pipeline is more complex than Aggregating - Expression - Filter - ReadFromStorage, @@ -5043,7 +5174,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg { selectBestProjection( reader, - metadata_snapshot, + storage_snapshot, query_info, analysis_result.required_columns, candidate, @@ -5063,7 +5194,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg { selectBestProjection( reader, - metadata_snapshot, + storage_snapshot, query_info, analysis_result.required_columns, candidate, @@ -5103,12 +5234,12 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg QueryProcessingStage::Enum MergeTreeData::getQueryProcessingStage( ContextPtr query_context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info) const { if (to_stage >= QueryProcessingStage::Enum::WithMergeableState) { - if (auto projection = getQueryProcessingStageWithAggregateProjection(query_context, metadata_snapshot, query_info)) + if (auto projection = getQueryProcessingStageWithAggregateProjection(query_context, storage_snapshot, query_info)) { query_info.projection = std::move(projection); if (query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) @@ -5509,6 +5640,7 @@ try if (result_part) { + part_log_elem.disk_name = result_part->volume->getDisk()->getName(); part_log_elem.path_on_disk = result_part->getFullPath(); part_log_elem.bytes_compressed_on_disk = result_part->getBytesOnDisk(); part_log_elem.rows = result_part->rows_count; @@ -5697,7 +5829,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge /// replica will actually move the part from disk to some /// zero-copy storage other replicas will just fetch /// metainformation. - if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part, disk); lock) + if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part->name, disk); lock) { cloned_part = parts_mover.clonePart(moving_part); parts_mover.swapClonedPart(cloned_part); @@ -6032,6 +6164,52 @@ ReservationPtr MergeTreeData::balancedReservation( return reserved_space; } +ColumnsDescription MergeTreeData::getObjectColumns( + const DataPartsVector & parts, const ColumnsDescription & storage_columns) +{ + return DB::getObjectColumns( + parts.begin(), parts.end(), + storage_columns, [](const auto & part) -> const auto & { return part->getColumns(); }); +} + +ColumnsDescription MergeTreeData::getObjectColumns( + boost::iterator_range range, const ColumnsDescription & storage_columns) +{ + return DB::getObjectColumns( + range.begin(), range.end(), + storage_columns, [](const auto & part) -> const auto & { return part->getColumns(); }); +} + +void MergeTreeData::resetObjectColumnsFromActiveParts(const DataPartsLock & /*lock*/) +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + const auto & columns = metadata_snapshot->getColumns(); + if (!hasObjectColumns(columns)) + return; + + auto range = getDataPartsStateRange(DataPartState::Active); + object_columns = getObjectColumns(range, columns); +} + +void MergeTreeData::updateObjectColumns(const DataPartPtr & part, const DataPartsLock & /*lock*/) +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + const auto & columns = metadata_snapshot->getColumns(); + if (!hasObjectColumns(columns)) + return; + + DB::updateObjectColumns(object_columns, part->getColumns()); +} + +StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const +{ + auto snapshot_data = std::make_unique(); + + auto lock = lockParts(); + snapshot_data->parts = getDataPartsVectorUnlocked({DataPartState::Active}, lock); + return std::make_shared(*this, metadata_snapshot, object_columns, std::move(snapshot_data)); +} + CurrentlySubmergingEmergingTagger::~CurrentlySubmergingEmergingTagger() { std::lock_guard lock(storage.currently_submerging_emerging_mutex); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 93add8d6935..fb839e5a0dd 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -3,30 +3,31 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include #include #include #include #include #include #include +#include +#include +#include +#include +#include +#include #include #include -#include #include #include #include -#include -#include +#include +#include +#include +#include #include #include -#include +#include +#include #include @@ -38,6 +39,9 @@ namespace DB { +/// Number of streams is not number parts, but number or parts*files, hence 1000. +const size_t DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE = 1000; + class AlterCommands; class MergeTreePartsMover; class MergeTreeDataMergerMutator; @@ -243,7 +247,7 @@ public: class Transaction : private boost::noncopyable { public: - Transaction(MergeTreeData & data_) : data(data_) {} + explicit Transaction(MergeTreeData & data_) : data(data_) {} DataPartsVector commit(MergeTreeData::DataPartsLock * acquired_parts_lock = nullptr); @@ -396,12 +400,12 @@ public: ContextPtr query_context) const; std::optional getQueryProcessingStageWithAggregateProjection( - ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info) const; + ContextPtr query_context, const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info) const; QueryProcessingStage::Enum getQueryProcessingStage( ContextPtr query_context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & info) const override; ReservationPtr reserveSpace(UInt64 expected_size, VolumePtr & volume) const; @@ -416,10 +420,21 @@ public: bool supportsSubcolumns() const override { return true; } + bool supportsDynamicSubcolumns() const override { return true; } + NamesAndTypesList getVirtuals() const override; bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, ContextPtr, const StorageMetadataPtr & metadata_snapshot) const override; + /// Snapshot for MergeTree contains the current set of data parts + /// at the moment of the start of query. + struct SnapshotData : public StorageSnapshot::Data + { + DataPartsVector parts; + }; + + StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const override; + /// Load the set of data parts from disk. Call once - immediately after the object is created. void loadDataParts(bool skip_sanity_checks); @@ -430,13 +445,23 @@ public: /// Returns a copy of the list so that the caller shouldn't worry about locks. DataParts getDataParts(const DataPartStates & affordable_states) const; + DataPartsVector getDataPartsVectorUnlocked( + const DataPartStates & affordable_states, + const DataPartsLock & lock, + DataPartStateVector * out_states = nullptr, + bool require_projection_parts = false) const; + /// Returns sorted list of the parts with specified states /// out_states will contain snapshot of each part state DataPartsVector getDataPartsVector( - const DataPartStates & affordable_states, DataPartStateVector * out_states = nullptr, bool require_projection_parts = false) const; + const DataPartStates & affordable_states, + DataPartStateVector * out_states = nullptr, + bool require_projection_parts = false) const; /// Returns absolutely all parts (and snapshot of their states) - DataPartsVector getAllDataPartsVector(DataPartStateVector * out_states = nullptr, bool require_projection_parts = false) const; + DataPartsVector getAllDataPartsVector( + DataPartStateVector * out_states = nullptr, + bool require_projection_parts = false) const; /// Returns all detached parts DetachedPartsInfo getDetachedParts() const; @@ -566,7 +591,7 @@ public: /// Delete all directories which names begin with "tmp" /// Must be called with locked lockForShare() because it's using relative_data_path. - size_t clearOldTemporaryDirectories(const MergeTreeDataMergerMutator & merger_mutator, size_t custom_directories_lifetime_seconds); + size_t clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds); size_t clearEmptyParts(); @@ -649,15 +674,18 @@ public: ContextPtr context, TableLockHolder & table_lock_holder); + /// Storage has data to backup. + bool hasDataToBackup() const override { return true; } + /// Prepares entries to backup data of the storage. - BackupEntries backup(const ASTs & partitions, ContextPtr context) override; + BackupEntries backupData(ContextPtr context, const ASTs & partitions) override; static BackupEntries backupDataParts(const DataPartsVector & data_parts); /// Extract data from the backup and put it to the storage. - RestoreDataTasks restoreDataPartsFromBackup( + RestoreTaskPtr restoreDataParts( + const std::unordered_set & partition_ids, const BackupPtr & backup, const String & data_path_in_backup, - const std::unordered_set & partition_ids, SimpleIncrement * increment); /// Moves partition to specified Disk @@ -688,6 +716,12 @@ public: return column_sizes; } + const ColumnsDescription & getObjectColumns() const { return object_columns; } + + /// Creates desciprion of columns of data type Object from the range of data parts. + static ColumnsDescription getObjectColumns( + const DataPartsVector & parts, const ColumnsDescription & storage_columns); + IndexSizeByName getSecondaryIndexSizes() const override { auto lock = lockParts(); @@ -876,7 +910,7 @@ public: /// Lock part in zookeeper for shared data in several nodes /// Overridden in StorageReplicatedMergeTree - virtual void lockSharedData(const IMergeTreeDataPart &) const {} + virtual void lockSharedData(const IMergeTreeDataPart &, bool = false) const {} /// NOLINT /// Unlock shared data part in zookeeper /// Overridden in StorageReplicatedMergeTree @@ -906,7 +940,6 @@ public: mutable std::mutex currently_submerging_emerging_mutex; protected: - friend class IMergeTreeDataPart; friend class MergeTreeDataMergerMutator; friend struct ReplicatedMergeTreeTableMetadata; @@ -978,6 +1011,11 @@ protected: DataPartsIndexes::index::type & data_parts_by_info; DataPartsIndexes::index::type & data_parts_by_state_and_info; + /// Current descriprion of columns of data type Object. + /// It changes only when set of parts is changed and is + /// protected by @data_parts_mutex. + ColumnsDescription object_columns; + MergeTreePartsMover parts_mover; /// Executors are common for both ReplicatedMergeTree and plain MergeTree @@ -1014,6 +1052,10 @@ protected: return {begin, end}; } + /// Creates desciprion of columns of data type Object from the range of data parts. + static ColumnsDescription getObjectColumns( + boost::iterator_range range, const ColumnsDescription & storage_columns); + std::optional totalRowsByPartitionPredicateImpl( const SelectQueryInfo & query_info, ContextPtr context, const DataPartsVector & parts) const; @@ -1197,9 +1239,14 @@ private: MutableDataPartsVector & parts_from_wal, DataPartsLock & part_lock); + void resetObjectColumnsFromActiveParts(const DataPartsLock & lock); + void updateObjectColumns(const DataPartPtr & part, const DataPartsLock & lock); + /// Create zero-copy exclusive lock for part and disk. Useful for coordination of /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree. - virtual std::optional tryCreateZeroCopyExclusiveLock(const DataPartPtr &, const DiskPtr &) { return std::nullopt; } + virtual std::optional tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; } + + TemporaryParts temporary_parts; }; /// RAII struct to record big parts that are submerging or emerging. diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index cb9fa7e6086..22a868f218e 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -303,7 +303,6 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge( SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( FutureMergedMutatedPartPtr future_part, - UInt64 & available_disk_space, const AllowedMergingPredicate & can_merge, const String & partition_id, bool final, @@ -355,6 +354,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti ++it; } + auto available_disk_space = data.getStoragePolicy()->getMaxUnreservedFreeSpace(); /// Enough disk space to cover the new merge with a margin. auto required_disk_space = sum_bytes * DISK_USAGE_COEFFICIENT_TO_SELECT; if (available_disk_space <= required_disk_space) @@ -382,7 +382,6 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); future_part->assign(std::move(parts)); - available_disk_space -= required_disk_space; return SelectPartsDecision::SELECTED; } @@ -694,11 +693,10 @@ MergeTreeDataMergerMutator::getColumnsForNewDataPart( } } - bool is_wide_part = isWidePart(source_part); SerializationInfoByName new_serialization_infos; for (const auto & [name, info] : serialization_infos) { - if (is_wide_part && removed_columns.count(name)) + if (removed_columns.count(name)) continue; auto it = renamed_columns_from_to.find(name); @@ -710,7 +708,7 @@ MergeTreeDataMergerMutator::getColumnsForNewDataPart( /// In compact parts we read all columns, because they all stored in a /// single file - if (!is_wide_part) + if (!isWidePart(source_part)) return {updated_header.getNamesAndTypesList(), new_serialization_infos}; Names source_column_names = source_part->getColumns().getNames(); @@ -783,10 +781,4 @@ ExecuteTTLType MergeTreeDataMergerMutator::shouldExecuteTTL(const StorageMetadat } -bool MergeTreeDataMergerMutator::hasTemporaryPart(const std::string & basename) const -{ - std::lock_guard lock(tmp_parts_lock); - return tmp_parts.contains(basename); -} - } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index bcac642eb16..e64c13ca6c3 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -81,7 +81,6 @@ public: */ SelectPartsDecision selectAllPartsToMergeWithinPartition( FutureMergedMutatedPartPtr future_part, - UInt64 & available_disk_space, const AllowedMergingPredicate & can_merge, const String & partition_id, bool final, @@ -177,7 +176,6 @@ private: bool need_remove_expired_values, const MergeTreeData::MergingParams & merging_params) const; -private: MergeTreeData & data; const size_t max_tasks_count; @@ -193,26 +191,6 @@ private: ITTLMergeSelector::PartitionIdToTTLs next_recompress_ttl_merge_times_by_partition; /// Performing TTL merges independently for each partition guarantees that /// there is only a limited number of TTL merges and no partition stores data, that is too stale - -public: - /// Returns true if passed part name is active. - /// (is the destination for one of active mutation/merge). - /// - /// NOTE: that it accept basename (i.e. dirname), not the path, - /// since later requires canonical form. - bool hasTemporaryPart(const std::string & basename) const; - -private: - /// Set of active temporary paths that is used as the destination. - /// List of such paths is required to avoid trying to remove them during cleanup. - /// - /// NOTE: It is pretty short, so use STL is fine. - std::unordered_set tmp_parts; - /// Lock for "tmp_parts". - /// - /// NOTE: mutable is required to mark hasTemporaryPath() const - mutable std::mutex tmp_parts_lock; - }; diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 1df97dc9241..737e89979a6 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -231,8 +231,10 @@ void MergeTreeDataPartChecksums::addFile(const String & file_name, UInt64 file_s void MergeTreeDataPartChecksums::add(MergeTreeDataPartChecksums && rhs_checksums) { - for (auto & checksum : rhs_checksums.files) - files[std::move(checksum.first)] = std::move(checksum.second); + for (auto && checksum : rhs_checksums.files) + { + files[checksum.first] = std::move(checksum.second); + } rhs_checksums.files.clear(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index 06f1fb06f25..15acb88aa0f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -26,7 +26,7 @@ struct MergeTreeDataPartChecksum UInt64 uncompressed_size {}; uint128 uncompressed_hash {}; - MergeTreeDataPartChecksum() {} + MergeTreeDataPartChecksum() = default; MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_) : file_size(file_size_), file_hash(file_hash_) {} MergeTreeDataPartChecksum(UInt64 file_size_, uint128 file_hash_, UInt64 uncompressed_size_, uint128 uncompressed_hash_) : file_size(file_size_), file_hash(file_hash_), is_compressed(true), diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.h b/src/Storages/MergeTree/MergeTreeDataPartType.h index fecd9d00cdc..7cf23c7a045 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -25,7 +25,7 @@ public: }; MergeTreeDataPartType() : value(UNKNOWN) {} - MergeTreeDataPartType(Value value_) : value(value_) {} + MergeTreeDataPartType(Value value_) : value(value_) {} /// NOLINT bool operator==(const MergeTreeDataPartType & other) const { diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 646b672a1e9..63d2fe41e48 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -117,7 +117,7 @@ static RelativeSize convertAbsoluteSampleSizeToRelative(const ASTPtr & node, siz QueryPlanPtr MergeTreeDataSelectExecutor::read( const Names & column_names_to_return, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, const UInt64 max_block_size, @@ -130,13 +130,17 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( return std::make_unique(); const auto & settings = context->getSettingsRef(); + const auto & metadata_for_reading = storage_snapshot->getMetadataForQuery(); + + const auto & snapshot_data = assert_cast(*storage_snapshot->data); + const auto & parts = snapshot_data.parts; + if (!query_info.projection) { auto plan = readFromParts( - query_info.merge_tree_select_result_ptr ? MergeTreeData::DataPartsVector{} : data.getDataPartsVector(), + query_info.merge_tree_select_result_ptr ? MergeTreeData::DataPartsVector{} : parts, column_names_to_return, - metadata_snapshot, - metadata_snapshot, + storage_snapshot, query_info, context, max_block_size, @@ -146,7 +150,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( enable_parallel_reading); if (plan->isInitialized() && settings.allow_experimental_projection_optimization && settings.force_optimize_projection - && !metadata_snapshot->projections.empty()) + && !metadata_for_reading->projections.empty()) throw Exception( "No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1", ErrorCodes::PROJECTION_NOT_USED); @@ -178,8 +182,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( projection_plan = readFromParts( {}, query_info.projection->required_columns, - metadata_snapshot, - query_info.projection->desc->metadata, + storage_snapshot, query_info, context, max_block_size, @@ -822,7 +825,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd { auto [it, inserted] = merged_indices.try_emplace({index_helper->index.type, index_helper->getGranularity()}); if (inserted) - it->second.condition = index_helper->createIndexMergedCondtition(query_info, metadata_snapshot); + it->second.condition = index_helper->createIndexMergedCondition(query_info, metadata_snapshot); it->second.addIndex(index_helper); } @@ -1204,8 +1207,7 @@ MergeTreeDataSelectAnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names_to_return, - const StorageMetadataPtr & metadata_snapshot_base, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, const UInt64 max_block_size, @@ -1237,8 +1239,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( virt_column_names, data, query_info, - metadata_snapshot, - metadata_snapshot_base, + storage_snapshot, context, max_block_size, num_streams, @@ -1751,8 +1752,6 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( PartFilterCounters & counters, Poco::Logger * log) { - const Settings & settings = query_context->getSettings(); - /// process_parts prepare parts that have to be read for the query, /// returns false if duplicated parts' UUID have been met auto select_parts = [&] (MergeTreeData::DataPartsVector & selected_parts) -> bool @@ -1807,14 +1806,11 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( counters.num_granules_after_partition_pruner += num_granules; /// populate UUIDs and exclude ignored parts if enabled - if (part->uuid != UUIDHelpers::Nil) + if (part->uuid != UUIDHelpers::Nil && pinned_part_uuids->contains(part->uuid)) { - if (settings.experimental_query_deduplication_send_all_part_uuids || pinned_part_uuids->contains(part->uuid)) - { - auto result = temp_part_uuids.insert(part->uuid); - if (!result.second) - throw Exception("Found a part with the same UUID on the same replica.", ErrorCodes::LOGICAL_ERROR); - } + auto result = temp_part_uuids.insert(part->uuid); + if (!result.second) + throw Exception("Found a part with the same UUID on the same replica.", ErrorCodes::LOGICAL_ERROR); } selected_parts.push_back(part_or_projection); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 3dde324ce22..e0647aa1ed2 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -28,7 +28,7 @@ public: QueryPlanPtr read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, UInt64 max_block_size, @@ -41,8 +41,7 @@ public: QueryPlanPtr readFromParts( MergeTreeData::DataPartsVector parts, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot_base, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, UInt64 max_block_size, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index d16b5274a45..4805a273c70 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -281,6 +282,16 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( { TemporaryPart temp_part; Block & block = block_with_partition.block; + auto columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); + auto storage_snapshot = data.getStorageSnapshot(metadata_snapshot); + + if (!storage_snapshot->object_columns.empty()) + { + auto extended_storage_columns = storage_snapshot->getColumns( + GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects()); + + convertObjectsToTuples(columns, block, extended_storage_columns); + } static const String TMP_PREFIX = "tmp_insert_"; @@ -357,7 +368,6 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( for (const auto & ttl_entry : move_ttl_entries) updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false); - NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); ReservationPtr reservation = data.reserveSpacePreferringTTLRules(metadata_snapshot, expected_size, move_ttl_infos, time(nullptr), 0, true); VolumePtr volume = data.getStoragePolicy()->getVolume(0); @@ -426,7 +436,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( auto compression_codec = data.getContext()->chooseCompressionCodec(0, 0); const auto & index_factory = MergeTreeIndexFactory::instance(); - auto out = std::make_unique(new_data_part, metadata_snapshot,columns, + auto out = std::make_unique(new_data_part, metadata_snapshot, columns, index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec); out->writeWithPermutation(block, perm_ptr); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp index 2057dec957e..7b194de8103 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.cpp @@ -552,7 +552,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseASTEquals( for (const auto & f : value_field.get()) { - if ((f.isNull() && !is_nullable) || f.IsDecimal(f.getType())) + if ((f.isNull() && !is_nullable) || f.isDecimal(f.getType())) return false; mutable_column->insert(convertFieldToType(f, *actual_type, value_type.get())); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h index 5c6559ba298..27fd701c67b 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionBloomFilter.h @@ -38,7 +38,7 @@ public: ALWAYS_TRUE, }; - RPNElement(Function function_ = FUNCTION_UNKNOWN) : function(function_) {} + RPNElement(Function function_ = FUNCTION_UNKNOWN) : function(function_) {} /// NOLINT Function function = FUNCTION_UNKNOWN; std::vector> predicate; diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index f87584c9cd6..5ecb7b537e2 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -710,9 +710,14 @@ void bloomFilterIndexValidator(const IndexDescription & index, bool /*attach*/) const auto & array_type = assert_cast(*index_data_type); data_type = WhichDataType(array_type.getNestedType()); } + else if (data_type.isLowCarnality()) + { + const auto & low_cardinality = assert_cast(*index_data_type); + data_type = WhichDataType(low_cardinality.getDictionaryType()); + } if (!data_type.isString() && !data_type.isFixedString()) - throw Exception("Bloom filter index can be used only with `String`, `FixedString` column or Array with `String` or `FixedString` values column.", ErrorCodes::INCORRECT_QUERY); + throw Exception("Bloom filter index can be used only with `String`, `FixedString`, `LowCardinality(String)`, `LowCardinality(FixedString)` column or Array with `String` or `FixedString` values column.", ErrorCodes::INCORRECT_QUERY); } if (index.type == NgramTokenExtractor::getName()) diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.h b/src/Storages/MergeTree/MergeTreeIndexFullText.h index 1826719df0b..5f5956553dc 100644 --- a/src/Storages/MergeTree/MergeTreeIndexFullText.h +++ b/src/Storages/MergeTree/MergeTreeIndexFullText.h @@ -102,7 +102,7 @@ private: ALWAYS_TRUE, }; - RPNElement( + RPNElement( /// NOLINT Function function_ = FUNCTION_UNKNOWN, size_t key_column_ = 0, std::unique_ptr && const_bloom_filter_ = nullptr) : function(function_), key_column(key_column_), bloom_filter(std::move(const_bloom_filter_)) {} diff --git a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp index 30995a162dc..088029d9e8e 100644 --- a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp @@ -18,7 +18,7 @@ MergeTreeIndexGranuleHypothesis::MergeTreeIndexGranuleHypothesis(const String & { } -MergeTreeIndexGranuleHypothesis::MergeTreeIndexGranuleHypothesis(const String & index_name_, const bool met_) +MergeTreeIndexGranuleHypothesis::MergeTreeIndexGranuleHypothesis(const String & index_name_, bool met_) : index_name(index_name_), is_empty(false), met(met_) { } @@ -84,7 +84,7 @@ MergeTreeIndexConditionPtr MergeTreeIndexHypothesis::createIndexCondition( throw Exception("Not supported", ErrorCodes::LOGICAL_ERROR); } -MergeTreeIndexMergedConditionPtr MergeTreeIndexHypothesis::createIndexMergedCondtition( +MergeTreeIndexMergedConditionPtr MergeTreeIndexHypothesis::createIndexMergedCondition( const SelectQueryInfo & query_info, StorageMetadataPtr storage_metadata) const { return std::make_shared( diff --git a/src/Storages/MergeTree/MergeTreeIndexHypothesis.h b/src/Storages/MergeTree/MergeTreeIndexHypothesis.h index bbdf70a052c..578bb6f3f7a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexHypothesis.h +++ b/src/Storages/MergeTree/MergeTreeIndexHypothesis.h @@ -16,7 +16,7 @@ struct MergeTreeIndexGranuleHypothesis : public IMergeTreeIndexGranule MergeTreeIndexGranuleHypothesis( const String & index_name_, - const bool met_); + bool met_); void serializeBinary(WriteBuffer & ostr) const override; void deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version) override; @@ -55,7 +55,7 @@ private: class MergeTreeIndexHypothesis : public IMergeTreeIndex { public: - MergeTreeIndexHypothesis( + explicit MergeTreeIndexHypothesis( const IndexDescription & index_) : IMergeTreeIndex(index_) {} @@ -70,7 +70,7 @@ public: MergeTreeIndexConditionPtr createIndexCondition( const SelectQueryInfo & query, ContextPtr context) const override; - MergeTreeIndexMergedConditionPtr createIndexMergedCondtition( + MergeTreeIndexMergedConditionPtr createIndexMergedCondition( const SelectQueryInfo & query_info, StorageMetadataPtr storage_metadata) const override; bool mayBenefitFromIndexForIn(const ASTPtr & node) const override; diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.h b/src/Storages/MergeTree/MergeTreeIndexMinMax.h index 0e05e25fb36..9f78c86a498 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMinMax.h +++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.h @@ -68,7 +68,7 @@ private: class MergeTreeIndexMinMax : public IMergeTreeIndex { public: - MergeTreeIndexMinMax(const IndexDescription & index_) + explicit MergeTreeIndexMinMax(const IndexDescription & index_) : IMergeTreeIndex(index_) {} @@ -83,7 +83,7 @@ public: bool mayBenefitFromIndexForIn(const ASTPtr & node) const override; const char* getSerializedFileExtension() const override { return ".idx2"; } - MergeTreeIndexFormat getDeserializedFormat(const DiskPtr disk, const std::string & path_prefix) const override; + MergeTreeIndexFormat getDeserializedFormat(const DiskPtr disk, const std::string & path_prefix) const override; /// NOLINT }; } diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 03a3b811de6..95ec8acc3b9 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -54,7 +54,7 @@ MergeTreeIndexReader::MergeTreeIndexReader( std::move(settings)); version = index_format.version; - stream->adjustForRange(MarkRange(0, getLastMark(all_mark_ranges_))); + stream->adjustRightMark(getLastMark(all_mark_ranges_)); stream->seekToStart(); } diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h index 1e001d01ada..984a2bb7762 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.h +++ b/src/Storages/MergeTree/MergeTreeIndices.h @@ -29,7 +29,7 @@ struct MergeTreeIndexFormat MergeTreeIndexVersion version; const char* extension; - operator bool() const { return version != 0; } + operator bool() const { return version != 0; } /// NOLINT }; /// Stores some info about a single block of data. @@ -122,7 +122,7 @@ using MergeTreeIndexMergedConditions = std::vector & x) const { UInt8 type = Field::Types::Decimal32; diff --git a/src/Storages/MergeTree/MergeTreePartsMover.h b/src/Storages/MergeTree/MergeTreePartsMover.h index a1afadec7fa..45bbce01d00 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.h +++ b/src/Storages/MergeTree/MergeTreePartsMover.h @@ -36,7 +36,7 @@ private: using AllowedMovingPredicate = std::function &, String * reason)>; public: - MergeTreePartsMover(MergeTreeData * data_) + explicit MergeTreePartsMover(MergeTreeData * data_) : data(data_) , log(&Poco::Logger::get("MergeTreePartsMover")) { @@ -59,7 +59,6 @@ public: /// merge or mutation. void swapClonedPart(const std::shared_ptr & cloned_parts) const; -public: /// Can stop background moves and moves from queries ActionBlocker moves_blocker; diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 98b8de6a0ea..d8dba458203 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -695,10 +695,10 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar { auto block = prev_reader->sample_block.cloneWithColumns(read_result.columns); auto block_before_prewhere = read_result.block_before_prewhere; - for (auto & ctn : block) + for (const auto & column : block) { - if (block_before_prewhere.has(ctn.name)) - block_before_prewhere.erase(ctn.name); + if (block_before_prewhere.has(column.name)) + block_before_prewhere.erase(column.name); } if (block_before_prewhere) @@ -707,11 +707,11 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar { auto old_columns = block_before_prewhere.getColumns(); filterColumns(old_columns, read_result.getFilterOriginal()->getData()); - block_before_prewhere.setColumns(std::move(old_columns)); + block_before_prewhere.setColumns(old_columns); } - for (auto && ctn : block_before_prewhere) - block.insert(std::move(ctn)); + for (auto & column : block_before_prewhere) + block.insert(std::move(column)); } merge_tree_reader->evaluateMissingDefaults(block, columns); } diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index c89affb5365..87839edc46f 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -18,21 +18,21 @@ namespace ErrorCodes namespace DB { MergeTreeReadPool::MergeTreeReadPool( - const size_t threads_, - const size_t sum_marks_, - const size_t min_marks_for_concurrent_read_, + size_t threads_, + size_t sum_marks_, + size_t min_marks_for_concurrent_read_, RangesInDataParts && parts_, const MergeTreeData & data_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, const Names & column_names_, const BackoffSettings & backoff_settings_, size_t preferred_block_size_bytes_, - const bool do_not_steal_tasks_) + bool do_not_steal_tasks_) : backoff_settings{backoff_settings_} , backoff_state{threads_} , data{data_} - , metadata_snapshot{metadata_snapshot_} + , storage_snapshot{storage_snapshot_} , column_names{column_names_} , do_not_steal_tasks{do_not_steal_tasks_} , predict_block_size_bytes{preferred_block_size_bytes_ > 0} @@ -45,7 +45,7 @@ MergeTreeReadPool::MergeTreeReadPool( } -MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, const size_t thread, const Names & ordered_names) +MergeTreeReadTaskPtr MergeTreeReadPool::getTask(size_t min_marks_to_read, size_t thread, const Names & ordered_names) { const std::lock_guard lock{mutex}; @@ -146,10 +146,10 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, Block MergeTreeReadPool::getHeader() const { - return metadata_snapshot->getSampleBlockForColumns(column_names, data.getVirtuals(), data.getStorageID()); + return storage_snapshot->getSampleBlockForColumns(column_names); } -void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInfo info) +void MergeTreeReadPool::profileFeedback(ReadBufferFromFileBase::ProfileInfo info) { if (backoff_settings.min_read_latency_ms == 0 || do_not_steal_tasks) return; @@ -192,7 +192,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf std::vector MergeTreeReadPool::fillPerPartInfo(const RangesInDataParts & parts) { std::vector per_part_sum_marks; - Block sample_block = metadata_snapshot->getSampleBlock(); + Block sample_block = storage_snapshot->metadata->getSampleBlock(); is_part_on_remote_disk.resize(parts.size()); for (const auto i : collections::range(0, parts.size())) @@ -209,7 +209,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo(const RangesInDataParts & per_part_sum_marks.push_back(sum_marks); - auto task_columns = getReadTaskColumns(data, metadata_snapshot, part.data_part, column_names, prewhere_info); + auto task_columns = getReadTaskColumns(data, storage_snapshot, part.data_part, column_names, prewhere_info); auto size_predictor = !predict_block_size_bytes ? nullptr : MergeTreeBaseSelectProcessor::getSizePredictor(part.data_part, task_columns, sample_block); @@ -232,8 +232,8 @@ std::vector MergeTreeReadPool::fillPerPartInfo(const RangesInDataParts & void MergeTreeReadPool::fillPerThreadInfo( - const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, - const RangesInDataParts & parts, const size_t min_marks_for_concurrent_read) + size_t threads, size_t sum_marks, std::vector per_part_sum_marks, + const RangesInDataParts & parts, size_t min_marks_for_concurrent_read) { threads_tasks.resize(threads); if (parts.empty()) diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index aac4d5016a2..0c93c701448 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -40,7 +40,7 @@ public: size_t min_concurrency = 1; /// Constants above is just an example. - BackoffSettings(const Settings & settings) + explicit BackoffSettings(const Settings & settings) : min_read_latency_ms(settings.read_backoff_min_latency_ms.totalMilliseconds()), max_throughput(settings.read_backoff_max_throughput), min_interval_between_events_ms(settings.read_backoff_min_interval_between_events_ms.totalMilliseconds()), @@ -63,27 +63,27 @@ private: Stopwatch time_since_prev_event {CLOCK_MONOTONIC_COARSE}; size_t num_events = 0; - BackoffState(size_t threads) : current_threads(threads) {} + explicit BackoffState(size_t threads) : current_threads(threads) {} }; BackoffState backoff_state; public: MergeTreeReadPool( - const size_t threads_, const size_t sum_marks_, const size_t min_marks_for_concurrent_read_, - RangesInDataParts && parts_, const MergeTreeData & data_, const StorageMetadataPtr & metadata_snapshot_, + size_t threads_, size_t sum_marks_, size_t min_marks_for_concurrent_read_, + RangesInDataParts && parts_, const MergeTreeData & data_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, const Names & column_names_, const BackoffSettings & backoff_settings_, size_t preferred_block_size_bytes_, - const bool do_not_steal_tasks_ = false); + bool do_not_steal_tasks_ = false); - MergeTreeReadTaskPtr getTask(const size_t min_marks_to_read, const size_t thread, const Names & ordered_names); + MergeTreeReadTaskPtr getTask(size_t min_marks_to_read, size_t thread, const Names & ordered_names); /** Each worker could call this method and pass information about read performance. * If read performance is too low, pool could decide to lower number of threads: do not assign more tasks to several threads. * This allows to overcome excessive load to disk subsystem, when reads are not from page cache. */ - void profileFeedback(const ReadBufferFromFileBase::ProfileInfo info); + void profileFeedback(ReadBufferFromFileBase::ProfileInfo info); Block getHeader() const; @@ -91,11 +91,11 @@ private: std::vector fillPerPartInfo(const RangesInDataParts & parts); void fillPerThreadInfo( - const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, - const RangesInDataParts & parts, const size_t min_marks_for_concurrent_read); + size_t threads, size_t sum_marks, std::vector per_part_sum_marks, + const RangesInDataParts & parts, size_t min_marks_for_concurrent_read); const MergeTreeData & data; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; const Names column_names; bool do_not_steal_tasks; bool predict_block_size_bytes; diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 6b265b9460c..b943c3c8718 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -26,14 +26,14 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) : IMergeTreeReader( - std::move(data_part_), - std::move(columns_), + data_part_, + columns_, metadata_snapshot_, uncompressed_cache_, mark_cache_, - std::move(mark_ranges_), - std::move(settings_), - std::move(avg_value_size_hints_)) + mark_ranges_, + settings_, + avg_value_size_hints_) , marks_loader( data_part->volume->getDisk(), mark_cache, @@ -52,6 +52,15 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( auto name_and_type = columns.begin(); for (size_t i = 0; i < columns_num; ++i, ++name_and_type) { + if (name_and_type->isSubcolumn()) + { + auto storage_column_from_part = getColumnFromPart( + {name_and_type->getNameInStorage(), name_and_type->getTypeInStorage()}); + + if (!storage_column_from_part.type->tryGetSubcolumnType(name_and_type->getSubcolumnName())) + continue; + } + auto column_from_part = getColumnFromPart(*name_and_type); auto position = data_part->getColumnPosition(column_from_part.getNameInStorage()); @@ -96,6 +105,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( cached_buffer = std::move(buffer); data_buffer = cached_buffer.get(); + compressed_data_buffer = cached_buffer.get(); } else { @@ -114,6 +124,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( non_cached_buffer = std::move(buffer); data_buffer = non_cached_buffer.get(); + compressed_data_buffer = non_cached_buffer.get(); } } catch (...) @@ -260,10 +271,7 @@ void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); try { - if (cached_buffer) - cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); - if (non_cached_buffer) - non_cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + compressed_data_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); } catch (Exception & e) { @@ -288,10 +296,7 @@ void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) return; last_right_offset = 0; // Zero value means the end of file. - if (cached_buffer) - cached_buffer->setReadUntilEnd(); - if (non_cached_buffer) - non_cached_buffer->setReadUntilEnd(); + data_buffer->setReadUntilEnd(); } else { @@ -299,10 +304,7 @@ void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) return; last_right_offset = right_offset; - if (cached_buffer) - cached_buffer->setReadUntilPosition(right_offset); - if (non_cached_buffer) - non_cached_buffer->setReadUntilPosition(right_offset); + data_buffer->setReadUntilPosition(right_offset); } } diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 381b212df3c..aa0eb949aa1 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -41,6 +41,7 @@ private: bool isContinuousReading(size_t mark, size_t column_position); ReadBuffer * data_buffer; + CompressedReadBufferBase * compressed_data_buffer; std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; diff --git a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp index 8a69183e858..c1b0067dbb0 100644 --- a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -20,9 +21,15 @@ MergeTreeReaderInMemory::MergeTreeReaderInMemory( const StorageMetadataPtr & metadata_snapshot_, MarkRanges mark_ranges_, MergeTreeReaderSettings settings_) - : IMergeTreeReader(data_part_, std::move(columns_), metadata_snapshot_, - nullptr, nullptr, std::move(mark_ranges_), - std::move(settings_), {}) + : IMergeTreeReader( + data_part_, + columns_, + metadata_snapshot_, + nullptr, + nullptr, + mark_ranges_, + settings_, + {}) , part_in_memory(std::move(data_part_)) { for (const auto & name_and_type : columns) diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 117ef43ecb2..b337bd62dd3 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -42,7 +42,8 @@ MergeTreeReaderStream::MergeTreeReaderStream( { size_t left_mark = mark_range.begin; size_t right_mark = mark_range.end; - auto [_, mark_range_bytes] = getRightOffsetAndBytesRange(left_mark, right_mark); + size_t left_offset = left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0; + auto mark_range_bytes = getRightOffset(right_mark) - left_offset; max_mark_range_bytes = std::max(max_mark_range_bytes, mark_range_bytes); sum_mark_range_bytes += mark_range_bytes; @@ -85,6 +86,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( cached_buffer = std::move(buffer); data_buffer = cached_buffer.get(); + compressed_data_buffer = cached_buffer.get(); } else { @@ -102,22 +104,21 @@ MergeTreeReaderStream::MergeTreeReaderStream( non_cached_buffer = std::move(buffer); data_buffer = non_cached_buffer.get(); + compressed_data_buffer = non_cached_buffer.get(); } } -std::pair MergeTreeReaderStream::getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark_non_included) +size_t MergeTreeReaderStream::getRightOffset(size_t right_mark_non_included) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. /// Special case, can happen in Collapsing/Replacing engines if (marks_count == 0) - return std::make_pair(0, 0); + return 0; - assert(left_mark < marks_count); assert(right_mark_non_included <= marks_count); - assert(left_mark <= right_mark_non_included); size_t result_right_offset; if (0 < right_mark_non_included && right_mark_non_included < marks_count) @@ -177,30 +178,20 @@ std::pair MergeTreeReaderStream::getRightOffsetAndBytesRange(siz } } else if (right_mark_non_included == 0) - { result_right_offset = marks_loader.getMark(right_mark_non_included).offset_in_compressed_file; - } else - { result_right_offset = file_size; - } - size_t mark_range_bytes = result_right_offset - (left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0); - - return std::make_pair(result_right_offset, mark_range_bytes); + return result_right_offset; } - void MergeTreeReaderStream::seekToMark(size_t index) { MarkInCompressedFile mark = marks_loader.getMark(index); try { - if (cached_buffer) - cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); - if (non_cached_buffer) - non_cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + compressed_data_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); } catch (Exception & e) { @@ -220,10 +211,7 @@ void MergeTreeReaderStream::seekToStart() { try { - if (cached_buffer) - cached_buffer->seek(0, 0); - if (non_cached_buffer) - non_cached_buffer->seek(0, 0); + compressed_data_buffer->seek(0, 0); } catch (Exception & e) { @@ -236,24 +224,21 @@ void MergeTreeReaderStream::seekToStart() } -void MergeTreeReaderStream::adjustForRange(MarkRange range) +void MergeTreeReaderStream::adjustRightMark(size_t right_mark) { /** * Note: this method is called multiple times for the same range of marks -- each time we * read from stream, but we must update last_right_offset only if it is bigger than * the last one to avoid redundantly cancelling prefetches. */ - auto [right_offset, _] = getRightOffsetAndBytesRange(range.begin, range.end); + auto right_offset = getRightOffset(right_mark); if (!right_offset) { if (last_right_offset && *last_right_offset == 0) return; last_right_offset = 0; // Zero value means the end of file. - if (cached_buffer) - cached_buffer->setReadUntilEnd(); - if (non_cached_buffer) - non_cached_buffer->setReadUntilEnd(); + data_buffer->setReadUntilEnd(); } else { @@ -261,10 +246,7 @@ void MergeTreeReaderStream::adjustForRange(MarkRange range) return; last_right_offset = right_offset; - if (cached_buffer) - cached_buffer->setReadUntilPosition(right_offset); - if (non_cached_buffer) - non_cached_buffer->setReadUntilPosition(right_offset); + data_buffer->setReadUntilPosition(right_offset); } } diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index 96f3ea31fec..0b6b86e7f82 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -34,12 +34,13 @@ public: * Does buffer need to know something about mark ranges bounds it is going to read? * (In case of MergeTree* tables). Mostly needed for reading from remote fs. */ - void adjustForRange(MarkRange range); + void adjustRightMark(size_t right_mark); ReadBuffer * data_buffer; + CompressedReadBufferBase * compressed_data_buffer; private: - std::pair getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark_non_included); + size_t getRightOffset(size_t right_mark_non_included); DiskPtr disk; std::string path_prefix; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 77c0e8332d4..7d7975e0bc0 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -36,14 +36,14 @@ MergeTreeReaderWide::MergeTreeReaderWide( const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) : IMergeTreeReader( - std::move(data_part_), - std::move(columns_), + data_part_, + columns_, metadata_snapshot_, uncompressed_cache_, - std::move(mark_cache_), - std::move(mark_ranges_), - std::move(settings_), - std::move(avg_value_size_hints_)) + mark_cache_, + mark_ranges_, + settings_, + avg_value_size_hints_) { try { @@ -212,7 +212,7 @@ static ReadBuffer * getStream( return nullptr; MergeTreeReaderStream & stream = *it->second; - stream.adjustForRange(MarkRange(seek_to_start ? 0 : from_mark, current_task_last_mark)); + stream.adjustRightMark(current_task_last_mark); if (seek_to_start) stream.seekToStart(); diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 2d4d3617cee..3245134c470 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -9,7 +9,7 @@ namespace DB MergeTreeSelectProcessor::MergeTreeSelectProcessor( const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const MergeTreeData::DataPartPtr & owned_data_part_, UInt64 max_block_size_rows_, size_t preferred_block_size_bytes_, @@ -25,13 +25,13 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( bool has_limit_below_one_block_, std::optional extension_) : MergeTreeBaseSelectProcessor{ - metadata_snapshot_->getSampleBlockForColumns(required_columns_, storage_.getVirtuals(), storage_.getStorageID()), - storage_, metadata_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, + storage_snapshot_->getSampleBlockForColumns(required_columns_), + storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, use_uncompressed_cache_, virt_column_names_, extension_}, required_columns{std::move(required_columns_)}, data_part{owned_data_part_}, - sample_block(metadata_snapshot_->getSampleBlock()), + sample_block(storage_snapshot_->metadata->getSampleBlock()), all_mark_ranges(std::move(mark_ranges_)), part_index_in_query(part_index_in_query_), has_limit_below_one_block(has_limit_below_one_block_), @@ -48,7 +48,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( void MergeTreeSelectProcessor::initializeReaders() { task_columns = getReadTaskColumns( - storage, metadata_snapshot, data_part, + storage, storage_snapshot, data_part, required_columns, prewhere_info); /// Will be used to distinguish between PREWHERE and WHERE columns when applying filter @@ -60,12 +60,12 @@ void MergeTreeSelectProcessor::initializeReaders() owned_mark_cache = storage.getContext()->getMarkCache(); - reader = data_part->getReader(task_columns.columns, metadata_snapshot, all_mark_ranges, - owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); + reader = data_part->getReader(task_columns.columns, storage_snapshot->getMetadataForQuery(), + all_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); if (prewhere_info) - pre_reader = data_part->getReader(task_columns.pre_columns, metadata_snapshot, all_mark_ranges, - owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); + pre_reader = data_part->getReader(task_columns.pre_columns, storage_snapshot->getMetadataForQuery(), + all_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 2ecdc3b59a8..4b3a46fc53c 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -18,7 +18,7 @@ class MergeTreeSelectProcessor : public MergeTreeBaseSelectProcessor public: MergeTreeSelectProcessor( const MergeTreeData & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot_, const MergeTreeData::DataPartPtr & owned_data_part, UInt64 max_block_size_rows, size_t preferred_block_size_bytes, diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 37012aa6570..5dbc59ba2d5 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -11,15 +11,15 @@ namespace ErrorCodes MergeTreeSequentialSource::MergeTreeSequentialSource( const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, MergeTreeData::DataPartPtr data_part_, Names columns_to_read_, bool read_with_direct_io_, bool take_column_types_from_storage, bool quiet) - : SourceWithProgress(metadata_snapshot_->getSampleBlockForColumns(columns_to_read_, storage_.getVirtuals(), storage_.getStorageID())) + : SourceWithProgress(storage_snapshot_->getSampleBlockForColumns(columns_to_read_)) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , data_part(std::move(data_part_)) , columns_to_read(std::move(columns_to_read_)) , read_with_direct_io(read_with_direct_io_) @@ -41,11 +41,12 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( addTotalRowsApprox(data_part->rows_count); /// Add columns because we don't want to read empty blocks - injectRequiredColumns(storage, metadata_snapshot, data_part, columns_to_read); + injectRequiredColumns(storage, storage_snapshot->metadata, data_part, columns_to_read); NamesAndTypesList columns_for_reader; if (take_column_types_from_storage) { - columns_for_reader = metadata_snapshot->getColumns().getByNames(ColumnsDescription::AllPhysical, columns_to_read, false); + auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects(); + columns_for_reader = storage_snapshot->getColumnsByNames(options, columns_to_read); } else { @@ -63,7 +64,7 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( .save_marks_in_cache = false }; - reader = data_part->getReader(columns_for_reader, metadata_snapshot, + reader = data_part->getReader(columns_for_reader, storage_snapshot->metadata, MarkRanges{MarkRange(0, data_part->getMarksCount())}, /* uncompressed_cache = */ nullptr, mark_cache.get(), reader_settings); } diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.h b/src/Storages/MergeTree/MergeTreeSequentialSource.h index 7eefdd9335b..962b2035b16 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.h +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.h @@ -14,7 +14,7 @@ class MergeTreeSequentialSource : public SourceWithProgress public: MergeTreeSequentialSource( const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, MergeTreeData::DataPartPtr data_part_, Names columns_to_read_, bool read_with_direct_io_, @@ -35,7 +35,7 @@ protected: private: const MergeTreeData & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; /// Data part will not be removed if the pointer owns it MergeTreeData::DataPartPtr data_part; @@ -58,7 +58,6 @@ private: /// current row at which we stop reading size_t current_row = 0; -private: /// Closes readers and unlock part locks void finish(); }; diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index b120c230005..c1cc3b6ed3c 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -122,7 +122,7 @@ void MergeTreeSettings::sanityCheck(const Settings & query_settings) const { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "min_bytes_to_rebalance_partition_over_jbod: {} is lower than specified max_bytes_to_merge_at_max_space_in_pool / 150: {}", + "min_bytes_to_rebalance_partition_over_jbod: {} is lower than specified max_bytes_to_merge_at_max_space_in_pool / 1024: {}", min_bytes_to_rebalance_partition_over_jbod, max_bytes_to_merge_at_max_space_in_pool / 1024); } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 6861599a1ac..3eb59b25562 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -150,6 +150,7 @@ struct Settings; M(UInt64, replicated_max_parallel_fetches_for_table, 0, "Obsolete setting, does nothing.", 0) \ M(Bool, write_final_mark, true, "Obsolete setting, does nothing.", 0) /// Settings that should not change after the creation of a table. + /// NOLINTNEXTLINE #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \ M(index_granularity) diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index ea252954b7e..97bbfc17e9d 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -52,7 +52,14 @@ void MergeTreeSink::consume(Chunk chunk) auto block = getHeader().cloneWithColumns(chunk.detachColumns()); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); - std::vector partitions; + + using DelayedPartitions = std::vector; + DelayedPartitions partitions; + + const Settings & settings = context->getSettingsRef(); + size_t streams = 0; + bool support_parallel_write = false; + for (auto & current_block : part_blocks) { Stopwatch watch; @@ -67,9 +74,12 @@ void MergeTreeSink::consume(Chunk chunk) if (!temp_part.part) continue; + if (!support_parallel_write && temp_part.part->volume->getDisk()->supportParallelWrite()) + support_parallel_write = true; + if (storage.getDeduplicationLog()) { - const String & dedup_token = context->getSettingsRef().insert_deduplication_token; + const String & dedup_token = settings.insert_deduplication_token; if (!dedup_token.empty()) { /// multiple blocks can be inserted within the same insert query @@ -79,6 +89,24 @@ void MergeTreeSink::consume(Chunk chunk) } } + size_t max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE; + if (!support_parallel_write || settings.max_insert_delayed_streams_for_parallel_write.changed) + max_insert_delayed_streams_for_parallel_write = settings.max_insert_delayed_streams_for_parallel_write; + + /// In case of too much columns/parts in block, flush explicitly. + streams += temp_part.streams.size(); + if (streams > max_insert_delayed_streams_for_parallel_write) + { + finishDelayedChunk(); + delayed_chunk = std::make_unique(); + delayed_chunk->partitions = std::move(partitions); + finishDelayedChunk(); + + streams = 0; + support_parallel_write = false; + partitions = DelayedPartitions{}; + } + partitions.emplace_back(MergeTreeSink::DelayedChunk::Partition { .temp_part = std::move(temp_part), diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp index 6a44da06f1f..063f018b1a4 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.cpp @@ -13,15 +13,15 @@ namespace ErrorCodes } MergeTreeThreadSelectProcessor::MergeTreeThreadSelectProcessor( - const size_t thread_, + size_t thread_, const MergeTreeReadPoolPtr & pool_, - const size_t min_marks_to_read_, - const UInt64 max_block_size_rows_, + size_t min_marks_to_read_, + UInt64 max_block_size_rows_, size_t preferred_block_size_bytes_, size_t preferred_max_column_in_block_size_bytes_, const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, - const bool use_uncompressed_cache_, + const StorageSnapshotPtr & storage_snapshot_, + bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, ExpressionActionsSettings actions_settings, const MergeTreeReaderSettings & reader_settings_, @@ -29,7 +29,7 @@ MergeTreeThreadSelectProcessor::MergeTreeThreadSelectProcessor( std::optional extension_) : MergeTreeBaseSelectProcessor{ - pool_->getHeader(), storage_, metadata_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, + pool_->getHeader(), storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, use_uncompressed_cache_, virt_column_names_, extension_}, thread{thread_}, @@ -103,6 +103,7 @@ void MergeTreeThreadSelectProcessor::finalizeNewTask() /// Allows pool to reduce number of threads in case of too slow reads. auto profile_callback = [this](ReadBufferFromFileBase::ProfileInfo info_) { pool->profileFeedback(info_); }; + const auto & metadata_snapshot = storage_snapshot->metadata; if (!reader) { diff --git a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h index 110c4fa34e6..3bba42bed28 100644 --- a/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeThreadSelectProcessor.h @@ -15,15 +15,15 @@ class MergeTreeThreadSelectProcessor final : public MergeTreeBaseSelectProcessor { public: MergeTreeThreadSelectProcessor( - const size_t thread_, + size_t thread_, const std::shared_ptr & pool_, - const size_t min_marks_to_read_, - const UInt64 max_block_size_, + size_t min_marks_to_read_, + UInt64 max_block_size_, size_t preferred_block_size_bytes_, size_t preferred_max_column_in_block_size_bytes_, const MergeTreeData & storage_, - const StorageMetadataPtr & metadata_snapshot_, - const bool use_uncompressed_cache_, + const StorageSnapshotPtr & storage_snapshot_, + bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, ExpressionActionsSettings actions_settings, const MergeTreeReaderSettings & reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 7a85791d172..737e0c9d4b7 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -399,6 +399,7 @@ bool MergeTreeWhereOptimizer::cannotBeMoved(const ASTPtr & ptr, bool is_final) c return true; /// disallow GLOBAL IN, GLOBAL NOT IN + /// TODO why? if ("globalIn" == function_ptr->name || "globalNotIn" == function_ptr->name) return true; diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index 4aa7aa532a8..fa14fea94d1 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -79,8 +79,6 @@ private: /// Transform Conditions list to WHERE or PREWHERE expression. static ASTPtr reconstruct(const Conditions & conditions); - void optimizeConjunction(ASTSelectQuery & select, ASTFunction * const fun) const; - void optimizeArbitrary(ASTSelectQuery & select) const; UInt64 getIdentifiersColumnSize(const NameSet & identifiers) const; diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 9a41204c7e5..d7cddfe9c14 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -109,6 +109,8 @@ void MergeTreeWriteAheadLog::rotate(const std::unique_lock &) + toString(min_block_number) + "_" + toString(max_block_number) + WAL_FILE_EXTENSION; + /// Finalize stream before file rename + out->finalize(); disk->replaceFile(path, storage.getRelativeDataPath() + new_name); init(); } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index e262217d17e..f94c89e20bd 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -55,11 +55,15 @@ struct MergedBlockOutputStream::Finalizer::Impl { IMergeTreeDataPartWriter & writer; MergeTreeData::MutableDataPartPtr part; + NameSet files_to_remove_after_finish; std::vector> written_files; bool sync; - Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, bool sync_) - : writer(writer_), part(std::move(part_)), sync(sync_) {} + Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, const NameSet & files_to_remove_after_finish_, bool sync_) + : writer(writer_) + , part(std::move(part_)) + , files_to_remove_after_finish(files_to_remove_after_finish_) + , sync(sync_) {} void finish(); }; @@ -75,6 +79,10 @@ void MergedBlockOutputStream::Finalizer::Impl::finish() { writer.finish(sync); + auto disk = part->volume->getDisk(); + for (const auto & file_name: files_to_remove_after_finish) + disk->removeFile(part->getFullRelativePath() + file_name); + for (auto & file : written_files) { file->finalize(); @@ -97,8 +105,8 @@ MergedBlockOutputStream::Finalizer::~Finalizer() } } -MergedBlockOutputStream::Finalizer::Finalizer(Finalizer &&) = default; -MergedBlockOutputStream::Finalizer & MergedBlockOutputStream::Finalizer::operator=(Finalizer &&) = default; +MergedBlockOutputStream::Finalizer::Finalizer(Finalizer &&) noexcept = default; +MergedBlockOutputStream::Finalizer & MergedBlockOutputStream::Finalizer::operator=(Finalizer &&) noexcept = default; MergedBlockOutputStream::Finalizer::Finalizer(std::unique_ptr impl_) : impl(std::move(impl_)) {} void MergedBlockOutputStream::finalizePart( @@ -133,19 +141,20 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( projection_part->checksums.getTotalSizeOnDisk(), projection_part->checksums.getTotalChecksumUInt128()); + NameSet files_to_remove_after_sync; if (reset_columns) { auto part_columns = total_columns_list ? *total_columns_list : columns_list; auto serialization_infos = new_part->getSerializationInfos(); serialization_infos.replaceData(new_serialization_infos); - removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums); + files_to_remove_after_sync = removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums); new_part->setColumns(part_columns); new_part->setSerializationInfos(serialization_infos); } - auto finalizer = std::make_unique(*writer, new_part, sync); + auto finalizer = std::make_unique(*writer, new_part, files_to_remove_after_sync, sync); if (new_part->isStoredOnDisk()) finalizer->written_files = finalizePartOnDisk(new_part, checksums); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index b38395d56c2..c17cfd22cd8 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -42,8 +42,8 @@ public: explicit Finalizer(std::unique_ptr impl_); ~Finalizer(); - Finalizer(Finalizer &&); - Finalizer & operator=(Finalizer &&); + Finalizer(Finalizer &&) noexcept; + Finalizer & operator=(Finalizer &&) noexcept; void finish(); }; diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 2cc20186808..5a706165000 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -74,9 +74,18 @@ MergedColumnOnlyOutputStream::fillChecksums( serialization_infos.replaceData(new_serialization_infos); auto removed_files = removeEmptyColumnsFromPart(new_part, columns, serialization_infos, checksums); + + auto disk = new_part->volume->getDisk(); for (const String & removed_file : removed_files) + { + auto file_path = new_part->getFullRelativePath() + removed_file; + /// Can be called multiple times, don't need to remove file twice + if (disk->exists(file_path)) + disk->removeFile(file_path); + if (all_checksums.files.count(removed_file)) all_checksums.files.erase(removed_file); + } new_part->setColumns(columns); new_part->setSerializationInfos(serialization_infos); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 16dd578a7eb..3f220566260 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -13,7 +13,7 @@ namespace ProfileEvents namespace DB { -std::pair MutateFromLogEntryTask::prepare() +ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() { const String & source_part_name = entry.source_parts.at(0); const auto storage_settings_ptr = storage.getSettings(); @@ -23,7 +23,11 @@ std::pair MutateFromLogEntry if (!source_part) { LOG_DEBUG(log, "Source part {} for {} is not ready; will try to fetch it instead", source_part_name, entry.new_part_name); - return {false, {}}; + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; } if (source_part->name != source_part_name) @@ -33,7 +37,12 @@ std::pair MutateFromLogEntry "Possibly the mutation of this part is not needed and will be skipped. " "This shouldn't happen often.", source_part_name, source_part->name, entry.new_part_name); - return {false, {}}; + + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; } /// TODO - some better heuristic? @@ -48,7 +57,33 @@ std::pair MutateFromLogEntry if (!replica.empty()) { LOG_DEBUG(log, "Prefer to fetch {} from replica {}", entry.new_part_name, replica); - return {false, {}}; + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } + } + + /// In some use cases merging can be more expensive than fetching + /// and it may be better to spread merges tasks across the replicas + /// instead of doing exactly the same merge cluster-wise + + if (storage.merge_strategy_picker.shouldMergeOnSingleReplica(entry)) + { + std::optional replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); + if (replica_to_execute_merge) + { + LOG_DEBUG(log, + "Prefer fetching part {} from replica {} due to execute_merges_on_single_replica_time_threshold", + entry.new_part_name, replica_to_execute_merge.value()); + + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } } @@ -73,13 +108,41 @@ std::pair MutateFromLogEntry future_mutated_part->updatePath(storage, reserved_space.get()); future_mutated_part->type = source_part->getType(); + if (storage_settings_ptr->allow_remote_fs_zero_copy_replication) + { + if (auto disk = reserved_space->getDisk(); disk->getType() == DB::DiskType::S3) + { + String dummy; + if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, true, dummy).empty()) + { + LOG_DEBUG(log, "Mutation of part {} finished by some other replica, will download merged part", entry.new_part_name); + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = true, + .part_log_writer = {} + }; + } + + zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); + + if (!zero_copy_lock) + { + LOG_DEBUG(log, "Mutation of part {} started by some other replica, will wait it and fetch merged part", entry.new_part_name); + return PrepareResult{ + .prepared_successfully = false, + .need_to_check_missing_part_in_fetch = false, + .part_log_writer = {} + }; + } + } + } + + const Settings & settings = storage.getContext()->getSettingsRef(); merge_mutate_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_mutated_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); stopwatch_ptr = std::make_unique(); @@ -95,7 +158,7 @@ std::pair MutateFromLogEntry for (auto & item : future_mutated_part->parts) priority += item->getBytesOnDisk(); - return {true, [this] (const ExecutionStatus & execution_status) + return {true, true, [this] (const ExecutionStatus & execution_status) { storage.writePartLog( PartLogElement::MUTATE_PART, execution_status, stopwatch_ptr->elapsed(), @@ -140,6 +203,12 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit throw; } + if (zero_copy_lock) + { + LOG_DEBUG(log, "Removing zero-copy lock"); + zero_copy_lock->lock->unlock(); + } + /** With `ZSESSIONEXPIRED` or `ZOPERATIONTIMEOUT`, we can inadvertently roll back local changes to the parts. * This is not a problem, because in this case the entry will remain in the queue, and we will try again. */ diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.h b/src/Storages/MergeTree/MutateFromLogEntryTask.h index 5709e7b808a..6c13401b290 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.h +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { @@ -24,7 +25,7 @@ public: UInt64 getPriority() override { return priority; } private: - std::pair prepare() override; + ReplicatedMergeMutateTaskBase::PrepareResult prepare() override; bool finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) override; bool executeInnerTask() override @@ -41,6 +42,7 @@ private: MutationCommandsConstPtr commands; MergeTreeData::TransactionUniquePtr transaction_ptr{nullptr}; + std::optional zero_copy_lock; StopwatchUniquePtr stopwatch_ptr{nullptr}; MergeTreeData::MutableDataPartPtr new_part{nullptr}; diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 6b602484ff3..e3fa07dd0c0 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -32,9 +32,7 @@ void MutatePlainMergeTreeTask::prepare() merge_list_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory); + settings); stopwatch = std::make_unique(); @@ -96,6 +94,7 @@ bool MutatePlainMergeTreeTask::executeStep() { storage.updateMutationEntriesErrors(future_part, false, getCurrentExceptionMessage(false)); write_part_log(ExecutionStatus::fromCurrentException()); + tryLogCurrentException(__PRETTY_FUNCTION__); return false; } } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index cc690287ef6..1fe701c54ae 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -422,10 +422,11 @@ void finalizeMutatedPart( const CompressionCodecPtr & codec) { auto disk = new_data_part->volume->getDisk(); + auto part_path = fs::path(new_data_part->getFullRelativePath()); if (new_data_part->uuid != UUIDHelpers::Nil) { - auto out = disk->writeFile(new_data_part->getFullRelativePath() + IMergeTreeDataPart::UUID_FILE_NAME, 4096); + auto out = disk->writeFile(part_path / IMergeTreeDataPart::UUID_FILE_NAME, 4096); HashingWriteBuffer out_hashing(*out); writeUUIDText(new_data_part->uuid, out_hashing); new_data_part->checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); @@ -435,27 +436,36 @@ void finalizeMutatedPart( if (execute_ttl_type != ExecuteTTLType::NONE) { /// Write a file with ttl infos in json format. - auto out_ttl = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "ttl.txt", 4096); + auto out_ttl = disk->writeFile(part_path / "ttl.txt", 4096); HashingWriteBuffer out_hashing(*out_ttl); new_data_part->ttl_infos.write(out_hashing); new_data_part->checksums.files["ttl.txt"].file_size = out_hashing.count(); new_data_part->checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); } + if (!new_data_part->getSerializationInfos().empty()) + { + auto out = disk->writeFile(part_path / IMergeTreeDataPart::SERIALIZATION_FILE_NAME, 4096); + HashingWriteBuffer out_hashing(*out); + new_data_part->getSerializationInfos().writeJSON(out_hashing); + new_data_part->checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_size = out_hashing.count(); + new_data_part->checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_hash = out_hashing.getHash(); + } + { /// Write file with checksums. - auto out_checksums = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "checksums.txt", 4096); + auto out_checksums = disk->writeFile(part_path / "checksums.txt", 4096); new_data_part->checksums.write(*out_checksums); } /// close fd { - auto out = disk->writeFile(new_data_part->getFullRelativePath() + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096); + auto out = disk->writeFile(part_path / IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096); DB::writeText(queryToString(codec->getFullCodecDesc()), *out); } { /// Write a file with a description of columns. - auto out_columns = disk->writeFile(fs::path(new_data_part->getFullRelativePath()) / "columns.txt", 4096); + auto out_columns = disk->writeFile(part_path / "columns.txt", 4096); new_data_part->getColumns().writeText(*out_columns); } /// close fd @@ -466,7 +476,7 @@ void finalizeMutatedPart( new_data_part->modification_time = time(nullptr); new_data_part->loadProjections(false, false); new_data_part->setBytesOnDisk( - MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath())); + MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), part_path)); new_data_part->default_codec = codec; new_data_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); new_data_part->storage.lockSharedData(*new_data_part); @@ -520,7 +530,6 @@ struct MutationContext DiskPtr disk; String new_part_tmp_path; - SyncGuardPtr sync_guard; IMergedBlockOutputStreamPtr out{nullptr}; String mrk_extension; @@ -633,12 +642,7 @@ public: projection_future_part, projection.metadata, ctx->mutate_entry, - std::make_unique( - (*ctx->mutate_entry)->table_id, - projection_future_part, - settings.memory_profiler_step, - settings.memory_profiler_sample_probability, - settings.max_untracked_memory), + std::make_unique((*ctx->mutate_entry)->table_id, projection_future_part, settings), *ctx->holder, ctx->time_of_mutation, ctx->context, @@ -1310,21 +1314,17 @@ bool MutateTask::prepare() ctx->disk = ctx->new_data_part->volume->getDisk(); ctx->new_part_tmp_path = ctx->new_data_part->getFullRelativePath(); - if (ctx->data->getSettings()->fsync_part_directory) - ctx->sync_guard = ctx->disk->getDirectorySyncGuard(ctx->new_part_tmp_path); - /// Don't change granularity type while mutating subset of columns ctx->mrk_extension = ctx->source_part->index_granularity_info.is_adaptive ? getAdaptiveMrkExtension(ctx->new_data_part->getType()) : getNonAdaptiveMrkExtension(); - const auto data_settings = ctx->data-> getSettings(); + const auto data_settings = ctx->data->getSettings(); ctx->need_sync = needSyncPart(ctx->source_part->rows_count, ctx->source_part->getBytesOnDisk(), *data_settings); ctx->execute_ttl_type = ExecuteTTLType::NONE; if (ctx->mutating_pipeline.initialized()) ctx->execute_ttl_type = MergeTreeDataMergerMutator::shouldExecuteTTL(ctx->metadata_snapshot, ctx->interpreter->getColumnDependencies()); - /// All columns from part are changed and may be some more that were missing before in part /// TODO We can materialize compact part without copying data if (!isWidePart(ctx->source_part) @@ -1334,7 +1334,6 @@ bool MutateTask::prepare() } else /// TODO: check that we modify only non-key columns in this case. { - /// We will modify only some of the columns. Other columns and key values can be copied as-is. for (const auto & name_type : ctx->updated_header.getNamesAndTypesList()) ctx->updated_columns.emplace(name_type.name); diff --git a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h index 14ef91c0777..07c5c55d873 100644 --- a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h +++ b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h @@ -63,7 +63,7 @@ public: }; EntryState(): value(TODO) {} - EntryState(Value value_): value(value_) {} + EntryState(Value value_): value(value_) {} /// NOLINT Value value; @@ -173,7 +173,6 @@ private: void removePins(const Entry & entry, zkutil::ZooKeeperPtr zk); void syncStateFromZK(); -private: StorageReplicatedMergeTree & storage; String zookeeper_path; diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index d63781db67d..183808c9290 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -22,8 +22,8 @@ public: using AtomFromASTFunc = std::function< bool(const ASTPtr & node, ContextPtr context, Block & block_with_constants, RPNElement & out)>; - RPNBuilder(const SelectQueryInfo & query_info, ContextPtr context_, const AtomFromASTFunc & atomFromAST_) - : WithContext(context_), atomFromAST(atomFromAST_) + RPNBuilder(const SelectQueryInfo & query_info, ContextPtr context_, const AtomFromASTFunc & atom_from_ast_) + : WithContext(context_), atom_from_ast(atom_from_ast_) { /** Evaluation of expressions that depend only on constants. * For the index to be used, if it is written, for example `WHERE Date = toDate(now())`. @@ -79,7 +79,7 @@ private: } } - if (!atomFromAST(node, getContext(), block_with_constants, element)) + if (!atom_from_ast(node, getContext(), block_with_constants, element)) { element.function = RPNElement::FUNCTION_UNKNOWN; } @@ -114,7 +114,7 @@ private: return true; } - const AtomFromASTFunc & atomFromAST; + const AtomFromASTFunc & atom_from_ast; Block block_with_constants; RPN rpn; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 880e729e534..36a5957bf1d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -31,6 +31,7 @@ bool ReplicatedMergeMutateTaskBase::executeStep() { std::exception_ptr saved_exception; + bool retryable_error = false; try { /// We don't have any backoff for failed entries @@ -46,16 +47,19 @@ bool ReplicatedMergeMutateTaskBase::executeStep() { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. LOG_INFO(log, fmt::runtime(e.displayText())); + retryable_error = true; } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. LOG_INFO(log, fmt::runtime(e.message())); + retryable_error = true; } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily LOG_INFO(log, fmt::runtime(e.displayText())); + retryable_error = true; storage.cleanup_thread.wakeup(); } else @@ -80,7 +84,7 @@ bool ReplicatedMergeMutateTaskBase::executeStep() } - if (saved_exception) + if (!retryable_error && saved_exception) { std::lock_guard lock(storage.queue.state_mutex); @@ -140,9 +144,9 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() }; - auto execute_fetch = [&] () -> bool + auto execute_fetch = [&] (bool need_to_check_missing_part) -> bool { - if (storage.executeFetch(entry)) + if (storage.executeFetch(entry, need_to_check_missing_part)) return remove_processed_entry(); return false; @@ -160,12 +164,13 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() return remove_processed_entry(); } - bool res = false; - std::tie(res, part_log_writer) = prepare(); + auto prepare_result = prepare(); + + part_log_writer = prepare_result.part_log_writer; /// Avoid resheduling, execute fetch here, in the same thread. - if (!res) - return execute_fetch(); + if (!prepare_result.prepared_successfully) + return execute_fetch(prepare_result.need_to_check_missing_part_in_fetch); state = State::NEED_EXECUTE_INNER_MERGE; return true; @@ -194,7 +199,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() try { if (!finalize(part_log_writer)) - return execute_fetch(); + return execute_fetch(/* need_to_check_missing = */true); } catch (...) { diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h index ed529c43a7d..614d51602eb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.h @@ -38,7 +38,14 @@ public: protected: using PartLogWriter = std::function; - virtual std::pair prepare() = 0; + struct PrepareResult + { + bool prepared_successfully; + bool need_to_check_missing_part_in_fetch; + PartLogWriter part_log_writer; + }; + + virtual PrepareResult prepare() = 0; virtual bool finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) = 0; /// Will execute a part of inner MergeTask or MutateTask diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.h b/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.h index e5d3dd0a737..aa58e16a716 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAltersSequence.h @@ -29,7 +29,6 @@ private: bool data_finished = false; }; -private: /// alter_version -> AlterState. std::map queue_state; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 26bfd951d3d..3b6c727cd02 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -64,7 +64,7 @@ void ReplicatedMergeTreeCleanupThread::iterate() /// Both use relative_data_path which changes during rename, so we /// do it under share lock storage.clearOldWriteAheadLogs(); - storage.clearOldTemporaryDirectories(storage.merger_mutator, storage.getSettings()->temporary_directories_lifetime.totalSeconds()); + storage.clearOldTemporaryDirectories(storage.getSettings()->temporary_directories_lifetime.totalSeconds()); } /// This is loose condition: no problem if we actually had lost leadership at this moment diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h index 509b52ec07f..861de620926 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h @@ -24,7 +24,7 @@ class StorageReplicatedMergeTree; class ReplicatedMergeTreeCleanupThread { public: - ReplicatedMergeTreeCleanupThread(StorageReplicatedMergeTree & storage_); + explicit ReplicatedMergeTreeCleanupThread(StorageReplicatedMergeTree & storage_); void start() { task->activateAndSchedule(); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp index 7e0432f4602..44877c3da95 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp @@ -57,17 +57,6 @@ bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplica(const Re } -bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplicaShared(const ReplicatedMergeTreeLogEntryData & entry) const -{ - time_t threshold = remote_fs_execute_merges_on_single_replica_time_threshold; - return ( - threshold > 0 /// feature turned on - && entry.type == ReplicatedMergeTreeLogEntry::MERGE_PARTS /// it is a merge log entry - && entry.create_time + threshold > time(nullptr) /// not too much time waited - ); -} - - /// that will return the same replica name for ReplicatedMergeTreeLogEntry on all the replicas (if the replica set is the same). /// that way each replica knows who is responsible for doing a certain merge. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h index ca5ec601a99..91f5824f8fc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h @@ -42,7 +42,7 @@ struct ReplicatedMergeTreeLogEntryData; class ReplicatedMergeTreeMergeStrategyPicker: public boost::noncopyable { public: - ReplicatedMergeTreeMergeStrategyPicker(StorageReplicatedMergeTree & storage_); + explicit ReplicatedMergeTreeMergeStrategyPicker(StorageReplicatedMergeTree & storage_); /// triggers refreshing the cached state (list of replicas etc.) /// used when we get new merge event from the zookeeper queue ( see queueUpdatingTask() etc ) @@ -52,10 +52,6 @@ public: /// and we may need to do a fetch (or postpone) instead of merge bool shouldMergeOnSingleReplica(const ReplicatedMergeTreeLogEntryData & entry) const; - /// return true if remote_fs_execute_merges_on_single_replica_time_threshold feature is active - /// and we may need to do a fetch (or postpone) instead of merge - bool shouldMergeOnSingleReplicaShared(const ReplicatedMergeTreeLogEntryData & entry) const; - /// returns the replica name /// and it's not current replica should do the merge std::optional pickReplicaToExecuteMerge(const ReplicatedMergeTreeLogEntryData & entry); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h index a8ce4fedd6d..7c2c2401bf0 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h @@ -30,7 +30,7 @@ class StorageReplicatedMergeTree; class ReplicatedMergeTreePartCheckThread { public: - ReplicatedMergeTreePartCheckThread(StorageReplicatedMergeTree & storage_); + explicit ReplicatedMergeTreePartCheckThread(StorageReplicatedMergeTree & storage_); ~ReplicatedMergeTreePartCheckThread(); /// Processing of the queue to be checked is done in the background thread, which you must first start. @@ -42,12 +42,12 @@ public: { ReplicatedMergeTreePartCheckThread * parent; - TemporarilyStop(ReplicatedMergeTreePartCheckThread * parent_) : parent(parent_) + explicit TemporarilyStop(ReplicatedMergeTreePartCheckThread * parent_) : parent(parent_) { parent->stop(); } - TemporarilyStop(TemporarilyStop && old) : parent(old.parent) + TemporarilyStop(TemporarilyStop && old) noexcept : parent(old.parent) { old.parent = nullptr; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f4dbf8dbdbd..5f805c39ae2 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -971,7 +971,7 @@ ReplicatedMergeTreeQueue::StringSet ReplicatedMergeTreeQueue::moveSiblingPartsFo return parts_for_merge; } -bool ReplicatedMergeTreeQueue::checkReplaceRangeCanBeRemoved(const MergeTreePartInfo & part_info, const LogEntryPtr entry_ptr, const ReplicatedMergeTreeLogEntryData & current) const +bool ReplicatedMergeTreeQueue::checkReplaceRangeCanBeRemoved(const MergeTreePartInfo & part_info, LogEntryPtr entry_ptr, const ReplicatedMergeTreeLogEntryData & current) const { if (entry_ptr->type != LogEntry::REPLACE_RANGE) return false; @@ -1205,31 +1205,32 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( return false; } - bool should_execute_on_single_replica = merge_strategy_picker.shouldMergeOnSingleReplica(entry); - if (!should_execute_on_single_replica) + const auto data_settings = data.getSettings(); + if (data_settings->allow_remote_fs_zero_copy_replication) { - /// Separate check. If we use only s3, check remote_fs_execute_merges_on_single_replica_time_threshold as well. auto disks = storage.getDisks(); bool only_s3_storage = true; for (const auto & disk : disks) if (disk->getType() != DB::DiskType::S3) only_s3_storage = false; - if (!disks.empty() && only_s3_storage) - should_execute_on_single_replica = merge_strategy_picker.shouldMergeOnSingleReplicaShared(entry); + if (!disks.empty() && only_s3_storage && storage.checkZeroCopyLockExists(entry.new_part_name, disks[0])) + { + out_postpone_reason = "Not executing merge/mutation for the part " + entry.new_part_name + + ", waiting other replica to execute it and will fetch after."; + return false; + } } - if (should_execute_on_single_replica) + if (merge_strategy_picker.shouldMergeOnSingleReplica(entry)) { - auto replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); if (replica_to_execute_merge && !merge_strategy_picker.isMergeFinishedByReplica(replica_to_execute_merge.value(), entry)) { - out_postpone_reason = fmt::format( - "Not executing merge for the part {}, waiting for {} to execute merge.", - entry.new_part_name, replica_to_execute_merge.value()); - LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); + String reason = "Not executing merge for the part " + entry.new_part_name + + ", waiting for " + replica_to_execute_merge.value() + " to execute merge."; + out_postpone_reason = reason; return false; } } @@ -1242,7 +1243,6 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( * Setting max_bytes_to_merge_at_max_space_in_pool still working for regular merges, * because the leader replica does not assign merges of greater size (except OPTIMIZE PARTITION and OPTIMIZE FINAL). */ - const auto data_settings = data.getSettings(); bool ignore_max_size = false; if (entry.type == LogEntry::MERGE_PARTS) { @@ -1515,7 +1515,7 @@ ReplicatedMergeTreeQueue::SelectedEntryPtr ReplicatedMergeTreeQueue::selectEntry bool ReplicatedMergeTreeQueue::processEntry( std::function get_zookeeper, LogEntryPtr & entry, - const std::function func) + std::function func) { std::exception_ptr saved_exception; @@ -1674,6 +1674,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Marking mutation {} done because it is <= mutation_pointer ({})", znode, mutation_pointer); mutation.is_done = true; + mutation.latest_fail_reason.clear(); alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) { @@ -1718,6 +1719,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Mutation {} is done", entry->znode_name); it->second.is_done = true; + it->second.latest_fail_reason.clear(); if (entry->isAlterMutation()) { LOG_TRACE(log, "Finishing data alter with version {} for entry {}", entry->alter_version, entry->znode_name); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 208ce73e5f1..1d10c504b3c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -184,7 +184,7 @@ private: /// Check that entry_ptr is REPLACE_RANGE entry and can be removed from queue because current entry covers it bool checkReplaceRangeCanBeRemoved( - const MergeTreePartInfo & part_info, const LogEntryPtr entry_ptr, const ReplicatedMergeTreeLogEntryData & current) const; + const MergeTreePartInfo & part_info, LogEntryPtr entry_ptr, const ReplicatedMergeTreeLogEntryData & current) const; /// Ensures that only one thread is simultaneously updating mutations. std::mutex update_mutations_mutex; @@ -366,7 +366,7 @@ public: * If there was an exception during processing, it saves it in `entry`. * Returns true if there were no exceptions during the processing. */ - bool processEntry(std::function get_zookeeper, LogEntryPtr & entry, const std::function func); + bool processEntry(std::function get_zookeeper, LogEntryPtr & entry, std::function func); /// Count the number of merges and mutations of single parts in the queue. OperationsInQueue countMergesAndPartMutations() const; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h index f560850a6c6..4cdcc936e21 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h @@ -23,8 +23,8 @@ struct ReplicatedMergeTreeQuorumEntry size_t required_number_of_replicas{}; std::set replicas; - ReplicatedMergeTreeQuorumEntry() {} - ReplicatedMergeTreeQuorumEntry(const String & str) + ReplicatedMergeTreeQuorumEntry() = default; + explicit ReplicatedMergeTreeQuorumEntry(const String & str) { fromString(str); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index de34929b43e..dc52660f1f6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -240,7 +240,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() return; /// Firstly, remove parts from ZooKeeper - storage.tryRemovePartsFromZooKeeperWithRetries(failed_parts); + storage.removePartsFromZooKeeperWithRetries(failed_parts); for (const auto & part_name : failed_parts) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h index e62cff4baf6..99e56ffb366 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h @@ -22,7 +22,7 @@ class StorageReplicatedMergeTree; class ReplicatedMergeTreeRestartingThread { public: - ReplicatedMergeTreeRestartingThread(StorageReplicatedMergeTree & storage_); + explicit ReplicatedMergeTreeRestartingThread(StorageReplicatedMergeTree & storage_); void start() { task->activateAndSchedule(); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 86d36aab50c..550c586f7de 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -150,9 +150,14 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) if (quorum) checkQuorumPrecondition(zookeeper); + const Settings & settings = context->getSettingsRef(); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); - std::vector partitions; - String block_dedup_token; + + using DelayedPartitions = std::vector; + DelayedPartitions partitions; + + size_t streams = 0; + bool support_parallel_write = false; for (auto & current_block : part_blocks) { @@ -171,10 +176,12 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) if (deduplicate) { + String block_dedup_token; + /// We add the hash from the data and partition identifier to deduplication ID. /// That is, do not insert the same data to the same partition twice. - const String & dedup_token = context->getSettingsRef().insert_deduplication_token; + const String & dedup_token = settings.insert_deduplication_token; if (!dedup_token.empty()) { /// multiple blocks can be inserted within the same insert query @@ -182,6 +189,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) block_dedup_token = fmt::format("{}_{}", dedup_token, chunk_dedup_seqnum); ++chunk_dedup_seqnum; } + block_id = temp_part.part->getZeroLevelPartBlockID(block_dedup_token); LOG_DEBUG(log, "Wrote block with ID '{}', {} rows", block_id, current_block.block.rows()); } @@ -192,6 +200,24 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) UInt64 elapsed_ns = watch.elapsed(); + size_t max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE; + if (!support_parallel_write || settings.max_insert_delayed_streams_for_parallel_write.changed) + max_insert_delayed_streams_for_parallel_write = settings.max_insert_delayed_streams_for_parallel_write; + + /// In case of too much columns/parts in block, flush explicitly. + streams += temp_part.streams.size(); + if (streams > max_insert_delayed_streams_for_parallel_write) + { + finishDelayedChunk(zookeeper); + delayed_chunk = std::make_unique(); + delayed_chunk->partitions = std::move(partitions); + finishDelayedChunk(zookeeper); + + streams = 0; + support_parallel_write = false; + partitions = DelayedPartitions{}; + } + partitions.emplace_back(ReplicatedMergeTreeSink::DelayedChunk::Partition{ .temp_part = std::move(temp_part), .elapsed_ns = elapsed_ns, @@ -207,7 +233,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) /// value for `last_block_is_duplicate`, which is possible only after the part is committed. /// Othervide we can delay commit. /// TODO: we can also delay commit if there is no MVs. - if (!context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views) + if (!settings.deduplicate_blocks_in_dependent_materialized_views) finishDelayedChunk(zookeeper); } diff --git a/src/Storages/MergeTree/SimpleMergeSelector.cpp b/src/Storages/MergeTree/SimpleMergeSelector.cpp index 0775e021c76..434d44022df 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/SimpleMergeSelector.cpp @@ -202,7 +202,7 @@ void selectWithinPartition( SimpleMergeSelector::PartsRange SimpleMergeSelector::select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) + size_t max_total_size_to_merge) { Estimator estimator; diff --git a/src/Storages/MergeTree/SimpleMergeSelector.h b/src/Storages/MergeTree/SimpleMergeSelector.h index 3e104d1319a..11ffe8b672a 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.h +++ b/src/Storages/MergeTree/SimpleMergeSelector.h @@ -152,7 +152,7 @@ public: PartsRange select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) override; + size_t max_total_size_to_merge) override; private: const Settings settings; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 729b545e9a0..854f070d0e0 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -24,7 +24,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, @@ -35,8 +35,7 @@ public: .readFromParts( parts, column_names, - metadata_snapshot, - metadata_snapshot, + storage_snapshot, query_info, context, max_block_size, @@ -80,7 +79,7 @@ public: protected: /// Used in part mutation. - StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) + explicit StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_)) , parts({part_}) , storage(part_->storage) diff --git a/src/Storages/MergeTree/TTLMergeSelector.cpp b/src/Storages/MergeTree/TTLMergeSelector.cpp index 6a42ce039ac..d5657aa680d 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.cpp +++ b/src/Storages/MergeTree/TTLMergeSelector.cpp @@ -18,7 +18,7 @@ const String & getPartitionIdForPart(const ITTLMergeSelector::Part & part_info) IMergeSelector::PartsRange ITTLMergeSelector::select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) + size_t max_total_size_to_merge) { using Iterator = IMergeSelector::PartsRange::const_iterator; Iterator best_begin; diff --git a/src/Storages/MergeTree/TTLMergeSelector.h b/src/Storages/MergeTree/TTLMergeSelector.h index d41ba6f519d..88dc1fffee2 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.h +++ b/src/Storages/MergeTree/TTLMergeSelector.h @@ -30,7 +30,7 @@ public: PartsRange select( const PartsRanges & parts_ranges, - const size_t max_total_size_to_merge) override; + size_t max_total_size_to_merge) override; /// Get TTL value for part, may depend on child type and some settings in /// constructor. diff --git a/src/Storages/MergeTree/TemporaryParts.cpp b/src/Storages/MergeTree/TemporaryParts.cpp new file mode 100644 index 00000000000..4239c8232e5 --- /dev/null +++ b/src/Storages/MergeTree/TemporaryParts.cpp @@ -0,0 +1,24 @@ +#include + +namespace DB +{ + +bool TemporaryParts::contains(const std::string & basename) const +{ + std::lock_guard lock(mutex); + return parts.contains(basename); +} + +void TemporaryParts::add(std::string basename) +{ + std::lock_guard lock(mutex); + parts.emplace(std::move(basename)); +} + +void TemporaryParts::remove(const std::string & basename) +{ + std::lock_guard lock(mutex); + parts.erase(basename); +} + +} diff --git a/src/Storages/MergeTree/TemporaryParts.h b/src/Storages/MergeTree/TemporaryParts.h new file mode 100644 index 00000000000..bc9d270856f --- /dev/null +++ b/src/Storages/MergeTree/TemporaryParts.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +/// Manages set of active temporary paths that should not be cleaned by background thread. +class TemporaryParts : private boost::noncopyable +{ +private: + /// To add const qualifier for contains() + mutable std::mutex mutex; + + /// NOTE: It is pretty short, so use STL is fine. + std::unordered_set parts; + +public: + /// Returns true if passed part name is active. + /// (is the destination for one of active mutation/merge). + /// + /// NOTE: that it accept basename (i.e. dirname), not the path, + /// since later requires canonical form. + bool contains(const std::string & basename) const; + + void add(std::string basename); + void remove(const std::string & basename); +}; + +} diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 075e9e9fbc8..e5c21ed8d3d 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -98,7 +98,8 @@ IMergeTreeDataPart::Checksums checkDataPart( }; }; - SerializationInfoByName serialization_infos(columns_txt, {}); + auto ratio_of_defaults = data_part->storage.getSettings()->ratio_of_defaults_for_sparse_serialization; + SerializationInfoByName serialization_infos(columns_txt, SerializationInfo::Settings{ratio_of_defaults, false}); auto serialization_path = path + IMergeTreeDataPart::SERIALIZATION_FILE_NAME; if (disk->exists(serialization_path)) diff --git a/src/Storages/MergeTree/localBackup.cpp b/src/Storages/MergeTree/localBackup.cpp index 236a0c5b5e4..1a04aa4b678 100644 --- a/src/Storages/MergeTree/localBackup.cpp +++ b/src/Storages/MergeTree/localBackup.cpp @@ -42,15 +42,31 @@ static void localBackupImpl(const DiskPtr & disk, const String & source_path, co } } +namespace +{ class CleanupOnFail { public: - explicit CleanupOnFail(std::function && cleaner_) : cleaner(cleaner_), is_success(false) {} + explicit CleanupOnFail(std::function && cleaner_) + : cleaner(cleaner_) + {} ~CleanupOnFail() { if (!is_success) - cleaner(); + { + /// We are trying to handle race condition here. So if we was not + /// able to backup directory try to remove garbage, but it's ok if + /// it doesn't exist. + try + { + cleaner(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } } void success() @@ -60,8 +76,9 @@ public: private: std::function cleaner; - bool is_success; + bool is_success{false}; }; +} void localBackup(const DiskPtr & disk, const String & source_path, const String & destination_path, std::optional max_level) { @@ -73,11 +90,11 @@ void localBackup(const DiskPtr & disk, const String & source_path, const String size_t try_no = 0; const size_t max_tries = 10; - CleanupOnFail cleanup([&](){disk->removeRecursive(destination_path);}); + CleanupOnFail cleanup([disk, destination_path]() { disk->removeRecursive(destination_path); }); /** Files in the directory can be permanently added and deleted. * If some file is deleted during an attempt to make a backup, then try again, - * because it's important to take into account any new files that might appear. + * because it's important to take into account any new files that might appear. */ while (true) { diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index db040584536..5b963a544c8 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -645,6 +645,10 @@ void MaterializedPostgreSQLConsumer::addNested( assert(!storages.contains(postgres_table_name)); storages.emplace(postgres_table_name, nested_storage_info); + auto it = deleted_tables.find(postgres_table_name); + if (it != deleted_tables.end()) + deleted_tables.erase(it); + /// Replication consumer will read wall and check for currently processed table whether it is allowed to start applying /// changes to this table. waiting_list[postgres_table_name] = table_start_lsn; @@ -663,7 +667,9 @@ void MaterializedPostgreSQLConsumer::updateNested(const String & table_name, Sto void MaterializedPostgreSQLConsumer::removeNested(const String & postgres_table_name) { - storages.erase(postgres_table_name); + auto it = storages.find(postgres_table_name); + if (it != storages.end()) + storages.erase(it); deleted_tables.insert(postgres_table_name); } @@ -727,6 +733,7 @@ bool MaterializedPostgreSQLConsumer::readFromReplicationSlot() { if (e.code() == ErrorCodes::POSTGRESQL_REPLICATION_INTERNAL_ERROR) continue; + throw; } } diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h index f37cb3bffef..a01f9394190 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.h @@ -22,6 +22,9 @@ struct StorageInfo StorageInfo(StoragePtr storage_, const PostgreSQLTableStructure::Attributes & attributes_) : storage(storage_), attributes(attributes_) {} + + StorageInfo(StoragePtr storage_, PostgreSQLTableStructure::Attributes && attributes_) + : storage(storage_), attributes(std::move(attributes_)) {} }; using StorageInfos = std::unordered_map; @@ -123,7 +126,7 @@ private: static Int64 getLSNValue(const std::string & lsn) { UInt32 upper_half, lower_half; - std::sscanf(lsn.data(), "%X/%X", &upper_half, &lower_half); + std::sscanf(lsn.data(), "%X/%X", &upper_half, &lower_half); /// NOLINT return (static_cast(upper_half) << 32) + lower_half; } diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index c72dec824f0..d12e91f62e4 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -270,7 +270,7 @@ bool StorageMaterializedPostgreSQL::needRewriteQueryWithFinal(const Names & colu Pipe StorageMaterializedPostgreSQL::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & query_info, ContextPtr context_, QueryProcessingStage::Enum processed_stage, @@ -279,7 +279,7 @@ Pipe StorageMaterializedPostgreSQL::read( { auto nested_table = getNested(); - auto pipe = readFinalFromNestedStorage(nested_table, column_names, metadata_snapshot, + auto pipe = readFinalFromNestedStorage(nested_table, column_names, query_info, context_, processed_stage, max_block_size, num_streams); auto lock = lockForShare(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); @@ -291,11 +291,11 @@ Pipe StorageMaterializedPostgreSQL::read( std::shared_ptr StorageMaterializedPostgreSQL::getMaterializedColumnsDeclaration( - const String name, const String type, UInt64 default_value) + String name, String type, UInt64 default_value) { auto column_declaration = std::make_shared(); - column_declaration->name = name; + column_declaration->name = std::move(name); column_declaration->type = makeASTFunction(type); column_declaration->default_specifier = "MATERIALIZED"; @@ -352,7 +352,7 @@ ASTPtr StorageMaterializedPostgreSQL::getColumnDeclaration(const DataTypePtr & d ast_expression->name = "DateTime64"; ast_expression->arguments = std::make_shared(); ast_expression->arguments->children.emplace_back(std::make_shared(UInt32(6))); - return std::move(ast_expression); + return ast_expression; } return std::make_shared(data_type->getName()); @@ -382,8 +382,6 @@ ASTPtr StorageMaterializedPostgreSQL::getCreateNestedTableQuery( PostgreSQLTableStructurePtr table_structure, const ASTTableOverride * table_override) { auto create_table_query = std::make_shared(); - if (table_override) - applyTableOverrideToCreateQuery(*table_override, create_table_query.get()); auto table_id = getStorageID(); create_table_query->setTable(getNestedTableName()); @@ -496,12 +494,37 @@ ASTPtr StorageMaterializedPostgreSQL::getCreateNestedTableQuery( constraints = metadata_snapshot->getConstraints(); } - columns_declare_list->columns->children.emplace_back(getMaterializedColumnsDeclaration("_sign", "Int8", 1)); - columns_declare_list->columns->children.emplace_back(getMaterializedColumnsDeclaration("_version", "UInt64", 1)); - create_table_query->set(create_table_query->columns_list, columns_declare_list); - create_table_query->set(create_table_query->storage, storage); + if (table_override) + { + if (auto * columns = table_override->columns) + { + if (columns->columns) + { + for (const auto & override_column_ast : columns->columns->children) + { + auto * override_column = override_column_ast->as(); + if (override_column->name == "_sign" || override_column->name == "_version") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot override _sign and _version column"); + } + } + } + + create_table_query->set(create_table_query->columns_list, columns_declare_list); + + applyTableOverrideToCreateQuery(*table_override, create_table_query.get()); + + create_table_query->columns_list->columns->children.emplace_back(getMaterializedColumnsDeclaration("_sign", "Int8", 1)); + create_table_query->columns_list->columns->children.emplace_back(getMaterializedColumnsDeclaration("_version", "UInt64", 1)); + } + else + { + columns_declare_list->columns->children.emplace_back(getMaterializedColumnsDeclaration("_sign", "Int8", 1)); + columns_declare_list->columns->children.emplace_back(getMaterializedColumnsDeclaration("_version", "UInt64", 1)); + create_table_query->set(create_table_query->columns_list, columns_declare_list); + } + /// Add columns _sign and _version, so that they can be accessed from nested ReplacingMergeTree table if needed. ordinary_columns_and_types.push_back({"_sign", std::make_shared()}); ordinary_columns_and_types.push_back({"_version", std::make_shared()}); @@ -511,7 +534,7 @@ ASTPtr StorageMaterializedPostgreSQL::getCreateNestedTableQuery( storage_metadata.setConstraints(constraints); setInMemoryMetadata(storage_metadata); - return std::move(create_table_query); + return create_table_query; } diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h index ff9b95cad7c..e41eb8ee98f 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h @@ -85,7 +85,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context_, QueryProcessingStage::Enum processed_stage, @@ -135,7 +135,7 @@ protected: private: static std::shared_ptr getMaterializedColumnsDeclaration( - const String name, const String type, UInt64 default_value); + String name, String type, UInt64 default_value); ASTPtr getColumnDeclaration(const DataTypePtr & data_type) const; diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index 8f355c4a0dc..25b32a29f58 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Storages/RabbitMQ/RabbitMQSource.cpp b/src/Storages/RabbitMQ/RabbitMQSource.cpp index be2806eb42a..4946a3537f9 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSource.cpp @@ -8,12 +8,11 @@ namespace DB { -static std::pair getHeaders(StorageRabbitMQ & storage, const StorageMetadataPtr & metadata_snapshot) +static std::pair getHeaders(const StorageSnapshotPtr & storage_snapshot) { - auto non_virtual_header = metadata_snapshot->getSampleBlockNonMaterialized(); - auto virtual_header = metadata_snapshot->getSampleBlockForColumns( - {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id", "_timestamp"}, - storage.getVirtuals(), storage.getStorageID()); + auto non_virtual_header = storage_snapshot->metadata->getSampleBlockNonMaterialized(); + auto virtual_header = storage_snapshot->getSampleBlockForColumns( + {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id", "_timestamp"}); return {non_virtual_header, virtual_header}; } @@ -29,15 +28,15 @@ static Block getSampleBlock(const Block & non_virtual_header, const Block & virt RabbitMQSource::RabbitMQSource( StorageRabbitMQ & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, ContextPtr context_, const Names & columns, size_t max_block_size_, bool ack_in_suffix_) : RabbitMQSource( storage_, - metadata_snapshot_, - getHeaders(storage_, metadata_snapshot_), + storage_snapshot_, + getHeaders(storage_snapshot_), context_, columns, max_block_size_, @@ -47,7 +46,7 @@ RabbitMQSource::RabbitMQSource( RabbitMQSource::RabbitMQSource( StorageRabbitMQ & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, std::pair headers, ContextPtr context_, const Names & columns, @@ -55,7 +54,7 @@ RabbitMQSource::RabbitMQSource( bool ack_in_suffix_) : SourceWithProgress(getSampleBlock(headers.first, headers.second)) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , context(context_) , column_names(columns) , max_block_size(max_block_size_) diff --git a/src/Storages/RabbitMQ/RabbitMQSource.h b/src/Storages/RabbitMQ/RabbitMQSource.h index f3ceac8e1e5..ff46408db42 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.h +++ b/src/Storages/RabbitMQ/RabbitMQSource.h @@ -14,7 +14,7 @@ class RabbitMQSource : public SourceWithProgress public: RabbitMQSource( StorageRabbitMQ & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, ContextPtr context_, const Names & columns, size_t max_block_size_, @@ -34,7 +34,7 @@ public: private: StorageRabbitMQ & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; ContextPtr context; Names column_names; const size_t max_block_size; @@ -48,7 +48,7 @@ private: RabbitMQSource( StorageRabbitMQ & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, std::pair headers, ContextPtr context_, const Names & columns, diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index ac299657ae6..b5e120d9405 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -90,6 +90,8 @@ StorageRabbitMQ::StorageRabbitMQ( , is_attach(is_attach_) { auto parsed_address = parseAddress(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_host_port), 5672); + context_->getRemoteHostFilter().checkHostAndPort(parsed_address.first, toString(parsed_address.second)); + auto rabbitmq_username = rabbitmq_settings->rabbitmq_username.value; auto rabbitmq_password = rabbitmq_settings->rabbitmq_password.value; configuration = @@ -648,7 +650,7 @@ void StorageRabbitMQ::unbindExchange() Pipe StorageRabbitMQ::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /* query_info */, ContextPtr local_context, QueryProcessingStage::Enum /* processed_stage */, @@ -669,7 +671,7 @@ Pipe StorageRabbitMQ::read( std::lock_guard lock(loop_mutex); - auto sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + auto sample_block = storage_snapshot->getSampleBlockForColumns(column_names); auto modified_context = addSettings(local_context); if (!connection->isConnected()) @@ -688,7 +690,7 @@ Pipe StorageRabbitMQ::read( for (size_t i = 0; i < num_created_consumers; ++i) { auto rabbit_source = std::make_shared( - *this, metadata_snapshot, modified_context, column_names, 1, rabbitmq_settings->rabbitmq_commit_on_select); + *this, storage_snapshot, modified_context, column_names, 1, rabbitmq_settings->rabbitmq_commit_on_select); auto converting_dag = ActionsDAG::makeConvertingActions( rabbit_source->getPort().getHeader().getColumnsWithTypeAndName(), @@ -1024,9 +1026,9 @@ bool StorageRabbitMQ::streamToViews() InterpreterInsertQuery interpreter(insert, rabbitmq_context, false, true, true); auto block_io = interpreter.execute(); - auto metadata_snapshot = getInMemoryMetadataPtr(); + auto storage_snapshot = getStorageSnapshot(getInMemoryMetadataPtr()); auto column_names = block_io.pipeline.getHeader().getNames(); - auto sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + auto sample_block = storage_snapshot->getSampleBlockForColumns(column_names); auto block_size = getMaxBlockSize(); @@ -1039,7 +1041,7 @@ bool StorageRabbitMQ::streamToViews() for (size_t i = 0; i < num_created_consumers; ++i) { auto source = std::make_shared( - *this, metadata_snapshot, rabbitmq_context, column_names, block_size, false); + *this, storage_snapshot, rabbitmq_context, column_names, block_size, false); sources.emplace_back(source); pipes.emplace_back(source); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 9633326366d..394845bbc2f 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -40,7 +40,7 @@ public: /// Always return virtual columns in addition to required columns Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp index 58b98aaa4c6..cf1c5c35629 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp +++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp @@ -27,7 +27,6 @@ bool needRewriteQueryWithFinalForStorage(const Names & column_names, const Stora Pipe readFinalFromNestedStorage( StoragePtr nested_storage, const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -36,7 +35,7 @@ Pipe readFinalFromNestedStorage( { NameSet column_names_set = NameSet(column_names.begin(), column_names.end()); auto lock = nested_storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout); - const StorageMetadataPtr & nested_metadata = nested_storage->getInMemoryMetadataPtr(); + const auto & nested_metadata = nested_storage->getInMemoryMetadataPtr(); Block nested_header = nested_metadata->getSampleBlock(); ColumnWithTypeAndName & sign_column = nested_header.getByPosition(nested_header.columns() - 2); @@ -55,7 +54,8 @@ Pipe readFinalFromNestedStorage( filter_column_name = expressions->children.back()->getColumnName(); } - Pipe pipe = nested_storage->read(require_columns_name, nested_metadata, query_info, context, processed_stage, max_block_size, num_streams); + auto nested_snapshot = nested_storage->getStorageSnapshot(nested_metadata); + Pipe pipe = nested_storage->read(require_columns_name, nested_snapshot, query_info, context, processed_stage, max_block_size, num_streams); pipe.addTableLock(lock); pipe.addStorageHolder(nested_storage); diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.h b/src/Storages/ReadFinalForExternalReplicaStorage.h index 979945a38c7..f21b396513f 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.h +++ b/src/Storages/ReadFinalForExternalReplicaStorage.h @@ -16,7 +16,6 @@ bool needRewriteQueryWithFinalForStorage(const Names & column_names, const Stora Pipe readFinalFromNestedStorage( StoragePtr nested_storage, const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 3dbb5b18de9..05b30bb014e 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -240,7 +240,7 @@ public: WriteBufferFromString wb(serialized_keys[rows_processed]); key_column_type->getDefaultSerialization()->serializeBinary(*it, wb); wb.finalize(); - slices_keys[rows_processed] = std::move(serialized_keys[rows_processed]); + slices_keys[rows_processed] = serialized_keys[rows_processed]; ++it; ++rows_processed; @@ -432,19 +432,19 @@ void StorageEmbeddedRocksDB::initDB() Pipe StorageEmbeddedRocksDB::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); FieldVectorPtr keys; bool all_scan = false; - Block sample_block = metadata_snapshot->getSampleBlock(); + Block sample_block = storage_snapshot->metadata->getSampleBlock(); auto primary_key_data_type = sample_block.getByName(primary_key).type; std::tie(keys, all_scan) = getFilterKeys(primary_key, primary_key_data_type, query_info); if (all_scan) diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h index d45d3dd320c..52a08cbefd4 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h @@ -28,7 +28,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index f97c09471c3..801e1b80a20 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -145,10 +146,10 @@ StorageBuffer::StorageBuffer( class BufferSource : public SourceWithProgress { public: - BufferSource(const Names & column_names_, StorageBuffer::Buffer & buffer_, const StorageBuffer & storage, const StorageMetadataPtr & metadata_snapshot) - : SourceWithProgress( - metadata_snapshot->getSampleBlockForColumns(column_names_, storage.getVirtuals(), storage.getStorageID())) - , column_names_and_types(metadata_snapshot->getColumns().getByNames(ColumnsDescription::All, column_names_, true)) + BufferSource(const Names & column_names_, StorageBuffer::Buffer & buffer_, const StorageSnapshotPtr & storage_snapshot) + : SourceWithProgress(storage_snapshot->getSampleBlockForColumns(column_names_)) + , column_names_and_types(storage_snapshot->getColumnsByNames( + GetColumnsOptions(GetColumnsOptions::All).withSubcolumns(), column_names_)) , buffer(buffer_) {} String getName() const override { return "Buffer"; } @@ -189,7 +190,7 @@ private: QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage( ContextPtr local_context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr &, + const StorageSnapshotPtr &, SelectQueryInfo & query_info) const { if (destination_id) @@ -201,7 +202,8 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage( /// TODO: Find a way to support projections for StorageBuffer query_info.ignore_projections = true; - return destination->getQueryProcessingStage(local_context, to_stage, destination->getInMemoryMetadataPtr(), query_info); + const auto & destination_metadata = destination->getInMemoryMetadataPtr(); + return destination->getQueryProcessingStage(local_context, to_stage, destination->getStorageSnapshot(destination_metadata), query_info); } return QueryProcessingStage::FetchColumns; @@ -210,7 +212,7 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage( Pipe StorageBuffer::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -218,7 +220,7 @@ Pipe StorageBuffer::read( const unsigned num_streams) { QueryPlan plan; - read(plan, column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + read(plan, column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); return plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(local_context), BuildQueryPipelineSettings::fromContext(local_context)); @@ -227,13 +229,15 @@ Pipe StorageBuffer::read( void StorageBuffer::read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { + const auto & metadata_snapshot = storage_snapshot->metadata; + if (destination_id) { auto destination = DatabaseCatalog::instance().getTable(destination_id, local_context); @@ -244,13 +248,14 @@ void StorageBuffer::read( auto destination_lock = destination->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); auto destination_metadata_snapshot = destination->getInMemoryMetadataPtr(); + auto destination_snapshot = destination->getStorageSnapshot(destination_metadata_snapshot); const bool dst_has_same_structure = std::all_of(column_names.begin(), column_names.end(), [metadata_snapshot, destination_metadata_snapshot](const String& column_name) { const auto & dest_columns = destination_metadata_snapshot->getColumns(); const auto & our_columns = metadata_snapshot->getColumns(); - auto dest_columm = dest_columns.tryGetColumnOrSubcolumn(ColumnsDescription::AllPhysical, column_name); - return dest_columm && dest_columm->type->equals(*our_columns.getColumnOrSubcolumn(ColumnsDescription::AllPhysical, column_name).type); + auto dest_columm = dest_columns.tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column_name); + return dest_columm && dest_columm->type->equals(*our_columns.getColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column_name).type); }); if (dst_has_same_structure) @@ -260,7 +265,7 @@ void StorageBuffer::read( /// The destination table has the same structure of the requested columns and we can simply read blocks from there. destination->read( - query_plan, column_names, destination_metadata_snapshot, query_info, + query_plan, column_names, destination_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); } else @@ -295,7 +300,7 @@ void StorageBuffer::read( else { destination->read( - query_plan, columns_intersection, destination_metadata_snapshot, query_info, + query_plan, columns_intersection, destination_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); if (query_plan.isInitialized()) @@ -352,7 +357,7 @@ void StorageBuffer::read( Pipes pipes_from_buffers; pipes_from_buffers.reserve(num_shards); for (auto & buf : buffers) - pipes_from_buffers.emplace_back(std::make_shared(column_names, buf, *this, metadata_snapshot)); + pipes_from_buffers.emplace_back(std::make_shared(column_names, buf, storage_snapshot)); pipe_from_buffers = Pipe::unitePipes(std::move(pipes_from_buffers)); } @@ -474,6 +479,14 @@ static void appendBlock(const Block & from, Block & to) const IColumn & col_from = *from.getByPosition(column_no).column.get(); last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); + /// In case of ColumnAggregateFunction aggregate states will + /// be allocated from the query context but can be destroyed from the + /// server context (in case of background flush), and thus memory + /// will be leaked from the query, but only tracked memory, not + /// memory itself. + /// + /// To avoid this, prohibit sharing the aggregate states. + last_col->ensureOwnership(); last_col->insertRangeFrom(col_from, 0, rows); to.getByPosition(column_no).column = std::move(last_col); @@ -1000,7 +1013,8 @@ void StorageBuffer::reschedule() size_t min = std::max(min_thresholds.time - time_passed, 1); size_t max = std::max(max_thresholds.time - time_passed, 1); - flush_handle->scheduleAfter(std::min(min, max) * 1000); + size_t flush = std::max(flush_thresholds.time - time_passed, 1); + flush_handle->scheduleAfter(std::min({min, max, flush}) * 1000); } void StorageBuffer::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index f04619a1d21..f589560008a 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -58,11 +58,11 @@ public: std::string getName() const override { return "Buffer"; } QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override; + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override; Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -72,7 +72,7 @@ public: void read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index da8c5f115b2..b3d0c1f17de 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -163,7 +163,7 @@ void StorageDictionary::checkTableCanBeDetached() const Pipe StorageDictionary::read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & /*query_info*/, ContextPtr local_context, QueryProcessingStage::Enum /*processed_stage*/, @@ -295,7 +295,7 @@ void StorageDictionary::alter(const AlterCommands & params, ContextPtr alter_con } std::lock_guard lock(dictionary_config_mutex); - configuration->setString("dictionary.comment", std::move(new_comment)); + configuration->setString("dictionary.comment", new_comment); } void registerStorageDictionary(StorageFactory & factory) diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 855d02b0947..bf9e6853233 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -26,7 +26,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index da648aa4e5c..1a390f784a2 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -55,7 +57,10 @@ #include #include #include +#include #include +#include +#include #include #include @@ -118,6 +123,7 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES; extern const int ARGUMENT_OUT_OF_BOUND; + extern const int TOO_LARGE_DISTRIBUTED_DEPTH; } namespace ActionLocks @@ -274,9 +280,9 @@ void replaceConstantExpressions( ContextPtr context, const NamesAndTypesList & columns, ConstStoragePtr storage, - const StorageMetadataPtr & metadata_snapshot) + const StorageSnapshotPtr & storage_snapshot) { - auto syntax_result = TreeRewriter(context).analyze(node, columns, storage, metadata_snapshot); + auto syntax_result = TreeRewriter(context).analyze(node, columns, storage, storage_snapshot); Block block_with_constants = KeyCondition::getBlockWithConstants(node, syntax_result, context); InDepthNodeVisitor visitor(block_with_constants); @@ -420,7 +426,7 @@ StorageDistributed::StorageDistributed( QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( ContextPtr local_context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info) const { const auto & settings = local_context->getSettingsRef(); @@ -434,7 +440,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( /// (Anyway it will be calculated in the read()) if (nodes > 1 && settings.optimize_skip_unused_shards) { - ClusterPtr optimized_cluster = getOptimizedCluster(local_context, metadata_snapshot, query_info.query); + ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info.query); if (optimized_cluster) { LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", @@ -587,9 +593,62 @@ std::optional StorageDistributed::getOptimizedQueryP return QueryProcessingStage::Complete; } +static bool requiresObjectColumns(const ColumnsDescription & all_columns, ASTPtr query) +{ + if (!hasObjectColumns(all_columns)) + return false; + + if (!query) + return true; + + RequiredSourceColumnsVisitor::Data columns_context; + RequiredSourceColumnsVisitor(columns_context).visit(query); + + auto required_columns = columns_context.requiredColumns(); + for (const auto & required_column : required_columns) + { + auto name_in_storage = Nested::splitName(required_column).first; + auto column_in_storage = all_columns.tryGetPhysical(name_in_storage); + + if (column_in_storage && isObject(column_in_storage->type)) + return true; + } + + return false; +} + +StorageSnapshotPtr StorageDistributed::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const +{ + return getStorageSnapshotForQuery(metadata_snapshot, nullptr); +} + +StorageSnapshotPtr StorageDistributed::getStorageSnapshotForQuery( + const StorageMetadataPtr & metadata_snapshot, const ASTPtr & query) const +{ + /// If query doesn't use columns of type Object, don't deduce + /// concrete types for them, because it required extra round trip. + auto snapshot_data = std::make_unique(); + if (!requiresObjectColumns(metadata_snapshot->getColumns(), query)) + return std::make_shared(*this, metadata_snapshot, ColumnsDescription{}, std::move(snapshot_data)); + + snapshot_data->objects_by_shard = getExtendedObjectsOfRemoteTables( + *getCluster(), + StorageID{remote_database, remote_table}, + metadata_snapshot->getColumns(), + getContext()); + + auto object_columns = DB::getObjectColumns( + snapshot_data->objects_by_shard.begin(), + snapshot_data->objects_by_shard.end(), + metadata_snapshot->getColumns(), + [](const auto & shard_num_and_columns) -> const auto & { return shard_num_and_columns.second; }); + + return std::make_shared(*this, metadata_snapshot, object_columns, std::move(snapshot_data)); +} + Pipe StorageDistributed::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -597,7 +656,7 @@ Pipe StorageDistributed::read( const unsigned num_streams) { QueryPlan plan; - read(plan, column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + read(plan, column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); return plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(local_context), BuildQueryPipelineSettings::fromContext(local_context)); @@ -606,7 +665,7 @@ Pipe StorageDistributed::read( void StorageDistributed::read( QueryPlan & query_plan, const Names &, - const StorageMetadataPtr &, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -639,9 +698,12 @@ void StorageDistributed::read( if (!remote_table_function_ptr) main_table = StorageID{remote_database, remote_table}; + const auto & snapshot_data = assert_cast(*storage_snapshot->data); ClusterProxy::SelectStreamFactory select_stream_factory = ClusterProxy::SelectStreamFactory( header, + snapshot_data.objects_by_shard, + storage_snapshot, processed_stage); ClusterProxy::executeQuery( @@ -705,6 +767,9 @@ SinkToStoragePtr StorageDistributed::write(const ASTPtr &, const StorageMetadata QueryPipelineBuilderPtr StorageDistributed::distributedWrite(const ASTInsertQuery & query, ContextPtr local_context) { const Settings & settings = local_context->getSettingsRef(); + if (settings.max_distributed_depth && local_context->getClientInfo().distributed_depth >= settings.max_distributed_depth) + throw Exception("Maximum distributed depth exceeded", ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH); + std::shared_ptr storage_src; auto & select = query.select->as(); auto new_query = std::dynamic_pointer_cast(query.clone()); @@ -719,28 +784,60 @@ QueryPipelineBuilderPtr StorageDistributed::distributedWrite(const ASTInsertQuer storage_src = std::dynamic_pointer_cast(joined_tables.getLeftTableStorage()); if (storage_src) { - const auto select_with_union_query = std::make_shared(); - select_with_union_query->list_of_selects = std::make_shared(); + /// Unwrap view() function. + if (storage_src->remote_table_function_ptr) + { + const TableFunctionPtr src_table_function = + TableFunctionFactory::instance().get(storage_src->remote_table_function_ptr, local_context); + const TableFunctionView * view_function = + assert_cast(src_table_function.get()); + new_query->select = view_function->getSelectQuery().clone(); + } + else + { + const auto select_with_union_query = std::make_shared(); + select_with_union_query->list_of_selects = std::make_shared(); - auto new_select_query = std::dynamic_pointer_cast(select_query->clone()); - select_with_union_query->list_of_selects->children.push_back(new_select_query); + auto new_select_query = std::dynamic_pointer_cast(select_query->clone()); + select_with_union_query->list_of_selects->children.push_back(new_select_query); - new_select_query->replaceDatabaseAndTable(storage_src->getRemoteDatabaseName(), storage_src->getRemoteTableName()); + new_select_query->replaceDatabaseAndTable(storage_src->getRemoteDatabaseName(), storage_src->getRemoteTableName()); - new_query->select = select_with_union_query; + new_query->select = select_with_union_query; + } } } } } - if (!storage_src || storage_src->getClusterName() != getClusterName()) + const Cluster::AddressesWithFailover & src_addresses = storage_src ? storage_src->getCluster()->getShardsAddresses() : Cluster::AddressesWithFailover{}; + const Cluster::AddressesWithFailover & dst_addresses = getCluster()->getShardsAddresses(); + /// Compare addresses instead of cluster name, to handle remote()/cluster(). + /// (since for remote()/cluster() the getClusterName() is empty string) + if (src_addresses != dst_addresses) { + /// The warning should be produced only for root queries, + /// since in case of parallel_distributed_insert_select=1, + /// it will produce warning for the rewritten insert, + /// since destination table is still Distributed there. + if (local_context->getClientInfo().distributed_depth == 0) + { + LOG_WARNING(log, + "Parallel distributed INSERT SELECT is not possible " + "(source cluster={} ({} addresses), destination cluster={} ({} addresses))", + storage_src ? storage_src->getClusterName() : "", + src_addresses.size(), + getClusterName(), + dst_addresses.size()); + } return nullptr; } if (settings.parallel_distributed_insert_select == PARALLEL_DISTRIBUTED_INSERT_SELECT_ALL) { new_query->table_id = StorageID(getRemoteDatabaseName(), getRemoteTableName()); + /// Reset table function for INSERT INTO remote()/cluster() + new_query->table_function.reset(); } const auto & cluster = getCluster(); @@ -757,12 +854,15 @@ QueryPipelineBuilderPtr StorageDistributed::distributedWrite(const ASTInsertQuer new_query_str = buf.str(); } + ContextMutablePtr query_context = Context::createCopy(local_context); + ++query_context->getClientInfo().distributed_depth; + for (size_t shard_index : collections::range(0, shards_info.size())) { const auto & shard_info = shards_info[shard_index]; if (shard_info.isLocal()) { - InterpreterInsertQuery interpreter(new_query, local_context); + InterpreterInsertQuery interpreter(new_query, query_context); pipelines.emplace_back(std::make_unique()); pipelines.back()->init(interpreter.execute().pipeline); } @@ -776,7 +876,7 @@ QueryPipelineBuilderPtr StorageDistributed::distributedWrite(const ASTInsertQuer /// INSERT SELECT query returns empty block auto remote_query_executor - = std::make_shared(shard_info.pool, std::move(connections), new_query_str, Block{}, local_context); + = std::make_shared(shard_info.pool, std::move(connections), new_query_str, Block{}, query_context); pipelines.emplace_back(std::make_unique()); pipelines.back()->init(Pipe(std::make_shared(remote_query_executor, false, settings.async_socket_for_remote))); pipelines.back()->setSinks([](const Block & header, QueryPipelineBuilder::StreamType) -> ProcessorPtr @@ -1036,7 +1136,7 @@ ClusterPtr StorageDistributed::getCluster() const } ClusterPtr StorageDistributed::getOptimizedCluster( - ContextPtr local_context, const StorageMetadataPtr & metadata_snapshot, const ASTPtr & query_ptr) const + ContextPtr local_context, const StorageSnapshotPtr & storage_snapshot, const ASTPtr & query_ptr) const { ClusterPtr cluster = getCluster(); const Settings & settings = local_context->getSettingsRef(); @@ -1045,7 +1145,7 @@ ClusterPtr StorageDistributed::getOptimizedCluster( if (has_sharding_key && sharding_key_is_usable) { - ClusterPtr optimized = skipUnusedShards(cluster, query_ptr, metadata_snapshot, local_context); + ClusterPtr optimized = skipUnusedShards(cluster, query_ptr, storage_snapshot, local_context); if (optimized) return optimized; } @@ -1101,7 +1201,7 @@ IColumn::Selector StorageDistributed::createSelector(const ClusterPtr cluster, c ClusterPtr StorageDistributed::skipUnusedShards( ClusterPtr cluster, const ASTPtr & query_ptr, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, ContextPtr local_context) const { const auto & select = query_ptr->as(); @@ -1121,7 +1221,7 @@ ClusterPtr StorageDistributed::skipUnusedShards( condition_ast = select.prewhere() ? select.prewhere()->clone() : select.where()->clone(); } - replaceConstantExpressions(condition_ast, local_context, metadata_snapshot->getColumns().getAll(), shared_from_this(), metadata_snapshot); + replaceConstantExpressions(condition_ast, local_context, storage_snapshot->metadata->getColumns().getAll(), shared_from_this(), storage_snapshot); size_t limit = local_context->getSettingsRef().optimize_skip_unused_shards_limit; if (!limit || limit > SSIZE_MAX) @@ -1422,3 +1522,4 @@ void registerStorageDistributed(StorageFactory & factory) } } + diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index e47e0fddd6c..317463783ee 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ public: bool supportsFinal() const override { return true; } bool supportsPrewhere() const override { return true; } bool supportsSubcolumns() const override { return true; } + bool supportsDynamicSubcolumns() const override { return true; } StoragePolicyPtr getStoragePolicy() const override; /// Do not apply moving to PREWHERE optimization for distributed tables, @@ -59,12 +61,24 @@ public: bool isRemote() const override { return true; } + /// Snapshot for StorageDistributed contains descriptions + /// of columns of type Object for each shard at the moment + /// of the start of query. + struct SnapshotData : public StorageSnapshot::Data + { + ColumnsDescriptionByShardNum objects_by_shard; + }; + + StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const override; + StorageSnapshotPtr getStorageSnapshotForQuery( + const StorageMetadataPtr & metadata_snapshot, const ASTPtr & query) const override; + QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override; + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override; Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -74,7 +88,7 @@ public: void read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -114,8 +128,6 @@ public: /// Used by InterpreterInsertQuery std::string getRemoteDatabaseName() const { return remote_database; } std::string getRemoteTableName() const { return remote_table; } - /// Returns empty string if tables is used by TableFunctionRemote - std::string getClusterName() const { return cluster_name; } ClusterPtr getCluster() const; /// Used by InterpreterSystemQuery @@ -177,10 +189,10 @@ private: /// Apply the following settings: /// - optimize_skip_unused_shards /// - force_optimize_skip_unused_shards - ClusterPtr getOptimizedCluster(ContextPtr, const StorageMetadataPtr & metadata_snapshot, const ASTPtr & query_ptr) const; + ClusterPtr getOptimizedCluster(ContextPtr, const StorageSnapshotPtr & storage_snapshot, const ASTPtr & query_ptr) const; ClusterPtr skipUnusedShards( - ClusterPtr cluster, const ASTPtr & query_ptr, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const; + ClusterPtr cluster, const ASTPtr & query_ptr, const StorageSnapshotPtr & storage_snapshot, ContextPtr context) const; /// This method returns optimal query processing stage. /// @@ -201,6 +213,7 @@ private: std::optional getOptimizedQueryProcessingStage(const SelectQueryInfo & query_info, const Settings & settings) const; size_t getRandomShardIndex(const Cluster::ShardsInfo & shards); + std::string getClusterName() const { return cluster_name.empty() ? "" : cluster_name; } const DistributedSettings & getDistributedSettingsRef() const { return distributed_settings; } diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index 21143438725..d9e97f98d56 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -105,7 +105,7 @@ StorageExecutable::StorageExecutable( Pipe StorageExecutable::read( const Names & /*column_names*/, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, @@ -142,7 +142,7 @@ Pipe StorageExecutable::read( if (settings.is_executable_pool) transformToSingleBlockSources(inputs); - auto sample_block = metadata_snapshot->getSampleBlock(); + auto sample_block = storage_snapshot->metadata->getSampleBlock(); ShellCommandSourceConfiguration configuration; configuration.max_block_size = max_block_size; diff --git a/src/Storages/StorageExecutable.h b/src/Storages/StorageExecutable.h index b6248abae97..ede98ea5b47 100644 --- a/src/Storages/StorageExecutable.h +++ b/src/Storages/StorageExecutable.h @@ -31,7 +31,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 40a2ad0b85e..18b8d4c037a 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -172,7 +172,7 @@ StorageExternalDistributed::StorageExternalDistributed( Pipe StorageExternalDistributed::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -184,7 +184,7 @@ Pipe StorageExternalDistributed::read( { pipes.emplace_back(shard->read( column_names, - metadata_snapshot, + storage_snapshot, query_info, context, processed_stage, diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h index 33a58d324da..57767db10b0 100644 --- a/src/Storages/StorageExternalDistributed.h +++ b/src/Storages/StorageExternalDistributed.h @@ -32,7 +32,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 9a2ec0789cd..761b4ecdeb1 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -179,8 +179,9 @@ std::unique_ptr createReadBuffer( method = chooseCompressionMethod(current_path, compression_method); } - /// For clickhouse-local add progress callback to display progress bar. - if (context->getApplicationType() == Context::ApplicationType::LOCAL) + /// For clickhouse-local and clickhouse-client add progress callback to display progress bar. + if (context->getApplicationType() == Context::ApplicationType::LOCAL + || context->getApplicationType() == Context::ApplicationType::CLIENT) { auto & in = static_cast(*nested_buffer); in.setProgressCallback(context); @@ -273,22 +274,38 @@ ColumnsDescription StorageFile::getTableStructureFromFile( return ColumnsDescription(source->getOutputs().front().getHeader().getNamesAndTypesList()); } - auto read_buffer_creator = [&]() + std::string exception_messages; + bool read_buffer_creator_was_used = false; + for (const auto & path : paths) { - String path; - auto it = std::find_if(paths.begin(), paths.end(), [](const String & p){ return std::filesystem::exists(p); }); - if (it == paths.end()) - throw Exception( - ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, - "Cannot extract table structure from {} format file, because there are no files with provided path. You must specify " - "table structure manually", - format); + auto read_buffer_creator = [&]() + { + read_buffer_creator_was_used = true; - path = *it; - return createReadBuffer(path, false, "File", -1, compression_method, context); - }; + if (!std::filesystem::exists(path)) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot extract table structure from {} format file, because there are no files with provided path. You must specify " + "table structure manually", + format); - return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + return createReadBuffer(path, false, "File", -1, compression_method, context); + }; + + try + { + return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + } + catch (...) + { + if (paths.size() == 1 || !read_buffer_creator_was_used) + throw; + + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } + } + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from files failed. Errors:\n{}", exception_messages); } bool StorageFile::isColumnOriented() const @@ -431,28 +448,27 @@ public: static Block getBlockForSource( const StorageFilePtr & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const ColumnsDescription & columns_description, const FilesInfoPtr & files_info) { if (storage->isColumnOriented()) - return metadata_snapshot->getSampleBlockForColumns( - columns_description.getNamesOfPhysical(), storage->getVirtuals(), storage->getStorageID()); + return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); else - return getHeader(metadata_snapshot, files_info->need_path_column, files_info->need_file_column); + return getHeader(storage_snapshot->metadata, files_info->need_path_column, files_info->need_file_column); } StorageFileSource( std::shared_ptr storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, ContextPtr context_, UInt64 max_block_size_, FilesInfoPtr files_info_, ColumnsDescription columns_description_, std::unique_ptr read_buf_) - : SourceWithProgress(getBlockForSource(storage_, metadata_snapshot_, columns_description_, files_info_)) + : SourceWithProgress(getBlockForSource(storage_, storage_snapshot_, columns_description_, files_info_)) , storage(std::move(storage_)) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , files_info(std::move(files_info_)) , read_buf(std::move(read_buf_)) , columns_description(std::move(columns_description_)) @@ -502,8 +518,8 @@ public: auto get_block_for_format = [&]() -> Block { if (storage->isColumnOriented()) - return metadata_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); - return metadata_snapshot->getSampleBlock(); + return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); + return storage_snapshot->metadata->getSampleBlock(); }; auto format = context->getInputFormat( @@ -566,7 +582,7 @@ public: private: std::shared_ptr storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; FilesInfoPtr files_info; String current_path; Block sample_block; @@ -587,7 +603,7 @@ private: Pipe StorageFile::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, @@ -601,7 +617,7 @@ Pipe StorageFile::read( if (paths.size() == 1 && !fs::exists(paths[0])) { if (context->getSettingsRef().engine_file_empty_if_not_exists) - return Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); + return Pipe(std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names))); else throw Exception("File " + paths[0] + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); } @@ -628,7 +644,9 @@ Pipe StorageFile::read( /// Set total number of bytes to process. For progress bar. auto progress_callback = context->getFileProgressCallback(); - if (context->getApplicationType() == Context::ApplicationType::LOCAL && progress_callback) + if ((context->getApplicationType() == Context::ApplicationType::LOCAL + || context->getApplicationType() == Context::ApplicationType::CLIENT) + && progress_callback) progress_callback(FileProgress(0, total_bytes_to_read)); for (size_t i = 0; i < num_streams; ++i) @@ -637,9 +655,9 @@ Pipe StorageFile::read( { if (isColumnOriented()) return ColumnsDescription{ - metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()).getNamesAndTypesList()}; + storage_snapshot->getSampleBlockForColumns(column_names).getNamesAndTypesList()}; else - return metadata_snapshot->getColumns(); + return storage_snapshot->metadata->getColumns(); }; /// In case of reading from fd we have to check whether we have already created @@ -651,7 +669,7 @@ Pipe StorageFile::read( read_buffer = std::move(peekable_read_buffer_from_fd); pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, get_columns_for_format(), std::move(read_buffer))); + this_ptr, storage_snapshot, context, max_block_size, files_info, get_columns_for_format(), std::move(read_buffer))); } return Pipe::unitePipes(std::move(pipes)); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index ed5431d5e03..86e75588e14 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -12,9 +12,6 @@ namespace DB { -class StorageFileBlockInputStream; -class StorageFileBlockOutputStream; - class StorageFile final : public shared_ptr_helper, public IStorage { friend struct shared_ptr_helper; @@ -25,7 +22,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -67,7 +64,7 @@ public: /// Is is useful because column oriented formats could effectively skip unknown columns /// So we can create a header of only required columns in read method and ask /// format to read only them. Note: this hack cannot be done with ordinary formats like TSV. - bool isColumnOriented() const; + bool isColumnOriented() const override; bool supportsPartitionBy() const override { return true; } diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index 19e8f78d877..aa7b17191b6 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -172,7 +172,7 @@ ColumnPtr fillColumnWithRandomData( auto data_column = fillColumnWithRandomData(nested_type, offset, max_array_length, max_string_length, rng, context); - return ColumnArray::create(std::move(data_column), std::move(offsets_column)); + return ColumnArray::create(data_column, std::move(offsets_column)); } case TypeIndex::Tuple: @@ -198,7 +198,7 @@ ColumnPtr fillColumnWithRandomData( for (UInt64 i = 0; i < limit; ++i) null_map[i] = rng() % 16 == 0; /// No real motivation for this. - return ColumnNullable::create(std::move(nested_column), std::move(null_map_column)); + return ColumnNullable::create(nested_column, std::move(null_map_column)); } case TypeIndex::UInt8: @@ -395,7 +395,7 @@ protected: for (const auto & elem : block_to_fill) columns.emplace_back(fillColumnWithRandomData(elem.type, block_size, max_array_length, max_string_length, rng, context)); - columns = Nested::flatten(block_to_fill.cloneWithColumns(std::move(columns))).getColumns(); + columns = Nested::flatten(block_to_fill.cloneWithColumns(columns)).getColumns(); return {std::move(columns), block_size}; } @@ -486,19 +486,19 @@ void registerStorageGenerateRandom(StorageFactory & factory) Pipe StorageGenerateRandom::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); Pipes pipes; pipes.reserve(num_streams); - const ColumnsDescription & our_columns = metadata_snapshot->getColumns(); + const ColumnsDescription & our_columns = storage_snapshot->metadata->getColumns(); Block block_header; for (const auto & name : column_names) { diff --git a/src/Storages/StorageGenerateRandom.h b/src/Storages/StorageGenerateRandom.h index d11a43b1dd6..2894b17d409 100644 --- a/src/Storages/StorageGenerateRandom.h +++ b/src/Storages/StorageGenerateRandom.h @@ -17,7 +17,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 061319bc1c0..15a761a5b84 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -300,6 +301,16 @@ ColumnDependencies StorageInMemoryMetadata::getColumnDependencies(const NameSet } +Block StorageInMemoryMetadata::getSampleBlockInsertable() const +{ + Block res; + + for (const auto & column : getColumns().getInsertable()) + res.insert({column.type->createColumn(), column.type, column.name}); + + return res; +} + Block StorageInMemoryMetadata::getSampleBlockNonMaterialized() const { Block res; @@ -332,39 +343,6 @@ Block StorageInMemoryMetadata::getSampleBlock() const return res; } -Block StorageInMemoryMetadata::getSampleBlockForColumns( - const Names & column_names, const NamesAndTypesList & virtuals, const StorageID & storage_id) const -{ - Block res; - - HashMapWithSavedHash virtuals_map; - - /// Virtual columns must be appended after ordinary, because user can - /// override them. - for (const auto & column : virtuals) - virtuals_map[column.name] = &column.type; - - for (const auto & name : column_names) - { - auto column = getColumns().tryGetColumnOrSubcolumn(ColumnsDescription::All, name); - if (column) - { - res.insert({column->type->createColumn(), column->type, column->name}); - } - else if (auto * it = virtuals_map.find(name); it != virtuals_map.end()) - { - const auto & type = *it->getMapped(); - res.insert({type->createColumn(), type, name}); - } - else - throw Exception( - "Column " + backQuote(name) + " not found in table " + (storage_id.empty() ? "" : storage_id.getNameForLogs()), - ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); - } - - return res; -} - const KeyDescription & StorageInMemoryMetadata::getPartitionKey() const { return partition_key; @@ -489,18 +467,6 @@ namespace using NamesAndTypesMap = HashMapWithSavedHash; using UniqueStrings = HashSetWithSavedHash; - String listOfColumns(const NamesAndTypesList & available_columns) - { - WriteBufferFromOwnString ss; - for (auto it = available_columns.begin(); it != available_columns.end(); ++it) - { - if (it != available_columns.begin()) - ss << ", "; - ss << it->name; - } - return ss.str(); - } - NamesAndTypesMap getColumnsMap(const NamesAndTypesList & columns) { NamesAndTypesMap res; @@ -529,36 +495,16 @@ namespace } } -void StorageInMemoryMetadata::check(const Names & column_names, const NamesAndTypesList & virtuals, const StorageID & storage_id) const +String listOfColumns(const NamesAndTypesList & available_columns) { - if (column_names.empty()) + WriteBufferFromOwnString ss; + for (auto it = available_columns.begin(); it != available_columns.end(); ++it) { - auto list_of_columns = listOfColumns(getColumns().getAllPhysicalWithSubcolumns()); - throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, - "Empty list of columns queried. There are columns: {}", list_of_columns); - } - - const auto virtuals_map = getColumnsMap(virtuals); - UniqueStrings unique_names; - - for (const auto & name : column_names) - { - bool has_column = getColumns().hasColumnOrSubcolumn(ColumnsDescription::AllPhysical, name) - || virtuals_map.find(name) != nullptr; - - if (!has_column) - { - auto list_of_columns = listOfColumns(getColumns().getAllPhysicalWithSubcolumns()); - throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, - "There is no column with name {} in table {}. There are columns: {}", - backQuote(name), storage_id.getNameForLogs(), list_of_columns); - } - - if (unique_names.end() != unique_names.find(name)) - throw Exception(ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE, "Column {} queried more than once", name); - - unique_names.insert(name); + if (it != available_columns.begin()) + ss << ", "; + ss << it->name; } + return ss.str(); } void StorageInMemoryMetadata::check(const NamesAndTypesList & provided_columns) const @@ -579,7 +525,10 @@ void StorageInMemoryMetadata::check(const NamesAndTypesList & provided_columns) listOfColumns(available_columns)); const auto * available_type = it->getMapped(); - if (!column.type->equals(*available_type) && !isCompatibleEnumTypes(available_type, column.type.get())) + + if (!isObject(*available_type) + && !column.type->equals(*available_type) + && !isCompatibleEnumTypes(available_type, column.type.get())) throw Exception( ErrorCodes::TYPE_MISMATCH, "Type mismatch for column {}. Column has type {}, got type {}", @@ -626,7 +575,9 @@ void StorageInMemoryMetadata::check(const NamesAndTypesList & provided_columns, const auto * provided_column_type = it->getMapped(); const auto * available_column_type = jt->getMapped(); - if (!provided_column_type->equals(*available_column_type) && !isCompatibleEnumTypes(available_column_type, provided_column_type)) + if (!isObject(*provided_column_type) + && !provided_column_type->equals(*available_column_type) + && !isCompatibleEnumTypes(available_column_type, provided_column_type)) throw Exception( ErrorCodes::TYPE_MISMATCH, "Type mismatch for column {}. Column has type {}, got type {}", @@ -668,7 +619,9 @@ void StorageInMemoryMetadata::check(const Block & block, bool need_all) const listOfColumns(available_columns)); const auto * available_type = it->getMapped(); - if (!column.type->equals(*available_type) && !isCompatibleEnumTypes(available_type, column.type.get())) + if (!isObject(*available_type) + && !column.type->equals(*available_type) + && !isCompatibleEnumTypes(available_type, column.type.get())) throw Exception( ErrorCodes::TYPE_MISMATCH, "Type mismatch for column {}. Column has type {}, got type {}", diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index a8912d44f2c..a9ab96909f4 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -55,6 +55,9 @@ struct StorageInMemoryMetadata StorageInMemoryMetadata(const StorageInMemoryMetadata & other); StorageInMemoryMetadata & operator=(const StorageInMemoryMetadata & other); + StorageInMemoryMetadata(StorageInMemoryMetadata && other) = default; + StorageInMemoryMetadata & operator=(StorageInMemoryMetadata && other) = default; + /// NOTE: Thread unsafe part. You should modify same StorageInMemoryMetadata /// structure from different threads. It should be used as MultiVersion /// object. See example in IStorage. @@ -151,6 +154,9 @@ struct StorageInMemoryMetadata /// Block with ordinary + materialized columns. Block getSampleBlock() const; + /// Block with ordinary + ephemeral. + Block getSampleBlockInsertable() const; + /// Block with ordinary columns. Block getSampleBlockNonMaterialized() const; @@ -159,13 +165,6 @@ struct StorageInMemoryMetadata /// Storage metadata. Block getSampleBlockWithVirtuals(const NamesAndTypesList & virtuals) const; - - /// Block with ordinary + materialized + aliases + virtuals. Virtuals have - /// to be explicitly specified, because they are part of Storage type, not - /// Storage metadata. StorageID required only for more clear exception - /// message. - Block getSampleBlockForColumns( - const Names & column_names, const NamesAndTypesList & virtuals = {}, const StorageID & storage_id = StorageID::createEmpty()) const; /// Returns structure with partition key. const KeyDescription & getPartitionKey() const; /// Returns ASTExpressionList of partition key expression for storage or nullptr if there is none. @@ -228,10 +227,6 @@ struct StorageInMemoryMetadata const SelectQueryDescription & getSelectQuery() const; bool hasSelectQuery() const; - /// Verify that all the requested names are in the table and are set correctly: - /// list of names is not empty and the names do not repeat. - void check(const Names & column_names, const NamesAndTypesList & virtuals, const StorageID & storage_id) const; - /// Check that all the requested names are in the table and have the correct types. void check(const NamesAndTypesList & columns) const; @@ -247,4 +242,6 @@ struct StorageInMemoryMetadata using StorageMetadataPtr = std::shared_ptr; using MultiVersionStorageMetadataPtr = MultiVersion; +String listOfColumns(const NamesAndTypesList & available_columns); + } diff --git a/src/Storages/StorageInput.cpp b/src/Storages/StorageInput.cpp index 2ed7a77b59d..a21a14cc240 100644 --- a/src/Storages/StorageInput.cpp +++ b/src/Storages/StorageInput.cpp @@ -52,7 +52,7 @@ void StorageInput::setPipe(Pipe pipe_) Pipe StorageInput::read( const Names & /*column_names*/, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, @@ -66,7 +66,7 @@ Pipe StorageInput::read( { /// Send structure to the client. query_context->initializeInput(shared_from_this()); - return Pipe(std::make_shared(query_context, metadata_snapshot->getSampleBlock())); + return Pipe(std::make_shared(query_context, storage_snapshot->metadata->getSampleBlock())); } if (pipe.empty()) diff --git a/src/Storages/StorageInput.h b/src/Storages/StorageInput.h index b28bc143bb0..4c44213a06b 100644 --- a/src/Storages/StorageInput.h +++ b/src/Storages/StorageInput.h @@ -20,7 +20,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index 3a9b42f7371..ecd182457e2 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -64,7 +65,7 @@ StorageJoin::StorageJoin( throw Exception{"Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE}; table_join = std::make_shared(limits, use_nulls, kind, strictness, key_names); - join = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + join = std::make_shared(table_join, getRightSampleBlock(), overwrite); restore(); } @@ -82,7 +83,7 @@ SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataP return StorageSetOrJoinBase::write(query, metadata_snapshot, context); } -void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, TableExclusiveLockHolder &) +void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr context, TableExclusiveLockHolder &) { std::lock_guard mutate_lock(mutate_mutex); TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); @@ -92,7 +93,7 @@ void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_s disk->createDirectories(path + "tmp/"); increment = 0; - join = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + join = std::make_shared(table_join, getRightSampleBlock(), overwrite); } void StorageJoin::checkMutationIsPossible(const MutationCommands & commands, const Settings & /* settings */) const @@ -116,7 +117,7 @@ void StorageJoin::mutate(const MutationCommands & commands, ContextPtr context) auto compressed_backup_buf = CompressedWriteBuffer(*backup_buf); auto backup_stream = NativeWriter(compressed_backup_buf, 0, metadata_snapshot->getSampleBlock()); - auto new_data = std::make_shared(table_join, metadata_snapshot->getSampleBlock().sortColumns(), overwrite); + auto new_data = std::make_shared(table_join, getRightSampleBlock(), overwrite); // New scope controls lifetime of InputStream. { @@ -167,8 +168,10 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, if ((analyzed_join->forceNullableRight() && !use_nulls) || (!analyzed_join->forceNullableRight() && isLeftOrFull(analyzed_join->kind()) && use_nulls)) - throw Exception("Table " + getStorageID().getNameForLogs() + " needs the same join_use_nulls setting as present in LEFT or FULL JOIN.", - ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); + throw Exception( + ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN, + "Table {} needs the same join_use_nulls setting as present in LEFT or FULL JOIN", + getStorageID().getNameForLogs()); /// TODO: check key columns @@ -177,7 +180,7 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr analyzed_join, /// Qualifies will be added by join implementation (HashJoin) analyzed_join->setRightKeys(key_names); - HashJoinPtr join_clone = std::make_shared(analyzed_join, metadata_snapshot->getSampleBlock().sortColumns()); + HashJoinPtr join_clone = std::make_shared(analyzed_join, getRightSampleBlock()); RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); join_clone->setLock(holder); @@ -576,16 +579,16 @@ private: // TODO: multiple stream read and index read Pipe StorageJoin::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); - Block source_sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + Block source_sample_block = storage_snapshot->getSampleBlockForColumns(column_names); RWLockImpl::LockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Read, context); return Pipe(std::make_shared(join, std::move(holder), max_block_size, source_sample_block)); } diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index ee685830a6c..ea71ff4be8f 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -50,7 +51,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -60,6 +61,22 @@ public: std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; + Block getRightSampleBlock() const + { + auto metadata_snapshot = getInMemoryMetadataPtr(); + Block block = metadata_snapshot->getSampleBlock().sortColumns(); + if (use_nulls && isLeftOrFull(kind)) + { + for (auto & col : block) + { + JoinCommon::convertColumnToNullable(col); + } + } + return block; + } + + bool useNulls() const { return use_nulls; } + private: Block sample_block; const Names key_names; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 5f0bd240f64..11116780734 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -25,9 +25,10 @@ #include #include -#include +#include #include #include +#include #include #include @@ -171,7 +172,7 @@ Chunk LogSource::generate() } if (!column->empty()) - res.insert(ColumnWithTypeAndName(std::move(column), name_type.type, name_type.name)); + res.insert(ColumnWithTypeAndName(column, name_type.type, name_type.name)); } if (res) @@ -732,8 +733,21 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & renameInMemory(new_table_id); } -void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) +static std::chrono::seconds getLockTimeout(ContextPtr context) { + const Settings & settings = context->getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + +void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr context, TableExclusiveLockHolder &) +{ + WriteLock lock{rwlock, getLockTimeout(context)}; + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + disk->clearDirectory(table_path); for (auto & data_file : data_files) @@ -750,26 +764,16 @@ void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr } -static std::chrono::seconds getLockTimeout(ContextPtr context) -{ - const Settings & settings = context->getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); - return std::chrono::seconds{lock_timeout}; -} - - Pipe StorageLog::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); auto lock_timeout = getLockTimeout(context); loadMarks(lock_timeout); @@ -779,7 +783,7 @@ Pipe StorageLog::read( throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); if (!num_data_files || !file_checker.getFileSize(data_files[INDEX_WITH_REAL_ROW_COUNT].path)) - return Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); + return Pipe(std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names))); const Marks & marks_with_real_row_count = data_files[INDEX_WITH_REAL_ROW_COUNT].marks; size_t num_marks = marks_with_real_row_count.size(); @@ -788,7 +792,8 @@ Pipe StorageLog::read( if (num_streams > max_streams) num_streams = max_streams; - auto all_columns = metadata_snapshot->getColumns().getByNames(ColumnsDescription::All, column_names, true); + auto options = GetColumnsOptions(GetColumnsOptions::All).withSubcolumns(); + auto all_columns = storage_snapshot->getColumnsByNames(options, column_names); all_columns = Nested::convertToSubcolumns(all_columns); std::vector offsets; @@ -883,7 +888,7 @@ IStorage::ColumnSizeByName StorageLog::getColumnSizes() const } -BackupEntries StorageLog::backup(const ASTs & partitions, ContextPtr context) +BackupEntries StorageLog::backupData(ContextPtr context, const ASTs & partitions) { if (!partitions.empty()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); @@ -909,12 +914,12 @@ BackupEntries StorageLog::backup(const ASTs & partitions, ContextPtr context) { /// We make a copy of the data file because it can be changed later in write() or in truncate(). String data_file_name = fileName(data_file.path); - String temp_file_path = temp_dir + "/" + data_file_name; - disk->copy(data_file.path, disk, temp_file_path); + String hardlink_file_path = temp_dir + "/" + data_file_name; + disk->createHardLink(data_file.path, hardlink_file_path); backup_entries.emplace_back( data_file_name, - std::make_unique( - disk, temp_file_path, file_checker.getFileSize(data_file.path), std::nullopt, temp_dir_owner)); + std::make_unique( + disk, hardlink_file_path, file_checker.getFileSize(data_file.path), std::nullopt, temp_dir_owner)); } /// __marks.mrk @@ -922,12 +927,12 @@ BackupEntries StorageLog::backup(const ASTs & partitions, ContextPtr context) { /// We make a copy of the data file because it can be changed later in write() or in truncate(). String marks_file_name = fileName(marks_file_path); - String temp_file_path = temp_dir + "/" + marks_file_name; - disk->copy(marks_file_path, disk, temp_file_path); + String hardlink_file_path = temp_dir + "/" + marks_file_name; + disk->createHardLink(marks_file_path, hardlink_file_path); backup_entries.emplace_back( marks_file_name, - std::make_unique( - disk, temp_file_path, file_checker.getFileSize(marks_file_path), std::nullopt, temp_dir_owner)); + std::make_unique( + disk, hardlink_file_path, file_checker.getFileSize(marks_file_path), std::nullopt, temp_dir_owner)); } /// sizes.json @@ -948,43 +953,57 @@ BackupEntries StorageLog::backup(const ASTs & partitions, ContextPtr context) return backup_entries; } -RestoreDataTasks StorageLog::restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context) +class LogRestoreTask : public IRestoreTask { - if (!partitions.empty()) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); + using WriteLock = StorageLog::WriteLock; + using Mark = StorageLog::Mark; - auto restore_task = [this, backup, data_path_in_backup, context]() +public: + LogRestoreTask( + std::shared_ptr storage_, const BackupPtr & backup_, const String & data_path_in_backup_, ContextMutablePtr context_) + : storage(storage_), backup(backup_), data_path_in_backup(data_path_in_backup_), context(context_) + { + } + + RestoreTasks run() override { auto lock_timeout = getLockTimeout(context); - WriteLock lock{rwlock, lock_timeout}; + WriteLock lock{storage->rwlock, lock_timeout}; if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + const auto num_data_files = storage->num_data_files; if (!num_data_files) - return; + return {}; + + auto & file_checker = storage->file_checker; /// Load the marks if not loaded yet. We have to do that now because we're going to update these marks. - loadMarks(lock); + storage->loadMarks(lock); /// If there were no files, save zero file sizes to be able to rollback in case of error. - saveFileSizes(lock); + storage->saveFileSizes(lock); try { /// Append data files. + auto & data_files = storage->data_files; for (const auto & data_file : data_files) { String file_path_in_backup = data_path_in_backup + fileName(data_file.path); auto backup_entry = backup->readFile(file_path_in_backup); + const auto & disk = storage->disk; auto in = backup_entry->getReadBuffer(); - auto out = disk->writeFile(data_file.path, max_compress_block_size, WriteMode::Append); + auto out = disk->writeFile(data_file.path, storage->max_compress_block_size, WriteMode::Append); copyData(*in, *out); } + const bool use_marks_file = storage->use_marks_file; if (use_marks_file) { /// Append marks. size_t num_extra_marks = 0; + const auto & marks_file_path = storage->marks_file_path; String file_path_in_backup = data_path_in_backup + fileName(marks_file_path); size_t file_size = backup->getFileSize(file_path_in_backup); if (file_size % (num_data_files * sizeof(Mark)) != 0) @@ -1023,19 +1042,34 @@ RestoreDataTasks StorageLog::restoreFromBackup(const BackupPtr & backup, const S } /// Finish writing. - saveMarks(lock); - saveFileSizes(lock); + storage->saveMarks(lock); + storage->saveFileSizes(lock); } catch (...) { /// Rollback partial writes. file_checker.repair(); - removeUnsavedMarks(lock); + storage->removeUnsavedMarks(lock); throw; } - }; - return {restore_task}; + return {}; + } + +private: + std::shared_ptr storage; + BackupPtr backup; + String data_path_in_backup; + ContextMutablePtr context; +}; + +RestoreTaskPtr StorageLog::restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings &) +{ + if (!partitions.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); + + return std::make_unique( + typeid_cast>(shared_from_this()), backup, data_path_in_backup, context); } diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h index 8b2ef0ccac1..b9255c16f2b 100644 --- a/src/Storages/StorageLog.h +++ b/src/Storages/StorageLog.h @@ -23,6 +23,7 @@ class StorageLog final : public shared_ptr_helper, public IStorage { friend class LogSource; friend class LogSink; + friend class LogRestoreTask; friend struct shared_ptr_helper; public: @@ -31,7 +32,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -51,8 +52,9 @@ public: bool supportsSubcolumns() const override { return true; } ColumnSizeByName getColumnSizes() const override; - BackupEntries backup(const ASTs & partitions, ContextPtr context) override; - RestoreDataTasks restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context) override; + bool hasDataToBackup() const override { return true; } + BackupEntries backupData(ContextPtr context, const ASTs & partitions) override; + RestoreTaskPtr restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings & restore_settings) override; protected: /** Attach the table with the appropriate name, along the appropriate path (with / at the end), diff --git a/src/Storages/StorageMaterializedMySQL.cpp b/src/Storages/StorageMaterializedMySQL.cpp index 07817aa634e..dbc0dd9ae92 100644 --- a/src/Storages/StorageMaterializedMySQL.cpp +++ b/src/Storages/StorageMaterializedMySQL.cpp @@ -34,7 +34,7 @@ bool StorageMaterializedMySQL::needRewriteQueryWithFinal(const Names & column_na Pipe StorageMaterializedMySQL::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & /*storage_snapshot*/, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -44,7 +44,7 @@ Pipe StorageMaterializedMySQL::read( if (const auto * db = typeid_cast(database)) db->rethrowExceptionIfNeeded(); - return readFinalFromNestedStorage(nested_storage, column_names, metadata_snapshot, + return readFinalFromNestedStorage(nested_storage, column_names, query_info, context, processed_stage, max_block_size, num_streams); } diff --git a/src/Storages/StorageMaterializedMySQL.h b/src/Storages/StorageMaterializedMySQL.h index 8ba98c3000f..953d83360fd 100644 --- a/src/Storages/StorageMaterializedMySQL.h +++ b/src/Storages/StorageMaterializedMySQL.h @@ -25,7 +25,7 @@ public: bool needRewriteQueryWithFinal(const Names & column_names) const override; Pipe read( - const Names & column_names, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info, + const Names & column_names, const StorageSnapshotPtr & metadata_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; SinkToStoragePtr write(const ASTPtr &, const StorageMetadataPtr &, ContextPtr) override { throwNotAllowed(); } @@ -38,7 +38,7 @@ public: void drop() override { nested_storage->drop(); } private: - [[noreturn]] void throwNotAllowed() const + [[noreturn]] static void throwNotAllowed() { throw Exception("This method is not allowed for MaterializedMySQL", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 7c5ef5ac04c..0c79c31eb7a 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -25,6 +25,9 @@ #include #include +#include +#include + namespace DB { @@ -132,19 +135,20 @@ StorageMaterializedView::StorageMaterializedView( QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage( ContextPtr local_context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr &, + const StorageSnapshotPtr &, SelectQueryInfo & query_info) const { /// TODO: Find a way to support projections for StorageMaterializedView. Why do we use different /// metadata for materialized view and target table? If they are the same, we can get rid of all /// converting and use it just like a normal view. query_info.ignore_projections = true; - return getTargetTable()->getQueryProcessingStage(local_context, to_stage, getTargetTable()->getInMemoryMetadataPtr(), query_info); + const auto & target_metadata = getTargetTable()->getInMemoryMetadataPtr(); + return getTargetTable()->getQueryProcessingStage(local_context, to_stage, getTargetTable()->getStorageSnapshot(target_metadata), query_info); } Pipe StorageMaterializedView::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -152,7 +156,7 @@ Pipe StorageMaterializedView::read( const unsigned num_streams) { QueryPlan plan; - read(plan, column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + read(plan, column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); return plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(local_context), BuildQueryPipelineSettings::fromContext(local_context)); @@ -161,7 +165,7 @@ Pipe StorageMaterializedView::read( void StorageMaterializedView::read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -171,15 +175,16 @@ void StorageMaterializedView::read( auto storage = getTargetTable(); auto lock = storage->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); auto target_metadata_snapshot = storage->getInMemoryMetadataPtr(); + auto target_storage_snapshot = storage->getStorageSnapshot(target_metadata_snapshot); if (query_info.order_optimizer) query_info.input_order_info = query_info.order_optimizer->getInputOrder(target_metadata_snapshot, local_context); - storage->read(query_plan, column_names, target_metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + storage->read(query_plan, column_names, target_storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); if (query_plan.isInitialized()) { - auto mv_header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, query_info, local_context, processed_stage); + auto mv_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); auto target_header = query_plan.getCurrentDataStream().header; /// No need to convert columns that does not exists in MV @@ -428,6 +433,20 @@ Strings StorageMaterializedView::getDataPaths() const return {}; } +BackupEntries StorageMaterializedView::backupData(ContextPtr context_, const ASTs & partitions_) +{ + if (!hasInnerTable()) + return {}; + return getTargetTable()->backupData(context_, partitions_); +} + +RestoreTaskPtr StorageMaterializedView::restoreData(ContextMutablePtr context_, const ASTs & partitions_, const BackupPtr & backup_, const String & data_path_in_backup_, const StorageRestoreSettings & restore_settings_) +{ + if (!hasInnerTable()) + return {}; + return getTargetTable()->restoreData(context_, partitions_, backup_, data_path_in_backup_, restore_settings_); +} + ActionLock StorageMaterializedView::getActionLock(StorageActionBlockType type) { if (has_inner_table) diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index 395560c1ca7..41c97fbc4d8 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -66,7 +66,7 @@ public: void shutdown() override; QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override; + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override; StoragePtr getTargetTable() const; StoragePtr tryGetTargetTable() const; @@ -78,7 +78,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -88,7 +88,7 @@ public: void read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -97,6 +97,10 @@ public: Strings getDataPaths() const override; + bool hasDataToBackup() const override { return hasInnerTable(); } + BackupEntries backupData(ContextPtr context_, const ASTs & partitions_) override; + RestoreTaskPtr restoreData(ContextMutablePtr context_, const ASTs & partitions_, const BackupPtr & backup, const String & data_path_in_backup_, const StorageRestoreSettings & restore_settings_) override; + private: /// Will be initialized in constructor StorageID target_table_id = StorageID::createEmpty(); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 72851472b79..30be297194a 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -2,9 +2,13 @@ #include #include +#include +#include #include #include #include +#include +#include #include #include @@ -13,6 +17,17 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + namespace DB { @@ -20,6 +35,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int NOT_IMPLEMENTED; } @@ -30,13 +46,13 @@ public: MemorySource( Names column_names_, - const StorageMemory & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, std::shared_ptr data_, std::shared_ptr> parallel_execution_index_, InitializerFunc initializer_func_ = {}) - : SourceWithProgress(metadata_snapshot->getSampleBlockForColumns(column_names_, storage.getVirtuals(), storage.getStorageID())) - , column_names_and_types(metadata_snapshot->getColumns().getByNames(ColumnsDescription::All, column_names_, true)) + : SourceWithProgress(storage_snapshot->getSampleBlockForColumns(column_names_)) + , column_names_and_types(storage_snapshot->getColumnsByNames( + GetColumnsOptions(GetColumnsOptions::All).withSubcolumns().withExtendedObjects(), column_names_)) , data(data_) , parallel_execution_index(parallel_execution_index_) , initializer_func(std::move(initializer_func_)) @@ -62,12 +78,20 @@ protected: } const Block & src = (*data)[current_index]; - Columns columns; - columns.reserve(column_names_and_types.size()); - /// Add only required columns to `res`. - for (const auto & elem : column_names_and_types) - columns.emplace_back(getColumnFromBlock(src, elem)); + Columns columns; + size_t num_columns = column_names_and_types.size(); + columns.reserve(num_columns); + + auto name_and_type = column_names_and_types.begin(); + for (size_t i = 0; i < num_columns; ++i) + { + columns.emplace_back(tryGetColumnFromBlock(src, *name_and_type)); + ++name_and_type; + } + + fillMissingColumns(columns, src.rows(), column_names_and_types, /*metadata_snapshot=*/ nullptr); + assert(std::all_of(columns.begin(), columns.end(), [](const auto & column) { return column != nullptr; })); return Chunk(std::move(columns), src.rows()); } @@ -101,7 +125,7 @@ public: const StorageMetadataPtr & metadata_snapshot_) : SinkToStorage(metadata_snapshot_->getSampleBlock()) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_.getStorageSnapshot(metadata_snapshot_)) { } @@ -110,7 +134,15 @@ public: void consume(Chunk chunk) override { auto block = getHeader().cloneWithColumns(chunk.getColumns()); - metadata_snapshot->check(block, true); + storage_snapshot->metadata->check(block, true); + if (!storage_snapshot->object_columns.empty()) + { + auto columns = storage_snapshot->metadata->getColumns().getAllPhysical().filter(block.getNames()); + auto extended_storage_columns = storage_snapshot->getColumns( + GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects()); + + convertObjectsToTuples(columns, block, extended_storage_columns); + } if (storage.compress) { @@ -151,7 +183,7 @@ private: Blocks new_blocks; StorageMemory & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; }; @@ -170,17 +202,36 @@ StorageMemory::StorageMemory( setInMemoryMetadata(storage_metadata); } +StorageSnapshotPtr StorageMemory::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const +{ + auto snapshot_data = std::make_unique(); + snapshot_data->blocks = data.get(); + + if (!hasObjectColumns(metadata_snapshot->getColumns())) + return std::make_shared(*this, metadata_snapshot, ColumnsDescription{}, std::move(snapshot_data)); + + auto object_columns = getObjectColumns( + snapshot_data->blocks->begin(), + snapshot_data->blocks->end(), + metadata_snapshot->getColumns(), + [](const auto & block) -> const auto & { return block.getColumnsWithTypeAndName(); }); + + return std::make_shared(*this, metadata_snapshot, object_columns, std::move(snapshot_data)); +} Pipe StorageMemory::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, size_t /*max_block_size*/, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); + + const auto & snapshot_data = assert_cast(*storage_snapshot->data); + auto current_data = snapshot_data.blocks; if (delay_read_for_global_subqueries) { @@ -194,17 +245,15 @@ Pipe StorageMemory::read( return Pipe(std::make_shared( column_names, - *this, - metadata_snapshot, + storage_snapshot, nullptr /* data */, nullptr /* parallel execution index */, - [this](std::shared_ptr & data_to_initialize) + [current_data](std::shared_ptr & data_to_initialize) { - data_to_initialize = data.get(); + data_to_initialize = current_data; })); } - auto current_data = data.get(); size_t size = current_data->size(); if (num_streams > size) @@ -216,7 +265,7 @@ Pipe StorageMemory::read( for (size_t stream = 0; stream < num_streams; ++stream) { - pipes.emplace_back(std::make_shared(column_names, *this, metadata_snapshot, current_data, parallel_execution_index)); + pipes.emplace_back(std::make_shared(column_names, storage_snapshot, current_data, parallel_execution_index)); } return Pipe::unitePipes(std::move(pipes)); @@ -327,6 +376,193 @@ void StorageMemory::truncate( total_size_rows.store(0, std::memory_order_relaxed); } + +class MemoryBackupEntriesBatch : public shared_ptr_helper, public IBackupEntriesBatch +{ +private: + friend struct shared_ptr_helper; + + MemoryBackupEntriesBatch( + const StorageMetadataPtr & metadata_snapshot_, const std::shared_ptr blocks_, UInt64 max_compress_block_size_) + : IBackupEntriesBatch({"data.bin", "index.mrk", "sizes.json"}) + , metadata_snapshot(metadata_snapshot_) + , blocks(blocks_) + , max_compress_block_size(max_compress_block_size_) + { + } + + static constexpr const size_t kDataBinPos = 0; + static constexpr const size_t kIndexMrkPos = 1; + static constexpr const size_t kSizesJsonPos = 2; + static constexpr const size_t kSize = 3; + + void initialize() + { + std::call_once(initialized_flag, [this]() + { + temp_dir_owner.emplace(); + auto temp_dir = temp_dir_owner->path(); + fs::create_directories(temp_dir); + + /// Writing data.bin + constexpr char data_file_name[] = "data.bin"; + String data_file_path = temp_dir + "/" + data_file_name; + IndexForNativeFormat index; + { + auto data_out_compressed = std::make_unique(data_file_path); + CompressedWriteBuffer data_out{*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size}; + NativeWriter block_out{data_out, 0, metadata_snapshot->getSampleBlock(), false, &index}; + for (const auto & block : *blocks) + block_out.write(block); + } + + /// Writing index.mrk + constexpr char index_file_name[] = "index.mrk"; + String index_file_path = temp_dir + "/" + index_file_name; + { + auto index_out_compressed = std::make_unique(index_file_path); + CompressedWriteBuffer index_out{*index_out_compressed}; + index.write(index_out); + } + + /// Writing sizes.json + constexpr char sizes_file_name[] = "sizes.json"; + String sizes_file_path = temp_dir + "/" + sizes_file_name; + FileChecker file_checker{sizes_file_path}; + file_checker.update(data_file_path); + file_checker.update(index_file_path); + file_checker.save(); + + file_paths[kDataBinPos] = data_file_path; + file_sizes[kDataBinPos] = file_checker.getFileSize(data_file_path); + + file_paths[kIndexMrkPos] = index_file_path; + file_sizes[kIndexMrkPos] = file_checker.getFileSize(index_file_path); + + file_paths[kSizesJsonPos] = sizes_file_path; + file_sizes[kSizesJsonPos] = fs::file_size(sizes_file_path); + + /// We don't need to keep `blocks` any longer. + blocks.reset(); + metadata_snapshot.reset(); + }); + } + + std::unique_ptr getReadBuffer(size_t index) override + { + initialize(); + return createReadBufferFromFileBase(file_paths[index], {}); + } + + UInt64 getSize(size_t index) override + { + initialize(); + return file_sizes[index]; + } + + StorageMetadataPtr metadata_snapshot; + std::shared_ptr blocks; + UInt64 max_compress_block_size; + std::once_flag initialized_flag; + std::optional temp_dir_owner; + std::array file_paths; + std::array file_sizes; +}; + + +BackupEntries StorageMemory::backupData(ContextPtr context, const ASTs & partitions) +{ + if (!partitions.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); + + return MemoryBackupEntriesBatch::create(getInMemoryMetadataPtr(), data.get(), context->getSettingsRef().max_compress_block_size) + ->getBackupEntries(); +} + + +class MemoryRestoreTask : public IRestoreTask +{ +public: + MemoryRestoreTask( + std::shared_ptr storage_, const BackupPtr & backup_, const String & data_path_in_backup_, ContextMutablePtr context_) + : storage(storage_), backup(backup_), data_path_in_backup(data_path_in_backup_), context(context_) + { + } + + RestoreTasks run() override + { + /// Our data are in the StripeLog format. + + /// Reading index.mrk + IndexForNativeFormat index; + { + String index_file_path = data_path_in_backup + "index.mrk"; + auto backup_entry = backup->readFile(index_file_path); + auto in = backup_entry->getReadBuffer(); + CompressedReadBuffer compressed_in{*in}; + index.read(compressed_in); + } + + /// Reading data.bin + Blocks new_blocks; + size_t new_bytes = 0; + size_t new_rows = 0; + { + String data_file_path = data_path_in_backup + "data.bin"; + auto backup_entry = backup->readFile(data_file_path); + std::unique_ptr in = backup_entry->getReadBuffer(); + std::optional temp_data_copy; + if (!dynamic_cast(in.get())) + { + temp_data_copy.emplace(); + auto temp_data_copy_out = std::make_unique(temp_data_copy->path()); + copyData(*in, *temp_data_copy_out); + temp_data_copy_out.reset(); + in = createReadBufferFromFileBase(temp_data_copy->path(), {}); + } + std::unique_ptr in_from_file{static_cast(in.release())}; + CompressedReadBufferFromFile compressed_in{std::move(in_from_file)}; + NativeReader block_in{compressed_in, 0, index.blocks.begin(), index.blocks.end()}; + + while (auto block = block_in.read()) + { + new_bytes += block.bytes(); + new_rows += block.rows(); + new_blocks.push_back(std::move(block)); + } + } + + /// Append old blocks with the new ones. + auto old_blocks = storage->data.get(); + Blocks old_and_new_blocks = *old_blocks; + old_and_new_blocks.insert(old_and_new_blocks.end(), std::make_move_iterator(new_blocks.begin()), std::make_move_iterator(new_blocks.end())); + + /// Finish restoring. + storage->data.set(std::make_unique(std::move(old_and_new_blocks))); + storage->total_size_bytes += new_bytes; + storage->total_size_rows += new_rows; + + return {}; + } + +private: + std::shared_ptr storage; + BackupPtr backup; + String data_path_in_backup; + ContextMutablePtr context; +}; + + +RestoreTaskPtr StorageMemory::restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings &) +{ + if (!partitions.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); + + return std::make_unique( + typeid_cast>(shared_from_this()), backup, data_path_in_backup, context); +} + + std::optional StorageMemory::totalRows(const Settings &) const { /// All modifications of these counters are done under mutex which automatically guarantees synchronization/consistency diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index 063802faf1a..20f47828846 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -22,6 +22,7 @@ namespace DB class StorageMemory final : public shared_ptr_helper, public IStorage { friend class MemorySink; +friend class MemoryRestoreTask; friend struct shared_ptr_helper; public: @@ -29,9 +30,18 @@ public: size_t getSize() const { return data.get()->size(); } + /// Snapshot for StorageMemory contains current set of blocks + /// at the moment of the start of query. + struct SnapshotData : public StorageSnapshot::Data + { + std::shared_ptr blocks; + }; + + StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot) const override; + Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -40,6 +50,7 @@ public: bool supportsParallelInsert() const override { return true; } bool supportsSubcolumns() const override { return true; } + bool supportsDynamicSubcolumns() const override { return true; } /// Smaller blocks (e.g. 64K rows) are better for CPU cache. bool prefersLargeBlocks() const override { return false; } @@ -55,6 +66,10 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override; + bool hasDataToBackup() const override { return true; } + BackupEntries backupData(ContextPtr context, const ASTs & partitions) override; + RestoreTaskPtr restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings & restore_settings) override; + std::optional totalRows(const Settings &) const override; std::optional totalBytes(const Settings &) const override; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 433fdb5b0b5..96e6070e09e 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -169,7 +169,7 @@ bool StorageMerge::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, Cont QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( ContextPtr local_context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr &, + const StorageSnapshotPtr &, SelectQueryInfo & query_info) const { /// In case of JOIN the first stage (which includes JOIN) @@ -200,7 +200,8 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( ++selected_table_size; stage_in_source_tables = std::max( stage_in_source_tables, - table->getQueryProcessingStage(local_context, to_stage, table->getInMemoryMetadataPtr(), query_info)); + table->getQueryProcessingStage(local_context, to_stage, + table->getStorageSnapshot(table->getInMemoryMetadataPtr()), query_info)); } iterator->next(); @@ -235,7 +236,7 @@ SelectQueryInfo StorageMerge::getModifiedQueryInfo( Pipe StorageMerge::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -251,9 +252,9 @@ Pipe StorageMerge::read( for (const auto & column_name : column_names) { - if (column_name == "_database" && isVirtualColumn(column_name, metadata_snapshot)) + if (column_name == "_database" && isVirtualColumn(column_name, storage_snapshot->metadata)) has_database_virtual_column = true; - else if (column_name == "_table" && isVirtualColumn(column_name, metadata_snapshot)) + else if (column_name == "_table" && isVirtualColumn(column_name, storage_snapshot->metadata)) has_table_virtual_column = true; else real_column_names.push_back(column_name); @@ -266,7 +267,7 @@ Pipe StorageMerge::read( modified_context->setSetting("optimize_move_to_prewhere", false); /// What will be result structure depending on query processed stage in source tables? - Block header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, query_info, local_context, processed_stage); + Block header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); /** First we make list of selected tables to find out its size. * This is necessary to correctly pass the recommended number of threads to each table. @@ -337,9 +338,11 @@ Pipe StorageMerge::read( Aliases aliases; auto storage_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_columns = storage_metadata_snapshot->getColumns(); + auto nested_storage_snaphsot = storage->getStorageSnapshot(storage_metadata_snapshot); auto modified_query_info = getModifiedQueryInfo(query_info, modified_context, storage->getStorageID(), storage->as()); - auto syntax_result = TreeRewriter(local_context).analyzeSelect(modified_query_info.query, TreeRewriterResult({}, storage, storage_metadata_snapshot)); + auto syntax_result = TreeRewriter(local_context).analyzeSelect( + modified_query_info.query, TreeRewriterResult({}, storage, nested_storage_snaphsot)); Names column_names_as_aliases; bool with_aliases = processed_stage == QueryProcessingStage::FetchColumns && !storage_columns.getAliases().empty(); @@ -374,7 +377,8 @@ Pipe StorageMerge::read( } syntax_result = TreeRewriter(local_context).analyze( - required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage_metadata_snapshot); + required_columns_expr_list, storage_columns.getAllPhysical(), storage, storage->getStorageSnapshot(storage_metadata_snapshot)); + auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, local_context).getActionsDAG(true); column_names_as_aliases = alias_actions->getRequiredColumns().getNames(); @@ -383,7 +387,7 @@ Pipe StorageMerge::read( } auto source_pipe = createSources( - storage_metadata_snapshot, + nested_storage_snaphsot, modified_query_info, processed_stage, max_block_size, @@ -411,7 +415,7 @@ Pipe StorageMerge::read( } Pipe StorageMerge::createSources( - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & modified_query_info, const QueryProcessingStage::Enum & processed_stage, const UInt64 max_block_size, @@ -449,16 +453,16 @@ Pipe StorageMerge::createSources( } auto storage_stage - = storage->getQueryProcessingStage(modified_context, QueryProcessingStage::Complete, metadata_snapshot, modified_query_info); + = storage->getQueryProcessingStage(modified_context, QueryProcessingStage::Complete, storage_snapshot, modified_query_info); if (processed_stage <= storage_stage) { /// If there are only virtual columns in query, you must request at least one other column. if (real_column_names.empty()) - real_column_names.push_back(ExpressionActions::getSmallestColumn(metadata_snapshot->getColumns().getAllPhysical())); + real_column_names.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical())); pipe = storage->read( real_column_names, - metadata_snapshot, + storage_snapshot, modified_query_info, modified_context, processed_stage, @@ -538,7 +542,7 @@ Pipe StorageMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, metadata_snapshot, aliases, modified_context, modified_query_info.query, pipe, processed_stage); + convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, modified_query_info.query, pipe, processed_stage); pipe.addTableLock(struct_lock); pipe.addStorageHolder(storage); @@ -730,7 +734,7 @@ void StorageMerge::convertingSourceStream( for (const auto & alias : aliases) { pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type)); - ASTPtr expr = std::move(alias.expression); + ASTPtr expr = alias.expression; auto syntax_result = TreeRewriter(local_context).analyze(expr, pipe_columns); auto expression_analyzer = ExpressionAnalyzer{alias.expression, syntax_result, local_context}; diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index e0d81531325..b7bdd957164 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -30,11 +30,11 @@ public: bool canMoveConditionsToPrewhere() const override; QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override; + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override; Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -116,7 +116,7 @@ protected: using Aliases = std::vector; Pipe createSources( - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, UInt64 max_block_size, diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 6cd1e2d66af..7f4c3deca37 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -3,7 +3,7 @@ #include #include - +#include #include #include #include @@ -108,7 +108,7 @@ void StorageMergeTree::startup() /// Temporary directories contain incomplete results of merges (after forced restart) /// and don't allow to reinitialize them, so delete each of them immediately - clearOldTemporaryDirectories(merger_mutator, 0); + clearOldTemporaryDirectories(0); /// NOTE background task will also do the above cleanups periodically. time_after_previous_cleanup_parts.restart(); @@ -193,7 +193,7 @@ StorageMergeTree::~StorageMergeTree() void StorageMergeTree::read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -207,13 +207,13 @@ void StorageMergeTree::read( LOG_TRACE(log, "Parallel reading from replicas enabled {}", enable_parallel_reading); if (auto plan = reader.read( - column_names, metadata_snapshot, query_info, local_context, max_block_size, num_streams, processed_stage, nullptr, enable_parallel_reading)) + column_names, storage_snapshot, query_info, local_context, max_block_size, num_streams, processed_stage, nullptr, enable_parallel_reading)) query_plan = std::move(*plan); } Pipe StorageMergeTree::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -221,7 +221,7 @@ Pipe StorageMergeTree::read( const unsigned num_streams) { QueryPlan plan; - read(plan, column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + read(plan, column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); return plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(local_context), BuildQueryPipelineSettings::fromContext(local_context)); @@ -745,9 +745,8 @@ std::shared_ptr StorageMergeTree::selectPartsToMerge( { while (true) { - UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace(); select_decision = merger_mutator.selectAllPartsToMergeWithinPartition( - future_part, disk_space, can_merge, partition_id, final, metadata_snapshot, out_disable_reason, optimize_skip_merged_partitions); + future_part, can_merge, partition_id, final, metadata_snapshot, out_disable_reason, optimize_skip_merged_partitions); auto timeout_ms = getSettings()->lock_acquire_timeout_for_background_operations.totalMilliseconds(); auto timeout = std::chrono::milliseconds(timeout_ms); @@ -1057,13 +1056,13 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign } bool scheduled = false; - if (time_after_previous_cleanup_temporary_directories.compareAndRestartDeferred( + if (auto lock = time_after_previous_cleanup_temporary_directories.compareAndRestartDeferred( getSettings()->merge_tree_clear_old_temporary_directories_interval_seconds)) { assignee.scheduleCommonTask(ExecutableLambdaAdapter::create( [this, share_lock] () { - return clearOldTemporaryDirectories(merger_mutator, getSettings()->temporary_directories_lifetime.totalSeconds()); + return clearOldTemporaryDirectories(getSettings()->temporary_directories_lifetime.totalSeconds()); }, common_assignee_trigger, getStorageID()), /* need_trigger */ false); scheduled = true; } @@ -1645,9 +1644,9 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ } -RestoreDataTasks StorageMergeTree::restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr local_context) +RestoreTaskPtr StorageMergeTree::restoreData(ContextMutablePtr local_context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings &) { - return restoreDataPartsFromBackup(backup, data_path_in_backup, getPartitionIDsFromQuery(partitions, local_context), &increment); + return restoreDataParts(getPartitionIDsFromQuery(partitions, local_context), backup, data_path_in_backup, &increment); } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index d3970449ceb..a1fc310d912 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -43,7 +43,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -53,7 +53,7 @@ public: void read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -97,7 +97,7 @@ public: CheckResults checkData(const ASTPtr & query, ContextPtr context) override; - RestoreDataTasks restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context) override; + RestoreTaskPtr restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings & restore_settings) override; bool scheduleDataProcessingJob(BackgroundJobsAssignee & assignee) override; diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 9b25b44c0e7..964b78083f7 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -90,7 +90,7 @@ void StorageMongoDB::connectIfNotConnected() Pipe StorageMongoDB::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, @@ -99,12 +99,12 @@ Pipe StorageMongoDB::read( { connectIfNotConnected(); - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); Block sample_block; for (const String & column_name : column_names) { - auto column_data = metadata_snapshot->getColumns().getPhysical(column_name); + auto column_data = storage_snapshot->metadata->getColumns().getPhysical(column_name); sample_block.insert({ column_data.type, column_data.name }); } @@ -156,6 +156,8 @@ StorageMongoDBConfiguration StorageMongoDB::getConfiguration(ASTs engine_args, C } + context->getRemoteHostFilter().checkHostAndPort(configuration.host, toString(configuration.port)); + return configuration; } diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 16b85364c5c..549d444d7bb 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -22,7 +22,7 @@ public: StorageMongoDB( const StorageID & table_id_, const std::string & host_, - short unsigned int port_, + uint16_t port_, const std::string & database_name_, const std::string & collection_name_, const std::string & username_, @@ -36,7 +36,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -49,7 +49,7 @@ private: void connectIfNotConnected(); const std::string host; - const short unsigned int port; + const uint16_t port; /// NOLINT const std::string database_name; const std::string collection_name; const std::string username; diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 83cf2b07b21..5e7c2ae95ae 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -75,17 +75,17 @@ StorageMySQL::StorageMySQL( Pipe StorageMySQL::read( const Names & column_names_, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info_, ContextPtr context_, QueryProcessingStage::Enum /*processed_stage*/, size_t /*max_block_size*/, unsigned) { - metadata_snapshot->check(column_names_, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names_); String query = transformQueryForExternalDatabase( query_info_, - metadata_snapshot->getColumns().getOrdinary(), + storage_snapshot->metadata->getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, remote_table_name, @@ -95,7 +95,7 @@ Pipe StorageMySQL::read( Block sample_block; for (const String & column_name : column_names_) { - auto column_data = metadata_snapshot->getColumns().getPhysical(column_name); + auto column_data = storage_snapshot->metadata->getColumns().getPhysical(column_name); WhichDataType which(column_data.type); /// Convert enum to string. @@ -178,7 +178,7 @@ public: { /// Avoid Excessive copy when block is small enough if (block.rows() <= max_rows) - return Blocks{std::move(block)}; + return {block}; const size_t split_block_size = ceil(block.rows() * 1.0 / max_rows); Blocks split_blocks(split_block_size); @@ -281,13 +281,13 @@ StorageMySQLConfiguration StorageMySQL::getConfiguration(ASTs engine_args, Conte configuration.table = engine_args[2]->as().value.safeGet(); configuration.username = engine_args[3]->as().value.safeGet(); configuration.password = engine_args[4]->as().value.safeGet(); - if (engine_args.size() >= 6) configuration.replace_query = engine_args[5]->as().value.safeGet(); if (engine_args.size() == 7) configuration.on_duplicate_clause = engine_args[6]->as().value.safeGet(); } - + for (const auto & address : configuration.addresses) + context_->getRemoteHostFilter().checkHostAndPort(address.first, toString(address.second)); if (configuration.replace_query && !configuration.on_duplicate_clause.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only one of 'replace_query' and 'on_duplicate_clause' can be specified, or none of them"); diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index fe2ee8439bc..03ebaaf87d7 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -44,7 +44,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageNull.h b/src/Storages/StorageNull.h index 5fef7f984e4..c5b2e2bf161 100644 --- a/src/Storages/StorageNull.h +++ b/src/Storages/StorageNull.h @@ -23,7 +23,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo &, ContextPtr /*context*/, QueryProcessingStage::Enum /*processing_stage*/, @@ -31,7 +31,7 @@ public: unsigned) override { return Pipe( - std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); + std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names))); } bool supportsParallelInsert() const override { return true; } @@ -61,11 +61,11 @@ protected: const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_, const String & comment) : IStorage(table_id_) { - StorageInMemoryMetadata metadata_; - metadata_.setColumns(columns_description_); - metadata_.setConstraints(constraints_); - metadata_.setComment(comment); - setInMemoryMetadata(metadata_); + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns_description_); + storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); + setInMemoryMetadata(storage_metadata); } }; diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 5042f911149..c50086d2c52 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -76,26 +76,26 @@ StoragePostgreSQL::StoragePostgreSQL( Pipe StoragePostgreSQL::read( const Names & column_names_, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info_, ContextPtr context_, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size_, unsigned) { - metadata_snapshot->check(column_names_, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names_); /// Connection is already made to the needed database, so it should not be present in the query; /// remote_table_schema is empty if it is not specified, will access only table_name. String query = transformQueryForExternalDatabase( - query_info_, metadata_snapshot->getColumns().getOrdinary(), + query_info_, storage_snapshot->metadata->getColumns().getOrdinary(), IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_); LOG_TRACE(log, "Query: {}", query); Block sample_block; for (const String & column_name : column_names_) { - auto column_data = metadata_snapshot->getColumns().getPhysical(column_name); + auto column_data = storage_snapshot->metadata->getColumns().getPhysical(column_name); WhichDataType which(column_data.type); if (which.isEnum()) column_data.type = std::make_shared(); @@ -425,7 +425,6 @@ StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_a configuration.host = configuration.addresses[0].first; configuration.port = configuration.addresses[0].second; } - configuration.database = engine_args[1]->as().value.safeGet(); configuration.table = engine_args[2]->as().value.safeGet(); configuration.username = engine_args[3]->as().value.safeGet(); @@ -436,6 +435,8 @@ StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_a if (engine_args.size() >= 7) configuration.on_conflict = engine_args[6]->as().value.safeGet(); } + for (const auto & address : configuration.addresses) + context->getRemoteHostFilter().checkHostAndPort(address.first, toString(address.second)); return configuration; } diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index 7d8752c91b9..ae41a713285 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -35,7 +35,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 894b470ef22..d5af81ced3d 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -35,12 +35,13 @@ public: QueryProcessingStage::Enum getQueryProcessingStage( ContextPtr context, QueryProcessingStage::Enum to_stage, - const StorageMetadataPtr &, + const StorageSnapshotPtr &, SelectQueryInfo & info) const override { /// TODO: Find a way to support projections for StorageProxy info.ignore_projections = true; - return getNested()->getQueryProcessingStage(context, to_stage, getNested()->getInMemoryMetadataPtr(), info); + const auto & nested_metadata = getNested()->getInMemoryMetadataPtr(); + return getNested()->getQueryProcessingStage(context, to_stage, getNested()->getStorageSnapshot(nested_metadata), info); } Pipe watch( @@ -56,14 +57,14 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override { - return getNested()->read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + return getNested()->read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); } SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c57e62f940a..b013b24f17b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -451,7 +451,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } /// Temporary directories contain uninitialized results of Merges or Fetches (after forced restart), /// don't allow to reinitialize them, delete each of them immediately. - clearOldTemporaryDirectories(merger_mutator, 0); + clearOldTemporaryDirectories(0); clearOldWriteAheadLogs(); } @@ -1290,6 +1290,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: { String columns_str; String checksums_str; + if (zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str) && zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str)) { @@ -1559,7 +1560,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) } -bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) +bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry, bool need_to_check_missing_part) { /// Looking for covering part. After that entry.actual_new_part_name may be filled. String replica = findReplicaHavingCoveringPart(entry, true); @@ -1683,6 +1684,10 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (replica.empty()) { ProfileEvents::increment(ProfileEvents::ReplicatedPartFailedFetches); + + if (!need_to_check_missing_part) + return false; + throw Exception("No active replica has part " + entry.new_part_name + " or covering part", ErrorCodes::NO_REPLICA_HAS_PART); } } @@ -1819,7 +1824,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) } /// Forcibly remove parts from ZooKeeper - tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); + removePartsFromZooKeeperWithRetries(parts_to_remove); if (entry.detach) LOG_DEBUG(log, "Detached {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); @@ -1941,7 +1946,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (parts_to_add.empty()) { LOG_INFO(log, "All parts from REPLACE PARTITION command have been already attached"); - tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); + removePartsFromZooKeeperWithRetries(parts_to_remove); return true; } @@ -2185,7 +2190,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) throw; } - tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); + removePartsFromZooKeeperWithRetries(parts_to_remove); res_parts.clear(); parts_to_remove.clear(); cleanup_thread.wakeup(); @@ -2417,7 +2422,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo "or removed broken part from ZooKeeper", source_replica); } - tryRemovePartsFromZooKeeperWithRetries(parts_to_remove_from_zk); + removePartsFromZooKeeperWithRetries(parts_to_remove_from_zk); auto local_active_parts = getDataParts(); @@ -3064,7 +3069,8 @@ void StorageReplicatedMergeTree::mergeSelectingTask() desired_mutation_version->second, merge_pred.getVersion()); - if (create_result == CreateMergeEntryResult::Ok) + if (create_result == CreateMergeEntryResult::Ok || + create_result == CreateMergeEntryResult::LogUpdated) break; } } @@ -3786,24 +3792,41 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (source_part) { - MinimalisticDataPartChecksums source_part_checksums; - source_part_checksums.computeTotalChecksums(source_part->checksums); + auto source_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( + source_part->getColumns(), source_part->checksums); - MinimalisticDataPartChecksums desired_checksums; String part_path = fs::path(source_replica_path) / "parts" / part_name; String part_znode = zookeeper->get(part_path); + std::optional desired_part_header; if (!part_znode.empty()) - desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums(); + { + desired_part_header = ReplicatedMergeTreePartHeader::fromString(part_znode); + } else { - String desired_checksums_str = zookeeper->get(fs::path(part_path) / "checksums"); - desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str); + String columns_str; + String checksums_str; + + if (zookeeper->tryGet(fs::path(part_path) / "columns", columns_str) && + zookeeper->tryGet(fs::path(part_path) / "checksums", checksums_str)) + { + desired_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksumsZNodes(columns_str, checksums_str); + } + else + { + LOG_INFO(log, "Not checking checksums of part {} with replica {} because part was removed from ZooKeeper", part_name, source_replica_path); + } } - if (source_part_checksums == desired_checksums) + /// Checking both checksums and columns hash. For example we can have empty part + /// with same checksums but different columns. And we attaching it exception will + /// be thrown. + if (desired_part_header + && source_part_header.getColumnsHash() == desired_part_header->getColumnsHash() + && source_part_header.getChecksums() == desired_part_header->getChecksums()) { - LOG_TRACE(log, "Found local part {} with the same checksums as {}", source_part->name, part_name); + LOG_TRACE(log, "Found local part {} with the same checksums and columns hash as {}", source_part->name, part_name); part_to_clone = source_part; } } @@ -4197,7 +4220,7 @@ ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock StorageReplicatedMerg void StorageReplicatedMergeTree::read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -4216,14 +4239,14 @@ void StorageReplicatedMergeTree::read( { auto max_added_blocks = std::make_shared(getMaxAddedBlocks()); if (auto plan = reader.read( - column_names, metadata_snapshot, query_info, local_context, + column_names, storage_snapshot, query_info, local_context, max_block_size, num_streams, processed_stage, std::move(max_added_blocks), enable_parallel_reading)) query_plan = std::move(*plan); return; } if (auto plan = reader.read( - column_names, metadata_snapshot, query_info, local_context, + column_names, storage_snapshot, query_info, local_context, max_block_size, num_streams, processed_stage, nullptr, enable_parallel_reading)) { query_plan = std::move(*plan); @@ -4232,7 +4255,7 @@ void StorageReplicatedMergeTree::read( Pipe StorageReplicatedMergeTree::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -4240,7 +4263,7 @@ Pipe StorageReplicatedMergeTree::read( const unsigned num_streams) { QueryPlan plan; - read(plan, column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + read(plan, column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); return plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(local_context), BuildQueryPipelineSettings::fromContext(local_context)); @@ -4349,7 +4372,6 @@ bool StorageReplicatedMergeTree::optimize( }; auto zookeeper = getZooKeeperAndAssertNotReadonly(); - UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace(); const auto storage_settings_ptr = getSettings(); auto metadata_snapshot = getInMemoryMetadataPtr(); std::vector merge_entries; @@ -4382,7 +4404,7 @@ bool StorageReplicatedMergeTree::optimize( else { select_decision = merger_mutator.selectAllPartsToMergeWithinPartition( - future_merged_part, disk_space, can_merge, partition_id, final, metadata_snapshot, + future_merged_part, can_merge, partition_id, final, metadata_snapshot, &disable_reason, query_context->getSettingsRef().optimize_skip_merged_partitions); } @@ -5986,16 +6008,16 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() } -bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(DataPartsVector & parts, size_t max_retries) +void StorageReplicatedMergeTree::removePartsFromZooKeeperWithRetries(DataPartsVector & parts, size_t max_retries) { Strings part_names_to_remove; for (const auto & part : parts) part_names_to_remove.emplace_back(part->name); - return tryRemovePartsFromZooKeeperWithRetries(part_names_to_remove, max_retries); + return removePartsFromZooKeeperWithRetries(part_names_to_remove, max_retries); } -bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const Strings & part_names, size_t max_retries) +void StorageReplicatedMergeTree::removePartsFromZooKeeperWithRetries(const Strings & part_names, size_t max_retries) { size_t num_tries = 0; bool success = false; @@ -6060,7 +6082,8 @@ bool StorageReplicatedMergeTree::tryRemovePartsFromZooKeeperWithRetries(const St std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } - return success; + if (!success) + throw Exception(ErrorCodes::UNFINISHED, "Failed to remove parts from ZooKeeper after {} retries", num_tries); } void StorageReplicatedMergeTree::removePartsFromZooKeeper( @@ -6373,7 +6396,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( lock.assumeUnlocked(); /// Forcibly remove replaced parts from ZooKeeper - tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); + removePartsFromZooKeeperWithRetries(parts_to_remove); /// Speedup removing of replaced parts from filesystem parts_to_remove.clear(); @@ -6580,7 +6603,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta for (auto & lock : ephemeral_locks) lock.assumeUnlocked(); - tryRemovePartsFromZooKeeperWithRetries(parts_to_remove); + removePartsFromZooKeeperWithRetries(parts_to_remove); parts_to_remove.clear(); cleanup_thread.wakeup(); @@ -7131,9 +7154,9 @@ void StorageReplicatedMergeTree::createTableSharedID() if (!zookeeper->tryGet(zookeeper_table_id_path, id)) { UUID table_id_candidate; - auto storage_id = getStorageID(); - if (storage_id.uuid != UUIDHelpers::Nil) - table_id_candidate = storage_id.uuid; + auto local_storage_id = getStorageID(); + if (local_storage_id.uuid != UUIDHelpers::Nil) + table_id_candidate = local_storage_id.uuid; else table_id_candidate = UUIDHelpers::generateV4(); @@ -7154,10 +7177,35 @@ void StorageReplicatedMergeTree::createTableSharedID() } -void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) const +void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const { - if (!part.volume) + if (!disk || !disk->supportZeroCopyReplication()) return; + + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return; + + String id = part_id; + boost::replace_all(id, "/", "_"); + + Strings zc_zookeeper_paths = getZeroCopyPartPath(*getSettings(), disk->getType(), getTableSharedID(), + part_name, zookeeper_path); + + for (const auto & zc_zookeeper_path : zc_zookeeper_paths) + { + String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name; + + LOG_TRACE(log, "Set zookeeper temporary ephemeral lock {}", zookeeper_node); + createZeroCopyLockNode(zookeeper, zookeeper_node, zkutil::CreateMode::Ephemeral, false); + } +} + +void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock) const +{ + if (!part.volume || !part.isStoredOnDisk()) + return; + DiskPtr disk = part.volume->getDisk(); if (!disk || !disk->supportZeroCopyReplication()) return; @@ -7175,8 +7223,9 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) { String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name; - LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); - createZeroCopyLockNode(zookeeper, zookeeper_node); + LOG_TRACE(log, "Set zookeeper persistent lock {}", zookeeper_node); + + createZeroCopyLockNode(zookeeper, zookeeper_node, zkutil::CreateMode::Persistent, replace_existing_lock); } } @@ -7189,21 +7238,28 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, const String & name) const { - if (!part.volume) + if (!part.volume || !part.isStoredOnDisk()) return true; + DiskPtr disk = part.volume->getDisk(); if (!disk || !disk->supportZeroCopyReplication()) return true; - zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); - if (!zookeeper) + /// If part is temporary refcount file may be absent + auto ref_count_path = fs::path(part.getFullRelativePath()) / IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK; + if (disk->exists(ref_count_path)) + { + auto ref_count = disk->getRefCount(ref_count_path); + if (ref_count > 0) /// Keep part shard info for frozen backups + return false; + } + else + { + /// Temporary part with some absent file cannot be locked in shared mode return true; + } - auto ref_count = part.getNumberOfRefereneces(); - if (ref_count > 0) /// Keep part shard info for frozen backups - return false; - - return unlockSharedDataByID(part.getUniqueId(), getTableSharedID(), name, replica_name, disk, zookeeper, *getSettings(), log, + return unlockSharedDataByID(part.getUniqueId(), getTableSharedID(), name, replica_name, disk, getZooKeeper(), *getSettings(), log, zookeeper_path); } @@ -7216,7 +7272,7 @@ bool StorageReplicatedMergeTree::unlockSharedDataByID(String part_id, const Stri Strings zc_zookeeper_paths = getZeroCopyPartPath(settings, disk->getType(), table_uuid, part_name, zookeeper_path_old); - bool res = true; + bool part_has_no_more_locks = true; for (const auto & zc_zookeeper_path : zc_zookeeper_paths) { @@ -7236,7 +7292,7 @@ bool StorageReplicatedMergeTree::unlockSharedDataByID(String part_id, const Stri if (!children.empty()) { LOG_TRACE(logger, "Found zookeper locks for {}", zookeeper_part_uniq_node); - res = false; + part_has_no_more_locks = false; continue; } @@ -7265,7 +7321,7 @@ bool StorageReplicatedMergeTree::unlockSharedDataByID(String part_id, const Stri } } - return res; + return part_has_no_more_locks; } @@ -7387,8 +7443,31 @@ Strings StorageReplicatedMergeTree::getZeroCopyPartPath(const MergeTreeSettings return res; } +bool StorageReplicatedMergeTree::checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk) +{ + auto path = getZeroCopyPartPath(part_name, disk); + if (path) + { + /// FIXME + auto lock_path = fs::path(*path) / "part_exclusive_lock"; + if (getZooKeeper()->exists(lock_path)) + { + return true; + } + } -std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusiveLock(const DataPartPtr & part, const DiskPtr & disk) + return false; +} + +std::optional StorageReplicatedMergeTree::getZeroCopyPartPath(const String & part_name, const DiskPtr & disk) +{ + if (!disk || !disk->supportZeroCopyReplication()) + return std::nullopt; + + return getZeroCopyPartPath(*getSettings(), disk->getType(), getTableSharedID(), part_name, zookeeper_path)[0]; +} + +std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) { if (!disk || !disk->supportZeroCopyReplication()) return std::nullopt; @@ -7397,8 +7476,7 @@ std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusi if (!zookeeper) return std::nullopt; - String zc_zookeeper_path = getZeroCopyPartPath(*getSettings(), disk->getType(), getTableSharedID(), - part->name, zookeeper_path)[0]; + String zc_zookeeper_path = *getZeroCopyPartPath(part_name, disk); /// Just recursively create ancestors for lock zookeeper->createAncestors(zc_zookeeper_path); @@ -7633,7 +7711,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP } -void StorageReplicatedMergeTree::createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node) +void StorageReplicatedMergeTree::createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node, int32_t mode, bool replace_existing_lock) { /// In rare case other replica can remove path between createAncestors and createIfNotExists /// So we make up to 5 attempts @@ -7642,9 +7720,28 @@ void StorageReplicatedMergeTree::createZeroCopyLockNode(const zkutil::ZooKeeperP { try { - zookeeper->createAncestors(zookeeper_node); - zookeeper->createIfNotExists(zookeeper_node, "lock"); - break; + /// Ephemeral locks can be created only when we fetch shared data. + /// So it never require to create ancestors. If we create them + /// race condition with source replica drop is possible. + if (mode == zkutil::CreateMode::Persistent) + zookeeper->createAncestors(zookeeper_node); + + if (replace_existing_lock && zookeeper->exists(zookeeper_node)) + { + Coordination::Requests ops; + ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_node, -1)); + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_node, "", mode)); + Coordination::Responses responses; + auto error = zookeeper->tryMulti(ops, responses); + if (error == Coordination::Error::ZOK) + break; + } + else + { + auto error = zookeeper->tryCreate(zookeeper_node, "", mode); + if (error == Coordination::Error::ZOK || error == Coordination::Error::ZNODEEXISTS) + break; + } } catch (const zkutil::KeeperException & e) { @@ -7673,10 +7770,12 @@ public: table_shared_id = storage.getTableSharedID(); } - void save(DiskPtr disk, const String & path) const + void save(DiskPtr data_disk, const String & path) const { + auto metadata_disk = data_disk->getMetadataDiskIfExistsOrSelf(); + auto file_path = getFileName(path); - auto buffer = disk->writeMetaFile(file_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + auto buffer = metadata_disk->writeFile(file_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); writeIntText(version, *buffer); buffer->write("\n", 1); writeBoolText(is_replicated, *buffer); @@ -7691,12 +7790,14 @@ public: buffer->write("\n", 1); } - bool load(DiskPtr disk, const String & path) + bool load(DiskPtr data_disk, const String & path) { + auto metadata_disk = data_disk->getMetadataDiskIfExistsOrSelf(); auto file_path = getFileName(path); - if (!disk->exists(file_path)) + + if (!metadata_disk->exists(file_path)) return false; - auto buffer = disk->readMetaFile(file_path, ReadSettings(), {}); + auto buffer = metadata_disk->readFile(file_path, ReadSettings(), {}); readIntText(version, *buffer); if (version != 1) { @@ -7717,9 +7818,10 @@ public: return true; } - static void clean(DiskPtr disk, const String & path) + static void clean(DiskPtr data_disk, const String & path) { - disk->removeMetaFileIfExists(getFileName(path)); + auto metadata_disk = data_disk->getMetadataDiskIfExistsOrSelf(); + metadata_disk->removeFileIfExists(getFileName(path)); } private: @@ -7773,22 +7875,18 @@ bool StorageReplicatedMergeTree::removeSharedDetachedPart(DiskPtr disk, const St zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - if (zookeeper) + fs::path checksums = fs::path(path) / IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK; + if (disk->exists(checksums)) { - fs::path checksums = fs::path(path) / "checksums.txt"; - if (disk->exists(checksums)) + if (disk->getRefCount(checksums) == 0) { - auto ref_count = disk->getRefCount(checksums); - if (ref_count == 0) - { - String id = disk->getUniqueId(checksums); - keep_shared = !StorageReplicatedMergeTree::unlockSharedDataByID(id, table_uuid, part_name, - detached_replica_name, disk, zookeeper, getContext()->getReplicatedMergeTreeSettings(), log, - detached_zookeeper_path); - } - else - keep_shared = true; + String id = disk->getUniqueId(checksums); + keep_shared = !StorageReplicatedMergeTree::unlockSharedDataByID(id, table_uuid, part_name, + detached_replica_name, disk, zookeeper, getContext()->getReplicatedMergeTreeSettings(), log, + detached_zookeeper_path); } + else + keep_shared = true; } disk->removeSharedRecursive(path, keep_shared); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index f60d3f974ca..c567447e9f2 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -98,7 +98,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -108,7 +108,7 @@ public: void read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -231,7 +231,9 @@ public: bool executeFetchShared(const String & source_replica, const String & new_part_name, const DiskPtr & disk, const String & path); /// Lock part in zookeeper for use shared data in several nodes - void lockSharedData(const IMergeTreeDataPart & part) const override; + void lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock) const override; + + void lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const; /// Unlock shared data part in zookeeper /// Return true if data unlocked @@ -281,7 +283,7 @@ public: // Return table id, common for different replicas String getTableSharedID() const; - static const String getDefaultZooKeeperName() { return default_zookeeper_name; } + static String getDefaultZooKeeperName() { return default_zookeeper_name; } /// Check if there are new broken disks and enqueue part recovery tasks. void checkBrokenDisks(); @@ -489,8 +491,9 @@ private: void removePartsFromZooKeeper(zkutil::ZooKeeperPtr & zookeeper, const Strings & part_names, NameSet * parts_should_be_retried = nullptr); - bool tryRemovePartsFromZooKeeperWithRetries(const Strings & part_names, size_t max_retries = 5); - bool tryRemovePartsFromZooKeeperWithRetries(DataPartsVector & parts, size_t max_retries = 5); + /// Remove parts from ZooKeeper, throw exception if unable to do so after max_retries. + void removePartsFromZooKeeperWithRetries(const Strings & part_names, size_t max_retries = 5); + void removePartsFromZooKeeperWithRetries(DataPartsVector & parts, size_t max_retries = 5); /// Removes a part from ZooKeeper and adds a task to the queue to download it. It is supposed to do this with broken parts. void removePartAndEnqueueFetch(const String & part_name); @@ -517,7 +520,7 @@ private: /// NOTE: Attention! First of all tries to find covering part on other replica /// and set it into entry.actual_new_part_name. After that tries to fetch this new covering part. /// If fetch was not successful, clears entry.actual_new_part_name. - bool executeFetch(LogEntry & entry); + bool executeFetch(LogEntry & entry, bool need_to_check_missing_part=true); bool executeReplaceRange(const LogEntry & entry); void executeClonePartFromShard(const LogEntry & entry); @@ -758,7 +761,7 @@ private: static Strings getZeroCopyPartPath(const MergeTreeSettings & settings, DiskType disk_type, const String & table_uuid, const String & part_name, const String & zookeeper_path_old); - static void createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node); + static void createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node, int32_t mode = zkutil::CreateMode::Persistent, bool replace_existing_lock = false); bool removeDetachedPart(DiskPtr disk, const String & path, const String & part_name, bool is_freezed) override; @@ -771,9 +774,14 @@ private: // Create table id if needed void createTableSharedID(); + + bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk); + + std::optional getZeroCopyPartPath(const String & part_name, const DiskPtr & disk); + /// Create ephemeral lock in zookeeper for part and disk which support zero copy replication. /// If somebody already holding the lock -- return std::nullopt. - std::optional tryCreateZeroCopyExclusiveLock(const DataPartPtr & part, const DiskPtr & disk) override; + std::optional tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) override; protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. @@ -830,6 +838,7 @@ String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, const * PS. Perhaps it would be better to add a flag to the DataPart that a part is inserted into ZK. * But here it's too easy to get confused with the consistency of this flag. */ +/// NOLINTNEXTLINE #define MAX_AGE_OF_LOCAL_PART_THAT_WASNT_ADDED_TO_ZOOKEEPER (5 * 60) } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 9a85644d825..f319bd1097b 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -233,7 +233,7 @@ StorageS3Source::StorageS3Source( const ColumnsDescription & columns_, UInt64 max_block_size_, UInt64 max_single_read_retries_, - const String compression_hint_, + String compression_hint_, const std::shared_ptr & client_, const String & bucket_, std::shared_ptr file_iterator_) @@ -245,7 +245,7 @@ StorageS3Source::StorageS3Source( , columns_desc(columns_) , max_block_size(max_block_size_) , max_single_read_retries(max_single_read_retries_) - , compression_hint(compression_hint_) + , compression_hint(std::move(compression_hint_)) , client(client_) , sample_block(sample_block_) , format_settings(format_settings_) @@ -615,9 +615,14 @@ std::shared_ptr StorageS3::createFileIterator( } } +bool StorageS3::isColumnOriented() const +{ + return FormatFactory::instance().checkIfFormatIsColumnOriented(format_name); +} + Pipe StorageS3::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr local_context, QueryProcessingStage::Enum /*processed_stage*/, @@ -639,6 +644,20 @@ Pipe StorageS3::read( std::shared_ptr iterator_wrapper = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, local_context); + ColumnsDescription columns_description; + Block block_for_format; + if (isColumnOriented()) + { + columns_description = ColumnsDescription{ + storage_snapshot->getSampleBlockForColumns(column_names).getNamesAndTypesList()}; + block_for_format = storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); + } + else + { + columns_description = storage_snapshot->metadata->getColumns(); + block_for_format = storage_snapshot->metadata->getSampleBlock(); + } + for (size_t i = 0; i < num_streams; ++i) { pipes.emplace_back(std::make_shared( @@ -646,10 +665,10 @@ Pipe StorageS3::read( need_file_column, format_name, getName(), - metadata_snapshot->getSampleBlock(), + block_for_format, local_context, format_settings, - metadata_snapshot->getColumns(), + columns_description, max_block_size, max_single_read_retries, compression_method, @@ -888,23 +907,44 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( ContextPtr ctx) { std::vector keys = {client_auth.uri.key}; - auto read_buffer_creator = [&]() + auto file_iterator = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, ctx); + + std::string current_key; + std::string exception_messages; + bool read_buffer_creator_was_used = false; + do { - auto file_iterator = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, ctx); - String current_key = (*file_iterator)(); - if (current_key.empty()) - throw Exception( - ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, - "Cannot extract table structure from {} format file, because there are no files with provided path in S3. You must specify " - "table structure manually", - format); + current_key = (*file_iterator)(); + auto read_buffer_creator = [&]() + { + read_buffer_creator_was_used = true; + if (current_key.empty()) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot extract table structure from {} format file, because there are no files with provided path in S3. You must specify " + "table structure manually", + format); - return wrapReadBufferWithCompressionMethod( - std::make_unique(client_auth.client, client_auth.uri.bucket, current_key, max_single_read_retries, ctx->getReadSettings()), - chooseCompressionMethod(current_key, compression_method)); - }; + return wrapReadBufferWithCompressionMethod( + std::make_unique( + client_auth.client, client_auth.uri.bucket, current_key, max_single_read_retries, ctx->getReadSettings()), + chooseCompressionMethod(current_key, compression_method)); + }; - return readSchemaFromFormat(format, format_settings, read_buffer_creator, ctx); + try + { + return readSchemaFromFormat(format, format_settings, read_buffer_creator, ctx); + } + catch (...) + { + if (!is_key_with_globs || !read_buffer_creator_was_used) + throw; + + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } + } while (!current_key.empty()); + + throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "All attempts to extract table structure from s3 files failed. Errors:\n{}", exception_messages); } diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 03b54706b4a..300b7becb93 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -71,7 +71,7 @@ public: const ColumnsDescription & columns_, UInt64 max_block_size_, UInt64 max_single_read_retries_, - const String compression_hint_, + String compression_hint_, const std::shared_ptr & client_, const String & bucket, std::shared_ptr file_iterator_); @@ -146,7 +146,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -218,6 +218,8 @@ private: bool is_key_with_globs, const std::optional & format_settings, ContextPtr ctx); + + bool isColumnOriented() const override; }; } diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index 57220c68347..b5549b32554 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -73,7 +73,7 @@ StorageS3Cluster::StorageS3Cluster( /// The code executes on initiator Pipe StorageS3Cluster::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -132,12 +132,12 @@ Pipe StorageS3Cluster::read( } } - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); return Pipe::unitePipes(std::move(pipes)); } QueryProcessingStage::Enum StorageS3Cluster::getQueryProcessingStage( - ContextPtr context, QueryProcessingStage::Enum to_stage, const StorageMetadataPtr &, SelectQueryInfo &) const + ContextPtr context, QueryProcessingStage::Enum to_stage, const StorageSnapshotPtr &, SelectQueryInfo &) const { /// Initiator executes query on remote node. if (context->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY) diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index d1e02c5a730..6d64c56020f 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -25,11 +25,11 @@ class StorageS3Cluster : public shared_ptr_helper, public ISto public: std::string getName() const override { return "S3Cluster"; } - Pipe read(const Names &, const StorageMetadataPtr &, SelectQueryInfo &, + Pipe read(const Names &, const StorageSnapshotPtr &, SelectQueryInfo &, ContextPtr, QueryProcessingStage::Enum, size_t /*max_block_size*/, unsigned /*num_streams*/) override; QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override; + getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageSnapshotPtr &, SelectQueryInfo &) const override; NamesAndTypesList getVirtuals() const override; diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index f93584ab374..bc4e2b1dfe8 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -52,7 +52,7 @@ StorageSQLite::StorageSQLite( Pipe StorageSQLite::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context_, QueryProcessingStage::Enum, @@ -62,11 +62,11 @@ Pipe StorageSQLite::read( if (!sqlite_db) sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true); - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); String query = transformQueryForExternalDatabase( query_info, - metadata_snapshot->getColumns().getOrdinary(), + storage_snapshot->metadata->getColumns().getOrdinary(), IdentifierQuotingStyle::DoubleQuotes, "", remote_table_name, @@ -76,7 +76,7 @@ Pipe StorageSQLite::read( Block sample_block; for (const String & column_name : column_names) { - auto column_data = metadata_snapshot->getColumns().getPhysical(column_name); + auto column_data = storage_snapshot->metadata->getColumns().getPhysical(column_name); sample_block.insert({column_data.type, column_data.name}); } diff --git a/src/Storages/StorageSQLite.h b/src/Storages/StorageSQLite.h index e8fd0771ff4..367e6ee9e80 100644 --- a/src/Storages/StorageSQLite.h +++ b/src/Storages/StorageSQLite.h @@ -36,7 +36,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp new file mode 100644 index 00000000000..e214afc6a90 --- /dev/null +++ b/src/Storages/StorageSnapshot.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NOT_FOUND_COLUMN_IN_BLOCK; + extern const int EMPTY_LIST_OF_COLUMNS_QUERIED; + extern const int NO_SUCH_COLUMN_IN_TABLE; + extern const int COLUMN_QUERIED_MORE_THAN_ONCE; +} + +void StorageSnapshot::init() +{ + for (const auto & [name, type] : storage.getVirtuals()) + virtual_columns[name] = type; +} + +NamesAndTypesList StorageSnapshot::getColumns(const GetColumnsOptions & options) const +{ + auto all_columns = getMetadataForQuery()->getColumns().get(options); + + if (options.with_extended_objects) + extendObjectColumns(all_columns, object_columns, options.with_subcolumns); + + if (options.with_virtuals) + { + /// Virtual columns must be appended after ordinary, + /// because user can override them. + if (!virtual_columns.empty()) + { + NameSet column_names; + for (const auto & column : all_columns) + column_names.insert(column.name); + + for (const auto & [name, type] : virtual_columns) + if (!column_names.count(name)) + all_columns.emplace_back(name, type); + } + } + + return all_columns; +} + +NamesAndTypesList StorageSnapshot::getColumnsByNames(const GetColumnsOptions & options, const Names & names) const +{ + NamesAndTypesList res; + const auto & columns = getMetadataForQuery()->getColumns(); + for (const auto & name : names) + { + auto column = columns.tryGetColumn(options, name); + if (column && !isObject(column->type)) + { + res.emplace_back(std::move(*column)); + continue; + } + + if (options.with_extended_objects) + { + auto object_column = object_columns.tryGetColumn(options, name); + if (object_column) + { + res.emplace_back(std::move(*object_column)); + continue; + } + } + + if (options.with_virtuals) + { + auto it = virtual_columns.find(name); + if (it != virtual_columns.end()) + { + res.emplace_back(name, it->second); + continue; + } + } + + throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "There is no column {} in table", name); + } + + return res; +} + +Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) const +{ + Block res; + + const auto & columns = getMetadataForQuery()->getColumns(); + for (const auto & name : column_names) + { + auto column = columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, name); + auto object_column = object_columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, name); + + if (column && !object_column) + { + res.insert({column->type->createColumn(), column->type, column->name}); + } + else if (object_column) + { + res.insert({object_column->type->createColumn(), object_column->type, object_column->name}); + } + else if (auto it = virtual_columns.find(name); it != virtual_columns.end()) + { + /// Virtual columns must be appended after ordinary, because user can + /// override them. + const auto & type = it->second; + res.insert({type->createColumn(), type, name}); + } + else + { + throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, + "Column {} not found in table {}", backQuote(name), storage.getStorageID().getNameForLogs()); + } + } + + return res; +} + +namespace +{ + using DenseHashSet = google::dense_hash_set; +} + +void StorageSnapshot::check(const Names & column_names) const +{ + const auto & columns = getMetadataForQuery()->getColumns(); + auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(); + + if (column_names.empty()) + { + auto list_of_columns = listOfColumns(columns.get(options)); + throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, + "Empty list of columns queried. There are columns: {}", list_of_columns); + } + + DenseHashSet unique_names; + unique_names.set_empty_key(StringRef()); + + for (const auto & name : column_names) + { + bool has_column = columns.hasColumnOrSubcolumn(GetColumnsOptions::AllPhysical, name) + || object_columns.hasColumnOrSubcolumn(GetColumnsOptions::AllPhysical, name) + || virtual_columns.count(name); + + if (!has_column) + { + auto list_of_columns = listOfColumns(columns.get(options)); + throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, + "There is no column with name {} in table {}. There are columns: {}", + backQuote(name), storage.getStorageID().getNameForLogs(), list_of_columns); + } + + if (unique_names.count(name)) + throw Exception(ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE, "Column {} queried more than once", name); + + unique_names.insert(name); + } +} + +DataTypePtr StorageSnapshot::getConcreteType(const String & column_name) const +{ + auto object_column = object_columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name); + if (object_column) + return object_column->type; + + return metadata->getColumns().get(column_name).type; +} + +} diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h new file mode 100644 index 00000000000..46244827f6c --- /dev/null +++ b/src/Storages/StorageSnapshot.h @@ -0,0 +1,86 @@ +#pragma once +#include + +namespace DB +{ + +class IStorage; + +/// Snapshot of storage that fixes set columns that can be read in query. +/// There are 3 sources of columns: regular columns from metadata, +/// dynamic columns from object Types, virtual columns. +struct StorageSnapshot +{ + const IStorage & storage; + const StorageMetadataPtr metadata; + const ColumnsDescription object_columns; + + /// Additional data, on which set of columns may depend. + /// E.g. data parts in MergeTree, list of blocks in Memory, etc. + struct Data + { + virtual ~Data() = default; + }; + + using DataPtr = std::unique_ptr; + const DataPtr data; + + /// Projection that is used in query. + mutable const ProjectionDescription * projection = nullptr; + + StorageSnapshot( + const IStorage & storage_, + const StorageMetadataPtr & metadata_) + : storage(storage_), metadata(metadata_) + { + init(); + } + + StorageSnapshot( + const IStorage & storage_, + const StorageMetadataPtr & metadata_, + const ColumnsDescription & object_columns_) + : storage(storage_), metadata(metadata_), object_columns(object_columns_) + { + init(); + } + + StorageSnapshot( + const IStorage & storage_, + const StorageMetadataPtr & metadata_, + const ColumnsDescription & object_columns_, + DataPtr data_) + : storage(storage_), metadata(metadata_), object_columns(object_columns_), data(std::move(data_)) + { + init(); + } + + /// Get all available columns with types according to options. + NamesAndTypesList getColumns(const GetColumnsOptions & options) const; + + /// Get columns with types according to options only for requested names. + NamesAndTypesList getColumnsByNames(const GetColumnsOptions & options, const Names & names) const; + + /// Block with ordinary + materialized + aliases + virtuals + subcolumns. + Block getSampleBlockForColumns(const Names & column_names) const; + + /// Verify that all the requested names are in the table and are set correctly: + /// list of names is not empty and the names do not repeat. + void check(const Names & column_names) const; + + DataTypePtr getConcreteType(const String & column_name) const; + + void addProjection(const ProjectionDescription * projection_) const { projection = projection_; } + + /// If we have a projection then we should use its metadata. + StorageMetadataPtr getMetadataForQuery() const { return projection ? projection->metadata : metadata; } + +private: + void init(); + + std::unordered_map virtual_columns; +}; + +using StorageSnapshotPtr = std::shared_ptr; + +} diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index c401d27a8fc..274789f012b 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -35,9 +35,10 @@ #include #include -#include +#include #include #include +#include #include #include @@ -63,14 +64,13 @@ class StripeLogSource final : public SourceWithProgress { public: static Block getHeader( - const StorageStripeLog & storage, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const Names & column_names, IndexForNativeFormat::Blocks::const_iterator index_begin, IndexForNativeFormat::Blocks::const_iterator index_end) { if (index_begin == index_end) - return metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID()); + return storage_snapshot->getSampleBlockForColumns(column_names); /// TODO: check if possible to always return storage.getSampleBlock() @@ -87,16 +87,16 @@ public: StripeLogSource( const StorageStripeLog & storage_, - const StorageMetadataPtr & metadata_snapshot_, + const StorageSnapshotPtr & storage_snapshot_, const Names & column_names, ReadSettings read_settings_, std::shared_ptr indices_, IndexForNativeFormat::Blocks::const_iterator index_begin_, IndexForNativeFormat::Blocks::const_iterator index_end_, size_t file_size_) - : SourceWithProgress(getHeader(storage_, metadata_snapshot_, column_names, index_begin_, index_end_)) + : SourceWithProgress(getHeader(storage_snapshot_, column_names, index_begin_, index_end_)) , storage(storage_) - , metadata_snapshot(metadata_snapshot_) + , storage_snapshot(storage_snapshot_) , read_settings(std::move(read_settings_)) , indices(indices_) , index_begin(index_begin_) @@ -131,7 +131,7 @@ protected: private: const StorageStripeLog & storage; - StorageMetadataPtr metadata_snapshot; + StorageSnapshotPtr storage_snapshot; ReadSettings read_settings; std::shared_ptr indices; @@ -343,14 +343,14 @@ static std::chrono::seconds getLockTimeout(ContextPtr context) Pipe StorageStripeLog::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); auto lock_timeout = getLockTimeout(context); loadIndices(lock_timeout); @@ -361,7 +361,7 @@ Pipe StorageStripeLog::read( size_t data_file_size = file_checker.getFileSize(data_file_path); if (!data_file_size) - return Pipe(std::make_shared(metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()))); + return Pipe(std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names))); auto indices_for_selected_columns = std::make_shared(indices.extractIndexForColumns(NameSet{column_names.begin(), column_names.end()})); @@ -382,7 +382,7 @@ Pipe StorageStripeLog::read( std::advance(end, (stream + 1) * size / num_streams); pipes.emplace_back(std::make_shared( - *this, metadata_snapshot, column_names, read_settings, indices_for_selected_columns, begin, end, data_file_size)); + *this, storage_snapshot, column_names, read_settings, indices_for_selected_columns, begin, end, data_file_size)); } /// We do not keep read lock directly at the time of reading, because we read ranges of data that do not change. @@ -491,7 +491,7 @@ void StorageStripeLog::saveFileSizes(const WriteLock & /* already locked for wri } -BackupEntries StorageStripeLog::backup(const ASTs & partitions, ContextPtr context) +BackupEntries StorageStripeLog::backupData(ContextPtr context, const ASTs & partitions) { if (!partitions.empty()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); @@ -516,24 +516,24 @@ BackupEntries StorageStripeLog::backup(const ASTs & partitions, ContextPtr conte { /// We make a copy of the data file because it can be changed later in write() or in truncate(). String data_file_name = fileName(data_file_path); - String temp_file_path = temp_dir + "/" + data_file_name; - disk->copy(data_file_path, disk, temp_file_path); + String hardlink_file_path = temp_dir + "/" + data_file_name; + disk->createHardLink(data_file_path, hardlink_file_path); backup_entries.emplace_back( data_file_name, - std::make_unique( - disk, temp_file_path, file_checker.getFileSize(data_file_path), std::nullopt, temp_dir_owner)); + std::make_unique( + disk, hardlink_file_path, file_checker.getFileSize(data_file_path), std::nullopt, temp_dir_owner)); } /// index.mrk { /// We make a copy of the data file because it can be changed later in write() or in truncate(). String index_file_name = fileName(index_file_path); - String temp_file_path = temp_dir + "/" + index_file_name; - disk->copy(index_file_path, disk, temp_file_path); + String hardlink_file_path = temp_dir + "/" + index_file_name; + disk->createHardLink(index_file_path, hardlink_file_path); backup_entries.emplace_back( index_file_name, - std::make_unique( - disk, temp_file_path, file_checker.getFileSize(index_file_path), std::nullopt, temp_dir_owner)); + std::make_unique( + disk, hardlink_file_path, file_checker.getFileSize(index_file_path), std::nullopt, temp_dir_owner)); } /// sizes.json @@ -553,37 +553,51 @@ BackupEntries StorageStripeLog::backup(const ASTs & partitions, ContextPtr conte return backup_entries; } -RestoreDataTasks StorageStripeLog::restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context) +class StripeLogRestoreTask : public IRestoreTask { - if (!partitions.empty()) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); + using WriteLock = StorageStripeLog::WriteLock; - auto restore_task = [this, backup, data_path_in_backup, context]() +public: + StripeLogRestoreTask( + const std::shared_ptr storage_, + const BackupPtr & backup_, + const String & data_path_in_backup_, + ContextMutablePtr context_) + : storage(storage_), backup(backup_), data_path_in_backup(data_path_in_backup_), context(context_) { - WriteLock lock{rwlock, getLockTimeout(context)}; + } + + RestoreTasks run() override + { + WriteLock lock{storage->rwlock, getLockTimeout(context)}; if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + auto & file_checker = storage->file_checker; + /// Load the indices if not loaded yet. We have to do that now because we're going to update these indices. - loadIndices(lock); + storage->loadIndices(lock); /// If there were no files, save zero file sizes to be able to rollback in case of error. - saveFileSizes(lock); + storage->saveFileSizes(lock); try { /// Append the data file. - auto old_data_size = file_checker.getFileSize(data_file_path); + auto old_data_size = file_checker.getFileSize(storage->data_file_path); { + const auto & data_file_path = storage->data_file_path; String file_path_in_backup = data_path_in_backup + fileName(data_file_path); auto backup_entry = backup->readFile(file_path_in_backup); + const auto & disk = storage->disk; auto in = backup_entry->getReadBuffer(); - auto out = disk->writeFile(data_file_path, max_compress_block_size, WriteMode::Append); + auto out = disk->writeFile(data_file_path, storage->max_compress_block_size, WriteMode::Append); copyData(*in, *out); } /// Append the index. { + const auto & index_file_path = storage->index_file_path; String index_path_in_backup = data_path_in_backup + fileName(index_file_path); IndexForNativeFormat extra_indices; auto backup_entry = backup->readFile(index_path_in_backup); @@ -598,23 +612,38 @@ RestoreDataTasks StorageStripeLog::restoreFromBackup(const BackupPtr & backup, c column.location.offset_in_compressed_file += old_data_size; } - insertAtEnd(indices.blocks, std::move(extra_indices.blocks)); + insertAtEnd(storage->indices.blocks, std::move(extra_indices.blocks)); } /// Finish writing. - saveIndices(lock); - saveFileSizes(lock); + storage->saveIndices(lock); + storage->saveFileSizes(lock); + return {}; } catch (...) { /// Rollback partial writes. file_checker.repair(); - removeUnsavedIndices(lock); + storage->removeUnsavedIndices(lock); throw; } + } - }; - return {restore_task}; +private: + std::shared_ptr storage; + BackupPtr backup; + String data_path_in_backup; + ContextMutablePtr context; +}; + + +RestoreTaskPtr StorageStripeLog::restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings &) +{ + if (!partitions.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table engine {} doesn't support partitions", getName()); + + return std::make_unique( + typeid_cast>(shared_from_this()), backup, data_path_in_backup, context); } diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h index 579e2f991e7..223b662d13c 100644 --- a/src/Storages/StorageStripeLog.h +++ b/src/Storages/StorageStripeLog.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -23,6 +24,7 @@ class StorageStripeLog final : public shared_ptr_helper, publi { friend class StripeLogSource; friend class StripeLogSink; + friend class StripeLogRestoreTask; friend struct shared_ptr_helper; public: @@ -32,7 +34,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -50,8 +52,9 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder&) override; - BackupEntries backup(const ASTs & partitions, ContextPtr context) override; - RestoreDataTasks restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context) override; + bool hasDataToBackup() const override { return true; } + BackupEntries backupData(ContextPtr context, const ASTs & partitions) override; + RestoreTaskPtr restoreData(ContextMutablePtr context, const ASTs & partitions, const BackupPtr & backup, const String & data_path_in_backup, const StorageRestoreSettings & restore_settings) override; protected: StorageStripeLog( diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 0b7ab30fa24..4616421b24a 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -92,7 +93,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -103,12 +104,12 @@ public: for (const auto & c : column_names) cnames += c + " "; auto storage = getNested(); - auto nested_metadata = storage->getInMemoryMetadataPtr(); - auto pipe = storage->read(column_names, nested_metadata, query_info, context, + auto nested_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr()); + auto pipe = storage->read(column_names, nested_snapshot, query_info, context, processed_stage, max_block_size, num_streams); if (!pipe.empty() && add_conversion) { - auto to_header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, + auto to_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, context, processed_stage); auto convert_actions_dag = ActionsDAG::makeConvertingActions( @@ -148,7 +149,7 @@ public: if (nested) StorageProxy::renameInMemory(new_table_id); else - IStorage::renameInMemory(new_table_id); + IStorage::renameInMemory(new_table_id); /// NOLINT } bool isView() const override { return false; } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 36b9853ac0e..5c8a7ea2be5 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -3,30 +3,35 @@ #include #include #include -#include #include +#include +#include +#include +#include +#include #include #include #include -#include -#include #include #include #include #include -#include #include #include +#include "Common/ThreadStatus.h" +#include +#include "IO/HTTPCommon.h" +#include "IO/ReadWriteBufferFromHTTP.h" -#include +#include +#include #include #include -#include #include -#include +#include namespace DB @@ -37,13 +42,13 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; } static bool urlWithGlobs(const String & uri) { - return (uri.find('{') != std::string::npos && uri.find('}') != std::string::npos) - || uri.find('|') != std::string::npos; + return (uri.find('{') != std::string::npos && uri.find('}') != std::string::npos) || uri.find('|') != std::string::npos; } @@ -87,8 +92,7 @@ IStorageURLBase::IStorageURLBase( namespace { - ReadWriteBufferFromHTTP::HTTPHeaderEntries getHeaders( - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_) + ReadWriteBufferFromHTTP::HTTPHeaderEntries getHeaders(const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_) { ReadWriteBufferFromHTTP::HTTPHeaderEntries headers(headers_.begin(), headers_.end()); // Propagate OpenTelemetry trace context, if any, downstream. @@ -97,13 +101,11 @@ namespace const auto & thread_trace_context = CurrentThread::get().thread_trace_context; if (thread_trace_context.trace_id != UUID()) { - headers.emplace_back("traceparent", - thread_trace_context.composeTraceparentHeader()); + headers.emplace_back("traceparent", thread_trace_context.composeTraceparentHeader()); if (!thread_trace_context.tracestate.empty()) { - headers.emplace_back("tracestate", - thread_trace_context.tracestate); + headers.emplace_back("tracestate", thread_trace_context.tracestate); } } } @@ -113,8 +115,7 @@ namespace class StorageURLSource : public SourceWithProgress { - - using URIParams = std::vector>; + using URIParams = std::vector>; public: struct URIInfo @@ -159,11 +160,11 @@ namespace UInt64 max_block_size, const ConnectionTimeouts & timeouts, const String & compression_method, + size_t download_threads, const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_ = {}, const URIParams & params = {}, bool glob_url = false) - : SourceWithProgress(sample_block), name(std::move(name_)) - , uri_info(uri_info_) + : SourceWithProgress(sample_block), name(std::move(name_)), uri_info(uri_info_) { auto headers = getHeaders(headers_); @@ -173,66 +174,42 @@ namespace if (uri_options.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty url list"); - if (uri_options.size() > 1) - { - read_buf = getFirstAvailableURLReadBuffer( - uri_options, context, params, http_method, - callback, timeouts, compression_method, credentials, headers); - } - else - { - ReadSettings read_settings = context->getReadSettings(); - bool skip_url_not_found_error = glob_url && read_settings.http_skip_not_found_url_for_globs; - auto request_uri = Poco::URI(uri_options[0]); + auto first_option = uri_options.begin(); + read_buf = getFirstAvailableURLReadBuffer( + first_option, + uri_options.end(), + context, + params, + http_method, + callback, + timeouts, + compression_method, + credentials, + headers, + glob_url, + uri_options.size() == 1, + download_threads); - for (const auto & [param, value] : params) - request_uri.addQueryParameter(param, value); - - setCredentials(credentials, request_uri); - - read_buf = wrapReadBufferWithCompressionMethod( - std::make_unique( - request_uri, - http_method, - callback, - timeouts, - credentials, - context->getSettingsRef().max_http_get_redirects, - DBMS_DEFAULT_BUFFER_SIZE, - read_settings, - headers, - ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter(), - /* delay_initiliazation */true, - /* use_external_buffer */false, - /* skip_url_not_found_error */skip_url_not_found_error), - chooseCompressionMethod(request_uri.getPath(), compression_method)); - } - - auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); + auto input_format + = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); QueryPipelineBuilder builder; builder.init(Pipe(input_format)); - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared(cur_header, columns, *input_format, context); - }); + builder.addSimpleTransform( + [&](const Block & cur_header) + { return std::make_shared(cur_header, columns, *input_format, context); }); pipeline = std::make_unique(QueryPipelineBuilder::getPipeline(std::move(builder))); reader = std::make_unique(*pipeline); }; } - String getName() const override - { - return name; - } + String getName() const override { return name; } Chunk generate() override { while (true) { - if (!reader) { auto current_uri_pos = uri_info->next_uri_to_read.fetch_add(1); @@ -258,7 +235,8 @@ namespace } static std::unique_ptr getFirstAvailableURLReadBuffer( - const std::vector & urls, + std::vector::const_iterator & option, + const std::vector::const_iterator & end, ContextPtr context, const URIParams & params, const String & http_method, @@ -266,14 +244,18 @@ namespace const ConnectionTimeouts & timeouts, const String & compression_method, Poco::Net::HTTPBasicCredentials & credentials, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers) + const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + bool glob_url, + bool delay_initialization, + size_t download_threads) { String first_exception_message; ReadSettings read_settings = context->getReadSettings(); - for (auto option = urls.begin(); option != urls.end(); ++option) + size_t options = std::distance(option, end); + for (; option != end; ++option) { - bool skip_url_not_found_error = read_settings.http_skip_not_found_url_for_globs && option == std::prev(urls.end()); + bool skip_url_not_found_error = glob_url && read_settings.http_skip_not_found_url_for_globs && option == std::prev(end); auto request_uri = Poco::URI(*option); for (const auto & [param, value] : params) @@ -281,8 +263,137 @@ namespace setCredentials(credentials, request_uri); + const auto settings = context->getSettings(); try { + if (download_threads > 1) + { + try + { + ReadWriteBufferFromHTTP buffer( + request_uri, + Poco::Net::HTTPRequest::HTTP_HEAD, + callback, + timeouts, + credentials, + settings.max_http_get_redirects, + DBMS_DEFAULT_BUFFER_SIZE, + read_settings, + headers, + ReadWriteBufferFromHTTP::Range{0, std::nullopt}, + &context->getRemoteHostFilter(), + true, + /* use_external_buffer */ false, + /* skip_url_not_found_error */ skip_url_not_found_error); + + Poco::Net::HTTPResponse res; + + for (size_t i = 0; i < settings.http_max_tries; ++i) + { + try + { + buffer.callWithRedirects(res, Poco::Net::HTTPRequest::HTTP_HEAD, true); + break; + } + catch (const Poco::Exception & e) + { + LOG_TRACE( + &Poco::Logger::get("StorageURLSource"), + "HTTP HEAD request to `{}` failed at try {}/{}. " + "Error: {}.", + request_uri.toString(), + i + 1, + settings.http_max_tries, + e.displayText()); + if (!ReadWriteBufferFromHTTP::isRetriableError(res.getStatus())) + { + throw; + } + } + } + + // to check if Range header is supported, we need to send a request with it set + const bool supports_ranges = (res.has("Accept-Ranges") && res.get("Accept-Ranges") == "bytes") + || (res.has("Content-Range") && res.get("Content-Range").starts_with("bytes")); + LOG_TRACE( + &Poco::Logger::get("StorageURLSource"), + fmt::runtime(supports_ranges ? "HTTP Range is supported" : "HTTP Range is not supported")); + + + if (supports_ranges && res.getStatus() == Poco::Net::HTTPResponse::HTTP_PARTIAL_CONTENT + && res.hasContentLength()) + { + LOG_TRACE( + &Poco::Logger::get("StorageURLSource"), + "Using ParallelReadBuffer with {} workers with chunks of {} bytes", + download_threads, + settings.max_download_buffer_size); + + auto read_buffer_factory = std::make_unique( + res.getContentLength(), + settings.max_download_buffer_size, + request_uri, + http_method, + callback, + timeouts, + credentials, + settings.max_http_get_redirects, + DBMS_DEFAULT_BUFFER_SIZE, + read_settings, + headers, + &context->getRemoteHostFilter(), + delay_initialization, + /* use_external_buffer */ false, + /* skip_url_not_found_error */ skip_url_not_found_error); + + ThreadGroupStatusPtr running_group = CurrentThread::isInitialized() && CurrentThread::get().getThreadGroup() + ? CurrentThread::get().getThreadGroup() + : MainThreadStatus::getInstance().getThreadGroup(); + + ContextPtr query_context + = CurrentThread::isInitialized() ? CurrentThread::get().getQueryContext() : nullptr; + + auto worker_cleanup = [has_running_group = running_group == nullptr](ThreadStatus & thread_status) + { + if (has_running_group) + thread_status.detachQuery(false); + }; + + auto worker_setup = [query_context = std::move(query_context), + running_group = std::move(running_group)](ThreadStatus & thread_status) + { + /// Save query context if any, because cache implementation needs it. + if (query_context) + thread_status.attachQueryContext(query_context); + + /// To be able to pass ProfileEvents. + if (running_group) + thread_status.attachQuery(running_group); + }; + + + return wrapReadBufferWithCompressionMethod( + std::make_unique( + std::move(read_buffer_factory), + &IOThreadPool::get(), + download_threads, + std::move(worker_setup), + std::move(worker_cleanup)), + chooseCompressionMethod(request_uri.getPath(), compression_method)); + } + } + catch (const Poco::Exception & e) + { + LOG_TRACE( + &Poco::Logger::get("StorageURLSource"), + "Failed to setup ParallelReadBuffer because of an exception:\n{}.\nFalling back to the single-threaded " + "buffer", + e.displayText()); + } + } + + LOG_TRACE(&Poco::Logger::get("StorageURLSource"), "Using single-threaded read buffer"); + return wrapReadBufferWithCompressionMethod( std::make_unique( request_uri, @@ -290,15 +401,15 @@ namespace callback, timeouts, credentials, - context->getSettingsRef().max_http_get_redirects, + settings.max_http_get_redirects, DBMS_DEFAULT_BUFFER_SIZE, read_settings, headers, ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter(), - /* delay_initiliazation */false, - /* use_external_buffer */false, - /* skip_url_not_found_error */skip_url_not_found_error), + &context->getRemoteHostFilter(), + delay_initialization, + /* use_external_buffer */ false, + /* skip_url_not_found_error */ skip_url_not_found_error), chooseCompressionMethod(request_uri.getPath(), compression_method)); } catch (...) @@ -306,14 +417,14 @@ namespace if (first_exception_message.empty()) first_exception_message = getCurrentExceptionMessage(false); - if (urls.size() == 1) + if (options == 1) throw; tryLogCurrentException(__PRETTY_FUNCTION__); } } - throw Exception(ErrorCodes::NETWORK_ERROR, "All uri ({}) options are unreachable: {}", urls.size(), first_exception_message); + throw Exception(ErrorCodes::NETWORK_ERROR, "All uri ({}) options are unreachable: {}", options, first_exception_message); } private: @@ -346,12 +457,13 @@ StorageURLSink::StorageURLSink( : SinkToStorage(sample_block) { std::string content_type = FormatFactory::instance().getContentType(format, context, format_settings); + std::string content_encoding = toContentEncodingName(compression_method); write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(Poco::URI(uri), http_method, content_type, timeouts), - compression_method, 3); - writer = FormatFactory::instance().getOutputFormat(format, *write_buf, sample_block, - context, {} /* write callback */, format_settings); + std::make_unique(Poco::URI(uri), http_method, content_type, content_encoding, timeouts), + compression_method, + 3); + writer = FormatFactory::instance().getOutputFormat(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); } @@ -380,15 +492,15 @@ public: const ConnectionTimeouts & timeouts_, const CompressionMethod compression_method_, const String & http_method_) - : PartitionedSink(partition_by, context_, sample_block_) - , uri(uri_) - , format(format_) - , format_settings(format_settings_) - , sample_block(sample_block_) - , context(context_) - , timeouts(timeouts_) - , compression_method(compression_method_) - , http_method(http_method_) + : PartitionedSink(partition_by, context_, sample_block_) + , uri(uri_) + , format(format_) + , format_settings(format_settings_) + , sample_block(sample_block_) + , context(context_) + , timeouts(timeouts_) + , compression_method(compression_method_) + , http_method(http_method_) { } @@ -396,8 +508,8 @@ public: { auto partition_path = PartitionedSink::replaceWildcards(uri, partition_id); context->getRemoteHostFilter().checkURL(Poco::URI(partition_path)); - return std::make_shared(partition_path, format, - format_settings, sample_block, context, timeouts, compression_method, http_method); + return std::make_shared( + partition_path, format, format_settings, sample_block, context, timeouts, compression_method, http_method); } private: @@ -419,7 +531,7 @@ std::string IStorageURLBase::getReadMethod() const std::vector> IStorageURLBase::getReadURIParams( const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & /*storage_snapshot*/, const SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum & /*processed_stage*/, @@ -430,7 +542,7 @@ std::vector> IStorageURLBase::getReadURIPara std::function IStorageURLBase::getReadPOSTDataCallback( const Names & /*column_names*/, - const StorageMetadataPtr & /*metadata_snapshot*/, + const ColumnsDescription & /* columns_description */, const SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum & /*processed_stage*/, @@ -448,13 +560,11 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( const std::optional & format_settings, ContextPtr context) { - ReadBufferCreator read_buffer_creator; Poco::Net::HTTPBasicCredentials credentials; + std::vector urls_to_check; if (urlWithGlobs(uri)) { - std::vector urls_to_check; - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; auto uri_descriptions = parseRemoteDescription(uri, 0, uri.size(), ',', max_addresses); for (const auto & description : uri_descriptions) @@ -462,11 +572,24 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( auto options = parseRemoteDescription(description, 0, description.size(), '|', max_addresses); urls_to_check.insert(urls_to_check.end(), options.begin(), options.end()); } + } + else + { + urls_to_check = {uri}; + } - read_buffer_creator = [&, urls_to_check]() + String exception_messages; + bool read_buffer_creator_was_used = false; + + std::vector::const_iterator option = urls_to_check.begin(); + do + { + auto read_buffer_creator = [&]() { + read_buffer_creator_was_used = true; return StorageURLSource::getFirstAvailableURLReadBuffer( - urls_to_check, + option, + urls_to_check.end(), context, {}, Poco::Net::HTTPRequest::HTTP_GET, @@ -474,47 +597,63 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( ConnectionTimeouts::getHTTPTimeouts(context), compression_method, credentials, - headers); + headers, + false, + false, + context->getSettingsRef().max_download_threads); }; - } - else - { - read_buffer_creator = [&]() + + try { - auto parsed_uri = Poco::URI(uri); - StorageURLSource::setCredentials(credentials, parsed_uri); + return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + } + catch (...) + { + if (urls_to_check.size() == 1 || !read_buffer_creator_was_used) + throw; - return wrapReadBufferWithCompressionMethod( - std::make_unique( - parsed_uri, - Poco::Net::HTTPRequest::HTTP_GET, - nullptr, - ConnectionTimeouts::getHTTPTimeouts(context), - credentials, - context->getSettingsRef().max_http_get_redirects, - DBMS_DEFAULT_BUFFER_SIZE, - context->getReadSettings(), - headers, - ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter(), - /* delay_initiliazation */true), - chooseCompressionMethod(parsed_uri.getPath(), compression_method)); - }; - } + exception_messages += getCurrentExceptionMessage(false) + "\n"; + } - return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); + } while (++option < urls_to_check.end()); + + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "All attempts to extract table structure from urls failed. Errors:\n{}", + exception_messages); +} + +bool IStorageURLBase::isColumnOriented() const +{ + return FormatFactory::instance().checkIfFormatIsColumnOriented(format_name); } Pipe IStorageURLBase::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { - auto params = getReadURIParams(column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size); + auto params = getReadURIParams(column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size); + + ColumnsDescription columns_description; + Block block_for_format; + if (isColumnOriented()) + { + columns_description = ColumnsDescription{ + storage_snapshot->getSampleBlockForColumns(column_names).getNamesAndTypesList()}; + block_for_format = storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); + } + else + { + columns_description = storage_snapshot->metadata->getColumns(); + block_for_format = storage_snapshot->metadata->getSampleBlock(); + } + + size_t max_download_threads = local_context->getSettingsRef().max_download_threads; if (urlWithGlobs(uri)) { @@ -532,23 +671,26 @@ Pipe IStorageURLBase::read( Pipes pipes; pipes.reserve(num_streams); + size_t download_threads = num_streams >= max_download_threads ? 1 : (max_download_threads / num_streams); for (size_t i = 0; i < num_streams; ++i) { pipes.emplace_back(std::make_shared( uri_info, getReadMethod(), - getReadPOSTDataCallback( - column_names, metadata_snapshot, query_info, - local_context, processed_stage, max_block_size), + getReadPOSTDataCallback(column_names, columns_description, query_info, local_context, processed_stage, max_block_size), format_name, format_settings, getName(), - getHeaderBlock(column_names, metadata_snapshot), + block_for_format, local_context, - metadata_snapshot->getColumns(), + columns_description, max_block_size, ConnectionTimeouts::getHTTPTimeouts(local_context), - compression_method, headers, params, /* glob_url */true)); + compression_method, + download_threads, + headers, + params, + /* glob_url */ true)); } return Pipe::unitePipes(std::move(pipes)); } @@ -559,50 +701,66 @@ Pipe IStorageURLBase::read( return Pipe(std::make_shared( uri_info, getReadMethod(), - getReadPOSTDataCallback( - column_names, metadata_snapshot, query_info, - local_context, processed_stage, max_block_size), + getReadPOSTDataCallback(column_names, columns_description, query_info, local_context, processed_stage, max_block_size), format_name, format_settings, getName(), - getHeaderBlock(column_names, metadata_snapshot), + block_for_format, local_context, - metadata_snapshot->getColumns(), + columns_description, max_block_size, ConnectionTimeouts::getHTTPTimeouts(local_context), - compression_method, headers, params)); + compression_method, + max_download_threads, + headers, + params)); } } Pipe StorageURLWithFailover::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned /*num_streams*/) { - auto params = getReadURIParams(column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size); + ColumnsDescription columns_description; + Block block_for_format; + if (isColumnOriented()) + { + columns_description = ColumnsDescription{ + storage_snapshot->getSampleBlockForColumns(column_names).getNamesAndTypesList()}; + block_for_format = storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical()); + } + else + { + columns_description = storage_snapshot->metadata->getColumns(); + block_for_format = storage_snapshot->metadata->getSampleBlock(); + } + + auto params = getReadURIParams(column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size); auto uri_info = std::make_shared(); uri_info->uri_list_to_read.emplace_back(uri_options); - auto pipe = Pipe(std::make_shared( + auto pipe = Pipe(std::make_shared( uri_info, getReadMethod(), - getReadPOSTDataCallback( - column_names, metadata_snapshot, query_info, - local_context, processed_stage, max_block_size), + getReadPOSTDataCallback(column_names, columns_description, query_info, local_context, processed_stage, max_block_size), format_name, format_settings, getName(), - getHeaderBlock(column_names, metadata_snapshot), + block_for_format, local_context, - metadata_snapshot->getColumns(), + columns_description, max_block_size, ConnectionTimeouts::getHTTPTimeouts(local_context), - compression_method, headers, params)); + compression_method, + local_context->getSettingsRef().max_download_threads, + headers, + params)); std::shuffle(uri_options.begin(), uri_options.end(), thread_local_rng); return pipe; } @@ -622,17 +780,26 @@ SinkToStoragePtr IStorageURLBase::write(const ASTPtr & query, const StorageMetad { return std::make_shared( partition_by_ast, - uri, format_name, - format_settings, metadata_snapshot->getSampleBlock(), context, + uri, + format_name, + format_settings, + metadata_snapshot->getSampleBlock(), + context, ConnectionTimeouts::getHTTPTimeouts(context), - chooseCompressionMethod(uri, compression_method), http_method); + chooseCompressionMethod(uri, compression_method), + http_method); } else { - return std::make_shared(uri, format_name, - format_settings, metadata_snapshot->getSampleBlock(), context, + return std::make_shared( + uri, + format_name, + format_settings, + metadata_snapshot->getSampleBlock(), + context, ConnectionTimeouts::getHTTPTimeouts(context), - chooseCompressionMethod(uri, compression_method), http_method); + chooseCompressionMethod(uri, compression_method), + http_method); } } @@ -649,8 +816,19 @@ StorageURL::StorageURL( const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_, const String & http_method_, ASTPtr partition_by_) - : IStorageURLBase(uri_, context_, table_id_, format_name_, format_settings_, - columns_, constraints_, comment, compression_method_, headers_, http_method_, partition_by_) + : IStorageURLBase( + uri_, + context_, + table_id_, + format_name_, + format_settings_, + columns_, + constraints_, + comment, + compression_method_, + headers_, + http_method_, + partition_by_) { context_->getRemoteHostFilter().checkURL(Poco::URI(uri)); } @@ -672,7 +850,7 @@ StorageURLWithFailover::StorageURLWithFailover( Poco::URI poco_uri(uri_option); context_->getRemoteHostFilter().checkURL(poco_uri); LOG_DEBUG(&Poco::Logger::get("StorageURLDistributed"), "Adding URL option: {}", uri_option); - uri_options.emplace_back(std::move(uri_option)); + uri_options.emplace_back(uri_option); } } @@ -701,8 +879,7 @@ FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Argum // Apply changes from SETTINGS clause, with validation. user_format_settings.applyChanges(args.storage_def->settings->changes); - format_settings = getFormatSettings(args.getContext(), - user_format_settings); + format_settings = getFormatSettings(args.getContext(), user_format_settings); } else { @@ -721,12 +898,12 @@ URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, Contex auto [common_configuration, storage_specific_args] = named_collection.value(); configuration.set(common_configuration); - if (!configuration.http_method.empty() - && configuration.http_method != Poco::Net::HTTPRequest::HTTP_POST + if (!configuration.http_method.empty() && configuration.http_method != Poco::Net::HTTPRequest::HTTP_POST && configuration.http_method != Poco::Net::HTTPRequest::HTTP_PUT) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Http method can be POST or PUT (current: {}). For insert default is POST, for select GET", - configuration.http_method); + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Http method can be POST or PUT (current: {}). For insert default is POST, for select GET", + configuration.http_method); if (!storage_specific_args.empty()) { @@ -744,7 +921,8 @@ URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, Contex { if (args.empty() || args.size() > 3) throw Exception( - "Storage URL requires 1, 2 or 3 arguments: url, name of used format (taken from file extension by default) and optional compression method.", + "Storage URL requires 1, 2 or 3 arguments: url, name of used format (taken from file extension by default) and optional " + "compression method.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & arg : args) @@ -766,43 +944,45 @@ URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, Contex void registerStorageURL(StorageFactory & factory) { - factory.registerStorage("URL", [](const StorageFactory::Arguments & args) - { - ASTs & engine_args = args.engine_args; - auto configuration = StorageURL::getConfiguration(engine_args, args.getLocalContext()); - auto format_settings = StorageURL::getFormatSettingsFromArgs(args); - - ReadWriteBufferFromHTTP::HTTPHeaderEntries headers; - for (const auto & [header, value] : configuration.headers) + factory.registerStorage( + "URL", + [](const StorageFactory::Arguments & args) { - auto value_literal = value.safeGet(); - if (header == "Range") - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Range headers are not allowed"); - headers.emplace_back(std::make_pair(header, value_literal)); - } + ASTs & engine_args = args.engine_args; + auto configuration = StorageURL::getConfiguration(engine_args, args.getLocalContext()); + auto format_settings = StorageURL::getFormatSettingsFromArgs(args); - ASTPtr partition_by; - if (args.storage_def->partition_by) - partition_by = args.storage_def->partition_by->clone(); + ReadWriteBufferFromHTTP::HTTPHeaderEntries headers; + for (const auto & [header, value] : configuration.headers) + { + auto value_literal = value.safeGet(); + if (header == "Range") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Range headers are not allowed"); + headers.emplace_back(std::make_pair(header, value_literal)); + } - return StorageURL::create( - configuration.url, - args.table_id, - configuration.format, - format_settings, - args.columns, - args.constraints, - args.comment, - args.getContext(), - configuration.compression_method, - headers, - configuration.http_method, - partition_by); - }, - { - .supports_settings = true, - .supports_schema_inference = true, - .source_access_type = AccessType::URL, - }); + ASTPtr partition_by; + if (args.storage_def->partition_by) + partition_by = args.storage_def->partition_by->clone(); + + return StorageURL::create( + configuration.url, + args.table_id, + configuration.format, + format_settings, + args.columns, + args.constraints, + args.comment, + args.getContext(), + configuration.compression_method, + headers, + configuration.http_method, + partition_by); + }, + { + .supports_settings = true, + .supports_schema_inference = true, + .source_access_type = AccessType::URL, + }); } } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 790f01135d3..a035b1bb93d 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -30,7 +30,7 @@ class IStorageURLBase : public IStorage public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -80,7 +80,7 @@ protected: virtual std::vector> getReadURIParams( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum & processed_stage, @@ -88,14 +88,16 @@ protected: virtual std::function getReadPOSTDataCallback( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const ColumnsDescription & columns_description, const SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size) const; + bool isColumnOriented() const override; + private: - virtual Block getHeaderBlock(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const = 0; + virtual Block getHeaderBlock(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) const = 0; }; class StorageURLSink : public SinkToStorage @@ -143,9 +145,9 @@ public: return "URL"; } - Block getHeaderBlock(const Names & /*column_names*/, const StorageMetadataPtr & metadata_snapshot) const override + Block getHeaderBlock(const Names & /*column_names*/, const StorageSnapshotPtr & storage_snapshot) const override { - return metadata_snapshot->getSampleBlock(); + return storage_snapshot->metadata->getSampleBlock(); } static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); @@ -170,7 +172,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageValues.cpp b/src/Storages/StorageValues.cpp index 650782afbba..2a3e1743983 100644 --- a/src/Storages/StorageValues.cpp +++ b/src/Storages/StorageValues.cpp @@ -22,14 +22,14 @@ StorageValues::StorageValues( Pipe StorageValues::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, size_t /*max_block_size*/, unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); /// Get only required columns. Block block; diff --git a/src/Storages/StorageValues.h b/src/Storages/StorageValues.h index 69b2f757046..21156ec27cc 100644 --- a/src/Storages/StorageValues.h +++ b/src/Storages/StorageValues.h @@ -17,7 +17,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index bcf7d7856cf..68b16de5a80 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -107,7 +107,7 @@ StorageView::StorageView( Pipe StorageView::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -115,7 +115,7 @@ Pipe StorageView::read( const unsigned num_streams) { QueryPlan plan; - read(plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + read(plan, column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams); return plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); @@ -124,14 +124,14 @@ Pipe StorageView::read( void StorageView::read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - ASTPtr current_inner_query = metadata_snapshot->getSelectQuery().inner_query; + ASTPtr current_inner_query = storage_snapshot->metadata->getSelectQuery().inner_query; if (query_info.view_query) { @@ -154,7 +154,7 @@ void StorageView::read( query_plan.addStep(std::move(materializing)); /// And also convert to expected structure. - const auto & expected_header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + const auto & expected_header = storage_snapshot->getSampleBlockForColumns(column_names); const auto & header = query_plan.getCurrentDataStream().header; const auto * select_with_union = current_inner_query->as(); diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index 5ca23434356..f49736afe4a 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -23,7 +23,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -33,14 +33,14 @@ public: void read( QueryPlan & query_plan, const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; - void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name, const StorageMetadataPtr & metadata_snapshot) const + static void replaceWithSubquery(ASTSelectQuery & select_query, ASTPtr & view_name, const StorageMetadataPtr & metadata_snapshot) { replaceWithSubquery(select_query, metadata_snapshot->getSelectQuery().inner_query->clone(), view_name); } diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index 90ac04ed250..d9a2a77515e 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -57,7 +57,7 @@ std::string StorageXDBC::getReadMethod() const std::vector> StorageXDBC::getReadURIParams( const Names & /* column_names */, - const StorageMetadataPtr & /* metadata_snapshot */, + const StorageSnapshotPtr & /*storage_snapshot*/, const SelectQueryInfo & /*query_info*/, ContextPtr /*context*/, QueryProcessingStage::Enum & /*processed_stage*/, @@ -68,14 +68,14 @@ std::vector> StorageXDBC::getReadURIParams( std::function StorageXDBC::getReadPOSTDataCallback( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const ColumnsDescription & columns_description, const SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum & /*processed_stage*/, size_t /*max_block_size*/) const { String query = transformQueryForExternalDatabase(query_info, - metadata_snapshot->getColumns().getOrdinary(), + columns_description.getOrdinary(), bridge_helper->getIdentifierQuotingStyle(), remote_database_name, remote_table_name, @@ -85,7 +85,7 @@ std::function StorageXDBC::getReadPOSTDataCallback( NamesAndTypesList cols; for (const String & name : column_names) { - auto column_data = metadata_snapshot->getColumns().getPhysical(name); + auto column_data = columns_description.getPhysical(name); cols.emplace_back(column_data.name, column_data.type); } @@ -101,20 +101,20 @@ std::function StorageXDBC::getReadPOSTDataCallback( Pipe StorageXDBC::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); bridge_helper->startBridgeSync(); - return IStorageURLBase::read(column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); + return IStorageURLBase::read(column_names, storage_snapshot, query_info, local_context, processed_stage, max_block_size, num_streams); } -SinkToStoragePtr StorageXDBC::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context) +SinkToStoragePtr StorageXDBC::write(const ASTPtr & /* query */, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context) { bridge_helper->startBridgeSync(); @@ -140,9 +140,14 @@ SinkToStoragePtr StorageXDBC::write(const ASTPtr & /*query*/, const StorageMetad chooseCompressionMethod(uri, compression_method)); } -Block StorageXDBC::getHeaderBlock(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const +bool StorageXDBC::isColumnOriented() const { - return metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + return true; +} + +Block StorageXDBC::getHeaderBlock(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) const +{ + return storage_snapshot->getSampleBlockForColumns(column_names); } std::string StorageXDBC::getName() const diff --git a/src/Storages/StorageXDBC.h b/src/Storages/StorageXDBC.h index 4438e1c4737..514171026fc 100644 --- a/src/Storages/StorageXDBC.h +++ b/src/Storages/StorageXDBC.h @@ -21,7 +21,7 @@ class StorageXDBC : public IStorageURLBase public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -51,7 +51,7 @@ private: std::vector> getReadURIParams( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum & processed_stage, @@ -59,13 +59,15 @@ private: std::function getReadPOSTDataCallback( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const ColumnsDescription & columns_description, const SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum & processed_stage, size_t max_block_size) const override; - Block getHeaderBlock(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const override; + Block getHeaderBlock(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) const override; + + bool isColumnOriented() const override; }; } diff --git a/src/Storages/System/CMakeLists.txt b/src/Storages/System/CMakeLists.txt index 133761cbe22..efc4c0ed37b 100644 --- a/src/Storages/System/CMakeLists.txt +++ b/src/Storages/System/CMakeLists.txt @@ -39,7 +39,13 @@ if(Git_FOUND) ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) endif() -configure_file (StorageSystemBuildOptions.generated.cpp.in ${CONFIG_BUILD}) +function(generate_system_build_options) + include(${ClickHouse_SOURCE_DIR}/src/configure_config.cmake) + include(${ClickHouse_SOURCE_DIR}/src/Functions/configure_config.cmake) + include(${ClickHouse_SOURCE_DIR}/src/Formats/configure_config.cmake) + configure_file(StorageSystemBuildOptions.generated.cpp.in ${CONFIG_BUILD}) +endfunction() +generate_system_build_options() include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(storages_system .) diff --git a/src/Storages/System/IStorageSystemOneBlock.h b/src/Storages/System/IStorageSystemOneBlock.h index 33086498730..7cc94a3f2f6 100644 --- a/src/Storages/System/IStorageSystemOneBlock.h +++ b/src/Storages/System/IStorageSystemOneBlock.h @@ -32,26 +32,25 @@ protected: virtual void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const = 0; public: - IStorageSystemOneBlock(const StorageID & table_id_) : IStorage(table_id_) + explicit IStorageSystemOneBlock(const StorageID & table_id_) : IStorage(table_id_) { - StorageInMemoryMetadata metadata_; - metadata_.setColumns(ColumnsDescription(Self::getNamesAndTypes(), Self::getNamesAndAliases())); - setInMemoryMetadata(metadata_); + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(ColumnsDescription(Self::getNamesAndTypes(), Self::getNamesAndAliases())); + setInMemoryMetadata(storage_metadata); } Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, size_t /*max_block_size*/, unsigned /*num_streams*/) override { - auto virtuals_names_and_types = getVirtuals(); - metadata_snapshot->check(column_names, virtuals_names_and_types, getStorageID()); + storage_snapshot->check(column_names); - Block sample_block = metadata_snapshot->getSampleBlockWithVirtuals(virtuals_names_and_types); + Block sample_block = storage_snapshot->metadata->getSampleBlockWithVirtuals(getVirtuals()); MutableColumns res_columns = sample_block.cloneEmptyColumns(); fillData(res_columns, context, query_info); diff --git a/src/Storages/System/StorageSystemAsynchronousInserts.cpp b/src/Storages/System/StorageSystemAsynchronousInserts.cpp index 0fa6c1b653c..80fc070c83a 100644 --- a/src/Storages/System/StorageSystemAsynchronousInserts.cpp +++ b/src/Storages/System/StorageSystemAsynchronousInserts.cpp @@ -41,10 +41,10 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co if (!insert_queue) return; - auto queue = insert_queue->getQueue(); + auto [queue, queue_lock] = insert_queue->getQueueLocked(); for (const auto & [key, elem] : queue) { - std::lock_guard lock(elem->mutex); + std::lock_guard elem_lock(elem->mutex); if (!elem->data) continue; @@ -62,8 +62,19 @@ void StorageSystemAsynchronousInserts::fillData(MutableColumns & res_columns, Co size_t i = 0; res_columns[i++]->insert(queryToString(insert_query)); - res_columns[i++]->insert(insert_query.table_id.getDatabaseName()); - res_columns[i++]->insert(insert_query.table_id.getTableName()); + + /// If query is "INSERT INTO FUNCTION" then table_id is empty. + if (insert_query.table_id) + { + res_columns[i++]->insert(insert_query.table_id.getDatabaseName()); + res_columns[i++]->insert(insert_query.table_id.getTableName()); + } + else + { + res_columns[i++]->insertDefault(); + res_columns[i++]->insertDefault(); + } + res_columns[i++]->insert(insert_query.format); res_columns[i++]->insert(time_in_microseconds(elem->data->first_update)); res_columns[i++]->insert(time_in_microseconds(elem->data->last_update)); diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 5c25322b4f0..d7034cf828b 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -11,7 +11,6 @@ const char * auto_config_build[] "VERSION_DATE", "@VERSION_DATE@", "BUILD_TYPE", "@CMAKE_BUILD_TYPE@", "SYSTEM_PROCESSOR", "@CMAKE_SYSTEM_PROCESSOR@", - "LIBRARY_ARCHITECTURE", "@CMAKE_LIBRARY_ARCHITECTURE@", "CMAKE_VERSION", "@CMAKE_VERSION@", "C_COMPILER", "@CMAKE_C_COMPILER@", "C_COMPILER_VERSION", "@CMAKE_C_COMPILER_VERSION@", @@ -19,7 +18,7 @@ const char * auto_config_build[] "CXX_COMPILER_VERSION", "@CMAKE_CXX_COMPILER_VERSION@", "C_FLAGS", "@FULL_C_FLAGS_NORMALIZED@", "CXX_FLAGS", "@FULL_CXX_FLAGS_NORMALIZED@", - "LINK_FLAGS", "@CMAKE_EXE_LINKER_FLAGS_NORMALIZED@", + "LINK_FLAGS", "@FULL_EXE_LINKER_FLAGS_NORMALIZED@", "BUILD_COMPILE_DEFINITIONS", "@BUILD_COMPILE_DEFINITIONS@", "STATIC", "@USE_STATIC_LIBRARIES@", "SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@", diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index d847c00846c..082b46f5a7e 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -304,20 +304,20 @@ private: Pipe StorageSystemColumns::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t max_block_size, const unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); /// Create a mask of what columns are needed in the result. NameSet names_set(column_names.begin(), column_names.end()); - Block sample_block = metadata_snapshot->getSampleBlock(); + Block sample_block = storage_snapshot->metadata->getSampleBlock(); Block header; std::vector columns_mask(sample_block.columns()); diff --git a/src/Storages/System/StorageSystemColumns.h b/src/Storages/System/StorageSystemColumns.h index dc184b1ae42..126deef1921 100644 --- a/src/Storages/System/StorageSystemColumns.h +++ b/src/Storages/System/StorageSystemColumns.h @@ -19,7 +19,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 87bd266af96..85cb51862e8 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -1,12 +1,16 @@ -// autogenerated by ./StorageSystemContributors.sh +// autogenerated by tests/ci/version_helper.py const char * auto_contributors[] { "0xflotus", + "13DaGGeR", "20018712", "243f6a88 85a308d3", "243f6a8885a308d313198a2e037", "3ldar-nasyrov", + "7", "821008736@qq.com", "ANDREI STAROVEROV", + "Aaron Katz", + "Adri Fernandez", "Ahmed Dardery", "Aimiyoo", "Akazz", @@ -19,15 +23,15 @@ const char * auto_contributors[] { "Aleksandrov Vladimir", "Aleksei Levushkin", "Aleksei Semiglazov", - "Aleksey Akulovich", "Aleksey", + "Aleksey Akulovich", + "Alex", "Alex Bocharov", "Alex Cao", "Alex Karo", "Alex Krash", "Alex Ryndin", "Alex Zatelepin", - "Alex", "Alexander Avdonkin", "Alexander Bezpiatov", "Alexander Burmak", @@ -55,20 +59,21 @@ const char * auto_contributors[] { "Alexandr Kondratev", "Alexandr Krasheninnikov", "Alexandr Orlov", - "Alexandra Latysheva", "Alexandra", + "Alexandra Latysheva", "Alexandre Snarskii", "Alexei Averchenko", + "Alexey", "Alexey Arno", "Alexey Boykov", "Alexey Dushechkin", "Alexey Elymanov", + "Alexey Gusev", "Alexey Ilyukhov", "Alexey Milovidov", "Alexey Tronov", "Alexey Vasiliev", "Alexey Zatelepin", - "Alexey", "Alexsey Shestakov", "Ali Demirci", "Aliaksandr Pliutau", @@ -84,14 +89,17 @@ const char * auto_contributors[] { "Anastasiya Tsarkova", "Anatoly Pugachev", "Andr0901", + "Andre Marianiello", "Andreas Hunkeler", "AndreevDm", "Andrei Bodrov", "Andrei Ch", "Andrei Chulkov", "Andrei Nekrashevich", + "Andrew", "Andrew Grigorev", "Andrew Onyshchuk", + "Andrey", "Andrey Chulkov", "Andrey Dudin", "Andrey Kadochnikov", @@ -103,12 +111,13 @@ const char * auto_contributors[] { "Andrey Torsunov", "Andrey Urusov", "Andrey Z", - "Andrey", + "Andrii Buriachevskyi", "Andy Liang", "Andy Yang", "Anmol Arora", - "Anna Shakhova", "Anna", + "Anna Shakhova", + "Anselmo D. Adams", "Anthony N. Simon", "Anton Ivashkin", "Anton Kobzev", @@ -121,6 +130,7 @@ const char * auto_contributors[] { "Anton Tikhonov", "Anton Yuzhaninov", "Anton Zhabolenko", + "Antonio Andelic", "Ariel Robaldo", "Arsen Hakobyan", "Arslan G", @@ -136,9 +146,9 @@ const char * auto_contributors[] { "Arthur Petukhovsky", "Arthur Tokarchuk", "Arthur Wong", + "Artur", "Artur Beglaryan", "Artur Filatenkov", - "Artur", "AsiaKorushkina", "Atri Sharma", "Avogar", @@ -149,6 +159,7 @@ const char * auto_contributors[] { "BanyRule", "Baudouin Giard", "BayoNet", + "Ben", "Benjamin Naecker", "Bertrand Junqua", "Bharat Nallan", @@ -156,12 +167,13 @@ const char * auto_contributors[] { "Bill", "BiteTheDDDDt", "BlahGeek", - "Bogdan Voronin", "Bogdan", + "Bogdan Voronin", "BohuTANG", "Bolinov", "BoloniniD", "Boris Granveaud", + "Boris Kuschel", "Bowen Masco", "Braulio Valdivielso", "Brett Hoerner", @@ -173,6 +185,8 @@ const char * auto_contributors[] { "Chen Yufei", "Chienlung Cheung", "Christian", + "Christoph Wurm", + "Chun-Sheng, Li", "Ciprian Hacman", "Clement Rodriguez", "ClickHouse Admin", @@ -181,12 +195,15 @@ const char * auto_contributors[] { "Colum", "Constantin S. Pan", "Constantine Peresypkin", + "CoolT2", "CurtizJ", + "DF5HSE", "DIAOZHAFENG", "Daniel Bershatsky", "Daniel Dao", "Daniel Qin", "Danila Kutenin", + "Dao", "Dao Minh Thuc", "Daria Mozhaeva", "Dario", @@ -198,13 +215,16 @@ const char * auto_contributors[] { "Denis Zhuravlev", "Denny Crane", "Derek Perkins", + "DimaAmega", "Ding Xiang Fei", "Dmitriev Mikhail", "Dmitrii Kovalkov", + "Dmitrii Mokhnatkin", "Dmitrii Raev", + "Dmitriy", "Dmitriy Dorofeev", "Dmitriy Lushnikov", - "Dmitriy", + "Dmitry", "Dmitry Belyavtsev", "Dmitry Bilunov", "Dmitry Galuza", @@ -217,7 +237,6 @@ const char * auto_contributors[] { "Dmitry Rubashkin", "Dmitry S..ky / skype: dvska-at-skype", "Dmitry Ukolov", - "Dmitry", "Doge", "Dongdong Yang", "DoomzD", @@ -232,8 +251,8 @@ const char * auto_contributors[] { "Elizaveta Mironyuk", "Elykov Alexandr", "Emmanuel Donin de Rosière", - "Eric Daniel", "Eric", + "Eric Daniel", "Erixonich", "Ernest Poletaev", "Eugene Klimov", @@ -243,9 +262,9 @@ const char * auto_contributors[] { "Evgeniia Sudarikova", "Evgeniy Gatov", "Evgeniy Udodov", + "Evgeny", "Evgeny Konkov", "Evgeny Markov", - "Evgeny", "Ewout", "FArthur-cmd", "Fabian Stäber", @@ -254,10 +273,12 @@ const char * auto_contributors[] { "Fan()", "FawnD2", "Federico Ceratto", + "Federico Rodriguez", "FeehanG", "FgoDt", "Filatenkov Artur", "Filipe Caixeta", + "Filippov Denis", "Flowyi", "Francisco Barón", "Frank Chen", @@ -269,8 +290,10 @@ const char * auto_contributors[] { "Gagan Arneja", "Gao Qiang", "Gary Dotzler", - "George G", + "Gaurav Kumar", + "Geoff Genz", "George", + "George G", "George3d6", "Georgy Ginzburg", "Gervasio Varela", @@ -278,28 +301,35 @@ const char * auto_contributors[] { "Gleb Novikov", "Gleb-Tretyakov", "Gregory", + "Grigory", "Grigory Buteyko", "Grigory Pervakov", - "Grigory", "Guillaume Tassery", "Guo Wei (William)", "Haavard Kvaalen", "Habibullah Oladepo", "Hamoon", + "Harry-Lee", + "HarryLeeIBM", "Hasitha Kanchana", "Hasnat", + "Heena Bansal", + "HeenaBansal2009", "Hiroaki Nakamura", "HuFuwang", "Hui Wang", + "ILya Limarenko", + "Igor", "Igor Hatarist", "Igor Mineev", + "Igor Nikonov", "Igor Strykhar", - "Igor", - "Igr Mineev", "Igr", + "Igr Mineev", "Ikko Ashimine", "Ildar Musin", "Ildus Kurbangaliev", + "Ilya", "Ilya Breev", "Ilya Golshtein", "Ilya Khomutov", @@ -310,10 +340,11 @@ const char * auto_contributors[] { "Ilya Shipitsin", "Ilya Skrypitsa", "Ilya Yatsishin", - "Ilya", + "IlyaTsoi", "ImgBotApp", - "Islam Israfilov (Islam93)", "Islam Israfilov", + "Islam Israfilov (Islam93)", + "Ivan", "Ivan A. Torgashov", "Ivan Babrou", "Ivan Blinkov", @@ -325,27 +356,29 @@ const char * auto_contributors[] { "Ivan Remen", "Ivan Starkov", "Ivan Zhukov", - "Ivan", "Jack Song", "JackyWoo", "Jacob Hayes", + "Jake Liu", "Jakub Kuklis", "JaosnHsieh", - "Jason Keirstead", "Jason", + "Jason Keirstead", "Javi Santana", "Javi santana bot", + "JaySon-Huang", "Jean Baptiste Favre", "Jeffrey Dang", "Jiading Guo", "Jiang Tao", "Jochen Schalanda", + "John", "John Hummel", "John Skopis", - "John", "Jonatas Freitas", "João Figueiredo", "Julian Zhou", + "Justin Hilliard", "Kang Liu", "Karl Pietrzak", "Keiji Yoshida", @@ -375,7 +408,9 @@ const char * auto_contributors[] { "Ky Li", "LB", "Latysheva Alexandra", + "Lemore", "Leonardo Cecchi", + "Leonid Krylov", "Leopold Schabel", "Lev Borodin", "Lewinma", @@ -391,9 +426,9 @@ const char * auto_contributors[] { "M0r64n", "MagiaGroz", "Maks Skorokhod", + "Maksim", "Maksim Fedotov", "Maksim Kita", - "Maksim", "Malte", "Marat IDRISOV", "Marek Vavrusa", @@ -412,10 +447,11 @@ const char * auto_contributors[] { "Masha", "Matthew Peveler", "Matwey V. Kornilov", + "Max", "Max Akhmedov", "Max Bruce", "Max Vetrov", - "Max", + "MaxTheHuman", "MaxWk", "Maxim Akhmedov", "Maxim Babenko", @@ -430,6 +466,7 @@ const char * auto_contributors[] { "Maxim Ulanovskiy", "MaximAL", "Mc.Spring", + "Meena-Renganathan", "MeiK", "Memo", "Metehan Çetinkaya", @@ -439,18 +476,21 @@ const char * auto_contributors[] { "Michael Monashev", "Michael Razuvaev", "Michael Smitasin", + "Michail Safronov", "Michal Lisowski", "MicrochipQ", "Miguel Fernández", "Mihail Fandyushin", "Mikahil Nacharov", + "Mike", "Mike F", "Mike Kot", - "Mike", + "Mikhail", "Mikhail Andreev", "Mikhail Cheshkov", "Mikhail Fandyushin", "Mikhail Filimonov", + "Mikhail Fursov", "Mikhail Gaidamaka", "Mikhail Korotov", "Mikhail Malafeev", @@ -458,18 +498,19 @@ const char * auto_contributors[] { "Mikhail Salosin", "Mikhail Surin", "Mikhail f. Shiryaev", - "Mikhail", "MikuSugar", "Milad Arabi", "Misko Lee", "Mohamad Fadhil", "Mohammad Hossein Sekhavat", + "Mojtaba Yaghoobzadeh", "Mostafa Dahab", "MovElb", "Mr.General", "Murat Kabilov", "MyroTk", "Mátyás Jani", + "N. Kolotov", "NIKITA MIKHAILOV", "Narek Galstyan", "Natasha Murashkina", @@ -477,15 +518,17 @@ const char * auto_contributors[] { "Neeke Gao", "Neng Liu", "NengLiu", - "Nickita Taranov", "Nickita", + "Nickita Taranov", "Nickolay Yastrebov", "Nico Mandery", "Nico Piderman", "Nicolae Vartolomei", + "Niek", "Nik", "Nikhil Nadig", "Nikhil Raman", + "Nikita", "Nikita Lapkov", "Nikita Mikhailov", "Nikita Mikhalev", @@ -495,13 +538,13 @@ const char * auto_contributors[] { "Nikita Vasilev", "Nikolai Kochetov", "Nikolai Sorokin", + "Nikolay", "Nikolay Degterinsky", "Nikolay Kirsh", "Nikolay Semyachkin", "Nikolay Shcheglov", "Nikolay Vasiliev", "Nikolay Volosatov", - "Nikolay", "Niu Zhaojie", "Odin Hultgren Van Der Horst", "Okada Haruki", @@ -517,11 +560,13 @@ const char * auto_contributors[] { "OnePiece", "Onehr7", "Orivej Desh", + "Orkhan Zeynalli", "Oskar Wojciski", "OuO", "PHO", "Paramtamtam", "Patrick Zippenfenig", + "Pavel", "Pavel Cheremushkin", "Pavel Kartaviy", "Pavel Kartavyy", @@ -531,7 +576,6 @@ const char * auto_contributors[] { "Pavel Medvedev", "Pavel Patrin", "Pavel Yakunin", - "Pavel", "Pavlo Bashynskiy", "Pawel Rog", "Peignon Melvyn", @@ -545,6 +589,7 @@ const char * auto_contributors[] { "Pysaoke", "Quid37", "Rafael David Tinoco", + "Rajkumar", "Ramazan Polat", "Ravengg", "Raúl Marín", @@ -556,8 +601,10 @@ const char * auto_contributors[] { "Ri", "Rich Raposa", "Robert Hodges", + "RogerYK", "Rohit Agarwal", "Romain Neutron", + "Roman", "Roman Bug", "Roman Chyrva", "Roman Lipovsky", @@ -566,13 +613,15 @@ const char * auto_contributors[] { "Roman Peshkurov", "Roman Tsisyk", "Roman Zhukov", - "Roman", - "Ruslan Savchenko", "Ruslan", + "Ruslan Savchenko", "Russ Frank", "Ruzal Ibragimov", + "Ryad ZENINE", "S.M.A. Djawadi", + "Saad Ur Rahman", "Sabyanin Maxim", + "Safronov Michail", "SaltTan", "Sami Kerola", "Samuel Chou", @@ -583,6 +632,7 @@ const char * auto_contributors[] { "Sergei Bocharov", "Sergei Semin", "Sergei Shtykov", + "Sergei Trifonov", "Sergei Tsetlin (rekub)", "Sergey Demurin", "Sergey Elantsev", @@ -614,26 +664,28 @@ const char * auto_contributors[] { "Stas Kelvich", "Stas Pavlovichev", "Stefan Thies", - "Stepan Herold", "Stepan", + "Stepan Herold", "Steve-金勇", "Stig Bakken", "Storozhuk Kostiantyn", "Stupnikov Andrey", "SuperBot", "SuperDJY", - "Sébastien Launay", + "Suzy Wang", "Sébastien", + "Sébastien Launay", + "TABLUM.IO", "TAC", "TCeason", "Tagir Kuskarov", "Tai White", "Taleh Zaliyev", "Tangaev", - "Tatiana Kirillova", "Tatiana", - "Teja Srivastasa", + "Tatiana Kirillova", "Teja", + "Teja Srivastasa", "Tema Novikov", "Tentoshka", "The-Alchemist", @@ -655,10 +707,10 @@ const char * auto_contributors[] { "UnamedRus", "V", "VDimir", + "Vadim", "Vadim Plakhtinskiy", "Vadim Skipin", "Vadim Volodin", - "Vadim", "VadimPE", "Val", "Valera Ryaboshapko", @@ -672,8 +724,8 @@ const char * auto_contributors[] { "Veniamin Gvozdikov", "Veselkov Konstantin", "Viachaslau Boben", - "Victor Tarnavsky", "Victor", + "Victor Tarnavsky", "Viktor Taranenko", "Vitalii S", "Vitaliy Fedorchenko", @@ -681,13 +733,15 @@ const char * auto_contributors[] { "Vitaliy Kozlovskiy", "Vitaliy Lyudvichenko", "Vitaliy Zakaznikov", + "Vitaly", + "Vitaly Artemyev", "Vitaly Baranov", "Vitaly Orlov", "Vitaly Samigullin", "Vitaly Stoyan", - "Vitaly", "Vivien Maisonneuve", "Vlad Arkhipov", + "Vladimir", "Vladimir Bunchuk", "Vladimir C", "Vladimir Ch", @@ -699,7 +753,6 @@ const char * auto_contributors[] { "Vladimir Kopysov", "Vladimir Kozbin", "Vladimir Smirnov", - "Vladimir", "Vladislav Rassokhin", "Vladislav Smirnov", "Vojtech Splichal", @@ -707,6 +760,7 @@ const char * auto_contributors[] { "Vsevolod Orlov", "Vxider", "Vyacheslav Alipov", + "W", "Wang Fenjin", "WangZengrui", "Weiqing Xu", @@ -714,8 +768,10 @@ const char * auto_contributors[] { "Winter Zhang", "Xianda Ke", "Xiang Zhou", + "Xin Wang", "Y Lu", "Yangkuan Liu", + "Yatian Xu", "Yatsishin Ilya", "Yağızcan Değirmenci", "Yegor Andreenko", @@ -724,13 +780,14 @@ const char * auto_contributors[] { "Yingfan Chen", "Yiğit Konur", "Yohann Jardin", + "Youenn Lebras", "Yuntao Wu", "Yuri Dyachenko", "Yurii Vlasenko", + "Yuriy", "Yuriy Baranov", "Yuriy Chernyshov", "Yuriy Korzhenevskiy", - "Yuriy", "Yury Karpovich", "Yury Stankevich", "ZhiYong Wang", @@ -756,6 +813,7 @@ const char * auto_contributors[] { "alex.lvxin", "alexander kozhikhov", "alexey-milovidov", + "alexeypavlenko", "alfredlu", "amesaru", "amoschen", @@ -810,10 +868,12 @@ const char * auto_contributors[] { "cms", "cmsxbc", "cn-ds", + "cnmade", "comunodi", "congbaoyangrou", "coraxster", "d.v.semenov", + "dalei2019", "damozhaeva", "dankondr", "daoready", @@ -849,6 +909,7 @@ const char * auto_contributors[] { "ezhaka", "f1yegor", "fancno", + "fanzhou", "fastio", "favstovol", "feihengye", @@ -863,8 +924,8 @@ const char * auto_contributors[] { "flow", "flynn", "foxxmary", - "frank chen", "frank", + "frank chen", "franklee", "fredchenbj", "freedomDR", @@ -877,8 +938,11 @@ const char * auto_contributors[] { "giordyb", "glockbender", "glushkovds", + "grantovsky", + "gulige", "guoleiyi", "gyuton", + "hanqf-git", "hao.he", "hchen9", "hcz", @@ -890,6 +954,7 @@ const char * auto_contributors[] { "huangzhaowei", "hustnn", "huzhichengdd", + "ianton-ru", "ice1x", "idfer", "igomac", @@ -907,8 +972,8 @@ const char * auto_contributors[] { "jasine", "jasperzhu", "javartisan", - "javi santana", "javi", + "javi santana", "jennyma", "jetgm", "jianmei zhang", @@ -937,6 +1002,8 @@ const char * auto_contributors[] { "levie", "levushkin aleksej", "levysh", + "lgbo", + "lgbo-ustc", "lhuang0928", "lhuang09287750", "liang.huang", @@ -947,6 +1014,7 @@ const char * auto_contributors[] { "listar", "litao91", "liu-bov", + "liuneng1994", "liuyangkuan", "liuyimin", "liyang", @@ -983,11 +1051,15 @@ const char * auto_contributors[] { "mikael", "mikepop7", "millb", + "minhthucdao", + "mlkui", "mnkonkova", "mo-avatar", "morty", "moscas", + "mreddy017", "msaf1980", + "msirm", "muzzlerator", "mwish", "myrrc", @@ -1007,6 +1079,7 @@ const char * auto_contributors[] { "ocadaruma", "ogorbacheva", "olegkv", + "olevino", "olgarev", "orantius", "p0ny", @@ -1014,6 +1087,7 @@ const char * auto_contributors[] { "pawelsz-rb", "pdv-ru", "peshkurov", + "peter279k", "philip.han", "pingyu", "potya", @@ -1040,8 +1114,10 @@ const char * auto_contributors[] { "roverxu", "ruct", "ryzuo", + "s-kat", "santaux", "satanson", + "save-my-heart", "sdk2", "serebrserg", "sev7e0", @@ -1068,8 +1144,10 @@ const char * auto_contributors[] { "taiyang-li", "tao jiang", "tavplubix", + "tekeri", "templarzq", "terrylin", + "tesw yew isal", "tianzhou", "tiger.yan", "tison", @@ -1080,6 +1158,8 @@ const char * auto_contributors[] { "unegare", "unknown", "urgordeadbeef", + "usurai", + "vahid-sohrabloo", "vdimir", "velom", "vesslanjin", @@ -1121,8 +1201,11 @@ const char * auto_contributors[] { "zhangxiao018", "zhangxiao871", "zhen ni", + "zhifeng", "zhongyuankai", + "zhoubintao", "zhukai", + "zkun", "zlx19950903", "zvonand", "zvrr", @@ -1150,6 +1233,7 @@ const char * auto_contributors[] { "曲正鹏", "木木夕120", "未来星___费", + "李扬", "极客青年", "枢木", "董海镔", @@ -1159,5 +1243,6 @@ const char * auto_contributors[] { "靳阳", "黄朝晖", "黄璞", + "박동철", "박현우", nullptr}; diff --git a/src/Storages/System/StorageSystemContributors.sh b/src/Storages/System/StorageSystemContributors.sh index 9b714a94207..a724fdc9d39 100755 --- a/src/Storages/System/StorageSystemContributors.sh +++ b/src/Storages/System/StorageSystemContributors.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +echo "THIS IS HEAVILY DEPRECATED, USE tests/ci/version_helper.py:update_contributors()" set -x # doesn't actually cd to directory, but return absolute path diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp index d7fc06da953..42b214bf101 100644 --- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp +++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp @@ -165,18 +165,18 @@ private: Pipe StorageSystemDataSkippingIndices::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /* processed_stage */, size_t max_block_size, unsigned int /* num_streams */) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); NameSet names_set(column_names.begin(), column_names.end()); - Block sample_block = metadata_snapshot->getSampleBlock(); + Block sample_block = storage_snapshot->metadata->getSampleBlock(); Block header; std::vector columns_mask(sample_block.columns()); diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.h b/src/Storages/System/StorageSystemDataSkippingIndices.h index d86890f5e27..93511d0d591 100644 --- a/src/Storages/System/StorageSystemDataSkippingIndices.h +++ b/src/Storages/System/StorageSystemDataSkippingIndices.h @@ -16,7 +16,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -26,7 +26,7 @@ public: bool isSystemStorage() const override { return true; } protected: - StorageSystemDataSkippingIndices(const StorageID & table_id_); + explicit StorageSystemDataSkippingIndices(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 5a24809d05a..4797dff2fd1 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -31,7 +31,7 @@ StorageSystemDetachedParts::StorageSystemDetachedParts(const StorageID & table_i Pipe StorageSystemDetachedParts::read( const Names & /* column_names */, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, @@ -41,7 +41,7 @@ Pipe StorageSystemDetachedParts::read( StoragesInfoStream stream(query_info, context); /// Create the result. - Block block = metadata_snapshot->getSampleBlock(); + Block block = storage_snapshot->metadata->getSampleBlock(); MutableColumns new_columns = block.cloneEmptyColumns(); while (StoragesInfo info = stream.next()) diff --git a/src/Storages/System/StorageSystemDetachedParts.h b/src/Storages/System/StorageSystemDetachedParts.h index ece9d495500..8ed11eb306c 100644 --- a/src/Storages/System/StorageSystemDetachedParts.h +++ b/src/Storages/System/StorageSystemDetachedParts.h @@ -25,12 +25,12 @@ protected: Pipe read( const Names & /* column_names */, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, - const size_t /*max_block_size*/, - const unsigned /*num_streams*/) override; + size_t /*max_block_size*/, + unsigned /*num_streams*/) override; }; } diff --git a/src/Storages/System/StorageSystemDisks.cpp b/src/Storages/System/StorageSystemDisks.cpp index 749717922da..3841abc2f2d 100644 --- a/src/Storages/System/StorageSystemDisks.cpp +++ b/src/Storages/System/StorageSystemDisks.cpp @@ -28,14 +28,14 @@ StorageSystemDisks::StorageSystemDisks(const StorageID & table_id_) Pipe StorageSystemDisks::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); MutableColumnPtr col_name = ColumnString::create(); MutableColumnPtr col_path = ColumnString::create(); @@ -65,7 +65,7 @@ Pipe StorageSystemDisks::read( UInt64 num_rows = res_columns.at(0)->size(); Chunk chunk(std::move(res_columns), num_rows); - return Pipe(std::make_shared(metadata_snapshot->getSampleBlock(), std::move(chunk))); + return Pipe(std::make_shared(storage_snapshot->metadata->getSampleBlock(), std::move(chunk))); } } diff --git a/src/Storages/System/StorageSystemDisks.h b/src/Storages/System/StorageSystemDisks.h index 2541dedd8fc..2640ab7149b 100644 --- a/src/Storages/System/StorageSystemDisks.h +++ b/src/Storages/System/StorageSystemDisks.h @@ -22,7 +22,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -32,7 +32,7 @@ public: bool isSystemStorage() const override { return true; } protected: - StorageSystemDisks(const StorageID & table_id_); + explicit StorageSystemDisks(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemModels.cpp b/src/Storages/System/StorageSystemModels.cpp index 3df48e830bb..4a4dbbc69df 100644 --- a/src/Storages/System/StorageSystemModels.cpp +++ b/src/Storages/System/StorageSystemModels.cpp @@ -38,7 +38,7 @@ void StorageSystemModels::fillData(MutableColumns & res_columns, ContextPtr cont if (load_result.object) { - const auto model_ptr = std::static_pointer_cast(load_result.object); + const auto model_ptr = std::static_pointer_cast(load_result.object); res_columns[3]->insert(model_ptr->getTypeName()); } else diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp index c09279e65ac..2e48bb857ce 100644 --- a/src/Storages/System/StorageSystemNumbers.cpp +++ b/src/Storages/System/StorageSystemNumbers.cpp @@ -124,14 +124,14 @@ StorageSystemNumbers::StorageSystemNumbers(const StorageID & table_id, bool mult Pipe StorageSystemNumbers::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo &, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); if (limit && *limit < max_block_size) { diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index 32105bb055d..5f3a12c530d 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -31,7 +31,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/System/StorageSystemOne.cpp b/src/Storages/System/StorageSystemOne.cpp index 7558ae0ae92..f262c981b83 100644 --- a/src/Storages/System/StorageSystemOne.cpp +++ b/src/Storages/System/StorageSystemOne.cpp @@ -22,14 +22,14 @@ StorageSystemOne::StorageSystemOne(const StorageID & table_id_) Pipe StorageSystemOne::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo &, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); Block header{ColumnWithTypeAndName( DataTypeUInt8().createColumn(), diff --git a/src/Storages/System/StorageSystemOne.h b/src/Storages/System/StorageSystemOne.h index cc1d5e05b75..b0ca389b76f 100644 --- a/src/Storages/System/StorageSystemOne.h +++ b/src/Storages/System/StorageSystemOne.h @@ -23,7 +23,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index f4dd9cbd45d..26b1b151073 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -26,7 +26,7 @@ namespace ErrorCodes extern const int TABLE_IS_DROPPED; } -bool StorageSystemPartsBase::hasStateColumn(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const +bool StorageSystemPartsBase::hasStateColumn(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) { bool has_state_column = false; Names real_column_names; @@ -41,7 +41,7 @@ bool StorageSystemPartsBase::hasStateColumn(const Names & column_names, const St /// Do not check if only _state column is requested if (!(has_state_column && real_column_names.empty())) - metadata_snapshot->check(real_column_names, {}, getStorageID()); + storage_snapshot->check(real_column_names); return has_state_column; } @@ -235,14 +235,14 @@ StoragesInfo StoragesInfoStream::next() Pipe StorageSystemPartsBase::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - bool has_state_column = hasStateColumn(column_names, metadata_snapshot); + bool has_state_column = hasStateColumn(column_names, storage_snapshot); StoragesInfoStream stream(query_info, context); @@ -250,7 +250,7 @@ Pipe StorageSystemPartsBase::read( NameSet names_set(column_names.begin(), column_names.end()); - Block sample = metadata_snapshot->getSampleBlock(); + Block sample = storage_snapshot->metadata->getSampleBlock(); Block header; std::vector columns_mask(sample.columns()); diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index 87247f96b24..0daa01a6b99 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -23,7 +23,7 @@ struct StoragesInfo bool need_inactive_parts = false; MergeTreeData * data = nullptr; - operator bool() const { return storage != nullptr; } + operator bool() const { return storage != nullptr; } /// NOLINT MergeTreeData::DataPartsVector getParts(MergeTreeData::DataPartStateVector & state, bool has_state_column, bool require_projection_parts = false) const; }; @@ -58,7 +58,7 @@ class StorageSystemPartsBase : public IStorage public: Pipe read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -70,7 +70,7 @@ public: bool isSystemStorage() const override { return true; } private: - bool hasStateColumn(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const; + static bool hasStateColumn(const Names & column_names, const StorageSnapshotPtr & storage_snapshot); protected: const FormatSettings format_settings; diff --git a/src/Storages/System/StorageSystemPartsColumns.cpp b/src/Storages/System/StorageSystemPartsColumns.cpp index f5e9b82c136..a9341abb9cd 100644 --- a/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/src/Storages/System/StorageSystemPartsColumns.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -64,7 +65,11 @@ StorageSystemPartsColumns::StorageSystemPartsColumns(const StorageID & table_id_ {"serialization_kind", std::make_shared()}, {"subcolumns.names", std::make_shared(std::make_shared())}, {"subcolumns.types", std::make_shared(std::make_shared())}, - {"subcolumns.serializations", std::make_shared(std::make_shared())} + {"subcolumns.serializations", std::make_shared(std::make_shared())}, + {"subcolumns.bytes_on_disk", std::make_shared(std::make_shared())}, + {"subcolumns.data_compressed_bytes", std::make_shared(std::make_shared())}, + {"subcolumns.data_uncompressed_bytes", std::make_shared(std::make_shared())}, + {"subcolumns.marks_bytes", std::make_shared(std::make_shared())}, } ) { @@ -228,13 +233,43 @@ void StorageSystemPartsColumns::processNextStorage( Array subcolumn_names; Array subcolumn_types; - Array subcolumn_sers; + Array subcolumn_serializations; + Array subcolumn_bytes_on_disk; + Array subcolumn_data_compressed_bytes; + Array subcolumn_data_uncompressed_bytes; + Array subcolumn_marks_bytes; - IDataType::forEachSubcolumn([&](const auto &, const auto & name, const auto & data) + IDataType::forEachSubcolumn([&](const auto & subpath, const auto & name, const auto & data) { + /// We count only final subcolumns, which are represented by files on disk + /// and skip intermediate suibcolumns of types Tuple and Nested. + if (isTuple(data.type) || isNested(data.type)) + return; + subcolumn_names.push_back(name); subcolumn_types.push_back(data.type->getName()); - subcolumn_sers.push_back(ISerialization::kindToString(data.serialization->getKind())); + subcolumn_serializations.push_back(ISerialization::kindToString(data.serialization->getKind())); + + ColumnSize size; + NameAndTypePair subcolumn(column.name, name, column.type, data.type); + String file_name = ISerialization::getFileNameForStream(subcolumn, subpath); + + auto bin_checksum = part->checksums.files.find(file_name + ".bin"); + if (bin_checksum != part->checksums.files.end()) + { + size.data_compressed += bin_checksum->second.file_size; + size.data_uncompressed += bin_checksum->second.uncompressed_size; + } + + auto mrk_checksum = part->checksums.files.find(file_name + part->index_granularity_info.marks_file_extension); + if (mrk_checksum != part->checksums.files.end()) + size.marks += mrk_checksum->second.file_size; + + subcolumn_bytes_on_disk.push_back(size.data_compressed + size.marks); + subcolumn_data_compressed_bytes.push_back(size.data_compressed); + subcolumn_data_uncompressed_bytes.push_back(size.data_uncompressed); + subcolumn_marks_bytes.push_back(size.marks); + }, { serialization, column.type, nullptr, nullptr }); if (columns_mask[src_index++]) @@ -242,7 +277,15 @@ void StorageSystemPartsColumns::processNextStorage( if (columns_mask[src_index++]) columns[res_index++]->insert(subcolumn_types); if (columns_mask[src_index++]) - columns[res_index++]->insert(subcolumn_sers); + columns[res_index++]->insert(subcolumn_serializations); + if (columns_mask[src_index++]) + columns[res_index++]->insert(subcolumn_bytes_on_disk); + if (columns_mask[src_index++]) + columns[res_index++]->insert(subcolumn_data_compressed_bytes); + if (columns_mask[src_index++]) + columns[res_index++]->insert(subcolumn_data_uncompressed_bytes); + if (columns_mask[src_index++]) + columns[res_index++]->insert(subcolumn_marks_bytes); if (has_state_column) columns[res_index++]->insert(part->stateString()); diff --git a/src/Storages/System/StorageSystemPartsColumns.h b/src/Storages/System/StorageSystemPartsColumns.h index c3e3eaefcf7..b8c52ca16ef 100644 --- a/src/Storages/System/StorageSystemPartsColumns.h +++ b/src/Storages/System/StorageSystemPartsColumns.h @@ -21,7 +21,7 @@ public: std::string getName() const override { return "SystemPartsColumns"; } protected: - StorageSystemPartsColumns(const StorageID & table_id_); + explicit StorageSystemPartsColumns(const StorageID & table_id_); void processNextStorage( MutableColumns & columns, std::vector & columns_mask, const StoragesInfo & info, bool has_state_column) override; }; diff --git a/src/Storages/System/StorageSystemProcesses.cpp b/src/Storages/System/StorageSystemProcesses.cpp index 5e6ba37226c..efa01561ad4 100644 --- a/src/Storages/System/StorageSystemProcesses.cpp +++ b/src/Storages/System/StorageSystemProcesses.cpp @@ -48,6 +48,7 @@ NamesAndTypesList StorageSystemProcesses::getNamesAndTypes() {"forwarded_for", std::make_shared()}, {"quota_key", std::make_shared()}, + {"distributed_depth", std::make_shared()}, {"elapsed", std::make_shared()}, {"is_cancelled", std::make_shared()}, @@ -115,6 +116,7 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, ContextPtr c res_columns[i++]->insert(process.client_info.forwarded_for); res_columns[i++]->insert(process.client_info.quota_key); + res_columns[i++]->insert(process.client_info.distributed_depth); res_columns[i++]->insert(process.elapsed_seconds); res_columns[i++]->insert(process.is_cancelled); diff --git a/src/Storages/System/StorageSystemProjectionPartsColumns.h b/src/Storages/System/StorageSystemProjectionPartsColumns.h index 10e80877285..5679f5e9093 100644 --- a/src/Storages/System/StorageSystemProjectionPartsColumns.h +++ b/src/Storages/System/StorageSystemProjectionPartsColumns.h @@ -21,7 +21,7 @@ public: std::string getName() const override { return "SystemProjectionPartsColumns"; } protected: - StorageSystemProjectionPartsColumns(const StorageID & table_id_); + explicit StorageSystemProjectionPartsColumns(const StorageID & table_id_); void processNextStorage( MutableColumns & columns, std::vector & columns_mask, const StoragesInfo & info, bool has_state_column) override; }; diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 467226c3b7a..e018ccc0733 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -61,14 +61,14 @@ StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_) Pipe StorageSystemReplicas::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); const auto access = context->getAccess(); const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); @@ -149,7 +149,7 @@ Pipe StorageSystemReplicas::read( col_engine = filtered_block.getByName("engine").column; } - MutableColumns res_columns = metadata_snapshot->getSampleBlock().cloneEmptyColumns(); + MutableColumns res_columns = storage_snapshot->metadata->getSampleBlock().cloneEmptyColumns(); for (size_t i = 0, size = col_database->size(); i < size; ++i) { @@ -203,8 +203,6 @@ Pipe StorageSystemReplicas::read( res_columns[col_num++]->insert(std::move(replica_is_active_values)); } - Block header = metadata_snapshot->getSampleBlock(); - Columns fin_columns; fin_columns.reserve(res_columns.size()); @@ -218,7 +216,7 @@ Pipe StorageSystemReplicas::read( UInt64 num_rows = fin_columns.at(0)->size(); Chunk chunk(std::move(fin_columns), num_rows); - return Pipe(std::make_shared(metadata_snapshot->getSampleBlock(), std::move(chunk))); + return Pipe(std::make_shared(storage_snapshot->metadata->getSampleBlock(), std::move(chunk))); } diff --git a/src/Storages/System/StorageSystemReplicas.h b/src/Storages/System/StorageSystemReplicas.h index cf457efe250..1b93d10367b 100644 --- a/src/Storages/System/StorageSystemReplicas.h +++ b/src/Storages/System/StorageSystemReplicas.h @@ -20,7 +20,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -30,7 +30,7 @@ public: bool isSystemStorage() const override { return true; } protected: - StorageSystemReplicas(const StorageID & table_id_); + explicit StorageSystemReplicas(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 455d715d5da..cd4f3dab109 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include namespace DB @@ -43,7 +43,8 @@ NamesAndTypesList StorageSystemRowPolicies::getNamesAndTypes() {"apply_to_except", std::make_shared(std::make_shared())} }; - boost::range::push_back(names_and_types, std::move(extra_names_and_types)); + insertAtEnd(names_and_types, extra_names_and_types); + return names_and_types; } diff --git a/src/Storages/System/StorageSystemStackTrace.h b/src/Storages/System/StorageSystemStackTrace.h index a5827e32e6f..da4315d3ffa 100644 --- a/src/Storages/System/StorageSystemStackTrace.h +++ b/src/Storages/System/StorageSystemStackTrace.h @@ -27,7 +27,7 @@ public: String getName() const override { return "SystemStackTrace"; } static NamesAndTypesList getNamesAndTypes(); - StorageSystemStackTrace(const StorageID & table_id_); + explicit StorageSystemStackTrace(const StorageID & table_id_); protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/src/Storages/System/StorageSystemStoragePolicies.cpp b/src/Storages/System/StorageSystemStoragePolicies.cpp index 036e4748e65..04c98e6be9c 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.cpp +++ b/src/Storages/System/StorageSystemStoragePolicies.cpp @@ -38,14 +38,14 @@ StorageSystemStoragePolicies::StorageSystemStoragePolicies(const StorageID & tab Pipe StorageSystemStoragePolicies::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & /*query_info*/, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); MutableColumnPtr col_policy_name = ColumnString::create(); MutableColumnPtr col_volume_name = ColumnString::create(); @@ -89,7 +89,7 @@ Pipe StorageSystemStoragePolicies::read( UInt64 num_rows = res_columns.at(0)->size(); Chunk chunk(std::move(res_columns), num_rows); - return Pipe(std::make_shared(metadata_snapshot->getSampleBlock(), std::move(chunk))); + return Pipe(std::make_shared(storage_snapshot->metadata->getSampleBlock(), std::move(chunk))); } } diff --git a/src/Storages/System/StorageSystemStoragePolicies.h b/src/Storages/System/StorageSystemStoragePolicies.h index f202299db1f..e2890c42897 100644 --- a/src/Storages/System/StorageSystemStoragePolicies.h +++ b/src/Storages/System/StorageSystemStoragePolicies.h @@ -22,7 +22,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -32,7 +32,7 @@ public: bool isSystemStorage() const override { return true; } protected: - StorageSystemStoragePolicies(const StorageID & table_id_); + explicit StorageSystemStoragePolicies(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 24e3fe4f7a9..98a07d0f4c3 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -509,8 +509,8 @@ protected: loading_dependencies_tables.reserve(info.dependencies.size()); for (auto && dependency : info.dependencies) { - loading_dependencies_databases.push_back(std::move(dependency.database)); - loading_dependencies_tables.push_back(std::move(dependency.table)); + loading_dependencies_databases.push_back(dependency.database); + loading_dependencies_tables.push_back(dependency.table); } Array loading_dependent_databases; @@ -519,8 +519,8 @@ protected: loading_dependent_tables.reserve(info.dependencies.size()); for (auto && dependent : info.dependent_database_objects) { - loading_dependent_databases.push_back(std::move(dependent.database)); - loading_dependent_tables.push_back(std::move(dependent.table)); + loading_dependent_databases.push_back(dependent.database); + loading_dependent_tables.push_back(dependent.table); } if (columns_mask[src_index++]) @@ -556,20 +556,20 @@ private: Pipe StorageSystemTables::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, const size_t max_block_size, const unsigned /*num_streams*/) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); /// Create a mask of what columns are needed in the result. NameSet names_set(column_names.begin(), column_names.end()); - Block sample_block = metadata_snapshot->getSampleBlock(); + Block sample_block = storage_snapshot->metadata->getSampleBlock(); Block res_block; std::vector columns_mask(sample_block.columns()); diff --git a/src/Storages/System/StorageSystemTables.h b/src/Storages/System/StorageSystemTables.h index 808dc862e8d..7f6a099a824 100644 --- a/src/Storages/System/StorageSystemTables.h +++ b/src/Storages/System/StorageSystemTables.h @@ -20,7 +20,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, @@ -30,7 +30,7 @@ public: bool isSystemStorage() const override { return true; } protected: - StorageSystemTables(const StorageID & table_id_); + explicit StorageSystemTables(const StorageID & table_id_); }; } diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index ca88fa688a0..d9b94f21c61 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -102,17 +102,27 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte column_storage.insertData(storage_name.data(), storage_name.length()); column_auth_type.push_back(static_cast(auth_data.getType())); - if ( - auth_data.getType() == AuthenticationType::LDAP || - auth_data.getType() == AuthenticationType::KERBEROS - ) + if (auth_data.getType() == AuthenticationType::LDAP || + auth_data.getType() == AuthenticationType::KERBEROS || + auth_data.getType() == AuthenticationType::SSL_CERTIFICATE) { Poco::JSON::Object auth_params_json; if (auth_data.getType() == AuthenticationType::LDAP) + { auth_params_json.set("server", auth_data.getLDAPServerName()); + } else if (auth_data.getType() == AuthenticationType::KERBEROS) + { auth_params_json.set("realm", auth_data.getKerberosRealm()); + } + else if (auth_data.getType() == AuthenticationType::SSL_CERTIFICATE) + { + Poco::JSON::Array::Ptr arr = new Poco::JSON::Array(); + for (const auto & common_name : auth_data.getSSLCertificateCommonNames()) + arr->add(common_name); + auth_params_json.set("common_names", arr); + } std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); diff --git a/src/Storages/System/StorageSystemZeros.cpp b/src/Storages/System/StorageSystemZeros.cpp index 624fc54998c..b6a623c3071 100644 --- a/src/Storages/System/StorageSystemZeros.cpp +++ b/src/Storages/System/StorageSystemZeros.cpp @@ -92,14 +92,14 @@ StorageSystemZeros::StorageSystemZeros(const StorageID & table_id_, bool multith Pipe StorageSystemZeros::read( const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo &, ContextPtr /*context*/, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + storage_snapshot->check(column_names); bool use_multiple_streams = multithreaded; diff --git a/src/Storages/System/StorageSystemZeros.h b/src/Storages/System/StorageSystemZeros.h index f5b2bb43117..bf72352b7be 100644 --- a/src/Storages/System/StorageSystemZeros.h +++ b/src/Storages/System/StorageSystemZeros.h @@ -22,7 +22,7 @@ public: Pipe read( const Names & column_names, - const StorageMetadataPtr & /*metadata_snapshot*/, + const StorageSnapshotPtr & storage_snapshot, SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum processed_stage, diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index f2b2102c7ff..879951df162 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include namespace DB @@ -47,14 +49,23 @@ NamesAndTypesList StorageSystemZooKeeper::getNamesAndTypes() }; } -using Paths = Strings; +/// Type of path to be fetched +enum class ZkPathType +{ + Exact, /// Fetch all nodes under this path + Prefix, /// Fetch all nodes starting with this prefix, recursively (multiple paths may match prefix) + Recurse, /// Fatch all nodes under this path, recursively +}; + +/// List of paths to be feched from zookeeper +using Paths = std::deque>; static String pathCorrected(const String & path) { String path_corrected; /// path should starts with '/', otherwise ZBADARGUMENTS will be thrown in /// ZooKeeper::sendThread and the session will fail. - if (path[0] != '/') + if (path.empty() || path[0] != '/') path_corrected = '/'; path_corrected += path; /// In all cases except the root, path must not end with a slash. @@ -64,7 +75,7 @@ static String pathCorrected(const String & path) } -static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) +static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context, bool allow_unrestricted) { const auto * function = elem.as(); if (!function) @@ -73,7 +84,7 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) if (function->name == "and") { for (const auto & child : function->arguments->children) - if (extractPathImpl(*child, res, context)) + if (extractPathImpl(*child, res, context, allow_unrestricted)) return true; return false; @@ -110,7 +121,7 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) set.checkColumnsNumber(1); const auto & set_column = *set.getSetElements()[0]; for (size_t row = 0; row < set_column.size(); ++row) - res.emplace_back(set_column[row].safeGet()); + res.emplace_back(set_column[row].safeGet(), ZkPathType::Exact); } else { @@ -121,12 +132,12 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) if (String str; literal->value.tryGet(str)) { - res.emplace_back(str); + res.emplace_back(str, ZkPathType::Exact); } else if (Tuple tuple; literal->value.tryGet(tuple)) { for (auto element : tuple) - res.emplace_back(element.safeGet()); + res.emplace_back(element.safeGet(), ZkPathType::Exact); } else return false; @@ -156,7 +167,61 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) if (literal->value.getType() != Field::Types::String) return false; - res.emplace_back(literal->value.safeGet()); + res.emplace_back(literal->value.safeGet(), ZkPathType::Exact); + return true; + } + else if (allow_unrestricted && function->name == "like") + { + const ASTIdentifier * ident; + ASTPtr value; + if ((ident = args.children.at(0)->as())) + value = args.children.at(1); + else if ((ident = args.children.at(1)->as())) + value = args.children.at(0); + else + return false; + + if (ident->name() != "path") + return false; + + auto evaluated = evaluateConstantExpressionAsLiteral(value, context); + const auto * literal = evaluated->as(); + if (!literal) + return false; + + if (literal->value.getType() != Field::Types::String) + return false; + + String pattern = literal->value.safeGet(); + bool has_metasymbol = false; + String prefix; // pattern prefix before the first metasymbol occurrence + for (size_t i = 0; i < pattern.size(); i++) + { + char c = pattern[i]; + // Handle escaping of metasymbols + if (c == '\\' && i + 1 < pattern.size()) + { + char c2 = pattern[i + 1]; + if (c2 == '_' || c2 == '%') + { + prefix.append(1, c2); + i++; // to skip two bytes + continue; + } + } + + // Stop prefix on the first metasymbols occurrence + if (c == '_' || c == '%') + { + has_metasymbol = true; + break; + } + + prefix.append(1, c); + } + + res.emplace_back(prefix, has_metasymbol ? ZkPathType::Prefix : ZkPathType::Exact); + return true; } @@ -166,39 +231,60 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) /** Retrieve from the query a condition of the form `path = 'path'`, from conjunctions in the WHERE clause. */ -static Paths extractPath(const ASTPtr & query, ContextPtr context) +static Paths extractPath(const ASTPtr & query, ContextPtr context, bool allow_unrestricted) { const auto & select = query->as(); if (!select.where()) - return Paths(); + return allow_unrestricted ? Paths{{"/", ZkPathType::Recurse}} : Paths(); Paths res; - return extractPathImpl(*select.where(), res, context) ? res : Paths(); + return extractPathImpl(*select.where(), res, context, allow_unrestricted) ? res : Paths(); } void StorageSystemZooKeeper::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const { - const Paths & paths = extractPath(query_info.query, context); - if (paths.empty()) - throw Exception("SELECT from system.zookeeper table must contain condition like path = 'path' or path IN ('path1','path2'...) or path IN (subquery) in WHERE clause.", ErrorCodes::BAD_ARGUMENTS); + Paths paths = extractPath(query_info.query, context, context->getSettingsRef().allow_unrestricted_reads_from_keeper); zkutil::ZooKeeperPtr zookeeper = context->getZooKeeper(); - std::unordered_set paths_corrected; - for (const auto & path : paths) - { - const String & path_corrected = pathCorrected(path); - auto [it, inserted] = paths_corrected.emplace(path_corrected); - if (!inserted) /// Do not repeat processing. - continue; + if (paths.empty()) + throw Exception("SELECT from system.zookeeper table must contain condition like path = 'path' or path IN ('path1','path2'...) or path IN (subquery) in WHERE clause unless `set allow_unrestricted_reads_from_keeper = 'true'`.", ErrorCodes::BAD_ARGUMENTS); - zkutil::Strings nodes = zookeeper->getChildren(path_corrected); + std::unordered_set added; + while (!paths.empty()) + { + auto [path, path_type] = std::move(paths.front()); + paths.pop_front(); + + String prefix; + if (path_type == ZkPathType::Prefix) + { + prefix = path; + size_t last_slash = prefix.rfind('/'); + path = prefix.substr(0, last_slash == String::npos ? 0 : last_slash); + } + + String path_corrected = pathCorrected(path); + + /// Node can be deleted concurrently. It's Ok, we don't provide any + /// consistency guarantees for system.zookeeper table. + zkutil::Strings nodes; + zookeeper->tryGetChildren(path_corrected, nodes); String path_part = path_corrected; if (path_part == "/") path_part.clear(); + if (!prefix.empty()) + { + // Remove nodes that do not match specified prefix + nodes.erase(std::remove_if(nodes.begin(), nodes.end(), [&prefix, &path_part] (const String & node) + { + return (path_part + '/' + node).substr(0, prefix.size()) != prefix; + }), nodes.end()); + } + std::vector> futures; futures.reserve(nodes.size()); for (const String & node : nodes) @@ -210,6 +296,11 @@ void StorageSystemZooKeeper::fillData(MutableColumns & res_columns, ContextPtr c if (res.error == Coordination::Error::ZNONODE) continue; /// Node was deleted meanwhile. + // Deduplication + String key = path_part + '/' + nodes[i]; + if (auto [it, inserted] = added.emplace(key); !inserted) + continue; + const Coordination::Stat & stat = res.stat; size_t col_num = 0; @@ -228,6 +319,11 @@ void StorageSystemZooKeeper::fillData(MutableColumns & res_columns, ContextPtr c res_columns[col_num++]->insert(stat.pzxid); res_columns[col_num++]->insert( path); /// This is the original path. In order to process the request, condition in WHERE should be triggered. + + if (path_type != ZkPathType::Exact && res.stat.numChildren > 0) + { + paths.emplace_back(key, ZkPathType::Recurse); + } } } } diff --git a/src/Storages/System/attachInformationSchemaTables.cpp b/src/Storages/System/attachInformationSchemaTables.cpp index 68a1eac305e..61a91685324 100644 --- a/src/Storages/System/attachInformationSchemaTables.cpp +++ b/src/Storages/System/attachInformationSchemaTables.cpp @@ -32,6 +32,8 @@ static void createInformationSchemaView(ContextMutablePtr context, IDatabase & d auto & ast_create = ast->as(); assert(view_name == ast_create.getTable()); + ast_create.attach = false; + ast_create.setDatabase(database.getDatabaseName()); if (is_uppercase) ast_create.setTable(Poco::toUpper(view_name)); diff --git a/src/Storages/System/attachSystemTablesImpl.h b/src/Storages/System/attachSystemTablesImpl.h index 4f83a0a4fda..b6080d15f2c 100644 --- a/src/Storages/System/attachSystemTablesImpl.h +++ b/src/Storages/System/attachSystemTablesImpl.h @@ -12,13 +12,13 @@ void attach(ContextPtr context, IDatabase & system_database, const String & tabl assert(system_database.getDatabaseName() == DatabaseCatalog::SYSTEM_DATABASE); if (system_database.getUUID() == UUIDHelpers::Nil) { - /// Attach to Ordinary database + /// Attach to Ordinary database. auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name); system_database.attachTable(context, table_name, StorageT::create(table_id, std::forward(args)...)); } else { - /// Attach to Atomic database + /// Attach to Atomic database. /// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages /// and path is actually not used auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4()); diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 69303264482..ccf924f2827 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -112,6 +112,7 @@ TTLDescription::TTLDescription(const TTLDescription & other) , aggregate_descriptions(other.aggregate_descriptions) , destination_type(other.destination_type) , destination_name(other.destination_name) + , if_exists(other.if_exists) , recompression_codec(other.recompression_codec) { if (other.expression) @@ -149,6 +150,7 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) aggregate_descriptions = other.aggregate_descriptions; destination_type = other.destination_type; destination_name = other.destination_name; + if_exists = other.if_exists; if (other.recompression_codec) recompression_codec = other.recompression_codec->clone(); @@ -185,9 +187,10 @@ TTLDescription TTLDescription::getTTLFromAST( } else /// rows TTL { + result.mode = ttl_element->mode; result.destination_type = ttl_element->destination_type; result.destination_name = ttl_element->destination_name; - result.mode = ttl_element->mode; + result.if_exists = ttl_element->if_exists; if (ttl_element->mode == TTLMode::DELETE) { diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 17020392013..8f60eb604b5 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -75,6 +75,10 @@ struct TTLDescription /// Name of destination disk or volume String destination_name; + /// If true, do nothing if DISK or VOLUME doesn't exist . + /// Only valid for table MOVE TTLs. + bool if_exists = false; + /// Codec name which will be used to recompress data ASTPtr recompression_codec; diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 37c913f58a9..644ab5d57c2 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -57,6 +57,7 @@ namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; extern const int BAD_ARGUMENTS; + extern const int SYNTAX_ERROR; extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int INCORRECT_QUERY; @@ -262,7 +263,13 @@ namespace IntervalKind strToIntervalKind(const String& interval_str) { - if (interval_str == "Second") + if (interval_str == "Nanosecond") + return IntervalKind::Nanosecond; + else if (interval_str == "Microsecond") + return IntervalKind::Microsecond; + else if (interval_str == "Millisecond") + return IntervalKind::Millisecond; + else if (interval_str == "Second") return IntervalKind::Second; else if (interval_str == "Minute") return IntervalKind::Minute; @@ -307,6 +314,12 @@ namespace { switch (kind) { + case IntervalKind::Nanosecond: + throw Exception("Fractional seconds are not supported by windows yet", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Microsecond: + throw Exception("Fractional seconds are not supported by windows yet", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Millisecond: + throw Exception("Fractional seconds are not supported by windows yet", ErrorCodes::SYNTAX_ERROR); #define CASE_WINDOW_KIND(KIND) \ case IntervalKind::KIND: { \ return AddTime::execute(time_sec, num_units, time_zone); \ @@ -639,10 +652,43 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( "The first argument of time window function should not be a constant value.", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW); + ToIdentifierMatcher::Data query_data; + query_data.window_id_name = window_id_name; + query_data.window_id_alias = window_id_alias; + ToIdentifierMatcher::Visitor to_identifier_visitor(query_data); + + ReplaceFunctionNowData time_now_data; + ReplaceFunctionNowVisitor time_now_visitor(time_now_data); + ReplaceFunctionWindowMatcher::Data func_hop_data; + ReplaceFunctionWindowMatcher::Visitor func_window_visitor(func_hop_data); + + DropTableIdentifierMatcher::Data drop_table_identifier_data; + DropTableIdentifierMatcher::Visitor drop_table_identifier_visitor(drop_table_identifier_data); + + auto visit = [&](const IAST * ast) + { + auto node = ast->clone(); + QueryNormalizer(normalizer_data).visit(node); + /// now() -> ____timestamp + if (is_time_column_func_now) + { + time_now_visitor.visit(node); + function_now_timezone = time_now_data.now_timezone; + } + drop_table_identifier_visitor.visit(node); + /// tumble/hop -> windowID + func_window_visitor.visit(node); + to_identifier_visitor.visit(node); + node->setAlias(""); + return node; + }; + auto new_storage = std::make_shared(); /// storage != nullptr in case create window view with ENGINE syntax if (storage) { + new_storage->set(new_storage->engine, storage->engine->clone()); + if (storage->ttl_table) throw Exception( ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW, @@ -654,46 +700,14 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( "The ENGINE of WindowView must be MergeTree family of table engines " "including the engines with replication support"); - ToIdentifierMatcher::Data query_data; - query_data.window_id_name = window_id_name; - query_data.window_id_alias = window_id_alias; - ToIdentifierMatcher::Visitor to_identifier_visitor(query_data); - - ReplaceFunctionNowData time_now_data; - ReplaceFunctionNowVisitor time_now_visitor(time_now_data); - ReplaceFunctionWindowMatcher::Data func_hop_data; - ReplaceFunctionWindowMatcher::Visitor func_window_visitor(func_hop_data); - - DropTableIdentifierMatcher::Data drop_table_identifier_data; - DropTableIdentifierMatcher::Visitor drop_table_identifier_visitor(drop_table_identifier_data); - - new_storage->set(new_storage->engine, storage->engine->clone()); - - auto visit = [&](const IAST * ast, IAST *& field) - { - if (ast) - { - auto node = ast->clone(); - QueryNormalizer(normalizer_data).visit(node); - /// now() -> ____timestamp - if (is_time_column_func_now) - { - time_now_visitor.visit(node); - function_now_timezone = time_now_data.now_timezone; - } - drop_table_identifier_visitor.visit(node); - /// tumble/hop -> windowID - func_window_visitor.visit(node); - to_identifier_visitor.visit(node); - node->setAlias(""); - new_storage->set(field, node); - } - }; - - visit(storage->partition_by, new_storage->partition_by); - visit(storage->primary_key, new_storage->primary_key); - visit(storage->order_by, new_storage->order_by); - visit(storage->sample_by, new_storage->sample_by); + if (storage->partition_by) + new_storage->set(new_storage->partition_by, visit(storage->partition_by)); + if (storage->primary_key) + new_storage->set(new_storage->primary_key, visit(storage->primary_key)); + if (storage->order_by) + new_storage->set(new_storage->order_by, visit(storage->order_by)); + if (storage->sample_by) + new_storage->set(new_storage->sample_by, visit(storage->sample_by)); if (storage->settings) new_storage->set(new_storage->settings, storage->settings->clone()); @@ -702,8 +716,21 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( { new_storage->set(new_storage->engine, makeASTFunction("AggregatingMergeTree")); - new_storage->set(new_storage->order_by, std::make_shared(window_id_column_name)); - new_storage->set(new_storage->primary_key, std::make_shared(window_id_column_name)); + if (inner_select_query->groupBy()->children.size() == 1) //GROUP BY windowID + { + auto node = visit(inner_select_query->groupBy()->children[0].get()); + new_storage->set(new_storage->order_by, std::make_shared(node->getColumnName())); + } + else + { + auto group_by_function = makeASTFunction("tuple"); + for (auto & child : inner_select_query->groupBy()->children) + { + auto node = visit(child.get()); + group_by_function->arguments->children.push_back(std::make_shared(node->getColumnName())); + } + new_storage->set(new_storage->order_by, group_by_function); + } } auto new_columns = std::make_shared(); @@ -724,6 +751,12 @@ UInt32 StorageWindowView::getWindowLowerBound(UInt32 time_sec) switch (window_interval_kind) { + case IntervalKind::Nanosecond: + throw Exception("Fractional seconds are not supported by windows yet", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Microsecond: + throw Exception("Fractional seconds are not supported by windows yet", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Millisecond: + throw Exception("Fractional seconds are not supported by windows yet", ErrorCodes::SYNTAX_ERROR); #define CASE_WINDOW_KIND(KIND) \ case IntervalKind::KIND: \ { \ @@ -759,6 +792,13 @@ UInt32 StorageWindowView::getWindowUpperBound(UInt32 time_sec) switch (window_interval_kind) { + case IntervalKind::Nanosecond: + throw Exception("Fractional seconds are not supported by window view yet", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Microsecond: + throw Exception("Fractional seconds are not supported by window view yet", ErrorCodes::SYNTAX_ERROR); + case IntervalKind::Millisecond: + throw Exception("Fractional seconds are not supported by window view yet", ErrorCodes::SYNTAX_ERROR); + #define CASE_WINDOW_KIND(KIND) \ case IntervalKind::KIND: \ { \ diff --git a/src/Storages/examples/active_parts.py b/src/Storages/examples/active_parts.py index a818a76017d..d82c5ca96bf 100644 --- a/src/Storages/examples/active_parts.py +++ b/src/Storages/examples/active_parts.py @@ -9,7 +9,9 @@ import re parts = {} for s in sys.stdin.read().split(): - m = re.match('^([0-9]{6})[0-9]{2}_([0-9]{6})[0-9]{2}_([0-9]+)_([0-9]+)_([0-9]+)$', s) + m = re.match( + "^([0-9]{6})[0-9]{2}_([0-9]{6})[0-9]{2}_([0-9]+)_([0-9]+)_([0-9]+)$", s + ) if m == None: continue m1 = m.group(1) @@ -18,7 +20,7 @@ for s in sys.stdin.read().split(): i2 = int(m.group(4)) l = int(m.group(5)) if m1 != m2: - raise Exception('not in single month: ' + s) + raise Exception("not in single month: " + s) if m1 not in parts: parts[m1] = [] parts[m1].append((i1, i2, l, s)) @@ -27,13 +29,13 @@ for m, ps in sorted(parts.items()): ps.sort(key=lambda i1_i2_l_s: (i1_i2_l_s[0], -i1_i2_l_s[1], -i1_i2_l_s[2])) (x2, y2, l2, s2) = (-1, -1, -1, -1) for x1, y1, l1, s1 in ps: - if x1 >= x2 and y1 <= y2 and l1 < l2 and (x1, y1) != (x2, y2): # 2 contains 1 + if x1 >= x2 and y1 <= y2 and l1 < l2 and (x1, y1) != (x2, y2): # 2 contains 1 pass - elif x1 > y2: # 1 is to the right of 2 + elif x1 > y2: # 1 is to the right of 2 if x1 != y2 + 1 and y2 != -1: - print() # to see the missing numbers + print() # to see the missing numbers (x2, y2, l2, s2) = (x1, y1, l1, s1) print(s1) else: - raise Exception('invalid parts intersection: ' + s1 + ' and ' + s2) + raise Exception("invalid parts intersection: " + s1 + " and " + s2) print() diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index 532abb8e2f3..8fa4d02e8e1 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -58,7 +58,6 @@ ColumnsDescription getStructureOfRemoteTableInShard( } ColumnsDescription res; - auto new_context = ClusterProxy::updateSettingsForCluster(cluster, context, context->getSettingsRef()); /// Expect only needed columns from the result of DESC TABLE. NOTE 'comment' column is ignored for compatibility reasons. @@ -150,4 +149,69 @@ ColumnsDescription getStructureOfRemoteTable( ErrorCodes::NO_REMOTE_SHARD_AVAILABLE); } +ColumnsDescriptionByShardNum getExtendedObjectsOfRemoteTables( + const Cluster & cluster, + const StorageID & remote_table_id, + const ColumnsDescription & storage_columns, + ContextPtr context) +{ + const auto & shards_info = cluster.getShardsInfo(); + auto query = "DESC TABLE " + remote_table_id.getFullTableName(); + + auto new_context = ClusterProxy::updateSettingsForCluster(cluster, context, context->getSettingsRef()); + new_context->setSetting("describe_extend_object_types", true); + + /// Expect only needed columns from the result of DESC TABLE. + Block sample_block + { + { ColumnString::create(), std::make_shared(), "name" }, + { ColumnString::create(), std::make_shared(), "type" }, + }; + + auto execute_query_on_shard = [&](const auto & shard_info) + { + /// Execute remote query without restrictions (because it's not real user query, but part of implementation) + RemoteQueryExecutor executor(shard_info.pool, query, sample_block, new_context); + + executor.setPoolMode(PoolMode::GET_ONE); + executor.setMainTable(remote_table_id); + + ColumnsDescription res; + while (auto block = executor.read()) + { + const auto & name_col = *block.getByName("name").column; + const auto & type_col = *block.getByName("type").column; + + size_t size = name_col.size(); + for (size_t i = 0; i < size; ++i) + { + auto name = get(name_col[i]); + auto type_name = get(type_col[i]); + + auto storage_column = storage_columns.tryGetPhysical(name); + if (storage_column && isObject(storage_column->type)) + res.add(ColumnDescription(std::move(name), DataTypeFactory::instance().get(type_name))); + } + } + + return res; + }; + + ColumnsDescriptionByShardNum columns; + for (const auto & shard_info : shards_info) + { + auto res = execute_query_on_shard(shard_info); + + /// Expect at least some columns. + /// This is a hack to handle the empty block case returned by Connection when skip_unavailable_shards is set. + if (!res.empty()) + columns.emplace(shard_info.shard_num, std::move(res)); + } + + if (columns.empty()) + throw NetException("All attempts to get table structure failed", ErrorCodes::NO_REMOTE_SHARD_AVAILABLE); + + return columns; +} + } diff --git a/src/Storages/getStructureOfRemoteTable.h b/src/Storages/getStructureOfRemoteTable.h index 3f77236c756..62f93dccf1a 100644 --- a/src/Storages/getStructureOfRemoteTable.h +++ b/src/Storages/getStructureOfRemoteTable.h @@ -8,6 +8,7 @@ namespace DB { + class Context; struct StorageID; @@ -19,4 +20,14 @@ ColumnsDescription getStructureOfRemoteTable( ContextPtr context, const ASTPtr & table_func_ptr = nullptr); + +using ColumnsDescriptionByShardNum = std::unordered_map; + +/// Returns descriptions of columns of type Object for each shard. +ColumnsDescriptionByShardNum getExtendedObjectsOfRemoteTables( + const Cluster & cluster, + const StorageID & remote_table_id, + const ColumnsDescription & storage_columns, + ContextPtr context); + } diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index a48b764b62c..4cda9d6c9f5 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -117,15 +117,16 @@ std::string readData(DB::StoragePtr & table, const DB::ContextPtr context) { using namespace DB; auto metadata_snapshot = table->getInMemoryMetadataPtr(); + auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot); Names column_names; column_names.push_back("a"); SelectQueryInfo query_info; QueryProcessingStage::Enum stage = table->getQueryProcessingStage( - context, QueryProcessingStage::Complete, metadata_snapshot, query_info); + context, QueryProcessingStage::Complete, storage_snapshot, query_info); - QueryPipeline pipeline(table->read(column_names, metadata_snapshot, query_info, context, stage, 8192, 1)); + QueryPipeline pipeline(table->read(column_names, storage_snapshot, query_info, context, stage, 8192, 1)); Block sample; { diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 576d1ea23ff..c58f93e310a 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -1,9 +1,21 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_table_functions .) +if (TARGET ch_contrib::hivemetastore) + add_headers_and_sources(clickhouse_table_functions Hive) +endif () -list(REMOVE_ITEM clickhouse_table_functions_sources ITableFunction.cpp TableFunctionFactory.cpp) -list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFunctionFactory.h) +list(REMOVE_ITEM clickhouse_table_functions_sources + ITableFunction.cpp + TableFunctionView.cpp + TableFunctionFactory.cpp) +list(REMOVE_ITEM clickhouse_table_functions_headers + ITableFunction.h + TableFunctionView.h + TableFunctionFactory.h) add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) -target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) +target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) +if (TARGET ch_contrib::hivemetastore) + target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::hivemetastore ch_contrib::hdfs) +endif () diff --git a/src/TableFunctions/Hive/TableFunctionHive.cpp b/src/TableFunctions/Hive/TableFunctionHive.cpp new file mode 100644 index 00000000000..e7de55181c3 --- /dev/null +++ b/src/TableFunctions/Hive/TableFunctionHive.cpp @@ -0,0 +1,91 @@ +#include +#if USE_HIVE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + namespace ErrorCodes + { + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + } + + void TableFunctionHive::parseArguments(const ASTPtr & ast_function_, ContextPtr context_) + { + ASTs & args_func = ast_function_->children; + if (args_func.size() != 1) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments.", getName()); + + ASTs & args = args_func.at(0)->children; + + const auto message = fmt::format( + "The signature of function {} is:\n" + " - hive_url, hive_database, hive_table, structure, partition_by_keys", + getName()); + + if (args.size() != 5) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, message); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context_); + + hive_metastore_url = args[0]->as().value.safeGet(); + hive_database = args[1]->as().value.safeGet(); + hive_table = args[2]->as().value.safeGet(); + table_structure = args[3]->as().value.safeGet(); + partition_by_def = args[4]->as().value.safeGet(); + + actual_columns = parseColumnsListFromString(table_structure, context_); + } + + ColumnsDescription TableFunctionHive::getActualTableStructure(ContextPtr /*context_*/) const { return actual_columns; } + + StoragePtr TableFunctionHive::executeImpl( + const ASTPtr & /*ast_function_*/, + ContextPtr context_, + const std::string & table_name_, + ColumnsDescription /*cached_columns_*/) const + { + const Settings & settings = context_->getSettings(); + ParserLambdaExpression partition_by_parser; + ASTPtr partition_by_ast = parseQuery( + partition_by_parser, + "(" + partition_by_def + ")", + "partition by declaration list", + settings.max_query_size, + settings.max_parser_depth); + StoragePtr storage; + storage = StorageHive::create( + hive_metastore_url, + hive_database, + hive_table, + StorageID(getDatabaseName(), table_name_), + actual_columns, + ConstraintsDescription{}, + "", + partition_by_ast, + std::make_unique(), + context_); + + return storage; + } + + + void registerTableFunctionHive(TableFunctionFactory & factory_) { factory_.registerFunction(); } + +} +#endif diff --git a/src/TableFunctions/Hive/TableFunctionHive.h b/src/TableFunctions/Hive/TableFunctionHive.h new file mode 100644 index 00000000000..0973bdda329 --- /dev/null +++ b/src/TableFunctions/Hive/TableFunctionHive.h @@ -0,0 +1,38 @@ +#pragma once +#include +#if USE_HIVE +#include +#include +namespace DB +{ +class Context; +class TableFunctionHive : public ITableFunction +{ +public: + static constexpr auto name = "hive"; + static constexpr auto storage_type_name = "hive"; + std::string getName() const override { return name; } + + bool hasStaticStructure() const override { return true; } + + StoragePtr executeImpl( + const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; + + const char * getStorageTypeName() const override { return storage_type_name; } + ColumnsDescription getActualTableStructure(ContextPtr) const override; + void parseArguments(const ASTPtr & ast_function_, ContextPtr context_) override; + +private: + Poco::Logger * logger = &Poco::Logger::get("TableFunctionHive"); + + String cluster_name; + String hive_metastore_url; + String hive_database; + String hive_table; + String table_structure; + String partition_by_def; + + ColumnsDescription actual_columns; +}; +} +#endif diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index 93cf5057e88..9c8d694865b 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -52,6 +52,16 @@ public: /// Returns actual table structure probably requested from remote server, may fail virtual ColumnsDescription getActualTableStructure(ContextPtr /*context*/) const = 0; + /// Check if table function needs a structure hint from SELECT query in case of + /// INSERT INTO FUNCTION ... SELECT ... + /// It's used for schema inference. + virtual bool needStructureHint() const { return false; } + + /// Set a structure hint from SELECT query in case of + /// INSERT INTO FUNCTION ... SELECT ... + /// This hint could be used not to repeat schema in function arguments. + virtual void setStructureHint(const ColumnsDescription &) {} + /// Create storage according to the query. StoragePtr execute(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns_ = {}, bool use_global_context = false) const; diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 5328abd1654..3388a7ec9f6 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -95,6 +95,9 @@ StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, ColumnsDescription columns; if (structure != "auto") columns = parseColumnsListFromString(structure, context); + else if (!structure_hint.empty()) + columns = structure_hint; + StoragePtr storage = getStorage(filename, format, columns, context, table_name, compression_method); storage->startup(); return storage; diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 6e00aac9c37..cd85f20fdc0 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -12,6 +12,10 @@ class Context; */ class ITableFunctionFileLike : public ITableFunction { +public: + bool needStructureHint() const override { return structure == "auto"; } + + void setStructureHint(const ColumnsDescription & structure_hint_) override { structure_hint = structure_hint_; } protected: void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; @@ -20,6 +24,7 @@ protected: String format = "auto"; String structure = "auto"; String compression_method = "auto"; + ColumnsDescription structure_hint; private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; diff --git a/src/TableFunctions/TableFunctionExecutable.cpp b/src/TableFunctions/TableFunctionExecutable.cpp index 41ba2db5c33..18d7d8867e8 100644 --- a/src/TableFunctions/TableFunctionExecutable.cpp +++ b/src/TableFunctions/TableFunctionExecutable.cpp @@ -78,7 +78,7 @@ StoragePtr TableFunctionExecutable::executeImpl(const ASTPtr & /*ast_function*/, auto global_context = context->getGlobalContext(); ExecutableSettings settings; settings.script_name = script_name; - settings.script_arguments = std::move(arguments); + settings.script_arguments = arguments; auto storage = StorageExecutable::create(storage_id, format, settings, input_queries, getActualTableStructure(context), ConstraintsDescription{}); storage->startup(); diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 192846f7f11..b09bb8b6ae1 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -41,6 +41,7 @@ ColumnsDescription TableFunctionFile::getActualTableStructure(ContextPtr context return StorageFile::getTableStructureFromFile(format, paths, compression_method, std::nullopt, context); } + return parseColumnsListFromString(structure, context); } diff --git a/src/TableFunctions/TableFunctionRemote.h b/src/TableFunctions/TableFunctionRemote.h index 976397ddc45..6f28f1ec9de 100644 --- a/src/TableFunctions/TableFunctionRemote.h +++ b/src/TableFunctions/TableFunctionRemote.h @@ -10,10 +10,10 @@ namespace DB /* remote ('address', db, table) - creates a temporary StorageDistributed. * To get the table structure, a DESC TABLE request is made to the remote server. - * For example + * For example: * SELECT count() FROM remote('example01-01-1', merge, hits) - go to `example01-01-1`, in the merge database, the hits table. * An expression that generates a set of shards and replicas can also be specified as the host name - see below. - * Also, there is a cluster version of the function: cluster('existing_cluster_name', 'db', 'table') + * Also, there is a cluster version of the function: cluster('existing_cluster_name', 'db', 'table'). */ class TableFunctionRemote : public ITableFunction { diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index f91ce36c3c4..f844772983a 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -32,6 +32,7 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con " - url\n" " - url, format\n" \ " - url, format, structure\n" \ + " - url, access_key_id, secret_access_key\n" \ " - url, format, structure, compression_method\n" \ " - url, access_key_id, secret_access_key, format\n" " - url, access_key_id, secret_access_key, format, structure\n" \ @@ -75,7 +76,6 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con { {1, {{}}}, {2, {{"format", 1}}}, - {3, {{"format", 1}, {"structure", 2}}}, {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}} }; @@ -83,14 +83,26 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con std::map args_to_idx; /// For 4 arguments we support 2 possible variants: /// s3(source, format, structure, compression_method) and s3(source, access_key_id, access_key_id, format) - /// We can distinguish them by looking at the 4-th argument: check if it's a format name or not. + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. if (args.size() == 4) { - auto last_arg = args[3]->as().value.safeGet(); - if (FormatFactory::instance().getAllFormats().contains(last_arg)) - args_to_idx = {{"access_key_id", 1}, {"access_key_id", 2}, {"format", 3}}; - else + auto second_arg = args[1]->as().value.safeGet(); + if (FormatFactory::instance().getAllFormats().contains(second_arg)) args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; + + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; + } + /// For 3 arguments we support 2 possible variants: + /// s3(source, format, structure) and s3(source, access_key_id, access_key_id) + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. + else if (args.size() == 3) + { + auto second_arg = args[1]->as().value.safeGet(); + if (FormatFactory::instance().getAllFormats().contains(second_arg)) + args_to_idx = {{"format", 1}, {"structure", 2}}; + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; } else { @@ -156,6 +168,8 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context ColumnsDescription columns; if (s3_configuration->structure != "auto") columns = parseColumnsListFromString(s3_configuration->structure, context); + else if (!structure_hint.empty()) + columns = structure_hint; StoragePtr storage = StorageS3::create( s3_uri, @@ -169,7 +183,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size, max_connections, - getActualTableStructure(context), + columns, ConstraintsDescription{}, String{}, context, diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 374e653072e..06a327593b0 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -13,7 +13,7 @@ namespace DB class Context; -/* s3(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary storage for a file in S3 +/* s3(source, [access_key_id, secret_access_key,] format, structure[, compression]) - creates a temporary storage for a file in S3. */ class TableFunctionS3 : public ITableFunction { @@ -25,6 +25,10 @@ public: } bool hasStaticStructure() const override { return s3_configuration->structure != "auto"; } + bool needStructureHint() const override { return s3_configuration->structure == "auto"; } + + void setStructureHint(const ColumnsDescription & structure_hint_) override { structure_hint = structure_hint_; } + protected: StoragePtr executeImpl( const ASTPtr & ast_function, @@ -38,6 +42,7 @@ protected: void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; std::optional s3_configuration; + ColumnsDescription structure_hint; }; class TableFunctionCOS : public TableFunctionS3 diff --git a/src/TableFunctions/TableFunctionS3Cluster.h b/src/TableFunctions/TableFunctionS3Cluster.h index cc857725ce6..35d18631ae1 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.h +++ b/src/TableFunctions/TableFunctionS3Cluster.h @@ -13,7 +13,7 @@ namespace DB class Context; /** - * s3Cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure) + * s3cluster(cluster_name, source, [access_key_id, secret_access_key,] format, structure) * A table function, which allows to process many files from S3 on a specific cluster * On initiator it creates a connection to _all_ nodes in cluster, discloses asterics * in S3 file path and dispatch each file dynamically. diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index 798a37dc478..35483b1a04a 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -10,7 +10,7 @@ namespace DB class Context; -/* url(source, format[, structure, compression]) - creates a temporary storage from url +/* url(source, format[, structure, compression]) - creates a temporary storage from url. */ class TableFunctionURL : public ITableFunctionFileLike { diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index 07019d26067..595e8f9cf41 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -109,7 +109,7 @@ void TableFunctionValues::parseArguments(const ASTPtr & ast_function, ContextPtr "Cannot determine common structure for {} function arguments: the amount of columns is differ for different arguments", getName()); for (size_t j = 0; j != arg_types.size(); ++j) - data_types[j] = getLeastSupertype({data_types[j], arg_types[j]}); + data_types[j] = getLeastSupertype(DataTypes{data_types[j], arg_types[j]}); } NamesAndTypesList names_and_types; diff --git a/src/TableFunctions/TableFunctionValues.h b/src/TableFunctions/TableFunctionValues.h index f01bcf6e20e..61ce5158086 100644 --- a/src/TableFunctions/TableFunctionValues.h +++ b/src/TableFunctions/TableFunctionValues.h @@ -5,7 +5,7 @@ namespace DB { /* values(structure, values...) - creates a temporary storage filling columns with values - * values is case-insensitive table function + * values is case-insensitive table function. */ class TableFunctionValues : public ITableFunction { diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 2cab8aeca25..e9fcbb219a3 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -15,6 +15,12 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } + +const ASTSelectWithUnionQuery & TableFunctionView::getSelectQuery() const +{ + return *create.select; +} + void TableFunctionView::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/) { const auto * function = ast_function->as(); diff --git a/src/TableFunctions/TableFunctionView.h b/src/TableFunctions/TableFunctionView.h index c20b45e7546..4afb049e738 100644 --- a/src/TableFunctions/TableFunctionView.h +++ b/src/TableFunctions/TableFunctionView.h @@ -16,6 +16,9 @@ class TableFunctionView : public ITableFunction public: static constexpr auto name = "view"; std::string getName() const override { return name; } + + const ASTSelectWithUnionQuery & getSelectQuery() const; + private: StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const String & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "View"; } diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index ed08972e74d..9e09fac665a 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -31,6 +31,10 @@ void registerTableFunctions() registerTableFunctionHDFSCluster(factory); #endif +#if USE_HIVE + registerTableFunctionHive(factory); +#endif + registerTableFunctionODBC(factory); registerTableFunctionJDBC(factory); diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 72ca185f656..e39d21cb580 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -29,6 +29,10 @@ void registerTableFunctionHDFS(TableFunctionFactory & factory); void registerTableFunctionHDFSCluster(TableFunctionFactory & factory); #endif +#if USE_HIVE +void registerTableFunctionHive(TableFunctionFactory & factory); +#endif + void registerTableFunctionODBC(TableFunctionFactory & factory); void registerTableFunctionJDBC(TableFunctionFactory & factory); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c9858910837..22c89aaafa7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,7 +7,7 @@ else () include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) endif () -option (ENABLE_CLICKHOUSE_TEST "Install clickhouse-test script and relevant tests scenarios" ON) +option (ENABLE_CLICKHOUSE_TEST "Install clickhouse-test script and relevant tests scenarios" OFF) if (ENABLE_CLICKHOUSE_TEST) install (PROGRAMS clickhouse-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 319a6fc3fa5..c330d1c725b 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -7,8 +7,14 @@ import sys from github import Github -from env_helper import GITHUB_REPOSITORY, TEMP_PATH, REPO_COPY, REPORTS_PATH, GITHUB_SERVER_URL, \ - GITHUB_RUN_ID +from env_helper import ( + GITHUB_REPOSITORY, + TEMP_PATH, + REPO_COPY, + REPORTS_PATH, + GITHUB_SERVER_URL, + GITHUB_RUN_ID, +) from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo @@ -19,19 +25,24 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickh from stopwatch import Stopwatch from rerun_helper import RerunHelper -IMAGE_NAME = 'clickhouse/fuzzer' +IMAGE_NAME = "clickhouse/fuzzer" + def get_run_command(pr_number, sha, download_url, workspace_path, image): - return f'docker run --network=host --volume={workspace_path}:/workspace ' \ - '--cap-add syslog --cap-add sys_admin --cap-add=SYS_PTRACE ' \ - f'-e PR_TO_TEST={pr_number} -e SHA_TO_TEST={sha} -e BINARY_URL_TO_DOWNLOAD="{download_url}" '\ - f'{image}' + return ( + f"docker run --network=host --volume={workspace_path}:/workspace " + "--cap-add syslog --cap-add sys_admin --cap-add=SYS_PTRACE " + f'-e PR_TO_TEST={pr_number} -e SHA_TO_TEST={sha} -e BINARY_URL_TO_DOWNLOAD="{download_url}" ' + f"{image}" + ) + def get_commit(gh, commit_sha): repo = gh.get_repo(GITHUB_REPOSITORY) commit = repo.get_commit(commit_sha) return commit + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -64,7 +75,7 @@ if __name__ == "__main__": raise Exception("No build URLs found") for url in urls: - if url.endswith('/clickhouse'): + if url.endswith("/clickhouse"): build_url = url break else: @@ -72,16 +83,20 @@ if __name__ == "__main__": logging.info("Got build url %s", build_url) - workspace_path = os.path.join(temp_path, 'workspace') + workspace_path = os.path.join(temp_path, "workspace") if not os.path.exists(workspace_path): os.makedirs(workspace_path) - run_command = get_run_command(pr_info.number, pr_info.sha, build_url, workspace_path, docker_image) + run_command = get_run_command( + pr_info.number, pr_info.sha, build_url, workspace_path, docker_image + ) logging.info("Going to run %s", run_command) run_log_path = os.path.join(temp_path, "runlog.log") - with open(run_log_path, 'w', encoding='utf-8') as log: - with subprocess.Popen(run_command, shell=True, stderr=log, stdout=log) as process: + with open(run_log_path, "w", encoding="utf-8") as log: + with subprocess.Popen( + run_command, shell=True, stderr=log, stdout=log + ) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") @@ -90,56 +105,70 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - check_name_lower = check_name.lower().replace('(', '').replace(')', '').replace(' ', '') - s3_prefix = f'{pr_info.number}/{pr_info.sha}/fuzzer_{check_name_lower}/' + check_name_lower = ( + check_name.lower().replace("(", "").replace(")", "").replace(" ", "") + ) + s3_prefix = f"{pr_info.number}/{pr_info.sha}/fuzzer_{check_name_lower}/" paths = { - 'runlog.log': run_log_path, - 'main.log': os.path.join(workspace_path, 'main.log'), - 'server.log': os.path.join(workspace_path, 'server.log'), - 'fuzzer.log': os.path.join(workspace_path, 'fuzzer.log'), - 'report.html': os.path.join(workspace_path, 'report.html'), - 'core.gz': os.path.join(workspace_path, 'core.gz'), + "runlog.log": run_log_path, + "main.log": os.path.join(workspace_path, "main.log"), + "server.log": os.path.join(workspace_path, "server.log"), + "fuzzer.log": os.path.join(workspace_path, "fuzzer.log"), + "report.html": os.path.join(workspace_path, "report.html"), + "core.gz": os.path.join(workspace_path, "core.gz"), } - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") for f in paths: try: - paths[f] = s3_helper.upload_test_report_to_s3(paths[f], s3_prefix + '/' + f) + paths[f] = s3_helper.upload_test_report_to_s3(paths[f], s3_prefix + "/" + f) except Exception as ex: logging.info("Exception uploading file %s text %s", f, ex) - paths[f] = '' + paths[f] = "" report_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" - if paths['runlog.log']: - report_url = paths['runlog.log'] - if paths['main.log']: - report_url = paths['main.log'] - if paths['server.log']: - report_url = paths['server.log'] - if paths['fuzzer.log']: - report_url = paths['fuzzer.log'] - if paths['report.html']: - report_url = paths['report.html'] + if paths["runlog.log"]: + report_url = paths["runlog.log"] + if paths["main.log"]: + report_url = paths["main.log"] + if paths["server.log"]: + report_url = paths["server.log"] + if paths["fuzzer.log"]: + report_url = paths["fuzzer.log"] + if paths["report.html"]: + report_url = paths["report.html"] # Try to get status message saved by the fuzzer try: - with open(os.path.join(workspace_path, 'status.txt'), 'r', encoding='utf-8') as status_f: - status = status_f.readline().rstrip('\n') + with open( + os.path.join(workspace_path, "status.txt"), "r", encoding="utf-8" + ) as status_f: + status = status_f.readline().rstrip("\n") - with open(os.path.join(workspace_path, 'description.txt'), 'r', encoding='utf-8') as desc_f: - description = desc_f.readline().rstrip('\n')[:140] + with open( + os.path.join(workspace_path, "description.txt"), "r", encoding="utf-8" + ) as desc_f: + description = desc_f.readline().rstrip("\n")[:140] except: - status = 'failure' - description = 'Task failed: $?=' + str(retcode) + status = "failure" + description = "Task failed: $?=" + str(retcode) - if 'fail' in status: - test_result = [(description, 'FAIL')] + if "fail" in status: + test_result = [(description, "FAIL")] else: - test_result = [(description, 'OK')] + test_result = [(description, "OK")] ch_helper = ClickHouseHelper() - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_result, status, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_result, + status, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + check_name, + ) logging.info("Result: '%s', '%s', '%s'", status, description, report_url) print(f"::notice ::Report url: {report_url}") diff --git a/tests/ci/bugfix_validate_check.py b/tests/ci/bugfix_validate_check.py new file mode 100644 index 00000000000..7c130a766a1 --- /dev/null +++ b/tests/ci/bugfix_validate_check.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import itertools +import os +import sys + +NO_CHANGES_MSG = "Nothing to run" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("report1") + parser.add_argument("report2") + return parser.parse_args() + + +def post_commit_status_from_file(file_path): + res = [] + with open(file_path, "r", encoding="utf-8") as f: + fin = csv.reader(f, delimiter="\t") + res = list(itertools.islice(fin, 1)) + if len(res) < 1: + raise Exception(f'Can\'t read from "{file_path}"') + if len(res[0]) != 3: + raise Exception(f'Can\'t read from "{file_path}"') + return res[0] + + +def process_results(file_path): + state, report_url, description = post_commit_status_from_file(file_path) + prefix = os.path.basename(os.path.dirname(file_path)) + print( + f"::notice:: bugfix check: {prefix} - {state}: {description} Report url: {report_url}" + ) + return state == "success" + + +def main(args): + is_ok = False + is_ok = process_results(args.report1) or is_ok + is_ok = process_results(args.report2) or is_ok + sys.exit(0 if is_ok else 1) + + +if __name__ == "__main__": + main(parse_args()) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index e21e7d0138d..24af9e5194c 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -21,6 +21,8 @@ from ci_config import CI_CONFIG, BuildConfig from docker_pull_helper import get_image_with_version from tee_popen import TeePopen +IMAGE_NAME = "clickhouse/binary-builder" + def get_build_config(build_check_name: str, build_name: str) -> BuildConfig: if build_check_name == "ClickHouse build check (actions)": @@ -52,7 +54,6 @@ def get_packager_cmd( build_version: str, image_version: str, ccache_path: str, - pr_info: PRInfo, ) -> str: package_type = build_config["package_type"] comp = build_config["compiler"] @@ -73,9 +74,8 @@ def get_packager_cmd( cmd += " --cache=ccache" cmd += " --ccache_dir={}".format(ccache_path) - if "alien_pkgs" in build_config and build_config["alien_pkgs"]: - if pr_info.number == 0 or "release" in pr_info.labels: - cmd += " --alien-pkgs rpm tgz" + if "additional_pkgs" in build_config and build_config["additional_pkgs"]: + cmd += " --additional-pkgs" cmd += " --docker-image-version={}".format(image_version) cmd += " --version={}".format(build_version) @@ -86,13 +86,6 @@ def get_packager_cmd( return cmd -def get_image_name(build_config: BuildConfig) -> str: - if build_config["package_type"] != "deb": - return "clickhouse/binary-builder" - else: - return "clickhouse/deb-builder" - - def build_clickhouse( packager_cmd: str, logs_path: str, build_output_path: str ) -> Tuple[str, bool]: @@ -175,7 +168,7 @@ def get_release_or_pr( # for pushes to master - major version, but not for performance builds # they havily relies on a fixed path for build package and nobody going # to deploy them somewhere, so it's ok. - return ".".join(version.as_tuple()[:2]) + return f"{version.major}.{version.minor}" # PR number for anything else return str(pr_info.number) @@ -218,7 +211,7 @@ def main(): s3_helper = S3Helper("https://s3.amazonaws.com") - version = get_version_from_repo(REPO_COPY) + version = get_version_from_repo() release_or_pr = get_release_or_pr(pr_info, build_config, version) s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name)) @@ -240,6 +233,7 @@ def main(): "https://s3.amazonaws.com/clickhouse-builds/" + url.replace("+", "%2B").replace(" ", "%20") ) + success = len(build_urls) > 0 create_json_artifact( TEMP_PATH, build_name, @@ -247,21 +241,24 @@ def main(): build_urls, build_config, 0, - len(build_urls) > 0, + success, ) - return + # Fail build job if not successeded + if not success: + sys.exit(1) + else: + sys.exit(0) - image_name = get_image_name(build_config) - docker_image = get_image_with_version(IMAGES_PATH, image_name) + docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME) image_version = docker_image.version - logging.info("Got version from repo %s", version.get_version_string()) + logging.info("Got version from repo %s", version.string) version_type = "testing" if "release" in pr_info.labels or "release-lts" in pr_info.labels: version_type = "stable" - update_version_local(REPO_COPY, pr_info.sha, version, version_type) + update_version_local(REPO_COPY, version, version_type) logging.info("Updated local files with version") @@ -290,10 +287,9 @@ def main(): build_config, os.path.join(REPO_COPY, "docker/packager"), build_output_path, - version.get_version_string(), + version.string, image_version, ccache_path, - pr_info, ) logging.info("Going to run packager with %s", packager_cmd) diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index a85558ebe33..1cee5fd42de 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -6,7 +6,13 @@ import os import sys from github import Github -from env_helper import REPORTS_PATH, TEMP_PATH, GITHUB_REPOSITORY, GITHUB_SERVER_URL, GITHUB_RUN_ID +from env_helper import ( + REPORTS_PATH, + TEMP_PATH, + GITHUB_REPOSITORY, + GITHUB_SERVER_URL, + GITHUB_RUN_ID, +) from report import create_build_html_report from s3_helper import S3Helper from get_robot_token import get_best_robot_token @@ -15,8 +21,19 @@ from commit_status_helper import get_commit from ci_config import CI_CONFIG from rerun_helper import RerunHelper -class BuildResult(): - def __init__(self, compiler, build_type, sanitizer, bundled, splitted, status, elapsed_seconds, with_coverage): + +class BuildResult: + def __init__( + self, + compiler, + build_type, + sanitizer, + bundled, + splitted, + status, + elapsed_seconds, + with_coverage, + ): self.compiler = compiler self.build_type = build_type self.sanitizer = sanitizer @@ -26,54 +43,72 @@ class BuildResult(): self.elapsed_seconds = elapsed_seconds self.with_coverage = with_coverage + def group_by_artifacts(build_urls): - groups = {'deb': [], 'binary': [], 'tgz': [], 'rpm': [], 'performance': []} + groups = { + "apk": [], + "deb": [], + "binary": [], + "tgz": [], + "rpm": [], + "performance": [], + } for url in build_urls: - if url.endswith('performance.tgz'): - groups['performance'].append(url) - elif url.endswith('.deb') or url.endswith('.buildinfo') or url.endswith('.changes') or url.endswith('.tar.gz'): - groups['deb'].append(url) - elif url.endswith('.rpm'): - groups['rpm'].append(url) - elif url.endswith('.tgz'): - groups['tgz'].append(url) + if url.endswith("performance.tgz"): + groups["performance"].append(url) + elif ( + url.endswith(".deb") + or url.endswith(".buildinfo") + or url.endswith(".changes") + or url.endswith(".tar.gz") + ): + groups["deb"].append(url) + elif url.endswith(".apk"): + groups["apk"].append(url) + elif url.endswith(".rpm"): + groups["rpm"].append(url) + elif url.endswith(".tgz"): + groups["tgz"].append(url) else: - groups['binary'].append(url) + groups["binary"].append(url) return groups + def process_report(build_report): - build_config = build_report['build_config'] + build_config = build_report["build_config"] build_result = BuildResult( - compiler=build_config['compiler'], - build_type=build_config['build_type'], - sanitizer=build_config['sanitizer'], - bundled=build_config['bundled'], - splitted=build_config['splitted'], - status="success" if build_report['status'] else "failure", - elapsed_seconds=build_report['elapsed_seconds'], - with_coverage=False + compiler=build_config["compiler"], + build_type=build_config["build_type"], + sanitizer=build_config["sanitizer"], + bundled=build_config["bundled"], + splitted=build_config["splitted"], + status="success" if build_report["status"] else "failure", + elapsed_seconds=build_report["elapsed_seconds"], + with_coverage=False, ) build_results = [] build_urls = [] build_logs_urls = [] - urls_groups = group_by_artifacts(build_report['build_urls']) + urls_groups = group_by_artifacts(build_report["build_urls"]) found_group = False for _, group_urls in urls_groups.items(): if group_urls: build_results.append(build_result) build_urls.append(group_urls) - build_logs_urls.append(build_report['log_url']) + build_logs_urls.append(build_report["log_url"]) found_group = True if not found_group: build_results.append(build_result) build_urls.append([""]) - build_logs_urls.append(build_report['log_url']) + build_logs_urls.append(build_report["log_url"]) return build_results, build_urls, build_logs_urls + def get_build_name_from_file_name(file_name): - return file_name.replace('build_urls_', '').replace('.json', '') + return file_name.replace("build_urls_", "").replace(".json", "") + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -99,17 +134,25 @@ if __name__ == "__main__": build_reports_map = {} for root, dirs, files in os.walk(reports_path): for f in files: - if f.startswith("build_urls_") and f.endswith('.json'): + if f.startswith("build_urls_") and f.endswith(".json"): logging.info("Found build report json %s", f) build_name = get_build_name_from_file_name(f) if build_name in reports_order: - with open(os.path.join(root, f), 'r') as file_handler: + with open(os.path.join(root, f), "r") as file_handler: build_report = json.load(file_handler) build_reports_map[build_name] = build_report else: - logging.info("Skipping report %s for build %s, it's not in our reports list", f, build_name) + logging.info( + "Skipping report %s for build %s, it's not in our reports list", + f, + build_name, + ) - build_reports = [build_reports_map[build_name] for build_name in reports_order if build_name in build_reports_map] + build_reports = [ + build_reports_map[build_name] + for build_name in reports_order + if build_name in build_reports_map + ] build_results = [] build_artifacts = [] @@ -127,7 +170,7 @@ if __name__ == "__main__": logging.info("No builds, failing check") sys.exit(1) - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") pr_info = PRInfo() @@ -137,7 +180,9 @@ if __name__ == "__main__": branch_name = "PR #{}".format(pr_info.number) branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_info.number}" commit_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commit/{pr_info.sha}" - task_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID or '0'}" + task_url = ( + f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID or '0'}" + ) report = create_build_html_report( build_check_name, build_results, @@ -146,18 +191,22 @@ if __name__ == "__main__": task_url, branch_url, branch_name, - commit_url + commit_url, ) - report_path = os.path.join(temp_path, 'report.html') - with open(report_path, 'w') as f: + report_path = os.path.join(temp_path, "report.html") + with open(report_path, "w") as f: f.write(report) logging.info("Going to upload prepared report") - context_name_for_path = build_check_name.lower().replace(' ', '_') - s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path + context_name_for_path = build_check_name.lower().replace(" ", "_") + s3_path_prefix = ( + str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path + ) - url = s3_helper.upload_build_file_to_s3(report_path, s3_path_prefix + "/report.html") + url = s3_helper.upload_build_file_to_s3( + report_path, s3_path_prefix + "/report.html" + ) logging.info("Report url %s", url) total_builds = len(build_results) @@ -180,4 +229,9 @@ if __name__ == "__main__": print("::notice ::Report url: {}".format(url)) commit = get_commit(gh, pr_info.sha) - commit.create_status(context=build_check_name, description=description, state=summary_status, target_url=url) + commit.create_status( + context=build_check_name, + description=description, + state=summary_status, + target_url=url, + ) diff --git a/tests/ci/ccache_utils.py b/tests/ci/ccache_utils.py index f21f1a8c965..7b0b0f01aa3 100644 --- a/tests/ci/ccache_utils.py +++ b/tests/ci/ccache_utils.py @@ -13,16 +13,19 @@ from compress_files import decompress_fast, compress_fast DOWNLOAD_RETRIES_COUNT = 5 + def dowload_file_with_progress(url, path): logging.info("Downloading from %s to temp path %s", url, path) for i in range(DOWNLOAD_RETRIES_COUNT): try: - with open(path, 'wb') as f: + with open(path, "wb") as f: response = requests.get(url, stream=True) response.raise_for_status() - total_length = response.headers.get('content-length') + total_length = response.headers.get("content-length") if total_length is None or int(total_length) == 0: - logging.info("No content-length, will download file without progress") + logging.info( + "No content-length, will download file without progress" + ) f.write(response.content) else: dl = 0 @@ -34,8 +37,8 @@ def dowload_file_with_progress(url, path): if sys.stdout.isatty(): done = int(50 * dl / total_length) percent = int(100 * float(dl) / total_length) - eq_str = '=' * done - space_str = ' ' * (50 - done) + eq_str = "=" * done + space_str = " " * (50 - done) sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%") sys.stdout.flush() break @@ -52,7 +55,9 @@ def dowload_file_with_progress(url, path): logging.info("Downloading finished") -def get_ccache_if_not_exists(path_to_ccache_dir, s3_helper, current_pr_number, temp_path): +def get_ccache_if_not_exists( + path_to_ccache_dir, s3_helper, current_pr_number, temp_path +): ccache_name = os.path.basename(path_to_ccache_dir) cache_found = False prs_to_check = [current_pr_number] @@ -93,13 +98,16 @@ def get_ccache_if_not_exists(path_to_ccache_dir, s3_helper, current_pr_number, t else: logging.info("ccache downloaded") + def upload_ccache(path_to_ccache_dir, s3_helper, current_pr_number, temp_path): logging.info("Uploading cache %s for pr %s", path_to_ccache_dir, current_pr_number) ccache_name = os.path.basename(path_to_ccache_dir) compressed_cache_path = os.path.join(temp_path, ccache_name + ".tar.gz") compress_fast(path_to_ccache_dir, compressed_cache_path) - s3_path = str(current_pr_number) + "/ccaches/" + os.path.basename(compressed_cache_path) + s3_path = ( + str(current_pr_number) + "/ccaches/" + os.path.basename(compressed_cache_path) + ) logging.info("Will upload %s to path %s", compressed_cache_path, s3_path) s3_helper.upload_build_file_to_s3(compressed_cache_path, s3_path) logging.info("Upload finished") diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py index 91a018f158f..4bbd30cd186 100644 --- a/tests/ci/cherry_pick.py +++ b/tests/ci/cherry_pick.py @@ -20,21 +20,29 @@ if __name__ == "__main__": if not os.path.exists(temp_path): os.makedirs(temp_path) - sys.path.append(os.path.join(repo_path, "utils/github")) - with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): token = get_parameter_from_ssm("github_robot_token_1") - bp = Backport(token, os.environ.get("REPO_OWNER"), os.environ.get("REPO_NAME"), os.environ.get("REPO_TEAM")) + bp = Backport( + token, + os.environ.get("REPO_OWNER"), + os.environ.get("REPO_NAME"), + os.environ.get("REPO_TEAM"), + ) + def cherrypick_run(token, pr, branch): - return CherryPick(token, - os.environ.get("REPO_OWNER"), os.environ.get("REPO_NAME"), - os.environ.get("REPO_TEAM"), pr, branch - ).execute(repo_path, False) + return CherryPick( + token, + os.environ.get("REPO_OWNER"), + os.environ.get("REPO_NAME"), + os.environ.get("REPO_TEAM"), + pr, + branch, + ).execute(repo_path, False) try: - bp.execute(repo_path, 'origin', None, cherrypick_run) + bp.execute(repo_path, "origin", None, cherrypick_run) except subprocess.CalledProcessError as e: logging.error(e.output) diff --git a/tests/ci/cherry_pick_utils/backport.py b/tests/ci/cherry_pick_utils/backport.py index 9227dbf4108..615c0d19ffa 100644 --- a/tests/ci/cherry_pick_utils/backport.py +++ b/tests/ci/cherry_pick_utils/backport.py @@ -17,7 +17,9 @@ import sys class Backport: def __init__(self, token, owner, name, team): - self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7) + self._gh = RemoteRepo( + token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7 + ) self._token = token self.default_branch_name = self._gh.default_branch self.ssh_url = self._gh.ssh_url @@ -28,7 +30,7 @@ class Backport: def getBranchesWithRelease(self): branches = set() for pull_request in self._gh.find_pull_requests("release"): - branches.add(pull_request['headRefName']) + branches.add(pull_request["headRefName"]) return branches def execute(self, repo, upstream, until_commit, run_cherrypick): @@ -44,11 +46,11 @@ class Backport: branches.append(branch) if not branches: - logging.info('No release branches found!') + logging.info("No release branches found!") return for branch in branches: - logging.info('Found release branch: %s', branch[0]) + logging.info("Found release branch: %s", branch[0]) if not until_commit: until_commit = branches[0][1] @@ -56,73 +58,128 @@ class Backport: backport_map = {} - RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$') - RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$') - RE_BACKPORTED = re.compile(r'^v(\d+\.\d+)-backported$') + RE_MUST_BACKPORT = re.compile(r"^v(\d+\.\d+)-must-backport$") + RE_NO_BACKPORT = re.compile(r"^v(\d+\.\d+)-no-backport$") + RE_BACKPORTED = re.compile(r"^v(\d+\.\d+)-backported$") # pull-requests are sorted by ancestry from the most recent. for pr in pull_requests: - while repo.comparator(branches[-1][1]) >= repo.comparator(pr['mergeCommit']['oid']): - logging.info("PR #{} is already inside {}. Dropping this branch for further PRs".format(pr['number'], branches[-1][0])) + while repo.comparator(branches[-1][1]) >= repo.comparator( + pr["mergeCommit"]["oid"] + ): + logging.info( + "PR #{} is already inside {}. Dropping this branch for further PRs".format( + pr["number"], branches[-1][0] + ) + ) branches.pop() - logging.info("Processing PR #{}".format(pr['number'])) + logging.info("Processing PR #{}".format(pr["number"])) assert len(branches) branch_set = set([branch[0] for branch in branches]) # First pass. Find all must-backports - for label in pr['labels']['nodes']: - if label['name'] == 'pr-must-backport': - backport_map[pr['number']] = branch_set.copy() + for label in pr["labels"]["nodes"]: + if label["name"] == "pr-must-backport": + backport_map[pr["number"]] = branch_set.copy() continue - matched = RE_MUST_BACKPORT.match(label['name']) + matched = RE_MUST_BACKPORT.match(label["name"]) if matched: - if pr['number'] not in backport_map: - backport_map[pr['number']] = set() - backport_map[pr['number']].add(matched.group(1)) + if pr["number"] not in backport_map: + backport_map[pr["number"]] = set() + backport_map[pr["number"]].add(matched.group(1)) # Second pass. Find all no-backports - for label in pr['labels']['nodes']: - if label['name'] == 'pr-no-backport' and pr['number'] in backport_map: - del backport_map[pr['number']] + for label in pr["labels"]["nodes"]: + if label["name"] == "pr-no-backport" and pr["number"] in backport_map: + del backport_map[pr["number"]] break - matched_no_backport = RE_NO_BACKPORT.match(label['name']) - matched_backported = RE_BACKPORTED.match(label['name']) - if matched_no_backport and pr['number'] in backport_map and matched_no_backport.group(1) in backport_map[pr['number']]: - backport_map[pr['number']].remove(matched_no_backport.group(1)) - logging.info('\tskipping %s because of forced no-backport', matched_no_backport.group(1)) - elif matched_backported and pr['number'] in backport_map and matched_backported.group(1) in backport_map[pr['number']]: - backport_map[pr['number']].remove(matched_backported.group(1)) - logging.info('\tskipping %s because it\'s already backported manually', matched_backported.group(1)) + matched_no_backport = RE_NO_BACKPORT.match(label["name"]) + matched_backported = RE_BACKPORTED.match(label["name"]) + if ( + matched_no_backport + and pr["number"] in backport_map + and matched_no_backport.group(1) in backport_map[pr["number"]] + ): + backport_map[pr["number"]].remove(matched_no_backport.group(1)) + logging.info( + "\tskipping %s because of forced no-backport", + matched_no_backport.group(1), + ) + elif ( + matched_backported + and pr["number"] in backport_map + and matched_backported.group(1) in backport_map[pr["number"]] + ): + backport_map[pr["number"]].remove(matched_backported.group(1)) + logging.info( + "\tskipping %s because it's already backported manually", + matched_backported.group(1), + ) for pr, branches in list(backport_map.items()): - logging.info('PR #%s needs to be backported to:', pr) + logging.info("PR #%s needs to be backported to:", pr) for branch in branches: - logging.info('\t%s, and the status is: %s', branch, run_cherrypick(self._token, pr, branch)) + logging.info( + "\t%s, and the status is: %s", + branch, + run_cherrypick(self._token, pr, branch), + ) # print API costs - logging.info('\nGitHub API total costs per query:') + logging.info("\nGitHub API total costs per query:") for name, value in list(self._gh.api_costs.items()): - logging.info('%s : %s', name, value) + logging.info("%s : %s", name, value) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--token', type=str, required=True, help='token for Github access') - parser.add_argument('--repo', type=str, required=True, help='path to full repository', metavar='PATH') - parser.add_argument('--til', type=str, help='check PRs from HEAD til this commit', metavar='COMMIT') - parser.add_argument('--dry-run', action='store_true', help='do not create or merge any PRs', default=False) - parser.add_argument('--verbose', '-v', action='store_true', help='more verbose output', default=False) - parser.add_argument('--upstream', '-u', type=str, help='remote name of upstream in repository', default='origin') + parser.add_argument( + "--token", type=str, required=True, help="token for Github access" + ) + parser.add_argument( + "--repo", + type=str, + required=True, + help="path to full repository", + metavar="PATH", + ) + parser.add_argument( + "--til", type=str, help="check PRs from HEAD til this commit", metavar="COMMIT" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="do not create or merge any PRs", + default=False, + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="more verbose output", + default=False, + ) + parser.add_argument( + "--upstream", + "-u", + type=str, + help="remote name of upstream in repository", + default="origin", + ) args = parser.parse_args() if args.verbose: - logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.DEBUG) + logging.basicConfig( + format="%(message)s", stream=sys.stdout, level=logging.DEBUG + ) else: - logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO) + logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.INFO) - cherrypick_run = lambda token, pr, branch: CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch).execute(args.repo, args.dry_run) - bp = Backport(args.token, 'ClickHouse', 'ClickHouse', 'core') + cherrypick_run = lambda token, pr, branch: CherryPick( + token, "ClickHouse", "ClickHouse", "core", pr, branch + ).execute(args.repo, args.dry_run) + bp = Backport(args.token, "ClickHouse", "ClickHouse", "core") bp.execute(args.repo, args.upstream, args.til, cherrypick_run) diff --git a/tests/ci/cherry_pick_utils/cherrypick.py b/tests/ci/cherry_pick_utils/cherrypick.py index 8bedf54fefa..c6469fa62a9 100644 --- a/tests/ci/cherry_pick_utils/cherrypick.py +++ b/tests/ci/cherry_pick_utils/cherrypick.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -''' +""" Backports changes from PR to release branch. Requires multiple separate runs as part of the implementation. @@ -12,7 +12,7 @@ First run should do the following: Second run checks PR from previous run to be merged or at least being mergeable. If it's not merged then try to merge it. Third run creates PR from backport branch (with merged previous PR) to release branch. -''' +""" try: from clickhouse.utils.github.query import Query as RemoteRepo @@ -29,13 +29,13 @@ import sys class CherryPick: class Status(Enum): - DISCARDED = 'discarded' - NOT_INITIATED = 'not started' - FIRST_MERGEABLE = 'waiting for 1st stage' - FIRST_CONFLICTS = 'conflicts on 1st stage' - SECOND_MERGEABLE = 'waiting for 2nd stage' - SECOND_CONFLICTS = 'conflicts on 2nd stage' - MERGED = 'backported' + DISCARDED = "discarded" + NOT_INITIATED = "not started" + FIRST_MERGEABLE = "waiting for 1st stage" + FIRST_CONFLICTS = "conflicts on 1st stage" + SECOND_MERGEABLE = "waiting for 2nd stage" + SECOND_CONFLICTS = "conflicts on 2nd stage" + MERGED = "backported" def _run(self, args): out = subprocess.check_output(args).rstrip() @@ -50,51 +50,90 @@ class CherryPick: # TODO: check if pull-request is merged. - self.merge_commit_oid = self._pr['mergeCommit']['oid'] + self.merge_commit_oid = self._pr["mergeCommit"]["oid"] self.target_branch = target_branch - self.backport_branch = 'backport/{branch}/{pr}'.format(branch=target_branch, pr=pr_number) - self.cherrypick_branch = 'cherrypick/{branch}/{oid}'.format(branch=target_branch, oid=self.merge_commit_oid) + self.backport_branch = "backport/{branch}/{pr}".format( + branch=target_branch, pr=pr_number + ) + self.cherrypick_branch = "cherrypick/{branch}/{oid}".format( + branch=target_branch, oid=self.merge_commit_oid + ) def getCherryPickPullRequest(self): - return self._gh.find_pull_request(base=self.backport_branch, head=self.cherrypick_branch) + return self._gh.find_pull_request( + base=self.backport_branch, head=self.cherrypick_branch + ) def createCherryPickPullRequest(self, repo_path): DESCRIPTION = ( - 'This pull-request is a first step of an automated backporting.\n' - 'It contains changes like after calling a local command `git cherry-pick`.\n' - 'If you intend to continue backporting this changes, then resolve all conflicts if any.\n' - 'Otherwise, if you do not want to backport them, then just close this pull-request.\n' - '\n' - 'The check results does not matter at this step - you can safely ignore them.\n' - 'Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n' + "This pull-request is a first step of an automated backporting.\n" + "It contains changes like after calling a local command `git cherry-pick`.\n" + "If you intend to continue backporting this changes, then resolve all conflicts if any.\n" + "Otherwise, if you do not want to backport them, then just close this pull-request.\n" + "\n" + "The check results does not matter at this step - you can safely ignore them.\n" + "Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n" ) # FIXME: replace with something better than os.system() - git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@yandex-team.ru', '-c', 'user.name=robot-clickhouse'] - base_commit_oid = self._pr['mergeCommit']['parents']['nodes'][0]['oid'] + git_prefix = [ + "git", + "-C", + repo_path, + "-c", + "user.email=robot-clickhouse@yandex-team.ru", + "-c", + "user.name=robot-clickhouse", + ] + base_commit_oid = self._pr["mergeCommit"]["parents"]["nodes"][0]["oid"] # Create separate branch for backporting, and make it look like real cherry-pick. - self._run(git_prefix + ['checkout', '-f', self.target_branch]) - self._run(git_prefix + ['checkout', '-B', self.backport_branch]) - self._run(git_prefix + ['merge', '-s', 'ours', '--no-edit', base_commit_oid]) + self._run(git_prefix + ["checkout", "-f", self.target_branch]) + self._run(git_prefix + ["checkout", "-B", self.backport_branch]) + self._run(git_prefix + ["merge", "-s", "ours", "--no-edit", base_commit_oid]) # Create secondary branch to allow pull request with cherry-picked commit. - self._run(git_prefix + ['branch', '-f', self.cherrypick_branch, self.merge_commit_oid]) + self._run( + git_prefix + ["branch", "-f", self.cherrypick_branch, self.merge_commit_oid] + ) - self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)]) - self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.cherrypick_branch)]) + self._run( + git_prefix + + [ + "push", + "-f", + "origin", + "{branch}:{branch}".format(branch=self.backport_branch), + ] + ) + self._run( + git_prefix + + [ + "push", + "-f", + "origin", + "{branch}:{branch}".format(branch=self.cherrypick_branch), + ] + ) # Create pull-request like a local cherry-pick - pr = self._gh.create_pull_request(source=self.cherrypick_branch, target=self.backport_branch, - title='Cherry pick #{number} to {target}: {title}'.format( - number=self._pr['number'], target=self.target_branch, - title=self._pr['title'].replace('"', '\\"')), - description='Original pull-request #{}\n\n{}'.format(self._pr['number'], DESCRIPTION)) + pr = self._gh.create_pull_request( + source=self.cherrypick_branch, + target=self.backport_branch, + title="Cherry pick #{number} to {target}: {title}".format( + number=self._pr["number"], + target=self.target_branch, + title=self._pr["title"].replace('"', '\\"'), + ), + description="Original pull-request #{}\n\n{}".format( + self._pr["number"], DESCRIPTION + ), + ) # FIXME: use `team` to leave a single eligible assignee. - self._gh.add_assignee(pr, self._pr['author']) - self._gh.add_assignee(pr, self._pr['mergedBy']) + self._gh.add_assignee(pr, self._pr["author"]) + self._gh.add_assignee(pr, self._pr["mergedBy"]) self._gh.set_label(pr, "do not test") self._gh.set_label(pr, "pr-cherrypick") @@ -102,36 +141,76 @@ class CherryPick: return pr def mergeCherryPickPullRequest(self, cherrypick_pr): - return self._gh.merge_pull_request(cherrypick_pr['id']) + return self._gh.merge_pull_request(cherrypick_pr["id"]) def getBackportPullRequest(self): - return self._gh.find_pull_request(base=self.target_branch, head=self.backport_branch) + return self._gh.find_pull_request( + base=self.target_branch, head=self.backport_branch + ) def createBackportPullRequest(self, cherrypick_pr, repo_path): DESCRIPTION = ( - 'This pull-request is a last step of an automated backporting.\n' - 'Treat it as a standard pull-request: look at the checks and resolve conflicts.\n' - 'Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n' + "This pull-request is a last step of an automated backporting.\n" + "Treat it as a standard pull-request: look at the checks and resolve conflicts.\n" + "Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n" ) - git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@clickhouse.com', '-c', 'user.name=robot-clickhouse'] + git_prefix = [ + "git", + "-C", + repo_path, + "-c", + "user.email=robot-clickhouse@clickhouse.com", + "-c", + "user.name=robot-clickhouse", + ] - pr_title = 'Backport #{number} to {target}: {title}'.format( - number=self._pr['number'], target=self.target_branch, - title=self._pr['title'].replace('"', '\\"')) + pr_title = "Backport #{number} to {target}: {title}".format( + number=self._pr["number"], + target=self.target_branch, + title=self._pr["title"].replace('"', '\\"'), + ) - self._run(git_prefix + ['checkout', '-f', self.backport_branch]) - self._run(git_prefix + ['pull', '--ff-only', 'origin', self.backport_branch]) - self._run(git_prefix + ['reset', '--soft', self._run(git_prefix + ['merge-base', 'origin/' + self.target_branch, self.backport_branch])]) - self._run(git_prefix + ['commit', '-a', '--allow-empty', '-m', pr_title]) - self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)]) + self._run(git_prefix + ["checkout", "-f", self.backport_branch]) + self._run(git_prefix + ["pull", "--ff-only", "origin", self.backport_branch]) + self._run( + git_prefix + + [ + "reset", + "--soft", + self._run( + git_prefix + + [ + "merge-base", + "origin/" + self.target_branch, + self.backport_branch, + ] + ), + ] + ) + self._run(git_prefix + ["commit", "-a", "--allow-empty", "-m", pr_title]) + self._run( + git_prefix + + [ + "push", + "-f", + "origin", + "{branch}:{branch}".format(branch=self.backport_branch), + ] + ) - pr = self._gh.create_pull_request(source=self.backport_branch, target=self.target_branch, title=pr_title, - description='Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION)) + pr = self._gh.create_pull_request( + source=self.backport_branch, + target=self.target_branch, + title=pr_title, + description="Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}".format( + self._pr["number"], cherrypick_pr["number"], DESCRIPTION + ), + ) # FIXME: use `team` to leave a single eligible assignee. - self._gh.add_assignee(pr, self._pr['author']) - self._gh.add_assignee(pr, self._pr['mergedBy']) + self._gh.add_assignee(pr, self._pr["author"]) + self._gh.add_assignee(pr, self._pr["mergedBy"]) self._gh.set_label(pr, "pr-backport") @@ -142,23 +221,43 @@ class CherryPick: if not pr1: if not dry_run: pr1 = self.createCherryPickPullRequest(repo_path) - logging.debug('Created PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + logging.debug( + "Created PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) else: return CherryPick.Status.NOT_INITIATED else: - logging.debug('Found PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + logging.debug( + "Found PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) - if not pr1['merged'] and pr1['mergeable'] == 'MERGEABLE' and not pr1['closed']: + if not pr1["merged"] and pr1["mergeable"] == "MERGEABLE" and not pr1["closed"]: if not dry_run: pr1 = self.mergeCherryPickPullRequest(pr1) - logging.debug('Merged PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + logging.debug( + "Merged PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) - if not pr1['merged']: - logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + if not pr1["merged"]: + logging.debug( + "Waiting for PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) - if pr1['closed']: + if pr1["closed"]: return CherryPick.Status.DISCARDED - elif pr1['mergeable'] == 'CONFLICTING': + elif pr1["mergeable"] == "CONFLICTING": return CherryPick.Status.FIRST_CONFLICTS else: return CherryPick.Status.FIRST_MERGEABLE @@ -167,31 +266,58 @@ class CherryPick: if not pr2: if not dry_run: pr2 = self.createBackportPullRequest(pr1, repo_path) - logging.debug('Created PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) + logging.debug( + "Created PR with backport of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr2["url"], + ) else: return CherryPick.Status.FIRST_MERGEABLE else: - logging.debug('Found PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) + logging.debug( + "Found PR with backport of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr2["url"], + ) - if pr2['merged']: + if pr2["merged"]: return CherryPick.Status.MERGED - elif pr2['closed']: + elif pr2["closed"]: return CherryPick.Status.DISCARDED - elif pr2['mergeable'] == 'CONFLICTING': + elif pr2["mergeable"] == "CONFLICTING": return CherryPick.Status.SECOND_CONFLICTS else: return CherryPick.Status.SECOND_MERGEABLE if __name__ == "__main__": - logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.DEBUG) + logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG) parser = argparse.ArgumentParser() - parser.add_argument('--token', '-t', type=str, required=True, help='token for Github access') - parser.add_argument('--pr', type=str, required=True, help='PR# to cherry-pick') - parser.add_argument('--branch', '-b', type=str, required=True, help='target branch name for cherry-pick') - parser.add_argument('--repo', '-r', type=str, required=True, help='path to full repository', metavar='PATH') + parser.add_argument( + "--token", "-t", type=str, required=True, help="token for Github access" + ) + parser.add_argument("--pr", type=str, required=True, help="PR# to cherry-pick") + parser.add_argument( + "--branch", + "-b", + type=str, + required=True, + help="target branch name for cherry-pick", + ) + parser.add_argument( + "--repo", + "-r", + type=str, + required=True, + help="path to full repository", + metavar="PATH", + ) args = parser.parse_args() - cp = CherryPick(args.token, 'ClickHouse', 'ClickHouse', 'core', args.pr, args.branch) + cp = CherryPick( + args.token, "ClickHouse", "ClickHouse", "core", args.pr, args.branch + ) cp.execute(args.repo) diff --git a/tests/ci/cherry_pick_utils/local.py b/tests/ci/cherry_pick_utils/local.py index 2ad8d4b8b71..571c9102ba0 100644 --- a/tests/ci/cherry_pick_utils/local.py +++ b/tests/ci/cherry_pick_utils/local.py @@ -20,13 +20,14 @@ class RepositoryBase: return -1 else: return 1 + self.comparator = functools.cmp_to_key(cmp) def get_head_commit(self): return self._repo.commit(self._default) def iterate(self, begin, end): - rev_range = '{}...{}'.format(begin, end) + rev_range = "{}...{}".format(begin, end) for commit in self._repo.iter_commits(rev_range, first_parent=True): yield commit @@ -39,27 +40,35 @@ class Repository(RepositoryBase): self._default = self._remote.refs[default_branch_name] def get_release_branches(self): - ''' + """ Returns sorted list of tuples: * remote branch (git.refs.remote.RemoteReference), * base commit (git.Commit), * head (git.Commit)). List is sorted by commits in ascending order. - ''' + """ release_branches = [] - RE_RELEASE_BRANCH_REF = re.compile(r'^refs/remotes/.+/\d+\.\d+$') + RE_RELEASE_BRANCH_REF = re.compile(r"^refs/remotes/.+/\d+\.\d+$") - for branch in [r for r in self._remote.refs if RE_RELEASE_BRANCH_REF.match(r.path)]: + for branch in [ + r for r in self._remote.refs if RE_RELEASE_BRANCH_REF.match(r.path) + ]: base = self._repo.merge_base(self._default, self._repo.commit(branch)) if not base: - logging.info('Branch %s is not based on branch %s. Ignoring.', branch.path, self._default) + logging.info( + "Branch %s is not based on branch %s. Ignoring.", + branch.path, + self._default, + ) elif len(base) > 1: - logging.info('Branch %s has more than one base commit. Ignoring.', branch.path) + logging.info( + "Branch %s has more than one base commit. Ignoring.", branch.path + ) else: release_branches.append((os.path.basename(branch.name), base[0])) - return sorted(release_branches, key=lambda x : self.comparator(x[1])) + return sorted(release_branches, key=lambda x: self.comparator(x[1])) class BareRepository(RepositoryBase): @@ -68,24 +77,32 @@ class BareRepository(RepositoryBase): self._default = self._repo.branches[default_branch_name] def get_release_branches(self): - ''' + """ Returns sorted list of tuples: * branch (git.refs.head?), * base commit (git.Commit), * head (git.Commit)). List is sorted by commits in ascending order. - ''' + """ release_branches = [] - RE_RELEASE_BRANCH_REF = re.compile(r'^refs/heads/\d+\.\d+$') + RE_RELEASE_BRANCH_REF = re.compile(r"^refs/heads/\d+\.\d+$") - for branch in [r for r in self._repo.branches if RE_RELEASE_BRANCH_REF.match(r.path)]: + for branch in [ + r for r in self._repo.branches if RE_RELEASE_BRANCH_REF.match(r.path) + ]: base = self._repo.merge_base(self._default, self._repo.commit(branch)) if not base: - logging.info('Branch %s is not based on branch %s. Ignoring.', branch.path, self._default) + logging.info( + "Branch %s is not based on branch %s. Ignoring.", + branch.path, + self._default, + ) elif len(base) > 1: - logging.info('Branch %s has more than one base commit. Ignoring.', branch.path) + logging.info( + "Branch %s has more than one base commit. Ignoring.", branch.path + ) else: release_branches.append((os.path.basename(branch.name), base[0])) - return sorted(release_branches, key=lambda x : self.comparator(x[1])) + return sorted(release_branches, key=lambda x: self.comparator(x[1])) diff --git a/tests/ci/cherry_pick_utils/parser.py b/tests/ci/cherry_pick_utils/parser.py index 570410ba23d..d8348e6d964 100644 --- a/tests/ci/cherry_pick_utils/parser.py +++ b/tests/ci/cherry_pick_utils/parser.py @@ -1,19 +1,20 @@ # -*- coding: utf-8 -*- + class Description: - '''Parsed description representation - ''' + """Parsed description representation""" + MAP_CATEGORY_TO_LABEL = { - 'New Feature': 'pr-feature', - 'Bug Fix': 'pr-bugfix', - 'Improvement': 'pr-improvement', - 'Performance Improvement': 'pr-performance', + "New Feature": "pr-feature", + "Bug Fix": "pr-bugfix", + "Improvement": "pr-improvement", + "Performance Improvement": "pr-performance", # 'Backward Incompatible Change': doesn't match anything - 'Build/Testing/Packaging Improvement': 'pr-build', - 'Non-significant (changelog entry is not needed)': 'pr-non-significant', - 'Non-significant (changelog entry is not required)': 'pr-non-significant', - 'Non-significant': 'pr-non-significant', - 'Documentation (changelog entry is not required)': 'pr-documentation', + "Build/Testing/Packaging Improvement": "pr-build", + "Non-significant (changelog entry is not needed)": "pr-non-significant", + "Non-significant (changelog entry is not required)": "pr-non-significant", + "Non-significant": "pr-non-significant", + "Documentation (changelog entry is not required)": "pr-documentation", # 'Other': doesn't match anything } @@ -21,7 +22,7 @@ class Description: self.label_name = str() self.legal = False - self._parse(pull_request['bodyText']) + self._parse(pull_request["bodyText"]) def _parse(self, text): lines = text.splitlines() @@ -38,14 +39,17 @@ class Description: category = stripped next_category = False - if stripped == 'I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en': + if ( + stripped + == "I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en" + ): self.legal = True category_headers = ( - 'Category (leave one):', - 'Changelog category (leave one):', - 'Changelog category:', - 'Category:' + "Category (leave one):", + "Changelog category (leave one):", + "Changelog category:", + "Category:", ) if stripped in category_headers: @@ -55,6 +59,6 @@ class Description: self.label_name = Description.MAP_CATEGORY_TO_LABEL[category] else: if not category: - print('Cannot find category in pr description') + print("Cannot find category in pr description") else: - print(('Unknown category: ' + category)) + print(("Unknown category: " + category)) diff --git a/tests/ci/cherry_pick_utils/query.py b/tests/ci/cherry_pick_utils/query.py index a9a8f4f1cd1..40eb5bf3604 100644 --- a/tests/ci/cherry_pick_utils/query.py +++ b/tests/ci/cherry_pick_utils/query.py @@ -5,11 +5,11 @@ import time class Query: - ''' + """ Implements queries to the Github API using GraphQL - ''' + """ - _PULL_REQUEST = ''' + _PULL_REQUEST = """ author {{ ... on User {{ id @@ -47,7 +47,7 @@ class Query: number title url - ''' + """ def __init__(self, token, owner, name, team, max_page_size=100, min_page_size=10): self._PULL_REQUEST = Query._PULL_REQUEST.format(min_page_size=min_page_size) @@ -63,14 +63,14 @@ class Query: self.api_costs = {} repo = self.get_repository() - self._id = repo['id'] - self.ssh_url = repo['sshUrl'] - self.default_branch = repo['defaultBranchRef']['name'] + self._id = repo["id"] + self.ssh_url = repo["sshUrl"] + self.default_branch = repo["defaultBranchRef"]["name"] self.members = set(self.get_members()) def get_repository(self): - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ defaultBranchRef {{ name @@ -78,19 +78,19 @@ class Query: id sshUrl }} - ''' + """ query = _QUERY.format(owner=self._owner, name=self._name) - return self._run(query)['repository'] + return self._run(query)["repository"] def get_members(self): - '''Get all team members for organization + """Get all team members for organization Returns: members: a map of members' logins to ids - ''' + """ - _QUERY = ''' + _QUERY = """ organization(login: "{organization}") {{ team(slug: "{team}") {{ members(first: {max_page_size} {next}) {{ @@ -105,43 +105,54 @@ class Query: }} }} }} - ''' + """ members = {} not_end = True - query = _QUERY.format(organization=self._owner, team=self._team, - max_page_size=self._max_page_size, - next='') + query = _QUERY.format( + organization=self._owner, + team=self._team, + max_page_size=self._max_page_size, + next="", + ) while not_end: - result = self._run(query)['organization']['team'] + result = self._run(query)["organization"]["team"] if result is None: break - result = result['members'] - not_end = result['pageInfo']['hasNextPage'] - query = _QUERY.format(organization=self._owner, team=self._team, - max_page_size=self._max_page_size, - next='after: "{}"'.format(result["pageInfo"]["endCursor"])) + result = result["members"] + not_end = result["pageInfo"]["hasNextPage"] + query = _QUERY.format( + organization=self._owner, + team=self._team, + max_page_size=self._max_page_size, + next='after: "{}"'.format(result["pageInfo"]["endCursor"]), + ) - members += dict([(node['login'], node['id']) for node in result['nodes']]) + members += dict([(node["login"], node["id"]) for node in result["nodes"]]) return members def get_pull_request(self, number): - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ pullRequest(number: {number}) {{ {pull_request_data} }} }} - ''' + """ - query = _QUERY.format(owner=self._owner, name=self._name, number=number, - pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) - return self._run(query)['repository']['pullRequest'] + query = _QUERY.format( + owner=self._owner, + name=self._name, + number=number, + pull_request_data=self._PULL_REQUEST, + min_page_size=self._min_page_size, + ) + return self._run(query)["repository"]["pullRequest"] def find_pull_request(self, base, head): - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ pullRequests(first: {min_page_size} baseRefName: "{base}" headRefName: "{head}") {{ nodes {{ @@ -150,21 +161,27 @@ class Query: totalCount }} }} - ''' + """ - query = _QUERY.format(owner=self._owner, name=self._name, base=base, head=head, - pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) - result = self._run(query)['repository']['pullRequests'] - if result['totalCount'] > 0: - return result['nodes'][0] + query = _QUERY.format( + owner=self._owner, + name=self._name, + base=base, + head=head, + pull_request_data=self._PULL_REQUEST, + min_page_size=self._min_page_size, + ) + result = self._run(query)["repository"]["pullRequests"] + if result["totalCount"] > 0: + return result["nodes"][0] else: return {} def find_pull_requests(self, label_name): - ''' + """ Get all pull-requests filtered by label name - ''' - _QUERY = ''' + """ + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ pullRequests(first: {min_page_size} labels: "{label_name}" states: OPEN) {{ nodes {{ @@ -172,18 +189,23 @@ class Query: }} }} }} - ''' + """ - query = _QUERY.format(owner=self._owner, name=self._name, label_name=label_name, - pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) - return self._run(query)['repository']['pullRequests']['nodes'] + query = _QUERY.format( + owner=self._owner, + name=self._name, + label_name=label_name, + pull_request_data=self._PULL_REQUEST, + min_page_size=self._min_page_size, + ) + return self._run(query)["repository"]["pullRequests"]["nodes"] def get_pull_requests(self, before_commit): - ''' + """ Get all merged pull-requests from the HEAD of default branch to the last commit (excluding) - ''' + """ - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ defaultBranchRef {{ target {{ @@ -221,44 +243,60 @@ class Query: }} }} }} - ''' + """ pull_requests = [] not_end = True - query = _QUERY.format(owner=self._owner, name=self._name, - max_page_size=self._max_page_size, - min_page_size=self._min_page_size, - pull_request_data=self._PULL_REQUEST, - next='') + query = _QUERY.format( + owner=self._owner, + name=self._name, + max_page_size=self._max_page_size, + min_page_size=self._min_page_size, + pull_request_data=self._PULL_REQUEST, + next="", + ) while not_end: - result = self._run(query)['repository']['defaultBranchRef']['target']['history'] - not_end = result['pageInfo']['hasNextPage'] - query = _QUERY.format(owner=self._owner, name=self._name, - max_page_size=self._max_page_size, - min_page_size=self._min_page_size, - pull_request_data=self._PULL_REQUEST, - next='after: "{}"'.format(result["pageInfo"]["endCursor"])) + result = self._run(query)["repository"]["defaultBranchRef"]["target"][ + "history" + ] + not_end = result["pageInfo"]["hasNextPage"] + query = _QUERY.format( + owner=self._owner, + name=self._name, + max_page_size=self._max_page_size, + min_page_size=self._min_page_size, + pull_request_data=self._PULL_REQUEST, + next='after: "{}"'.format(result["pageInfo"]["endCursor"]), + ) - for commit in result['nodes']: + for commit in result["nodes"]: # FIXME: maybe include `before_commit`? - if str(commit['oid']) == str(before_commit): + if str(commit["oid"]) == str(before_commit): not_end = False break # TODO: fetch all pull-requests that were merged in a single commit. - assert commit['associatedPullRequests']['totalCount'] <= self._min_page_size + assert ( + commit["associatedPullRequests"]["totalCount"] + <= self._min_page_size + ) - for pull_request in commit['associatedPullRequests']['nodes']: - if(pull_request['baseRepository']['nameWithOwner'] == '{}/{}'.format(self._owner, self._name) and - pull_request['baseRefName'] == self.default_branch and - pull_request['mergeCommit']['oid'] == commit['oid']): + for pull_request in commit["associatedPullRequests"]["nodes"]: + if ( + pull_request["baseRepository"]["nameWithOwner"] + == "{}/{}".format(self._owner, self._name) + and pull_request["baseRefName"] == self.default_branch + and pull_request["mergeCommit"]["oid"] == commit["oid"] + ): pull_requests.append(pull_request) return pull_requests - def create_pull_request(self, source, target, title, description="", draft=False, can_modify=True): - _QUERY = ''' + def create_pull_request( + self, source, target, title, description="", draft=False, can_modify=True + ): + _QUERY = """ createPullRequest(input: {{ baseRefName: "{target}", headRefName: "{source}", @@ -272,15 +310,22 @@ class Query: {pull_request_data} }} }} - ''' + """ - query = _QUERY.format(target=target, source=source, id=self._id, title=title, body=description, - draft="true" if draft else "false", modify="true" if can_modify else "false", - pull_request_data=self._PULL_REQUEST) - return self._run(query, is_mutation=True)['createPullRequest']['pullRequest'] + query = _QUERY.format( + target=target, + source=source, + id=self._id, + title=title, + body=description, + draft="true" if draft else "false", + modify="true" if can_modify else "false", + pull_request_data=self._PULL_REQUEST, + ) + return self._run(query, is_mutation=True)["createPullRequest"]["pullRequest"] def merge_pull_request(self, id): - _QUERY = ''' + _QUERY = """ mergePullRequest(input: {{ pullRequestId: "{id}" }}) {{ @@ -288,35 +333,35 @@ class Query: {pull_request_data} }} }} - ''' + """ query = _QUERY.format(id=id, pull_request_data=self._PULL_REQUEST) - return self._run(query, is_mutation=True)['mergePullRequest']['pullRequest'] + return self._run(query, is_mutation=True)["mergePullRequest"]["pullRequest"] # FIXME: figure out how to add more assignees at once def add_assignee(self, pr, assignee): - _QUERY = ''' + _QUERY = """ addAssigneesToAssignable(input: {{ assignableId: "{id1}", assigneeIds: "{id2}" }}) {{ clientMutationId }} - ''' + """ - query = _QUERY.format(id1=pr['id'], id2=assignee['id']) + query = _QUERY.format(id1=pr["id"], id2=assignee["id"]) self._run(query, is_mutation=True) def set_label(self, pull_request, label_name): - ''' + """ Set label by name to the pull request Args: pull_request: JSON object returned by `get_pull_requests()` label_name (string): label name - ''' + """ - _GET_LABEL = ''' + _GET_LABEL = """ repository(owner: "{owner}" name: "{name}") {{ labels(first: {max_page_size} {next} query: "{label_name}") {{ pageInfo {{ @@ -330,36 +375,44 @@ class Query: }} }} }} - ''' + """ - _SET_LABEL = ''' + _SET_LABEL = """ addLabelsToLabelable(input: {{ labelableId: "{pr_id}", labelIds: "{label_id}" }}) {{ clientMutationId }} - ''' + """ labels = [] not_end = True - query = _GET_LABEL.format(owner=self._owner, name=self._name, label_name=label_name, - max_page_size=self._max_page_size, - next='') + query = _GET_LABEL.format( + owner=self._owner, + name=self._name, + label_name=label_name, + max_page_size=self._max_page_size, + next="", + ) while not_end: - result = self._run(query)['repository']['labels'] - not_end = result['pageInfo']['hasNextPage'] - query = _GET_LABEL.format(owner=self._owner, name=self._name, label_name=label_name, - max_page_size=self._max_page_size, - next='after: "{}"'.format(result["pageInfo"]["endCursor"])) + result = self._run(query)["repository"]["labels"] + not_end = result["pageInfo"]["hasNextPage"] + query = _GET_LABEL.format( + owner=self._owner, + name=self._name, + label_name=label_name, + max_page_size=self._max_page_size, + next='after: "{}"'.format(result["pageInfo"]["endCursor"]), + ) - labels += [label for label in result['nodes']] + labels += [label for label in result["nodes"]] if not labels: return - query = _SET_LABEL.format(pr_id=pull_request['id'], label_id=labels[0]['id']) + query = _SET_LABEL.format(pr_id=pull_request["id"], label_id=labels[0]["id"]) self._run(query, is_mutation=True) def _run(self, query, is_mutation=False): @@ -385,19 +438,21 @@ class Query: status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) return session - headers = {'Authorization': 'bearer {}'.format(self._token)} + headers = {"Authorization": "bearer {}".format(self._token)} if is_mutation: - query = ''' + query = """ mutation {{ {query} }} - '''.format(query=query) + """.format( + query=query + ) else: - query = ''' + query = """ query {{ {query} rateLimit {{ @@ -405,23 +460,38 @@ class Query: remaining }} }} - '''.format(query=query) + """.format( + query=query + ) while True: - request = requests_retry_session().post('https://api.github.com/graphql', json={'query': query}, headers=headers) + request = requests_retry_session().post( + "https://api.github.com/graphql", json={"query": query}, headers=headers + ) if request.status_code == 200: result = request.json() - if 'errors' in result: - raise Exception('Errors occurred: {}\nOriginal query: {}'.format(result["errors"], query)) + if "errors" in result: + raise Exception( + "Errors occurred: {}\nOriginal query: {}".format( + result["errors"], query + ) + ) if not is_mutation: import inspect + caller = inspect.getouterframes(inspect.currentframe(), 2)[1][3] if caller not in list(self.api_costs.keys()): self.api_costs[caller] = 0 - self.api_costs[caller] += result['data']['rateLimit']['cost'] + self.api_costs[caller] += result["data"]["rateLimit"]["cost"] - return result['data'] + return result["data"] else: import json - raise Exception('Query failed with code {code}:\n{json}'.format(code=request.status_code, json=json.dumps(request.json(), indent=4))) + + raise Exception( + "Query failed with code {code}:\n{json}".format( + code=request.status_code, + json=json.dumps(request.json(), indent=4), + ) + ) diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json deleted file mode 100644 index 19afdd172d5..00000000000 --- a/tests/ci/ci_config.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "build_config": [ - { - "compiler": "clang-13", - "build-type": "", - "sanitizer": "", - "package-type": "deb", - "bundled": "bundled", - "splitted": "unsplitted", - "alien_pkgs": true, - "tidy": "disable", - "with_coverage": false - }, - { - "compiler": "clang-13", - "build-type": "", - "sanitizer": "", - "package-type": "performance", - "bundled": "bundled", - "splitted": "unsplitted", - "tidy": "disable", - "with_coverage": false - }, - { - "compiler": "gcc-11", - "build-type": "", - "sanitizer": "", - "package-type": "binary", - "bundled": "bundled", - "splitted": "unsplitted", - "tidy": "disable", - "with_coverage": false - }, - { - "compiler": "clang-13", - "build-type": "", - "sanitizer": "", - "package-type": "binary", - "bundled": "bundled", - "splitted": "unsplitted", - "tidy": "disable", - "with_coverage": false - } - ], - "tests_config": { - "Testflows check": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang-tidy": "disable", - "with_coverage": false - } - }, - "Release": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang-tidy": "disable", - "with_coverage": false - } - }, - "ClickHouse Keeper Jepsen": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "binary", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang-tidy": "disable", - "with_coverage": false - } - } - } -} diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 70fdf06d40b..5c63b3f1ad1 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -14,7 +14,7 @@ CI_CONFIG = { "package_type": "deb", "bundled": "bundled", "splitted": "unsplitted", - "alien_pkgs": True, + "additional_pkgs": True, "tidy": "disable", "with_coverage": False, }, @@ -45,7 +45,7 @@ CI_CONFIG = { "package_type": "deb", "bundled": "bundled", "splitted": "unsplitted", - "alien_pkgs": True, + "additional_pkgs": True, "tidy": "disable", "with_coverage": False, }, @@ -231,7 +231,6 @@ CI_CONFIG = { }, "Stateful tests (aarch64, actions)": { "required_build": "package_aarch64", - "force_tests": True, }, "Stateful tests (release, DatabaseOrdinary, actions)": { "required_build": "package_release", @@ -259,7 +258,6 @@ CI_CONFIG = { }, "Stateless tests (aarch64, actions)": { "required_build": "package_aarch64", - "force_tests": True, }, "Stateless tests (release, wide parts enabled, actions)": { "required_build": "package_release", @@ -270,6 +268,9 @@ CI_CONFIG = { "Stateless tests (release, DatabaseReplicated, actions)": { "required_build": "package_release", }, + "Stateless tests (release, s3 storage, actions)": { + "required_build": "package_release", + }, "Stress test (address, actions)": { "required_build": "package_asan", }, @@ -348,6 +349,9 @@ CI_CONFIG = { "Stateless tests flaky check (address, actions)": { "required_build": "package_asan", }, + "Stateless tests bugfix validate check (address, actions)": { + "required_build": "package_asan", + }, "ClickHouse Keeper Jepsen (actions)": { "required_build": "binary_release", }, diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 0d8aee552f5..9d8a7463b3e 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -6,6 +6,7 @@ import json import requests # type: ignore from get_robot_token import get_parameter_from_ssm + class ClickHouseHelper: def __init__(self, url=None, user=None, password=None): self.url2 = None @@ -15,27 +16,35 @@ class ClickHouseHelper: url = get_parameter_from_ssm("clickhouse-test-stat-url") self.url2 = get_parameter_from_ssm("clickhouse-test-stat-url2") self.auth2 = { - 'X-ClickHouse-User': get_parameter_from_ssm("clickhouse-test-stat-login2"), - 'X-ClickHouse-Key': '' + "X-ClickHouse-User": get_parameter_from_ssm( + "clickhouse-test-stat-login2" + ), + "X-ClickHouse-Key": "", } self.url = url self.auth = { - 'X-ClickHouse-User': user if user is not None else get_parameter_from_ssm("clickhouse-test-stat-login"), - 'X-ClickHouse-Key': password if password is not None else get_parameter_from_ssm("clickhouse-test-stat-password") + "X-ClickHouse-User": user + if user is not None + else get_parameter_from_ssm("clickhouse-test-stat-login"), + "X-ClickHouse-Key": password + if password is not None + else get_parameter_from_ssm("clickhouse-test-stat-password"), } @staticmethod def _insert_json_str_info_impl(url, auth, db, table, json_str): params = { - 'database': db, - 'query': 'INSERT INTO {table} FORMAT JSONEachRow'.format(table=table), - 'date_time_input_format': 'best_effort', - 'send_logs_level': 'warning', + "database": db, + "query": "INSERT INTO {table} FORMAT JSONEachRow".format(table=table), + "date_time_input_format": "best_effort", + "send_logs_level": "warning", } for i in range(5): - response = requests.post(url, params=params, data=json_str, headers=auth, verify=False) + response = requests.post( + url, params=params, data=json_str, headers=auth, verify=False + ) logging.info("Response content '%s'", response.content) @@ -43,16 +52,25 @@ class ClickHouseHelper: break error = ( - "Cannot insert data into clickhouse at try " + str(i) - + ": HTTP code " + str(response.status_code) + ": '" - + str(response.text) + "'") + "Cannot insert data into clickhouse at try " + + str(i) + + ": HTTP code " + + str(response.status_code) + + ": '" + + str(response.text) + + "'" + ) if response.status_code >= 500: # A retriable error time.sleep(1) continue - logging.info("Request headers '%s', body '%s'", response.request.headers, response.request.body) + logging.info( + "Request headers '%s', body '%s'", + response.request.headers, + response.request.body, + ) raise Exception(error) else: @@ -72,18 +90,20 @@ class ClickHouseHelper: for event in events: jsons.append(json.dumps(event)) - self._insert_json_str_info(db, table, ','.join(jsons)) + self._insert_json_str_info(db, table, ",".join(jsons)) def _select_and_get_json_each_row(self, db, query): params = { - 'database': db, - 'query': query, - 'default_format': 'JSONEachRow', + "database": db, + "query": query, + "default_format": "JSONEachRow", } for i in range(5): response = None try: - response = requests.get(self.url, params=params, headers=self.auth, verify=False) + response = requests.get( + self.url, params=params, headers=self.auth, verify=False + ) response.raise_for_status() return response.text except Exception as ex: @@ -97,15 +117,21 @@ class ClickHouseHelper: def select_json_each_row(self, db, query): text = self._select_and_get_json_each_row(db, query) result = [] - for line in text.split('\n'): + for line in text.split("\n"): if line: result.append(json.loads(line)) return result + def prepare_tests_results_for_clickhouse( - pr_info, test_results, - check_status, check_duration, check_start_time, - report_url, check_name): + pr_info, + test_results, + check_status, + check_duration, + check_start_time, + report_url, + check_name, +): pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" base_ref = "master" @@ -147,13 +173,14 @@ def prepare_tests_results_for_clickhouse( test_time = 0 if len(test_result) > 2 and test_result[2]: test_time = test_result[2] - current_row['test_duration_ms'] = int(float(test_time) * 1000) - current_row['test_name'] = test_name - current_row['test_status'] = test_status + current_row["test_duration_ms"] = int(float(test_time) * 1000) + current_row["test_name"] = test_name + current_row["test_status"] = test_status result.append(current_row) return result + def mark_flaky_tests(clickhouse_helper, check_name, test_results): try: query = """ @@ -164,14 +191,16 @@ def mark_flaky_tests(clickhouse_helper, check_name, test_results): AND check_name = '{check_name}' AND (test_status = 'FAIL' OR test_status = 'FLAKY') AND pull_request_number = 0 - """.format(check_name=check_name) + """.format( + check_name=check_name + ) - tests_data = clickhouse_helper.select_json_each_row('gh-data', query) - master_failed_tests = {row['test_name'] for row in tests_data} - logging.info("Found flaky tests: %s", ', '.join(master_failed_tests)) + tests_data = clickhouse_helper.select_json_each_row("gh-data", query) + master_failed_tests = {row["test_name"] for row in tests_data} + logging.info("Found flaky tests: %s", ", ".join(master_failed_tests)) for test_result in test_results: - if test_result[1] == 'FAIL' and test_result[0] in master_failed_tests: - test_result[1] = 'FLAKY' + if test_result[1] == "FAIL" and test_result[0] in master_failed_tests: + test_result[1] = "FLAKY" except Exception as ex: logging.info("Exception happened during flaky tests fetch %s", ex) diff --git a/tests/ci/codebrowser_check.py b/tests/ci/codebrowser_check.py index 97fd58c3235..48c92e9f6ac 100644 --- a/tests/ci/codebrowser_check.py +++ b/tests/ci/codebrowser_check.py @@ -18,13 +18,16 @@ from tee_popen import TeePopen NAME = "Woboq Build (actions)" + def get_run_command(repo_path, output_path, image): - cmd = "docker run " + \ - f"--volume={repo_path}:/repo_folder " \ - f"--volume={output_path}:/test_output " \ - f"-e 'DATA=https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data' {image}" + cmd = ( + "docker run " + f"--volume={repo_path}:/repo_folder " + f"--volume={output_path}:/test_output " + f"-e 'DATA=https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data' {image}" + ) return cmd + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -37,8 +40,8 @@ if __name__ == "__main__": if not os.path.exists(temp_path): os.makedirs(temp_path) - docker_image = get_image_with_version(IMAGES_PATH, 'clickhouse/codebrowser') - s3_helper = S3Helper('https://s3.amazonaws.com') + docker_image = get_image_with_version(IMAGES_PATH, "clickhouse/codebrowser") + s3_helper = S3Helper("https://s3.amazonaws.com") result_path = os.path.join(temp_path, "result_path") if not os.path.exists(result_path): @@ -62,14 +65,20 @@ if __name__ == "__main__": report_path = os.path.join(result_path, "html_report") logging.info("Report path %s", report_path) s3_path_prefix = "codebrowser" - html_urls = s3_helper.fast_parallel_upload_dir(report_path, s3_path_prefix, 'clickhouse-test-reports') + html_urls = s3_helper.fast_parallel_upload_dir( + report_path, s3_path_prefix, "clickhouse-test-reports" + ) index_html = 'HTML report' test_results = [(index_html, "Look at the report")] - report_url = upload_results(s3_helper, 0, os.getenv("GITHUB_SHA"), test_results, [], NAME) + report_url = upload_results( + s3_helper, 0, os.getenv("GITHUB_SHA"), test_results, [], NAME + ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, os.getenv("GITHUB_SHA"), NAME, "Report built", "success", report_url) + post_commit_status( + gh, os.getenv("GITHUB_SHA"), NAME, "Report built", "success", report_url + ) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index dd57f742ff1..e379c9a2254 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -1,15 +1,23 @@ #!/usr/bin/env python3 import time +import os +import csv from env_helper import GITHUB_REPOSITORY from ci_config import CI_CONFIG RETRY = 5 -def override_status(status, check_name): - if CI_CONFIG["tests_config"][check_name].get("force_tests", False): +def override_status(status, check_name, invert=False): + if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False): return "success" + + if invert: + if status == "success": + return "error" + return "success" + return status @@ -43,3 +51,11 @@ def post_commit_status(gh, sha, check_name, description, state, report_url): if i == RETRY - 1: raise ex time.sleep(i) + + +def post_commit_status_to_file(file_path, description, state, report_url): + if os.path.exists(file_path): + raise Exception(f'File "{file_path}" already exists!') + with open(file_path, "w", encoding="utf-8") as f: + out = csv.writer(f, delimiter="\t") + out.writerow([state, report_url, description]) diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index 72626bd6364..d546fabf231 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -16,34 +16,40 @@ from build_download_helper import download_builds_filter from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions from commit_status_helper import post_commit_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper IMAGE_UBUNTU = "clickhouse/test-old-ubuntu" IMAGE_CENTOS = "clickhouse/test-old-centos" -MAX_GLIBC_VERSION = '2.4' +MAX_GLIBC_VERSION = "2.4" DOWNLOAD_RETRIES_COUNT = 5 CHECK_NAME = "Compatibility check (actions)" + def process_os_check(log_path): name = os.path.basename(log_path) - with open(log_path, 'r') as log: - line = log.read().split('\n')[0].strip() - if line != 'OK': + with open(log_path, "r") as log: + line = log.read().split("\n")[0].strip() + if line != "OK": return (name, "FAIL") else: return (name, "OK") + def process_glibc_check(log_path): bad_lines = [] - with open(log_path, 'r') as log: + with open(log_path, "r") as log: for line in log: if line.strip(): - columns = line.strip().split(' ') + columns = line.strip().split(" ") symbol_with_glibc = columns[-2] # sysconf@GLIBC_2.2.5 - _, version = symbol_with_glibc.split('@GLIBC_') - if version == 'PRIVATE': + _, version = symbol_with_glibc.split("@GLIBC_") + if version == "PRIVATE": bad_lines.append((symbol_with_glibc, "FAIL")) elif StrictVersion(version) > MAX_GLIBC_VERSION: bad_lines.append((symbol_with_glibc, "FAIL")) @@ -51,6 +57,7 @@ def process_glibc_check(log_path): bad_lines.append(("glibc check", "OK")) return bad_lines + def process_result(result_folder, server_log_folder): summary = process_glibc_check(os.path.join(result_folder, "glibc.log")) @@ -86,16 +93,18 @@ def process_result(result_folder, server_log_folder): return status, description, summary, result_logs -def get_run_commands(build_path, result_folder, server_log_folder, image_centos, image_ubuntu): +def get_run_commands( + build_path, result_folder, server_log_folder, image_centos, image_ubuntu +): return [ f"readelf -s {build_path}/usr/bin/clickhouse | grep '@GLIBC_' > {result_folder}/glibc.log", f"readelf -s {build_path}/usr/bin/clickhouse-odbc-bridge | grep '@GLIBC_' >> {result_folder}/glibc.log", - f"docker run --network=host --volume={build_path}/usr/bin/clickhouse:/clickhouse " \ - f"--volume={build_path}/etc/clickhouse-server:/config " \ - f"--volume={server_log_folder}:/var/log/clickhouse-server {image_ubuntu} > {result_folder}/ubuntu:12.04", - f"docker run --network=host --volume={build_path}/usr/bin/clickhouse:/clickhouse " \ - f"--volume={build_path}/etc/clickhouse-server:/config " \ - f"--volume={server_log_folder}:/var/log/clickhouse-server {image_centos} > {result_folder}/centos:5", + f"docker run --network=host --volume={build_path}/usr/bin/clickhouse:/clickhouse " + f"--volume={build_path}/etc/clickhouse-server:/config " + f"--volume={server_log_folder}:/var/log/clickhouse-server {image_ubuntu} > {result_folder}/ubuntu:12.04", + f"docker run --network=host --volume={build_path}/usr/bin/clickhouse:/clickhouse " + f"--volume={build_path}/etc/clickhouse-server:/config " + f"--volume={server_log_folder}:/var/log/clickhouse-server {image_centos} > {result_folder}/centos:5", ] @@ -124,14 +133,18 @@ if __name__ == "__main__": os.makedirs(packages_path) def url_filter(url): - return url.endswith('.deb') and ('clickhouse-common-static_' in url or 'clickhouse-server_' in url) + return url.endswith(".deb") and ( + "clickhouse-common-static_" in url or "clickhouse-server_" in url + ) download_builds_filter(CHECK_NAME, reports_path, packages_path, url_filter) for f in os.listdir(packages_path): - if '.deb' in f: + if ".deb" in f: full_path = os.path.join(packages_path, f) - subprocess.check_call(f"dpkg -x {full_path} {packages_path} && rm {full_path}", shell=True) + subprocess.check_call( + f"dpkg -x {full_path} {packages_path} && rm {full_path}", shell=True + ) server_log_path = os.path.join(temp_path, "server_log") if not os.path.exists(server_log_path): @@ -141,7 +154,9 @@ if __name__ == "__main__": if not os.path.exists(result_path): os.makedirs(result_path) - run_commands = get_run_commands(packages_path, result_path, server_log_path, docker_images[0], docker_images[1]) + run_commands = get_run_commands( + packages_path, result_path, server_log_path, docker_images[0], docker_images[1] + ) state = "success" for run_command in run_commands: @@ -154,15 +169,32 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - s3_helper = S3Helper('https://s3.amazonaws.com') - state, description, test_results, additional_logs = process_result(result_path, server_log_path) + s3_helper = S3Helper("https://s3.amazonaws.com") + state, description, test_results, additional_logs = process_result( + result_path, server_log_path + ) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, CHECK_NAME, test_results) - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, CHECK_NAME) + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + additional_logs, + CHECK_NAME, + ) print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, CHECK_NAME, description, state, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, CHECK_NAME) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + CHECK_NAME, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/compress_files.py b/tests/ci/compress_files.py index f3d2349408f..53170a4e9e3 100644 --- a/tests/ci/compress_files.py +++ b/tests/ci/compress_files.py @@ -3,20 +3,21 @@ import subprocess import logging import os + def compress_file_fast(path, archive_path): - if os.path.exists('/usr/bin/pigz'): + if os.path.exists("/usr/bin/pigz"): subprocess.check_call("pigz < {} > {}".format(path, archive_path), shell=True) else: subprocess.check_call("gzip < {} > {}".format(path, archive_path), shell=True) def compress_fast(path, archive_path, exclude=None): - pigz_part = '' - if os.path.exists('/usr/bin/pigz'): + pigz_part = "" + if os.path.exists("/usr/bin/pigz"): logging.info("pigz found, will compress and decompress faster") pigz_part = "--use-compress-program='pigz'" else: - pigz_part = '-z' + pigz_part = "-z" logging.info("no pigz, compressing with default tar") if exclude is None: @@ -31,21 +32,36 @@ def compress_fast(path, archive_path, exclude=None): path = os.path.dirname(path) else: path += "/.." - cmd = "tar {} {} -cf {} -C {} {}".format(pigz_part, exclude_part, archive_path, path, fname) + cmd = "tar {} {} -cf {} -C {} {}".format( + pigz_part, exclude_part, archive_path, path, fname + ) logging.debug("compress_fast cmd: %s", cmd) subprocess.check_call(cmd, shell=True) def decompress_fast(archive_path, result_path=None): - pigz_part = '' - if os.path.exists('/usr/bin/pigz'): - logging.info("pigz found, will compress and decompress faster ('%s' -> '%s')", archive_path, result_path) + pigz_part = "" + if os.path.exists("/usr/bin/pigz"): + logging.info( + "pigz found, will compress and decompress faster ('%s' -> '%s')", + archive_path, + result_path, + ) pigz_part = "--use-compress-program='pigz'" else: - pigz_part = '-z' - logging.info("no pigz, decompressing with default tar ('%s' -> '%s')", archive_path, result_path) + pigz_part = "-z" + logging.info( + "no pigz, decompressing with default tar ('%s' -> '%s')", + archive_path, + result_path, + ) if result_path is None: - subprocess.check_call("tar {} -xf {}".format(pigz_part, archive_path), shell=True) + subprocess.check_call( + "tar {} -xf {}".format(pigz_part, archive_path), shell=True + ) else: - subprocess.check_call("tar {} -xf {} -C {}".format(pigz_part, archive_path, result_path), shell=True) + subprocess.check_call( + "tar {} -xf {} -C {}".format(pigz_part, archive_path, result_path), + shell=True, + ) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 140ede3067f..818478f6430 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -349,14 +349,20 @@ def parse_args() -> argparse.Namespace: help="list of image paths to build instead of using pr_info + diff URL, " "e.g. 'docker/packager/binary'", ) + parser.add_argument("--reports", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-reports", - action="store_true", + action="store_false", + dest="reports", + default=argparse.SUPPRESS, help="don't push reports to S3 and github", ) + parser.add_argument("--push", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-push-images", - action="store_true", + action="store_false", + dest="push", + default=argparse.SUPPRESS, help="don't push images to docker hub", ) @@ -375,8 +381,7 @@ def main(): else: changed_json = os.path.join(TEMP_PATH, "changed_images.json") - push = not args.no_push_images - if push: + if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'robotclickhouse' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), @@ -408,7 +413,7 @@ def main(): images_processing_result = [] for image in changed_images: images_processing_result += process_image_with_parents( - image, image_versions, push + image, image_versions, args.push ) result_images[image.repo] = result_version @@ -437,7 +442,7 @@ def main(): print(f"::notice ::Report url: {url}") print(f'::set-output name=url_output::"{url}"') - if args.no_reports: + if not args.reports: return gh = Github(get_best_robot_token()) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 82d012bfe1a..8bd50819877 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -44,14 +44,20 @@ def parse_args() -> argparse.Namespace: default=RUNNER_TEMP, help="path to changed_images_*.json files", ) + parser.add_argument("--reports", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-reports", - action="store_true", + action="store_false", + dest="reports", + default=argparse.SUPPRESS, help="don't push reports to S3 and github", ) + parser.add_argument("--push", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-push-images", - action="store_true", + action="store_false", + dest="push", + default=argparse.SUPPRESS, help="don't push images to docker hub", ) @@ -63,7 +69,7 @@ def parse_args() -> argparse.Namespace: def load_images(path: str, suffix: str) -> Images: - with open(os.path.join(path, CHANGED_IMAGES.format(suffix)), "r") as images: + with open(os.path.join(path, CHANGED_IMAGES.format(suffix)), "rb") as images: return json.load(images) @@ -125,39 +131,37 @@ def merge_images(to_merge: Dict[str, Images]) -> Dict[str, List[List[str]]]: def create_manifest(image: str, tags: List[str], push: bool) -> Tuple[str, str]: tag = tags[0] manifest = f"{image}:{tag}" - cmd = "docker manifest create --amend {}".format( - " ".join((f"{image}:{t}" for t in tags)) - ) + cmd = "docker manifest create --amend " + " ".join((f"{image}:{t}" for t in tags)) logging.info("running: %s", cmd) - popen = subprocess.Popen( + with subprocess.Popen( cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True, - ) - retcode = popen.wait() - if retcode != 0: - output = popen.stdout.read() # type: ignore - logging.error("failed to create manifest for %s:\n %s\n", manifest, output) - return manifest, "FAIL" - if not push: - return manifest, "OK" + ) as popen: + retcode = popen.wait() + if retcode != 0: + output = popen.stdout.read() # type: ignore + logging.error("failed to create manifest for %s:\n %s\n", manifest, output) + return manifest, "FAIL" + if not push: + return manifest, "OK" cmd = f"docker manifest push {manifest}" logging.info("running: %s", cmd) - popen = subprocess.Popen( + with subprocess.Popen( cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True, - ) - retcode = popen.wait() - if retcode != 0: - output = popen.stdout.read() # type: ignore - logging.error("failed to push %s:\n %s\n", manifest, output) - return manifest, "FAIL" + ) as popen: + retcode = popen.wait() + if retcode != 0: + output = popen.stdout.read() # type: ignore + logging.error("failed to push %s:\n %s\n", manifest, output) + return manifest, "FAIL" return manifest, "OK" @@ -167,8 +171,7 @@ def main(): stopwatch = Stopwatch() args = parse_args() - push = not args.no_push_images - if push: + if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'robotclickhouse' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), @@ -189,12 +192,14 @@ def main(): test_results = [] # type: List[Tuple[str, str]] for image, versions in merged.items(): for tags in versions: - manifest, test_result = create_manifest(image, tags, push) + manifest, test_result = create_manifest(image, tags, args.push) test_results.append((manifest, test_result)) if test_result != "OK": status = "failure" - with open(os.path.join(args.path, "changed_images.json"), "w") as ci: + with open( + os.path.join(args.path, "changed_images.json"), "w", encoding="utf-8" + ) as ci: json.dump(changed_images, ci) pr_info = PRInfo() @@ -202,10 +207,10 @@ def main(): url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME) - print("::notice ::Report url: {}".format(url)) - print('::set-output name=url_output::"{}"'.format(url)) + print(f"::notice ::Report url: {url}") + print(f'::set-output name=url_output::"{url}"') - if args.no_reports: + if not args.reports: return if changed_images: diff --git a/tests/ci/docker_pull_helper.py b/tests/ci/docker_pull_helper.py index 50354da6801..ee7f3337cd9 100644 --- a/tests/ci/docker_pull_helper.py +++ b/tests/ci/docker_pull_helper.py @@ -6,23 +6,29 @@ import time import subprocess import logging +from typing import Optional + + class DockerImage: - def __init__(self, name, version=None): + def __init__(self, name, version: Optional[str] = None): self.name = name if version is None: - self.version = 'latest' + self.version = "latest" else: self.version = version def __str__(self): return f"{self.name}:{self.version}" -def get_images_with_versions(reports_path, required_image, pull=True): + +def get_images_with_versions( + reports_path, required_image, pull=True, version: Optional[str] = None +): images_path = None for root, _, files in os.walk(reports_path): for f in files: - if f == 'changed_images.json': - images_path = os.path.join(root, 'changed_images.json') + if f == "changed_images.json": + images_path = os.path.join(root, "changed_images.json") break if not images_path: @@ -32,7 +38,7 @@ def get_images_with_versions(reports_path, required_image, pull=True): if images_path is not None and os.path.exists(images_path): logging.info("Images file exists") - with open(images_path, 'r', encoding='utf-8') as images_fd: + with open(images_path, "r", encoding="utf-8") as images_fd: images = json.load(images_fd) logging.info("Got images %s", images) else: @@ -40,7 +46,7 @@ def get_images_with_versions(reports_path, required_image, pull=True): docker_images = [] for image_name in required_image: - docker_image = DockerImage(image_name) + docker_image = DockerImage(image_name, version) if image_name in images: docker_image.version = images[image_name] docker_images.append(docker_image) @@ -50,15 +56,22 @@ def get_images_with_versions(reports_path, required_image, pull=True): for i in range(10): try: logging.info("Pulling image %s", docker_image) - latest_error = subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True) + latest_error = subprocess.check_output( + f"docker pull {docker_image}", + stderr=subprocess.STDOUT, + shell=True, + ) break except Exception as ex: time.sleep(i * 3) logging.info("Got execption pulling docker %s", ex) else: - raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image} because of {latest_error}") + raise Exception( + f"Cannot pull dockerhub for image docker pull {docker_image} because of {latest_error}" + ) return docker_images -def get_image_with_version(reports_path, image, pull=True): - return get_images_with_versions(reports_path, [image], pull)[0] + +def get_image_with_version(reports_path, image, pull=True, version=None): + return get_images_with_versions(reports_path, [image], pull, version=version)[0] diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index 23e90aa5b60..58678b160a4 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -40,7 +40,9 @@ if __name__ == "__main__": if not pr_info.has_changes_in_documentation(): logging.info("No changes in documentation") commit = get_commit(gh, pr_info.sha) - commit.create_status(context=NAME, description="No changes in docs", state="success") + commit.create_status( + context=NAME, description="No changes in docs", state="success" + ) sys.exit(0) logging.info("Has changes in docs") @@ -48,15 +50,15 @@ if __name__ == "__main__": if not os.path.exists(temp_path): os.makedirs(temp_path) - docker_image = get_image_with_version(temp_path, 'clickhouse/docs-check') + docker_image = get_image_with_version(temp_path, "clickhouse/docs-check") - test_output = os.path.join(temp_path, 'docs_check_log') + test_output = os.path.join(temp_path, "docs_check_log") if not os.path.exists(test_output): os.makedirs(test_output) cmd = f"docker run --cap-add=SYS_PTRACE --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}" - run_log_path = os.path.join(test_output, 'runlog.log') + run_log_path = os.path.join(test_output, "runlog.log") logging.info("Running command: '%s'", cmd) with TeePopen(cmd, run_log_path) as process: @@ -82,10 +84,10 @@ if __name__ == "__main__": for f in files: path = os.path.join(test_output, f) additional_files.append(path) - with open(path, 'r', encoding='utf-8') as check_file: + with open(path, "r", encoding="utf-8") as check_file: for line in check_file: if "ERROR" in line: - lines.append((line.split(':')[-1], "FAIL")) + lines.append((line.split(":")[-1], "FAIL")) if lines: status = "failure" description = "Found errors in docs" @@ -94,12 +96,22 @@ if __name__ == "__main__": else: lines.append(("Non zero exit code", "FAIL")) - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") ch_helper = ClickHouseHelper() - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files, NAME) + report_url = upload_results( + s3_helper, pr_info.number, pr_info.sha, lines, additional_files, NAME + ) print("::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, NAME, description, status, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, lines, status, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, NAME) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + lines, + status, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + NAME, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/docs_release.py b/tests/ci/docs_release.py index 825bca0b68b..b6d47326f9b 100644 --- a/tests/ci/docs_release.py +++ b/tests/ci/docs_release.py @@ -34,19 +34,23 @@ if __name__ == "__main__": if not os.path.exists(temp_path): os.makedirs(temp_path) - docker_image = get_image_with_version(temp_path, 'clickhouse/docs-release') + docker_image = get_image_with_version(temp_path, "clickhouse/docs-release") - test_output = os.path.join(temp_path, 'docs_release_log') + test_output = os.path.join(temp_path, "docs_release_log") if not os.path.exists(test_output): os.makedirs(test_output) token = CLOUDFLARE_TOKEN - cmd = "docker run --cap-add=SYS_PTRACE --volume=$SSH_AUTH_SOCK:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent " \ - f"-e CLOUDFLARE_TOKEN={token} --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}" + cmd = ( + "docker run --cap-add=SYS_PTRACE --volume=$SSH_AUTH_SOCK:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent " + f"-e CLOUDFLARE_TOKEN={token} --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}" + ) - run_log_path = os.path.join(test_output, 'runlog.log') + run_log_path = os.path.join(test_output, "runlog.log") - with open(run_log_path, 'w', encoding='utf-8') as log, SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): + with open(run_log_path, "w", encoding="utf-8") as log, SSHKey( + "ROBOT_CLICKHOUSE_SSH_KEY" + ): with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process: retcode = process.wait() if retcode == 0: @@ -70,10 +74,10 @@ if __name__ == "__main__": for f in files: path = os.path.join(test_output, f) additional_files.append(path) - with open(path, 'r', encoding='utf-8') as check_file: + with open(path, "r", encoding="utf-8") as check_file: for line in check_file: if "ERROR" in line: - lines.append((line.split(':')[-1], "FAIL")) + lines.append((line.split(":")[-1], "FAIL")) if lines: status = "failure" description = "Found errors in docs" @@ -82,9 +86,13 @@ if __name__ == "__main__": else: lines.append(("Non zero exit code", "FAIL")) - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files, NAME) + report_url = upload_results( + s3_helper, pr_info.number, pr_info.sha, lines, additional_files, NAME + ) print("::notice ::Report url: {report_url}") commit = get_commit(gh, pr_info.sha) - commit.create_status(context=NAME, description=description, state=status, target_url=report_url) + commit.create_status( + context=NAME, description=description, state=status, target_url=report_url + ) diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py new file mode 100644 index 00000000000..16d0f9e4939 --- /dev/null +++ b/tests/ci/download_previous_release.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +########################################################################### +# # +# TODO (@vdimir, @Avogar) # +# Merge with one from https://github.com/ClickHouse/ClickHouse/pull/27928 # +# # +########################################################################### + +import re +import os +import logging + +import requests + +CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" + +CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb" + + +CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = ( + "clickhouse-common-static-dbg_{version}_amd64.deb" +) +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" + +PACKETS_DIR = "previous_release_package_folder/" +VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" + + +class Version: + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return list(map(int, self.version.split("."))) < list( + map(int, other.version.split(".")) + ) + + def __str__(self): + return self.version + + +class ReleaseInfo: + def __init__(self, version, release_type): + self.version = version + self.type = release_type + + def __repr__(self): + return f"ReleaseInfo: {self.version}-{self.type}" + + +def find_previous_release(server_version, releases): + releases.sort(key=lambda x: x.version, reverse=True) + + if server_version is None: + return True, releases[0] + + for release in releases: + if release.version < server_version: + return True, release + + return False, None + + +def get_previous_release(server_version=None): + page = 1 + found = False + while not found: + response = requests.get(CLICKHOUSE_TAGS_URL, {"page": page, "per_page": 100}) + if not response.ok: + raise Exception( + "Cannot load the list of tags from github: " + response.reason + ) + + releases_str = set(re.findall(VERSION_PATTERN, response.text)) + if len(releases_str) == 0: + raise Exception( + "Cannot find previous release for " + + str(server_version) + + " server version" + ) + + releases = list( + map( + lambda x: ReleaseInfo(Version(x.split("-")[0]), x.split("-")[1]), + releases_str, + ) + ) + found, previous_release = find_previous_release(server_version, releases) + page += 1 + + return previous_release + + +def download_packet(url, out_path): + """ + TODO: use dowload_build_with_progress from build_download_helper.py + """ + + response = requests.get(url) + logging.info("Downloading %s", url) + if response.ok: + open(out_path, "wb").write(response.content) + + +def download_packets(release, dest_path=PACKETS_DIR): + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + logging.info("Will download %s", release) + + download_packet( + CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format( + version=release.version, type=release.type + ), + out_path=os.path.join( + dest_path, + CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version), + ), + ) + + download_packet( + CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format( + version=release.version, type=release.type + ), + out_path=os.path.join( + dest_path, + CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version), + ), + ) + + download_packet( + CLICKHOUSE_SERVER_DOWNLOAD_URL.format( + version=release.version, type=release.type + ), + out_path=os.path.join( + dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version) + ), + ) + + download_packet( + CLICKHOUSE_CLIENT_DOWNLOAD_URL.format( + version=release.version, type=release.type + ), + out_path=os.path.join( + dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version) + ), + ) + + +def download_previous_release(dest_path): + current_release = get_previous_release(None) + download_packets(current_release, dest_path=dest_path) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + server_version = Version(input()) + previous_release = get_previous_release(server_version) + download_packets(previous_release) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 0eef886625a..64e04594786 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -15,38 +15,54 @@ from get_robot_token import get_best_robot_token from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen from ccache_utils import get_ccache_if_not_exists, upload_ccache -NAME = 'Fast test (actions)' +NAME = "Fast test (actions)" -def get_fasttest_cmd(workspace, output_path, ccache_path, repo_path, pr_number, commit_sha, image): - return f"docker run --cap-add=SYS_PTRACE " \ - f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " \ - f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " \ - f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} -e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " \ - f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse --volume={output_path}:/test_output "\ + +def get_fasttest_cmd( + workspace, output_path, ccache_path, repo_path, pr_number, commit_sha, image +): + return ( + f"docker run --cap-add=SYS_PTRACE " + f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " + f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " + f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " + f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + f"--volume={output_path}:/test_output " f"--volume={ccache_path}:/fasttest-workspace/ccache {image}" + ) def process_results(result_folder): test_results = [] additional_files = [] # Just upload all files from result_folder. - # If task provides processed results, then it's responsible for content of result_folder. + # If task provides processed results, then it's responsible for content of + # result_folder if os.path.exists(result_folder): - test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))] + test_files = [ + f + for f in os.listdir(result_folder) + if os.path.isfile(os.path.join(result_folder, f)) + ] additional_files = [os.path.join(result_folder, f) for f in test_files] status = [] status_path = os.path.join(result_folder, "check_status.tsv") if os.path.exists(status_path): logging.info("Found test_results.tsv") - with open(status_path, 'r', encoding='utf-8') as status_file: - status = list(csv.reader(status_file, delimiter='\t')) + with open(status_path, "r", encoding="utf-8") as status_file: + status = list(csv.reader(status_file, delimiter="\t")) if len(status) != 1 or len(status[0]) != 2: logging.info("Files in result folder %s", os.listdir(result_folder)) return "error", "Invalid check_status.tsv", test_results, additional_files @@ -54,15 +70,14 @@ def process_results(result_folder): results_path = os.path.join(result_folder, "test_results.tsv") if os.path.exists(results_path): - with open(results_path, 'r', encoding='utf-8') as results_file: - test_results = list(csv.reader(results_file, delimiter='\t')) + with open(results_path, "r", encoding="utf-8") as results_file: + test_results = list(csv.reader(results_file, delimiter="\t")) if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files return state, description, test_results, additional_files - if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -83,9 +98,9 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(temp_path, 'clickhouse/fasttest') + docker_image = get_image_with_version(temp_path, "clickhouse/fasttest") - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") workspace = os.path.join(temp_path, "fasttest-workspace") if not os.path.exists(workspace): @@ -108,15 +123,23 @@ if __name__ == "__main__": if not os.path.exists(repo_path): os.makedirs(repo_path) - run_cmd = get_fasttest_cmd(workspace, output_path, cache_path, repo_path, pr_info.number, pr_info.sha, docker_image) + run_cmd = get_fasttest_cmd( + workspace, + output_path, + cache_path, + repo_path, + pr_info.number, + pr_info.sha, + docker_image, + ) logging.info("Going to run fasttest with cmd %s", run_cmd) logs_path = os.path.join(temp_path, "fasttest-logs") if not os.path.exists(logs_path): os.makedirs(logs_path) - run_log_path = os.path.join(logs_path, 'runlog.log') - with TeePopen(run_cmd, run_log_path) as process: + run_log_path = os.path.join(logs_path, "runlog.log") + with TeePopen(run_cmd, run_log_path, timeout=40 * 60) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") @@ -131,19 +154,21 @@ if __name__ == "__main__": for f in test_output_files: additional_logs.append(os.path.join(output_path, f)) - test_log_exists = 'test_log.txt' in test_output_files or 'test_result.txt' in test_output_files - test_result_exists = 'test_results.tsv' in test_output_files + test_log_exists = ( + "test_log.txt" in test_output_files or "test_result.txt" in test_output_files + ) + test_result_exists = "test_results.tsv" in test_output_files test_results = [] - if 'submodule_log.txt' not in test_output_files: + if "submodule_log.txt" not in test_output_files: description = "Cannot clone repository" state = "failure" - elif 'cmake_log.txt' not in test_output_files: + elif "cmake_log.txt" not in test_output_files: description = "Cannot fetch submodules" state = "failure" - elif 'build_log.txt' not in test_output_files: + elif "build_log.txt" not in test_output_files: description = "Cannot finish cmake" state = "failure" - elif 'install_log.txt' not in test_output_files: + elif "install_log.txt" not in test_output_files: description = "Cannot build ClickHouse" state = "failure" elif not test_log_exists and not test_result_exists: @@ -158,16 +183,32 @@ if __name__ == "__main__": ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, NAME, test_results) - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, NAME, True) - print("::notice ::Report url: {}".format(report_url)) + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + [run_log_path] + additional_logs, + NAME, + True, + ) + print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, NAME, description, state, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, NAME) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + NAME, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) # Refuse other checks to run if fast test failed - if state != 'success': - if 'force-tests' in pr_info.labels: + if state != "success": + if "force-tests" in pr_info.labels: print("'force-tests' enabled, will report success") else: sys.exit(1) diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index 72f26daf4cd..79cea83b1c8 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -7,7 +7,7 @@ from pr_info import PRInfo from get_robot_token import get_best_robot_token from commit_status_helper import get_commit -NAME = 'Run Check (actions)' +NAME = "Run Check (actions)" def filter_statuses(statuses): @@ -36,4 +36,9 @@ if __name__ == "__main__": url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" statuses = filter_statuses(list(commit.get_statuses())) if NAME in statuses and statuses[NAME].state == "pending": - commit.create_status(context=NAME, description="All checks finished", state="success", target_url=url) + commit.create_status( + context=NAME, + description="All checks finished", + state="success", + target_url=url, + ) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 7328db26926..52ec5a0f8e9 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 +import argparse import csv import logging -import subprocess import os +import subprocess import sys from github import Github @@ -13,59 +14,95 @@ from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo from build_download_helper import download_all_deb_packages +from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version -from commit_status_helper import post_commit_status, get_commit, override_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from commit_status_helper import ( + post_commit_status, + get_commit, + override_status, + post_commit_status_to_file, +) +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen +NO_CHANGES_MSG = "Nothing to run" + + def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result = [] - if 'DatabaseReplicated' in check_name: + if "DatabaseReplicated" in check_name: result.append("USE_DATABASE_REPLICATED=1") - if 'DatabaseOrdinary' in check_name: + if "DatabaseOrdinary" in check_name: result.append("USE_DATABASE_ORDINARY=1") - if 'wide parts enabled' in check_name: + if "wide parts enabled" in check_name: result.append("USE_POLYMORPHIC_PARTS=1") + # temporary + if "s3 storage" in check_name: + result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") + if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") result.append(f"RUN_BY_HASH_TOTAL={run_by_hash_total}") return result + def get_image_name(check_name): - if 'stateless' in check_name.lower(): - return 'clickhouse/stateless-test' - if 'stateful' in check_name.lower(): - return 'clickhouse/stateful-test' + if "stateless" in check_name.lower(): + return "clickhouse/stateless-test" + if "stateful" in check_name.lower(): + return "clickhouse/stateful-test" else: raise Exception(f"Cannot deduce image name based on check name {check_name}") -def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): - additional_options = ['--hung-check'] - additional_options.append('--print-time') + +def get_run_command( + builds_path, + repo_tests_path, + result_path, + server_log_path, + kill_timeout, + additional_envs, + image, + flaky_check, + tests_to_run, +): + additional_options = ["--hung-check"] + additional_options.append("--print-time") if tests_to_run: additional_options += tests_to_run - additional_options_str = '-e ADDITIONAL_OPTIONS="' + ' '.join(additional_options) + '"' + additional_options_str = ( + '-e ADDITIONAL_OPTIONS="' + " ".join(additional_options) + '"' + ) - envs = [f'-e MAX_RUN_TIME={int(0.9 * kill_timeout)}', '-e S3_URL="https://clickhouse-datasets.s3.amazonaws.com"'] + envs = [ + f"-e MAX_RUN_TIME={int(0.9 * kill_timeout)}", + '-e S3_URL="https://clickhouse-datasets.s3.amazonaws.com"', + ] if flaky_check: - envs += ['-e NUM_TRIES=100', '-e MAX_RUN_TIME=1800'] + envs += ["-e NUM_TRIES=100", "-e MAX_RUN_TIME=1800"] - envs += [f'-e {e}' for e in additional_envs] + envs += [f"-e {e}" for e in additional_envs] - env_str = ' '.join(envs) + env_str = " ".join(envs) - return f"docker run --volume={builds_path}:/package_folder " \ - f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ - f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " \ + return ( + f"docker run --volume={builds_path}:/package_folder " + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " + f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" + ) def get_tests_to_run(pr_info): @@ -75,32 +112,43 @@ def get_tests_to_run(pr_info): return [] for fpath in pr_info.changed_files: - if 'tests/queries/0_stateless/0' in fpath: - logging.info('File %s changed and seems like stateless test', fpath) - fname = fpath.split('/')[3] + if "tests/queries/0_stateless/0" in fpath: + logging.info("File %s changed and seems like stateless test", fpath) + fname = fpath.split("/")[3] fname_without_ext = os.path.splitext(fname)[0] - result.add(fname_without_ext + '.') + result.add(fname_without_ext + ".") return list(result) + def process_results(result_folder, server_log_path): test_results = [] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible for content of result_folder. if os.path.exists(result_folder): - test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))] + test_files = [ + f + for f in os.listdir(result_folder) + if os.path.isfile(os.path.join(result_folder, f)) + ] additional_files = [os.path.join(result_folder, f) for f in test_files] if os.path.exists(server_log_path): - server_log_files = [f for f in os.listdir(server_log_path) if os.path.isfile(os.path.join(server_log_path, f))] - additional_files = additional_files + [os.path.join(server_log_path, f) for f in server_log_files] + server_log_files = [ + f + for f in os.listdir(server_log_path) + if os.path.isfile(os.path.join(server_log_path, f)) + ] + additional_files = additional_files + [ + os.path.join(server_log_path, f) for f in server_log_files + ] status = [] status_path = os.path.join(result_folder, "check_status.tsv") if os.path.exists(status_path): logging.info("Found test_results.tsv") - with open(status_path, 'r', encoding='utf-8') as status_file: - status = list(csv.reader(status_file, delimiter='\t')) + with open(status_path, "r", encoding="utf-8") as status_file: + status = list(csv.reader(status_file, delimiter="\t")) if len(status) != 1 or len(status[0]) != 2: logging.info("Files in result folder %s", os.listdir(result_folder)) @@ -115,14 +163,32 @@ def process_results(result_folder, server_log_path): logging.info("Files in result folder %s", os.listdir(result_folder)) return "error", "Not found test_results.tsv", test_results, additional_files - with open(results_path, 'r', encoding='utf-8') as results_file: - test_results = list(csv.reader(results_file, delimiter='\t')) + with open(results_path, "r", encoding="utf-8") as results_file: + test_results = list(csv.reader(results_file, delimiter="\t")) if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files return state, description, test_results, additional_files +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("check_name") + parser.add_argument("kill_timeout", type=int) + parser.add_argument( + "--validate-bugfix", + action="store_true", + help="Check that added tests failed on latest stable", + ) + parser.add_argument( + "--post-commit-status", + default="commit_status", + choices=["commit_status", "file"], + help="Where to public post commit status", + ) + return parser.parse_args() + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -132,18 +198,38 @@ if __name__ == "__main__": repo_path = REPO_COPY reports_path = REPORTS_PATH - check_name = sys.argv[1] - kill_timeout = int(sys.argv[2]) + args = parse_args() + check_name = args.check_name + kill_timeout = args.kill_timeout + validate_bugix_check = args.validate_bugfix - flaky_check = 'flaky' in check_name.lower() + flaky_check = "flaky" in check_name.lower() + + run_changed_tests = flaky_check or validate_bugix_check gh = Github(get_best_robot_token()) - pr_info = PRInfo(need_changed_files=flaky_check) + pr_info = PRInfo(need_changed_files=run_changed_tests) - if 'RUN_BY_HASH_NUM' in os.environ: - run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) - run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL')) - check_name_with_group = check_name + f' [{run_by_hash_num + 1}/{run_by_hash_total}]' + if not os.path.exists(temp_path): + os.makedirs(temp_path) + + if validate_bugix_check and "pr-bugfix" not in pr_info.labels: + if args.post_commit_status == "file": + post_commit_status_to_file( + os.path.join(temp_path, "post_commit_status.tsv"), + "Skipped (no pr-bugfix)", + "success", + "null", + ) + logging.info("Skipping '%s' (no pr-bugfix)", check_name) + sys.exit(0) + + if "RUN_BY_HASH_NUM" in os.environ: + run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM")) + run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL")) + check_name_with_group = ( + check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]" + ) else: run_by_hash_num = 0 run_by_hash_total = 0 @@ -154,15 +240,23 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) - if not os.path.exists(temp_path): - os.makedirs(temp_path) - tests_to_run = [] - if flaky_check: + if run_changed_tests: tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: commit = get_commit(gh, pr_info.sha) - commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state='success') + state = override_status("success", check_name, validate_bugix_check) + if args.post_commit_status == "commit_status": + commit.create_status( + context=check_name_with_group, + description=NO_CHANGES_MSG, + state=state, + ) + elif args.post_commit_status == "file": + fpath = os.path.join(temp_path, "post_commit_status.tsv") + post_commit_status_to_file( + fpath, description=NO_CHANGES_MSG, state=state, report_url="null" + ) sys.exit(0) image_name = get_image_name(check_name) @@ -174,7 +268,10 @@ if __name__ == "__main__": if not os.path.exists(packages_path): os.makedirs(packages_path) - download_all_deb_packages(check_name, reports_path, packages_path) + if validate_bugix_check: + download_previous_release(packages_path) + else: + download_all_deb_packages(check_name, reports_path, packages_path) server_log_path = os.path.join(temp_path, "server_log") if not os.path.exists(server_log_path): @@ -186,8 +283,23 @@ if __name__ == "__main__": run_log_path = os.path.join(result_path, "runlog.log") - additional_envs = get_additional_envs(check_name, run_by_hash_num, run_by_hash_total) - run_command = get_run_command(packages_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run) + additional_envs = get_additional_envs( + check_name, run_by_hash_num, run_by_hash_total + ) + if validate_bugix_check: + additional_envs.append("GLOBAL_TAGS=no-random-settings") + + run_command = get_run_command( + packages_path, + repo_tests_path, + result_path, + server_log_path, + kill_timeout, + additional_envs, + docker_image, + flaky_check, + tests_to_run, + ) logging.info("Going to run func tests: %s", run_command) with TeePopen(run_command, run_log_path) as process: @@ -199,24 +311,55 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") - state, description, test_results, additional_logs = process_results(result_path, server_log_path) - state = override_status(state, check_name) + state, description, test_results, additional_logs = process_results( + result_path, server_log_path + ) + state = override_status(state, check_name, validate_bugix_check) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name_with_group) + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + [run_log_path] + additional_logs, + check_name_with_group, + ) - print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + print(f"::notice:: {check_name} Report url: {report_url}") + if args.post_commit_status == "commit_status": + post_commit_status( + gh, pr_info.sha, check_name_with_group, description, state, report_url + ) + elif args.post_commit_status == "file": + post_commit_status_to_file( + os.path.join(temp_path, "post_commit_status.tsv"), + description, + state, + report_url, + ) + else: + raise Exception( + f'Unknown post_commit_status option "{args.post_commit_status}"' + ) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name_with_group) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + check_name_with_group, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) - if state != 'success': - if 'force-tests' in pr_info.labels: + if state != "success": + if "force-tests" in pr_info.labels: print("'force-tests' enabled, will report success") else: sys.exit(1) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index fae277fe319..cb79d9ae01a 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -2,13 +2,15 @@ import boto3 # type: ignore from github import Github # type: ignore + def get_parameter_from_ssm(name, decrypt=True, client=None): if not client: - client = boto3.client('ssm', region_name='us-east-1') - return client.get_parameter(Name=name, WithDecryption=decrypt)['Parameter']['Value'] + client = boto3.client("ssm", region_name="us-east-1") + return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + def get_best_robot_token(token_prefix_env_name="github_robot_token_", total_tokens=4): - client = boto3.client('ssm', region_name='us-east-1') + client = boto3.client("ssm", region_name="us-east-1") tokens = {} for i in range(1, total_tokens + 1): token_name = token_prefix_env_name + str(i) diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py new file mode 100644 index 00000000000..2d28c693087 --- /dev/null +++ b/tests/ci/git_helper.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +import argparse +import os.path as p +import re +import subprocess +from typing import Optional + +# ^ and $ match subline in `multiple\nlines` +# \A and \Z match only start and end of the whole string +RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z" +TAG_REGEXP = ( + r"\Av\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*-(testing|prestable|stable|lts)\Z" +) +SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z" + + +# Py 3.8 removeprefix and removesuffix +def removeprefix(string: str, prefix: str): + if string.startswith(prefix): + return string[len(prefix) :] # noqa: ignore E203, false positive + return string + + +def removesuffix(string: str, suffix: str): + if string.endswith(suffix): + return string[: -len(suffix)] + return string + + +def commit(name: str): + r = re.compile(SHA_REGEXP) + if not r.match(name): + raise argparse.ArgumentTypeError( + "commit hash should contain exactly 40 hex characters" + ) + return name + + +def release_branch(name: str): + r = re.compile(RELEASE_BRANCH_REGEXP) + if not r.match(name): + raise argparse.ArgumentTypeError("release branch should be as 12.1") + return name + + +class Runner: + """lightweight check_output wrapper with stripping last NEW_LINE""" + + def __init__(self, cwd: str = p.dirname(p.realpath(__file__))): + self.cwd = cwd + + def run(self, cmd: str, cwd: Optional[str] = None) -> str: + if cwd is None: + cwd = self.cwd + return subprocess.check_output( + cmd, shell=True, cwd=cwd, encoding="utf-8" + ).strip() + + +class Git: + """A small wrapper around subprocess to invoke git commands""" + + def __init__(self): + runner = Runner() + rel_root = runner.run("git rev-parse --show-cdup") + self.root = p.realpath(p.join(runner.cwd, rel_root)) + self._tag_pattern = re.compile(TAG_REGEXP) + runner.cwd = self.root + self.run = runner.run + self.new_branch = "" + self.branch = "" + self.sha = "" + self.sha_short = "" + self.description = "" + self.commits_since_tag = 0 + self.update() + + def update(self): + """Is used to refresh all attributes after updates, e.g. checkout or commit""" + self.branch = self.run("git branch --show-current") + self.sha = self.run("git rev-parse HEAD") + self.sha_short = self.sha[:11] + # The following command shows the most recent tag in a graph + # Format should match TAG_REGEXP + self.latest_tag = self.run("git describe --tags --abbrev=0") + # Format should be: {latest_tag}-{commits_since_tag}-g{sha_short} + self.description = self.run("git describe --tags --long") + self.commits_since_tag = int( + self.run(f"git rev-list {self.latest_tag}..HEAD --count") + ) + + def _check_tag(self, value: str): + if value == "": + return + if not self._tag_pattern.match(value): + raise Exception(f"last tag {value} doesn't match the pattern") + + @property + def latest_tag(self) -> str: + return self._latest_tag + + @latest_tag.setter + def latest_tag(self, value: str): + self._check_tag(value) + self._latest_tag = value + + @property + def new_tag(self) -> str: + return self._new_tag + + @new_tag.setter + def new_tag(self, value: str): + self._check_tag(value) + self._new_tag = value + + @property + def tweak(self) -> int: + if not self.latest_tag.endswith("-testing"): + # When we are on the tag, we still need to have tweak=1 to not + # break cmake with versions like 12.13.14.0 + return self.commits_since_tag or 1 + + version = self.latest_tag.split("-", maxsplit=1)[0] + return int(version.split(".")[-1]) + self.commits_since_tag diff --git a/tests/ci/git_test.py b/tests/ci/git_test.py new file mode 100644 index 00000000000..785c9b62cce --- /dev/null +++ b/tests/ci/git_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +from unittest.mock import patch +import os.path as p +import unittest + +from git_helper import Git, Runner + + +class TestRunner(unittest.TestCase): + def test_init(self): + runner = Runner() + self.assertEqual(runner.cwd, p.realpath(p.dirname(__file__))) + runner = Runner("/") + self.assertEqual(runner.cwd, "/") + + def test_run(self): + runner = Runner() + output = runner.run("echo 1") + self.assertEqual(output, "1") + + +class TestGit(unittest.TestCase): + def setUp(self): + """we use dummy git object""" + run_patcher = patch("git_helper.Runner.run", return_value="") + self.run_mock = run_patcher.start() + self.addCleanup(run_patcher.stop) + update_patcher = patch("git_helper.Git.update") + update_mock = update_patcher.start() + self.addCleanup(update_patcher.stop) + self.git = Git() + update_mock.assert_called_once() + self.run_mock.assert_called_once() + self.git.new_branch = "NEW_BRANCH_NAME" + self.git.new_tag = "v21.12.333.22222-stable" + self.git.branch = "old_branch" + self.git.sha = "" + self.git.sha_short = "" + self.git.latest_tag = "" + self.git.description = "" + self.git.commits_since_tag = 0 + + def test_tags(self): + self.git.new_tag = "v21.12.333.22222-stable" + self.git.latest_tag = "v21.12.333.22222-stable" + for tag_attr in ("new_tag", "latest_tag"): + self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-stable") + setattr(self.git, tag_attr, "") + self.assertEqual(getattr(self.git, tag_attr), "") + for tag in ( + "v21.12.333-stable", + "v21.12.333-prestable", + "21.12.333.22222-stable", + "v21.12.333.22222-production", + ): + with self.assertRaises(Exception): + setattr(self.git, tag_attr, tag) + + def test_tweak(self): + self.git.commits_since_tag = 0 + self.assertEqual(self.git.tweak, 1) + self.git.commits_since_tag = 2 + self.assertEqual(self.git.tweak, 2) + self.git.latest_tag = "v21.12.333.22222-testing" + self.assertEqual(self.git.tweak, 22224) + self.git.commits_since_tag = 0 + self.assertEqual(self.git.tweak, 22222) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index e87528dd528..30009414d6e 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -import os -import logging -import sys -import json -import subprocess +import argparse import csv +import json +import logging +import os +import subprocess +import sys from github import Github @@ -14,15 +15,26 @@ from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo from build_download_helper import download_all_deb_packages +from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions -from commit_status_helper import post_commit_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from commit_status_helper import ( + post_commit_status, + override_status, + post_commit_status_to_file, +) +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen +# When update, update +# integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too IMAGES = [ "clickhouse/integration-tests-runner", "clickhouse/mysql-golang-client", @@ -32,28 +44,33 @@ IMAGES = [ "clickhouse/postgresql-java-client", "clickhouse/integration-test", "clickhouse/kerberos-kdc", + "clickhouse/kerberized-hadoop", "clickhouse/integration-helper", "clickhouse/dotnet-client", ] -def get_json_params_dict(check_name, pr_info, docker_images, run_by_hash_total, run_by_hash_num): + +def get_json_params_dict( + check_name, pr_info, docker_images, run_by_hash_total, run_by_hash_num +): return { - 'context_name': check_name, - 'commit': pr_info.sha, - 'pull_request': pr_info.number, - 'pr_info': {'changed_files' : list(pr_info.changed_files)}, - 'docker_images_with_versions': docker_images, - 'shuffle_test_groups': False, - 'use_tmpfs': False, - 'disable_net_host': True, - 'run_by_hash_total': run_by_hash_total, - 'run_by_hash_num': run_by_hash_num, + "context_name": check_name, + "commit": pr_info.sha, + "pull_request": pr_info.number, + "pr_info": {"changed_files": list(pr_info.changed_files)}, + "docker_images_with_versions": docker_images, + "shuffle_test_groups": False, + "use_tmpfs": False, + "disable_net_host": True, + "run_by_hash_total": run_by_hash_total, + "run_by_hash_num": run_by_hash_num, } + def get_env_for_runner(build_path, repo_path, result_path, work_path): - binary_path = os.path.join(build_path, 'clickhouse') - odbc_bridge_path = os.path.join(build_path, 'clickhouse-odbc-bridge') - library_bridge_path = os.path.join(build_path, 'clickhouse-library-bridge') + binary_path = os.path.join(build_path, "clickhouse") + odbc_bridge_path = os.path.join(build_path, "clickhouse-odbc-bridge") + library_bridge_path = os.path.join(build_path, "clickhouse-library-bridge") my_env = os.environ.copy() my_env["CLICKHOUSE_TESTS_BUILD_PATH"] = build_path @@ -65,25 +82,30 @@ def get_env_for_runner(build_path, repo_path, result_path, work_path): my_env["CLICKHOUSE_TESTS_RESULT_PATH"] = result_path my_env["CLICKHOUSE_TESTS_BASE_CONFIG_DIR"] = f"{repo_path}/programs/server" my_env["CLICKHOUSE_TESTS_JSON_PARAMS_PATH"] = os.path.join(work_path, "params.json") - my_env["CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER"] = '0' + my_env["CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER"] = "0" return my_env + def process_results(result_folder): test_results = [] additional_files = [] # Just upload all files from result_folder. # If task provides processed results, then it's responsible for content of result_folder. if os.path.exists(result_folder): - test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))] + test_files = [ + f + for f in os.listdir(result_folder) + if os.path.isfile(os.path.join(result_folder, f)) + ] additional_files = [os.path.join(result_folder, f) for f in test_files] status = [] status_path = os.path.join(result_folder, "check_status.tsv") if os.path.exists(status_path): logging.info("Found test_results.tsv") - with open(status_path, 'r', encoding='utf-8') as status_file: - status = list(csv.reader(status_file, delimiter='\t')) + with open(status_path, "r", encoding="utf-8") as status_file: + status = list(csv.reader(status_file, delimiter="\t")) if len(status) != 1 or len(status[0]) != 2: logging.info("Files in result folder %s", os.listdir(result_folder)) @@ -92,13 +114,31 @@ def process_results(result_folder): results_path = os.path.join(result_folder, "test_results.tsv") if os.path.exists(results_path): - with open(results_path, 'r', encoding='utf-8') as results_file: - test_results = list(csv.reader(results_file, delimiter='\t')) + with open(results_path, "r", encoding="utf-8") as results_file: + test_results = list(csv.reader(results_file, delimiter="\t")) if len(test_results) == 0: return "error", "Empty test_results.tsv", test_results, additional_files return state, description, test_results, additional_files + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("check_name") + parser.add_argument( + "--validate-bugfix", + action="store_true", + help="Check that added tests failed on latest stable", + ) + parser.add_argument( + "--post-commit-status", + default="commit_status", + choices=["commit_status", "file"], + help="Where to public post commit status", + ) + return parser.parse_args() + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -108,12 +148,16 @@ if __name__ == "__main__": repo_path = REPO_COPY reports_path = REPORTS_PATH - check_name = sys.argv[1] + args = parse_args() + check_name = args.check_name + validate_bugix_check = args.validate_bugfix - if 'RUN_BY_HASH_NUM' in os.environ: - run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) - run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL')) - check_name_with_group = check_name + f' [{run_by_hash_num + 1}/{run_by_hash_total}]' + if "RUN_BY_HASH_NUM" in os.environ: + run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM")) + run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL")) + check_name_with_group = ( + check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]" + ) else: run_by_hash_num = 0 run_by_hash_total = 0 @@ -122,8 +166,19 @@ if __name__ == "__main__": if not os.path.exists(temp_path): os.makedirs(temp_path) - is_flaky_check = 'flaky' in check_name - pr_info = PRInfo(need_changed_files=is_flaky_check) + is_flaky_check = "flaky" in check_name + pr_info = PRInfo(need_changed_files=is_flaky_check or validate_bugix_check) + + if validate_bugix_check and "pr-bugfix" not in pr_info.labels: + if args.post_commit_status == "file": + post_commit_status_to_file( + os.path.join(temp_path, "post_commit_status.tsv"), + "Skipped (no pr-bugfix)", + "success", + "null", + ) + logging.info("Skipping '%s' (no pr-bugfix)", check_name) + sys.exit(0) gh = Github(get_best_robot_token()) @@ -146,13 +201,26 @@ if __name__ == "__main__": if not os.path.exists(build_path): os.makedirs(build_path) - download_all_deb_packages(check_name, reports_path, build_path) + if validate_bugix_check: + download_previous_release(build_path) + else: + download_all_deb_packages(check_name, reports_path, build_path) my_env = get_env_for_runner(build_path, repo_path, result_path, work_path) - json_path = os.path.join(work_path, 'params.json') - with open(json_path, 'w', encoding='utf-8') as json_params: - json_params.write(json.dumps(get_json_params_dict(check_name, pr_info, images_with_versions, run_by_hash_total, run_by_hash_num))) + json_path = os.path.join(work_path, "params.json") + with open(json_path, "w", encoding="utf-8") as json_params: + json_params.write( + json.dumps( + get_json_params_dict( + check_name, + pr_info, + images_with_versions, + run_by_hash_total, + run_by_hash_num, + ) + ) + ) output_path_log = os.path.join(result_path, "main_script_log.txt") @@ -169,14 +237,46 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) state, description, test_results, additional_logs = process_results(result_path) + state = override_status(state, check_name, validate_bugix_check) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) - s3_helper = S3Helper('https://s3.amazonaws.com') - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [output_path_log] + additional_logs, check_name_with_group, False) - print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + s3_helper = S3Helper("https://s3.amazonaws.com") + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + [output_path_log] + additional_logs, + check_name_with_group, + False, + ) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name_with_group) + print(f"::notice:: {check_name} Report url: {report_url}") + if args.post_commit_status == "commit_status": + post_commit_status( + gh, pr_info.sha, check_name_with_group, description, state, report_url + ) + elif args.post_commit_status == "file": + post_commit_status_to_file( + os.path.join(temp_path, "post_commit_status.tsv"), + description, + state, + report_url, + ) + else: + raise Exception( + f'Unknown post_commit_status option "{args.post_commit_status}"' + ) + + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + check_name_with_group, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/keeper_jepsen_check.py b/tests/ci/keeper_jepsen_check.py index b7acc92b0f3..24d720e67ab 100644 --- a/tests/ci/keeper_jepsen_check.py +++ b/tests/ci/keeper_jepsen_check.py @@ -24,10 +24,10 @@ from ssh import SSHKey from build_download_helper import get_build_name_for_check from rerun_helper import RerunHelper -JEPSEN_GROUP_NAME = 'jepsen_group' +JEPSEN_GROUP_NAME = "jepsen_group" DESIRED_INSTANCE_COUNT = 3 -IMAGE_NAME = 'clickhouse/keeper-jepsen-test' -CHECK_NAME = 'ClickHouse Keeper Jepsen (actions)' +IMAGE_NAME = "clickhouse/keeper-jepsen-test" +CHECK_NAME = "ClickHouse Keeper Jepsen (actions)" SUCCESSFUL_TESTS_ANCHOR = "# Successful tests" @@ -35,45 +35,58 @@ INTERMINATE_TESTS_ANCHOR = "# Indeterminate tests" CRASHED_TESTS_ANCHOR = "# Crashed tests" FAILED_TESTS_ANCHOR = "# Failed tests" + def _parse_jepsen_output(path): test_results = [] - current_type = '' - with open(path, 'r') as f: + current_type = "" + with open(path, "r") as f: for line in f: if SUCCESSFUL_TESTS_ANCHOR in line: - current_type = 'OK' + current_type = "OK" elif INTERMINATE_TESTS_ANCHOR in line or CRASHED_TESTS_ANCHOR in line: - current_type = 'ERROR' + current_type = "ERROR" elif FAILED_TESTS_ANCHOR in line: - current_type = 'FAIL' + current_type = "FAIL" - if (line.startswith('store/clickhouse-keeper') or line.startswith('clickhouse-keeper')) and current_type: + if ( + line.startswith("store/clickhouse-keeper") + or line.startswith("clickhouse-keeper") + ) and current_type: test_results.append((line.strip(), current_type)) return test_results + def get_autoscaling_group_instances_ids(asg_client, group_name): - group_description = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name]) - our_group = group_description['AutoScalingGroups'][0] + group_description = asg_client.describe_auto_scaling_groups( + AutoScalingGroupNames=[group_name] + ) + our_group = group_description["AutoScalingGroups"][0] instance_ids = [] - for instance in our_group['Instances']: - if instance['LifecycleState'] == 'InService' and instance['HealthStatus'] == 'Healthy': - instance_ids.append(instance['InstanceId']) + for instance in our_group["Instances"]: + if ( + instance["LifecycleState"] == "InService" + and instance["HealthStatus"] == "Healthy" + ): + instance_ids.append(instance["InstanceId"]) return instance_ids + def get_instances_addresses(ec2_client, instance_ids): - ec2_response = ec2_client.describe_instances(InstanceIds = instance_ids) + ec2_response = ec2_client.describe_instances(InstanceIds=instance_ids) instance_ips = [] - for instances in ec2_response['Reservations']: - for ip in instances['Instances']: - instance_ips.append(ip['PrivateIpAddress']) + for instances in ec2_response["Reservations"]: + for ip in instances["Instances"]: + instance_ips.append(ip["PrivateIpAddress"]) return instance_ips def prepare_autoscaling_group_and_get_hostnames(): - asg_client = boto3.client('autoscaling', region_name='us-east-1') - asg_client.set_desired_capacity(AutoScalingGroupName=JEPSEN_GROUP_NAME, DesiredCapacity=DESIRED_INSTANCE_COUNT) + asg_client = boto3.client("autoscaling", region_name="us-east-1") + asg_client.set_desired_capacity( + AutoScalingGroupName=JEPSEN_GROUP_NAME, DesiredCapacity=DESIRED_INSTANCE_COUNT + ) instances = get_autoscaling_group_instances_ids(asg_client, JEPSEN_GROUP_NAME) counter = 0 @@ -84,13 +97,15 @@ def prepare_autoscaling_group_and_get_hostnames(): if counter > 30: raise Exception("Cannot wait autoscaling group") - ec2_client = boto3.client('ec2', region_name='us-east-1') + ec2_client = boto3.client("ec2", region_name="us-east-1") return get_instances_addresses(ec2_client, instances) def clear_autoscaling_group(): - asg_client = boto3.client('autoscaling', region_name='us-east-1') - asg_client.set_desired_capacity(AutoScalingGroupName=JEPSEN_GROUP_NAME, DesiredCapacity=0) + asg_client = boto3.client("autoscaling", region_name="us-east-1") + asg_client.set_desired_capacity( + AutoScalingGroupName=JEPSEN_GROUP_NAME, DesiredCapacity=0 + ) instances = get_autoscaling_group_instances_ids(asg_client, JEPSEN_GROUP_NAME) counter = 0 while len(instances) > 0: @@ -103,15 +118,28 @@ def clear_autoscaling_group(): def save_nodes_to_file(instances, temp_path): nodes_path = os.path.join(temp_path, "nodes.txt") - with open(nodes_path, 'w') as f: + with open(nodes_path, "w") as f: f.write("\n".join(instances)) f.flush() return nodes_path -def get_run_command(ssh_auth_sock, ssh_sock_dir, pr_info, nodes_path, repo_path, build_url, result_path, docker_image): - return f"docker run --network=host -v '{ssh_sock_dir}:{ssh_sock_dir}' -e SSH_AUTH_SOCK={ssh_auth_sock} " \ - f"-e PR_TO_TEST={pr_info.number} -e SHA_TO_TEST={pr_info.sha} -v '{nodes_path}:/nodes.txt' -v {result_path}:/test_output " \ - f"-e 'CLICKHOUSE_PACKAGE={build_url}' -v '{repo_path}:/ch' -e 'CLICKHOUSE_REPO_PATH=/ch' -e NODES_USERNAME=ubuntu {docker_image}" + +def get_run_command( + ssh_auth_sock, + ssh_sock_dir, + pr_info, + nodes_path, + repo_path, + build_url, + result_path, + docker_image, +): + return ( + f"docker run --network=host -v '{ssh_sock_dir}:{ssh_sock_dir}' -e SSH_AUTH_SOCK={ssh_auth_sock} " + f"-e PR_TO_TEST={pr_info.number} -e SHA_TO_TEST={pr_info.sha} -v '{nodes_path}:/nodes.txt' -v {result_path}:/test_output " + f"-e 'CLICKHOUSE_PACKAGE={build_url}' -v '{repo_path}:/ch' -e 'CLICKHOUSE_REPO_PATH=/ch' -e NODES_USERNAME=ubuntu {docker_image}" + ) + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -120,9 +148,14 @@ if __name__ == "__main__": pr_info = PRInfo() - logging.info("Start at PR number %s, commit sha %s labels %s", pr_info.number, pr_info.sha, pr_info.labels) + logging.info( + "Start at PR number %s, commit sha %s labels %s", + pr_info.number, + pr_info.sha, + pr_info.labels, + ) - if pr_info.number != 0 and 'jepsen-test' not in pr_info.labels: + if pr_info.number != 0 and "jepsen-test" not in pr_info.labels: logging.info("Not jepsen test label in labels list, skipping") sys.exit(0) @@ -149,8 +182,8 @@ if __name__ == "__main__": build_name = get_build_name_for_check(CHECK_NAME) if pr_info.number == 0: - version = get_version_from_repo(REPO_COPY) - release_or_pr = ".".join(version.as_tuple()[:2]) + version = get_version_from_repo() + release_or_pr = f"{version.major}.{version.minor}" else: # PR number for anything else release_or_pr = str(pr_info.number) @@ -167,13 +200,22 @@ if __name__ == "__main__": head = requests.head(build_url) counter += 1 if counter >= 180: - post_commit_status(gh, pr_info.sha, CHECK_NAME, "Cannot fetch build to run", "error", "") - raise Exception("Cannot fetch build") + logging.warning("Cannot fetch build in 30 minutes, exiting") + sys.exit(0) - with SSHKey(key_value=get_parameter_from_ssm("jepsen_ssh_key") + '\n'): - ssh_auth_sock = os.environ['SSH_AUTH_SOCK'] + with SSHKey(key_value=get_parameter_from_ssm("jepsen_ssh_key") + "\n"): + ssh_auth_sock = os.environ["SSH_AUTH_SOCK"] auth_sock_dir = os.path.dirname(ssh_auth_sock) - cmd = get_run_command(ssh_auth_sock, auth_sock_dir, pr_info, nodes_path, REPO_COPY, build_url, result_path, docker_image) + cmd = get_run_command( + ssh_auth_sock, + auth_sock_dir, + pr_info, + nodes_path, + REPO_COPY, + build_url, + result_path, + docker_image, + ) logging.info("Going to run jepsen: %s", cmd) run_log_path = os.path.join(TEMP_PATH, "runlog.log") @@ -185,31 +227,49 @@ if __name__ == "__main__": else: logging.info("Run failed") - status = 'success' - description = 'No invalid analysis found ヽ(‘ー`)ノ' - jepsen_log_path = os.path.join(result_path, 'jepsen_run_all_tests.log') + status = "success" + description = "No invalid analysis found ヽ(‘ー`)ノ" + jepsen_log_path = os.path.join(result_path, "jepsen_run_all_tests.log") additional_data = [] try: test_result = _parse_jepsen_output(jepsen_log_path) - if any(r[1] == 'FAIL' for r in test_result): - status = 'failure' - description = 'Found invalid analysis (ﾉಥ益ಥ）ﾉ ┻━┻' + if any(r[1] == "FAIL" for r in test_result): + status = "failure" + description = "Found invalid analysis (ﾉಥ益ಥ）ﾉ ┻━┻" - compress_fast(os.path.join(result_path, 'store'), os.path.join(result_path, 'jepsen_store.tar.gz')) - additional_data.append(os.path.join(result_path, 'jepsen_store.tar.gz')) + compress_fast( + os.path.join(result_path, "store"), + os.path.join(result_path, "jepsen_store.tar.gz"), + ) + additional_data.append(os.path.join(result_path, "jepsen_store.tar.gz")) except Exception as ex: print("Exception", ex) - status = 'failure' - description = 'No Jepsen output log' - test_result = [('No Jepsen output log', 'FAIL')] + status = "failure" + description = "No Jepsen output log" + test_result = [("No Jepsen output log", "FAIL")] - s3_helper = S3Helper('https://s3.amazonaws.com') - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_result, [run_log_path] + additional_data, CHECK_NAME) + s3_helper = S3Helper("https://s3.amazonaws.com") + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_result, + [run_log_path] + additional_data, + CHECK_NAME, + ) print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, CHECK_NAME, description, status, report_url) ch_helper = ClickHouseHelper() - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_result, status, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, CHECK_NAME) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_result, + status, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + CHECK_NAME, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) clear_autoscaling_group() diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index ea2f3c5196a..761b1ac9257 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -19,13 +19,26 @@ from commit_status_helper import get_commit, post_commit_status from tee_popen import TeePopen from rerun_helper import RerunHelper -IMAGE_NAME = 'clickhouse/performance-comparison' +IMAGE_NAME = "clickhouse/performance-comparison" -def get_run_command(workspace, result_path, pr_to_test, sha_to_test, additional_env, image): - return f"docker run --privileged --volume={workspace}:/workspace --volume={result_path}:/output " \ - f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio " \ - f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} " \ + +def get_run_command( + workspace, + result_path, + repo_tests_path, + pr_to_test, + sha_to_test, + additional_env, + image, +): + return ( + f"docker run --privileged --volume={workspace}:/workspace --volume={result_path}:/output " + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " + f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio " + f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} " f"{image}" + ) + class RamDrive: def __init__(self, path, size): @@ -36,18 +49,22 @@ class RamDrive: if not os.path.exists(self.path): os.makedirs(self.path) - subprocess.check_call(f"sudo mount -t tmpfs -o rw,size={self.size} tmpfs {self.path}", shell=True) + subprocess.check_call( + f"sudo mount -t tmpfs -o rw,size={self.size} tmpfs {self.path}", shell=True + ) def __exit__(self, exc_type, exc_val, exc_tb): subprocess.check_call(f"sudo umount {self.path}", shell=True) + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) temp_path = os.getenv("TEMP_PATH", os.path.abspath(".")) repo_path = os.getenv("REPO_COPY", os.path.abspath("../../")) + repo_tests_path = os.path.join(repo_path, "tests") ramdrive_path = os.getenv("RAMDRIVE_PATH", os.path.join(temp_path, "ramdrive")) # currently unused, doesn't make tests more stable - ramdrive_size = os.getenv("RAMDRIVE_SIZE", '0G') + ramdrive_size = os.getenv("RAMDRIVE_SIZE", "0G") reports_path = os.getenv("REPORTS_PATH", "./reports") check_name = sys.argv[1] @@ -55,14 +72,14 @@ if __name__ == "__main__": if not os.path.exists(temp_path): os.makedirs(temp_path) - with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file: + with open(os.getenv("GITHUB_EVENT_PATH"), "r", encoding="utf-8") as event_file: event = json.load(event_file) gh = Github(get_best_robot_token()) pr_info = PRInfo(event) commit = get_commit(gh, pr_info.sha) - docker_env = '' + docker_env = "" docker_env += " -e S3_URL=https://s3.amazonaws.com/clickhouse-builds" @@ -73,13 +90,16 @@ if __name__ == "__main__": task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}" docker_env += ' -e CHPC_ADD_REPORT_LINKS="Job (actions) Tested commit"'.format( - task_url, pr_link) + task_url, pr_link + ) - if 'RUN_BY_HASH_TOTAL' in os.environ: - run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL')) - run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) - docker_env += f' -e CHPC_TEST_RUN_BY_HASH_TOTAL={run_by_hash_total} -e CHPC_TEST_RUN_BY_HASH_NUM={run_by_hash_num}' - check_name_with_group = check_name + f' [{run_by_hash_num + 1}/{run_by_hash_total}]' + if "RUN_BY_HASH_TOTAL" in os.environ: + run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL")) + run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM")) + docker_env += f" -e CHPC_TEST_RUN_BY_HASH_TOTAL={run_by_hash_total} -e CHPC_TEST_RUN_BY_HASH_NUM={run_by_hash_num}" + check_name_with_group = ( + check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]" + ) else: check_name_with_group = check_name @@ -90,12 +110,20 @@ if __name__ == "__main__": docker_image = get_image_with_version(reports_path, IMAGE_NAME) - #with RamDrive(ramdrive_path, ramdrive_size): + # with RamDrive(ramdrive_path, ramdrive_size): result_path = ramdrive_path if not os.path.exists(result_path): os.makedirs(result_path) - run_command = get_run_command(result_path, result_path, pr_info.number, pr_info.sha, docker_env, docker_image) + run_command = get_run_command( + result_path, + result_path, + repo_tests_path, + pr_info.number, + pr_info.sha, + docker_env, + docker_image, + ) logging.info("Going to run command %s", run_command) run_log_path = os.path.join(temp_path, "runlog.log") with TeePopen(run_command, run_log_path) as process: @@ -108,74 +136,83 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) paths = { - 'compare.log': os.path.join(result_path, 'compare.log'), - 'output.7z': os.path.join(result_path, 'output.7z'), - 'report.html': os.path.join(result_path, 'report.html'), - 'all-queries.html': os.path.join(result_path, 'all-queries.html'), - 'queries.rep': os.path.join(result_path, 'queries.rep'), - 'all-query-metrics.tsv': os.path.join(result_path, 'report/all-query-metrics.tsv'), - 'runlog.log': run_log_path, + "compare.log": os.path.join(result_path, "compare.log"), + "output.7z": os.path.join(result_path, "output.7z"), + "report.html": os.path.join(result_path, "report.html"), + "all-queries.html": os.path.join(result_path, "all-queries.html"), + "queries.rep": os.path.join(result_path, "queries.rep"), + "all-query-metrics.tsv": os.path.join( + result_path, "report/all-query-metrics.tsv" + ), + "runlog.log": run_log_path, } - check_name_prefix = check_name_with_group.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_') - s3_prefix = f'{pr_info.number}/{pr_info.sha}/{check_name_prefix}/' - s3_helper = S3Helper('https://s3.amazonaws.com') + check_name_prefix = ( + check_name_with_group.lower() + .replace(" ", "_") + .replace("(", "_") + .replace(")", "_") + .replace(",", "_") + ) + s3_prefix = f"{pr_info.number}/{pr_info.sha}/{check_name_prefix}/" + s3_helper = S3Helper("https://s3.amazonaws.com") for file in paths: try: - paths[file] = s3_helper.upload_test_report_to_s3(paths[file], - s3_prefix + file) + paths[file] = s3_helper.upload_test_report_to_s3( + paths[file], s3_prefix + file + ) except Exception: - paths[file] = '' + paths[file] = "" traceback.print_exc() # Upload all images and flamegraphs to S3 try: s3_helper.upload_test_folder_to_s3( - os.path.join(result_path, 'images'), - s3_prefix + 'images' + os.path.join(result_path, "images"), s3_prefix + "images" ) except Exception: traceback.print_exc() # Try to fetch status from the report. - status = '' - message = '' + status = "" + message = "" try: - report_text = open(os.path.join(result_path, 'report.html'), 'r').read() - status_match = re.search('', report_text) - message_match = re.search('', report_text) + report_text = open(os.path.join(result_path, "report.html"), "r").read() + status_match = re.search("", report_text) + message_match = re.search("", report_text) if status_match: status = status_match.group(1).strip() if message_match: message = message_match.group(1).strip() # TODO: Remove me, always green mode for the first time - status = 'success' + status = "success" except Exception: traceback.print_exc() - status = 'failure' - message = 'Failed to parse the report.' + status = "failure" + message = "Failed to parse the report." if not status: - status = 'failure' - message = 'No status in report.' + status = "failure" + message = "No status in report." elif not message: - status = 'failure' - message = 'No message in report.' + status = "failure" + message = "No message in report." report_url = task_url - if paths['runlog.log']: - report_url = paths['runlog.log'] + if paths["runlog.log"]: + report_url = paths["runlog.log"] - if paths['compare.log']: - report_url = paths['compare.log'] + if paths["compare.log"]: + report_url = paths["compare.log"] - if paths['output.7z']: - report_url = paths['output.7z'] + if paths["output.7z"]: + report_url = paths["output.7z"] - if paths['report.html']: - report_url = paths['report.html'] + if paths["report.html"]: + report_url = paths["report.html"] - - post_commit_status(gh, pr_info.sha, check_name_with_group, message, status, report_url) + post_commit_status( + gh, pr_info.sha, check_name_with_group, message, status, report_url + ) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 1a6ed5645de..64e22712059 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -78,7 +78,7 @@ class PRInfo: else: github_event = PRInfo.default_event.copy() self.event = github_event - self.changed_files = set([]) + self.changed_files = set() self.body = "" ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): @@ -159,6 +159,7 @@ class PRInfo: f"compare/{github_event['before']}...{self.sha}" ) else: + self.number = pull_request["number"] self.labels = {label["name"] for label in pull_request["labels"]} self.base_ref = pull_request["base"]["ref"] @@ -208,6 +209,7 @@ class PRInfo: else: diff_object = PatchSet(response.text) self.changed_files = {f.path for f in diff_object} + print("Fetched info about %d changed files", len(self.changed_files)) def get_dict(self): return { diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index d2241d10b6c..ccbf1918602 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -4,23 +4,12 @@ import argparse import logging import os import re -from typing import Tuple +from typing import List, Tuple from artifactory import ArtifactorySaaSPath # type: ignore from build_download_helper import dowload_build_with_progress - - -# Py 3.8 removeprefix and removesuffix -def removeprefix(string: str, prefix: str): - if string.startswith(prefix): - return string[len(prefix) :] # noqa: ignore E203, false positive - return string - - -def removesuffix(string: str, suffix: str): - if string.endswith(suffix): - return string[: -len(suffix)] - return string +from env_helper import RUNNER_TEMP +from git_helper import TAG_REGEXP, commit, removeprefix, removesuffix # Necessary ENV variables @@ -31,7 +20,7 @@ def getenv(name: str, default: str = None): raise KeyError(f"Necessary {name} environment is not set") -TEMP_PATH = getenv("TEMP_PATH", ".") +TEMP_PATH = os.path.join(RUNNER_TEMP, "push_to_artifactory") # One of the following ENVs is necessary JFROG_API_KEY = getenv("JFROG_API_KEY", "") JFROG_TOKEN = getenv("JFROG_TOKEN", "") @@ -44,7 +33,6 @@ class Packages: ("clickhouse-common-static", "amd64"), ("clickhouse-common-static-dbg", "amd64"), ("clickhouse-server", "all"), - ("clickhouse-test", "all"), ) def __init__(self, version: str): @@ -58,11 +46,11 @@ class Packages: for name, arch in self.packages ) - self.tgz = tuple("{}-{}.tgz".format(name, version) for name, _ in self.packages) + self.tgz = tuple(f"{name}-{version}.tgz" for name, _ in self.packages) def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: - raise ValueError("{} not in {}".format(deb_pkg, self.deb)) + raise ValueError(f"{deb_pkg} not in {self.deb}") return removesuffix(deb_pkg, ".deb").split("_")[-1] @staticmethod @@ -124,7 +112,7 @@ class S3: class Release: def __init__(self, name: str): - r = re.compile(r"^v\d{2}[.]\d+[.]\d+[.]\d+-(testing|prestable|stable|lts)$") + r = re.compile(TAG_REGEXP) # Automatically remove refs/tags/ if full refname passed here name = removeprefix(name, "refs/tags/") if not r.match(name): @@ -212,15 +200,6 @@ class Artifactory: return self.__path_helper("_tgz", package_file) -def commit(name: str): - r = re.compile(r"^([0-9]|[a-f]){40}$") - if not r.match(name): - raise argparse.ArgumentTypeError( - "commit hash should contain exactly 40 hex characters" - ) - return name - - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -275,15 +254,21 @@ def parse_args() -> argparse.Namespace: default="https://clickhousedb.jfrog.io/artifactory", help="SaaS Artifactory url", ) + parser.add_argument("--artifactory", default=True, help=argparse.SUPPRESS) parser.add_argument( "-n", "--no-artifactory", - action="store_true", + action="store_false", + dest="artifactory", + default=argparse.SUPPRESS, help="do not push packages to artifactory", ) + parser.add_argument("--force-download", default=True, help=argparse.SUPPRESS) parser.add_argument( "--no-force-download", - action="store_true", + action="store_false", + dest="force_download", + default=argparse.SUPPRESS, help="do not download packages again if they exist already", ) @@ -298,45 +283,48 @@ def parse_args() -> argparse.Namespace: return args -def process_deb(s3: S3, art_client: Artifactory): +def process_deb(s3: S3, art_clients: List[Artifactory]): s3.download_deb() - if art_client is not None: + for art_client in art_clients: art_client.deploy_deb(s3.packages) -def process_rpm(s3: S3, art_client: Artifactory): +def process_rpm(s3: S3, art_clients: List[Artifactory]): s3.download_rpm() - if art_client is not None: + for art_client in art_clients: art_client.deploy_rpm(s3.packages) -def process_tgz(s3: S3, art_client: Artifactory): +def process_tgz(s3: S3, art_clients: List[Artifactory]): s3.download_tgz() - if art_client is not None: + for art_client in art_clients: art_client.deploy_tgz(s3.packages) def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") args = parse_args() + os.makedirs(TEMP_PATH, exist_ok=True) s3 = S3( args.bucket_name, args.pull_request, args.commit, args.check_name, args.release.version, - not args.no_force_download, + args.force_download, ) - art_client = None - if not args.no_artifactory: - art_client = Artifactory(args.artifactory_url, args.release.type) + art_clients = [] + if args.artifactory: + art_clients.append(Artifactory(args.artifactory_url, args.release.type)) + if args.release.type == "lts": + art_clients.append(Artifactory(args.artifactory_url, "stable")) if args.deb: - process_deb(s3, art_client) + process_deb(s3, art_clients) if args.rpm: - process_rpm(s3, art_client) + process_rpm(s3, art_clients) if args.tgz: - process_tgz(s3, art_client) + process_tgz(s3, art_clients) if __name__ == "__main__": diff --git a/tests/ci/pvs_check.py b/tests/ci/pvs_check.py deleted file mode 100644 index af543211c16..00000000000 --- a/tests/ci/pvs_check.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 - -# pylint: disable=line-too-long - -import os -import json -import logging -import sys -from github import Github - -from env_helper import REPO_COPY, TEMP_PATH, GITHUB_RUN_ID, GITHUB_REPOSITORY, GITHUB_SERVER_URL -from s3_helper import S3Helper -from pr_info import PRInfo -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -from upload_result_helper import upload_results -from commit_status_helper import get_commit -from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from stopwatch import Stopwatch -from rerun_helper import RerunHelper -from tee_popen import TeePopen - -NAME = 'PVS Studio (actions)' -LICENCE_NAME = 'Free license: ClickHouse, Yandex' -HTML_REPORT_FOLDER = 'pvs-studio-html-report' -TXT_REPORT_NAME = 'pvs-studio-task-report.txt' - - -def _process_txt_report(path): - warnings = [] - errors = [] - with open(path, 'r') as report_file: - for line in report_file: - if 'viva64' in line: - continue - - if 'warn' in line: - warnings.append(':'.join(line.split('\t')[0:2])) - elif 'err' in line: - errors.append(':'.join(line.split('\t')[0:2])) - - return warnings, errors - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - stopwatch = Stopwatch() - - repo_path = REPO_COPY - temp_path = TEMP_PATH - - pr_info = PRInfo() - # this check modify repository so copy it to the temp directory - logging.info("Repo copy path %s", repo_path) - - gh = Github(get_best_robot_token()) - rerun_helper = RerunHelper(gh, pr_info, NAME) - if rerun_helper.is_already_finished_by_status(): - logging.info("Check is already finished according to github status, exiting") - sys.exit(0) - - images_path = os.path.join(temp_path, 'changed_images.json') - docker_image = 'clickhouse/pvs-test' - if os.path.exists(images_path): - logging.info("Images file exists") - with open(images_path, 'r') as images_fd: - images = json.load(images_fd) - logging.info("Got images %s", images) - if 'clickhouse/pvs-test' in images: - docker_image += ':' + images['clickhouse/pvs-test'] - - logging.info("Got docker image %s", docker_image) - - s3_helper = S3Helper('https://s3.amazonaws.com') - - licence_key = get_parameter_from_ssm('pvs_studio_key') - cmd = f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --volume={repo_path}:/repo_folder --volume={temp_path}:/test_output -e LICENCE_NAME='{LICENCE_NAME}' -e LICENCE_KEY='{licence_key}' {docker_image}" - commit = get_commit(gh, pr_info.sha) - - run_log_path = os.path.join(temp_path, 'run_log.log') - - with TeePopen(cmd, run_log_path) as process: - retcode = process.wait() - if retcode != 0: - logging.info("Run failed") - else: - logging.info("Run Ok") - - if retcode != 0: - commit.create_status(context=NAME, description='PVS report failed to build', state='error', - target_url=f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}") - sys.exit(1) - - try: - s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_') - html_urls = s3_helper.upload_test_folder_to_s3(os.path.join(temp_path, HTML_REPORT_FOLDER), s3_path_prefix) - index_html = None - - for url in html_urls: - if 'index.html' in url: - index_html = 'HTML report'.format(url) - break - - if not index_html: - commit.create_status(context=NAME, description='PVS report failed to build', state='error', - target_url=f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}") - sys.exit(1) - - txt_report = os.path.join(temp_path, TXT_REPORT_NAME) - warnings, errors = _process_txt_report(txt_report) - errors = errors + warnings - - status = 'success' - test_results = [(index_html, "Look at the report"), ("Errors count not checked", "OK")] - description = "Total errors {}".format(len(errors)) - additional_logs = [txt_report, os.path.join(temp_path, 'pvs-studio.log')] - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, NAME) - - print("::notice ::Report url: {}".format(report_url)) - commit = get_commit(gh, pr_info.sha) - commit.create_status(context=NAME, description=description, state=status, target_url=report_url) - - ch_helper = ClickHouseHelper() - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, status, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, NAME) - ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) - except Exception as ex: - print("Got an exception", ex) - sys.exit(1) diff --git a/tests/ci/release.py b/tests/ci/release.py new file mode 100755 index 00000000000..ca1a66dbbac --- /dev/null +++ b/tests/ci/release.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python + + +from contextlib import contextmanager +from typing import List, Optional +import argparse +import logging + +from git_helper import commit, release_branch +from version_helper import ( + FILE_WITH_VERSION_PATH, + ClickHouseVersion, + VersionType, + get_abs_path, + get_version_from_repo, + update_cmake_version, +) + + +class Repo: + VALID = ("ssh", "https", "origin") + + def __init__(self, repo: str, protocol: str): + self._repo = repo + self._url = "" + self.url = protocol + + @property + def url(self) -> str: + return self._url + + @url.setter + def url(self, protocol: str): + if protocol == "ssh": + self._url = f"git@github.com:{self}.git" + elif protocol == "https": + self._url = f"https://github.com/{self}.git" + elif protocol == "origin": + self._url = protocol + else: + raise Exception(f"protocol must be in {self.VALID}") + + def __str__(self): + return self._repo + + +class Release: + BIG = ("major", "minor") + SMALL = ("patch",) + + def __init__(self, repo: Repo, release_commit: str, release_type: str): + self.repo = repo + self._release_commit = "" + self.release_commit = release_commit + self.release_type = release_type + self._version = get_version_from_repo() + self._git = self._version._git + self._release_branch = "" + self._rollback_stack = [] # type: List[str] + + def run(self, cmd: str, cwd: Optional[str] = None) -> str: + cwd_text = "" + if cwd: + cwd_text = f" (CWD='{cwd}')" + logging.info("Running command%s:\n %s", cwd_text, cmd) + return self._git.run(cmd, cwd) + + def set_release_branch(self): + # Get the actual version for the commit before check + with self._checkout(self.release_commit, True): + self.read_version() + self.release_branch = f"{self.version.major}.{self.version.minor}" + + self.read_version() + + def read_version(self): + self._git.update() + self.version = get_version_from_repo() + + def check_prerequisites(self): + """ + Check tooling installed in the system + """ + self.run("gh auth status") + self.run("git status") + + def do(self, check_dirty: bool, check_branch: bool, with_prestable: bool): + self.check_prerequisites() + + if check_dirty: + logging.info("Checking if repo is clean") + self.run("git diff HEAD --exit-code") + + self.set_release_branch() + + if check_branch: + self.check_branch() + + with self._checkout(self.release_commit, True): + if self.release_type in self.BIG: + # Checkout to the commit, it will provide the correct current version + if with_prestable: + with self.prestable(): + logging.info("Prestable part of the releasing is done") + else: + logging.info("Skipping prestable stage") + + with self.testing(): + logging.info("Testing part of the releasing is done") + + elif self.release_type in self.SMALL: + with self.stable(): + logging.info("Stable part of the releasing is done") + + self.log_rollback() + + def check_no_tags_after(self): + tags_after_commit = self.run(f"git tag --contains={self.release_commit}") + if tags_after_commit: + raise Exception( + f"Commit {self.release_commit} belongs to following tags:\n" + f"{tags_after_commit}\nChoose another commit" + ) + + def check_branch(self): + if self.release_type in self.BIG: + # Commit to spin up the release must belong to a main branch + branch = "master" + output = self.run(f"git branch --contains={self.release_commit} {branch}") + if branch not in output: + raise Exception( + f"commit {self.release_commit} must belong to {branch} for " + f"{self.release_type} release" + ) + return + elif self.release_type in self.SMALL: + output = self.run( + f"git branch --contains={self.release_commit} {self.release_branch}" + ) + if self.release_branch not in output: + raise Exception( + f"commit {self.release_commit} must be in " + f"'{self.release_branch}' branch for {self.release_type} release" + ) + return + + def log_rollback(self): + if self._rollback_stack: + rollback = self._rollback_stack + rollback.reverse() + logging.info( + "To rollback the action run the following commands:\n %s", + "\n ".join(rollback), + ) + + @contextmanager + def prestable(self): + self.check_no_tags_after() + # Create release branch + self.read_version() + with self._create_branch(self.release_branch, self.release_commit): + with self._checkout(self.release_branch, True): + self.read_version() + self.version.with_description(VersionType.PRESTABLE) + with self._create_gh_release(True): + with self._bump_prestable_version(): + # At this point everything will rollback automatically + yield + + @contextmanager + def stable(self): + self.check_no_tags_after() + self.read_version() + version_type = VersionType.STABLE + if self.version.minor % 5 == 3: # our 3 and 8 are LTS + version_type = VersionType.LTS + self.version.with_description(version_type) + with self._create_gh_release(False): + self.version = self.version.update(self.release_type) + self.version.with_description(version_type) + update_cmake_version(self.version) + cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + # Checkouting the commit of the branch and not the branch itself, + # then we are able to skip rollback + with self._checkout(f"{self.release_branch}@{{0}}", False): + current_commit = self.run("git rev-parse HEAD") + self.run( + f"git commit -m " + f"'Update version to {self.version.string}' '{cmake_path}'" + ) + with self._push( + "HEAD", with_rollback_on_fail=False, remote_ref=self.release_branch + ): + # DO NOT PUT ANYTHING ELSE HERE + # The push must be the last action and mean the successful release + self._rollback_stack.append( + f"git push {self.repo.url} " + f"+{current_commit}:{self.release_branch}" + ) + yield + + @contextmanager + def testing(self): + # Create branch for a version bump + self.read_version() + self.version = self.version.update(self.release_type) + helper_branch = f"{self.version.major}.{self.version.minor}-prepare" + with self._create_branch(helper_branch, self.release_commit): + with self._checkout(helper_branch, True): + with self._bump_testing_version(helper_branch): + yield + + @property + def version(self) -> ClickHouseVersion: + return self._version + + @version.setter + def version(self, version: ClickHouseVersion): + if not isinstance(version, ClickHouseVersion): + raise ValueError(f"version must be ClickHouseVersion, not {type(version)}") + self._version = version + + @property + def release_branch(self) -> str: + return self._release_branch + + @release_branch.setter + def release_branch(self, branch: str): + self._release_branch = release_branch(branch) + + @property + def release_commit(self) -> str: + return self._release_commit + + @release_commit.setter + def release_commit(self, release_commit: str): + self._release_commit = commit(release_commit) + + @contextmanager + def _bump_prestable_version(self): + # Update only git, origal version stays the same + self._git.update() + new_version = self.version.patch_update() + new_version.with_description("prestable") + update_cmake_version(new_version) + cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + self.run( + f"git commit -m 'Update version to {new_version.string}' '{cmake_path}'" + ) + with self._push(self.release_branch): + with self._create_gh_label( + f"v{self.release_branch}-must-backport", "10dbed" + ): + with self._create_gh_label( + f"v{self.release_branch}-affected", "c2bfff" + ): + self.run( + f"gh pr create --repo {self.repo} --title " + f"'Release pull request for branch {self.release_branch}' " + f"--head {self.release_branch} --label release " + "--body 'This PullRequest is a part of ClickHouse release " + "cycle. It is used by CI system only. Do not perform any " + "changes with it.'" + ) + # Here the prestable part is done + yield + + @contextmanager + def _bump_testing_version(self, helper_branch: str): + self.read_version() + self.version = self.version.update(self.release_type) + self.version.with_description("testing") + update_cmake_version(self.version) + cmake_path = get_abs_path(FILE_WITH_VERSION_PATH) + self.run( + f"git commit -m 'Update version to {self.version.string}' '{cmake_path}'" + ) + with self._push(helper_branch): + body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md") + self.run( + f"gh pr create --repo {self.repo} --title 'Update version after " + f"release' --head {helper_branch} --body-file '{body_file}'" + ) + # Here the prestable part is done + yield + + @contextmanager + def _checkout(self, ref: str, with_checkout_back: bool = False): + orig_ref = self._git.branch or self._git.sha + need_rollback = False + if ref not in (self._git.branch, self._git.sha): + need_rollback = True + self.run(f"git checkout {ref}") + # checkout is not put into rollback_stack intentionally + rollback_cmd = f"git checkout {orig_ref}" + try: + yield + except BaseException: + logging.warning("Rolling back checked out %s for %s", ref, orig_ref) + self.run(f"git reset --hard; git checkout {orig_ref}") + raise + else: + if with_checkout_back and need_rollback: + self.run(rollback_cmd) + + @contextmanager + def _create_branch(self, name: str, start_point: str = ""): + self.run(f"git branch {name} {start_point}") + rollback_cmd = f"git branch -D {name}" + self._rollback_stack.append(rollback_cmd) + try: + yield + except BaseException: + logging.warning("Rolling back created branch %s", name) + self.run(rollback_cmd) + raise + + @contextmanager + def _create_gh_label(self, label: str, color_hex: str): + # API call, https://docs.github.com/en/rest/reference/issues#create-a-label + self.run( + f"gh api repos/{self.repo}/labels -f name={label} -f color={color_hex}" + ) + rollback_cmd = f"gh api repos/{self.repo}/labels/{label} -X DELETE" + self._rollback_stack.append(rollback_cmd) + try: + yield + except BaseException: + logging.warning("Rolling back label %s", label) + self.run(rollback_cmd) + raise + + @contextmanager + def _create_gh_release(self, as_prerelease: bool): + with self._create_tag(): + # Preserve tag if version is changed + tag = self.version.describe + prerelease = "" + if as_prerelease: + prerelease = "--prerelease" + self.run( + f"gh release create {prerelease} --draft --repo {self.repo} " + f"--title 'Release {tag}' '{tag}'" + ) + rollback_cmd = f"gh release delete --yes --repo {self.repo} '{tag}'" + self._rollback_stack.append(rollback_cmd) + try: + yield + except BaseException: + logging.warning("Rolling back release publishing") + self.run(rollback_cmd) + raise + + @contextmanager + def _create_tag(self): + tag = self.version.describe + self.run(f"git tag -a -m 'Release {tag}' '{tag}'") + rollback_cmd = f"git tag -d '{tag}'" + self._rollback_stack.append(rollback_cmd) + try: + with self._push(f"'{tag}'"): + yield + except BaseException: + logging.warning("Rolling back tag %s", tag) + self.run(rollback_cmd) + raise + + @contextmanager + def _push(self, ref: str, with_rollback_on_fail: bool = True, remote_ref: str = ""): + if remote_ref == "": + remote_ref = ref + + self.run(f"git push {self.repo.url} {ref}:{remote_ref}") + if with_rollback_on_fail: + rollback_cmd = f"git push -d {self.repo.url} {remote_ref}" + self._rollback_stack.append(rollback_cmd) + + try: + yield + except BaseException: + if with_rollback_on_fail: + logging.warning("Rolling back pushed ref %s", ref) + self.run(rollback_cmd) + + raise + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Script to release a new ClickHouse version, requires `git` and " + "`gh` (github-cli) commands", + ) + + parser.add_argument( + "--commit", + required=True, + type=commit, + help="commit create a release", + ) + parser.add_argument( + "--repo", + default="ClickHouse/ClickHouse", + help="repository to create the release", + ) + parser.add_argument( + "--remote-protocol", + "-p", + default="ssh", + choices=Repo.VALID, + help="repo protocol for git commands remote, 'origin' is a special case and " + "uses 'origin' as a remote", + ) + parser.add_argument( + "--type", + default="minor", + choices=Release.BIG + Release.SMALL, + dest="release_type", + help="a release type, new branch is created only for 'major' and 'minor'", + ) + parser.add_argument("--with-prestable", default=True, help=argparse.SUPPRESS) + parser.add_argument( + "--no-prestable", + dest="with_prestable", + action="store_false", + default=argparse.SUPPRESS, + help=f"if set, for release types in {Release.BIG} skip creating prestable " + "release and release branch", + ) + parser.add_argument("--check-dirty", default=True, help=argparse.SUPPRESS) + parser.add_argument( + "--no-check-dirty", + dest="check_dirty", + action="store_false", + default=argparse.SUPPRESS, + help="(dangerous) if set, skip check repository for uncommited changes", + ) + parser.add_argument("--check-branch", default=True, help=argparse.SUPPRESS) + parser.add_argument( + "--no-check-branch", + dest="check_branch", + action="store_false", + default=argparse.SUPPRESS, + help="(debug or development only) if set, skip the branch check for a run. " + "By default, 'major' and 'minor' types workonly for master, and 'patch' works " + "only for a release branches, that name " + "should be the same as '$MAJOR.$MINOR' version, e.g. 22.2", + ) + + return parser.parse_args() + + +def main(): + logging.basicConfig(level=logging.INFO) + args = parse_args() + repo = Repo(args.repo, args.remote_protocol) + release = Release(repo, args.commit, args.release_type) + + release.do(args.check_dirty, args.check_branch, args.with_prestable) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/report.py b/tests/ci/report.py index 156e6096605..c79a5406998 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -92,16 +92,27 @@ HTML_TEST_PART = """ """ -BASE_HEADERS = ['Test name', 'Test status'] +BASE_HEADERS = ["Test name", "Test status"] + + +class ReportColorTheme: + class ReportColor: + yellow = "#FFB400" + red = "#F00" + green = "#0A0" + blue = "#00B4FF" + + default = (ReportColor.green, ReportColor.red, ReportColor.yellow) + bugfixcheck = (ReportColor.yellow, ReportColor.blue, ReportColor.blue) def _format_header(header, branch_name, branch_url=None): - result = ' '.join([w.capitalize() for w in header.split(' ')]) + result = " ".join([w.capitalize() for w in header.split(" ")]) result = result.replace("Clickhouse", "ClickHouse") result = result.replace("clickhouse", "ClickHouse") - if 'ClickHouse' not in result: - result = 'ClickHouse ' + result - result += ' for ' + if "ClickHouse" not in result: + result = "ClickHouse " + result + result += " for " if branch_url: result += '{name}'.format(url=branch_url, name=branch_name) else: @@ -109,26 +120,57 @@ def _format_header(header, branch_name, branch_url=None): return result -def _get_status_style(status): +def _get_status_style(status, colortheme=None): + ok_statuses = ("OK", "success", "PASSED") + fail_statuses = ("FAIL", "failure", "error", "FAILED", "Timeout") + + if colortheme is None: + colortheme = ReportColorTheme.default + style = "font-weight: bold;" - if status in ('OK', 'success', 'PASSED'): - style += 'color: #0A0;' - elif status in ('FAIL', 'failure', 'error', 'FAILED', 'Timeout'): - style += 'color: #F00;' + if status in ok_statuses: + style += f"color: {colortheme[0]};" + elif status in fail_statuses: + style += f"color: {colortheme[1]};" else: - style += 'color: #FFB400;' + style += f"color: {colortheme[2]};" return style -def _get_html_url(url): +def _get_html_url_name(url): if isinstance(url, str): - return '{name}'.format(url=url, name=os.path.basename(url).replace('%2B', '+').replace('%20', ' ')) + return os.path.basename(url).replace("%2B", "+").replace("%20", " ") if isinstance(url, tuple): - return '{name}'.format(url=url[0], name=url[1].replace('%2B', '+').replace('%20', ' ')) - return '' + return url[1].replace("%2B", "+").replace("%20", " ") + return None -def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=None, with_raw_logs=False): +def _get_html_url(url): + href = None + name = None + if isinstance(url, str): + href, name = url, _get_html_url_name(url) + if isinstance(url, tuple): + href, name = url[0], _get_html_url_name(url) + if href and name: + return '{name}'.format( + href=href, name=_get_html_url_name(url) + ) + return "" + + +def create_test_html_report( + header, + test_result, + raw_log_url, + task_url, + branch_url, + branch_name, + commit_url, + additional_urls=None, + with_raw_logs=False, + statuscolors=None, +): if additional_urls is None: additional_urls = [] @@ -152,11 +194,11 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u has_test_logs = True row = "" - is_fail = test_status in ('FAIL', 'FLAKY') + is_fail = test_status in ("FAIL", "FLAKY") if is_fail and with_raw_logs and test_logs is not None: - row = "" + row = '' row += "" + test_name + "" - style = _get_status_style(test_status) + style = _get_status_style(test_status, colortheme=statuscolors) # Allow to quickly scroll to the first failure. is_fail_id = "" @@ -164,7 +206,13 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u num_fails = num_fails + 1 is_fail_id = 'id="fail' + str(num_fails) + '" ' - row += ''.format(style) + test_status + "" + row += ( + "'.format(style) + + test_status + + "" + ) if test_time is not None: row += "" + test_time + "" @@ -176,26 +224,26 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u row += "" rows_part += row if test_logs is not None and with_raw_logs: - row = "" + row = '' # TODO: compute colspan too - row += "
" + test_logs + "
" + row += '
' + test_logs + "
" row += "" rows_part += row headers = BASE_HEADERS if has_test_time: - headers.append('Test time, sec.') + headers.append("Test time, sec.") if has_test_logs and not with_raw_logs: - headers.append('Logs') + headers.append("Logs") - headers = ''.join(['' + h + '' for h in headers]) + headers = "".join(["" + h + "" for h in headers]) test_part = HTML_TEST_PART.format(headers=headers, rows=rows_part) else: test_part = "" - additional_html_urls = "" - for url in additional_urls: - additional_html_urls += ' ' + _get_html_url(url) + additional_html_urls = " ".join( + [_get_html_url(url) for url in sorted(additional_urls, key=_get_html_url_name)] + ) result = HTML_BASE_TEST_TEMPLATE.format( title=_format_header(header, branch_name), @@ -206,7 +254,7 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u test_part=test_part, branch_name=branch_name, commit_url=commit_url, - additional_urls=additional_html_urls + additional_urls=additional_html_urls, ) return result @@ -270,9 +318,20 @@ tr:hover td {{filter: brightness(95%);}} LINK_TEMPLATE = '{text}' -def create_build_html_report(header, build_results, build_logs_urls, artifact_urls_list, task_url, branch_url, branch_name, commit_url): +def create_build_html_report( + header, + build_results, + build_logs_urls, + artifact_urls_list, + task_url, + branch_url, + branch_name, + commit_url, +): rows = "" - for (build_result, build_log_url, artifact_urls) in zip(build_results, build_logs_urls, artifact_urls_list): + for (build_result, build_log_url, artifact_urls) in zip( + build_results, build_logs_urls, artifact_urls_list + ): row = "" row += "{}".format(build_result.compiler) if build_result.build_type: @@ -299,18 +358,20 @@ def create_build_html_report(header, build_results, build_logs_urls, artifact_ur if build_result.elapsed_seconds: delta = datetime.timedelta(seconds=build_result.elapsed_seconds) else: - delta = 'unknown' + delta = "unknown" - row += '{}'.format(str(delta)) + row += "{}".format(str(delta)) links = "" link_separator = "
" if artifact_urls: for artifact_url in artifact_urls: - links += LINK_TEMPLATE.format(text=os.path.basename(artifact_url.replace('%2B', '+').replace('%20', ' ')), url=artifact_url) + links += LINK_TEMPLATE.format( + text=_get_html_url_name(artifact_url), url=artifact_url + ) links += link_separator if links: - links = links[:-len(link_separator)] + links = links[: -len(link_separator)] row += "{}".format(links) row += "" @@ -321,4 +382,5 @@ def create_build_html_report(header, build_results, build_logs_urls, artifact_ur rows=rows, task_url=task_url, branch_name=branch_name, - commit_url=commit_url) + commit_url=commit_url, + ) diff --git a/tests/ci/rerun_helper.py b/tests/ci/rerun_helper.py index 0ba50334d28..35363593db6 100644 --- a/tests/ci/rerun_helper.py +++ b/tests/ci/rerun_helper.py @@ -2,6 +2,7 @@ from commit_status_helper import get_commit + def _filter_statuses(statuses): """ Squash statuses to latest state @@ -19,7 +20,6 @@ def _filter_statuses(statuses): class RerunHelper: - def __init__(self, gh, pr_info, check_name): self.gh = gh self.pr_info = pr_info @@ -30,6 +30,9 @@ class RerunHelper: def is_already_finished_by_status(self): # currently we agree even for failed statuses for status in self.statuses: - if self.check_name in status.context and status.state in ('success', 'failure'): + if self.check_name in status.context and status.state in ( + "success", + "failure", + ): return True return False diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index a2403e61ac1..5b89082532d 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -69,6 +69,7 @@ TRUSTED_CONTRIBUTORS = { "s-mx", # Maxim Sabyanin, former employee, present contributor "sevirov", # technical writer, Yandex "spongedu", # Seasoned contributor + "taiyang-li", "ucasFL", # Amos Bird's friend "vdimir", # Employee "vzakaznikov", diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 902b97fdb95..91e67135f6f 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -34,30 +34,59 @@ def _flatten_list(lst): class S3Helper: def __init__(self, host): - self.session = boto3.session.Session(region_name='us-east-1') - self.client = self.session.client('s3', endpoint_url=host) + self.session = boto3.session.Session(region_name="us-east-1") + self.client = self.session.client("s3", endpoint_url=host) def _upload_file_to_s3(self, bucket_name, file_path, s3_path): - logging.debug("Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path) + logging.debug( + "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path + ) metadata = {} if os.path.getsize(file_path) < 64 * 1024 * 1024: - if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"): - metadata['ContentType'] = "text/plain; charset=utf-8" - logging.info("Content type %s for file path %s", "text/plain; charset=utf-8", file_path) + if ( + s3_path.endswith("txt") + or s3_path.endswith("log") + or s3_path.endswith("err") + or s3_path.endswith("out") + ): + metadata["ContentType"] = "text/plain; charset=utf-8" + logging.info( + "Content type %s for file path %s", + "text/plain; charset=utf-8", + file_path, + ) elif s3_path.endswith("html"): - metadata['ContentType'] = "text/html; charset=utf-8" - logging.info("Content type %s for file path %s", "text/html; charset=utf-8", file_path) + metadata["ContentType"] = "text/html; charset=utf-8" + logging.info( + "Content type %s for file path %s", + "text/html; charset=utf-8", + file_path, + ) elif s3_path.endswith("css"): - metadata['ContentType'] = "text/css; charset=utf-8" - logging.info("Content type %s for file path %s", "text/css; charset=utf-8", file_path) + metadata["ContentType"] = "text/css; charset=utf-8" + logging.info( + "Content type %s for file path %s", + "text/css; charset=utf-8", + file_path, + ) elif s3_path.endswith("js"): - metadata['ContentType'] = "text/javascript; charset=utf-8" - logging.info("Content type %s for file path %s", "text/css; charset=utf-8", file_path) + metadata["ContentType"] = "text/javascript; charset=utf-8" + logging.info( + "Content type %s for file path %s", + "text/css; charset=utf-8", + file_path, + ) else: logging.info("No content type provied for %s", file_path) else: - if re.search(r'\.(txt|log|err|out)$', s3_path) or re.search(r'\.log\..*(?{result_folder}/{RESULT_LOG_NAME}" + return ( + f"docker run --network=host --volume={build_path}:/package_folder" + f" --volume={server_log_folder}:/var/log/clickhouse-server" + f" --volume={result_folder}:/test_output" + f" {docker_image} >{result_folder}/{RESULT_LOG_NAME}" + ) if __name__ == "__main__": @@ -76,8 +85,8 @@ if __name__ == "__main__": for root, _, files in os.walk(reports_path): for f in files: - if f == 'changed_images.json': - images_path = os.path.join(root, 'changed_images.json') + if f == "changed_images.json": + images_path = os.path.join(root, "changed_images.json") break docker_image = get_image_with_version(reports_path, DOCKER_IMAGE) @@ -96,7 +105,9 @@ if __name__ == "__main__": if not os.path.exists(result_path): os.makedirs(result_path) - run_command = get_run_command(packages_path, result_path, server_log_path, docker_image) + run_command = get_run_command( + packages_path, result_path, server_log_path, docker_image + ) logging.info("Going to run command %s", run_command) with subprocess.Popen(run_command, shell=True) as process: @@ -110,13 +121,30 @@ if __name__ == "__main__": print("Result path", os.listdir(result_path)) print("Server log path", os.listdir(server_log_path)) - state, description, test_results, additional_logs = process_result(result_path, server_log_path) + state, description, test_results, additional_logs = process_result( + result_path, server_log_path + ) ch_helper = ClickHouseHelper() - s3_helper = S3Helper('https://s3.amazonaws.com') - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, CHECK_NAME) + s3_helper = S3Helper("https://s3.amazonaws.com") + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + additional_logs, + CHECK_NAME, + ) print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, CHECK_NAME, description, state, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, CHECK_NAME) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + CHECK_NAME, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/ssh.py b/tests/ci/ssh.py index f6309e31d0f..275f26fd65f 100644 --- a/tests/ci/ssh.py +++ b/tests/ci/ssh.py @@ -27,15 +27,19 @@ class SSHAgent: self._env_backup["SSH_OPTIONS"] = os.environ.get("SSH_OPTIONS") # set ENV from stdout of ssh-agent - for line in self._run(['ssh-agent']).splitlines(): + for line in self._run(["ssh-agent"]).splitlines(): name, _, value = line.partition(b"=") if _ == b"=": value = value.split(b";", 1)[0] self._env[name.decode()] = value.decode() os.environ[name.decode()] = value.decode() - ssh_options = "," + os.environ["SSH_OPTIONS"] if os.environ.get("SSH_OPTIONS") else "" - os.environ["SSH_OPTIONS"] = f"{ssh_options}UserKnownHostsFile=/dev/null,StrictHostKeyChecking=no" + ssh_options = ( + "," + os.environ["SSH_OPTIONS"] if os.environ.get("SSH_OPTIONS") else "" + ) + os.environ[ + "SSH_OPTIONS" + ] = f"{ssh_options}UserKnownHostsFile=/dev/null,StrictHostKeyChecking=no" def add(self, key): key_pub = self._key_pub(key) @@ -89,7 +93,13 @@ class SSHAgent: @staticmethod def _run(cmd, stdin=None): shell = isinstance(cmd, str) - with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE if stdin else None, shell=shell) as p: + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE if stdin else None, + shell=shell, + ) as p: stdout, stderr = p.communicate(stdin) if stdout.strip().decode() == "The agent has no identities.": @@ -101,6 +111,7 @@ class SSHAgent: return stdout + class SSHKey: def __init__(self, key_name=None, key_value=None): if key_name is None and key_value is None: diff --git a/tests/ci/stopwatch.py b/tests/ci/stopwatch.py index b6ae8674df1..db174550c03 100644 --- a/tests/ci/stopwatch.py +++ b/tests/ci/stopwatch.py @@ -2,7 +2,8 @@ import datetime -class Stopwatch(): + +class Stopwatch: def __init__(self): self.start_time = datetime.datetime.utcnow() self.start_time_str_value = self.start_time.strftime("%Y-%m-%d %H:%M:%S") diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 1f811b1e25d..32c181140e2 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -16,50 +16,77 @@ from build_download_helper import download_all_deb_packages from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen -def get_run_command(build_path, result_folder, server_log_folder, image): - cmd = "docker run --cap-add=SYS_PTRACE -e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + \ - f"--volume={build_path}:/package_folder " \ - f"--volume={result_folder}:/test_output " \ - f"--volume={server_log_folder}:/var/log/clickhouse-server {image}" +def get_run_command( + build_path, result_folder, repo_tests_path, server_log_folder, image +): + cmd = ( + "docker run --cap-add=SYS_PTRACE " + "-e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + f"--volume={build_path}:/package_folder " + f"--volume={result_folder}:/test_output " + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " + f"--volume={server_log_folder}:/var/log/clickhouse-server {image}" + ) return cmd + def process_results(result_folder, server_log_path, run_log_path): test_results = [] additional_files = [] # Just upload all files from result_folder. - # If task provides processed results, then it's responsible for content of result_folder. + # If task provides processed results, then it's responsible for content + # of result_folder. if os.path.exists(result_folder): - test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))] + test_files = [ + f + for f in os.listdir(result_folder) + if os.path.isfile(os.path.join(result_folder, f)) + ] additional_files = [os.path.join(result_folder, f) for f in test_files] if os.path.exists(server_log_path): - server_log_files = [f for f in os.listdir(server_log_path) if os.path.isfile(os.path.join(server_log_path, f))] - additional_files = additional_files + [os.path.join(server_log_path, f) for f in server_log_files] + server_log_files = [ + f + for f in os.listdir(server_log_path) + if os.path.isfile(os.path.join(server_log_path, f)) + ] + additional_files = additional_files + [ + os.path.join(server_log_path, f) for f in server_log_files + ] additional_files.append(run_log_path) status_path = os.path.join(result_folder, "check_status.tsv") if not os.path.exists(status_path): - return "failure", "check_status.tsv doesn't exists", test_results, additional_files + return ( + "failure", + "check_status.tsv doesn't exists", + test_results, + additional_files, + ) logging.info("Found check_status.tsv") - with open(status_path, 'r', encoding='utf-8') as status_file: - status = list(csv.reader(status_file, delimiter='\t')) + with open(status_path, "r", encoding="utf-8") as status_file: + status = list(csv.reader(status_file, delimiter="\t")) if len(status) != 1 or len(status[0]) != 2: return "error", "Invalid check_status.tsv", test_results, additional_files state, description = status[0][0], status[0][1] results_path = os.path.join(result_folder, "test_results.tsv") - with open(results_path, 'r', encoding='utf-8') as results_file: - test_results = list(csv.reader(results_file, delimiter='\t')) + with open(results_path, "r", encoding="utf-8") as results_file: + test_results = list(csv.reader(results_file, delimiter="\t")) if len(test_results) == 0: raise Exception("Empty results") @@ -72,6 +99,7 @@ if __name__ == "__main__": stopwatch = Stopwatch() temp_path = TEMP_PATH repo_path = REPO_COPY + repo_tests_path = os.path.join(repo_path, "tests") reports_path = REPORTS_PATH check_name = sys.argv[1] @@ -88,7 +116,7 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(reports_path, 'clickhouse/stress-test') + docker_image = get_image_with_version(reports_path, "clickhouse/stress-test") packages_path = os.path.join(temp_path, "packages") if not os.path.exists(packages_path): @@ -106,7 +134,9 @@ if __name__ == "__main__": run_log_path = os.path.join(temp_path, "runlog.log") - run_command = get_run_command(packages_path, result_path, server_log_path, docker_image) + run_command = get_run_command( + packages_path, result_path, repo_tests_path, server_log_path, docker_image + ) logging.info("Going to run func tests: %s", run_command) with TeePopen(run_command, run_log_path) as process: @@ -118,16 +148,32 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - - s3_helper = S3Helper('https://s3.amazonaws.com') - state, description, test_results, additional_logs = process_results(result_path, server_log_path, run_log_path) + s3_helper = S3Helper("https://s3.amazonaws.com") + state, description, test_results, additional_logs = process_results( + result_path, server_log_path, run_log_path + ) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name) + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + additional_logs, + check_name, + ) print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, check_name, description, state, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + check_name, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/team_keys_lambda/app.py b/tests/ci/team_keys_lambda/app.py index 8a82e8dd353..ad153664b86 100644 --- a/tests/ci/team_keys_lambda/app.py +++ b/tests/ci/team_keys_lambda/app.py @@ -105,4 +105,4 @@ if __name__ == "__main__": args = parser.parse_args() keys = main(args.token, args.organization, args.team) - print(f"Just shoing off the keys:\n{keys}") + print(f"# Just shoing off the keys:\n{keys}") diff --git a/tests/ci/tee_popen.py b/tests/ci/tee_popen.py index 20302dacb97..7270cd6fb03 100644 --- a/tests/ci/tee_popen.py +++ b/tests/ci/tee_popen.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 from subprocess import Popen, PIPE, STDOUT -import sys +from threading import Thread +from time import sleep +import logging import os +import sys # Very simple tee logic implementation. You can specify shell command, output @@ -11,11 +14,23 @@ import os # stdout. class TeePopen: # pylint: disable=W0102 - def __init__(self, command, log_file, env=os.environ.copy()): + def __init__(self, command, log_file, env=os.environ.copy(), timeout=None): self.command = command self.log_file = log_file self.env = env self.process = None + self.timeout = timeout + + def _check_timeout(self): + sleep(self.timeout) + while self.process.poll() is None: + logging.warning( + "Killing process %s, timeout %s exceeded", + self.process.pid, + self.timeout, + ) + os.killpg(self.process.pid, 9) + sleep(10) def __enter__(self): self.process = Popen( @@ -23,11 +38,16 @@ class TeePopen: shell=True, universal_newlines=True, env=self.env, + start_new_session=True, # signall will be sent to all children stderr=STDOUT, stdout=PIPE, bufsize=1, ) self.log_file = open(self.log_file, "w", encoding="utf-8") + if self.timeout is not None and self.timeout > 0: + t = Thread(target=self._check_timeout) + t.daemon = True # does not block the program from exit + t.start() return self def __exit__(self, t, value, traceback): diff --git a/tests/ci/termination_lambda/app.py b/tests/ci/termination_lambda/app.py index 5de3d1531f2..14a0b2d1250 100644 --- a/tests/ci/termination_lambda/app.py +++ b/tests/ci/termination_lambda/app.py @@ -8,18 +8,19 @@ import json import time from collections import namedtuple + def get_key_and_app_from_aws(): import boto3 + secret_name = "clickhouse_github_secret_key" session = boto3.session.Session() client = session.client( - service_name='secretsmanager', + service_name="secretsmanager", ) - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - data = json.loads(get_secret_value_response['SecretString']) - return data['clickhouse-app-key'], int(data['clickhouse-app-id']) + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + data = json.loads(get_secret_value_response["SecretString"]) + return data["clickhouse-app-key"], int(data["clickhouse-app-id"]) + def get_installation_id(jwt_token): headers = { @@ -29,117 +30,152 @@ def get_installation_id(jwt_token): response = requests.get("https://api.github.com/app/installations", headers=headers) response.raise_for_status() data = response.json() - return data[0]['id'] + return data[0]["id"] + def get_access_token(jwt_token, installation_id): headers = { "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", } - response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers) + response = requests.post( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers=headers, + ) response.raise_for_status() data = response.json() - return data['token'] + return data["token"] -RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy']) +RunnerDescription = namedtuple( + "RunnerDescription", ["id", "name", "tags", "offline", "busy"] +) + def list_runners(access_token): headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", } - response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", headers=headers) + response = requests.get( + "https://api.github.com/orgs/ClickHouse/actions/runners?per_page=100", + headers=headers, + ) response.raise_for_status() data = response.json() - total_runners = data['total_count'] - runners = data['runners'] + total_runners = data["total_count"] + runners = data["runners"] total_pages = int(total_runners / 100 + 1) for i in range(2, total_pages + 1): - response = requests.get(f"https://api.github.com/orgs/ClickHouse/actions/runners?page={i}&per_page=100", headers=headers) + response = requests.get( + f"https://api.github.com/orgs/ClickHouse/actions/runners?page={i}&per_page=100", + headers=headers, + ) response.raise_for_status() data = response.json() - runners += data['runners'] + runners += data["runners"] print("Total runners", len(runners)) result = [] for runner in runners: - tags = [tag['name'] for tag in runner['labels']] - desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags, - offline=runner['status']=='offline', busy=runner['busy']) + tags = [tag["name"] for tag in runner["labels"]] + desc = RunnerDescription( + id=runner["id"], + name=runner["name"], + tags=tags, + offline=runner["status"] == "offline", + busy=runner["busy"], + ) result.append(desc) return result + def push_metrics_to_cloudwatch(listed_runners, namespace): import boto3 - client = boto3.client('cloudwatch') + + client = boto3.client("cloudwatch") metrics_data = [] busy_runners = sum(1 for runner in listed_runners if runner.busy) - metrics_data.append({ - 'MetricName': 'BusyRunners', - 'Value': busy_runners, - 'Unit': 'Count', - }) + metrics_data.append( + { + "MetricName": "BusyRunners", + "Value": busy_runners, + "Unit": "Count", + } + ) total_active_runners = sum(1 for runner in listed_runners if not runner.offline) - metrics_data.append({ - 'MetricName': 'ActiveRunners', - 'Value': total_active_runners, - 'Unit': 'Count', - }) + metrics_data.append( + { + "MetricName": "ActiveRunners", + "Value": total_active_runners, + "Unit": "Count", + } + ) total_runners = len(listed_runners) - metrics_data.append({ - 'MetricName': 'TotalRunners', - 'Value': total_runners, - 'Unit': 'Count', - }) + metrics_data.append( + { + "MetricName": "TotalRunners", + "Value": total_runners, + "Unit": "Count", + } + ) if total_active_runners == 0: busy_ratio = 100 else: busy_ratio = busy_runners / total_active_runners * 100 - metrics_data.append({ - 'MetricName': 'BusyRunnersRatio', - 'Value': busy_ratio, - 'Unit': 'Percent', - }) + metrics_data.append( + { + "MetricName": "BusyRunnersRatio", + "Value": busy_ratio, + "Unit": "Percent", + } + ) - client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data) + client.put_metric_data(Namespace="RunnersMetrics", MetricData=metrics_data) def how_many_instances_to_kill(event_data): - data_array = event_data['CapacityToTerminate'] + data_array = event_data["CapacityToTerminate"] to_kill_by_zone = {} for av_zone in data_array: - zone_name = av_zone['AvailabilityZone'] - to_kill = av_zone['Capacity'] + zone_name = av_zone["AvailabilityZone"] + to_kill = av_zone["Capacity"] if zone_name not in to_kill_by_zone: to_kill_by_zone[zone_name] = 0 to_kill_by_zone[zone_name] += to_kill return to_kill_by_zone + def get_candidates_to_be_killed(event_data): - data_array = event_data['Instances'] + data_array = event_data["Instances"] instances_by_zone = {} for instance in data_array: - zone_name = instance['AvailabilityZone'] - instance_id = instance['InstanceId'] + zone_name = instance["AvailabilityZone"] + instance_id = instance["InstanceId"] if zone_name not in instances_by_zone: instances_by_zone[zone_name] = [] instances_by_zone[zone_name].append(instance_id) return instances_by_zone + def delete_runner(access_token, runner): headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", } - response = requests.delete(f"https://api.github.com/orgs/ClickHouse/actions/runners/{runner.id}", headers=headers) + response = requests.delete( + f"https://api.github.com/orgs/ClickHouse/actions/runners/{runner.id}", + headers=headers, + ) response.raise_for_status() - print(f"Response code deleting {runner.name} with id {runner.id} is {response.status_code}") + print( + f"Response code deleting {runner.name} with id {runner.id} is {response.status_code}" + ) return response.status_code == 204 @@ -166,12 +202,16 @@ def main(github_secret_key, github_app_id, event): num_to_kill = to_kill_by_zone[zone] candidates = instances_by_zone[zone] if num_to_kill > len(candidates): - raise Exception(f"Required to kill {num_to_kill}, but have only {len(candidates)} candidates in AV {zone}") + raise Exception( + f"Required to kill {num_to_kill}, but have only {len(candidates)} candidates in AV {zone}" + ) delete_for_av = [] for candidate in candidates: if candidate not in set([runner.name for runner in runners]): - print(f"Candidate {candidate} was not in runners list, simply delete it") + print( + f"Candidate {candidate} was not in runners list, simply delete it" + ) instances_to_kill.append(candidate) for candidate in candidates: @@ -183,57 +223,76 @@ def main(github_secret_key, github_app_id, event): for runner in runners: if runner.name == candidate: if not runner.busy: - print(f"Runner {runner.name} is not busy and can be deleted from AV {zone}") + print( + f"Runner {runner.name} is not busy and can be deleted from AV {zone}" + ) delete_for_av.append(runner) else: print(f"Runner {runner.name} is busy, not going to delete it") break if len(delete_for_av) < num_to_kill: - print(f"Checked all candidates for av {zone}, get to delete {len(delete_for_av)}, but still cannot get required {num_to_kill}") + print( + f"Checked all candidates for av {zone}, get to delete {len(delete_for_av)}, but still cannot get required {num_to_kill}" + ) to_delete_runners += delete_for_av - print("Got instances to kill: ", ', '.join(instances_to_kill)) - print("Going to delete runners:", ', '.join([runner.name for runner in to_delete_runners])) + print("Got instances to kill: ", ", ".join(instances_to_kill)) + print( + "Going to delete runners:", + ", ".join([runner.name for runner in to_delete_runners]), + ) for runner in to_delete_runners: if delete_runner(access_token, runner): - print(f"Runner with name {runner.name} and id {runner.id} successfuly deleted from github") + print( + f"Runner with name {runner.name} and id {runner.id} successfuly deleted from github" + ) instances_to_kill.append(runner.name) else: print(f"Cannot delete {runner.name} from github") ## push metrics - #runners = list_runners(access_token) - #push_metrics_to_cloudwatch(runners, 'RunnersMetrics') + # runners = list_runners(access_token) + # push_metrics_to_cloudwatch(runners, 'RunnersMetrics') - response = { - "InstanceIDs": instances_to_kill - } + response = {"InstanceIDs": instances_to_kill} print(response) return response + def handler(event, context): private_key, app_id = get_key_and_app_from_aws() return main(private_key, app_id, event) + if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Get list of runners and their states') - parser.add_argument('-p', '--private-key-path', help='Path to file with private key') - parser.add_argument('-k', '--private-key', help='Private key') - parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True) + parser = argparse.ArgumentParser(description="Get list of runners and their states") + parser.add_argument( + "-p", "--private-key-path", help="Path to file with private key" + ) + parser.add_argument("-k", "--private-key", help="Private key") + parser.add_argument( + "-a", "--app-id", type=int, help="GitHub application ID", required=True + ) args = parser.parse_args() if not args.private_key_path and not args.private_key: - print("Either --private-key-path or --private-key must be specified", file=sys.stderr) + print( + "Either --private-key-path or --private-key must be specified", + file=sys.stderr, + ) if args.private_key_path and args.private_key: - print("Either --private-key-path or --private-key must be specified", file=sys.stderr) + print( + "Either --private-key-path or --private-key must be specified", + file=sys.stderr, + ) if args.private_key: private_key = args.private_key else: - with open(args.private_key_path, 'r') as key_file: + with open(args.private_key_path, "r") as key_file: private_key = key_file.read() sample_event = { @@ -243,41 +302,41 @@ if __name__ == "__main__": { "AvailabilityZone": "us-east-1b", "Capacity": 1, - "InstanceMarketOption": "OnDemand" + "InstanceMarketOption": "OnDemand", }, { "AvailabilityZone": "us-east-1c", "Capacity": 2, - "InstanceMarketOption": "OnDemand" - } + "InstanceMarketOption": "OnDemand", + }, ], "Instances": [ { "AvailabilityZone": "us-east-1b", "InstanceId": "i-08d0b3c1a137e02a5", "InstanceType": "t2.nano", - "InstanceMarketOption": "OnDemand" + "InstanceMarketOption": "OnDemand", }, { "AvailabilityZone": "us-east-1c", "InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal", "InstanceType": "t2.nano", - "InstanceMarketOption": "OnDemand" + "InstanceMarketOption": "OnDemand", }, { "AvailabilityZone": "us-east-1c", "InstanceId": "ip-172-31-27-227.eu-west-1.compute.internal", "InstanceType": "t2.nano", - "InstanceMarketOption": "OnDemand" + "InstanceMarketOption": "OnDemand", }, { "AvailabilityZone": "us-east-1c", "InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal", "InstanceType": "t2.nano", - "InstanceMarketOption": "OnDemand" - } + "InstanceMarketOption": "OnDemand", + }, ], - "Cause": "SCALE_IN" + "Cause": "SCALE_IN", } main(private_key, args.app_id, sample_event) diff --git a/tests/ci/token_lambda/app.py b/tests/ci/token_lambda/app.py index 731d6c040de..e3b768fca36 100644 --- a/tests/ci/token_lambda/app.py +++ b/tests/ci/token_lambda/app.py @@ -7,6 +7,7 @@ import sys import json import time + def get_installation_id(jwt_token): headers = { "Authorization": f"Bearer {jwt_token}", @@ -15,40 +16,48 @@ def get_installation_id(jwt_token): response = requests.get("https://api.github.com/app/installations", headers=headers) response.raise_for_status() data = response.json() - return data[0]['id'] + return data[0]["id"] + def get_access_token(jwt_token, installation_id): headers = { "Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json", } - response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers) + response = requests.post( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers=headers, + ) response.raise_for_status() data = response.json() - return data['token'] + return data["token"] + def get_runner_registration_token(access_token): headers = { "Authorization": f"token {access_token}", "Accept": "application/vnd.github.v3+json", } - response = requests.post("https://api.github.com/orgs/ClickHouse/actions/runners/registration-token", headers=headers) + response = requests.post( + "https://api.github.com/orgs/ClickHouse/actions/runners/registration-token", + headers=headers, + ) response.raise_for_status() data = response.json() - return data['token'] + return data["token"] + def get_key_and_app_from_aws(): import boto3 + secret_name = "clickhouse_github_secret_key" session = boto3.session.Session() client = session.client( - service_name='secretsmanager', + service_name="secretsmanager", ) - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - data = json.loads(get_secret_value_response['SecretString']) - return data['clickhouse-app-key'], int(data['clickhouse-app-id']) + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + data = json.loads(get_secret_value_response["SecretString"]) + return data["clickhouse-app-key"], int(data["clickhouse-app-id"]) def main(github_secret_key, github_app_id, push_to_ssm, ssm_parameter_name): @@ -67,40 +76,65 @@ def main(github_secret_key, github_app_id, push_to_ssm, ssm_parameter_name): import boto3 print("Trying to put params into ssm manager") - client = boto3.client('ssm') + client = boto3.client("ssm") client.put_parameter( Name=ssm_parameter_name, Value=runner_registration_token, - Type='SecureString', - Overwrite=True) + Type="SecureString", + Overwrite=True, + ) else: - print("Not push token to AWS Parameter Store, just print:", runner_registration_token) + print( + "Not push token to AWS Parameter Store, just print:", + runner_registration_token, + ) def handler(event, context): private_key, app_id = get_key_and_app_from_aws() - main(private_key, app_id, True, 'github_runner_registration_token') + main(private_key, app_id, True, "github_runner_registration_token") + if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Get new token from github to add runners') - parser.add_argument('-p', '--private-key-path', help='Path to file with private key') - parser.add_argument('-k', '--private-key', help='Private key') - parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True) - parser.add_argument('--push-to-ssm', action='store_true', help='Store received token in parameter store') - parser.add_argument('--ssm-parameter-name', default='github_runner_registration_token', help='AWS paramater store parameter name') + parser = argparse.ArgumentParser( + description="Get new token from github to add runners" + ) + parser.add_argument( + "-p", "--private-key-path", help="Path to file with private key" + ) + parser.add_argument("-k", "--private-key", help="Private key") + parser.add_argument( + "-a", "--app-id", type=int, help="GitHub application ID", required=True + ) + parser.add_argument( + "--push-to-ssm", + action="store_true", + help="Store received token in parameter store", + ) + parser.add_argument( + "--ssm-parameter-name", + default="github_runner_registration_token", + help="AWS paramater store parameter name", + ) args = parser.parse_args() if not args.private_key_path and not args.private_key: - print("Either --private-key-path or --private-key must be specified", file=sys.stderr) + print( + "Either --private-key-path or --private-key must be specified", + file=sys.stderr, + ) if args.private_key_path and args.private_key: - print("Either --private-key-path or --private-key must be specified", file=sys.stderr) + print( + "Either --private-key-path or --private-key must be specified", + file=sys.stderr, + ) if args.private_key: private_key = args.private_key else: - with open(args.private_key_path, 'r') as key_file: + with open(args.private_key_path, "r") as key_file: private_key = key_file.read() main(private_key, args.app_id, args.push_to_ssm, args.ssm_parameter_name) diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 06faa5704af..84c4faa822d 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -15,32 +15,38 @@ from build_download_helper import download_unit_tests from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status -from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse +from clickhouse_helper import ( + ClickHouseHelper, + mark_flaky_tests, + prepare_tests_results_for_clickhouse, +) from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen -IMAGE_NAME = 'clickhouse/unit-test' +IMAGE_NAME = "clickhouse/unit-test" + def get_test_name(line): - elements = reversed(line.split(' ')) + elements = reversed(line.split(" ")) for element in elements: - if '(' not in element and ')' not in element: + if "(" not in element and ")" not in element: return element raise Exception(f"No test name in line '{line}'") + def process_result(result_folder): - OK_SIGN = 'OK ]' - FAILED_SIGN = 'FAILED ]' - SEGFAULT = 'Segmentation fault' - SIGNAL = 'received signal SIG' - PASSED = 'PASSED' + OK_SIGN = "OK ]" + FAILED_SIGN = "FAILED ]" + SEGFAULT = "Segmentation fault" + SIGNAL = "received signal SIG" + PASSED = "PASSED" summary = [] total_counter = 0 failed_counter = 0 - result_log_path = f'{result_folder}/test_result.txt' + result_log_path = f"{result_folder}/test_result.txt" if not os.path.exists(result_log_path): logging.info("No output log on path %s", result_log_path) return "error", "No output log", summary, [] @@ -48,7 +54,7 @@ def process_result(result_folder): status = "success" description = "" passed = False - with open(result_log_path, 'r', encoding='utf-8') as test_result: + with open(result_log_path, "r", encoding="utf-8") as test_result: for line in test_result: if OK_SIGN in line: logging.info("Found ok line: '%s'", line) @@ -56,7 +62,7 @@ def process_result(result_folder): logging.info("Test name: '%s'", test_name) summary.append((test_name, "OK")) total_counter += 1 - elif FAILED_SIGN in line and 'listed below' not in line and 'ms)' in line: + elif FAILED_SIGN in line and "listed below" not in line and "ms)" in line: logging.info("Found fail line: '%s'", line) test_name = get_test_name(line.strip()) logging.info("Test name: '%s'", test_name) @@ -85,7 +91,9 @@ def process_result(result_folder): status = "failure" if not description: - description += f"fail: {failed_counter}, passed: {total_counter - failed_counter}" + description += ( + f"fail: {failed_counter}, passed: {total_counter - failed_counter}" + ) return status, description, summary, [result_log_path] @@ -139,15 +147,30 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) - s3_helper = S3Helper('https://s3.amazonaws.com') + s3_helper = S3Helper("https://s3.amazonaws.com") state, description, test_results, additional_logs = process_result(test_output) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) - report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name) + report_url = upload_results( + s3_helper, + pr_info.number, + pr_info.sha, + test_results, + [run_log_path] + additional_logs, + check_name, + ) print(f"::notice ::Report url: {report_url}") post_commit_status(gh, pr_info.sha, check_name, description, state, report_url) - prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name) + prepared_events = prepare_tests_results_for_clickhouse( + pr_info, + test_results, + state, + stopwatch.duration_seconds, + stopwatch.start_time_str, + report_url, + check_name, + ) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 5a5e8d3f36a..f7b74e8d5dd 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -3,10 +3,12 @@ import logging import ast from env_helper import GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID -from report import create_test_html_report +from report import ReportColorTheme, create_test_html_report -def process_logs(s3_client, additional_logs, s3_path_prefix, test_results, with_raw_logs): +def process_logs( + s3_client, additional_logs, s3_path_prefix, test_results, with_raw_logs +): processed_logs = {} # Firstly convert paths of logs from test_results to urls to s3. for test_result in test_results: @@ -21,8 +23,8 @@ def process_logs(s3_client, additional_logs, s3_path_prefix, test_results, with_ test_log_urls.append(processed_logs[log_path]) elif log_path: url = s3_client.upload_test_report_to_s3( - log_path, - s3_path_prefix + "/" + os.path.basename(log_path)) + log_path, s3_path_prefix + "/" + os.path.basename(log_path) + ) test_log_urls.append(url) processed_logs[log_path] = url @@ -33,15 +35,29 @@ def process_logs(s3_client, additional_logs, s3_path_prefix, test_results, with_ if log_path: additional_urls.append( s3_client.upload_test_report_to_s3( - log_path, - s3_path_prefix + "/" + os.path.basename(log_path))) + log_path, s3_path_prefix + "/" + os.path.basename(log_path) + ) + ) return additional_urls -def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files, check_name, with_raw_logs=True): - s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_') - additional_urls = process_logs(s3_client, additional_files, s3_path_prefix, test_results, with_raw_logs) +def upload_results( + s3_client, + pr_number, + commit_sha, + test_results, + additional_files, + check_name, + with_raw_logs=True, + statuscolors=None, +): + s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace( + " ", "_" + ).replace("(", "_").replace(")", "_").replace(",", "_") + additional_urls = process_logs( + s3_client, additional_files, s3_path_prefix, test_results, with_raw_logs + ) branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master" branch_name = "master" @@ -58,10 +74,25 @@ def upload_results(s3_client, pr_number, commit_sha, test_results, additional_fi else: raw_log_url = task_url - html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, with_raw_logs) - with open('report.html', 'w', encoding='utf-8') as f: + statuscolors = ( + ReportColorTheme.bugfixcheck if "bugfix validate check" in check_name else None + ) + + html_report = create_test_html_report( + check_name, + test_results, + raw_log_url, + task_url, + branch_url, + branch_name, + commit_url, + additional_urls, + with_raw_logs, + statuscolors=statuscolors, + ) + with open("report.html", "w", encoding="utf-8") as f: f.write(html_report) - url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html") + url = s3_client.upload_test_report_to_s3("report.html", s3_path_prefix + ".html") logging.info("Search result in url %s", url) return url diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py old mode 100644 new mode 100755 index e207dac4671..02e22ee0c4d --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -1,128 +1,191 @@ #!/usr/bin/env python3 -import os -import subprocess import datetime +import logging +import os.path as p +import subprocess +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from typing import Dict, Tuple, Union + +from git_helper import Git, removeprefix FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" CHANGELOG_IN_PATH = "debian/changelog.in" CHANGELOG_PATH = "debian/changelog" -CONTRIBUTORS_SCRIPT_DIR = "src/Storages/System/" +GENERATED_CONTRIBUTORS = "src/Storages/System/StorageSystemContributors.generated.cpp" + +# It has {{ for plain "{" +CONTRIBUTORS_TEMPLATE = """// autogenerated by {executer} +const char * auto_contributors[] {{ +{contributors} + nullptr}}; +""" + +VERSIONS = Dict[str, Union[int, str]] + +VERSIONS_TEMPLATE = """# This variables autochanged by release_lib.sh: + +# NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, +# only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. +SET(VERSION_REVISION {revision}) +SET(VERSION_MAJOR {major}) +SET(VERSION_MINOR {minor}) +SET(VERSION_PATCH {patch}) +SET(VERSION_GITHASH {githash}) +SET(VERSION_DESCRIBE {describe}) +SET(VERSION_STRING {string}) +# end of autochange +""" + +git = Git() -class ClickHouseVersion(): - def __init__(self, major, minor, patch, tweak, revision): - self.major = major - self.minor = minor - self.patch = patch - self.tweak = tweak - self.revision = revision +class ClickHouseVersion: + """Immutable version class. On update returns a new instance""" - def minor_update(self): + def __init__( + self, + major: Union[int, str], + minor: Union[int, str], + patch: Union[int, str], + revision: Union[int, str], + git: Git, + ): + self._major = int(major) + self._minor = int(minor) + self._patch = int(patch) + self._revision = int(revision) + self._git = git + self._describe = "" + + def update(self, part: str) -> "ClickHouseVersion": + """If part is valid, returns a new version""" + method = getattr(self, f"{part}_update") + return method() + + def major_update(self) -> "ClickHouseVersion": + return ClickHouseVersion(self.major + 1, 1, 1, self.revision + 1, self._git) + + def minor_update(self) -> "ClickHouseVersion": return ClickHouseVersion( - self.major, - self.minor + 1, - 1, - 1, - self.revision + 1) + self.major, self.minor + 1, 1, self.revision + 1, self._git + ) - def patch_update(self): + def patch_update(self) -> "ClickHouseVersion": return ClickHouseVersion( - self.major, - self.minor, - self.patch + 1, - 1, - self.revision) + self.major, self.minor, self.patch + 1, self.revision, self._git + ) - def tweak_update(self): - return ClickHouseVersion( - self.major, - self.minor, - self.patch, - self.tweak + 1, - self.revision) + @property + def major(self) -> int: + return self._major - def get_version_string(self): - return '.'.join([ - str(self.major), - str(self.minor), - str(self.patch), - str(self.tweak) - ]) + @property + def minor(self) -> int: + return self._minor - def as_tuple(self): + @property + def patch(self) -> int: + return self._patch + + @property + def tweak(self) -> int: + return self._git.tweak + + @property + def revision(self) -> int: + return self._revision + + @property + def githash(self) -> str: + return self._git.sha + + @property + def describe(self): + return self._describe + + @property + def string(self): + return ".".join( + (str(self.major), str(self.minor), str(self.patch), str(self.tweak)) + ) + + def as_dict(self) -> VERSIONS: + return { + "revision": self.revision, + "major": self.major, + "minor": self.minor, + "patch": self.patch, + "tweak": self.tweak, + "githash": self.githash, + "describe": self.describe, + "string": self.string, + } + + def as_tuple(self) -> Tuple[int, int, int, int]: return (self.major, self.minor, self.patch, self.tweak) + def with_description(self, version_type): + if version_type not in VersionType.VALID: + raise ValueError(f"version type {version_type} not in {VersionType.VALID}") + self._describe = f"v{self.string}-{version_type}" -class VersionType(): + +class VersionType: + LTS = "lts" + PRESTABLE = "prestable" STABLE = "stable" TESTING = "testing" + VALID = (TESTING, PRESTABLE, STABLE, LTS) -def build_version_description(version, version_type): - return "v" + version.get_version_string() + "-" + version_type +def get_abs_path(path: str) -> str: + return p.abspath(p.join(git.root, path)) -def _get_version_from_line(line): - _, ver_with_bracket = line.strip().split(' ') - return ver_with_bracket[:-1] +def read_versions(versions_path: str = FILE_WITH_VERSION_PATH) -> VERSIONS: + versions = {} + path_to_file = get_abs_path(versions_path) + with open(path_to_file, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line.startswith("SET("): + continue -def get_tweak_from_git_describe(repo_path): - # something like v21.12.1.8816-testing-358-g81942b8128 - # or v21.11.4.14-stable-31-gd6aab025e0 - output = subprocess.check_output(f"cd {repo_path} && git describe --long", shell=True).decode('utf-8') - commits_number = int(output.split('-')[-2]) - # for testing releases we have to also add fourth number of - # the previous tag - if 'testing' in output: - previous_version = output.split('-')[0] - previous_version_commits = int(previous_version.split('.')[3]) - commits_number += previous_version_commits + value = 0 # type: Union[int, str] + name, value = line[4:-1].split(maxsplit=1) + name = removeprefix(name, "VERSION_").lower() + try: + value = int(value) + except ValueError: + pass + versions[name] = value - return commits_number + return versions -def get_version_from_repo(repo_path): - path_to_file = os.path.join(repo_path, FILE_WITH_VERSION_PATH) - major = 0 - minor = 0 - patch = 0 - tweak = get_tweak_from_git_describe(repo_path) - version_revision = 0 - with open(path_to_file, 'r') as ver_file: - for line in ver_file: - if "VERSION_MAJOR" in line and "math" not in line and "SET" in line: - major = _get_version_from_line(line) - elif "VERSION_MINOR" in line and "math" not in line and "SET" in line: - minor = _get_version_from_line(line) - elif "VERSION_PATCH" in line and "math" not in line and "SET" in line: - patch = _get_version_from_line(line) - elif "VERSION_REVISION" in line and "math" not in line: - version_revision = _get_version_from_line(line) - return ClickHouseVersion(major, minor, patch, tweak, version_revision) - - -def _update_cmake_version(repo_path, version, sha, version_type): - cmd = """sed -i --follow-symlinks -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION {revision}/g;" \ - -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE {version_desc}/g;" \ - -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH {sha}/g;" \ - -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR {major}/g;" \ - -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR {minor}/g;" \ - -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH {patch}/g;" \ - -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING {version_string}/g;" \ - {path}""".format( - revision=version.revision, - version_desc=build_version_description(version, version_type), - sha=sha, - major=version.major, - minor=version.minor, - patch=version.patch, - version_string=version.get_version_string(), - path=os.path.join(repo_path, FILE_WITH_VERSION_PATH), +def get_version_from_repo( + versions_path: str = FILE_WITH_VERSION_PATH, +) -> ClickHouseVersion: + versions = read_versions(versions_path) + return ClickHouseVersion( + versions["major"], + versions["minor"], + versions["patch"], + versions["revision"], + git, ) - subprocess.check_call(cmd, shell=True) -def _update_changelog(repo_path, version): +def update_cmake_version( + version: ClickHouseVersion, + versions_path: str = FILE_WITH_VERSION_PATH, +): + path_to_file = get_abs_path(versions_path) + with open(path_to_file, "w", encoding="utf-8") as f: + f.write(VERSIONS_TEMPLATE.format_map(version.as_dict())) + + +def _update_changelog(repo_path: str, version: ClickHouseVersion): cmd = """sed \ -e "s/[@]VERSION_STRING[@]/{version_str}/g" \ -e "s/[@]DATE[@]/{date}/g" \ @@ -130,24 +193,117 @@ def _update_changelog(repo_path, version): -e "s/[@]EMAIL[@]/clickhouse-release@yandex-team.ru/g" \ < {in_path} > {changelog_path} """.format( - version_str=version.get_version_string(), + version_str=version.string, date=datetime.datetime.now().strftime("%a, %d %b %Y %H:%M:%S") + " +0300", - in_path=os.path.join(repo_path, CHANGELOG_IN_PATH), - changelog_path=os.path.join(repo_path, CHANGELOG_PATH) + in_path=p.join(repo_path, CHANGELOG_IN_PATH), + changelog_path=p.join(repo_path, CHANGELOG_PATH), ) subprocess.check_call(cmd, shell=True) -def _update_contributors(repo_path): - cmd = "cd {} && ./StorageSystemContributors.sh".format(os.path.join(repo_path, CONTRIBUTORS_SCRIPT_DIR)) + +def update_contributors( + relative_contributors_path: str = GENERATED_CONTRIBUTORS, force: bool = False +): + # Check if we have shallow checkout by comparing number of lines + # '--is-shallow-repository' is in git since 2.15, 2017-10-30 + if git.run("git rev-parse --is-shallow-repository") == "true" and not force: + logging.warning("The repository is shallow, refusing to update contributors") + return + + contributors = git.run("git shortlog HEAD --summary") + contributors = sorted( + [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in contributors.split("\n")], + ) + contributors = [f' "{c}",' for c in contributors] + + executer = p.relpath(p.realpath(__file__), git.root) + content = CONTRIBUTORS_TEMPLATE.format( + executer=executer, contributors="\n".join(contributors) + ) + contributors_path = get_abs_path(relative_contributors_path) + with open(contributors_path, "w", encoding="utf-8") as cfd: + cfd.write(content) + + +def _update_dockerfile(repo_path: str, version: ClickHouseVersion): + version_str_for_docker = ".".join( + [str(version.major), str(version.minor), str(version.patch), "*"] + ) + cmd = "ls -1 {path}/docker/*/Dockerfile | xargs sed -i -r -e 's/ARG version=.+$/ARG version='{ver}'/'".format( + path=repo_path, ver=version_str_for_docker + ) subprocess.check_call(cmd, shell=True) -def _update_dockerfile(repo_path, version): - version_str_for_docker = '.'.join([str(version.major), str(version.minor), str(version.patch), '*']) - cmd = "ls -1 {path}/docker/*/Dockerfile | xargs sed -i -r -e 's/ARG version=.+$/ARG version='{ver}'/'".format(path=repo_path, ver=version_str_for_docker) - subprocess.check_call(cmd, shell=True) -def update_version_local(repo_path, sha, version, version_type="testing"): - _update_contributors(repo_path) - _update_cmake_version(repo_path, version, sha, version_type) +def update_version_local(repo_path, version, version_type="testing"): + update_contributors() + version.with_description(version_type) + update_cmake_version(version, version_type) _update_changelog(repo_path, version) _update_dockerfile(repo_path, version) + + +def main(): + """The simplest thing it does - reads versions from cmake and produce the + environment variables that may be sourced in bash scripts""" + parser = ArgumentParser( + formatter_class=ArgumentDefaultsHelpFormatter, + description="The script reads versions from cmake and produce ENV variables", + ) + parser.add_argument( + "--version-path", + "-p", + default=FILE_WITH_VERSION_PATH, + help="relative path to the cmake file with versions", + ) + parser.add_argument( + "--version-type", + "-t", + choices=VersionType.VALID, + help="optional parameter to generate DESCRIBE", + ) + parser.add_argument( + "--export", + "-e", + action="store_true", + help="if the ENV variables should be exported", + ) + parser.add_argument( + "--update", + "-u", + choices=("major", "minor", "patch"), + help="the version part to update, tweak is always calculated from commits", + ) + parser.add_argument( + "--update-contributors", + "-c", + action="store_true", + help=f"update {GENERATED_CONTRIBUTORS} file and exit, " + "doesn't work on shallow repo", + ) + args = parser.parse_args() + + if args.update_contributors: + update_contributors() + return + + version = get_version_from_repo(args.version_path) + + if args.update: + version = version.update(args.update) + + if args.version_type: + version.with_description(args.version_type) + + if args.update: + update_cmake_version(version) + + for k, v in version.as_dict().items(): + name = f"CLICKHOUSE_VERSION_{k.upper()}" + print(f"{name}='{v}'") + if args.export: + print(f"export {name}") + + +if __name__ == "__main__": + main() diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index 9cea6db3da2..50b9d9bfedc 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -114,7 +114,8 @@ TRUSTED_CONTRIBUTORS = { "s-mx", # Maxim Sabyanin, former employee, present contributor "sevirov", # technical writer, Yandex "spongedu", # Seasoned contributor - "ucasfl", # Amos Bird's friend + "taiyang-li", + "ucasFL", # Amos Bird's friend "vdimir", # Employee "vzakaznikov", "YiuRULE", diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 4566cabf1e7..9dd05cacce4 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -59,6 +59,8 @@ MAX_RETRIES = 3 TEST_FILE_EXTENSIONS = ['.sql', '.sql.j2', '.sh', '.py', '.expect'] +VERSION_PATTERN = r"^((\d+\.)?(\d+\.)?(\d+\.)?\d+)$" + def stringhash(s): # default hash() function consistent @@ -202,6 +204,29 @@ def get_processlist(args): return clickhouse_execute_json(args, 'SHOW PROCESSLIST') +def get_processlist_after_test(args): + log_comment = args.testcase_basename + database = args.testcase_database + if args.replicated_database: + return clickhouse_execute_json(args, f""" + SELECT materialize((hostName(), tcpPort())) as host, * + FROM clusterAllReplicas('test_cluster_database_replicated', system.processes) + WHERE + query NOT LIKE '%system.processes%' AND + Settings['log_comment'] = '{log_comment}' AND + current_database = '{database}' + """) + else: + return clickhouse_execute_json(args, f""" + SELECT * + FROM system.processes + WHERE + query NOT LIKE '%system.processes%' AND + Settings['log_comment'] = '{log_comment}' AND + current_database = '{database}' + """) + + # collect server stacktraces using gdb def get_stacktraces_from_gdb(server_pid): try: @@ -329,13 +354,35 @@ class FailureReason(enum.Enum): FAST_ONLY = "running fast tests only" NO_LONG = "not running long tests" REPLICATED_DB = "replicated-database" + S3_STORAGE = "s3-storage" BUILD = "not running for current build" + BACKWARD_INCOMPATIBLE = "test is backward incompatible" # UNKNOWN reasons NO_REFERENCE = "no reference file" INTERNAL_ERROR = "Test internal error: " +class SettingsRandomizer: + settings = { + "max_insert_threads": lambda: 0 if random.random() < 0.5 else random.randint(1, 16), + "group_by_two_level_threshold": lambda: 1 if random.random() < 0.1 else 2 ** 60 if random.random() < 0.11 else 100000, + "group_by_two_level_threshold_bytes": lambda: 1 if random.random() < 0.1 else 2 ** 60 if random.random() < 0.11 else 50000000, + "distributed_aggregation_memory_efficient": lambda: random.randint(0, 1), + "fsync_metadata": lambda: random.randint(0, 1), + "priority": lambda: int(abs(random.gauss(0, 2))), + "output_format_parallel_formatting": lambda: random.randint(0, 1), + "input_format_parallel_parsing": lambda: random.randint(0, 1), + } + + @staticmethod + def get_random_settings(): + random_settings = [] + for setting, generator in SettingsRandomizer.settings.items(): + random_settings.append(setting + "=" + str(generator()) + "") + return random_settings + + class TestResult: def __init__(self, case_name: str, status: TestStatus, reason: Optional[FailureReason], total_time: float, description: str): self.case_name: str = case_name @@ -383,7 +430,7 @@ class TestCase: testcase_args.testcase_start_time = datetime.now() testcase_basename = os.path.basename(case_file) - testcase_args.testcase_client = f"{testcase_args.client} --log_comment='{testcase_basename}'" + testcase_args.testcase_client = f"{testcase_args.client} --log_comment '{testcase_basename}'" testcase_args.testcase_basename = testcase_basename if testcase_args.database: @@ -416,10 +463,36 @@ class TestCase: return testcase_args + def add_random_settings(self, client_options): + if self.tags and 'no-random-settings' in self.tags: + return client_options + + if len(self.base_url_params) == 0: + os.environ['CLICKHOUSE_URL_PARAMS'] = '&'.join(self.random_settings) + else: + os.environ['CLICKHOUSE_URL_PARAMS'] = self.base_url_params + '&' + '&'.join(self.random_settings) + + new_options = " --allow_repeated_settings --" + " --".join(self.random_settings) + os.environ['CLICKHOUSE_CLIENT_OPT'] = self.base_client_options + new_options + ' ' + return client_options + new_options + + def remove_random_settings_from_env(self): + os.environ['CLICKHOUSE_URL_PARAMS'] = self.base_url_params + os.environ['CLICKHOUSE_CLIENT_OPT'] = self.base_client_options + + def add_info_about_settings(self, description): + if self.tags and 'no-random-settings' in self.tags: + return description + + return description + "\n" + "Settings used in the test: " + "--" + " --".join(self.random_settings) + "\n" + def __init__(self, suite, case: str, args, is_concurrent: bool): self.case: str = case # case file name self.tags: Set[str] = suite.all_tags[case] if case in suite.all_tags else set() + for tag in os.getenv("GLOBAL_TAGS", "").split(","): + self.tags.add(tag.strip()) + self.case_file: str = os.path.join(suite.suite_path, case) (self.name, self.ext) = os.path.splitext(case) @@ -431,6 +504,32 @@ class TestCase: self.testcase_args = None self.runs_count = 0 + self.random_settings = SettingsRandomizer.get_random_settings() + self.base_url_params = os.environ['CLICKHOUSE_URL_PARAMS'] if 'CLICKHOUSE_URL_PARAMS' in os.environ else '' + self.base_client_options = os.environ['CLICKHOUSE_CLIENT_OPT'] if 'CLICKHOUSE_CLIENT_OPT' in os.environ else '' + + # Check if test contains tag "no-backward-compatibility-check" and we should skip it + def check_backward_incompatible_tag(self) -> bool: + for tag in self.tags: + if tag.startswith("no-backward-compatibility-check"): + split = tag.split(':') + + # If version is not specified in tag, always skip this test. + if len(split) == 1: + return True + version_from_tag = split[1] + + # Check if extracted string from tag is a real ClickHouse version, if not - always skip test. + if re.match(VERSION_PATTERN, version_from_tag) is None: + return True + + server_version = str(clickhouse_execute(args, "SELECT version()").decode()) + # If server version is less or equal from the version specified in tag, we should skip this test. + if list(map(int, server_version.split('.'))) <= list(map(int, version_from_tag.split('.'))): + return True + + return False + # should skip test, should increment skipped_total, skip reason def should_skip_test(self, suite) -> Optional[FailureReason]: tags = self.tags @@ -463,6 +562,12 @@ class TestCase: elif tags and ('no-replicated-database' in tags) and args.replicated_database: return FailureReason.REPLICATED_DB + elif args.backward_compatibility_check and self.check_backward_incompatible_tag(): + return FailureReason.BACKWARD_INCOMPATIBLE + + elif tags and ('no-s3-storage' in tags) and args.s3_storage: + return FailureReason.S3_STORAGE + elif tags: for build_flag in args.build_flags: if 'no-' + build_flag in tags: @@ -620,6 +725,16 @@ class TestCase: proc.stdout is None or 'Exception' not in proc.stdout) need_drop_database = not maybe_passed + left_queries_check = args.no_left_queries_check is False + if self.tags and 'no-left-queries-check' in self.tags: + left_queries_check = False + if left_queries_check: + processlist = get_processlist_after_test(args) + if processlist: + print(colored(f"\nFound queries left in processlist after running {args.testcase_basename} (database={database}):", args, "red", attrs=["bold"])) + print(json.dumps(processlist, indent=4)) + exit_code.value = 1 + if need_drop_database: seconds_left = max(args.timeout - (datetime.now() - start_time).total_seconds(), 20) try: @@ -668,10 +783,13 @@ class TestCase: self.runs_count += 1 self.testcase_args = self.configure_testcase_args(args, self.case_file, suite.suite_tmp_path) + client_options = self.add_random_settings(client_options) proc, stdout, stderr, total_time = self.run_single_test(server_logs_level, client_options) result = self.process_result_impl(proc, stdout, stderr, total_time) result.check_if_need_retry(args, stdout, stderr, self.runs_count) + if result.status == TestStatus.FAIL: + result.description = self.add_info_about_settings(result.description) return result except KeyboardInterrupt as e: raise e @@ -679,17 +797,20 @@ class TestCase: return TestResult(self.name, TestStatus.FAIL, FailureReason.INTERNAL_QUERY_FAIL, 0., - self.get_description_from_exception_info(sys.exc_info())) + self.add_info_about_settings(self.get_description_from_exception_info(sys.exc_info()))) except (ConnectionRefusedError, ConnectionResetError): return TestResult(self.name, TestStatus.FAIL, FailureReason.SERVER_DIED, 0., - self.get_description_from_exception_info(sys.exc_info())) + self.add_info_about_settings(self.get_description_from_exception_info(sys.exc_info()))) except: return TestResult(self.name, TestStatus.UNKNOWN, FailureReason.INTERNAL_ERROR, 0., self.get_description_from_exception_info(sys.exc_info())) + finally: + self.remove_random_settings_from_env() + class TestSuite: @staticmethod @@ -1073,11 +1194,15 @@ def collect_build_flags(args): if value == 0: result.append(BuildFlags.POLYMORPHIC_PARTS) - use_flags = clickhouse_execute(args, "SELECT name FROM system.build_options WHERE name like 'USE_%' AND value in ('ON', '1');") + use_flags = clickhouse_execute(args, "SELECT name FROM system.build_options WHERE name like 'USE_%' AND value in ('ON', '1')") for use_flag in use_flags.strip().splitlines(): use_flag = use_flag.decode().lower() result.append(use_flag) + system_processor = clickhouse_execute(args, "SELECT value FROM system.build_options WHERE name = 'SYSTEM_PROCESSOR' LIMIT 1").strip() + if system_processor: + result.append(f'cpu-{system_processor.decode().lower()}') + return result @@ -1349,6 +1474,7 @@ if __name__ == '__main__': parser.add_argument('--order', default='desc', choices=['asc', 'desc', 'random'], help='Run order') parser.add_argument('--testname', action='store_true', default=None, dest='testname', help='Make query with test name before test run') parser.add_argument('--hung-check', action='store_true', default=False) + parser.add_argument('--no-left-queries-check', action='store_true', default=False) parser.add_argument('--force-color', action='store_true', default=False) parser.add_argument('--database', help='Database for tests (random name test_XXXXXX by default)') parser.add_argument('--no-drop-if-fail', action='store_true', help='Do not drop database for test if test has failed') @@ -1369,6 +1495,7 @@ if __name__ == '__main__': parser.add_argument('--client-option', nargs='+', help='Specify additional client argument') parser.add_argument('--print-time', action='store_true', dest='print_time', help='Print test time') parser.add_argument('--check-zookeeper-session', action='store_true', help='Check ZooKeeper session uptime to determine if failed test should be retried') + parser.add_argument('--s3-storage', action='store_true', default=False, help='Run tests over s3 storage') parser.add_argument('--run-by-hash-num', type=int, help='Run tests matching crc32(test_name) % run_by_hash_total == run_by_hash_num') parser.add_argument('--run-by-hash-total', type=int, help='Total test groups for crc32(test_name) % run_by_hash_total == run_by_hash_num') @@ -1381,6 +1508,8 @@ if __name__ == '__main__': group.add_argument('--shard', action='store_true', default=None, dest='shard', help='Run sharding related tests (required to clickhouse-server listen 127.0.0.2 127.0.0.3)') group.add_argument('--no-shard', action='store_false', default=None, dest='shard', help='Do not run shard related tests') + group.add_argument('--backward-compatibility-check', action='store_true', help='Run tests for further backwoard compatibility testing by ignoring all' + 'drop queries in tests for collecting data from new version of server') args = parser.parse_args() if args.queries and not os.path.isdir(args.queries): @@ -1457,6 +1586,9 @@ if __name__ == '__main__': else: args.client_database = 'default' + if args.backward_compatibility_check: + args.client += ' --fake-drop' + if args.client_option: # Set options for client if 'CLICKHOUSE_CLIENT_OPT' in os.environ: diff --git a/tests/config/config.d/s3_storage_policy_by_default.xml b/tests/config/config.d/s3_storage_policy_by_default.xml new file mode 100644 index 00000000000..b4a2d697c78 --- /dev/null +++ b/tests/config/config.d/s3_storage_policy_by_default.xml @@ -0,0 +1,26 @@ + + + + + s3 + http://localhost:11111/test/test/ + clickhouse + clickhouse + 1 + 22548578304 + + + + + +
+ s3 +
+ + + + + + s3 + + diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml new file mode 100644 index 00000000000..2e43f735605 --- /dev/null +++ b/tests/config/config.d/storage_conf.xml @@ -0,0 +1,23 @@ + + + + + s3 + http://localhost:11111/test/00170_test/ + clickhouse + clickhouse + 1 + 22548578304 + + + + + +
+ s3_cache +
+ + + + + diff --git a/tests/config/install.sh b/tests/config/install.sh index 731a4317f44..c499ffa88f7 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -78,6 +78,15 @@ fi if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then ln -sf $SRC_PATH/users.d/database_ordinary.xml $DEST_SERVER_PATH/users.d/ fi + +if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then + ln -sf $SRC_PATH/config.d/s3_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ +fi + +if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then + ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/ +fi + if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ln -sf $SRC_PATH/users.d/database_replicated.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/config.d/database_replicated.xml $DEST_SERVER_PATH/config.d/ diff --git a/tests/instructions/pvs-studio.txt b/tests/instructions/pvs-studio.txt deleted file mode 100644 index 6ffc266aa6f..00000000000 --- a/tests/instructions/pvs-studio.txt +++ /dev/null @@ -1,15 +0,0 @@ -https://www.viva64.com/ru/m/0036/ - -# Analyze project with 4 threads. It takes about six minutes. - -pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 - -# Generate a report with "general" diagnostics of severity 1 and 2, in "tasks" format (simple text file): - -plog-converter -a GA:1,2 -t tasklist -o project.tasks pvs-studio.log - -# Generate an HTML report: - -plog-converter -a GA:1,2 -t fullhtml -o ./pvs-studio-html-report pvs-studio.log - -# Open ./pvs-studio-html-report/index.html in your browser. diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 40cb2c6fdd7..05e56d2a910 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 -import logging -import subprocess -import os -import glob -import time -import shutil from collections import defaultdict -import random -import json import csv -# for crc32 -import zlib +import glob +import json +import logging +import os +import random +import shutil +import subprocess +import time +import zlib # for crc32 MAX_RETRY = 3 @@ -22,57 +21,67 @@ CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse" CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge" CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge" -TRIES_COUNT = 10 +FLAKY_TRIES_COUNT = 10 MAX_TIME_SECONDS = 3600 -MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes -TASK_TIMEOUT = 8 * 60 * 60 # 8 hours +MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes +TASK_TIMEOUT = 8 * 60 * 60 # 8 hours + +NO_CHANGES_MSG = "Nothing to run" + def stringhash(s): - return zlib.crc32(s.encode('utf-8')) + return zlib.crc32(s.encode("utf-8")) -def get_tests_to_run(pr_info): - result = set([]) - changed_files = pr_info['changed_files'] + +def get_changed_tests_to_run(pr_info, repo_path): + result = set() + changed_files = pr_info["changed_files"] if changed_files is None: return [] for fpath in changed_files: - if 'tests/integration/test_' in fpath: - logging.info('File %s changed and seems like integration test', fpath) - result.add(fpath.split('/')[2]) - return list(result) + if "tests/integration/test_" in fpath: + logging.info("File %s changed and seems like integration test", fpath) + result.add(fpath.split("/")[2]) + return filter_existing_tests(result, repo_path) def filter_existing_tests(tests_to_run, repo_path): result = [] for relative_test_path in tests_to_run: - if os.path.exists(os.path.join(repo_path, 'tests/integration', relative_test_path)): + if os.path.exists( + os.path.join(repo_path, "tests/integration", relative_test_path) + ): result.append(relative_test_path) else: - logging.info("Skipping test %s, seems like it was removed", relative_test_path) + logging.info( + "Skipping test %s, seems like it was removed", relative_test_path + ) return result def _get_deselect_option(tests): - return ' '.join(['--deselect {}'.format(t) for t in tests]) + return " ".join([f"--deselect {t}" for t in tests]) + # https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks def chunks(lst, n): """Yield successive n-sized chunks from lst.""" for i in range(0, len(lst), n): - yield lst[i:i + n] + yield lst[i : i + n] + def get_counters(fname): counters = { - "ERROR": set([]), - "PASSED": set([]), - "FAILED": set([]), + "ERROR": set([]), + "PASSED": set([]), + "FAILED": set([]), "SKIPPED": set([]), } - with open(fname, 'r') as out: + with open(fname, "r") as out: for line in out: line = line.strip() # Example of log: @@ -81,10 +90,10 @@ def get_counters(fname): # [gw0] [ 7%] ERROR test_mysql_protocol/test.py::test_golang_client # # And only the line with test status should be matched - if not('.py::' in line and ' ' in line): + if not (".py::" in line and " " in line): continue - line_arr = line.strip().split(' ') + line_arr = line.strip().split(" ") if len(line_arr) < 2: logging.debug("Strange line %s", line) continue @@ -97,9 +106,9 @@ def get_counters(fname): if state in counters: counters[state].add(test_name) else: - # will skip lines line: - # 30.76s call test_host_ip_change/test.py::test_ip_change_drop_dns_cache - # 5.71s teardown test_host_ip_change/test.py::test_user_access_ip_change[node1] + # will skip lines like: + # 30.76s call test_host_ip_change/test.py::test_ip_drop_cache + # 5.71s teardown test_host_ip_change/test.py::test_ip_change[node1] # and similar logging.debug("Strange state in line %s", line) @@ -109,13 +118,13 @@ def get_counters(fname): def parse_test_times(fname): read = False description_output = [] - with open(fname, 'r') as out: + with open(fname, "r") as out: for line in out: - if read and '==' in line: + if read and "==" in line: break if read and line.strip(): description_output.append(line.strip()) - if 'slowest durations' in line: + if "slowest durations" in line: read = True return description_output @@ -123,10 +132,10 @@ def parse_test_times(fname): def get_test_times(output): result = defaultdict(float) for line in output: - if '.py' in line: - line_arr = line.strip().split(' ') + if ".py" in line: + line_arr = line.strip().split(" ") test_time = line_arr[0] - test_name = ' '.join([elem for elem in line_arr[2:] if elem]) + test_name = " ".join([elem for elem in line_arr[2:] if elem]) if test_name not in result: result[test_name] = 0.0 result[test_name] += float(test_time[:-1]) @@ -134,21 +143,28 @@ def get_test_times(output): def clear_ip_tables_and_restart_daemons(): - logging.info("Dump iptables after run %s", subprocess.check_output("sudo iptables -L", shell=True)) + logging.info( + "Dump iptables after run %s", + subprocess.check_output("sudo iptables -L", shell=True), + ) try: logging.info("Killing all alive docker containers") - subprocess.check_output("timeout -s 9 10m docker kill $(docker ps -q)", shell=True) + subprocess.check_output( + "timeout -s 9 10m docker kill $(docker ps -q)", shell=True + ) except subprocess.CalledProcessError as err: logging.info("docker kill excepted: " + str(err)) try: logging.info("Removing all docker containers") - subprocess.check_output("timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True) + subprocess.check_output( + "timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True + ) except subprocess.CalledProcessError as err: logging.info("docker rm excepted: " + str(err)) # don't restart docker if it's disabled - if os.environ.get("CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER", '1') == '1': + if os.environ.get("CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER", "1") == "1": try: logging.info("Stopping docker daemon") subprocess.check_output("service docker stop", shell=True) @@ -177,27 +193,36 @@ def clear_ip_tables_and_restart_daemons(): # when rules will be empty, it will raise exception subprocess.check_output("sudo iptables -D DOCKER-USER 1", shell=True) except subprocess.CalledProcessError as err: - logging.info("All iptables rules cleared, " + str(iptables_iter) + "iterations, last error: " + str(err)) + logging.info( + "All iptables rules cleared, " + + str(iptables_iter) + + "iterations, last error: " + + str(err) + ) class ClickhouseIntegrationTestsRunner: - def __init__(self, result_path, params): self.result_path = result_path self.params = params - self.image_versions = self.params['docker_images_with_versions'] - self.shuffle_groups = self.params['shuffle_test_groups'] - self.flaky_check = 'flaky check' in self.params['context_name'] + self.image_versions = self.params["docker_images_with_versions"] + self.shuffle_groups = self.params["shuffle_test_groups"] + self.flaky_check = "flaky check" in self.params["context_name"] + self.bugfix_validate_check = ( + "bugfix validate check" in self.params["context_name"] + ) # if use_tmpfs is not set we assume it to be true, otherwise check - self.use_tmpfs = 'use_tmpfs' not in self.params or self.params['use_tmpfs'] - self.disable_net_host = 'disable_net_host' in self.params and self.params['disable_net_host'] + self.use_tmpfs = "use_tmpfs" not in self.params or self.params["use_tmpfs"] + self.disable_net_host = ( + "disable_net_host" in self.params and self.params["disable_net_host"] + ) self.start_time = time.time() self.soft_deadline_time = self.start_time + (TASK_TIMEOUT - MAX_TIME_IN_SANDBOX) - if 'run_by_hash_total' in self.params: - self.run_by_hash_total = self.params['run_by_hash_total'] - self.run_by_hash_num = self.params['run_by_hash_num'] + if "run_by_hash_total" in self.params: + self.run_by_hash_total = self.params["run_by_hash_total"] + self.run_by_hash_num = self.params["run_by_hash_num"] else: self.run_by_hash_total = 0 self.run_by_hash_num = 0 @@ -206,7 +231,7 @@ class ClickhouseIntegrationTestsRunner: return self.result_path def base_path(self): - return os.path.join(str(self.result_path), '../') + return os.path.join(str(self.result_path), "../") def should_skip_tests(self): return [] @@ -214,8 +239,10 @@ class ClickhouseIntegrationTestsRunner: def get_image_with_version(self, name): if name in self.image_versions: return name + ":" + self.image_versions[name] - logging.warn("Cannot find image %s in params list %s", name, self.image_versions) - if ':' not in name: + logging.warn( + "Cannot find image %s in params list %s", name, self.image_versions + ) + if ":" not in name: return name + ":latest" return name @@ -223,31 +250,44 @@ class ClickhouseIntegrationTestsRunner: name = self.get_images_names()[0] if name in self.image_versions: return self.image_versions[name] - logging.warn("Cannot find image %s in params list %s", name, self.image_versions) - return 'latest' + logging.warn( + "Cannot find image %s in params list %s", name, self.image_versions + ) + return "latest" def shuffle_test_groups(self): return self.shuffle_groups != 0 @staticmethod def get_images_names(): - return ["clickhouse/integration-tests-runner", "clickhouse/mysql-golang-client", - "clickhouse/mysql-java-client", "clickhouse/mysql-js-client", - "clickhouse/mysql-php-client", "clickhouse/postgresql-java-client", - "clickhouse/integration-test", "clickhouse/kerberos-kdc", - "clickhouse/dotnet-client", - "clickhouse/integration-helper", ] - + return [ + "clickhouse/dotnet-client", + "clickhouse/integration-helper", + "clickhouse/integration-test", + "clickhouse/integration-tests-runner", + "clickhouse/kerberized-hadoop", + "clickhouse/kerberos-kdc", + "clickhouse/mysql-golang-client", + "clickhouse/mysql-java-client", + "clickhouse/mysql-js-client", + "clickhouse/mysql-php-client", + "clickhouse/postgresql-java-client", + ] def _can_run_with(self, path, opt): - with open(path, 'r') as script: + with open(path, "r") as script: for line in script: if opt in line: return True return False def _install_clickhouse(self, debs_path): - for package in ('clickhouse-common-static_', 'clickhouse-server_', 'clickhouse-client', 'clickhouse-common-static-dbg_'): # order matters + for package in ( + "clickhouse-common-static_", + "clickhouse-server_", + "clickhouse-client", + "clickhouse-common-static-dbg_", + ): # order matters logging.info("Installing package %s", package) for f in os.listdir(debs_path): if package in f: @@ -255,10 +295,12 @@ class ClickhouseIntegrationTestsRunner: logging.info("Package found in %s", full_path) log_name = "install_" + f + ".log" log_path = os.path.join(str(self.path()), log_name) - with open(log_path, 'w') as log: + with open(log_path, "w") as log: cmd = "dpkg -x {} .".format(full_path) logging.info("Executing installation cmd %s", cmd) - retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() + retcode = subprocess.Popen( + cmd, shell=True, stderr=log, stdout=log + ).wait() if retcode == 0: logging.info("Installation of %s successfull", full_path) else: @@ -267,18 +309,35 @@ class ClickhouseIntegrationTestsRunner: else: raise Exception("Package with {} not found".format(package)) logging.info("Unstripping binary") - # logging.info("Unstring %s", subprocess.check_output("eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), shell=True)) + # logging.info( + # "Unstring %s", + # subprocess.check_output( + # "eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), + # shell=True, + # ), + # ) logging.info("All packages installed") os.chmod(CLICKHOUSE_BINARY_PATH, 0o777) os.chmod(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, 0o777) os.chmod(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, 0o777) - shutil.copy(CLICKHOUSE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH")) - shutil.copy(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH")) - shutil.copy(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH")) + shutil.copy( + CLICKHOUSE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH") + ) + shutil.copy( + CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, + os.getenv("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"), + ) + shutil.copy( + CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, + os.getenv("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH"), + ) def _compress_logs(self, dir, relpaths, result_path): - subprocess.check_call("tar czf {} -C {} {}".format(result_path, dir, ' '.join(relpaths)), shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + "tar czf {} -C {} {}".format(result_path, dir, " ".join(relpaths)), + shell=True, + ) def _get_runner_opts(self): result = [] @@ -292,22 +351,40 @@ class ClickhouseIntegrationTestsRunner: image_cmd = self._get_runner_image_cmd(repo_path) out_file = "all_tests.txt" out_file_full = "all_tests_full.txt" - cmd = "cd {repo_path}/tests/integration && " \ - "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " \ - "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' " \ + cmd = ( + "cd {repo_path}/tests/integration && " + "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " + "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' " "| grep -v 'SKIPPED' | sort -u > {out_file}".format( - repo_path=repo_path, runner_opts=self._get_runner_opts(), image_cmd=image_cmd, out_file=out_file, out_file_full=out_file_full) + repo_path=repo_path, + runner_opts=self._get_runner_opts(), + image_cmd=image_cmd, + out_file=out_file, + out_file_full=out_file_full, + ) + ) logging.info("Getting all tests with cmd '%s'", cmd) - subprocess.check_call(cmd, shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + cmd, shell=True + ) - all_tests_file_path = "{repo_path}/tests/integration/{out_file}".format(repo_path=repo_path, out_file=out_file) - if not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0: - all_tests_full_file_path = "{repo_path}/tests/integration/{out_file}".format(repo_path=repo_path, out_file=out_file_full) + all_tests_file_path = "{repo_path}/tests/integration/{out_file}".format( + repo_path=repo_path, out_file=out_file + ) + if ( + not os.path.isfile(all_tests_file_path) + or os.path.getsize(all_tests_file_path) == 0 + ): + all_tests_full_file_path = ( + "{repo_path}/tests/integration/{out_file}".format( + repo_path=repo_path, out_file=out_file_full + ) + ) if os.path.isfile(all_tests_full_file_path): # log runner output logging.info("runner output:") - with open(all_tests_full_file_path, 'r') as all_tests_full_file: + with open(all_tests_full_file_path, "r") as all_tests_full_file: for line in all_tests_full_file: line = line.rstrip() if line: @@ -315,7 +392,11 @@ class ClickhouseIntegrationTestsRunner: else: logging.info("runner output '%s' is empty", all_tests_full_file_path) - raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(all_tests_file_path)) + raise Exception( + "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format( + all_tests_file_path + ) + ) all_tests = [] with open(all_tests_file_path, "r") as all_tests_file: @@ -324,9 +405,18 @@ class ClickhouseIntegrationTestsRunner: return list(sorted(all_tests)) def _get_parallel_tests_skip_list(self, repo_path): - skip_list_file_path = "{}/tests/integration/parallel_skip.json".format(repo_path) - if not os.path.isfile(skip_list_file_path) or os.path.getsize(skip_list_file_path) == 0: - raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(skip_list_file_path)) + skip_list_file_path = "{}/tests/integration/parallel_skip.json".format( + repo_path + ) + if ( + not os.path.isfile(skip_list_file_path) + or os.path.getsize(skip_list_file_path) == 0 + ): + raise Exception( + "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format( + skip_list_file_path + ) + ) skip_list_tests = [] with open(skip_list_file_path, "r") as skip_list_file: @@ -336,7 +426,7 @@ class ClickhouseIntegrationTestsRunner: def group_test_by_file(self, tests): result = {} for test in tests: - test_file = test.split('::')[0] + test_file = test.split("::")[0] if test_file not in result: result[test_file] = [] result[test_file].append(test) @@ -344,7 +434,10 @@ class ClickhouseIntegrationTestsRunner: def _update_counters(self, main_counters, current_counters): for test in current_counters["PASSED"]: - if test not in main_counters["PASSED"] and test not in main_counters["FLAKY"]: + if ( + test not in main_counters["PASSED"] + and test not in main_counters["FLAKY"] + ): is_flaky = False if test in main_counters["FAILED"]: main_counters["FAILED"].remove(test) @@ -369,45 +462,63 @@ class ClickhouseIntegrationTestsRunner: main_counters[state].append(test) def _get_runner_image_cmd(self, repo_path): - image_cmd = '' - if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-image-version'): + image_cmd = "" + if self._can_run_with( + os.path.join(repo_path, "tests/integration", "runner"), + "--docker-image-version", + ): for img in self.get_images_names(): if img == "clickhouse/integration-tests-runner": runner_version = self.get_single_image_version() - logging.info("Can run with custom docker image version %s", runner_version) - image_cmd += ' --docker-image-version={} '.format(runner_version) + logging.info( + "Can run with custom docker image version %s", runner_version + ) + image_cmd += " --docker-image-version={} ".format(runner_version) else: - if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-compose-images-tags'): - image_cmd += '--docker-compose-images-tags={} '.format(self.get_image_with_version(img)) + if self._can_run_with( + os.path.join(repo_path, "tests/integration", "runner"), + "--docker-compose-images-tags", + ): + image_cmd += "--docker-compose-images-tags={} ".format( + self.get_image_with_version(img) + ) else: - image_cmd = '' + image_cmd = "" logging.info("Cannot run with custom docker image version :(") return image_cmd def _find_test_data_dirs(self, repo_path, test_names): relpaths = {} for test_name in test_names: - if '/' in test_name: - test_dir = test_name[:test_name.find('/')] + if "/" in test_name: + test_dir = test_name[: test_name.find("/")] else: test_dir = test_name if os.path.isdir(os.path.join(repo_path, "tests/integration", test_dir)): - for name in os.listdir(os.path.join(repo_path, "tests/integration", test_dir)): + for name in os.listdir( + os.path.join(repo_path, "tests/integration", test_dir) + ): relpath = os.path.join(os.path.join(test_dir, name)) - mtime = os.path.getmtime(os.path.join(repo_path, "tests/integration", relpath)) + mtime = os.path.getmtime( + os.path.join(repo_path, "tests/integration", relpath) + ) relpaths[relpath] = mtime return relpaths def _get_test_data_dirs_difference(self, new_snapshot, old_snapshot): res = set() for path in new_snapshot: - if (not path in old_snapshot) or (old_snapshot[path] != new_snapshot[path]): + if (path not in old_snapshot) or (old_snapshot[path] != new_snapshot[path]): res.add(path) return res - def try_run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): + def try_run_test_group( + self, repo_path, test_group, tests_in_group, num_tries, num_workers + ): try: - return self.run_test_group(repo_path, test_group, tests_in_group, num_tries, num_workers) + return self.run_test_group( + repo_path, test_group, tests_in_group, num_tries, num_workers + ) except Exception as e: logging.info("Failed to run {}:\n{}".format(str(test_group), str(e))) counters = { @@ -423,7 +534,9 @@ class ClickhouseIntegrationTestsRunner: tests_times[test] = 0 return counters, tests_times, [] - def run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): + def run_test_group( + self, repo_path, test_group, tests_in_group, num_tries, num_workers + ): counters = { "ERROR": [], "PASSED": [], @@ -441,7 +554,7 @@ class ClickhouseIntegrationTestsRunner: return counters, tests_times, [] image_cmd = self._get_runner_image_cmd(repo_path) - test_group_str = test_group.replace('/', '_').replace('.', '_') + test_group_str = test_group.replace("/", "_").replace(".", "_") log_paths = [] test_data_dirs = {} @@ -453,8 +566,8 @@ class ClickhouseIntegrationTestsRunner: test_names = set([]) for test_name in tests_in_group: if test_name not in counters["PASSED"]: - if '[' in test_name: - test_names.add(test_name[:test_name.find('[')]) + if "[" in test_name: + test_names.add(test_name[: test_name.find("[")]) else: test_names.add(test_name) @@ -464,47 +577,83 @@ class ClickhouseIntegrationTestsRunner: info_basename = test_group_str + "_" + str(i) + ".nfo" info_path = os.path.join(repo_path, "tests/integration", info_basename) - test_cmd = ' '.join([test for test in sorted(test_names)]) - parallel_cmd = " --parallel {} ".format(num_workers) if num_workers > 0 else "" + test_cmd = " ".join([test for test in sorted(test_names)]) + parallel_cmd = ( + " --parallel {} ".format(num_workers) if num_workers > 0 else "" + ) # -r -- show extra test summary: # -f -- (f)ailed # -E -- (E)rror # -p -- (p)assed # -s -- (s)kipped cmd = "cd {}/tests/integration && timeout -s 9 1h ./runner {} {} -t {} {} '-rfEps --run-id={} --color=no --durations=0 {}' | tee {}".format( - repo_path, self._get_runner_opts(), image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), info_path) + repo_path, + self._get_runner_opts(), + image_cmd, + test_cmd, + parallel_cmd, + i, + _get_deselect_option(self.should_skip_tests()), + info_path, + ) log_basename = test_group_str + "_" + str(i) + ".log" log_path = os.path.join(repo_path, "tests/integration", log_basename) - with open(log_path, 'w') as log: + with open(log_path, "w") as log: logging.info("Executing cmd: %s", cmd) - retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() + retcode = subprocess.Popen( + cmd, shell=True, stderr=log, stdout=log + ).wait() if retcode == 0: logging.info("Run %s group successfully", test_group) else: logging.info("Some tests failed") extra_logs_names = [log_basename] - log_result_path = os.path.join(str(self.path()), 'integration_run_' + log_basename) + log_result_path = os.path.join( + str(self.path()), "integration_run_" + log_basename + ) shutil.copy(log_path, log_result_path) log_paths.append(log_result_path) - for pytest_log_path in glob.glob(os.path.join(repo_path, "tests/integration/pytest*.log")): - new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(pytest_log_path) - os.rename(pytest_log_path, os.path.join(repo_path, "tests/integration", new_name)) + for pytest_log_path in glob.glob( + os.path.join(repo_path, "tests/integration/pytest*.log") + ): + new_name = ( + test_group_str + + "_" + + str(i) + + "_" + + os.path.basename(pytest_log_path) + ) + os.rename( + pytest_log_path, + os.path.join(repo_path, "tests/integration", new_name), + ) extra_logs_names.append(new_name) dockerd_log_path = os.path.join(repo_path, "tests/integration/dockerd.log") if os.path.exists(dockerd_log_path): - new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(dockerd_log_path) - os.rename(dockerd_log_path, os.path.join(repo_path, "tests/integration", new_name)) + new_name = ( + test_group_str + + "_" + + str(i) + + "_" + + os.path.basename(dockerd_log_path) + ) + os.rename( + dockerd_log_path, + os.path.join(repo_path, "tests/integration", new_name), + ) extra_logs_names.append(new_name) if os.path.exists(info_path): extra_logs_names.append(info_basename) new_counters = get_counters(info_path) for state, tests in new_counters.items(): - logging.info("Tests with %s state (%s): %s", state, len(tests), tests) + logging.info( + "Tests with %s state (%s): %s", state, len(tests), tests + ) times_lines = parse_test_times(info_path) new_tests_times = get_test_times(times_lines) self._update_counters(counters, new_counters) @@ -512,19 +661,35 @@ class ClickhouseIntegrationTestsRunner: tests_times[test_name] = test_time test_data_dirs_new = self._find_test_data_dirs(repo_path, test_names) - test_data_dirs_diff = self._get_test_data_dirs_difference(test_data_dirs_new, test_data_dirs) + test_data_dirs_diff = self._get_test_data_dirs_difference( + test_data_dirs_new, test_data_dirs + ) test_data_dirs = test_data_dirs_new if extra_logs_names or test_data_dirs_diff: - extras_result_path = os.path.join(str(self.path()), "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz") - self._compress_logs(os.path.join(repo_path, "tests/integration"), extra_logs_names + list(test_data_dirs_diff), extras_result_path) + extras_result_path = os.path.join( + str(self.path()), + "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz", + ) + self._compress_logs( + os.path.join(repo_path, "tests/integration"), + extra_logs_names + list(test_data_dirs_diff), + extras_result_path, + ) log_paths.append(extras_result_path) if len(counters["PASSED"]) + len(counters["FLAKY"]) == len(tests_in_group): logging.info("All tests from group %s passed", test_group) break - if len(counters["PASSED"]) + len(counters["FLAKY"]) >= 0 and len(counters["FAILED"]) == 0 and len(counters["ERROR"]) == 0: - logging.info("Seems like all tests passed but some of them are skipped or deselected. Ignoring them and finishing group.") + if ( + len(counters["PASSED"]) + len(counters["FLAKY"]) >= 0 + and len(counters["FAILED"]) == 0 + and len(counters["ERROR"]) == 0 + ): + logging.info( + "Seems like all tests passed but some of them are skipped or " + "deselected. Ignoring them and finishing group." + ) break else: # Mark all non tried tests as errors, with '::' in name @@ -532,49 +697,56 @@ class ClickhouseIntegrationTestsRunner: # we run whole test dirs like "test_odbc_interaction" and don't # want to mark them as error so we filter by '::'. for test in tests_in_group: - if (test not in counters["PASSED"] and - test not in counters["ERROR"] and - test not in counters["SKIPPED"] and - test not in counters["FAILED"] and - '::' in test): + if ( + test not in counters["PASSED"] + and test not in counters["ERROR"] + and test not in counters["SKIPPED"] + and test not in counters["FAILED"] + and "::" in test + ): counters["ERROR"].append(test) return counters, tests_times, log_paths - def run_flaky_check(self, repo_path, build_path): - pr_info = self.params['pr_info'] + def run_flaky_check(self, repo_path, build_path, should_fail=False): + pr_info = self.params["pr_info"] - # pytest swears, if we require to run some tests which was renamed or deleted - tests_to_run = filter_existing_tests(get_tests_to_run(pr_info), repo_path) + tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") - return 'success', 'Nothing to run', [('Nothing to run', 'OK')], '' + return "success", NO_CHANGES_MSG, [(NO_CHANGES_MSG, "OK")], "" self._install_clickhouse(build_path) - logging.info("Found '%s' tests to run", ' '.join(tests_to_run)) + logging.info("Found '%s' tests to run", " ".join(tests_to_run)) result_state = "success" description_prefix = "No flaky tests: " start = time.time() logging.info("Starting check with retries") final_retry = 0 logs = [] - for i in range(TRIES_COUNT): + tires_num = 1 if should_fail else FLAKY_TRIES_COUNT + for i in range(tires_num): final_retry += 1 logging.info("Running tests for the %s time", i) - counters, tests_times, log_paths = self.try_run_test_group(repo_path, "flaky", tests_to_run, 1, 1) + counters, tests_times, log_paths = self.try_run_test_group( + repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1 + ) logs += log_paths if counters["FAILED"]: - logging.info("Found failed tests: %s", ' '.join(counters["FAILED"])) - description_prefix = "Flaky tests found: " + logging.info("Found failed tests: %s", " ".join(counters["FAILED"])) + description_prefix = "Failed tests found: " result_state = "failure" - break + if not should_fail: + break if counters["ERROR"]: - description_prefix = "Flaky tests found: " - logging.info("Found error tests: %s", ' '.join(counters["ERROR"])) - # NOTE "error" result state will restart the whole test task, so we use "failure" here + description_prefix = "Failed tests found: " + logging.info("Found error tests: %s", " ".join(counters["ERROR"])) + # NOTE "error" result state will restart the whole test task, + # so we use "failure" here result_state = "failure" - break - assert len(counters["FLAKY"]) == 0 + if not should_fail: + break + assert len(counters["FLAKY"]) == 0 or should_fail logging.info("Try is OK, all tests passed, going to clear env") clear_ip_tables_and_restart_daemons() logging.info("And going to sleep for some time") @@ -591,17 +763,34 @@ class ClickhouseIntegrationTestsRunner: text_state = "FAIL" else: text_state = state - test_result += [(c + ' (✕' + str(final_retry) + ')', text_state, "{:.2f}".format(tests_times[c])) for c in counters[state]] - status_text = description_prefix + ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()]) + test_result += [ + ( + c + " (✕" + str(final_retry) + ")", + text_state, + "{:.2f}".format(tests_times[c]), + ) + for c in counters[state] + ] + status_text = description_prefix + ", ".join( + [ + str(n).lower().replace("failed", "fail") + ": " + str(len(c)) + for n, c in counters.items() + ] + ) return result_state, status_text, test_result, logs def run_impl(self, repo_path, build_path): - if self.flaky_check: - return self.run_flaky_check(repo_path, build_path) + if self.flaky_check or self.bugfix_validate_check: + return self.run_flaky_check( + repo_path, build_path, should_fail=self.bugfix_validate_check + ) self._install_clickhouse(build_path) - logging.info("Dump iptables before run %s", subprocess.check_output("sudo iptables -L", shell=True)) + logging.info( + "Dump iptables before run %s", + subprocess.check_output("sudo iptables -L", shell=True), + ) all_tests = self._get_all_tests(repo_path) if self.run_by_hash_total != 0: @@ -613,18 +802,36 @@ class ClickhouseIntegrationTestsRunner: all_tests = all_filtered_by_hash_tests parallel_skip_tests = self._get_parallel_tests_skip_list(repo_path) - logging.info("Found %s tests first 3 %s", len(all_tests), ' '.join(all_tests[:3])) - filtered_sequential_tests = list(filter(lambda test: test in all_tests, parallel_skip_tests)) - filtered_parallel_tests = list(filter(lambda test: test not in parallel_skip_tests, all_tests)) - not_found_tests = list(filter(lambda test: test not in all_tests, parallel_skip_tests)) - logging.info("Found %s tests first 3 %s, parallel %s, other %s", len(all_tests), ' '.join(all_tests[:3]), len(filtered_parallel_tests), len(filtered_sequential_tests)) - logging.info("Not found %s tests first 3 %s", len(not_found_tests), ' '.join(not_found_tests[:3])) + logging.info( + "Found %s tests first 3 %s", len(all_tests), " ".join(all_tests[:3]) + ) + filtered_sequential_tests = list( + filter(lambda test: test in all_tests, parallel_skip_tests) + ) + filtered_parallel_tests = list( + filter(lambda test: test not in parallel_skip_tests, all_tests) + ) + not_found_tests = list( + filter(lambda test: test not in all_tests, parallel_skip_tests) + ) + logging.info( + "Found %s tests first 3 %s, parallel %s, other %s", + len(all_tests), + " ".join(all_tests[:3]), + len(filtered_parallel_tests), + len(filtered_sequential_tests), + ) + logging.info( + "Not found %s tests first 3 %s", + len(not_found_tests), + " ".join(not_found_tests[:3]), + ) grouped_tests = self.group_test_by_file(filtered_sequential_tests) i = 0 for par_group in chunks(filtered_parallel_tests, PARALLEL_GROUP_SIZE): - grouped_tests["parallel{}".format(i)] = par_group - i+=1 + grouped_tests[f"parallel{i}"] = par_group + i += 1 logging.info("Found %s tests groups", len(grouped_tests)) counters = { @@ -646,12 +853,18 @@ class ClickhouseIntegrationTestsRunner: for group, tests in items_to_run: logging.info("Running test group %s containing %s tests", group, len(tests)) - group_counters, group_test_times, log_paths = self.try_run_test_group(repo_path, group, tests, MAX_RETRY, NUM_WORKERS) + group_counters, group_test_times, log_paths = self.try_run_test_group( + repo_path, group, tests, MAX_RETRY, NUM_WORKERS + ) total_tests = 0 for counter, value in group_counters.items(): - logging.info("Tests from group %s stats, %s count %s", group, counter, len(value)) + logging.info( + "Tests from group %s stats, %s count %s", group, counter, len(value) + ) counters[counter] += value - logging.info("Totally have %s with status %s", len(counters[counter]), counter) + logging.info( + "Totally have %s with status %s", len(counters[counter]), counter + ) total_tests += len(counters[counter]) logging.info("Totally finished tests %s/%s", total_tests, len(all_tests)) @@ -664,7 +877,9 @@ class ClickhouseIntegrationTestsRunner: break if counters["FAILED"] or counters["ERROR"]: - logging.info("Overall status failure, because we have tests in FAILED or ERROR state") + logging.info( + "Overall status failure, because we have tests in FAILED or ERROR state" + ) result_state = "failure" else: logging.info("Overall success!") @@ -678,42 +893,49 @@ class ClickhouseIntegrationTestsRunner: text_state = "FAIL" else: text_state = state - test_result += [(c, text_state, "{:.2f}".format(tests_times[c]), tests_log_paths[c]) for c in counters[state]] + test_result += [ + (c, text_state, "{:.2f}".format(tests_times[c]), tests_log_paths[c]) + for c in counters[state] + ] - failed_sum = len(counters['FAILED']) + len(counters['ERROR']) - status_text = "fail: {}, passed: {}, flaky: {}".format(failed_sum, len(counters['PASSED']), len(counters['FLAKY'])) + failed_sum = len(counters["FAILED"]) + len(counters["ERROR"]) + status_text = "fail: {}, passed: {}, flaky: {}".format( + failed_sum, len(counters["PASSED"]), len(counters["FLAKY"]) + ) if self.soft_deadline_time < time.time(): status_text = "Timeout, " + status_text result_state = "failure" - counters['FLAKY'] = [] + counters["FLAKY"] = [] if not counters or sum(len(counter) for counter in counters.values()) == 0: status_text = "No tests found for some reason! It's a bug" result_state = "failure" - if '(memory)' in self.params['context_name']: + if "(memory)" in self.params["context_name"]: result_state = "success" return result_state, status_text, test_result, [] + def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) + if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") repo_path = os.environ.get("CLICKHOUSE_TESTS_REPO_PATH") build_path = os.environ.get("CLICKHOUSE_TESTS_BUILD_PATH") result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH") params_path = os.environ.get("CLICKHOUSE_TESTS_JSON_PARAMS_PATH") - params = json.loads(open(params_path, 'r').read()) + params = json.loads(open(params_path, "r").read()) runner = ClickhouseIntegrationTestsRunner(result_path, params) logging.info("Running tests") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4b0a9a2835b..9dd10ce9b52 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,23 +5,34 @@ import os from helpers.test_tools import TSV from helpers.network import _NetworkManager + @pytest.fixture(autouse=True, scope="session") def cleanup_environment(): try: if int(os.environ.get("PYTEST_CLEANUP_CONTAINERS", 0)) == 1: logging.debug(f"Cleaning all iptables rules") _NetworkManager.clean_all_user_iptables_rules() - result = run_and_check(['docker ps | wc -l'], shell=True) + result = run_and_check(["docker ps | wc -l"], shell=True) if int(result) > 1: if int(os.environ.get("PYTEST_CLEANUP_CONTAINERS", 0)) != 1: - logging.warning(f"Docker containters({int(result)}) are running before tests run. They can be left from previous pytest run and cause test failures.\n"\ - "You can set env PYTEST_CLEANUP_CONTAINERS=1 or use runner with --cleanup-containers argument to enable automatic containers cleanup.") + logging.warning( + f"Docker containters({int(result)}) are running before tests run. They can be left from previous pytest run and cause test failures.\n" + "You can set env PYTEST_CLEANUP_CONTAINERS=1 or use runner with --cleanup-containers argument to enable automatic containers cleanup." + ) else: logging.debug("Trying to kill unstopped containers...") - run_and_check([f'docker kill $(docker container list --all --quiet)'], shell=True, nothrow=True) - run_and_check([f'docker rm $docker container list --all --quiet)'], shell=True, nothrow=True) + run_and_check( + [f"docker kill $(docker container list --all --quiet)"], + shell=True, + nothrow=True, + ) + run_and_check( + [f"docker rm $docker container list --all --quiet)"], + shell=True, + nothrow=True, + ) logging.debug("Unstopped containers killed") - r = run_and_check(['docker-compose', 'ps', '--services', '--all']) + r = run_and_check(["docker-compose", "ps", "--services", "--all"]) logging.debug(f"Docker ps before start:{r.stdout}") else: logging.debug(f"No running containers") @@ -31,8 +42,14 @@ def cleanup_environment(): yield + def pytest_addoption(parser): - parser.addoption("--run-id", default="", help="run-id is used as postfix in _instances_{} directory") + parser.addoption( + "--run-id", + default="", + help="run-id is used as postfix in _instances_{} directory", + ) + def pytest_configure(config): - os.environ['INTEGRATION_TESTS_RUN_ID'] = config.option.run_id + os.environ["INTEGRATION_TESTS_RUN_ID"] = config.option.run_id diff --git a/tests/integration/helpers/0_common_instance_config.xml b/tests/integration/helpers/0_common_instance_config.xml index 71a2f8f4b13..493366b1209 100644 --- a/tests/integration/helpers/0_common_instance_config.xml +++ b/tests/integration/helpers/0_common_instance_config.xml @@ -1,5 +1,5 @@ - Europe/Moscow + Etc/UTC 0.0.0.0 custom_ /var/lib/clickhouse/ diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index b0e764bf174..af49408abee 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -6,79 +6,117 @@ from threading import Timer class Client: - def __init__(self, host, port=9000, command='/usr/bin/clickhouse-client'): + def __init__(self, host, port=9000, command="/usr/bin/clickhouse-client"): self.host = host self.port = port self.command = [command] - if os.path.basename(command) == 'clickhouse': - self.command.append('client') + if os.path.basename(command) == "clickhouse": + self.command.append("client") - self.command += ['--host', self.host, '--port', str(self.port), '--stacktrace'] + self.command += ["--host", self.host, "--port", str(self.port), "--stacktrace"] - def query(self, sql, - stdin=None, - timeout=None, - settings=None, - user=None, - password=None, - database=None, - ignore_error=False, - query_id=None): - return self.get_query_request(sql, - stdin=stdin, - timeout=timeout, - settings=settings, - user=user, - password=password, - database=database, - ignore_error=ignore_error, - query_id=query_id).get_answer() + def query( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ignore_error=False, + query_id=None, + ): + return self.get_query_request( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ignore_error=ignore_error, + query_id=query_id, + ).get_answer() - def get_query_request(self, sql, - stdin=None, - timeout=None, - settings=None, - user=None, - password=None, - database=None, - ignore_error=False, - query_id=None): + def get_query_request( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ignore_error=False, + query_id=None, + ): command = self.command[:] if stdin is None: - command += ['--multiquery', '--testmode'] + command += ["--multiquery", "--testmode"] stdin = sql else: - command += ['--query', sql] + command += ["--query", sql] if settings is not None: for setting, value in settings.items(): - command += ['--' + setting, str(value)] + command += ["--" + setting, str(value)] if user is not None: - command += ['--user', user] + command += ["--user", user] if password is not None: - command += ['--password', password] + command += ["--password", password] if database is not None: - command += ['--database', database] + command += ["--database", database] if query_id is not None: - command += ['--query_id', query_id] + command += ["--query_id", query_id] return CommandRequest(command, stdin, timeout, ignore_error) - def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, - database=None): - return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, - password=password, database=database).get_error() + def query_and_get_error( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ): + return self.get_query_request( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ).get_error() - def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, - database=None): - return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, - password=password, database=database).get_answer_and_error() + def query_and_get_answer_with_error( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ): + return self.get_query_request( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ).get_answer_and_error() class QueryTimeoutExceedException(Exception): @@ -95,7 +133,7 @@ class QueryRuntimeException(Exception): class CommandRequest: def __init__(self, command, stdin=None, timeout=None, ignore_error=False): # Write data to tmp file to avoid PIPEs and execution blocking - stdin_file = tempfile.TemporaryFile(mode='w+') + stdin_file = tempfile.TemporaryFile(mode="w+") stdin_file.write(stdin) stdin_file.seek(0) self.stdout_file = tempfile.TemporaryFile() @@ -108,11 +146,19 @@ class CommandRequest: # can print some debug information there env = {} env["TSAN_OPTIONS"] = "verbosity=0" - self.process = sp.Popen(command, stdin=stdin_file, stdout=self.stdout_file, stderr=self.stderr_file, env=env, universal_newlines=True) + self.process = sp.Popen( + command, + stdin=stdin_file, + stdout=self.stdout_file, + stderr=self.stderr_file, + env=env, + universal_newlines=True, + ) self.timer = None self.process_finished_before_timeout = True if timeout is not None: + def kill_process(): if self.process.poll() is None: self.process_finished_before_timeout = False @@ -126,16 +172,25 @@ class CommandRequest: self.stdout_file.seek(0) self.stderr_file.seek(0) - stdout = self.stdout_file.read().decode('utf-8', errors='replace') - stderr = self.stderr_file.read().decode('utf-8', errors='replace') + stdout = self.stdout_file.read().decode("utf-8", errors="replace") + stderr = self.stderr_file.read().decode("utf-8", errors="replace") - if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error: + if ( + self.timer is not None + and not self.process_finished_before_timeout + and not self.ignore_error + ): logging.debug(f"Timed out. Last stdout:{stdout}, stderr:{stderr}") - raise QueryTimeoutExceedException('Client timed out!') + raise QueryTimeoutExceedException("Client timed out!") if (self.process.returncode != 0 or stderr) and not self.ignore_error: raise QueryRuntimeException( - 'Client failed! Return code: {}, stderr: {}'.format(self.process.returncode, stderr), self.process.returncode, stderr) + "Client failed! Return code: {}, stderr: {}".format( + self.process.returncode, stderr + ), + self.process.returncode, + stderr, + ) return stdout @@ -144,14 +199,22 @@ class CommandRequest: self.stdout_file.seek(0) self.stderr_file.seek(0) - stdout = self.stdout_file.read().decode('utf-8', errors='replace') - stderr = self.stderr_file.read().decode('utf-8', errors='replace') + stdout = self.stdout_file.read().decode("utf-8", errors="replace") + stderr = self.stderr_file.read().decode("utf-8", errors="replace") - if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error: - raise QueryTimeoutExceedException('Client timed out!') + if ( + self.timer is not None + and not self.process_finished_before_timeout + and not self.ignore_error + ): + raise QueryTimeoutExceedException("Client timed out!") - if (self.process.returncode == 0): - raise QueryRuntimeException('Client expected to be failed but succeeded! stdout: {}'.format(stdout), self.process.returncode, stderr) + if self.process.returncode == 0: + raise QueryRuntimeException( + "Client expected to be failed but succeeded! stdout: {}".format(stdout), + self.process.returncode, + stderr, + ) return stderr @@ -160,10 +223,14 @@ class CommandRequest: self.stdout_file.seek(0) self.stderr_file.seek(0) - stdout = self.stdout_file.read().decode('utf-8', errors='replace') - stderr = self.stderr_file.read().decode('utf-8', errors='replace') + stdout = self.stdout_file.read().decode("utf-8", errors="replace") + stderr = self.stderr_file.read().decode("utf-8", errors="replace") - if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error: - raise QueryTimeoutExceedException('Client timed out!') + if ( + self.timer is not None + and not self.process_finished_before_timeout + and not self.ignore_error + ): + raise QueryTimeoutExceedException("Client timed out!") return (stdout, stderr) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 627e3725232..d0b5e892f5b 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -23,8 +23,9 @@ import psycopg2 import pymongo import pymysql import requests -from confluent_kafka.avro.cached_schema_registry_client import \ - CachedSchemaRegistryClient +from confluent_kafka.avro.cached_schema_registry_client import ( + CachedSchemaRegistryClient, +) from dict2xml import dict2xml from kazoo.client import KazooClient from kazoo.exceptions import KazooException @@ -42,28 +43,48 @@ from .hdfs_api import HDFSApi HELPERS_DIR = p.dirname(__file__) CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") -LOCAL_DOCKER_COMPOSE_DIR = p.join(CLICKHOUSE_ROOT_DIR, "docker/test/integration/runner/compose/") -DEFAULT_ENV_NAME = '.env' +LOCAL_DOCKER_COMPOSE_DIR = p.join( + CLICKHOUSE_ROOT_DIR, "docker/test/integration/runner/compose/" +) +DEFAULT_ENV_NAME = ".env" SANITIZER_SIGN = "==================" # to create docker-compose env file def _create_env_file(path, variables): logging.debug(f"Env {variables} stored in {path}") - with open(path, 'w') as f: + with open(path, "w") as f: for var, value in list(variables.items()): f.write("=".join([var, value]) + "\n") return path -def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300, nothrow=False, detach=False): + +def run_and_check( + args, + env=None, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=300, + nothrow=False, + detach=False, +): if detach: - subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, shell=shell) + subprocess.Popen( + args, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=env, + shell=shell, + ) return logging.debug(f"Command:{args}") - res = subprocess.run(args, stdout=stdout, stderr=stderr, env=env, shell=shell, timeout=timeout) - out = res.stdout.decode('utf-8') - err = res.stderr.decode('utf-8') + res = subprocess.run( + args, stdout=stdout, stderr=stderr, env=env, shell=shell, timeout=timeout + ) + out = res.stdout.decode("utf-8") + err = res.stderr.decode("utf-8") # check_call(...) from subprocess does not print stderr, so we do it manually for outline in out.splitlines(): logging.debug(f"Stdout:{outline}") @@ -74,18 +95,22 @@ def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=su if env: logging.debug(f"Env:{env}") if not nothrow: - raise Exception(f"Command {args} return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}") + raise Exception( + f"Command {args} return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}" + ) return out + # Based on https://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python/2838309#2838309 def get_free_port(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind(("",0)) + s.bind(("", 0)) s.listen(1) port = s.getsockname()[1] s.close() return port + def retry_exception(num, delay, func, exception=Exception, *args, **kwargs): """ Retry if `func()` throws, `num` times. @@ -100,87 +125,109 @@ def retry_exception(num, delay, func, exception=Exception, *args, **kwargs): try: func(*args, **kwargs) time.sleep(delay) - except exception: # pylint: disable=broad-except + except exception: # pylint: disable=broad-except i += 1 continue return - raise StopIteration('Function did not finished successfully') + raise StopIteration("Function did not finished successfully") + def subprocess_check_call(args, detach=False, nothrow=False): # Uncomment for debugging - #logging.info('run:' + ' '.join(args)) + # logging.info('run:' + ' '.join(args)) return run_and_check(args, detach=detach, nothrow=nothrow) def get_odbc_bridge_path(): - path = os.environ.get('CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH') + path = os.environ.get("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH") if path is None: - server_path = os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH') + server_path = os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH") if server_path is not None: - return os.path.join(os.path.dirname(server_path), 'clickhouse-odbc-bridge') + return os.path.join(os.path.dirname(server_path), "clickhouse-odbc-bridge") else: - return '/usr/bin/clickhouse-odbc-bridge' + return "/usr/bin/clickhouse-odbc-bridge" return path + def get_library_bridge_path(): - path = os.environ.get('CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH') + path = os.environ.get("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH") if path is None: - server_path = os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH') + server_path = os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH") if server_path is not None: - return os.path.join(os.path.dirname(server_path), 'clickhouse-library-bridge') + return os.path.join( + os.path.dirname(server_path), "clickhouse-library-bridge" + ) else: - return '/usr/bin/clickhouse-library-bridge' + return "/usr/bin/clickhouse-library-bridge" return path + def get_docker_compose_path(): - compose_path = os.environ.get('DOCKER_COMPOSE_DIR') + compose_path = os.environ.get("DOCKER_COMPOSE_DIR") if compose_path is not None: return os.path.dirname(compose_path) else: - if os.path.exists(os.path.dirname('/compose/')): - return os.path.dirname('/compose/') # default in docker runner container + if os.path.exists(os.path.dirname("/compose/")): + return os.path.dirname("/compose/") # default in docker runner container else: - logging.debug(f"Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {LOCAL_DOCKER_COMPOSE_DIR}") + logging.debug( + f"Fallback docker_compose_path to LOCAL_DOCKER_COMPOSE_DIR: {LOCAL_DOCKER_COMPOSE_DIR}" + ) return LOCAL_DOCKER_COMPOSE_DIR + def check_kafka_is_available(kafka_id, kafka_port): - p = subprocess.Popen(('docker', - 'exec', - '-i', - kafka_id, - '/usr/bin/kafka-broker-api-versions', - '--bootstrap-server', - f'INSIDE://localhost:{kafka_port}'), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + ( + "docker", + "exec", + "-i", + kafka_id, + "/usr/bin/kafka-broker-api-versions", + "--bootstrap-server", + f"INSIDE://localhost:{kafka_port}", + ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) p.communicate() return p.returncode == 0 + def check_rabbitmq_is_available(rabbitmq_id): - p = subprocess.Popen(('docker', - 'exec', - '-i', - rabbitmq_id, - 'rabbitmqctl', - 'await_startup'), - stdout=subprocess.PIPE) + p = subprocess.Popen( + ("docker", "exec", "-i", rabbitmq_id, "rabbitmqctl", "await_startup"), + stdout=subprocess.PIPE, + ) p.communicate() return p.returncode == 0 + def enable_consistent_hash_plugin(rabbitmq_id): - p = subprocess.Popen(('docker', - 'exec', - '-i', - rabbitmq_id, - "rabbitmq-plugins", "enable", "rabbitmq_consistent_hash_exchange"), - stdout=subprocess.PIPE) + p = subprocess.Popen( + ( + "docker", + "exec", + "-i", + rabbitmq_id, + "rabbitmq-plugins", + "enable", + "rabbitmq_consistent_hash_exchange", + ), + stdout=subprocess.PIPE, + ) p.communicate() return p.returncode == 0 + def get_instances_dir(): - if 'INTEGRATION_TESTS_RUN_ID' in os.environ and os.environ['INTEGRATION_TESTS_RUN_ID']: - return '_instances_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) + if ( + "INTEGRATION_TESTS_RUN_ID" in os.environ + and os.environ["INTEGRATION_TESTS_RUN_ID"] + ): + return "_instances_" + shlex.quote(os.environ["INTEGRATION_TESTS_RUN_ID"]) else: - return '_instances' + return "_instances" class ClickHouseCluster: @@ -192,53 +239,87 @@ class ClickHouseCluster: these directories will contain logs, database files, docker-compose config, ClickHouse configs etc. """ - def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None, - odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None, - zookeeper_keyfile=None, zookeeper_certfile=None): + def __init__( + self, + base_path, + name=None, + base_config_dir=None, + server_bin_path=None, + client_bin_path=None, + odbc_bridge_bin_path=None, + library_bridge_bin_path=None, + zookeeper_config_path=None, + custom_dockerd_host=None, + zookeeper_keyfile=None, + zookeeper_certfile=None, + ): for param in list(os.environ.keys()): logging.debug("ENV %40s %s" % (param, os.environ[param])) self.base_path = base_path self.base_dir = p.dirname(base_path) - self.name = name if name is not None else '' + self.name = name if name is not None else "" - self.base_config_dir = base_config_dir or os.environ.get('CLICKHOUSE_TESTS_BASE_CONFIG_DIR', - '/etc/clickhouse-server/') + self.base_config_dir = base_config_dir or os.environ.get( + "CLICKHOUSE_TESTS_BASE_CONFIG_DIR", "/etc/clickhouse-server/" + ) self.server_bin_path = p.realpath( - server_bin_path or os.environ.get('CLICKHOUSE_TESTS_SERVER_BIN_PATH', '/usr/bin/clickhouse')) - self.odbc_bridge_bin_path = p.realpath(odbc_bridge_bin_path or get_odbc_bridge_path()) - self.library_bridge_bin_path = p.realpath(library_bridge_bin_path or get_library_bridge_path()) + server_bin_path + or os.environ.get("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse") + ) + self.odbc_bridge_bin_path = p.realpath( + odbc_bridge_bin_path or get_odbc_bridge_path() + ) + self.library_bridge_bin_path = p.realpath( + library_bridge_bin_path or get_library_bridge_path() + ) self.client_bin_path = p.realpath( - client_bin_path or os.environ.get('CLICKHOUSE_TESTS_CLIENT_BIN_PATH', '/usr/bin/clickhouse-client')) - self.zookeeper_config_path = p.join(self.base_dir, zookeeper_config_path) if zookeeper_config_path else p.join( - HELPERS_DIR, 'zookeeper_config.xml') + client_bin_path + or os.environ.get( + "CLICKHOUSE_TESTS_CLIENT_BIN_PATH", "/usr/bin/clickhouse-client" + ) + ) + self.zookeeper_config_path = ( + p.join(self.base_dir, zookeeper_config_path) + if zookeeper_config_path + else p.join(HELPERS_DIR, "zookeeper_config.xml") + ) - project_name = pwd.getpwuid(os.getuid()).pw_name + p.basename(self.base_dir) + self.name + project_name = ( + pwd.getpwuid(os.getuid()).pw_name + p.basename(self.base_dir) + self.name + ) # docker-compose removes everything non-alphanumeric from project names so we do it too. - self.project_name = re.sub(r'[^a-z0-9]', '', project_name.lower()) - instances_dir_name = '_instances' + self.project_name = re.sub(r"[^a-z0-9]", "", project_name.lower()) + instances_dir_name = "_instances" if self.name: - instances_dir_name += '_' + self.name + instances_dir_name += "_" + self.name - if 'INTEGRATION_TESTS_RUN_ID' in os.environ and os.environ['INTEGRATION_TESTS_RUN_ID']: - instances_dir_name += '_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) + if ( + "INTEGRATION_TESTS_RUN_ID" in os.environ + and os.environ["INTEGRATION_TESTS_RUN_ID"] + ): + instances_dir_name += "_" + shlex.quote( + os.environ["INTEGRATION_TESTS_RUN_ID"] + ) self.instances_dir = p.join(self.base_dir, instances_dir_name) - self.docker_logs_path = p.join(self.instances_dir, 'docker.log') + self.docker_logs_path = p.join(self.instances_dir, "docker.log") self.env_file = p.join(self.instances_dir, DEFAULT_ENV_NAME) self.env_variables = {} self.env_variables["TSAN_OPTIONS"] = "second_deadlock_stack=1" self.env_variables["CLICKHOUSE_WATCHDOG_ENABLE"] = "0" self.up_called = False - custom_dockerd_host = custom_dockerd_host or os.environ.get('CLICKHOUSE_TESTS_DOCKERD_HOST') + custom_dockerd_host = custom_dockerd_host or os.environ.get( + "CLICKHOUSE_TESTS_DOCKERD_HOST" + ) self.docker_api_version = os.environ.get("DOCKER_API_VERSION") self.docker_base_tag = os.environ.get("DOCKER_BASE_TAG", "latest") - self.base_cmd = ['docker-compose'] + self.base_cmd = ["docker-compose"] if custom_dockerd_host: - self.base_cmd += ['--host', custom_dockerd_host] - self.base_cmd += ['--env-file', self.env_file] - self.base_cmd += ['--project-name', self.project_name] + self.base_cmd += ["--host", custom_dockerd_host] + self.base_cmd += ["--env-file", self.env_file] + self.base_cmd += ["--project-name", self.project_name] self.base_zookeeper_cmd = None self.base_mysql_cmd = [] @@ -275,7 +356,7 @@ class ClickHouseCluster: self.with_minio = False self.minio_dir = os.path.join(self.instances_dir, "minio") - self.minio_certs_dir = None # source for certificates + self.minio_certs_dir = None # source for certificates self.minio_host = "minio1" self.minio_ip = None self.minio_bucket = "root" @@ -295,14 +376,16 @@ class ClickHouseCluster: self.hdfs_data_port = 50075 self.hdfs_dir = p.abspath(p.join(self.instances_dir, "hdfs")) self.hdfs_logs_dir = os.path.join(self.hdfs_dir, "logs") - self.hdfs_api = None # also for kerberized hdfs + self.hdfs_api = None # also for kerberized hdfs # available when with_kerberized_hdfs == True self.hdfs_kerberized_host = "kerberizedhdfs1" self.hdfs_kerberized_ip = None self.hdfs_kerberized_name_port = 50070 self.hdfs_kerberized_data_port = 1006 - self.hdfs_kerberized_dir = p.abspath(p.join(self.instances_dir, "kerberized_hdfs")) + self.hdfs_kerberized_dir = p.abspath( + p.join(self.instances_dir, "kerberized_hdfs") + ) self.hdfs_kerberized_logs_dir = os.path.join(self.hdfs_kerberized_dir, "logs") # available when with_kafka == True @@ -316,7 +399,9 @@ class ClickHouseCluster: # available when with_kerberozed_kafka == True self.kerberized_kafka_host = "kerberized_kafka1" self.kerberized_kafka_port = get_free_port() - self.kerberized_kafka_docker_id = self.get_instance_docker_id(self.kerberized_kafka_host) + self.kerberized_kafka_docker_id = self.get_instance_docker_id( + self.kerberized_kafka_host + ) # available when with_mongo == True self.mongo_host = "mongo1" @@ -388,7 +473,6 @@ class ClickHouseCluster: self.mysql_cluster_dir = p.abspath(p.join(self.instances_dir, "mysql")) self.mysql_cluster_logs_dir = os.path.join(self.mysql_dir, "logs") - # available when with_mysql8 == True self.mysql8_host = "mysql80" self.mysql8_port = 3306 @@ -404,7 +488,9 @@ class ClickHouseCluster: # available when with_zookeper == True self.use_keeper = True self.zookeeper_port = 2181 - self.keeper_instance_dir_prefix = p.join(p.abspath(self.instances_dir), "keeper") # if use_keeper = True + self.keeper_instance_dir_prefix = p.join( + p.abspath(self.instances_dir), "keeper" + ) # if use_keeper = True self.zookeeper_instance_dir_prefix = p.join(self.instances_dir, "zk") self.zookeeper_dirs_to_create = [] @@ -421,7 +507,11 @@ class ClickHouseCluster: logging.debug(f"CLUSTER INIT base_config_dir:{self.base_config_dir}") def cleanup(self): - if os.environ and 'DISABLE_CLEANUP' in os.environ and os.environ['DISABLE_CLEANUP'] == "1": + if ( + os.environ + and "DISABLE_CLEANUP" in os.environ + and os.environ["DISABLE_CLEANUP"] == "1" + ): logging.warning("Cleanup is disabled") return @@ -429,10 +519,12 @@ class ClickHouseCluster: try: unstopped_containers = self.get_running_containers() if unstopped_containers: - logging.debug(f"Trying to kill unstopped containers: {unstopped_containers}") + logging.debug( + f"Trying to kill unstopped containers: {unstopped_containers}" + ) for id in unstopped_containers: - run_and_check(f'docker kill {id}', shell=True, nothrow=True) - run_and_check(f'docker rm {id}', shell=True, nothrow=True) + run_and_check(f"docker kill {id}", shell=True, nothrow=True) + run_and_check(f"docker rm {id}", shell=True, nothrow=True) unstopped_containers = self.get_running_containers() if unstopped_containers: logging.debug(f"Left unstopped containers: {unstopped_containers}") @@ -465,9 +557,9 @@ class ClickHouseCluster: try: logging.debug("Trying to prune unused volumes...") - result = run_and_check(['docker volume ls | wc -l'], shell=True) - if int(result>0): - run_and_check(['docker', 'volume', 'prune', '-f']) + result = run_and_check(["docker volume ls | wc -l"], shell=True) + if int(result > 0): + run_and_check(["docker", "volume", "prune", "-f"]) logging.debug(f"Volumes pruned: {result}") except: pass @@ -485,7 +577,7 @@ class ClickHouseCluster: def get_client_cmd(self): cmd = self.client_bin_path - if p.basename(cmd) == 'clickhouse': + if p.basename(cmd) == "clickhouse": cmd += " client" return cmd @@ -495,310 +587,605 @@ class ClickHouseCluster: # container_name = project_name + '_' + instance_name + '_1' # We need to have "^/" and "$" in the "--filter name" option below to filter by exact name of the container, see # https://stackoverflow.com/questions/48767760/how-to-make-docker-container-ls-f-name-filter-by-exact-name - filter_name = f'^/{self.project_name}_.*_1$' + filter_name = f"^/{self.project_name}_.*_1$" # We want the command "docker container list" to show only containers' ID and their names, separated by colon. - format = '{{.ID}}:{{.Names}}' - containers = run_and_check(f"docker container list --all --filter name='{filter_name}' --format '{format}'", shell=True) - containers = dict(line.split(':', 1) for line in containers.decode('utf8').splitlines()) + format = "{{.ID}}:{{.Names}}" + containers = run_and_check( + f"docker container list --all --filter name='{filter_name}' --format '{format}'", + shell=True, + ) + containers = dict( + line.split(":", 1) for line in containers.decode("utf8").splitlines() + ) return containers - def copy_file_from_container_to_container(self, src_node, src_path, dst_node, dst_path): + def copy_file_from_container_to_container( + self, src_node, src_path, dst_node, dst_path + ): fname = os.path.basename(src_path) - run_and_check([f"docker cp {src_node.docker_id}:{src_path} {self.instances_dir}"], shell=True) - run_and_check([f"docker cp {self.instances_dir}/{fname} {dst_node.docker_id}:{dst_path}"], shell=True) + run_and_check( + [f"docker cp {src_node.docker_id}:{src_path} {self.instances_dir}"], + shell=True, + ) + run_and_check( + [f"docker cp {self.instances_dir}/{fname} {dst_node.docker_id}:{dst_path}"], + shell=True, + ) - def setup_zookeeper_secure_cmd(self, instance, env_variables, docker_compose_yml_dir): - logging.debug('Setup ZooKeeper Secure') - zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_zookeeper_secure.yml') - env_variables['ZOO_SECURE_CLIENT_PORT'] = str(self.zookeeper_secure_port) - env_variables['ZK_FS'] = 'bind' + def setup_zookeeper_secure_cmd( + self, instance, env_variables, docker_compose_yml_dir + ): + logging.debug("Setup ZooKeeper Secure") + zookeeper_docker_compose_path = p.join( + docker_compose_yml_dir, "docker_compose_zookeeper_secure.yml" + ) + env_variables["ZOO_SECURE_CLIENT_PORT"] = str(self.zookeeper_secure_port) + env_variables["ZK_FS"] = "bind" for i in range(1, 4): - zk_data_path = os.path.join(self.zookeeper_instance_dir_prefix + str(i), "data") - zk_log_path = os.path.join(self.zookeeper_instance_dir_prefix + str(i), "log") - env_variables['ZK_DATA' + str(i)] = zk_data_path - env_variables['ZK_DATA_LOG' + str(i)] = zk_log_path + zk_data_path = os.path.join( + self.zookeeper_instance_dir_prefix + str(i), "data" + ) + zk_log_path = os.path.join( + self.zookeeper_instance_dir_prefix + str(i), "log" + ) + env_variables["ZK_DATA" + str(i)] = zk_data_path + env_variables["ZK_DATA_LOG" + str(i)] = zk_log_path self.zookeeper_dirs_to_create += [zk_data_path, zk_log_path] logging.debug(f"DEBUG ZK: {self.zookeeper_dirs_to_create}") self.with_zookeeper_secure = True - self.base_cmd.extend(['--file', zookeeper_docker_compose_path]) - self.base_zookeeper_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', zookeeper_docker_compose_path] + self.base_cmd.extend(["--file", zookeeper_docker_compose_path]) + self.base_zookeeper_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + zookeeper_docker_compose_path, + ] return self.base_zookeeper_cmd def setup_zookeeper_cmd(self, instance, env_variables, docker_compose_yml_dir): - logging.debug('Setup ZooKeeper') - zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_zookeeper.yml') + logging.debug("Setup ZooKeeper") + zookeeper_docker_compose_path = p.join( + docker_compose_yml_dir, "docker_compose_zookeeper.yml" + ) - env_variables['ZK_FS'] = 'bind' + env_variables["ZK_FS"] = "bind" for i in range(1, 4): - zk_data_path = os.path.join(self.zookeeper_instance_dir_prefix + str(i), "data") - zk_log_path = os.path.join(self.zookeeper_instance_dir_prefix + str(i), "log") - env_variables['ZK_DATA' + str(i)] = zk_data_path - env_variables['ZK_DATA_LOG' + str(i)] = zk_log_path + zk_data_path = os.path.join( + self.zookeeper_instance_dir_prefix + str(i), "data" + ) + zk_log_path = os.path.join( + self.zookeeper_instance_dir_prefix + str(i), "log" + ) + env_variables["ZK_DATA" + str(i)] = zk_data_path + env_variables["ZK_DATA_LOG" + str(i)] = zk_log_path self.zookeeper_dirs_to_create += [zk_data_path, zk_log_path] logging.debug(f"DEBUG ZK: {self.zookeeper_dirs_to_create}") self.with_zookeeper = True - self.base_cmd.extend(['--file', zookeeper_docker_compose_path]) - self.base_zookeeper_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', zookeeper_docker_compose_path] + self.base_cmd.extend(["--file", zookeeper_docker_compose_path]) + self.base_zookeeper_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + zookeeper_docker_compose_path, + ] return self.base_zookeeper_cmd def setup_keeper_cmd(self, instance, env_variables, docker_compose_yml_dir): - logging.debug('Setup Keeper') - keeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_keeper.yml') + logging.debug("Setup Keeper") + keeper_docker_compose_path = p.join( + docker_compose_yml_dir, "docker_compose_keeper.yml" + ) binary_path = self.server_bin_path - if binary_path.endswith('-server'): - binary_path = binary_path[:-len('-server')] + if binary_path.endswith("-server"): + binary_path = binary_path[: -len("-server")] - env_variables['keeper_binary'] = binary_path - env_variables['image'] = "clickhouse/integration-test:" + self.docker_base_tag - env_variables['user'] = str(os.getuid()) - env_variables['keeper_fs'] = 'bind' + env_variables["keeper_binary"] = binary_path + env_variables["image"] = "clickhouse/integration-test:" + self.docker_base_tag + env_variables["user"] = str(os.getuid()) + env_variables["keeper_fs"] = "bind" for i in range(1, 4): keeper_instance_dir = self.keeper_instance_dir_prefix + f"{i}" logs_dir = os.path.join(keeper_instance_dir, "log") configs_dir = os.path.join(keeper_instance_dir, "config") coordination_dir = os.path.join(keeper_instance_dir, "coordination") - env_variables[f'keeper_logs_dir{i}'] = logs_dir - env_variables[f'keeper_config_dir{i}'] = configs_dir - env_variables[f'keeper_db_dir{i}'] = coordination_dir + env_variables[f"keeper_logs_dir{i}"] = logs_dir + env_variables[f"keeper_config_dir{i}"] = configs_dir + env_variables[f"keeper_db_dir{i}"] = coordination_dir self.zookeeper_dirs_to_create += [logs_dir, configs_dir, coordination_dir] logging.debug(f"DEBUG KEEPER: {self.zookeeper_dirs_to_create}") - self.with_zookeeper = True - self.base_cmd.extend(['--file', keeper_docker_compose_path]) - self.base_zookeeper_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', keeper_docker_compose_path] + self.base_cmd.extend(["--file", keeper_docker_compose_path]) + self.base_zookeeper_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + keeper_docker_compose_path, + ] return self.base_zookeeper_cmd def setup_mysql_client_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mysql_client = True - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_client.yml')]) - self.base_mysql_client_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_client.yml')] + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mysql_client.yml"), + ] + ) + self.base_mysql_client_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mysql_client.yml"), + ] return self.base_mysql_client_cmd def setup_mysql_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mysql = True - env_variables['MYSQL_HOST'] = self.mysql_host - env_variables['MYSQL_PORT'] = str(self.mysql_port) - env_variables['MYSQL_ROOT_HOST'] = '%' - env_variables['MYSQL_LOGS'] = self.mysql_logs_dir - env_variables['MYSQL_LOGS_FS'] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')]) - self.base_mysql_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql.yml')] + env_variables["MYSQL_HOST"] = self.mysql_host + env_variables["MYSQL_PORT"] = str(self.mysql_port) + env_variables["MYSQL_ROOT_HOST"] = "%" + env_variables["MYSQL_LOGS"] = self.mysql_logs_dir + env_variables["MYSQL_LOGS_FS"] = "bind" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_mysql.yml")] + ) + self.base_mysql_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mysql.yml"), + ] return self.base_mysql_cmd def setup_mysql8_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mysql8 = True - env_variables['MYSQL8_HOST'] = self.mysql8_host - env_variables['MYSQL8_PORT'] = str(self.mysql8_port) - env_variables['MYSQL8_ROOT_HOST'] = '%' - env_variables['MYSQL8_LOGS'] = self.mysql8_logs_dir - env_variables['MYSQL8_LOGS_FS'] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_8_0.yml')]) - self.base_mysql8_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_8_0.yml')] + env_variables["MYSQL8_HOST"] = self.mysql8_host + env_variables["MYSQL8_PORT"] = str(self.mysql8_port) + env_variables["MYSQL8_ROOT_HOST"] = "%" + env_variables["MYSQL8_LOGS"] = self.mysql8_logs_dir + env_variables["MYSQL8_LOGS_FS"] = "bind" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_mysql_8_0.yml")] + ) + self.base_mysql8_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mysql_8_0.yml"), + ] return self.base_mysql8_cmd def setup_mysql_cluster_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mysql_cluster = True - env_variables['MYSQL_CLUSTER_PORT'] = str(self.mysql_port) - env_variables['MYSQL_CLUSTER_ROOT_HOST'] = '%' - env_variables['MYSQL_CLUSTER_LOGS'] = self.mysql_cluster_logs_dir - env_variables['MYSQL_CLUSTER_LOGS_FS'] = "bind" + env_variables["MYSQL_CLUSTER_PORT"] = str(self.mysql_port) + env_variables["MYSQL_CLUSTER_ROOT_HOST"] = "%" + env_variables["MYSQL_CLUSTER_LOGS"] = self.mysql_cluster_logs_dir + env_variables["MYSQL_CLUSTER_LOGS_FS"] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_cluster.yml')]) - self.base_mysql_cluster_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_mysql_cluster.yml')] + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mysql_cluster.yml"), + ] + ) + self.base_mysql_cluster_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mysql_cluster.yml"), + ] return self.base_mysql_cluster_cmd def setup_postgres_cmd(self, instance, env_variables, docker_compose_yml_dir): - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')]) - env_variables['POSTGRES_PORT'] = str(self.postgres_port) - env_variables['POSTGRES_DIR'] = self.postgres_logs_dir - env_variables['POSTGRES_LOGS_FS'] = "bind" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_postgres.yml")] + ) + env_variables["POSTGRES_PORT"] = str(self.postgres_port) + env_variables["POSTGRES_DIR"] = self.postgres_logs_dir + env_variables["POSTGRES_LOGS_FS"] = "bind" self.with_postgres = True - self.base_postgres_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres.yml')] + self.base_postgres_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_postgres.yml"), + ] return self.base_postgres_cmd - def setup_postgres_cluster_cmd(self, instance, env_variables, docker_compose_yml_dir): + def setup_postgres_cluster_cmd( + self, instance, env_variables, docker_compose_yml_dir + ): self.with_postgres_cluster = True - env_variables['POSTGRES_PORT'] = str(self.postgres_port) - env_variables['POSTGRES2_DIR'] = self.postgres2_logs_dir - env_variables['POSTGRES3_DIR'] = self.postgres3_logs_dir - env_variables['POSTGRES4_DIR'] = self.postgres4_logs_dir - env_variables['POSTGRES_LOGS_FS'] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres_cluster.yml')]) - self.base_postgres_cluster_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_postgres_cluster.yml')] + env_variables["POSTGRES_PORT"] = str(self.postgres_port) + env_variables["POSTGRES2_DIR"] = self.postgres2_logs_dir + env_variables["POSTGRES3_DIR"] = self.postgres3_logs_dir + env_variables["POSTGRES4_DIR"] = self.postgres4_logs_dir + env_variables["POSTGRES_LOGS_FS"] = "bind" + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_postgres_cluster.yml"), + ] + ) + self.base_postgres_cluster_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_postgres_cluster.yml"), + ] def setup_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_hdfs = True - env_variables['HDFS_HOST'] = self.hdfs_host - env_variables['HDFS_NAME_PORT'] = str(self.hdfs_name_port) - env_variables['HDFS_DATA_PORT'] = str(self.hdfs_data_port) - env_variables['HDFS_LOGS'] = self.hdfs_logs_dir - env_variables['HDFS_FS'] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]) - self.base_hdfs_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')] + env_variables["HDFS_HOST"] = self.hdfs_host + env_variables["HDFS_NAME_PORT"] = str(self.hdfs_name_port) + env_variables["HDFS_DATA_PORT"] = str(self.hdfs_data_port) + env_variables["HDFS_LOGS"] = self.hdfs_logs_dir + env_variables["HDFS_FS"] = "bind" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_hdfs.yml")] + ) + self.base_hdfs_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_hdfs.yml"), + ] logging.debug("HDFS BASE CMD:{self.base_hdfs_cmd)}") return self.base_hdfs_cmd - def setup_kerberized_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir): + def setup_kerberized_hdfs_cmd( + self, instance, env_variables, docker_compose_yml_dir + ): self.with_kerberized_hdfs = True - env_variables['KERBERIZED_HDFS_HOST'] = self.hdfs_kerberized_host - env_variables['KERBERIZED_HDFS_NAME_PORT'] = str(self.hdfs_kerberized_name_port) - env_variables['KERBERIZED_HDFS_DATA_PORT'] = str(self.hdfs_kerberized_data_port) - env_variables['KERBERIZED_HDFS_LOGS'] = self.hdfs_kerberized_logs_dir - env_variables['KERBERIZED_HDFS_FS'] = "bind" - env_variables['KERBERIZED_HDFS_DIR'] = instance.path + '/' - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_hdfs.yml')]) - self.base_kerberized_hdfs_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_hdfs.yml')] + env_variables["KERBERIZED_HDFS_HOST"] = self.hdfs_kerberized_host + env_variables["KERBERIZED_HDFS_NAME_PORT"] = str(self.hdfs_kerberized_name_port) + env_variables["KERBERIZED_HDFS_DATA_PORT"] = str(self.hdfs_kerberized_data_port) + env_variables["KERBERIZED_HDFS_LOGS"] = self.hdfs_kerberized_logs_dir + env_variables["KERBERIZED_HDFS_FS"] = "bind" + env_variables["KERBERIZED_HDFS_DIR"] = instance.path + "/" + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_kerberized_hdfs.yml"), + ] + ) + self.base_kerberized_hdfs_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_kerberized_hdfs.yml"), + ] return self.base_kerberized_hdfs_cmd def setup_kafka_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_kafka = True - env_variables['KAFKA_HOST'] = self.kafka_host - env_variables['KAFKA_EXTERNAL_PORT'] = str(self.kafka_port) - env_variables['SCHEMA_REGISTRY_EXTERNAL_PORT'] = str(self.schema_registry_port) - env_variables['SCHEMA_REGISTRY_INTERNAL_PORT'] = "8081" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')]) - self.base_kafka_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_kafka.yml')] + env_variables["KAFKA_HOST"] = self.kafka_host + env_variables["KAFKA_EXTERNAL_PORT"] = str(self.kafka_port) + env_variables["SCHEMA_REGISTRY_EXTERNAL_PORT"] = str(self.schema_registry_port) + env_variables["SCHEMA_REGISTRY_INTERNAL_PORT"] = "8081" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_kafka.yml")] + ) + self.base_kafka_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_kafka.yml"), + ] return self.base_kafka_cmd - def setup_kerberized_kafka_cmd(self, instance, env_variables, docker_compose_yml_dir): + def setup_kerberized_kafka_cmd( + self, instance, env_variables, docker_compose_yml_dir + ): self.with_kerberized_kafka = True - env_variables['KERBERIZED_KAFKA_DIR'] = instance.path + '/' - env_variables['KERBERIZED_KAFKA_HOST'] = self.kerberized_kafka_host - env_variables['KERBERIZED_KAFKA_EXTERNAL_PORT'] = str(self.kerberized_kafka_port) - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_kafka.yml')]) - self.base_kerberized_kafka_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_kerberized_kafka.yml')] + env_variables["KERBERIZED_KAFKA_DIR"] = instance.path + "/" + env_variables["KERBERIZED_KAFKA_HOST"] = self.kerberized_kafka_host + env_variables["KERBERIZED_KAFKA_EXTERNAL_PORT"] = str( + self.kerberized_kafka_port + ) + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_kerberized_kafka.yml"), + ] + ) + self.base_kerberized_kafka_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_kerberized_kafka.yml"), + ] return self.base_kerberized_kafka_cmd def setup_redis_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_redis = True - env_variables['REDIS_HOST'] = self.redis_host - env_variables['REDIS_EXTERNAL_PORT'] = str(self.redis_port) - env_variables['REDIS_INTERNAL_PORT'] = "6379" + env_variables["REDIS_HOST"] = self.redis_host + env_variables["REDIS_EXTERNAL_PORT"] = str(self.redis_port) + env_variables["REDIS_INTERNAL_PORT"] = "6379" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')]) - self.base_redis_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_redis.yml')] + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_redis.yml")] + ) + self.base_redis_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_redis.yml"), + ] return self.base_redis_cmd def setup_rabbitmq_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_rabbitmq = True - env_variables['RABBITMQ_HOST'] = self.rabbitmq_host - env_variables['RABBITMQ_PORT'] = str(self.rabbitmq_port) - env_variables['RABBITMQ_LOGS'] = self.rabbitmq_logs_dir - env_variables['RABBITMQ_LOGS_FS'] = "bind" + env_variables["RABBITMQ_HOST"] = self.rabbitmq_host + env_variables["RABBITMQ_PORT"] = str(self.rabbitmq_port) + env_variables["RABBITMQ_LOGS"] = self.rabbitmq_logs_dir + env_variables["RABBITMQ_LOGS_FS"] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')]) - self.base_rabbitmq_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_rabbitmq.yml')] + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_rabbitmq.yml")] + ) + self.base_rabbitmq_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_rabbitmq.yml"), + ] return self.base_rabbitmq_cmd def setup_mongo_secure_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mongo = self.with_mongo_secure = True - env_variables['MONGO_HOST'] = self.mongo_host - env_variables['MONGO_EXTERNAL_PORT'] = str(self.mongo_port) - env_variables['MONGO_INTERNAL_PORT'] = "27017" - env_variables['MONGO_CONFIG_PATH'] = HELPERS_DIR - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo_secure.yml')]) - self.base_mongo_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo_secure.yml')] + env_variables["MONGO_HOST"] = self.mongo_host + env_variables["MONGO_EXTERNAL_PORT"] = str(self.mongo_port) + env_variables["MONGO_INTERNAL_PORT"] = "27017" + env_variables["MONGO_CONFIG_PATH"] = HELPERS_DIR + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mongo_secure.yml"), + ] + ) + self.base_mongo_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mongo_secure.yml"), + ] return self.base_mongo_cmd def setup_mongo_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_mongo = True - env_variables['MONGO_HOST'] = self.mongo_host - env_variables['MONGO_EXTERNAL_PORT'] = str(self.mongo_port) - env_variables['MONGO_INTERNAL_PORT'] = "27017" - env_variables['MONGO_NO_CRED_EXTERNAL_PORT'] = str(self.mongo_no_cred_port) - env_variables['MONGO_NO_CRED_INTERNAL_PORT'] = "27017" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')]) - self.base_mongo_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_mongo.yml')] + env_variables["MONGO_HOST"] = self.mongo_host + env_variables["MONGO_EXTERNAL_PORT"] = str(self.mongo_port) + env_variables["MONGO_INTERNAL_PORT"] = "27017" + env_variables["MONGO_NO_CRED_EXTERNAL_PORT"] = str(self.mongo_no_cred_port) + env_variables["MONGO_NO_CRED_INTERNAL_PORT"] = "27017" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_mongo.yml")] + ) + self.base_mongo_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_mongo.yml"), + ] return self.base_mongo_cmd def setup_minio_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_minio = True cert_d = p.join(self.minio_dir, "certs") - env_variables['MINIO_CERTS_DIR'] = cert_d - env_variables['MINIO_PORT'] = str(self.minio_port) - env_variables['SSL_CERT_FILE'] = p.join(self.base_dir, cert_d, 'public.crt') + env_variables["MINIO_CERTS_DIR"] = cert_d + env_variables["MINIO_PORT"] = str(self.minio_port) + env_variables["SSL_CERT_FILE"] = p.join(self.base_dir, cert_d, "public.crt") - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')]) - self.base_minio_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_minio.yml')] + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_minio.yml")] + ) + self.base_minio_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_minio.yml"), + ] return self.base_minio_cmd def setup_azurite_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_azurite = True - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_azurite.yml')]) - self.base_azurite_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_azurite.yml')] + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_azurite.yml")] + ) + self.base_azurite_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_azurite.yml"), + ] return self.base_azurite_cmd def setup_cassandra_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_cassandra = True - env_variables['CASSANDRA_PORT'] = str(self.cassandra_port) - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')]) - self.base_cassandra_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_cassandra.yml')] + env_variables["CASSANDRA_PORT"] = str(self.cassandra_port) + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_cassandra.yml")] + ) + self.base_cassandra_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_cassandra.yml"), + ] return self.base_cassandra_cmd def setup_jdbc_bridge_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_jdbc_bridge = True - env_variables['JDBC_DRIVER_LOGS'] = self.jdbc_driver_logs_dir - env_variables['JDBC_DRIVER_FS'] = "bind" - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_jdbc_bridge.yml')]) - self.base_jdbc_bridge_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_jdbc_bridge.yml')] + env_variables["JDBC_DRIVER_LOGS"] = self.jdbc_driver_logs_dir + env_variables["JDBC_DRIVER_FS"] = "bind" + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_jdbc_bridge.yml")] + ) + self.base_jdbc_bridge_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_jdbc_bridge.yml"), + ] return self.base_jdbc_bridge_cmd def setup_nginx_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_nginx = True - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_nginx.yml')]) - self.base_nginx_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_nginx.yml')] + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_nginx.yml")] + ) + self.base_nginx_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_nginx.yml"), + ] return self.base_nginx_cmd def setup_hive(self, instance, env_variables, docker_compose_yml_dir): self.with_hive = True - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hive.yml')]) - self.base_hive_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, - '--file', p.join(docker_compose_yml_dir, 'docker_compose_hive.yml')] + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_hive.yml")] + ) + self.base_hive_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_hive.yml"), + ] return self.base_hive_cmd - def add_instance(self, name, base_config_dir=None, main_configs=None, user_configs=None, dictionaries=None, - macros=None, with_zookeeper=False, with_zookeeper_secure=False, - with_mysql_client=False, with_mysql=False, with_mysql8=False, with_mysql_cluster=False, - with_kafka=False, with_kerberized_kafka=False, with_rabbitmq=False, clickhouse_path_dir=None, - with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, with_hdfs=False, - with_kerberized_hdfs=False, with_mongo=False, with_mongo_secure=False, with_nginx=False, - with_redis=False, with_minio=False, with_azurite=False, with_cassandra=False, with_jdbc_bridge=False, with_hive=False, - hostname=None, env_variables=None, image="clickhouse/integration-test", tag=None, - stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, external_dirs=None, tmpfs=None, - zookeeper_docker_compose_path=None, minio_certs_dir=None, use_keeper=True, - main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True, config_root_name="clickhouse", extra_configs=[]) -> 'ClickHouseInstance': + def add_instance( + self, + name, + base_config_dir=None, + main_configs=None, + user_configs=None, + dictionaries=None, + macros=None, + with_zookeeper=False, + with_zookeeper_secure=False, + with_mysql_client=False, + with_mysql=False, + with_mysql8=False, + with_mysql_cluster=False, + with_kafka=False, + with_kerberized_kafka=False, + with_rabbitmq=False, + clickhouse_path_dir=None, + with_odbc_drivers=False, + with_postgres=False, + with_postgres_cluster=False, + with_hdfs=False, + with_kerberized_hdfs=False, + with_mongo=False, + with_mongo_secure=False, + with_nginx=False, + with_redis=False, + with_minio=False, + with_azurite=False, + with_cassandra=False, + with_jdbc_bridge=False, + with_hive=False, + hostname=None, + env_variables=None, + image="clickhouse/integration-test", + tag=None, + stay_alive=False, + ipv4_address=None, + ipv6_address=None, + with_installed_binary=False, + external_dirs=None, + tmpfs=None, + zookeeper_docker_compose_path=None, + minio_certs_dir=None, + use_keeper=True, + main_config_name="config.xml", + users_config_name="users.xml", + copy_common_configs=True, + config_root_name="clickhouse", + extra_configs=[], + ) -> "ClickHouseInstance": """Add an instance to the cluster. @@ -812,10 +1199,13 @@ class ClickHouseCluster: """ if self.is_up: - raise Exception("Can\'t add instance %s: cluster is already up!" % name) + raise Exception("Can't add instance %s: cluster is already up!" % name) if name in self.instances: - raise Exception("Can\'t add instance `%s': there is already an instance with the same name!" % name) + raise Exception( + "Can't add instance `%s': there is already an instance with the same name!" + % name + ) if tag is None: tag = self.docker_base_tag @@ -826,13 +1216,17 @@ class ClickHouseCluster: # Code coverage files will be placed in database directory # (affect only WITH_COVERAGE=1 build) - env_variables['LLVM_PROFILE_FILE'] = '/var/lib/clickhouse/server_%h_%p_%m.profraw' + env_variables[ + "LLVM_PROFILE_FILE" + ] = "/var/lib/clickhouse/server_%h_%p_%m.profraw" instance = ClickHouseInstance( cluster=self, base_path=self.base_dir, name=name, - base_config_dir=base_config_dir if base_config_dir else self.base_config_dir, + base_config_dir=base_config_dir + if base_config_dir + else self.base_config_dir, custom_main_configs=main_configs or [], custom_user_configs=user_configs or [], custom_dictionaries=dictionaries or [], @@ -854,7 +1248,7 @@ class ClickHouseCluster: with_azurite=with_azurite, with_cassandra=with_cassandra, with_jdbc_bridge=with_jdbc_bridge, - with_hive = with_hive, + with_hive=with_hive, server_bin_path=self.server_bin_path, odbc_bridge_bin_path=self.odbc_bridge_bin_path, library_bridge_bin_path=self.library_bridge_bin_path, @@ -876,89 +1270,164 @@ class ClickHouseCluster: external_dirs=external_dirs, tmpfs=tmpfs or [], config_root_name=config_root_name, - extra_configs = extra_configs) + extra_configs=extra_configs, + ) docker_compose_yml_dir = get_docker_compose_path() self.instances[name] = instance if ipv4_address is not None or ipv6_address is not None: self.with_net_trics = True - self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_net.yml')]) + self.base_cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_net.yml")] + ) - self.base_cmd.extend(['--file', instance.docker_compose_path]) + self.base_cmd.extend(["--file", instance.docker_compose_path]) cmds = [] if with_zookeeper_secure and not self.with_zookeeper_secure: - cmds.append(self.setup_zookeeper_secure_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_zookeeper_secure_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_zookeeper and not self.with_zookeeper: if self.use_keeper: - cmds.append(self.setup_keeper_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_keeper_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) else: - cmds.append(self.setup_zookeeper_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_zookeeper_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_mysql_client and not self.with_mysql_client: - cmds.append(self.setup_mysql_client_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mysql_client_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_mysql and not self.with_mysql: - cmds.append(self.setup_mysql_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mysql_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_mysql8 and not self.with_mysql8: - cmds.append(self.setup_mysql8_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mysql8_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_mysql_cluster and not self.with_mysql_cluster: - cmds.append(self.setup_mysql_cluster_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mysql_cluster_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_postgres and not self.with_postgres: - cmds.append(self.setup_postgres_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_postgres_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_postgres_cluster and not self.with_postgres_cluster: - cmds.append(self.setup_postgres_cluster_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_postgres_cluster_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_odbc_drivers and not self.with_odbc_drivers: self.with_odbc_drivers = True if not self.with_mysql: - cmds.append(self.setup_mysql_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mysql_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if not self.with_postgres: - cmds.append(self.setup_postgres_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_postgres_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_kafka and not self.with_kafka: - cmds.append(self.setup_kafka_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_kafka_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_kerberized_kafka and not self.with_kerberized_kafka: - cmds.append(self.setup_kerberized_kafka_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_kerberized_kafka_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_rabbitmq and not self.with_rabbitmq: - cmds.append(self.setup_rabbitmq_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_rabbitmq_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_nginx and not self.with_nginx: - cmds.append(self.setup_nginx_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_nginx_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_hdfs and not self.with_hdfs: - cmds.append(self.setup_hdfs_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_hdfs_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_kerberized_hdfs and not self.with_kerberized_hdfs: - cmds.append(self.setup_kerberized_hdfs_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_kerberized_hdfs_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) - if (with_mongo or with_mongo_secure) and not (self.with_mongo or self.with_mongo_secure): + if (with_mongo or with_mongo_secure) and not ( + self.with_mongo or self.with_mongo_secure + ): if with_mongo_secure: - cmds.append(self.setup_mongo_secure_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mongo_secure_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) else: - cmds.append(self.setup_mongo_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_mongo_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if self.with_net_trics: for cmd in cmds: - cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_net.yml')]) + cmd.extend( + ["--file", p.join(docker_compose_yml_dir, "docker_compose_net.yml")] + ) if with_redis and not self.with_redis: - cmds.append(self.setup_redis_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_redis_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_minio and not self.with_minio: - cmds.append(self.setup_minio_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_minio_cmd(instance, env_variables, docker_compose_yml_dir) + ) if with_azurite and not self.with_azurite: - cmds.append(self.setup_azurite_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_azurite_cmd(instance, env_variables, docker_compose_yml_dir) + ) if minio_certs_dir is not None: if self.minio_certs_dir is None: @@ -967,31 +1436,49 @@ class ClickHouseCluster: raise Exception("Overwriting minio certs dir") if with_cassandra and not self.with_cassandra: - cmds.append(self.setup_cassandra_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_cassandra_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_jdbc_bridge and not self.with_jdbc_bridge: - cmds.append(self.setup_jdbc_bridge_cmd(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_jdbc_bridge_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) if with_hive: - cmds.append(self.setup_hive(instance, env_variables, docker_compose_yml_dir)) + cmds.append( + self.setup_hive(instance, env_variables, docker_compose_yml_dir) + ) - logging.debug("Cluster name:{} project_name:{}. Added instance name:{} tag:{} base_cmd:{} docker_compose_yml_dir:{}".format( - self.name, self.project_name, name, tag, self.base_cmd, docker_compose_yml_dir)) + logging.debug( + "Cluster name:{} project_name:{}. Added instance name:{} tag:{} base_cmd:{} docker_compose_yml_dir:{}".format( + self.name, + self.project_name, + name, + tag, + self.base_cmd, + docker_compose_yml_dir, + ) + ) return instance def get_instance_docker_id(self, instance_name): # According to how docker-compose names containers. - return self.project_name + '_' + instance_name + '_1' + return self.project_name + "_" + instance_name + "_1" def _replace(self, path, what, to): - with open(path, 'r') as p: + with open(path, "r") as p: data = p.read() data = data.replace(what, to) - with open(path, 'w') as p: + with open(path, "w") as p: p.write(data) def restart_instance_with_ip_change(self, node, new_ip): - if '::' in new_ip: + if "::" in new_ip: if node.ipv6_address is None: raise Exception("You should specity ipv6_address in add_node method") self._replace(node.docker_compose_path, node.ipv6_address, new_ip) @@ -1003,7 +1490,9 @@ class ClickHouseCluster: node.ipv4_address = new_ip run_and_check(self.base_cmd + ["stop", node.name]) run_and_check(self.base_cmd + ["rm", "--force", "--stop", node.name]) - run_and_check(self.base_cmd + ["up", "--force-recreate", "--no-deps", "-d", node.name]) + run_and_check( + self.base_cmd + ["up", "--force-recreate", "--no-deps", "-d", node.name] + ) node.ip_address = self.get_instance_ip(node.name) node.client = Client(node.ip_address, command=self.client_bin_path) @@ -1024,9 +1513,11 @@ class ClickHouseCluster: logging.debug("get_instance_ip instance_name={}".format(instance_name)) docker_id = self.get_instance_docker_id(instance_name) # for cont in self.docker_client.containers.list(): - # logging.debug("CONTAINERS LIST: ID={} NAME={} STATUS={}".format(cont.id, cont.name, cont.status)) + # logging.debug("CONTAINERS LIST: ID={} NAME={} STATUS={}".format(cont.id, cont.name, cont.status)) handle = self.docker_client.containers.get(docker_id) - return list(handle.attrs['NetworkSettings']['Networks'].values())[0]['IPAddress'] + return list(handle.attrs["NetworkSettings"]["Networks"].values())[0][ + "IPAddress" + ] def get_container_id(self, instance_name): return self.get_instance_docker_id(instance_name) @@ -1038,31 +1529,40 @@ class ClickHouseCluster: container_id = self.get_container_id(instance_name) return self.docker_client.api.logs(container_id).decode() - def exec_in_container(self, container_id, cmd, detach=False, nothrow=False, use_cli=True, **kwargs): + def exec_in_container( + self, container_id, cmd, detach=False, nothrow=False, use_cli=True, **kwargs + ): if use_cli: - logging.debug(f"run container_id:{container_id} detach:{detach} nothrow:{nothrow} cmd: {cmd}") + logging.debug( + f"run container_id:{container_id} detach:{detach} nothrow:{nothrow} cmd: {cmd}" + ) exec_cmd = ["docker", "exec"] - if 'user' in kwargs: - exec_cmd += ['-u', kwargs['user']] - result = subprocess_check_call(exec_cmd + [container_id] + cmd, detach=detach, nothrow=nothrow) + if "user" in kwargs: + exec_cmd += ["-u", kwargs["user"]] + result = subprocess_check_call( + exec_cmd + [container_id] + cmd, detach=detach, nothrow=nothrow + ) return result else: exec_id = self.docker_client.api.exec_create(container_id, cmd, **kwargs) output = self.docker_client.api.exec_start(exec_id, detach=detach) - exit_code = self.docker_client.api.exec_inspect(exec_id)['ExitCode'] + exit_code = self.docker_client.api.exec_inspect(exec_id)["ExitCode"] if exit_code: container_info = self.docker_client.api.inspect_container(container_id) - image_id = container_info.get('Image') + image_id = container_info.get("Image") image_info = self.docker_client.api.inspect_image(image_id) logging.debug(("Command failed in container {}: ".format(container_id))) pprint.pprint(container_info) logging.debug("") - logging.debug(("Container {} uses image {}: ".format(container_id, image_id))) + logging.debug( + ("Container {} uses image {}: ".format(container_id, image_id)) + ) pprint.pprint(image_info) logging.debug("") - message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format(' '.join(cmd), container_id, - exit_code, output) + message = 'Cmd "{}" failed in container {}. Return code {}. Output: {}'.format( + " ".join(cmd), container_id, exit_code, output + ) if nothrow: logging.debug(message) else: @@ -1076,12 +1576,20 @@ class ClickHouseCluster: data = fdata.read() encodedBytes = base64.b64encode(data.encode("utf-8")) encodedStr = str(encodedBytes, "utf-8") - self.exec_in_container(container_id, - ["bash", "-c", "echo {} | base64 --decode > {}".format(encodedStr, dest_path)], - user='root') + self.exec_in_container( + container_id, + [ + "bash", + "-c", + "echo {} | base64 --decode > {}".format(encodedStr, dest_path), + ], + user="root", + ) - def wait_for_url(self, url="http://localhost:8123/ping", conn_timeout=2, interval=2, timeout=60): - if not url.startswith('http'): + def wait_for_url( + self, url="http://localhost:8123/ping", conn_timeout=2, interval=2, timeout=60 + ): + if not url.startswith("http"): url = "http://" + url if interval <= 0: interval = 2 @@ -1093,29 +1601,44 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - requests.get(url, allow_redirects=True, timeout=conn_timeout, verify=False).raise_for_status() - logging.debug("{} is available after {} seconds".format(url, time.time() - start)) + requests.get( + url, allow_redirects=True, timeout=conn_timeout, verify=False + ).raise_for_status() + logging.debug( + "{} is available after {} seconds".format(url, time.time() - start) + ) return except Exception as ex: - logging.debug("{} Attempt {} failed, retrying in {} seconds".format(ex, attempts, interval)) + logging.debug( + "{} Attempt {} failed, retrying in {} seconds".format( + ex, attempts, interval + ) + ) attempts += 1 errors += [str(ex)] time.sleep(interval) - run_and_check(['docker', 'ps', '--all']) + run_and_check(["docker", "ps", "--all"]) logging.error("Can't connect to URL:{}".format(errors)) - raise Exception("Cannot wait URL {}(interval={}, timeout={}, attempts={})".format( - url, interval, timeout, attempts)) + raise Exception( + "Cannot wait URL {}(interval={}, timeout={}, attempts={})".format( + url, interval, timeout, attempts + ) + ) def wait_mysql_client_to_start(self, timeout=180): start = time.time() errors = [] - self.mysql_client_container = self.get_docker_handle(self.get_instance_docker_id(self.mysql_client_host)) + self.mysql_client_container = self.get_docker_handle( + self.get_instance_docker_id(self.mysql_client_host) + ) while time.time() - start < timeout: try: - info = self.mysql_client_container.client.api.inspect_container(self.mysql_client_container.name) - if info['State']['Health']['Status'] == 'healthy': + info = self.mysql_client_container.client.api.inspect_container( + self.mysql_client_container.name + ) + if info["State"]["Health"]["Status"] == "healthy": logging.debug("Mysql Client Container Started") return time.sleep(1) @@ -1123,17 +1646,22 @@ class ClickHouseCluster: errors += [str(ex)] time.sleep(1) - run_and_check(['docker', 'ps', '--all']) + run_and_check(["docker", "ps", "--all"]) logging.error("Can't connect to MySQL Client:{}".format(errors)) raise Exception("Cannot wait MySQL Client container") def wait_mysql_to_start(self, timeout=180): - self.mysql_ip = self.get_instance_ip('mysql57') + self.mysql_ip = self.get_instance_ip("mysql57") start = time.time() errors = [] while time.time() - start < timeout: try: - conn = pymysql.connect(user='root', password='clickhouse', host=self.mysql_ip, port=self.mysql_port) + conn = pymysql.connect( + user="root", + password="clickhouse", + host=self.mysql_ip, + port=self.mysql_port, + ) conn.close() logging.debug("Mysql Started") return @@ -1141,16 +1669,21 @@ class ClickHouseCluster: errors += [str(ex)] time.sleep(0.5) - run_and_check(['docker-compose', 'ps', '--services', '--all']) + run_and_check(["docker-compose", "ps", "--services", "--all"]) logging.error("Can't connect to MySQL:{}".format(errors)) raise Exception("Cannot wait MySQL container") def wait_mysql8_to_start(self, timeout=180): - self.mysql8_ip = self.get_instance_ip('mysql80') + self.mysql8_ip = self.get_instance_ip("mysql80") start = time.time() while time.time() - start < timeout: try: - conn = pymysql.connect(user='root', password='clickhouse', host=self.mysql8_ip, port=self.mysql8_port) + conn = pymysql.connect( + user="root", + password="clickhouse", + host=self.mysql8_ip, + port=self.mysql8_port, + ) conn.close() logging.debug("Mysql 8 Started") return @@ -1158,7 +1691,7 @@ class ClickHouseCluster: logging.debug("Can't connect to MySQL 8 " + str(ex)) time.sleep(0.5) - run_and_check(['docker-compose', 'ps', '--services', '--all']) + run_and_check(["docker-compose", "ps", "--services", "--all"]) raise Exception("Cannot wait MySQL 8 container") def wait_mysql_cluster_to_start(self, timeout=180): @@ -1170,7 +1703,12 @@ class ClickHouseCluster: while time.time() - start < timeout: try: for ip in [self.mysql2_ip, self.mysql3_ip, self.mysql4_ip]: - conn = pymysql.connect(user='root', password='clickhouse', host=ip, port=self.mysql_port) + conn = pymysql.connect( + user="root", + password="clickhouse", + host=ip, + port=self.mysql_port, + ) conn.close() logging.debug(f"Mysql Started {ip}") return @@ -1178,7 +1716,7 @@ class ClickHouseCluster: errors += [str(ex)] time.sleep(0.5) - run_and_check(['docker-compose', 'ps', '--services', '--all']) + run_and_check(["docker-compose", "ps", "--services", "--all"]) logging.error("Can't connect to MySQL:{}".format(errors)) raise Exception("Cannot wait MySQL container") @@ -1187,7 +1725,13 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - self.postgres_conn = psycopg2.connect(host=self.postgres_ip, port=self.postgres_port, database='postgres', user='postgres', password='mysecretpassword') + self.postgres_conn = psycopg2.connect( + host=self.postgres_ip, + port=self.postgres_port, + database="postgres", + user="postgres", + password="mysecretpassword", + ) self.postgres_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres_conn.autocommit = True logging.debug("Postgres Started") @@ -1205,7 +1749,13 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - self.postgres2_conn = psycopg2.connect(host=self.postgres2_ip, port=self.postgres_port, database='postgres', user='postgres', password='mysecretpassword') + self.postgres2_conn = psycopg2.connect( + host=self.postgres2_ip, + port=self.postgres_port, + database="postgres", + user="postgres", + password="mysecretpassword", + ) self.postgres2_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres2_conn.autocommit = True logging.debug("Postgres Cluster host 2 started") @@ -1215,7 +1765,13 @@ class ClickHouseCluster: time.sleep(0.5) while time.time() - start < timeout: try: - self.postgres3_conn = psycopg2.connect(host=self.postgres3_ip, port=self.postgres_port, database='postgres', user='postgres', password='mysecretpassword') + self.postgres3_conn = psycopg2.connect( + host=self.postgres3_ip, + port=self.postgres_port, + database="postgres", + user="postgres", + password="mysecretpassword", + ) self.postgres3_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres3_conn.autocommit = True logging.debug("Postgres Cluster host 3 started") @@ -1225,7 +1781,13 @@ class ClickHouseCluster: time.sleep(0.5) while time.time() - start < timeout: try: - self.postgres4_conn = psycopg2.connect(host=self.postgres4_ip, port=self.postgres_port, database='postgres', user='postgres', password='mysecretpassword') + self.postgres4_conn = psycopg2.connect( + host=self.postgres4_ip, + port=self.postgres_port, + database="postgres", + user="postgres", + password="mysecretpassword", + ) self.postgres4_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres4_conn.autocommit = True logging.debug("Postgres Cluster host 4 started") @@ -1261,10 +1823,15 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - self.exec_in_container(self.nginx_id, ["curl", "-X", "PUT", "-d", "Test", "http://test.com/test.txt"]) - res = self.exec_in_container(self.nginx_id, ["curl", "-X", "GET", "http://test.com/test.txt"]) - assert(res == 'Test') - print('nginx static files server is available') + self.exec_in_container( + self.nginx_id, + ["curl", "-X", "PUT", "-d", "Test", "http://test.com/test.txt"], + ) + res = self.exec_in_container( + self.nginx_id, ["curl", "-X", "GET", "http://test.com/test.txt"] + ) + assert res == "Test" + print("nginx static files server is available") return except Exception as ex: print("Can't connect to nginx: " + str(ex)) @@ -1275,9 +1842,9 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - for instance in ['zoo1', 'zoo2', 'zoo3']: + for instance in ["zoo1", "zoo2", "zoo3"]: conn = self.get_kazoo_client(instance) - conn.get_children('/') + conn.get_children("/") conn.stop() logging.debug("All instances of ZooKeeper Secure started") return @@ -1292,9 +1859,9 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - for instance in ['zoo1', 'zoo2', 'zoo3']: + for instance in ["zoo1", "zoo2", "zoo3"]: conn = self.get_kazoo_client(instance) - conn.get_children('/') + conn.get_children("/") conn.stop() logging.debug("All instances of ZooKeeper started") return @@ -1306,26 +1873,38 @@ class ClickHouseCluster: def make_hdfs_api(self, timeout=180, kerberized=False): if kerberized: - keytab = p.abspath(p.join(self.instances['node1'].path, "secrets/clickhouse.keytab")) - krb_conf = p.abspath(p.join(self.instances['node1'].path, "secrets/krb_long.conf")) + keytab = p.abspath( + p.join(self.instances["node1"].path, "secrets/clickhouse.keytab") + ) + krb_conf = p.abspath( + p.join(self.instances["node1"].path, "secrets/krb_long.conf") + ) self.hdfs_kerberized_ip = self.get_instance_ip(self.hdfs_kerberized_host) - kdc_ip = self.get_instance_ip('hdfskerberos') + kdc_ip = self.get_instance_ip("hdfskerberos") - self.hdfs_api = HDFSApi(user="root", - timeout=timeout, - kerberized=True, - principal="root@TEST.CLICKHOUSE.TECH", - keytab=keytab, - krb_conf=krb_conf, - host=self.hdfs_kerberized_host, - protocol="http", - proxy_port=self.hdfs_kerberized_name_port, - data_port=self.hdfs_kerberized_data_port, - hdfs_ip=self.hdfs_kerberized_ip, - kdc_ip=kdc_ip) + self.hdfs_api = HDFSApi( + user="root", + timeout=timeout, + kerberized=True, + principal="root@TEST.CLICKHOUSE.TECH", + keytab=keytab, + krb_conf=krb_conf, + host=self.hdfs_kerberized_host, + protocol="http", + proxy_port=self.hdfs_kerberized_name_port, + data_port=self.hdfs_kerberized_data_port, + hdfs_ip=self.hdfs_kerberized_ip, + kdc_ip=kdc_ip, + ) else: self.hdfs_ip = self.get_instance_ip(self.hdfs_host) - self.hdfs_api = HDFSApi(user="root", host=self.hdfs_host, data_port=self.hdfs_data_port, proxy_port=self.hdfs_name_port, hdfs_ip=self.hdfs_ip) + self.hdfs_api = HDFSApi( + user="root", + host=self.hdfs_host, + data_port=self.hdfs_data_port, + proxy_port=self.hdfs_name_port, + hdfs_ip=self.hdfs_ip, + ) def wait_kafka_is_available(self, kafka_docker_id, kafka_port, max_retries=50): retries = 0 @@ -1350,16 +1929,19 @@ class ClickHouseCluster: return except Exception as ex: - logging.exception("Can't connect to HDFS or preparations are not done yet " + str(ex)) + logging.exception( + "Can't connect to HDFS or preparations are not done yet " + str(ex) + ) time.sleep(1) raise Exception("Can't wait HDFS to start") def wait_mongo_to_start(self, timeout=30, secure=False): - connection_str = 'mongodb://{user}:{password}@{host}:{port}'.format( - host='localhost', port=self.mongo_port, user='root', password='clickhouse') + connection_str = "mongodb://{user}:{password}@{host}:{port}".format( + host="localhost", port=self.mongo_port, user="root", password="clickhouse" + ) if secure: - connection_str += '/?tls=true&tlsAllowInvalidCertificates=true' + connection_str += "/?tls=true&tlsAllowInvalidCertificates=true" connection = pymongo.MongoClient(connection_str) start = time.time() while time.time() - start < timeout: @@ -1375,13 +1957,16 @@ class ClickHouseCluster: self.minio_ip = self.get_instance_ip(self.minio_host) self.minio_redirect_ip = self.get_instance_ip(self.minio_redirect_host) - - os.environ['SSL_CERT_FILE'] = p.join(self.base_dir, self.minio_dir, 'certs', 'public.crt') - minio_client = Minio(f'{self.minio_ip}:{self.minio_port}', - access_key='minio', - secret_key='minio123', - secure=secure, - http_client=urllib3.PoolManager(cert_reqs='CERT_NONE')) # disable SSL check as we test ClickHouse and not Python library + os.environ["SSL_CERT_FILE"] = p.join( + self.base_dir, self.minio_dir, "certs", "public.crt" + ) + minio_client = Minio( + f"{self.minio_ip}:{self.minio_port}", + access_key="minio", + secret_key="minio123", + secure=secure, + http_client=urllib3.PoolManager(cert_reqs="CERT_NONE"), + ) # disable SSL check as we test ClickHouse and not Python library start = time.time() while time.time() - start < timeout: try: @@ -1414,12 +1999,15 @@ class ClickHouseCluster: def wait_azurite_to_start(self, timeout=180): from azure.storage.blob import BlobServiceClient + connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" time.sleep(1) start = time.time() while time.time() - start < timeout: try: - blob_service_client = BlobServiceClient.from_connection_string(connection_string) + blob_service_client = BlobServiceClient.from_connection_string( + connection_string + ) logging.debug(blob_service_client.get_account_information()) self.blob_service_client = blob_service_client return @@ -1430,7 +2018,9 @@ class ClickHouseCluster: raise Exception("Can't wait Azurite to start") def wait_schema_registry_to_start(self, timeout=180): - sr_client = CachedSchemaRegistryClient({"url":'http://localhost:{}'.format(self.schema_registry_port)}) + sr_client = CachedSchemaRegistryClient( + {"url": "http://localhost:{}".format(self.schema_registry_port)} + ) start = time.time() while time.time() - start < timeout: try: @@ -1445,12 +2035,26 @@ class ClickHouseCluster: def wait_cassandra_to_start(self, timeout=180): self.cassandra_ip = self.get_instance_ip(self.cassandra_host) - cass_client = cassandra.cluster.Cluster([self.cassandra_ip], port=self.cassandra_port, load_balancing_policy=RoundRobinPolicy()) + cass_client = cassandra.cluster.Cluster( + [self.cassandra_ip], + port=self.cassandra_port, + load_balancing_policy=RoundRobinPolicy(), + ) start = time.time() while time.time() - start < timeout: try: - logging.info(f"Check Cassandra Online {self.cassandra_id} {self.cassandra_ip} {self.cassandra_port}") - check = self.exec_in_container(self.cassandra_id, ["bash", "-c", f"/opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e 'describe keyspaces' {self.cassandra_ip} {self.cassandra_port}"], user='root') + logging.info( + f"Check Cassandra Online {self.cassandra_id} {self.cassandra_ip} {self.cassandra_port}" + ) + check = self.exec_in_container( + self.cassandra_id, + [ + "bash", + "-c", + f"/opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e 'describe keyspaces' {self.cassandra_ip} {self.cassandra_port}", + ], + user="root", + ) logging.info("Cassandra Online") cass_client.connect() logging.info("Connected Clients to Cassandra") @@ -1465,7 +2069,11 @@ class ClickHouseCluster: pytest_xdist_logging_to_separate_files.setup() logging.info("Running tests in {}".format(self.base_path)) - logging.debug("Cluster start called. is_up={}, destroy_dirs={}".format(self.is_up, destroy_dirs)) + logging.debug( + "Cluster start called. is_up={}, destroy_dirs={}".format( + self.is_up, destroy_dirs + ) + ) if self.is_up: return @@ -1484,15 +2092,25 @@ class ClickHouseCluster: shutil.rmtree(self.instances_dir) for instance in list(self.instances.values()): - logging.debug(('Setup directory for instance: {} destroy_dirs: {}'.format(instance.name, destroy_dirs))) + logging.debug( + ( + "Setup directory for instance: {} destroy_dirs: {}".format( + instance.name, destroy_dirs + ) + ) + ) instance.create_dir(destroy_dir=destroy_dirs) _create_env_file(os.path.join(self.env_file), self.env_variables) - self.docker_client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version=self.docker_api_version, timeout=600) + self.docker_client = docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=self.docker_api_version, + timeout=600, + ) - common_opts = ['--verbose', 'up', '-d'] + common_opts = ["--verbose", "up", "-d"] - images_pull_cmd = self.base_cmd + ['pull'] + images_pull_cmd = self.base_cmd + ["pull"] # sometimes dockerhub/proxy can be flaky for i in range(5): try: @@ -1505,9 +2123,11 @@ class ClickHouseCluster: time.sleep(i * 3) if self.with_zookeeper_secure and self.base_zookeeper_cmd: - logging.debug('Setup ZooKeeper Secure') - logging.debug(f'Creating internal ZooKeeper dirs: {self.zookeeper_dirs_to_create}') - for i in range(1,3): + logging.debug("Setup ZooKeeper Secure") + logging.debug( + f"Creating internal ZooKeeper dirs: {self.zookeeper_dirs_to_create}" + ) + for i in range(1, 3): if os.path.exists(self.zookeeper_instance_dir_prefix + f"{i}"): shutil.rmtree(self.zookeeper_instance_dir_prefix + f"{i}") for dir in self.zookeeper_dirs_to_create: @@ -1520,23 +2140,30 @@ class ClickHouseCluster: self.run_kazoo_commands_with_retries(command, repeats=5) if self.with_zookeeper and self.base_zookeeper_cmd: - logging.debug('Setup ZooKeeper') - logging.debug(f'Creating internal ZooKeeper dirs: {self.zookeeper_dirs_to_create}') + logging.debug("Setup ZooKeeper") + logging.debug( + f"Creating internal ZooKeeper dirs: {self.zookeeper_dirs_to_create}" + ) if self.use_keeper: - for i in range(1,4): + for i in range(1, 4): if os.path.exists(self.keeper_instance_dir_prefix + f"{i}"): shutil.rmtree(self.keeper_instance_dir_prefix + f"{i}") else: - for i in range(1,3): + for i in range(1, 3): if os.path.exists(self.zookeeper_instance_dir_prefix + f"{i}"): shutil.rmtree(self.zookeeper_instance_dir_prefix + f"{i}") for dir in self.zookeeper_dirs_to_create: os.makedirs(dir) - if self.use_keeper: # TODO: remove hardcoded paths from here - for i in range(1,4): - shutil.copy(os.path.join(HELPERS_DIR, f'keeper_config{i}.xml'), os.path.join(self.keeper_instance_dir_prefix + f"{i}", "config" )) + if self.use_keeper: # TODO: remove hardcoded paths from here + for i in range(1, 4): + shutil.copy( + os.path.join(HELPERS_DIR, f"keeper_config{i}.xml"), + os.path.join( + self.keeper_instance_dir_prefix + f"{i}", "config" + ), + ) run_and_check(self.base_zookeeper_cmd + common_opts, env=self.env) self.up_called = True @@ -1546,12 +2173,12 @@ class ClickHouseCluster: self.run_kazoo_commands_with_retries(command, repeats=5) if self.with_mysql_client and self.base_mysql_client_cmd: - logging.debug('Setup MySQL Client') + logging.debug("Setup MySQL Client") subprocess_check_call(self.base_mysql_client_cmd + common_opts) self.wait_mysql_client_to_start() if self.with_mysql and self.base_mysql_cmd: - logging.debug('Setup MySQL') + logging.debug("Setup MySQL") if os.path.exists(self.mysql_dir): shutil.rmtree(self.mysql_dir) os.makedirs(self.mysql_logs_dir) @@ -1561,7 +2188,7 @@ class ClickHouseCluster: self.wait_mysql_to_start() if self.with_mysql8 and self.base_mysql8_cmd: - logging.debug('Setup MySQL 8') + logging.debug("Setup MySQL 8") if os.path.exists(self.mysql8_dir): shutil.rmtree(self.mysql8_dir) os.makedirs(self.mysql8_logs_dir) @@ -1570,7 +2197,7 @@ class ClickHouseCluster: self.wait_mysql8_to_start() if self.with_mysql_cluster and self.base_mysql_cluster_cmd: - print('Setup MySQL') + print("Setup MySQL") if os.path.exists(self.mysql_cluster_dir): shutil.rmtree(self.mysql_cluster_dir) os.makedirs(self.mysql_cluster_logs_dir) @@ -1581,7 +2208,7 @@ class ClickHouseCluster: self.wait_mysql_cluster_to_start() if self.with_postgres and self.base_postgres_cmd: - logging.debug('Setup Postgres') + logging.debug("Setup Postgres") if os.path.exists(self.postgres_dir): shutil.rmtree(self.postgres_dir) os.makedirs(self.postgres_logs_dir) @@ -1592,7 +2219,7 @@ class ClickHouseCluster: self.wait_postgres_to_start() if self.with_postgres_cluster and self.base_postgres_cluster_cmd: - print('Setup Postgres') + print("Setup Postgres") os.makedirs(self.postgres2_logs_dir) os.chmod(self.postgres2_logs_dir, stat.S_IRWXU | stat.S_IRWXO) os.makedirs(self.postgres3_logs_dir) @@ -1604,33 +2231,43 @@ class ClickHouseCluster: self.wait_postgres_cluster_to_start() if self.with_kafka and self.base_kafka_cmd: - logging.debug('Setup Kafka') - subprocess_check_call(self.base_kafka_cmd + common_opts + ['--renew-anon-volumes']) + logging.debug("Setup Kafka") + subprocess_check_call( + self.base_kafka_cmd + common_opts + ["--renew-anon-volumes"] + ) self.up_called = True self.wait_kafka_is_available(self.kafka_docker_id, self.kafka_port) self.wait_schema_registry_to_start() if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: - logging.debug('Setup kerberized kafka') - run_and_check(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes']) + logging.debug("Setup kerberized kafka") + run_and_check( + self.base_kerberized_kafka_cmd + + common_opts + + ["--renew-anon-volumes"] + ) self.up_called = True - self.wait_kafka_is_available(self.kerberized_kafka_docker_id, self.kerberized_kafka_port, 100) + self.wait_kafka_is_available( + self.kerberized_kafka_docker_id, self.kerberized_kafka_port, 100 + ) if self.with_rabbitmq and self.base_rabbitmq_cmd: - logging.debug('Setup RabbitMQ') + logging.debug("Setup RabbitMQ") os.makedirs(self.rabbitmq_logs_dir) os.chmod(self.rabbitmq_logs_dir, stat.S_IRWXU | stat.S_IRWXO) for i in range(5): - subprocess_check_call(self.base_rabbitmq_cmd + common_opts + ['--renew-anon-volumes']) + subprocess_check_call( + self.base_rabbitmq_cmd + common_opts + ["--renew-anon-volumes"] + ) self.up_called = True - self.rabbitmq_docker_id = self.get_instance_docker_id('rabbitmq1') + self.rabbitmq_docker_id = self.get_instance_docker_id("rabbitmq1") logging.debug(f"RabbitMQ checking container try: {i}") - if self.wait_rabbitmq_to_start(throw=(i==4)): + if self.wait_rabbitmq_to_start(throw=(i == 4)): break if self.with_hdfs and self.base_hdfs_cmd: - logging.debug('Setup HDFS') + logging.debug("Setup HDFS") os.makedirs(self.hdfs_logs_dir) os.chmod(self.hdfs_logs_dir, stat.S_IRWXU | stat.S_IRWXO) subprocess_check_call(self.base_hdfs_cmd + common_opts) @@ -1639,7 +2276,7 @@ class ClickHouseCluster: self.wait_hdfs_to_start() if self.with_kerberized_hdfs and self.base_kerberized_hdfs_cmd: - logging.debug('Setup kerberized HDFS') + logging.debug("Setup kerberized HDFS") os.makedirs(self.hdfs_kerberized_logs_dir) os.chmod(self.hdfs_kerberized_logs_dir, stat.S_IRWXU | stat.S_IRWXO) run_and_check(self.base_kerberized_hdfs_cmd + common_opts) @@ -1648,26 +2285,28 @@ class ClickHouseCluster: self.wait_hdfs_to_start(check_marker=True) if self.with_nginx and self.base_nginx_cmd: - logging.debug('Setup nginx') - subprocess_check_call(self.base_nginx_cmd + common_opts + ['--renew-anon-volumes']) + logging.debug("Setup nginx") + subprocess_check_call( + self.base_nginx_cmd + common_opts + ["--renew-anon-volumes"] + ) self.up_called = True - self.nginx_docker_id = self.get_instance_docker_id('nginx') + self.nginx_docker_id = self.get_instance_docker_id("nginx") self.wait_nginx_to_start() if self.with_mongo and self.base_mongo_cmd: - logging.debug('Setup Mongo') + logging.debug("Setup Mongo") run_and_check(self.base_mongo_cmd + common_opts) self.up_called = True self.wait_mongo_to_start(30, secure=self.with_mongo_secure) if self.with_redis and self.base_redis_cmd: - logging.debug('Setup Redis') + logging.debug("Setup Redis") subprocess_check_call(self.base_redis_cmd + common_opts) self.up_called = True time.sleep(10) if self.with_hive and self.base_hive_cmd: - logging.debug('Setup hive') + logging.debug("Setup hive") subprocess_check_call(self.base_hive_cmd + common_opts) self.up_called = True time.sleep(300) @@ -1676,13 +2315,19 @@ class ClickHouseCluster: # Copy minio certificates to minio/certs os.mkdir(self.minio_dir) if self.minio_certs_dir is None: - os.mkdir(os.path.join(self.minio_dir, 'certs')) + os.mkdir(os.path.join(self.minio_dir, "certs")) else: - shutil.copytree(os.path.join(self.base_dir, self.minio_certs_dir), os.path.join(self.minio_dir, 'certs')) + shutil.copytree( + os.path.join(self.base_dir, self.minio_certs_dir), + os.path.join(self.minio_dir, "certs"), + ) minio_start_cmd = self.base_minio_cmd + common_opts - logging.info("Trying to create Minio instance by command %s", ' '.join(map(str, minio_start_cmd))) + logging.info( + "Trying to create Minio instance by command %s", + " ".join(map(str, minio_start_cmd)), + ) run_and_check(minio_start_cmd) self.up_called = True logging.info("Trying to connect to Minio...") @@ -1690,14 +2335,17 @@ class ClickHouseCluster: if self.with_azurite and self.base_azurite_cmd: azurite_start_cmd = self.base_azurite_cmd + common_opts - logging.info("Trying to create Azurite instance by command %s", ' '.join(map(str, azurite_start_cmd))) + logging.info( + "Trying to create Azurite instance by command %s", + " ".join(map(str, azurite_start_cmd)), + ) run_and_check(azurite_start_cmd) self.up_called = True logging.info("Trying to connect to Azurite") self.wait_azurite_to_start() if self.with_cassandra and self.base_cassandra_cmd: - subprocess_check_call(self.base_cassandra_cmd + ['up', '-d']) + subprocess_check_call(self.base_cassandra_cmd + ["up", "-d"]) self.up_called = True self.wait_cassandra_to_start() @@ -1705,13 +2353,20 @@ class ClickHouseCluster: os.makedirs(self.jdbc_driver_logs_dir) os.chmod(self.jdbc_driver_logs_dir, stat.S_IRWXU | stat.S_IRWXO) - subprocess_check_call(self.base_jdbc_bridge_cmd + ['up', '-d']) + subprocess_check_call(self.base_jdbc_bridge_cmd + ["up", "-d"]) self.up_called = True self.jdbc_bridge_ip = self.get_instance_ip(self.jdbc_bridge_host) - self.wait_for_url(f"http://{self.jdbc_bridge_ip}:{self.jdbc_bridge_port}/ping") + self.wait_for_url( + f"http://{self.jdbc_bridge_ip}:{self.jdbc_bridge_port}/ping" + ) - clickhouse_start_cmd = self.base_cmd + ['up', '-d', '--no-recreate'] - logging.debug(("Trying to create ClickHouse instance by command %s", ' '.join(map(str, clickhouse_start_cmd)))) + clickhouse_start_cmd = self.base_cmd + ["up", "-d", "--no-recreate"] + logging.debug( + ( + "Trying to create ClickHouse instance by command %s", + " ".join(map(str, clickhouse_start_cmd)), + ) + ) self.up_called = True run_and_check(clickhouse_start_cmd) logging.debug("ClickHouse instance created") @@ -1721,11 +2376,15 @@ class ClickHouseCluster: instance.docker_client = self.docker_client instance.ip_address = self.get_instance_ip(instance.name) - logging.debug(f"Waiting for ClickHouse start in {instance.name}, ip: {instance.ip_address}...") + logging.debug( + f"Waiting for ClickHouse start in {instance.name}, ip: {instance.ip_address}..." + ) instance.wait_for_start(start_timeout) logging.debug(f"ClickHouse {instance.name} started") - instance.client = Client(instance.ip_address, command=self.client_bin_path) + instance.client = Client( + instance.ip_address, command=self.client_bin_path + ) self.is_up = True @@ -1743,43 +2402,59 @@ class ClickHouseCluster: if self.up_called: with open(self.docker_logs_path, "w+") as f: try: - subprocess.check_call(self.base_cmd + ['logs'], stdout=f) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + self.base_cmd + ["logs"], stdout=f + ) except Exception as e: logging.debug("Unable to get logs from docker.") f.seek(0) for line in f: if SANITIZER_SIGN in line: - sanitizer_assert_instance = line.split('|')[0].strip() + sanitizer_assert_instance = line.split("|")[0].strip() break if kill: try: - run_and_check(self.base_cmd + ['stop', '--timeout', '20']) + run_and_check(self.base_cmd + ["stop", "--timeout", "20"]) except Exception as e: - logging.debug("Kill command failed during shutdown. {}".format(repr(e))) + logging.debug( + "Kill command failed during shutdown. {}".format(repr(e)) + ) logging.debug("Trying to kill forcefully") - run_and_check(self.base_cmd + ['kill']) + run_and_check(self.base_cmd + ["kill"]) # Check server logs for Fatal messages and sanitizer failures. # NOTE: we cannot do this via docker since in case of Fatal message container may already die. for name, instance in self.instances.items(): if instance.contains_in_log(SANITIZER_SIGN, from_host=True): - sanitizer_assert_instance = instance.grep_in_log(SANITIZER_SIGN, from_host=True, filename='stderr.log') - logging.error("Sanitizer in instance %s log %s", name, sanitizer_assert_instance) + sanitizer_assert_instance = instance.grep_in_log( + SANITIZER_SIGN, from_host=True, filename="stderr.log" + ) + logging.error( + "Sanitizer in instance %s log %s", + name, + sanitizer_assert_instance, + ) - if not ignore_fatal and instance.contains_in_log("Fatal", from_host=True): + if not ignore_fatal and instance.contains_in_log( + "Fatal", from_host=True + ): fatal_log = instance.grep_in_log("Fatal", from_host=True) - if 'Child process was terminated by signal 9 (KILL)' in fatal_log: + if "Child process was terminated by signal 9 (KILL)" in fatal_log: fatal_log = None continue logging.error("Crash in instance %s fatal log %s", name, fatal_log) try: - subprocess_check_call(self.base_cmd + ['down', '--volumes']) + subprocess_check_call(self.base_cmd + ["down", "--volumes"]) except Exception as e: - logging.debug("Down + remove orphans failed during shutdown. {}".format(repr(e))) + logging.debug( + "Down + remove orphans failed during shutdown. {}".format(repr(e)) + ) else: - logging.warning("docker-compose up was not called. Trying to export docker.log for running containers") + logging.warning( + "docker-compose up was not called. Trying to export docker.log for running containers" + ) self.cleanup() @@ -1794,23 +2469,25 @@ class ClickHouseCluster: if sanitizer_assert_instance is not None: raise Exception( - "Sanitizer assert found in {} for instance {}".format(self.docker_logs_path, sanitizer_assert_instance)) + "Sanitizer assert found in {} for instance {}".format( + self.docker_logs_path, sanitizer_assert_instance + ) + ) if fatal_log is not None: raise Exception("Fatal messages found: {}".format(fatal_log)) - def pause_container(self, instance_name): - subprocess_check_call(self.base_cmd + ['pause', instance_name]) + subprocess_check_call(self.base_cmd + ["pause", instance_name]) # subprocess_check_call(self.base_cmd + ['kill', '-s SIGSTOP', instance_name]) def unpause_container(self, instance_name): - subprocess_check_call(self.base_cmd + ['unpause', instance_name]) + subprocess_check_call(self.base_cmd + ["unpause", instance_name]) # subprocess_check_call(self.base_cmd + ['kill', '-s SIGCONT', instance_name]) def open_bash_shell(self, instance_name): - os.system(' '.join(self.base_cmd + ['exec', instance_name, '/bin/bash'])) + os.system(" ".join(self.base_cmd + ["exec", instance_name, "/bin/bash"])) def get_kazoo_client(self, zoo_instance_name): use_ssl = False @@ -1823,15 +2500,26 @@ class ClickHouseCluster: raise Exception("Cluster has no ZooKeeper") ip = self.get_instance_ip(zoo_instance_name) - logging.debug(f"get_kazoo_client: {zoo_instance_name}, ip:{ip}, port:{port}, use_ssl:{use_ssl}") - zk = KazooClient(hosts=f"{ip}:{port}", use_ssl=use_ssl, verify_certs=False, certfile=self.zookeeper_certfile, - keyfile=self.zookeeper_keyfile) + logging.debug( + f"get_kazoo_client: {zoo_instance_name}, ip:{ip}, port:{port}, use_ssl:{use_ssl}" + ) + zk = KazooClient( + hosts=f"{ip}:{port}", + use_ssl=use_ssl, + verify_certs=False, + certfile=self.zookeeper_certfile, + keyfile=self.zookeeper_keyfile, + ) zk.start() return zk - def run_kazoo_commands_with_retries(self, kazoo_callback, zoo_instance_name='zoo1', repeats=1, sleep_for=1): + def run_kazoo_commands_with_retries( + self, kazoo_callback, zoo_instance_name="zoo1", repeats=1, sleep_for=1 + ): zk = self.get_kazoo_client(zoo_instance_name) - logging.debug(f"run_kazoo_commands_with_retries: {zoo_instance_name}, {kazoo_callback}") + logging.debug( + f"run_kazoo_commands_with_retries: {zoo_instance_name}, {kazoo_callback}" + ) for i in range(repeats - 1): try: kazoo_callback(zk) @@ -1856,14 +2544,18 @@ class ClickHouseCluster: subprocess_check_call(self.base_zookeeper_cmd + ["start", n]) -CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/{main_config_file}" \ - " --log-file=/var/log/clickhouse-server/clickhouse-server.log " \ - " --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" +CLICKHOUSE_START_COMMAND = ( + "clickhouse server --config-file=/etc/clickhouse-server/{main_config_file}" + " --log-file=/var/log/clickhouse-server/clickhouse-server.log " + " --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" +) -CLICKHOUSE_STAY_ALIVE_COMMAND = 'bash -c "trap \'pkill tail\' INT TERM; {} --daemon; coproc tail -f /dev/null; wait $$!"'.format(CLICKHOUSE_START_COMMAND) +CLICKHOUSE_STAY_ALIVE_COMMAND = "bash -c \"trap 'pkill tail' INT TERM; {} --daemon; coproc tail -f /dev/null; wait $$!\"".format( + CLICKHOUSE_START_COMMAND +) # /run/xtables.lock passed inside for correct iptables --wait -DOCKER_COMPOSE_TEMPLATE = ''' +DOCKER_COMPOSE_TEMPLATE = """ version: '2.3' services: {name}: @@ -1906,22 +2598,62 @@ services: {ipv6_address} {net_aliases} {net_alias1} -''' +""" class ClickHouseInstance: - def __init__( - self, cluster, base_path, name, base_config_dir, custom_main_configs, custom_user_configs, - custom_dictionaries, - macros, with_zookeeper, zookeeper_config_path, with_mysql_client, with_mysql, with_mysql8, with_mysql_cluster, with_kafka, with_kerberized_kafka, - with_rabbitmq, with_nginx, with_kerberized_hdfs, with_mongo, with_redis, with_minio, with_azurite, with_jdbc_bridge, with_hive, - with_cassandra, server_bin_path, odbc_bridge_bin_path, library_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, with_postgres, with_postgres_cluster, - clickhouse_start_command=CLICKHOUSE_START_COMMAND, - main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True, - hostname=None, env_variables=None, - image="clickhouse/integration-test", tag="latest", - stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, external_dirs=None, tmpfs=None, config_root_name="clickhouse", extra_configs=[]): + self, + cluster, + base_path, + name, + base_config_dir, + custom_main_configs, + custom_user_configs, + custom_dictionaries, + macros, + with_zookeeper, + zookeeper_config_path, + with_mysql_client, + with_mysql, + with_mysql8, + with_mysql_cluster, + with_kafka, + with_kerberized_kafka, + with_rabbitmq, + with_nginx, + with_kerberized_hdfs, + with_mongo, + with_redis, + with_minio, + with_azurite, + with_jdbc_bridge, + with_hive, + with_cassandra, + server_bin_path, + odbc_bridge_bin_path, + library_bridge_bin_path, + clickhouse_path_dir, + with_odbc_drivers, + with_postgres, + with_postgres_cluster, + clickhouse_start_command=CLICKHOUSE_START_COMMAND, + main_config_name="config.xml", + users_config_name="users.xml", + copy_common_configs=True, + hostname=None, + env_variables=None, + image="clickhouse/integration-test", + tag="latest", + stay_alive=False, + ipv4_address=None, + ipv6_address=None, + with_installed_binary=False, + external_dirs=None, + tmpfs=None, + config_root_name="clickhouse", + extra_configs=[], + ): self.name = name self.base_cmd = cluster.base_cmd @@ -1931,13 +2663,27 @@ class ClickHouseInstance: self.external_dirs = external_dirs self.tmpfs = tmpfs or [] - self.base_config_dir = p.abspath(p.join(base_path, base_config_dir)) if base_config_dir else None - self.custom_main_config_paths = [p.abspath(p.join(base_path, c)) for c in custom_main_configs] - self.custom_user_config_paths = [p.abspath(p.join(base_path, c)) for c in custom_user_configs] - self.custom_dictionaries_paths = [p.abspath(p.join(base_path, c)) for c in custom_dictionaries] - self.custom_extra_config_paths = [p.abspath(p.join(base_path,c)) for c in extra_configs] - self.clickhouse_path_dir = p.abspath(p.join(base_path, clickhouse_path_dir)) if clickhouse_path_dir else None - self.kerberos_secrets_dir = p.abspath(p.join(base_path, 'secrets')) + self.base_config_dir = ( + p.abspath(p.join(base_path, base_config_dir)) if base_config_dir else None + ) + self.custom_main_config_paths = [ + p.abspath(p.join(base_path, c)) for c in custom_main_configs + ] + self.custom_user_config_paths = [ + p.abspath(p.join(base_path, c)) for c in custom_user_configs + ] + self.custom_dictionaries_paths = [ + p.abspath(p.join(base_path, c)) for c in custom_dictionaries + ] + self.custom_extra_config_paths = [ + p.abspath(p.join(base_path, c)) for c in extra_configs + ] + self.clickhouse_path_dir = ( + p.abspath(p.join(base_path, clickhouse_path_dir)) + if clickhouse_path_dir + else None + ) + self.kerberos_secrets_dir = p.abspath(p.join(base_path, "secrets")) self.macros = macros if macros is not None else {} self.with_zookeeper = with_zookeeper self.zookeeper_config_path = zookeeper_config_path @@ -1969,10 +2715,12 @@ class ClickHouseInstance: self.users_config_name = users_config_name self.copy_common_configs = copy_common_configs - self.clickhouse_start_command = clickhouse_start_command.replace("{main_config_file}", self.main_config_name) + self.clickhouse_start_command = clickhouse_start_command.replace( + "{main_config_file}", self.main_config_name + ) self.path = p.join(self.cluster.instances_dir, name) - self.docker_compose_path = p.join(self.path, 'docker-compose.yml') + self.docker_compose_path = p.join(self.path, "docker-compose.yml") self.env_variables = env_variables or {} self.env_file = self.cluster.env_file if with_odbc_drivers: @@ -1982,8 +2730,16 @@ class ClickHouseInstance: self.odbc_ini_path = "" if with_kerberized_kafka or with_kerberized_hdfs: - self.keytab_path = '- ' + os.path.dirname(self.docker_compose_path) + "/secrets:/tmp/keytab" - self.krb5_conf = '- ' + os.path.dirname(self.docker_compose_path) + "/secrets/krb.conf:/etc/krb5.conf:ro" + self.keytab_path = ( + "- " + + os.path.dirname(self.docker_compose_path) + + "/secrets:/tmp/keytab" + ) + self.krb5_conf = ( + "- " + + os.path.dirname(self.docker_compose_path) + + "/secrets/krb.conf:/etc/krb5.conf:ro" + ) else: self.keytab_path = "" self.krb5_conf = "" @@ -2000,54 +2756,81 @@ class ClickHouseInstance: self.is_up = False self.config_root_name = config_root_name - - def is_built_with_sanitizer(self, sanitizer_name=''): - build_opts = self.query("SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'") + def is_built_with_sanitizer(self, sanitizer_name=""): + build_opts = self.query( + "SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'" + ) return "-fsanitize={}".format(sanitizer_name) in build_opts def is_debug_build(self): - build_opts = self.query("SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'") - return 'NDEBUG' not in build_opts + build_opts = self.query( + "SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'" + ) + return "NDEBUG" not in build_opts def is_built_with_thread_sanitizer(self): - return self.is_built_with_sanitizer('thread') + return self.is_built_with_sanitizer("thread") def is_built_with_address_sanitizer(self): - return self.is_built_with_sanitizer('address') + return self.is_built_with_sanitizer("address") def is_built_with_memory_sanitizer(self): - return self.is_built_with_sanitizer('memory') + return self.is_built_with_sanitizer("memory") # Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer - def query(self, sql, - stdin=None, - timeout=None, - settings=None, - user=None, - password=None, - database=None, - ignore_error=False, - query_id=None): + def query( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ignore_error=False, + query_id=None, + ): logging.debug("Executing query %s on %s", sql, self.name) - return self.client.query(sql, - stdin=stdin, - timeout=timeout, - settings=settings, - user=user, - password=password, - database=database, - ignore_error=ignore_error, - query_id=query_id) + return self.client.query( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ignore_error=ignore_error, + query_id=query_id, + ) - def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, - ignore_error=False, - retry_count=20, sleep_time=0.5, check_callback=lambda x: True): + def query_with_retry( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ignore_error=False, + retry_count=20, + sleep_time=0.5, + check_callback=lambda x: True, + ): logging.debug(f"Executing query {sql} on {self.name}") result = None for i in range(retry_count): try: - result = self.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, - database=database, ignore_error=ignore_error) + result = self.query( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ignore_error=ignore_error, + ) if check_callback(result): return result time.sleep(sleep_time) @@ -2065,22 +2848,62 @@ class ClickHouseInstance: return self.client.get_query_request(sql, *args, **kwargs) # Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code - def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, - database=None): + def query_and_get_error( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ): logging.debug(f"Executing query {sql} on {self.name}") - return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, - password=password, database=database) + return self.client.query_and_get_error( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ) # The same as query_and_get_error but ignores successful query. - def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, - database=None): + def query_and_get_answer_with_error( + self, + sql, + stdin=None, + timeout=None, + settings=None, + user=None, + password=None, + database=None, + ): logging.debug(f"Executing query {sql} on {self.name}") - return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings, - user=user, password=password, database=database) + return self.client.query_and_get_answer_with_error( + sql, + stdin=stdin, + timeout=timeout, + settings=settings, + user=user, + password=password, + database=database, + ) # Connects to the instance via HTTP interface, sends a query and returns the answer - def http_query(self, sql, data=None, params=None, user=None, password=None, expect_fail_and_get_error=False, - port=8123, timeout=None, retry_strategy=None): + def http_query( + self, + sql, + data=None, + params=None, + user=None, + password=None, + expect_fail_and_get_error=False, + port=8123, + timeout=None, + retry_strategy=None, + ): logging.debug(f"Executing query {sql} on {self.name} via HTTP interface") if params is None: params = {} @@ -2093,7 +2916,7 @@ class ClickHouseInstance: if user and password: auth = requests.auth.HTTPBasicAuth(user, password) elif user: - auth = requests.auth.HTTPBasicAuth(user, '') + auth = requests.auth.HTTPBasicAuth(user, "") url = f"http://{self.ip_address}:{port}/?" + urllib.parse.urlencode(params) if retry_strategy is None: @@ -2114,35 +2937,57 @@ class ClickHouseInstance: if expect_fail_and_get_error: if r.ok: - raise Exception("ClickHouse HTTP server is expected to fail, but succeeded: " + r.text) + raise Exception( + "ClickHouse HTTP server is expected to fail, but succeeded: " + + r.text + ) return http_code_and_message() else: if not r.ok: - raise Exception("ClickHouse HTTP server returned " + http_code_and_message()) + raise Exception( + "ClickHouse HTTP server returned " + http_code_and_message() + ) return r.text # Connects to the instance via HTTP interface, sends a query and returns the answer - def http_request(self, url, method='GET', params=None, data=None, headers=None): + def http_request(self, url, method="GET", params=None, data=None, headers=None): logging.debug(f"Sending HTTP request {url} to {self.name}") url = "http://" + self.ip_address + ":8123/" + url - return requests.request(method=method, url=url, params=params, data=data, headers=headers) + return requests.request( + method=method, url=url, params=params, data=data, headers=headers + ) # Connects to the instance via HTTP interface, sends a query, expects an error and return the error message - def http_query_and_get_error(self, sql, data=None, params=None, user=None, password=None): + def http_query_and_get_error( + self, sql, data=None, params=None, user=None, password=None + ): logging.debug(f"Executing query {sql} on {self.name} via HTTP interface") - return self.http_query(sql=sql, data=data, params=params, user=user, password=password, - expect_fail_and_get_error=True) + return self.http_query( + sql=sql, + data=data, + params=params, + user=user, + password=password, + expect_fail_and_get_error=True, + ) def stop_clickhouse(self, stop_wait_sec=30, kill=False): if not self.stay_alive: - raise Exception("clickhouse can be stopped only with stay_alive=True instance") + raise Exception( + "clickhouse can be stopped only with stay_alive=True instance" + ) try: - ps_clickhouse = self.exec_in_container(["bash", "-c", "ps -C clickhouse"], nothrow=True, user='root') - if ps_clickhouse == " PID TTY STAT TIME COMMAND" : + ps_clickhouse = self.exec_in_container( + ["bash", "-c", "ps -C clickhouse"], nothrow=True, user="root" + ) + if ps_clickhouse == " PID TTY STAT TIME COMMAND": logging.warning("ClickHouse process already stopped") return - self.exec_in_container(["bash", "-c", "pkill {} clickhouse".format("-9" if kill else "")], user='root') + self.exec_in_container( + ["bash", "-c", "pkill {} clickhouse".format("-9" if kill else "")], + user="root", + ) start_time = time.time() stopped = False @@ -2157,19 +3002,34 @@ class ClickHouseInstance: if not stopped: pid = self.get_process_pid("clickhouse") if pid is not None: - logging.warning(f"Force kill clickhouse in stop_clickhouse. ps:{pid}") - self.exec_in_container(["bash", "-c", f"gdb -batch -ex 'thread apply all bt full' -p {pid} > {os.path.join(self.path, 'logs/stdout.log')}"], user='root') + logging.warning( + f"Force kill clickhouse in stop_clickhouse. ps:{pid}" + ) + self.exec_in_container( + [ + "bash", + "-c", + f"gdb -batch -ex 'thread apply all bt full' -p {pid} > {os.path.join(self.path, 'logs/stdout.log')}", + ], + user="root", + ) self.stop_clickhouse(kill=True) else: - ps_all = self.exec_in_container(["bash", "-c", "ps aux"], nothrow=True, user='root') - logging.warning(f"We want force stop clickhouse, but no clickhouse-server is running\n{ps_all}") + ps_all = self.exec_in_container( + ["bash", "-c", "ps aux"], nothrow=True, user="root" + ) + logging.warning( + f"We want force stop clickhouse, but no clickhouse-server is running\n{ps_all}" + ) return except Exception as e: logging.warning(f"Stop ClickHouse raised an error {e}") def start_clickhouse(self, start_wait_sec=60): if not self.stay_alive: - raise Exception("ClickHouse can be started again only with stay_alive=True instance") + raise Exception( + "ClickHouse can be started again only with stay_alive=True instance" + ) start_time = time.time() time_to_sleep = 0.5 @@ -2179,7 +3039,10 @@ class ClickHouseInstance: pid = self.get_process_pid("clickhouse") if pid is None: logging.debug("No clickhouse process running. Start new one.") - self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) + self.exec_in_container( + ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], + user=str(os.getuid()), + ) time.sleep(1) continue else: @@ -2188,13 +3051,16 @@ class ClickHouseInstance: self.wait_start(start_wait_sec + start_time - time.time()) return except Exception as e: - logging.warning(f"Current start attempt failed. Will kill {pid} just in case.") - self.exec_in_container(["bash", "-c", f"kill -9 {pid}"], user='root', nothrow=True) + logging.warning( + f"Current start attempt failed. Will kill {pid} just in case." + ) + self.exec_in_container( + ["bash", "-c", f"kill -9 {pid}"], user="root", nothrow=True + ) time.sleep(time_to_sleep) raise Exception("Cannot start ClickHouse, see additional info in logs") - def wait_start(self, start_wait_sec): start_time = time.time() last_err = None @@ -2203,7 +3069,7 @@ class ClickHouseInstance: pid = self.get_process_pid("clickhouse") if pid is None: raise Exception("ClickHouse server is not running. Check logs.") - exec_query_with_retry(self, 'select 20', retry_count = 10, silent=True) + exec_query_with_retry(self, "select 20", retry_count=10, silent=True) return except QueryRuntimeException as err: last_err = err @@ -2214,12 +3080,19 @@ class ClickHouseInstance: raise Exception("ClickHouse server is not running. Check logs.") if time.time() > start_time + start_wait_sec: break - logging.error(f"No time left to start. But process is still running. Will dump threads.") - ps_clickhouse = self.exec_in_container(["bash", "-c", "ps -C clickhouse"], nothrow=True, user='root') + logging.error( + f"No time left to start. But process is still running. Will dump threads." + ) + ps_clickhouse = self.exec_in_container( + ["bash", "-c", "ps -C clickhouse"], nothrow=True, user="root" + ) logging.info(f"PS RESULT:\n{ps_clickhouse}") pid = self.get_process_pid("clickhouse") if pid is not None: - self.exec_in_container(["bash", "-c", f"gdb -batch -ex 'thread apply all bt full' -p {pid}"], user='root') + self.exec_in_container( + ["bash", "-c", f"gdb -batch -ex 'thread apply all bt full' -p {pid}"], + user="root", + ) if last_err is not None: raise last_err @@ -2228,82 +3101,164 @@ class ClickHouseInstance: self.start_clickhouse(stop_start_wait_sec) def exec_in_container(self, cmd, detach=False, nothrow=False, **kwargs): - return self.cluster.exec_in_container(self.docker_id, cmd, detach, nothrow, **kwargs) + return self.cluster.exec_in_container( + self.docker_id, cmd, detach, nothrow, **kwargs + ) def rotate_logs(self): - self.exec_in_container(["bash", "-c", f"kill -HUP {self.get_process_pid('clickhouse server')}"], user='root') + self.exec_in_container( + ["bash", "-c", f"kill -HUP {self.get_process_pid('clickhouse server')}"], + user="root", + ) - def contains_in_log(self, substring, from_host=False, filename='clickhouse-server.log'): + def contains_in_log( + self, substring, from_host=False, filename="clickhouse-server.log" + ): if from_host: # We check fist file exists but want to look for all rotated logs as well - result = subprocess_check_call(["bash", "-c", - f'[ -f {self.logs_dir}/{filename} ] && zgrep -aH "{substring}" {self.logs_dir}/{filename}* || true' - ]) + result = subprocess_check_call( + [ + "bash", + "-c", + f'[ -f {self.logs_dir}/{filename} ] && zgrep -aH "{substring}" {self.logs_dir}/{filename}* || true', + ] + ) else: - result = self.exec_in_container(["bash", "-c", - f'[ -f /var/log/clickhouse-server/{filename} ] && zgrep -aH "{substring}" /var/log/clickhouse-server/{filename} || true' - ]) + result = self.exec_in_container( + [ + "bash", + "-c", + f'[ -f /var/log/clickhouse-server/{filename} ] && zgrep -aH "{substring}" /var/log/clickhouse-server/{filename} || true', + ] + ) return len(result) > 0 - def grep_in_log(self, substring, from_host=False, filename='clickhouse-server.log'): + def grep_in_log(self, substring, from_host=False, filename="clickhouse-server.log"): logging.debug(f"grep in log called %s", substring) if from_host: # We check fist file exists but want to look for all rotated logs as well - result = subprocess_check_call(["bash", "-c", - f'[ -f {self.logs_dir}/{filename} ] && zgrep -a "{substring}" {self.logs_dir}/{filename}* || true' - ]) + result = subprocess_check_call( + [ + "bash", + "-c", + f'[ -f {self.logs_dir}/{filename} ] && zgrep -a "{substring}" {self.logs_dir}/{filename}* || true', + ] + ) else: - result = self.exec_in_container(["bash", "-c", - f'[ -f /var/log/clickhouse-server/{filename} ] && zgrep -a "{substring}" /var/log/clickhouse-server/{filename}* || true' - ]) + result = self.exec_in_container( + [ + "bash", + "-c", + f'[ -f /var/log/clickhouse-server/{filename} ] && zgrep -a "{substring}" /var/log/clickhouse-server/{filename}* || true', + ] + ) logging.debug("grep result %s", result) return result def count_in_log(self, substring): result = self.exec_in_container( - ["bash", "-c", 'grep -a "{}" /var/log/clickhouse-server/clickhouse-server.log | wc -l'.format(substring)]) + [ + "bash", + "-c", + 'grep -a "{}" /var/log/clickhouse-server/clickhouse-server.log | wc -l'.format( + substring + ), + ] + ) return result - def wait_for_log_line(self, regexp, filename='/var/log/clickhouse-server/clickhouse-server.log', timeout=30, repetitions=1, look_behind_lines=100): + def wait_for_log_line( + self, + regexp, + filename="/var/log/clickhouse-server/clickhouse-server.log", + timeout=30, + repetitions=1, + look_behind_lines=100, + ): start_time = time.time() result = self.exec_in_container( - ["bash", "-c", 'timeout {} tail -Fn{} "{}" | grep -Em {} {}'.format(timeout, look_behind_lines, filename, repetitions, shlex.quote(regexp))]) + [ + "bash", + "-c", + 'timeout {} tail -Fn{} "{}" | grep -Em {} {}'.format( + timeout, + look_behind_lines, + filename, + repetitions, + shlex.quote(regexp), + ), + ] + ) # if repetitions>1 grep will return success even if not enough lines were collected, - if repetitions>1 and len(result.splitlines()) < repetitions: - logging.debug("wait_for_log_line: those lines were found during {} seconds:".format(timeout)) + if repetitions > 1 and len(result.splitlines()) < repetitions: + logging.debug( + "wait_for_log_line: those lines were found during {} seconds:".format( + timeout + ) + ) logging.debug(result) - raise Exception("wait_for_log_line: Not enough repetitions: {} found, while {} expected".format(len(result.splitlines()), repetitions)) + raise Exception( + "wait_for_log_line: Not enough repetitions: {} found, while {} expected".format( + len(result.splitlines()), repetitions + ) + ) wait_duration = time.time() - start_time - logging.debug('{} log line(s) matching "{}" appeared in a {:.3f} seconds'.format(repetitions, regexp, wait_duration)) + logging.debug( + '{} log line(s) matching "{}" appeared in a {:.3f} seconds'.format( + repetitions, regexp, wait_duration + ) + ) return wait_duration def path_exists(self, path): - return self.exec_in_container( - ["bash", "-c", "echo $(if [ -e '{}' ]; then echo 'yes'; else echo 'no'; fi)".format(path)]) == 'yes\n' + return ( + self.exec_in_container( + [ + "bash", + "-c", + "echo $(if [ -e '{}' ]; then echo 'yes'; else echo 'no'; fi)".format( + path + ), + ] + ) + == "yes\n" + ) def copy_file_to_container(self, local_path, dest_path): - return self.cluster.copy_file_to_container(self.docker_id, local_path, dest_path) + return self.cluster.copy_file_to_container( + self.docker_id, local_path, dest_path + ) def get_process_pid(self, process_name): - output = self.exec_in_container(["bash", "-c", - "ps ax | grep '{}' | grep -v 'grep' | grep -v 'coproc' | grep -v 'bash -c' | awk '{{print $1}}'".format( - process_name)]) + output = self.exec_in_container( + [ + "bash", + "-c", + "ps ax | grep '{}' | grep -v 'grep' | grep -v 'coproc' | grep -v 'bash -c' | awk '{{print $1}}'".format( + process_name + ), + ] + ) if output: try: - pid = int(output.split('\n')[0].strip()) + pid = int(output.split("\n")[0].strip()) return pid except: return None return None - def restart_with_original_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=15): + def restart_with_original_version( + self, stop_start_wait_sec=300, callback_onstop=None, signal=15 + ): begin_time = time.time() if not self.stay_alive: raise Exception("Cannot restart not stay alive container") - self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root') + self.exec_in_container( + ["bash", "-c", "pkill -{} clickhouse".format(signal)], user="root" + ) retries = int(stop_start_wait_sec / 0.5) local_counter = 0 # wait stop @@ -2316,18 +3271,41 @@ class ClickHouseInstance: # force kill if server hangs if self.get_process_pid("clickhouse server"): # server can die before kill, so don't throw exception, it's expected - self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(9)], nothrow=True, user='root') + self.exec_in_container( + ["bash", "-c", "pkill -{} clickhouse".format(9)], + nothrow=True, + user="root", + ) if callback_onstop: callback_onstop(self) - self.exec_in_container(["bash", "-c", "echo 'restart_with_original_version: From version' && /usr/bin/clickhouse server --version && echo 'To version' && /usr/share/clickhouse_original server --version"]) self.exec_in_container( - ["bash", "-c", "cp /usr/share/clickhouse_original /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse"], - user='root') - self.exec_in_container(["bash", "-c", - "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"], - user='root') - self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) + [ + "bash", + "-c", + "echo 'restart_with_original_version: From version' && /usr/bin/clickhouse server --version && echo 'To version' && /usr/share/clickhouse_original server --version", + ] + ) + self.exec_in_container( + [ + "bash", + "-c", + "cp /usr/share/clickhouse_original /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse", + ], + user="root", + ) + self.exec_in_container( + [ + "bash", + "-c", + "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse", + ], + user="root", + ) + self.exec_in_container( + ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], + user=str(os.getuid()), + ) # wait start time_left = begin_time + stop_start_wait_sec - time.time() @@ -2336,11 +3314,15 @@ class ClickHouseInstance: else: self.wait_start(time_left) - def restart_with_latest_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=15): + def restart_with_latest_version( + self, stop_start_wait_sec=300, callback_onstop=None, signal=15 + ): begin_time = time.time() if not self.stay_alive: raise Exception("Cannot restart not stay alive container") - self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root') + self.exec_in_container( + ["bash", "-c", "pkill -{} clickhouse".format(signal)], user="root" + ) retries = int(stop_start_wait_sec / 0.5) local_counter = 0 # wait stop @@ -2353,21 +3335,45 @@ class ClickHouseInstance: # force kill if server hangs if self.get_process_pid("clickhouse server"): # server can die before kill, so don't throw exception, it's expected - self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(9)], nothrow=True, user='root') + self.exec_in_container( + ["bash", "-c", "pkill -{} clickhouse".format(9)], + nothrow=True, + user="root", + ) if callback_onstop: callback_onstop(self) self.exec_in_container( ["bash", "-c", "cp /usr/bin/clickhouse /usr/share/clickhouse_original"], - user='root') + user="root", + ) self.exec_in_container( - ["bash", "-c", "cp /usr/share/clickhouse_fresh /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse"], - user='root') - self.exec_in_container(["bash", "-c", "echo 'restart_with_latest_version: From version' && /usr/share/clickhouse_original server --version && echo 'To version' /usr/share/clickhouse_fresh server --version"]) - self.exec_in_container(["bash", "-c", - "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"], - user='root') - self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) + [ + "bash", + "-c", + "cp /usr/share/clickhouse_fresh /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse", + ], + user="root", + ) + self.exec_in_container( + [ + "bash", + "-c", + "echo 'restart_with_latest_version: From version' && /usr/share/clickhouse_original server --version && echo 'To version' /usr/share/clickhouse_fresh server --version", + ] + ) + self.exec_in_container( + [ + "bash", + "-c", + "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse", + ], + user="root", + ) + self.exec_in_container( + ["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], + user=str(os.getuid()), + ) # wait start time_left = begin_time + stop_start_wait_sec - time.time() @@ -2392,8 +3398,11 @@ class ClickHouseInstance: raise Exception("Invalid timeout: {}".format(start_timeout)) if connection_timeout is not None and connection_timeout < start_timeout: - raise Exception("Connection timeout {} should be grater then start timeout {}" - .format(connection_timeout, start_timeout)) + raise Exception( + "Connection timeout {} should be grater then start timeout {}".format( + connection_timeout, start_timeout + ) + ) start_time = time.time() prev_rows_in_log = 0 @@ -2411,19 +3420,23 @@ class ClickHouseInstance: while True: handle.reload() status = handle.status - if status == 'exited': - raise Exception(f"Instance `{self.name}' failed to start. Container status: {status}, logs: {handle.logs().decode('utf-8')}") + if status == "exited": + raise Exception( + f"Instance `{self.name}' failed to start. Container status: {status}, logs: {handle.logs().decode('utf-8')}" + ) deadline = start_time + start_timeout # It is possible that server starts slowly. # If container is running, and there is some progress in log, check connection_timeout. - if connection_timeout and status == 'running' and has_new_rows_in_log(): + if connection_timeout and status == "running" and has_new_rows_in_log(): deadline = start_time + connection_timeout current_time = time.time() if current_time >= deadline: - raise Exception(f"Timed out while waiting for instance `{self.name}' with ip address {self.ip_address} to start. " \ - f"Container status: {status}, logs: {handle.logs().decode('utf-8')}") + raise Exception( + f"Timed out while waiting for instance `{self.name}' with ip address {self.ip_address} to start. " + f"Container status: {status}, logs: {handle.logs().decode('utf-8')}" + ) socket_timeout = min(start_timeout, deadline - current_time) @@ -2438,7 +3451,11 @@ class ClickHouseInstance: except socket.timeout: continue except socket.error as e: - if e.errno == errno.ECONNREFUSED or e.errno == errno.EHOSTUNREACH or e.errno == errno.ENETUNREACH: + if ( + e.errno == errno.ECONNREFUSED + or e.errno == errno.EHOSTUNREACH + or e.errno == errno.ENETUNREACH + ): time.sleep(0.1) else: raise @@ -2446,7 +3463,9 @@ class ClickHouseInstance: sock.close() def dict_to_xml(self, dictionary): - xml_str = dict2xml(dictionary, wrap=self.config_root_name, indent=" ", newlines=True) + xml_str = dict2xml( + dictionary, wrap=self.config_root_name, indent=" ", newlines=True + ) return xml_str @property @@ -2481,13 +3500,13 @@ class ClickHouseInstance: "Driver": "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so", "Setup": "/usr/lib/x86_64-linux-gnu/odbc/libodbcpsqlS.so", "ConnSettings": "", - } + }, } else: return {} def _create_odbc_config_file(self): - with open(self.odbc_ini_path.split(':')[0], 'w') as f: + with open(self.odbc_ini_path.split(":")[0], "w") as f: for driver_setup in list(self.odbc_drivers.values()): f.write("[{}]\n".format(driver_setup["DSN"])) for key, value in list(driver_setup.items()): @@ -2495,10 +3514,14 @@ class ClickHouseInstance: f.write(key + "=" + value + "\n") def replace_config(self, path_to_config, replacement): - self.exec_in_container(["bash", "-c", "echo '{}' > {}".format(replacement, path_to_config)]) + self.exec_in_container( + ["bash", "-c", "echo '{}' > {}".format(replacement, path_to_config)] + ) def replace_in_config(self, path_to_config, replace, replacement): - self.exec_in_container(["bash", "-c", f"sed -i 's/{replace}/{replacement}/g' {path_to_config}"]) + self.exec_in_container( + ["bash", "-c", f"sed -i 's/{replace}/{replacement}/g' {path_to_config}"] + ) def create_dir(self, destroy_dir=True): """Create the instance directory and all the needed files there.""" @@ -2510,54 +3533,64 @@ class ClickHouseInstance: os.makedirs(self.path) - instance_config_dir = p.abspath(p.join(self.path, 'configs')) + instance_config_dir = p.abspath(p.join(self.path, "configs")) os.makedirs(instance_config_dir) - print(f"Copy common default production configuration from {self.base_config_dir}. Files: {self.main_config_name}, {self.users_config_name}") + print( + f"Copy common default production configuration from {self.base_config_dir}. Files: {self.main_config_name}, {self.users_config_name}" + ) - shutil.copyfile(p.join(self.base_config_dir, self.main_config_name), p.join(instance_config_dir, self.main_config_name)) - shutil.copyfile(p.join(self.base_config_dir, self.users_config_name), p.join(instance_config_dir, self.users_config_name)) + shutil.copyfile( + p.join(self.base_config_dir, self.main_config_name), + p.join(instance_config_dir, self.main_config_name), + ) + shutil.copyfile( + p.join(self.base_config_dir, self.users_config_name), + p.join(instance_config_dir, self.users_config_name), + ) logging.debug("Create directory for configuration generated in this helper") # used by all utils with any config - conf_d_dir = p.abspath(p.join(instance_config_dir, 'conf.d')) + conf_d_dir = p.abspath(p.join(instance_config_dir, "conf.d")) os.mkdir(conf_d_dir) logging.debug("Create directory for common tests configuration") # used by server with main config.xml - self.config_d_dir = p.abspath(p.join(instance_config_dir, 'config.d')) + self.config_d_dir = p.abspath(p.join(instance_config_dir, "config.d")) os.mkdir(self.config_d_dir) - users_d_dir = p.abspath(p.join(instance_config_dir, 'users.d')) + users_d_dir = p.abspath(p.join(instance_config_dir, "users.d")) os.mkdir(users_d_dir) - dictionaries_dir = p.abspath(p.join(instance_config_dir, 'dictionaries')) + dictionaries_dir = p.abspath(p.join(instance_config_dir, "dictionaries")) os.mkdir(dictionaries_dir) - extra_conf_dir = p.abspath(p.join(instance_config_dir, 'extra_conf.d')) + extra_conf_dir = p.abspath(p.join(instance_config_dir, "extra_conf.d")) os.mkdir(extra_conf_dir) def write_embedded_config(name, dest_dir, fix_log_level=False): - with open(p.join(HELPERS_DIR, name), 'r') as f: + with open(p.join(HELPERS_DIR, name), "r") as f: data = f.read() - data = data.replace('clickhouse', self.config_root_name) + data = data.replace("clickhouse", self.config_root_name) if fix_log_level: - data = data.replace('test', 'trace') - with open(p.join(dest_dir, name), 'w') as r: + data = data.replace("test", "trace") + with open(p.join(dest_dir, name), "w") as r: r.write(data) logging.debug("Copy common configuration from helpers") # The file is named with 0_ prefix to be processed before other configuration overloads. if self.copy_common_configs: - need_fix_log_level = self.tag != 'latest' - write_embedded_config('0_common_instance_config.xml', self.config_d_dir, need_fix_log_level) + need_fix_log_level = self.tag != "latest" + write_embedded_config( + "0_common_instance_config.xml", self.config_d_dir, need_fix_log_level + ) - write_embedded_config('0_common_instance_users.xml', users_d_dir) + write_embedded_config("0_common_instance_users.xml", users_d_dir) if len(self.custom_dictionaries_paths): - write_embedded_config('0_common_enable_dictionaries.xml', self.config_d_dir) + write_embedded_config("0_common_enable_dictionaries.xml", self.config_d_dir) logging.debug("Generate and write macros file") macros = self.macros.copy() - macros['instance'] = self.name - with open(p.join(conf_d_dir, 'macros.xml'), 'w') as macros_config: + macros["instance"] = self.name + with open(p.join(conf_d_dir, "macros.xml"), "w") as macros_config: macros_config.write(self.dict_to_xml({"macros": macros})) # Put ZooKeeper config @@ -2565,10 +3598,14 @@ class ClickHouseInstance: shutil.copy(self.zookeeper_config_path, conf_d_dir) if self.with_kerberized_kafka or self.with_kerberized_hdfs: - shutil.copytree(self.kerberos_secrets_dir, p.abspath(p.join(self.path, 'secrets'))) + shutil.copytree( + self.kerberos_secrets_dir, p.abspath(p.join(self.path, "secrets")) + ) # Copy config.d configs - logging.debug(f"Copy custom test config files {self.custom_main_config_paths} to {self.config_d_dir}") + logging.debug( + f"Copy custom test config files {self.custom_main_config_paths} to {self.config_d_dir}" + ) for path in self.custom_main_config_paths: shutil.copy(path, self.config_d_dir) @@ -2582,16 +3619,18 @@ class ClickHouseInstance: for path in self.custom_extra_config_paths: shutil.copy(path, extra_conf_dir) - db_dir = p.abspath(p.join(self.path, 'database')) + db_dir = p.abspath(p.join(self.path, "database")) logging.debug(f"Setup database dir {db_dir}") if self.clickhouse_path_dir is not None: logging.debug(f"Database files taken from {self.clickhouse_path_dir}") shutil.copytree(self.clickhouse_path_dir, db_dir) - logging.debug(f"Database copied from {self.clickhouse_path_dir} to {db_dir}") + logging.debug( + f"Database copied from {self.clickhouse_path_dir} to {db_dir}" + ) else: os.mkdir(db_dir) - logs_dir = p.abspath(p.join(self.path, 'logs')) + logs_dir = p.abspath(p.join(self.path, "logs")) logging.debug(f"Setup logs dir {logs_dir}") os.mkdir(logs_dir) self.logs_dir = logs_dir @@ -2647,19 +3686,29 @@ class ClickHouseInstance: odbc_ini_path = "" if self.odbc_ini_path: self._create_odbc_config_file() - odbc_ini_path = '- ' + self.odbc_ini_path + odbc_ini_path = "- " + self.odbc_ini_path entrypoint_cmd = self.clickhouse_start_command if self.stay_alive: - entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace("{main_config_file}", self.main_config_name) + entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace( + "{main_config_file}", self.main_config_name + ) else: - entrypoint_cmd = '[' + ', '.join(map(lambda x: '"' + x + '"', entrypoint_cmd.split())) + ']' + entrypoint_cmd = ( + "[" + + ", ".join(map(lambda x: '"' + x + '"', entrypoint_cmd.split())) + + "]" + ) logging.debug("Entrypoint cmd: {}".format(entrypoint_cmd)) networks = app_net = ipv4_address = ipv6_address = net_aliases = net_alias1 = "" - if self.ipv4_address is not None or self.ipv6_address is not None or self.hostname != self.name: + if ( + self.ipv4_address is not None + or self.ipv6_address is not None + or self.hostname != self.name + ): networks = "networks:" app_net = "default:" if self.ipv4_address is not None: @@ -2672,51 +3721,70 @@ class ClickHouseInstance: if not self.with_installed_binary: binary_volume = "- " + self.server_bin_path + ":/usr/bin/clickhouse" - odbc_bridge_volume = "- " + self.odbc_bridge_bin_path + ":/usr/bin/clickhouse-odbc-bridge" - library_bridge_volume = "- " + self.library_bridge_bin_path + ":/usr/bin/clickhouse-library-bridge" + odbc_bridge_volume = ( + "- " + self.odbc_bridge_bin_path + ":/usr/bin/clickhouse-odbc-bridge" + ) + library_bridge_volume = ( + "- " + + self.library_bridge_bin_path + + ":/usr/bin/clickhouse-library-bridge" + ) else: binary_volume = "- " + self.server_bin_path + ":/usr/share/clickhouse_fresh" - odbc_bridge_volume = "- " + self.odbc_bridge_bin_path + ":/usr/share/clickhouse-odbc-bridge_fresh" - library_bridge_volume = "- " + self.library_bridge_bin_path + ":/usr/share/clickhouse-library-bridge_fresh" + odbc_bridge_volume = ( + "- " + + self.odbc_bridge_bin_path + + ":/usr/share/clickhouse-odbc-bridge_fresh" + ) + library_bridge_volume = ( + "- " + + self.library_bridge_bin_path + + ":/usr/share/clickhouse-library-bridge_fresh" + ) external_dirs_volumes = "" if self.external_dirs: for external_dir in self.external_dirs: - external_dir_abs_path = p.abspath(p.join(self.path, external_dir.lstrip('/'))) - logging.info(f'external_dir_abs_path={external_dir_abs_path}') + external_dir_abs_path = p.abspath( + p.join(self.path, external_dir.lstrip("/")) + ) + logging.info(f"external_dir_abs_path={external_dir_abs_path}") os.mkdir(external_dir_abs_path) - external_dirs_volumes += "- " + external_dir_abs_path + ":" + external_dir + "\n" + external_dirs_volumes += ( + "- " + external_dir_abs_path + ":" + external_dir + "\n" + ) - - with open(self.docker_compose_path, 'w') as docker_compose: - docker_compose.write(DOCKER_COMPOSE_TEMPLATE.format( - image=self.image, - tag=self.tag, - name=self.name, - hostname=self.hostname, - binary_volume=binary_volume, - odbc_bridge_volume=odbc_bridge_volume, - library_bridge_volume=library_bridge_volume, - instance_config_dir=instance_config_dir, - config_d_dir=self.config_d_dir, - db_dir=db_dir, - external_dirs_volumes=external_dirs_volumes, - tmpfs=str(self.tmpfs), - logs_dir=logs_dir, - depends_on=str(depends_on), - user=os.getuid(), - env_file=self.env_file, - odbc_ini_path=odbc_ini_path, - keytab_path=self.keytab_path, - krb5_conf=self.krb5_conf, - entrypoint_cmd=entrypoint_cmd, - networks=networks, - app_net=app_net, - ipv4_address=ipv4_address, - ipv6_address=ipv6_address, - net_aliases=net_aliases, - net_alias1=net_alias1, - )) + with open(self.docker_compose_path, "w") as docker_compose: + docker_compose.write( + DOCKER_COMPOSE_TEMPLATE.format( + image=self.image, + tag=self.tag, + name=self.name, + hostname=self.hostname, + binary_volume=binary_volume, + odbc_bridge_volume=odbc_bridge_volume, + library_bridge_volume=library_bridge_volume, + instance_config_dir=instance_config_dir, + config_d_dir=self.config_d_dir, + db_dir=db_dir, + external_dirs_volumes=external_dirs_volumes, + tmpfs=str(self.tmpfs), + logs_dir=logs_dir, + depends_on=str(depends_on), + user=os.getuid(), + env_file=self.env_file, + odbc_ini_path=odbc_ini_path, + keytab_path=self.keytab_path, + krb5_conf=self.krb5_conf, + entrypoint_cmd=entrypoint_cmd, + networks=networks, + app_net=app_net, + ipv4_address=ipv4_address, + ipv6_address=ipv6_address, + net_aliases=net_aliases, + net_alias1=net_alias1, + ) + ) def destroy_dir(self): if p.exists(self.path): @@ -2730,11 +3798,21 @@ class ClickHouseInstance: time.sleep(1) def get_backuped_s3_objects(self, disk, backup_name): - path = f'/var/lib/clickhouse/disks/{disk}/shadow/{backup_name}/store' + path = f"/var/lib/clickhouse/disks/{disk}/shadow/{backup_name}/store" self.wait_for_path_exists(path, 10) - command = ['find', path, '-type', 'f', - '-exec', 'grep', '-o', 'r[01]\\{64\\}-file-[[:lower:]]\\{32\\}', '{}', ';'] - return self.exec_in_container(command).split('\n') + command = [ + "find", + path, + "-type", + "f", + "-exec", + "grep", + "-o", + "r[01]\\{64\\}-file-[[:lower:]]\\{32\\}", + "{}", + ";", + ] + return self.exec_in_container(command).split("\n") class ClickHouseKiller(object): diff --git a/tests/integration/helpers/corrupt_part_data_on_disk.py b/tests/integration/helpers/corrupt_part_data_on_disk.py index 1a6f384da9e..676511ebbdf 100644 --- a/tests/integration/helpers/corrupt_part_data_on_disk.py +++ b/tests/integration/helpers/corrupt_part_data_on_disk.py @@ -1,14 +1,29 @@ def corrupt_part_data_on_disk(node, table, part_name): - part_path = node.query("SELECT path FROM system.parts WHERE table = '{}' and name = '{}'" - .format(table, part_name)).strip() + part_path = node.query( + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() corrupt_part_data_by_path(node, part_path) + def corrupt_part_data_by_path(node, part_path): print("Corrupting part", part_path, "at", node.name) - print("Will corrupt: ", - node.exec_in_container(['bash', '-c', 'cd {p} && ls *.bin | head -n 1'.format(p=part_path)])) + print( + "Will corrupt: ", + node.exec_in_container( + ["bash", "-c", "cd {p} && ls *.bin | head -n 1".format(p=part_path)] + ), + ) - node.exec_in_container(['bash', '-c', - 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format( - p=part_path)], privileged=True) + node.exec_in_container( + [ + "bash", + "-c", + "cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c 'echo \"1\" >> $1' -- {{}}".format( + p=part_path + ), + ], + privileged=True, + ) diff --git a/tests/integration/helpers/dictionary.py b/tests/integration/helpers/dictionary.py index 99c4e3a5a5d..aaa1b00a8a6 100644 --- a/tests/integration/helpers/dictionary.py +++ b/tests/integration/helpers/dictionary.py @@ -4,18 +4,18 @@ import copy class Layout(object): LAYOUTS_STR_DICT = { - 'flat': '', - 'hashed': '', - 'cache': '128', - 'ssd_cache': '/etc/clickhouse-server/dictionaries/all', - 'complex_key_hashed': '', - 'complex_key_hashed_one_key': '', - 'complex_key_hashed_two_keys': '', - 'complex_key_cache': '128', - 'complex_key_ssd_cache': '/etc/clickhouse-server/dictionaries/all', - 'range_hashed': '', - 'direct': '', - 'complex_key_direct': '' + "flat": "", + "hashed": "", + "cache": "128", + "ssd_cache": "/etc/clickhouse-server/dictionaries/all", + "complex_key_hashed": "", + "complex_key_hashed_one_key": "", + "complex_key_hashed_two_keys": "", + "complex_key_cache": "128", + "complex_key_ssd_cache": "/etc/clickhouse-server/dictionaries/all", + "range_hashed": "", + "direct": "", + "complex_key_direct": "", } def __init__(self, name): @@ -23,14 +23,14 @@ class Layout(object): self.is_complex = False self.is_simple = False self.is_ranged = False - if self.name.startswith('complex'): - self.layout_type = 'complex' + if self.name.startswith("complex"): + self.layout_type = "complex" self.is_complex = True - elif name.startswith('range'): - self.layout_type = 'ranged' + elif name.startswith("range"): + self.layout_type = "ranged" self.is_ranged = True else: - self.layout_type = 'simple' + self.layout_type = "simple" self.is_simple = True def get_str(self): @@ -38,8 +38,8 @@ class Layout(object): def get_key_block_name(self): if self.is_complex: - return 'key' - return 'id' + return "key" + return "id" class Row(object): @@ -59,8 +59,17 @@ class Row(object): class Field(object): - def __init__(self, name, field_type, is_key=False, is_range_key=False, default=None, hierarchical=False, - range_hash_type=None, default_value_for_get=None): + def __init__( + self, + name, + field_type, + is_key=False, + is_range_key=False, + default=None, + hierarchical=False, + range_hash_type=None, + default_value_for_get=None, + ): self.name = name self.field_type = field_type self.is_key = is_key @@ -72,30 +81,32 @@ class Field(object): self.default_value_for_get = default_value_for_get def get_attribute_str(self): - return ''' + return """ {name} {field_type} {default} {hierarchical} - '''.format( + """.format( name=self.name, field_type=self.field_type, - default=self.default if self.default else '', - hierarchical='true' if self.hierarchical else 'false', + default=self.default if self.default else "", + hierarchical="true" if self.hierarchical else "false", ) def get_simple_index_str(self): - return '{name}'.format(name=self.name) + return "{name}".format(name=self.name) def get_range_hash_str(self): if not self.range_hash_type: raise Exception("Field {} is not range hashed".format(self.name)) - return ''' + return """ {name} - '''.format(type=self.range_hash_type, name=self.name) + """.format( + type=self.range_hash_type, name=self.name + ) class DictionaryStructure(object): @@ -125,9 +136,14 @@ class DictionaryStructure(object): if not self.layout.is_complex and len(self.keys) > 1: raise Exception( - "More than one key {} field in non complex layout {}".format(len(self.keys), self.layout.name)) + "More than one key {} field in non complex layout {}".format( + len(self.keys), self.layout.name + ) + ) - if self.layout.is_ranged and (not self.range_key or len(self.range_fields) != 2): + if self.layout.is_ranged and ( + not self.range_key or len(self.range_fields) != 2 + ): raise Exception("Inconsistent configuration of ranged dictionary") def get_structure_str(self): @@ -148,7 +164,7 @@ class DictionaryStructure(object): for range_field in self.range_fields: ranged_strs.append(range_field.get_range_hash_str()) - return ''' + return """ {layout_str} @@ -158,12 +174,12 @@ class DictionaryStructure(object): {range_strs} {attributes_str} - '''.format( + """.format( layout_str=self.layout.get_str(), key_block_name=self.layout.get_key_block_name(), - key_str='\n'.join(key_strs), - attributes_str='\n'.join(fields_strs), - range_strs='\n'.join(ranged_strs), + key_str="\n".join(key_strs), + attributes_str="\n".join(fields_strs), + range_strs="\n".join(ranged_strs), ) def get_ordered_names(self): @@ -179,15 +195,19 @@ class DictionaryStructure(object): def get_all_fields(self): return self.keys + self.range_fields + self.ordinary_fields - def _get_dict_get_common_expression(self, dict_name, field, row, or_default, with_type, has): + def _get_dict_get_common_expression( + self, dict_name, field, row, or_default, with_type, has + ): if field in self.keys: - raise Exception("Trying to receive key field {} from dictionary".format(field.name)) + raise Exception( + "Trying to receive key field {} from dictionary".format(field.name) + ) if not self.layout.is_complex: if not or_default: - key_expr = ', toUInt64({})'.format(row.data[self.keys[0].name]) + key_expr = ", toUInt64({})".format(row.data[self.keys[0].name]) else: - key_expr = ', toUInt64({})'.format(self.keys[0].default_value_for_get) + key_expr = ", toUInt64({})".format(self.keys[0].default_value_for_get) else: key_exprs_strs = [] for key in self.keys: @@ -197,48 +217,57 @@ class DictionaryStructure(object): val = key.default_value_for_get if isinstance(val, str): val = "'" + val + "'" - key_exprs_strs.append('to{type}({value})'.format(type=key.field_type, value=val)) - key_expr = ', tuple(' + ','.join(key_exprs_strs) + ')' + key_exprs_strs.append( + "to{type}({value})".format(type=key.field_type, value=val) + ) + key_expr = ", tuple(" + ",".join(key_exprs_strs) + ")" - date_expr = '' + date_expr = "" if self.layout.is_ranged: val = row.data[self.range_key.name] if isinstance(val, str): val = "'" + val + "'" val = "to{type}({val})".format(type=self.range_key.field_type, val=val) - date_expr = ', ' + val + date_expr = ", " + val if or_default: - raise Exception("Can create 'dictGetOrDefault' query for ranged dictionary") + raise Exception( + "Can create 'dictGetOrDefault' query for ranged dictionary" + ) if or_default: - or_default_expr = 'OrDefault' + or_default_expr = "OrDefault" if field.default_value_for_get is None: raise Exception( - "Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format(field.name)) + "Can create 'dictGetOrDefault' query for field {} without default_value_for_get".format( + field.name + ) + ) val = field.default_value_for_get if isinstance(val, str): val = "'" + val + "'" - default_value_for_get = ', to{type}({value})'.format(type=field.field_type, value=val) + default_value_for_get = ", to{type}({value})".format( + type=field.field_type, value=val + ) else: - or_default_expr = '' - default_value_for_get = '' + or_default_expr = "" + default_value_for_get = "" if with_type: field_type = field.field_type else: - field_type = '' + field_type = "" field_name = ", '" + field.name + "'" if has: what = "Has" - field_type = '' - or_default = '' - field_name = '' - date_expr = '' - def_for_get = '' + field_type = "" + or_default = "" + field_name = "" + date_expr = "" + def_for_get = "" else: what = "Get" @@ -255,28 +284,38 @@ class DictionaryStructure(object): def get_get_expressions(self, dict_name, field, row): return [ - self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False, has=False), - self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=True, has=False), + self._get_dict_get_common_expression( + dict_name, field, row, or_default=False, with_type=False, has=False + ), + self._get_dict_get_common_expression( + dict_name, field, row, or_default=False, with_type=True, has=False + ), ] def get_get_or_default_expressions(self, dict_name, field, row): if not self.layout.is_ranged: return [ - self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=False, - has=False), - self._get_dict_get_common_expression(dict_name, field, row, or_default=True, with_type=True, has=False), + self._get_dict_get_common_expression( + dict_name, field, row, or_default=True, with_type=False, has=False + ), + self._get_dict_get_common_expression( + dict_name, field, row, or_default=True, with_type=True, has=False + ), ] return [] def get_has_expressions(self, dict_name, field, row): if not self.layout.is_ranged: - return [self._get_dict_get_common_expression(dict_name, field, row, or_default=False, with_type=False, - has=True)] + return [ + self._get_dict_get_common_expression( + dict_name, field, row, or_default=False, with_type=False, has=True + ) + ] return [] def get_hierarchical_expressions(self, dict_name, row): if self.layout.is_simple: - key_expr = 'toUInt64({})'.format(row.data[self.keys[0].name]) + key_expr = "toUInt64({})".format(row.data[self.keys[0].name]) return [ "dictGetHierarchy('{dict_name}', {key})".format( dict_name=dict_name, @@ -288,21 +327,31 @@ class DictionaryStructure(object): def get_is_in_expressions(self, dict_name, row, parent_row): if self.layout.is_simple: - child_key_expr = 'toUInt64({})'.format(row.data[self.keys[0].name]) - parent_key_expr = 'toUInt64({})'.format(parent_row.data[self.keys[0].name]) + child_key_expr = "toUInt64({})".format(row.data[self.keys[0].name]) + parent_key_expr = "toUInt64({})".format(parent_row.data[self.keys[0].name]) return [ "dictIsIn('{dict_name}', {child_key}, {parent_key})".format( dict_name=dict_name, child_key=child_key_expr, - parent_key=parent_key_expr, ) + parent_key=parent_key_expr, + ) ] return [] class Dictionary(object): - def __init__(self, name, structure, source, config_path, - table_name, fields, min_lifetime=3, max_lifetime=5): + def __init__( + self, + name, + structure, + source, + config_path, + table_name, + fields, + min_lifetime=3, + max_lifetime=5, + ): self.name = name self.structure = copy.deepcopy(structure) self.source = copy.deepcopy(source) @@ -313,9 +362,10 @@ class Dictionary(object): self.max_lifetime = max_lifetime def generate_config(self): - with open(self.config_path, 'w') as result: - if 'direct' not in self.structure.layout.get_str(): - result.write(''' + with open(self.config_path, "w") as result: + if "direct" not in self.structure.layout.get_str(): + result.write( + """ @@ -329,15 +379,17 @@ class Dictionary(object): - '''.format( - min_lifetime=self.min_lifetime, - max_lifetime=self.max_lifetime, - name=self.name, - structure=self.structure.get_structure_str(), - source=self.source.get_source_str(self.table_name), - )) + """.format( + min_lifetime=self.min_lifetime, + max_lifetime=self.max_lifetime, + name=self.name, + structure=self.structure.get_structure_str(), + source=self.source.get_source_str(self.table_name), + ) + ) else: - result.write(''' + result.write( + """ {name} @@ -347,38 +399,59 @@ class Dictionary(object): - '''.format( - min_lifetime=self.min_lifetime, - max_lifetime=self.max_lifetime, - name=self.name, - structure=self.structure.get_structure_str(), - source=self.source.get_source_str(self.table_name), - )) + """.format( + min_lifetime=self.min_lifetime, + max_lifetime=self.max_lifetime, + name=self.name, + structure=self.structure.get_structure_str(), + source=self.source.get_source_str(self.table_name), + ) + ) def prepare_source(self, cluster): self.source.prepare(self.structure, self.table_name, cluster) def load_data(self, data): if not self.source.prepared: - raise Exception("Cannot load data for dictionary {}, source is not prepared".format(self.name)) + raise Exception( + "Cannot load data for dictionary {}, source is not prepared".format( + self.name + ) + ) self.source.load_data(data, self.table_name) def get_select_get_queries(self, field, row): - return ['select {}'.format(expr) for expr in self.structure.get_get_expressions(self.name, field, row)] + return [ + "select {}".format(expr) + for expr in self.structure.get_get_expressions(self.name, field, row) + ] def get_select_get_or_default_queries(self, field, row): - return ['select {}'.format(expr) for expr in - self.structure.get_get_or_default_expressions(self.name, field, row)] + return [ + "select {}".format(expr) + for expr in self.structure.get_get_or_default_expressions( + self.name, field, row + ) + ] def get_select_has_queries(self, field, row): - return ['select {}'.format(expr) for expr in self.structure.get_has_expressions(self.name, field, row)] + return [ + "select {}".format(expr) + for expr in self.structure.get_has_expressions(self.name, field, row) + ] def get_hierarchical_queries(self, row): - return ['select {}'.format(expr) for expr in self.structure.get_hierarchical_expressions(self.name, row)] + return [ + "select {}".format(expr) + for expr in self.structure.get_hierarchical_expressions(self.name, row) + ] def get_is_in_queries(self, row, parent_row): - return ['select {}'.format(expr) for expr in self.structure.get_is_in_expressions(self.name, row, parent_row)] + return [ + "select {}".format(expr) + for expr in self.structure.get_is_in_expressions(self.name, row, parent_row) + ] def is_complex(self): return self.structure.layout.is_complex diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index 32ebdfa58c6..fd086fc4526 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -10,12 +10,19 @@ import pymongo import pymysql.cursors import redis import logging -from tzlocal import get_localzone class ExternalSource(object): - def __init__(self, name, internal_hostname, internal_port, - docker_hostname, docker_port, user, password): + def __init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ): self.name = name self.internal_hostname = internal_hostname self.internal_port = int(internal_port) @@ -25,17 +32,26 @@ class ExternalSource(object): self.password = password def get_source_str(self, table_name): - raise NotImplementedError("Method {} is not implemented for {}".format( - "get_source_config_part", self.__class__.__name__)) + raise NotImplementedError( + "Method {} is not implemented for {}".format( + "get_source_config_part", self.__class__.__name__ + ) + ) def prepare(self, structure, table_name, cluster): - raise NotImplementedError("Method {} is not implemented for {}".format( - "prepare_remote_source", self.__class__.__name__)) + raise NotImplementedError( + "Method {} is not implemented for {}".format( + "prepare_remote_source", self.__class__.__name__ + ) + ) # data is banch of Row def load_data(self, data): - raise NotImplementedError("Method {} is not implemented for {}".format( - "prepare_remote_source", self.__class__.__name__)) + raise NotImplementedError( + "Method {} is not implemented for {}".format( + "prepare_remote_source", self.__class__.__name__ + ) + ) def compatible_with_layout(self, layout): return True @@ -43,29 +59,32 @@ class ExternalSource(object): class SourceMySQL(ExternalSource): TYPE_MAPPING = { - 'UInt8': 'tinyint unsigned', - 'UInt16': 'smallint unsigned', - 'UInt32': 'int unsigned', - 'UInt64': 'bigint unsigned', - 'Int8': 'tinyint', - 'Int16': 'smallint', - 'Int32': 'int', - 'Int64': 'bigint', - 'UUID': 'varchar(36)', - 'Date': 'date', - 'DateTime': 'datetime', - 'String': 'text', - 'Float32': 'float', - 'Float64': 'double' + "UInt8": "tinyint unsigned", + "UInt16": "smallint unsigned", + "UInt32": "int unsigned", + "UInt64": "bigint unsigned", + "Int8": "tinyint", + "Int16": "smallint", + "Int32": "int", + "Int64": "bigint", + "UUID": "varchar(36)", + "Date": "date", + "DateTime": "datetime", + "String": "text", + "Float32": "float", + "Float64": "double", } def create_mysql_conn(self): - logging.debug(f"pymysql connect {self.user}, {self.password}, {self.internal_hostname}, {self.internal_port}") + logging.debug( + f"pymysql connect {self.user}, {self.password}, {self.internal_hostname}, {self.internal_port}" + ) self.connection = pymysql.connect( user=self.user, password=self.password, host=self.internal_hostname, - port=self.internal_port) + port=self.internal_port, + ) def execute_mysql_query(self, query): with warnings.catch_warnings(): @@ -75,7 +94,7 @@ class SourceMySQL(ExternalSource): self.connection.commit() def get_source_str(self, table_name): - return ''' + return """ 1 @@ -91,7 +110,7 @@ class SourceMySQL(ExternalSource): {password} test {tbl}
- '''.format( + """.format( hostname=self.docker_hostname, port=self.docker_port, user=self.user, @@ -103,14 +122,20 @@ class SourceMySQL(ExternalSource): if self.internal_hostname is None: self.internal_hostname = cluster.mysql_ip self.create_mysql_conn() - self.execute_mysql_query("create database if not exists test default character set 'utf8'") + self.execute_mysql_query( + "create database if not exists test default character set 'utf8'" + ) self.execute_mysql_query("drop table if exists test.{}".format(table_name)) fields_strs = [] - for field in structure.keys + structure.ordinary_fields + structure.range_fields: - fields_strs.append(field.name + ' ' + self.TYPE_MAPPING[field.field_type]) - create_query = '''create table test.{table_name} ( + for field in ( + structure.keys + structure.ordinary_fields + structure.range_fields + ): + fields_strs.append(field.name + " " + self.TYPE_MAPPING[field.field_type]) + create_query = """create table test.{table_name} ( {fields_str}); - '''.format(table_name=table_name, fields_str=','.join(fields_strs)) + """.format( + table_name=table_name, fields_str=",".join(fields_strs) + ) self.execute_mysql_query(create_query) self.ordered_names = structure.get_ordered_names() self.prepared = True @@ -128,18 +153,16 @@ class SourceMySQL(ExternalSource): else: data = str(data) sorted_row.append(data) - values_strs.append('(' + ','.join(sorted_row) + ')') - query = 'insert into test.{} ({}) values {}'.format( - table_name, - ','.join(self.ordered_names), - ','.join(values_strs)) + values_strs.append("(" + ",".join(sorted_row) + ")") + query = "insert into test.{} ({}) values {}".format( + table_name, ",".join(self.ordered_names), ",".join(values_strs) + ) self.execute_mysql_query(query) class SourceMongo(ExternalSource): - def get_source_str(self, table_name): - return ''' + return """ {host} {port} @@ -148,7 +171,7 @@ class SourceMongo(ExternalSource): test {tbl} - '''.format( + """.format( host=self.docker_hostname, port=self.docker_port, user=self.user, @@ -157,21 +180,29 @@ class SourceMongo(ExternalSource): ) def prepare(self, structure, table_name, cluster): - connection_str = 'mongodb://{user}:{password}@{host}:{port}'.format( - host=self.internal_hostname, port=self.internal_port, - user=self.user, password=self.password) + connection_str = "mongodb://{user}:{password}@{host}:{port}".format( + host=self.internal_hostname, + port=self.internal_port, + user=self.user, + password=self.password, + ) self.connection = pymongo.MongoClient(connection_str) self.converters = {} for field in structure.get_all_fields(): if field.field_type == "Date": - self.converters[field.name] = lambda x: datetime.datetime.strptime(x, "%Y-%m-%d") + self.converters[field.name] = lambda x: datetime.datetime.strptime( + x, "%Y-%m-%d" + ) elif field.field_type == "DateTime": - self.converters[field.name] = lambda x: get_localzone().localize( - datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")) + + def converter(x): + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + + self.converters[field.name] = converter else: self.converters[field.name] = lambda x: x - self.db = self.connection['test'] + self.db = self.connection["test"] self.db.add_user(self.user, self.password) self.prepared = True @@ -192,15 +223,15 @@ class SourceMongoURI(SourceMongo): def compatible_with_layout(self, layout): # It is enough to test one layout for this dictionary, since we're # only testing that the connection with URI works. - return layout.name == 'flat' + return layout.name == "flat" def get_source_str(self, table_name): - return ''' + return """ mongodb://{user}:{password}@{host}:{port}/test {tbl} - '''.format( + """.format( host=self.docker_hostname, port=self.docker_port, user=self.user, @@ -210,9 +241,8 @@ class SourceMongoURI(SourceMongo): class SourceClickHouse(ExternalSource): - def get_source_str(self, table_name): - return ''' + return """ {host} {port} @@ -221,7 +251,7 @@ class SourceClickHouse(ExternalSource): test {tbl}
 - '''.format( + """.format( host=self.docker_hostname, port=self.docker_port, user=self.user, @@ -233,11 +263,15 @@ class SourceClickHouse(ExternalSource): self.node = cluster.instances[self.docker_hostname] self.node.query("CREATE DATABASE IF NOT EXISTS test") fields_strs = [] - for field in structure.keys + structure.ordinary_fields + structure.range_fields: - fields_strs.append(field.name + ' ' + field.field_type) - create_query = '''CREATE TABLE test.{table_name} ( + for field in ( + structure.keys + structure.ordinary_fields + structure.range_fields + ): + fields_strs.append(field.name + " " + field.field_type) + create_query = """CREATE TABLE test.{table_name} ( {fields_str}) ENGINE MergeTree ORDER BY tuple(); - '''.format(table_name=table_name, fields_str=','.join(fields_strs)) + """.format( + table_name=table_name, fields_str=",".join(fields_strs) + ) self.node.query(create_query) self.ordered_names = structure.get_ordered_names() self.prepared = True @@ -255,31 +289,31 @@ class SourceClickHouse(ExternalSource): else: row_data = str(row_data) sorted_row.append(row_data) - values_strs.append('(' + ','.join(sorted_row) + ')') - query = 'INSERT INTO test.{} ({}) values {}'.format( - table_name, - ','.join(self.ordered_names), - ','.join(values_strs)) + values_strs.append("(" + ",".join(sorted_row) + ")") + query = "INSERT INTO test.{} ({}) values {}".format( + table_name, ",".join(self.ordered_names), ",".join(values_strs) + ) self.node.query(query) class SourceFile(ExternalSource): - def get_source_str(self, table_name): table_path = "/" + table_name + ".tsv" - return ''' + return """ {path} TabSeparated - '''.format( + """.format( path=table_path, ) def prepare(self, structure, table_name, cluster): self.node = cluster.instances[self.docker_hostname] path = "/" + table_name + ".tsv" - self.node.exec_in_container(["bash", "-c", "touch {}".format(path)], user="root") + self.node.exec_in_container( + ["bash", "-c", "touch {}".format(path)], user="root" + ) self.ordered_names = structure.get_ordered_names() self.prepared = True @@ -292,35 +326,45 @@ class SourceFile(ExternalSource): for name in self.ordered_names: sorted_row.append(str(row.data[name])) - str_data = '\t'.join(sorted_row) - self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], - user="root") + str_data = "\t".join(sorted_row) + self.node.exec_in_container( + [ + "bash", + "-c", + 'echo "{row}" >> {fname}'.format(row=str_data, fname=path), + ], + user="root", + ) def compatible_with_layout(self, layout): - return 'cache' not in layout.name and 'direct' not in layout.name + return "cache" not in layout.name and "direct" not in layout.name class _SourceExecutableBase(ExternalSource): - def _get_cmd(self, path): - raise NotImplementedError("Method {} is not implemented for {}".format( - "_get_cmd", self.__class__.__name__)) + raise NotImplementedError( + "Method {} is not implemented for {}".format( + "_get_cmd", self.__class__.__name__ + ) + ) def get_source_str(self, table_name): table_path = "/" + table_name + ".tsv" - return ''' + return """ {cmd} TabSeparated - '''.format( + """.format( cmd=self._get_cmd(table_path), ) def prepare(self, structure, table_name, cluster): self.node = cluster.instances[self.docker_hostname] path = "/" + table_name + ".tsv" - self.node.exec_in_container(["bash", "-c", "touch {}".format(path)], user="root") + self.node.exec_in_container( + ["bash", "-c", "touch {}".format(path)], user="root" + ) self.ordered_names = structure.get_ordered_names() self.prepared = True @@ -333,27 +377,31 @@ class _SourceExecutableBase(ExternalSource): for name in self.ordered_names: sorted_row.append(str(row.data[name])) - str_data = '\t'.join(sorted_row) - self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], - user='root') + str_data = "\t".join(sorted_row) + self.node.exec_in_container( + [ + "bash", + "-c", + 'echo "{row}" >> {fname}'.format(row=str_data, fname=path), + ], + user="root", + ) class SourceExecutableHashed(_SourceExecutableBase): - def _get_cmd(self, path): return "cat {}".format(path) def compatible_with_layout(self, layout): - return 'hashed' in layout.name + return "hashed" in layout.name class SourceExecutableCache(_SourceExecutableBase): - def _get_cmd(self, path): return "cat - >/dev/null;cat {}".format(path) def compatible_with_layout(self, layout): - return 'cache' in layout.name + return "cache" in layout.name class SourceHTTPBase(ExternalSource): @@ -361,10 +409,11 @@ class SourceHTTPBase(ExternalSource): def get_source_str(self, table_name): self.http_port = SourceHTTPBase.PORT_COUNTER - url = "{schema}://{host}:{port}/".format(schema=self._get_schema(), host=self.docker_hostname, - port=self.http_port) + url = "{schema}://{host}:{port}/".format( + schema=self._get_schema(), host=self.docker_hostname, port=self.http_port + ) SourceHTTPBase.PORT_COUNTER += 1 - return ''' + return """ {url} TabSeparated @@ -379,22 +428,37 @@ class SourceHTTPBase(ExternalSource): - '''.format(url=url) + """.format( + url=url + ) def prepare(self, structure, table_name, cluster): self.node = cluster.instances[self.docker_hostname] path = "/" + table_name + ".tsv" - self.node.exec_in_container(["bash", "-c", "touch {}".format(path)], user='root') + self.node.exec_in_container( + ["bash", "-c", "touch {}".format(path)], user="root" + ) script_dir = os.path.dirname(os.path.realpath(__file__)) - self.node.copy_file_to_container(os.path.join(script_dir, './http_server.py'), '/http_server.py') - self.node.copy_file_to_container(os.path.join(script_dir, './fake_cert.pem'), '/fake_cert.pem') - self.node.exec_in_container([ - "bash", - "-c", - "python3 /http_server.py --data-path={tbl} --schema={schema} --host={host} --port={port} --cert-path=/fake_cert.pem".format( - tbl=path, schema=self._get_schema(), host=self.docker_hostname, port=self.http_port) - ], detach=True) + self.node.copy_file_to_container( + os.path.join(script_dir, "./http_server.py"), "/http_server.py" + ) + self.node.copy_file_to_container( + os.path.join(script_dir, "./fake_cert.pem"), "/fake_cert.pem" + ) + self.node.exec_in_container( + [ + "bash", + "-c", + "python3 /http_server.py --data-path={tbl} --schema={schema} --host={host} --port={port} --cert-path=/fake_cert.pem".format( + tbl=path, + schema=self._get_schema(), + host=self.docker_hostname, + port=self.http_port, + ), + ], + detach=True, + ) self.ordered_names = structure.get_ordered_names() self.prepared = True @@ -407,9 +471,15 @@ class SourceHTTPBase(ExternalSource): for name in self.ordered_names: sorted_row.append(str(row.data[name])) - str_data = '\t'.join(sorted_row) - self.node.exec_in_container(["bash", "-c", "echo \"{row}\" >> {fname}".format(row=str_data, fname=path)], - user='root') + str_data = "\t".join(sorted_row) + self.node.exec_in_container( + [ + "bash", + "-c", + 'echo "{row}" >> {fname}'.format(row=str_data, fname=path), + ], + user="root", + ) class SourceHTTP(SourceHTTPBase): @@ -424,29 +494,46 @@ class SourceHTTPS(SourceHTTPBase): class SourceCassandra(ExternalSource): TYPE_MAPPING = { - 'UInt8': 'tinyint', - 'UInt16': 'smallint', - 'UInt32': 'int', - 'UInt64': 'bigint', - 'Int8': 'tinyint', - 'Int16': 'smallint', - 'Int32': 'int', - 'Int64': 'bigint', - 'UUID': 'uuid', - 'Date': 'date', - 'DateTime': 'timestamp', - 'String': 'text', - 'Float32': 'float', - 'Float64': 'double' + "UInt8": "tinyint", + "UInt16": "smallint", + "UInt32": "int", + "UInt64": "bigint", + "Int8": "tinyint", + "Int16": "smallint", + "Int32": "int", + "Int64": "bigint", + "UUID": "uuid", + "Date": "date", + "DateTime": "timestamp", + "String": "text", + "Float32": "float", + "Float64": "double", } - def __init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password): - ExternalSource.__init__(self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, - password) + def __init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ): + ExternalSource.__init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ) self.structure = dict() def get_source_str(self, table_name): - return ''' + return """ {host} {port} @@ -455,7 +542,7 @@ class SourceCassandra(ExternalSource): 1 "Int64_" < 1000000000000000000 - '''.format( + """.format( host=self.docker_hostname, port=self.docker_port, table=table_name, @@ -465,50 +552,79 @@ class SourceCassandra(ExternalSource): if self.internal_hostname is None: self.internal_hostname = cluster.cassandra_ip - self.client = cassandra.cluster.Cluster([self.internal_hostname], port=self.internal_port) + self.client = cassandra.cluster.Cluster( + [self.internal_hostname], port=self.internal_port + ) self.session = self.client.connect() self.session.execute( - "create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};") + "create keyspace if not exists test with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};" + ) self.session.execute('drop table if exists test."{}"'.format(table_name)) self.structure[table_name] = structure - columns = ['"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] for col in structure.get_all_fields()] + columns = [ + '"' + col.name + '" ' + self.TYPE_MAPPING[col.field_type] + for col in structure.get_all_fields() + ] keys = ['"' + col.name + '"' for col in structure.keys] query = 'create table test."{name}" ({columns}, primary key ({pk}));'.format( - name=table_name, columns=', '.join(columns), pk=', '.join(keys)) + name=table_name, columns=", ".join(columns), pk=", ".join(keys) + ) self.session.execute(query) self.prepared = True def get_value_to_insert(self, value, type): - if type == 'UUID': + if type == "UUID": return uuid.UUID(value) - elif type == 'DateTime': - local_datetime = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') - return get_localzone().localize(local_datetime) + elif type == "DateTime": + return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") return value def load_data(self, data, table_name): - names_and_types = [(field.name, field.field_type) for field in self.structure[table_name].get_all_fields()] + names_and_types = [ + (field.name, field.field_type) + for field in self.structure[table_name].get_all_fields() + ] columns = ['"' + col[0] + '"' for col in names_and_types] insert = 'insert into test."{table}" ({columns}) values ({args})'.format( - table=table_name, columns=','.join(columns), args=','.join(['%s'] * len(columns))) + table=table_name, + columns=",".join(columns), + args=",".join(["%s"] * len(columns)), + ) for row in data: - values = [self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) for col in names_and_types] + values = [ + self.get_value_to_insert(row.get_value_by_name(col[0]), col[1]) + for col in names_and_types + ] self.session.execute(insert, values) class SourceRedis(ExternalSource): def __init__( - self, name, internal_hostname, internal_port, docker_hostname, docker_port, user, password, db_index, - storage_type + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + db_index, + storage_type, ): super(SourceRedis, self).__init__( - name, internal_hostname, internal_port, docker_hostname, docker_port, user, password + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, ) self.storage_type = storage_type self.db_index = db_index def get_source_str(self, table_name): - return ''' + return """ {host} {port} @@ -516,7 +632,7 @@ class SourceRedis(ExternalSource): {db_index} {storage_type} - '''.format( + """.format( host=self.docker_hostname, port=self.docker_port, password=self.password, @@ -525,8 +641,12 @@ class SourceRedis(ExternalSource): ) def prepare(self, structure, table_name, cluster): - self.client = redis.StrictRedis(host=self.internal_hostname, port=self.internal_port, db=self.db_index, - password=self.password or None) + self.client = redis.StrictRedis( + host=self.internal_hostname, + port=self.internal_port, + db=self.db_index, + password=self.password or None, + ) self.prepared = True self.ordered_names = structure.get_ordered_names() @@ -542,33 +662,52 @@ class SourceRedis(ExternalSource): self.client.hset(*values) def compatible_with_layout(self, layout): - return layout.is_simple and self.storage_type == "simple" or layout.is_complex and self.storage_type == "hash_map" + return ( + layout.is_simple + and self.storage_type == "simple" + or layout.is_complex + and self.storage_type == "hash_map" + ) class SourceAerospike(ExternalSource): - def __init__(self, name, internal_hostname, internal_port, - docker_hostname, docker_port, user, password): - ExternalSource.__init__(self, name, internal_hostname, internal_port, - docker_hostname, docker_port, user, password) + def __init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ): + ExternalSource.__init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ) self.namespace = "test" self.set = "test_set" def get_source_str(self, table_name): print("AEROSPIKE get source str") - return ''' + return """ {host} {port} - '''.format( + """.format( host=self.docker_hostname, port=self.docker_port, ) def prepare(self, structure, table_name, cluster): - config = { - 'hosts': [(self.internal_hostname, self.internal_port)] - } + config = {"hosts": [(self.internal_hostname, self.internal_port)]} self.client = aerospike.client(config).connect() self.prepared = True print("PREPARED AEROSPIKE") @@ -603,10 +742,14 @@ class SourceAerospike(ExternalSource): for value in values: key = (self.namespace, self.set, value[0]) print(key) - self.client.put(key, {"bin_value": value[1]}, policy={"key": aerospike.POLICY_KEY_SEND}) + self.client.put( + key, + {"bin_value": value[1]}, + policy={"key": aerospike.POLICY_KEY_SEND}, + ) assert self.client.exists(key) else: - assert ("VALUES SIZE != 2") + assert "VALUES SIZE != 2" # print(values) diff --git a/tests/integration/helpers/hdfs_api.py b/tests/integration/helpers/hdfs_api.py index 3d2d647d0ed..5739496cb50 100644 --- a/tests/integration/helpers/hdfs_api.py +++ b/tests/integration/helpers/hdfs_api.py @@ -10,27 +10,44 @@ import socket import tempfile import logging import os + + class mk_krb_conf(object): def __init__(self, krb_conf, kdc_ip): self.krb_conf = krb_conf self.kdc_ip = kdc_ip self.amended_krb_conf = None + def __enter__(self): with open(self.krb_conf) as f: content = f.read() - amended_content = content.replace('hdfskerberos', self.kdc_ip) + amended_content = content.replace("hdfskerberos", self.kdc_ip) self.amended_krb_conf = tempfile.NamedTemporaryFile(delete=False, mode="w+") self.amended_krb_conf.write(amended_content) self.amended_krb_conf.close() return self.amended_krb_conf.name + def __exit__(self, type, value, traceback): if self.amended_krb_conf is not None: self.amended_krb_conf.close() + class HDFSApi(object): - def __init__(self, user, host, proxy_port, data_port, timeout=100, kerberized=False, principal=None, - keytab=None, krb_conf=None, - protocol = "http", hdfs_ip = None, kdc_ip = None): + def __init__( + self, + user, + host, + proxy_port, + data_port, + timeout=100, + kerberized=False, + principal=None, + keytab=None, + krb_conf=None, + protocol="http", + hdfs_ip=None, + kdc_ip=None, + ): self.host = host self.protocol = protocol self.proxy_port = proxy_port @@ -55,7 +72,11 @@ class HDFSApi(object): if kerberized: self._run_kinit() - self.kerberos_auth = reqkerb.HTTPKerberosAuth(mutual_authentication=reqkerb.DISABLED, hostname_override=self.host, principal=self.principal) + self.kerberos_auth = reqkerb.HTTPKerberosAuth( + mutual_authentication=reqkerb.DISABLED, + hostname_override=self.host, + principal=self.principal, + ) if self.kerberos_auth is None: print("failed to obtain kerberos_auth") else: @@ -70,7 +91,11 @@ class HDFSApi(object): os.environ["KRB5_CONFIG"] = instantiated_krb_conf - cmd = "(kinit -R -t {keytab} -k {principal} || (sleep 5 && kinit -R -t {keytab} -k {principal})) ; klist".format(instantiated_krb_conf=instantiated_krb_conf, keytab=self.keytab, principal=self.principal) + cmd = "(kinit -R -t {keytab} -k {principal} || (sleep 5 && kinit -R -t {keytab} -k {principal})) ; klist".format( + instantiated_krb_conf=instantiated_krb_conf, + keytab=self.keytab, + principal=self.principal, + ) start = time.time() @@ -79,10 +104,18 @@ class HDFSApi(object): res = subprocess.run(cmd, shell=True) if res.returncode != 0: # check_call(...) from subprocess does not print stderr, so we do it manually - logging.debug('Stderr:\n{}\n'.format(res.stderr.decode('utf-8'))) - logging.debug('Stdout:\n{}\n'.format(res.stdout.decode('utf-8'))) - logging.debug('Env:\n{}\n'.format(env)) - raise Exception('Command {} return non-zero code {}: {}'.format(args, res.returncode, res.stderr.decode('utf-8'))) + logging.debug( + "Stderr:\n{}\n".format(res.stderr.decode("utf-8")) + ) + logging.debug( + "Stdout:\n{}\n".format(res.stdout.decode("utf-8")) + ) + logging.debug("Env:\n{}\n".format(env)) + raise Exception( + "Command {} return non-zero code {}: {}".format( + args, res.returncode, res.stderr.decode("utf-8") + ) + ) logging.debug("KDC started, kinit successfully run") return @@ -97,28 +130,60 @@ class HDFSApi(object): for i in range(0, cnt): logging.debug(f"CALL: {str(kwargs)}") response_data = func(**kwargs) - logging.debug(f"response_data:{response_data.content} headers:{response_data.headers}") + logging.debug( + f"response_data:{response_data.content} headers:{response_data.headers}" + ) if response_data.status_code == expected_code: return response_data else: - logging.error(f"unexpected response_data.status_code {response_data.status_code} != {expected_code}") + logging.error( + f"unexpected response_data.status_code {response_data.status_code} != {expected_code}" + ) time.sleep(1) response_data.raise_for_status() - def read_data(self, path, universal_newlines=True): - logging.debug("read_data protocol:{} host:{} ip:{} proxy port:{} data port:{} path: {}".format(self.protocol, self.host, self.hdfs_ip, self.proxy_port, self.data_port, path)) - response = self.req_wrapper(requests.get, 307, url="{protocol}://{ip}:{port}/webhdfs/v1{path}?op=OPEN".format(protocol=self.protocol, ip=self.hdfs_ip, port=self.proxy_port, path=path), headers={'host': str(self.hdfs_ip)}, allow_redirects=False, verify=False, auth=self.kerberos_auth) + logging.debug( + "read_data protocol:{} host:{} ip:{} proxy port:{} data port:{} path: {}".format( + self.protocol, + self.host, + self.hdfs_ip, + self.proxy_port, + self.data_port, + path, + ) + ) + response = self.req_wrapper( + requests.get, + 307, + url="{protocol}://{ip}:{port}/webhdfs/v1{path}?op=OPEN".format( + protocol=self.protocol, ip=self.hdfs_ip, port=self.proxy_port, path=path + ), + headers={"host": str(self.hdfs_ip)}, + allow_redirects=False, + verify=False, + auth=self.kerberos_auth, + ) # additional_params = '&'.join(response.headers['Location'].split('&')[1:2]) location = None if self.kerberized: - location = response.headers['Location'].replace("kerberizedhdfs1:1006", "{}:{}".format(self.hdfs_ip, self.data_port)) + location = response.headers["Location"].replace( + "kerberizedhdfs1:1006", "{}:{}".format(self.hdfs_ip, self.data_port) + ) else: - location = response.headers['Location'].replace("hdfs1:50075", "{}:{}".format(self.hdfs_ip, self.data_port)) + location = response.headers["Location"].replace( + "hdfs1:50075", "{}:{}".format(self.hdfs_ip, self.data_port) + ) logging.debug("redirected to {}".format(location)) - response_data = self.req_wrapper(requests.get, 200, url=location, headers={'host': self.hdfs_ip}, - verify=False, auth=self.kerberos_auth) + response_data = self.req_wrapper( + requests.get, + 200, + url=location, + headers={"host": self.hdfs_ip}, + verify=False, + auth=self.kerberos_auth, + ) if universal_newlines: return response_data.text @@ -126,23 +191,38 @@ class HDFSApi(object): return response_data.content def write_data(self, path, content): - logging.debug("write_data protocol:{} host:{} port:{} path: {} user:{}, principal:{}".format( - self.protocol, self.host, self.proxy_port, path, self.user, self.principal)) - named_file = NamedTemporaryFile(mode='wb+') + logging.debug( + "write_data protocol:{} host:{} port:{} path: {} user:{}, principal:{}".format( + self.protocol, + self.host, + self.proxy_port, + path, + self.user, + self.principal, + ) + ) + named_file = NamedTemporaryFile(mode="wb+") fpath = named_file.name if isinstance(content, str): content = content.encode() named_file.write(content) named_file.flush() - response = self.req_wrapper(requests.put, 307, - url="{protocol}://{ip}:{port}/webhdfs/v1{path}?op=CREATE".format(protocol=self.protocol, ip=self.hdfs_ip, - port=self.proxy_port, - path=path, user=self.user), + response = self.req_wrapper( + requests.put, + 307, + url="{protocol}://{ip}:{port}/webhdfs/v1{path}?op=CREATE".format( + protocol=self.protocol, + ip=self.hdfs_ip, + port=self.proxy_port, + path=path, + user=self.user, + ), allow_redirects=False, - headers={'host': str(self.hdfs_ip)}, - params={'overwrite' : 'true'}, - verify=False, auth=self.kerberos_auth + headers={"host": str(self.hdfs_ip)}, + params={"overwrite": "true"}, + verify=False, + auth=self.kerberos_auth, ) logging.debug("HDFS api response:{}".format(response.headers)) @@ -150,23 +230,30 @@ class HDFSApi(object): # additional_params = '&'.join( # response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) if self.kerberized: - location = response.headers['Location'].replace("kerberizedhdfs1:1006", "{}:{}".format(self.hdfs_ip, self.data_port)) + location = response.headers["Location"].replace( + "kerberizedhdfs1:1006", "{}:{}".format(self.hdfs_ip, self.data_port) + ) else: - location = response.headers['Location'].replace("hdfs1:50075", "{}:{}".format(self.hdfs_ip, self.data_port)) + location = response.headers["Location"].replace( + "hdfs1:50075", "{}:{}".format(self.hdfs_ip, self.data_port) + ) with open(fpath, mode="rb") as fh: file_data = fh.read() - protocol = "http" # self.protocol - response = self.req_wrapper(requests.put, 201, + protocol = "http" # self.protocol + response = self.req_wrapper( + requests.put, + 201, url="{location}".format(location=location), data=file_data, - headers={'content-type':'text/plain', 'host': str(self.hdfs_ip)}, - params={'file': path, 'user.name' : self.user}, - allow_redirects=False, verify=False, auth=self.kerberos_auth + headers={"content-type": "text/plain", "host": str(self.hdfs_ip)}, + params={"file": path, "user.name": self.user}, + allow_redirects=False, + verify=False, + auth=self.kerberos_auth, ) logging.debug(f"{response.content} {response.headers}") - def write_gzip_data(self, path, content): if isinstance(content, str): content = content.encode() @@ -176,4 +263,10 @@ class HDFSApi(object): self.write_data(path, out.getvalue()) def read_gzip_data(self, path): - return gzip.GzipFile(fileobj=io.BytesIO(self.read_data(path, universal_newlines=False))).read().decode() + return ( + gzip.GzipFile( + fileobj=io.BytesIO(self.read_data(path, universal_newlines=False)) + ) + .read() + .decode() + ) diff --git a/tests/integration/helpers/http_server.py b/tests/integration/helpers/http_server.py index e62096dd33f..3f32c2be775 100644 --- a/tests/integration/helpers/http_server.py +++ b/tests/integration/helpers/http_server.py @@ -9,9 +9,14 @@ from http.server import BaseHTTPRequestHandler, HTTPServer # Decorator used to see if authentication works for external dictionary who use a HTTP source. def check_auth(fn): def wrapper(req): - auth_header = req.headers.get('authorization', None) - api_key = req.headers.get('api-key', None) - if not auth_header or auth_header != 'Basic Zm9vOmJhcg==' or not api_key or api_key != 'secret': + auth_header = req.headers.get("authorization", None) + api_key = req.headers.get("api-key", None) + if ( + not auth_header + or auth_header != "Basic Zm9vOmJhcg==" + or not api_key + or api_key != "secret" + ): req.send_response(401) else: fn(req) @@ -35,15 +40,15 @@ def start_server(server_address, data_path, schema, cert_path, address_family): def __send_headers(self): self.send_response(200) - self.send_header('Content-type', 'text/tsv') + self.send_header("Content-type", "text/tsv") self.end_headers() def __send_data(self, only_ids=None): - with open(data_path, 'r') as fl: - reader = csv.reader(fl, delimiter='\t') + with open(data_path, "r") as fl: + reader = csv.reader(fl, delimiter="\t") for row in reader: if not only_ids or (row[0] in only_ids): - self.wfile.write(('\t'.join(row) + '\n').encode()) + self.wfile.write(("\t".join(row) + "\n").encode()) def __read_and_decode_post_ids(self): data = self.__read_and_decode_post_data() @@ -51,7 +56,7 @@ def start_server(server_address, data_path, schema, cert_path, address_family): def __read_and_decode_post_data(self): transfer_encoding = self.headers.get("Transfer-encoding") - decoded = ""; + decoded = "" if transfer_encoding == "chunked": while True: s = self.rfile.readline().decode() @@ -69,19 +74,29 @@ def start_server(server_address, data_path, schema, cert_path, address_family): HTTPServer.address_family = socket.AF_INET6 httpd = HTTPServer(server_address, TSVHTTPHandler) if schema == "https": - httpd.socket = ssl.wrap_socket(httpd.socket, certfile=cert_path, server_side=True) + httpd.socket = ssl.wrap_socket( + httpd.socket, certfile=cert_path, server_side=True + ) httpd.serve_forever() if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple HTTP server returns data from file") + parser = argparse.ArgumentParser( + description="Simple HTTP server returns data from file" + ) parser.add_argument("--host", default="localhost") parser.add_argument("--port", default=5555, type=int) parser.add_argument("--data-path", required=True) parser.add_argument("--schema", choices=("http", "https"), required=True) parser.add_argument("--cert-path", default="./fake_cert.pem") - parser.add_argument('--address-family', choices=("ipv4", "ipv6"), default="ipv4") + parser.add_argument("--address-family", choices=("ipv4", "ipv6"), default="ipv4") args = parser.parse_args() - start_server((args.host, args.port), args.data_path, args.schema, args.cert_path, args.address_family) + start_server( + (args.host, args.port), + args.data_path, + args.schema, + args.cert_path, + args.address_family, + ) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 2bf0867c847..63fb2065f9d 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -22,26 +22,38 @@ class PartitionManager: self._netem_delayed_instances = [] _NetworkManager.get() - def drop_instance_zk_connections(self, instance, action='DROP'): + def drop_instance_zk_connections(self, instance, action="DROP"): self._check_instance(instance) - self._add_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action}) - self._add_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action}) + self._add_rule( + {"source": instance.ip_address, "destination_port": 2181, "action": action} + ) + self._add_rule( + {"destination": instance.ip_address, "source_port": 2181, "action": action} + ) - def restore_instance_zk_connections(self, instance, action='DROP'): + def restore_instance_zk_connections(self, instance, action="DROP"): self._check_instance(instance) - self._delete_rule({'source': instance.ip_address, 'destination_port': 2181, 'action': action}) - self._delete_rule({'destination': instance.ip_address, 'source_port': 2181, 'action': action}) + self._delete_rule( + {"source": instance.ip_address, "destination_port": 2181, "action": action} + ) + self._delete_rule( + {"destination": instance.ip_address, "source_port": 2181, "action": action} + ) - def partition_instances(self, left, right, port=None, action='DROP'): + def partition_instances(self, left, right, port=None, action="DROP"): self._check_instance(left) self._check_instance(right) def create_rule(src, dst): - rule = {'source': src.ip_address, 'destination': dst.ip_address, 'action': action} + rule = { + "source": src.ip_address, + "destination": dst.ip_address, + "action": action, + } if port is not None: - rule['destination_port'] = port + rule["destination_port"] = port return rule self._add_rule(create_rule(left, right)) @@ -57,7 +69,9 @@ class PartitionManager: while self._netem_delayed_instances: instance = self._netem_delayed_instances.pop() - instance.exec_in_container(["bash", "-c", "tc qdisc del dev eth0 root netem"], user="root") + instance.exec_in_container( + ["bash", "-c", "tc qdisc del dev eth0 root netem"], user="root" + ) def pop_rules(self): res = self._iptables_rules[:] @@ -71,7 +85,7 @@ class PartitionManager: @staticmethod def _check_instance(instance): if instance.ip_address is None: - raise Exception('Instance + ' + instance.name + ' is not launched!') + raise Exception("Instance + " + instance.name + " is not launched!") def _add_rule(self, rule): _NetworkManager.get().add_iptables_rule(**rule) @@ -82,7 +96,14 @@ class PartitionManager: self._iptables_rules.remove(rule) def _add_tc_netem_delay(self, instance, delay_ms): - instance.exec_in_container(["bash", "-c", "tc qdisc add dev eth0 root netem delay {}ms".format(delay_ms)], user="root") + instance.exec_in_container( + [ + "bash", + "-c", + "tc qdisc add dev eth0 root netem delay {}ms".format(delay_ms), + ], + user="root", + ) self._netem_delayed_instances.append(instance) def __enter__(self): @@ -127,12 +148,12 @@ class _NetworkManager: return cls._instance def add_iptables_rule(self, **kwargs): - cmd = ['iptables', '--wait', '-I', 'DOCKER-USER', '1'] + cmd = ["iptables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) def delete_iptables_rule(self, **kwargs): - cmd = ['iptables', '--wait', '-D', 'DOCKER-USER'] + cmd = ["iptables", "--wait", "-D", "DOCKER-USER"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) @@ -144,40 +165,66 @@ class _NetworkManager: res = subprocess.run("iptables --wait -D DOCKER-USER 1", shell=True) if res.returncode != 0: - logging.info("All iptables rules cleared, " + str(iptables_iter) + " iterations, last error: " + str(res.stderr)) + logging.info( + "All iptables rules cleared, " + + str(iptables_iter) + + " iterations, last error: " + + str(res.stderr) + ) return @staticmethod def _iptables_cmd_suffix( - source=None, destination=None, - source_port=None, destination_port=None, - action=None, probability=None, custom_args=None): + source=None, + destination=None, + source_port=None, + destination_port=None, + action=None, + probability=None, + custom_args=None, + ): ret = [] if probability is not None: - ret.extend(['-m', 'statistic', '--mode', 'random', '--probability', str(probability)]) - ret.extend(['-p', 'tcp']) + ret.extend( + [ + "-m", + "statistic", + "--mode", + "random", + "--probability", + str(probability), + ] + ) + ret.extend(["-p", "tcp"]) if source is not None: - ret.extend(['-s', source]) + ret.extend(["-s", source]) if destination is not None: - ret.extend(['-d', destination]) + ret.extend(["-d", destination]) if source_port is not None: - ret.extend(['--sport', str(source_port)]) + ret.extend(["--sport", str(source_port)]) if destination_port is not None: - ret.extend(['--dport', str(destination_port)]) + ret.extend(["--dport", str(destination_port)]) if action is not None: - ret.extend(['-j'] + action.split()) + ret.extend(["-j"] + action.split()) if custom_args is not None: ret.extend(custom_args) return ret def __init__( - self, - container_expire_timeout=50, container_exit_timeout=60, docker_api_version=os.environ.get("DOCKER_API_VERSION")): + self, + container_expire_timeout=50, + container_exit_timeout=60, + docker_api_version=os.environ.get("DOCKER_API_VERSION"), + ): self.container_expire_timeout = container_expire_timeout self.container_exit_timeout = container_exit_timeout - self._docker_client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version=docker_api_version, timeout=600) + self._docker_client = docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=docker_api_version, + timeout=600, + ) self._container = None @@ -194,29 +241,41 @@ class _NetworkManager: except docker.errors.NotFound: break except Exception as ex: - print("Error removing network blocade container, will try again", str(ex)) + print( + "Error removing network blocade container, will try again", + str(ex), + ) time.sleep(i) - image = subprocess.check_output("docker images -q clickhouse/integration-helper 2>/dev/null", shell=True) + image = subprocess.check_output( + "docker images -q clickhouse/integration-helper 2>/dev/null", shell=True + ) if not image.strip(): print("No network image helper, will try download") # for some reason docker api may hang if image doesn't exist, so we download it # before running for i in range(5): try: - subprocess.check_call("docker pull clickhouse/integration-helper", shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + "docker pull clickhouse/integration-helper", shell=True + ) break except: time.sleep(i) else: raise Exception("Cannot pull clickhouse/integration-helper image") - self._container = self._docker_client.containers.run('clickhouse/integration-helper', - auto_remove=True, - command=('sleep %s' % self.container_exit_timeout), - # /run/xtables.lock passed inside for correct iptables --wait - volumes={'/run/xtables.lock': {'bind': '/run/xtables.lock', 'mode': 'ro' }}, - detach=True, network_mode='host') + self._container = self._docker_client.containers.run( + "clickhouse/integration-helper", + auto_remove=True, + command=("sleep %s" % self.container_exit_timeout), + # /run/xtables.lock passed inside for correct iptables --wait + volumes={ + "/run/xtables.lock": {"bind": "/run/xtables.lock", "mode": "ro"} + }, + detach=True, + network_mode="host", + ) container_id = self._container.id self._container_expire_time = time.time() + self.container_expire_timeout @@ -233,8 +292,8 @@ class _NetworkManager: container = self._ensure_container() handle = self._docker_client.api.exec_create(container.id, cmd, **kwargs) - output = self._docker_client.api.exec_start(handle).decode('utf8') - exit_code = self._docker_client.api.exec_inspect(handle)['ExitCode'] + output = self._docker_client.api.exec_start(handle).decode("utf8") + exit_code = self._docker_client.api.exec_inspect(handle)["ExitCode"] if exit_code != 0: print(output) @@ -242,30 +301,56 @@ class _NetworkManager: return output + # Approximately mesure network I/O speed for interface class NetThroughput(object): def __init__(self, node): self.node = node # trying to get default interface and check it in /proc/net/dev - self.interface = self.node.exec_in_container(["bash", "-c", "awk '{print $1 \" \" $2}' /proc/net/route | grep 00000000 | awk '{print $1}'"]).strip() - check = self.node.exec_in_container(["bash", "-c", f'grep "^ *{self.interface}:" /proc/net/dev']).strip() - if not check: # if check is not successful just try eth{1-10} + self.interface = self.node.exec_in_container( + [ + "bash", + "-c", + "awk '{print $1 \" \" $2}' /proc/net/route | grep 00000000 | awk '{print $1}'", + ] + ).strip() + check = self.node.exec_in_container( + ["bash", "-c", f'grep "^ *{self.interface}:" /proc/net/dev'] + ).strip() + if not check: # if check is not successful just try eth{1-10} for i in range(10): try: - self.interface = self.node.exec_in_container(["bash", "-c", f"awk '{{print $1}}' /proc/net/route | grep 'eth{i}'"]).strip() + self.interface = self.node.exec_in_container( + [ + "bash", + "-c", + f"awk '{{print $1}}' /proc/net/route | grep 'eth{i}'", + ] + ).strip() break except Exception as ex: print(f"No interface eth{i}") else: - raise Exception("No interface eth{1-10} and default interface not specified in /proc/net/route, maybe some special network configuration") + raise Exception( + "No interface eth{1-10} and default interface not specified in /proc/net/route, maybe some special network configuration" + ) try: - check = self.node.exec_in_container(["bash", "-c", f'grep "^ *{self.interface}:" /proc/net/dev']).strip() + check = self.node.exec_in_container( + ["bash", "-c", f'grep "^ *{self.interface}:" /proc/net/dev'] + ).strip() if not check: - raise Exception(f"No such interface {self.interface} found in /proc/net/dev") + raise Exception( + f"No such interface {self.interface} found in /proc/net/dev" + ) except: - logging.error("All available interfaces %s", self.node.exec_in_container(["bash", "-c", "cat /proc/net/dev"])) - raise Exception(f"No such interface {self.interface} found in /proc/net/dev") + logging.error( + "All available interfaces %s", + self.node.exec_in_container(["bash", "-c", "cat /proc/net/dev"]), + ) + raise Exception( + f"No such interface {self.interface} found in /proc/net/dev" + ) self.current_in = self._get_in_bytes() self.current_out = self._get_out_bytes() @@ -273,27 +358,47 @@ class NetThroughput(object): def _get_in_bytes(self): try: - result = self.node.exec_in_container(['bash', '-c', f'awk "/^ *{self.interface}:/"\' {{ if ($1 ~ /.*:[0-9][0-9]*/) {{ sub(/^.*:/, "") ; print $1 }} else {{ print $2 }} }}\' /proc/net/dev']) + result = self.node.exec_in_container( + [ + "bash", + "-c", + f'awk "/^ *{self.interface}:/"\' {{ if ($1 ~ /.*:[0-9][0-9]*/) {{ sub(/^.*:/, "") ; print $1 }} else {{ print $2 }} }}\' /proc/net/dev', + ] + ) except: - raise Exception(f"Cannot receive in bytes from /proc/net/dev for interface {self.interface}") + raise Exception( + f"Cannot receive in bytes from /proc/net/dev for interface {self.interface}" + ) try: return int(result) except: - raise Exception(f"Got non-numeric in bytes '{result}' from /proc/net/dev for interface {self.interface}") + raise Exception( + f"Got non-numeric in bytes '{result}' from /proc/net/dev for interface {self.interface}" + ) def _get_out_bytes(self): try: - result = self.node.exec_in_container(['bash', '-c', f'awk "/^ *{self.interface}:/"\' {{ if ($1 ~ /.*:[0-9][0-9]*/) {{ print $9 }} else {{ print $10 }} }}\' /proc/net/dev']) + result = self.node.exec_in_container( + [ + "bash", + "-c", + f"awk \"/^ *{self.interface}:/\"' {{ if ($1 ~ /.*:[0-9][0-9]*/) {{ print $9 }} else {{ print $10 }} }}' /proc/net/dev", + ] + ) except: - raise Exception(f"Cannot receive out bytes from /proc/net/dev for interface {self.interface}") + raise Exception( + f"Cannot receive out bytes from /proc/net/dev for interface {self.interface}" + ) try: return int(result) except: - raise Exception(f"Got non-numeric out bytes '{result}' from /proc/net/dev for interface {self.interface}") + raise Exception( + f"Got non-numeric out bytes '{result}' from /proc/net/dev for interface {self.interface}" + ) - def measure_speed(self, measure='bytes'): + def measure_speed(self, measure="bytes"): new_in = self._get_in_bytes() new_out = self._get_out_bytes() current_time = time.time() @@ -304,11 +409,11 @@ class NetThroughput(object): self.current_in = new_in self.measure_time = current_time - if measure == 'bytes': + if measure == "bytes": return in_speed, out_speed - elif measure == 'kilobytes': - return in_speed / 1024., out_speed / 1024. - elif measure == 'megabytes': + elif measure == "kilobytes": + return in_speed / 1024.0, out_speed / 1024.0 + elif measure == "megabytes": return in_speed / (1024 * 1024), out_speed / (1024 * 1024) else: raise Exception(f"Unknown measure {measure}") diff --git a/tests/integration/helpers/postgres_utility.py b/tests/integration/helpers/postgres_utility.py index 16461ea3310..978b9a98fb4 100644 --- a/tests/integration/helpers/postgres_utility.py +++ b/tests/integration/helpers/postgres_utility.py @@ -23,11 +23,21 @@ postgres_table_template_5 = """ key Integer NOT NULL, value UUID, PRIMARY KEY(key)) """ -def get_postgres_conn(ip, port, database=False, auto_commit=True, database_name='postgres_database', replication=False): + +def get_postgres_conn( + ip, + port, + database=False, + auto_commit=True, + database_name="postgres_database", + replication=False, +): if database == True: conn_string = f"host={ip} port={port} dbname='{database_name}' user='postgres' password='mysecretpassword'" else: - conn_string = f"host={ip} port={port} user='postgres' password='mysecretpassword'" + conn_string = ( + f"host={ip} port={port} user='postgres' password='mysecretpassword'" + ) if replication: conn_string += " replication='database'" @@ -38,33 +48,41 @@ def get_postgres_conn(ip, port, database=False, auto_commit=True, database_name= conn.autocommit = True return conn -def create_replication_slot(conn, slot_name='user_slot'): + +def create_replication_slot(conn, slot_name="user_slot"): cursor = conn.cursor() - cursor.execute(f'CREATE_REPLICATION_SLOT {slot_name} LOGICAL pgoutput EXPORT_SNAPSHOT') + cursor.execute( + f"CREATE_REPLICATION_SLOT {slot_name} LOGICAL pgoutput EXPORT_SNAPSHOT" + ) result = cursor.fetchall() - print(result[0][0]) # slot name - print(result[0][1]) # start lsn - print(result[0][2]) # snapshot + print(result[0][0]) # slot name + print(result[0][1]) # start lsn + print(result[0][2]) # snapshot return result[0][2] -def drop_replication_slot(conn, slot_name='user_slot'): + +def drop_replication_slot(conn, slot_name="user_slot"): cursor = conn.cursor() cursor.execute(f"select pg_drop_replication_slot('{slot_name}')") def create_postgres_schema(cursor, schema_name): drop_postgres_schema(cursor, schema_name) - cursor.execute(f'CREATE SCHEMA {schema_name}') + cursor.execute(f"CREATE SCHEMA {schema_name}") + def drop_postgres_schema(cursor, schema_name): - cursor.execute(f'DROP SCHEMA IF EXISTS {schema_name} CASCADE') + cursor.execute(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE") -def create_postgres_table(cursor, table_name, replica_identity_full=False, template=postgres_table_template): +def create_postgres_table( + cursor, table_name, replica_identity_full=False, template=postgres_table_template +): drop_postgres_table(cursor, table_name) cursor.execute(template.format(table_name)) if replica_identity_full: - cursor.execute(f'ALTER TABLE {table_name} REPLICA IDENTITY FULL;') + cursor.execute(f"ALTER TABLE {table_name} REPLICA IDENTITY FULL;") + def drop_postgres_table(cursor, table_name): cursor.execute(f"""DROP TABLE IF EXISTS "{table_name}" """) @@ -74,6 +92,7 @@ def create_postgres_table_with_schema(cursor, schema_name, table_name): drop_postgres_table_with_schema(cursor, schema_name, table_name) cursor.execute(postgres_table_template_4.format(schema_name, table_name)) + def drop_postgres_table_with_schema(cursor, schema_name, table_name): cursor.execute(f"""DROP TABLE IF EXISTS "{schema_name}"."{table_name}" """) @@ -102,14 +121,14 @@ class PostgresManager: def prepare(self): conn = get_postgres_conn(ip=self.ip, port=self.port) cursor = conn.cursor() - self.create_postgres_db(cursor, 'postgres_database') + self.create_postgres_db(cursor, "postgres_database") self.create_clickhouse_postgres_db(ip=self.ip, port=self.port) def clear(self): if self.conn.closed == 0: self.conn.close() for db in self.created_materialized_postgres_db_list.copy(): - self.drop_materialized_db(db); + self.drop_materialized_db(db) for db in self.created_ch_postgres_db_list.copy(): self.drop_clickhouse_postgres_db(db) if len(self.created_postgres_db_list) > 0: @@ -122,38 +141,54 @@ class PostgresManager: self.conn = get_postgres_conn(ip=self.ip, port=self.port, database=True) return self.conn.cursor() - def create_postgres_db(self, cursor, name='postgres_database'): + def create_postgres_db(self, cursor, name="postgres_database"): self.drop_postgres_db(cursor, name) self.created_postgres_db_list.add(name) cursor.execute(f"CREATE DATABASE {name}") - def drop_postgres_db(self, cursor, name='postgres_database'): + def drop_postgres_db(self, cursor, name="postgres_database"): cursor.execute(f"DROP DATABASE IF EXISTS {name}") if name in self.created_postgres_db_list: self.created_postgres_db_list.remove(name) - def create_clickhouse_postgres_db(self, ip, port, name='postgres_database', database_name='postgres_database', schema_name=''): + def create_clickhouse_postgres_db( + self, + ip, + port, + name="postgres_database", + database_name="postgres_database", + schema_name="", + ): self.drop_clickhouse_postgres_db(name) self.created_ch_postgres_db_list.add(name) if len(schema_name) == 0: - self.instance.query(f''' + self.instance.query( + f""" CREATE DATABASE {name} - ENGINE = PostgreSQL('{ip}:{port}', '{database_name}', 'postgres', 'mysecretpassword')''') + ENGINE = PostgreSQL('{ip}:{port}', '{database_name}', 'postgres', 'mysecretpassword')""" + ) else: - self.instance.query(f''' + self.instance.query( + f""" CREATE DATABASE {name} - ENGINE = PostgreSQL('{ip}:{port}', '{database_name}', 'postgres', 'mysecretpassword', '{schema_name}')''') + ENGINE = PostgreSQL('{ip}:{port}', '{database_name}', 'postgres', 'mysecretpassword', '{schema_name}')""" + ) - def drop_clickhouse_postgres_db(self, name='postgres_database'): - self.instance.query(f'DROP DATABASE IF EXISTS {name}') + def drop_clickhouse_postgres_db(self, name="postgres_database"): + self.instance.query(f"DROP DATABASE IF EXISTS {name}") if name in self.created_ch_postgres_db_list: self.created_ch_postgres_db_list.remove(name) - - def create_materialized_db(self, ip, port, - materialized_database='test_database', postgres_database='postgres_database', - settings=[], table_overrides=''): + def create_materialized_db( + self, + ip, + port, + materialized_database="test_database", + postgres_database="postgres_database", + settings=[], + table_overrides="", + ): self.created_materialized_postgres_db_list.add(materialized_database) self.instance.query(f"DROP DATABASE IF EXISTS {materialized_database}") @@ -162,17 +197,17 @@ class PostgresManager: create_query += " SETTINGS " for i in range(len(settings)): if i != 0: - create_query += ', ' + create_query += ", " create_query += settings[i] create_query += table_overrides self.instance.query(create_query) - assert materialized_database in self.instance.query('SHOW DATABASES') + assert materialized_database in self.instance.query("SHOW DATABASES") - def drop_materialized_db(self, materialized_database='test_database'): - self.instance.query(f'DROP DATABASE IF EXISTS {materialized_database} NO DELAY') + def drop_materialized_db(self, materialized_database="test_database"): + self.instance.query(f"DROP DATABASE IF EXISTS {materialized_database} NO DELAY") if materialized_database in self.created_materialized_postgres_db_list: self.created_materialized_postgres_db_list.remove(materialized_database) - assert materialized_database not in self.instance.query('SHOW DATABASES') + assert materialized_database not in self.instance.query("SHOW DATABASES") def create_and_fill_postgres_table(self, table_name): conn = get_postgres_conn(ip=self.ip, port=self.port, database=True) @@ -180,82 +215,109 @@ class PostgresManager: self.create_and_fill_postgres_table_from_cursor(cursor, table_name) def create_and_fill_postgres_table_from_cursor(self, cursor, table_name): - create_postgres_table(cursor, table_name); - self.instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(50)") + create_postgres_table(cursor, table_name) + self.instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(50)" + ) def create_and_fill_postgres_tables(self, tables_num, numbers=50): conn = get_postgres_conn(ip=self.ip, port=self.port, database=True) cursor = conn.cursor() - self.create_and_fill_postgres_tables_from_cursor(cursor, tables_num, numbers=numbers) + self.create_and_fill_postgres_tables_from_cursor( + cursor, tables_num, numbers=numbers + ) - def create_and_fill_postgres_tables_from_cursor(self, cursor, tables_num, numbers=50): + def create_and_fill_postgres_tables_from_cursor( + self, cursor, tables_num, numbers=50 + ): for i in range(tables_num): - table_name = f'postgresql_replica_{i}' - create_postgres_table(cursor, table_name); + table_name = f"postgresql_replica_{i}" + create_postgres_table(cursor, table_name) if numbers > 0: - self.instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers({numbers})") + self.instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers({numbers})" + ) queries = [ - 'INSERT INTO postgresql_replica_{} select i, i from generate_series(0, 10000) as t(i);', - 'DELETE FROM postgresql_replica_{} WHERE (value*value) % 3 = 0;', - 'UPDATE postgresql_replica_{} SET value = value - 125 WHERE key % 2 = 0;', + "INSERT INTO postgresql_replica_{} select i, i from generate_series(0, 10000) as t(i);", + "DELETE FROM postgresql_replica_{} WHERE (value*value) % 3 = 0;", + "UPDATE postgresql_replica_{} SET value = value - 125 WHERE key % 2 = 0;", "UPDATE postgresql_replica_{} SET key=key+20000 WHERE key%2=0", - 'INSERT INTO postgresql_replica_{} select i, i from generate_series(40000, 50000) as t(i);', - 'DELETE FROM postgresql_replica_{} WHERE key % 10 = 0;', - 'UPDATE postgresql_replica_{} SET value = value + 101 WHERE key % 2 = 1;', + "INSERT INTO postgresql_replica_{} select i, i from generate_series(40000, 50000) as t(i);", + "DELETE FROM postgresql_replica_{} WHERE key % 10 = 0;", + "UPDATE postgresql_replica_{} SET value = value + 101 WHERE key % 2 = 1;", "UPDATE postgresql_replica_{} SET key=key+80000 WHERE key%2=1", - 'DELETE FROM postgresql_replica_{} WHERE value % 2 = 0;', - 'UPDATE postgresql_replica_{} SET value = value + 2000 WHERE key % 5 = 0;', - 'INSERT INTO postgresql_replica_{} select i, i from generate_series(200000, 250000) as t(i);', - 'DELETE FROM postgresql_replica_{} WHERE value % 3 = 0;', - 'UPDATE postgresql_replica_{} SET value = value * 2 WHERE key % 3 = 0;', + "DELETE FROM postgresql_replica_{} WHERE value % 2 = 0;", + "UPDATE postgresql_replica_{} SET value = value + 2000 WHERE key % 5 = 0;", + "INSERT INTO postgresql_replica_{} select i, i from generate_series(200000, 250000) as t(i);", + "DELETE FROM postgresql_replica_{} WHERE value % 3 = 0;", + "UPDATE postgresql_replica_{} SET value = value * 2 WHERE key % 3 = 0;", "UPDATE postgresql_replica_{} SET key=key+500000 WHERE key%2=1", - 'INSERT INTO postgresql_replica_{} select i, i from generate_series(1000000, 1050000) as t(i);', - 'DELETE FROM postgresql_replica_{} WHERE value % 9 = 2;', + "INSERT INTO postgresql_replica_{} select i, i from generate_series(1000000, 1050000) as t(i);", + "DELETE FROM postgresql_replica_{} WHERE value % 9 = 2;", "UPDATE postgresql_replica_{} SET key=key+10000000", - 'UPDATE postgresql_replica_{} SET value = value + 2 WHERE key % 3 = 1;', - 'DELETE FROM postgresql_replica_{} WHERE value%5 = 0;' - ] + "UPDATE postgresql_replica_{} SET value = value + 2 WHERE key % 3 = 1;", + "DELETE FROM postgresql_replica_{} WHERE value%5 = 0;", +] -def assert_nested_table_is_created(instance, table_name, materialized_database='test_database', schema_name=''): +def assert_nested_table_is_created( + instance, table_name, materialized_database="test_database", schema_name="" +): if len(schema_name) == 0: table = table_name else: table = schema_name + "." + table_name - print(f'Checking table {table} exists in {materialized_database}') - database_tables = instance.query(f'SHOW TABLES FROM {materialized_database}') + print(f"Checking table {table} exists in {materialized_database}") + database_tables = instance.query(f"SHOW TABLES FROM {materialized_database}") while table not in database_tables: time.sleep(0.2) - database_tables = instance.query(f'SHOW TABLES FROM {materialized_database}') + database_tables = instance.query(f"SHOW TABLES FROM {materialized_database}") - assert(table in database_tables) + assert table in database_tables -def assert_number_of_columns(instance, expected, table_name, database_name='test_database'): - result = instance.query(f"select count() from system.columns where table = '{table_name}' and database = '{database_name}' and not startsWith(name, '_')") - while (int(result) != expected): +def assert_number_of_columns( + instance, expected, table_name, database_name="test_database" +): + result = instance.query( + f"select count() from system.columns where table = '{table_name}' and database = '{database_name}' and not startsWith(name, '_')" + ) + while int(result) != expected: time.sleep(1) - result = instance.query(f"select count() from system.columns where table = '{table_name}' and database = '{database_name}' and not startsWith(name, '_')") - print('Number of columns ok') + result = instance.query( + f"select count() from system.columns where table = '{table_name}' and database = '{database_name}' and not startsWith(name, '_')" + ) + print("Number of columns ok") -def check_tables_are_synchronized(instance, table_name, order_by='key', postgres_database='postgres_database', materialized_database='test_database', schema_name=''): - assert_nested_table_is_created(instance, table_name, materialized_database, schema_name) +def check_tables_are_synchronized( + instance, + table_name, + order_by="key", + postgres_database="postgres_database", + materialized_database="test_database", + schema_name="", +): + assert_nested_table_is_created( + instance, table_name, materialized_database, schema_name + ) - table_path = '' + table_path = "" if len(schema_name) == 0: - table_path = f'{materialized_database}.{table_name}' + table_path = f"{materialized_database}.{table_name}" else: - table_path = f'{materialized_database}.`{schema_name}.{table_name}`' + table_path = f"{materialized_database}.`{schema_name}.{table_name}`" print(f"Checking table is synchronized: {table_path}") - result_query = f'select * from {table_path} order by {order_by};' + result_query = f"select * from {table_path} order by {order_by};" - expected = instance.query(f'select * from {postgres_database}.{table_name} order by {order_by};') + expected = instance.query( + f"select * from {postgres_database}.{table_name} order by {order_by};" + ) result = instance.query(result_query) for _ in range(30): @@ -265,9 +327,16 @@ def check_tables_are_synchronized(instance, table_name, order_by='key', postgres time.sleep(0.5) result = instance.query(result_query) - assert(result == expected) + assert result == expected -def check_several_tables_are_synchronized(instance, tables_num, order_by='key', postgres_database='postgres_database', materialized_database='test_database', schema_name=''): +def check_several_tables_are_synchronized( + instance, + tables_num, + order_by="key", + postgres_database="postgres_database", + materialized_database="test_database", + schema_name="", +): for i in range(tables_num): - check_tables_are_synchronized(instance, f'postgresql_replica_{i}'); + check_tables_are_synchronized(instance, f"postgresql_replica_{i}") diff --git a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py index ee9a52e042c..d424ad58fa4 100644 --- a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py +++ b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py @@ -5,17 +5,17 @@ import os.path # Without this function all workers will log to the same log file # and mix everything together making it much more difficult for troubleshooting. def setup(): - worker_name = os.environ.get('PYTEST_XDIST_WORKER', 'master') - if worker_name == 'master': + worker_name = os.environ.get("PYTEST_XDIST_WORKER", "master") + if worker_name == "master": return - logger = logging.getLogger('') + logger = logging.getLogger("") new_handlers = [] handlers_to_remove = [] for handler in logger.handlers: if isinstance(handler, logging.FileHandler): filename, ext = os.path.splitext(handler.baseFilename) - if not filename.endswith('-' + worker_name): - new_filename = filename + '-' + worker_name + if not filename.endswith("-" + worker_name): + new_filename = filename + "-" + worker_name new_handler = logging.FileHandler(new_filename + ext) new_handler.setFormatter(handler.formatter) new_handler.setLevel(handler.level) diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index ec3841f79d7..2afbae340be 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -13,12 +13,18 @@ class TSV: elif isinstance(contents, str) or isinstance(contents, str): raw_lines = contents.splitlines(True) elif isinstance(contents, list): - raw_lines = ['\t'.join(map(str, l)) if isinstance(l, list) else str(l) for l in contents] + raw_lines = [ + "\t".join(map(str, l)) if isinstance(l, list) else str(l) + for l in contents + ] elif isinstance(contents, TSV): self.lines = contents.lines return else: - raise TypeError("contents must be either file or string or list, actual type: " + type(contents).__name__) + raise TypeError( + "contents must be either file or string or list, actual type: " + + type(contents).__name__ + ) self.lines = [l.strip() for l in raw_lines if l.strip()] def __eq__(self, other): @@ -31,13 +37,18 @@ class TSV: return self != TSV(other) return self.lines != other.lines - def diff(self, other, n1='', n2=''): + def diff(self, other, n1="", n2=""): if not isinstance(other, TSV): return self.diff(TSV(other), n1=n1, n2=n2) - return list(line.rstrip() for line in difflib.unified_diff(self.lines, other.lines, fromfile=n1, tofile=n2))[2:] + return list( + line.rstrip() + for line in difflib.unified_diff( + self.lines, other.lines, fromfile=n1, tofile=n2 + ) + )[2:] def __str__(self): - return '\n'.join(self.lines) + return "\n".join(self.lines) def __repr__(self): return self.__str__() @@ -50,29 +61,70 @@ class TSV: return [line.split("\t") for line in contents.split("\n") if line.strip()] -def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_time=0.5, stdin=None, timeout=None, - settings=None, user=None, ignore_error=False, get_result=lambda x: x): +def assert_eq_with_retry( + instance, + query, + expectation, + retry_count=20, + sleep_time=0.5, + stdin=None, + timeout=None, + settings=None, + user=None, + ignore_error=False, + get_result=lambda x: x, +): expectation_tsv = TSV(expectation) for i in range(retry_count): try: - if TSV(get_result(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, - ignore_error=ignore_error))) == expectation_tsv: + if ( + TSV( + get_result( + instance.query( + query, + user=user, + stdin=stdin, + timeout=timeout, + settings=settings, + ignore_error=ignore_error, + ) + ) + ) + == expectation_tsv + ): break time.sleep(sleep_time) except Exception as ex: logging.exception(f"assert_eq_with_retry retry {i+1} exception {ex}") time.sleep(sleep_time) else: - val = TSV(get_result(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, - ignore_error=ignore_error))) + val = TSV( + get_result( + instance.query( + query, + user=user, + stdin=stdin, + timeout=timeout, + settings=settings, + ignore_error=ignore_error, + ) + ) + ) if expectation_tsv != val: - raise AssertionError("'{}' != '{}'\n{}".format(expectation_tsv, val, '\n'.join( - expectation_tsv.diff(val, n1="expectation", n2="query")))) + raise AssertionError( + "'{}' != '{}'\n{}".format( + expectation_tsv, + val, + "\n".join(expectation_tsv.diff(val, n1="expectation", n2="query")), + ) + ) + def assert_logs_contain(instance, substring): if not instance.contains_in_log(substring): raise AssertionError("'{}' not found in logs".format(substring)) + def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_time=0.5): for i in range(retry_count): try: @@ -85,7 +137,10 @@ def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_ti else: raise AssertionError("'{}' not found in logs".format(substring)) -def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, silent=False, settings={}): + +def exec_query_with_retry( + instance, query, retry_count=40, sleep_time=0.5, silent=False, settings={} +): exception = None for cnt in range(retry_count): try: @@ -96,16 +151,21 @@ def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, silen except Exception as ex: exception = ex if not silent: - logging.exception(f"Failed to execute query '{query}' on {cnt} try on instance '{instance.name}' will retry") + logging.exception( + f"Failed to execute query '{query}' on {cnt} try on instance '{instance.name}' will retry" + ) time.sleep(sleep_time) else: raise exception + def csv_compare(result, expected): csv_result = TSV(result) csv_expected = TSV(expected) mismatch = [] - max_len = len(csv_result) if len(csv_result) > len(csv_expected) else len(csv_expected) + max_len = ( + len(csv_result) if len(csv_result) > len(csv_expected) else len(csv_expected) + ) for i in range(max_len): if i >= len(csv_result): mismatch.append("-[%d]=%s" % (i, csv_expected.lines[i])) diff --git a/tests/integration/helpers/uclient.py b/tests/integration/helpers/uclient.py index 538722580af..45c8b8f64e2 100644 --- a/tests/integration/helpers/uclient.py +++ b/tests/integration/helpers/uclient.py @@ -8,30 +8,30 @@ sys.path.insert(0, os.path.join(CURDIR)) from . import uexpect -prompt = ':\) ' -end_of_block = r'.*\r\n.*\r\n' +prompt = ":\) " +end_of_block = r".*\r\n.*\r\n" class client(object): - def __init__(self, command=None, name='', log=None): - self.client = uexpect.spawn(['/bin/bash', '--noediting']) + def __init__(self, command=None, name="", log=None): + self.client = uexpect.spawn(["/bin/bash", "--noediting"]) if command is None: - command = '/usr/bin/clickhouse-client' + command = "/usr/bin/clickhouse-client" self.client.command = command - self.client.eol('\r') + self.client.eol("\r") self.client.logger(log, prefix=name) self.client.timeout(20) - self.client.expect('[#\$] ', timeout=2) + self.client.expect("[#\$] ", timeout=2) self.client.send(command) def __enter__(self): return self.client.__enter__() def __exit__(self, type, value, traceback): - self.client.reader['kill_event'].set() + self.client.reader["kill_event"].set() # send Ctrl-C - self.client.send('\x03', eol='') + self.client.send("\x03", eol="") time.sleep(0.3) - self.client.send('quit', eol='\r') - self.client.send('\x03', eol='') + self.client.send("quit", eol="\r") + self.client.send("\x03", eol="") return self.client.__exit__(type, value, traceback) diff --git a/tests/integration/helpers/uexpect.py b/tests/integration/helpers/uexpect.py index cd26e3ddbd3..757a3a7f199 100644 --- a/tests/integration/helpers/uexpect.py +++ b/tests/integration/helpers/uexpect.py @@ -25,7 +25,7 @@ class TimeoutError(Exception): self.timeout = timeout def __str__(self): - return 'Timeout %.3fs' % float(self.timeout) + return "Timeout %.3fs" % float(self.timeout) class ExpectTimeoutError(Exception): @@ -35,12 +35,12 @@ class ExpectTimeoutError(Exception): self.buffer = buffer def __str__(self): - s = 'Timeout %.3fs ' % float(self.timeout) + s = "Timeout %.3fs " % float(self.timeout) if self.pattern: - s += 'for %s ' % repr(self.pattern.pattern) + s += "for %s " % repr(self.pattern.pattern) if self.buffer: - s += 'buffer %s ' % repr(self.buffer[:]) - s += 'or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) + s += "buffer %s " % repr(self.buffer[:]) + s += "or '%s'" % ",".join(["%x" % ord(c) for c in self.buffer[:]]) return s @@ -55,12 +55,12 @@ class IO(object): TIMEOUT = Timeout class Logger(object): - def __init__(self, logger, prefix=''): + def __init__(self, logger, prefix=""): self._logger = logger self._prefix = prefix def write(self, data): - self._logger.write(('\n' + data).replace('\n', '\n' + self._prefix)) + self._logger.write(("\n" + data).replace("\n", "\n" + self._prefix)) def flush(self): self._logger.flush() @@ -77,7 +77,7 @@ class IO(object): self.reader = reader self._timeout = None self._logger = None - self._eol = '' + self._eol = "" def __enter__(self): return self @@ -85,7 +85,7 @@ class IO(object): def __exit__(self, type, value, traceback): self.close() - def logger(self, logger=None, prefix=''): + def logger(self, logger=None, prefix=""): if logger: self._logger = self.Logger(logger, prefix=prefix) return self._logger @@ -101,15 +101,15 @@ class IO(object): return self._eol def close(self, force=True): - self.reader['kill_event'].set() - os.system('pkill -TERM -P %d' % self.process.pid) + self.reader["kill_event"].set() + os.system("pkill -TERM -P %d" % self.process.pid) if force: self.process.kill() else: self.process.terminate() os.close(self.master) if self._logger: - self._logger.write('\n') + self._logger.write("\n") self._logger.flush() def send(self, data, eol=None): @@ -135,9 +135,9 @@ class IO(object): if self.buffer is not None: self.match = pattern.search(self.buffer, 0) if self.match is not None: - self.after = self.buffer[self.match.start():self.match.end()] - self.before = self.buffer[:self.match.start()] - self.buffer = self.buffer[self.match.end():] + self.after = self.buffer[self.match.start() : self.match.end()] + self.before = self.buffer[: self.match.start()] + self.buffer = self.buffer[self.match.end() :] break if timeleft < 0: break @@ -145,16 +145,16 @@ class IO(object): data = self.read(timeout=timeleft, raise_exception=True) except TimeoutError: if self._logger: - self._logger.write((self.buffer or '') + '\n') + self._logger.write((self.buffer or "") + "\n") self._logger.flush() exception = ExpectTimeoutError(pattern, timeout, self.buffer) self.buffer = None raise exception - timeleft -= (time.time() - start_time) + timeleft -= time.time() - start_time if data: self.buffer = (self.buffer + data) if self.buffer else data if self._logger: - self._logger.write((self.before or '') + (self.after or '')) + self._logger.write((self.before or "") + (self.after or "")) self._logger.flush() if self.match is None: exception = ExpectTimeoutError(pattern, timeout, self.buffer) @@ -163,7 +163,7 @@ class IO(object): return self.match def read(self, timeout=0, raise_exception=False): - data = '' + data = "" timeleft = timeout try: while timeleft >= 0: @@ -171,7 +171,7 @@ class IO(object): data += self.queue.get(timeout=timeleft) if data: break - timeleft -= (time.time() - start_time) + timeleft -= time.time() - start_time except Empty: if data: return data @@ -186,7 +186,14 @@ class IO(object): def spawn(command): master, slave = pty.openpty() - process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) + process = Popen( + command, + preexec_fn=os.setsid, + stdout=slave, + stdin=slave, + stderr=slave, + bufsize=1, + ) os.close(slave) queue = Queue() @@ -195,14 +202,19 @@ def spawn(command): thread.daemon = True thread.start() - return IO(process, master, queue, reader={'thread': thread, 'kill_event': reader_kill_event}) + return IO( + process, + master, + queue, + reader={"thread": thread, "kill_event": reader_kill_event}, + ) def reader(process, out, queue, kill_event): while True: try: # TODO: there are some issues with 1<<16 buffer size - data = os.read(out, 1<<17).decode(errors='replace') + data = os.read(out, 1 << 17).decode(errors="replace") queue.put(data) except: if kill_event.is_set(): diff --git a/tests/integration/helpers/utility.py b/tests/integration/helpers/utility.py index 69dfa53cd3e..0fd55569d92 100644 --- a/tests/integration/helpers/utility.py +++ b/tests/integration/helpers/utility.py @@ -11,11 +11,13 @@ class SafeThread(threading.Thread): super().__init__() self.target = target self.exception = None + def run(self): try: self.target() - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except self.exception = e + def join(self, timeout=None): super().join(timeout) if self.exception: @@ -24,7 +26,7 @@ class SafeThread(threading.Thread): def random_string(length): letters = string.ascii_letters - return ''.join(random.choice(letters) for i in range(length)) + return "".join(random.choice(letters) for i in range(length)) def generate_values(date_str, count, sign=1): @@ -34,10 +36,10 @@ def generate_values(date_str, count, sign=1): def replace_config(config_path, old, new): - config = open(config_path, 'r') + config = open(config_path, "r") config_lines = config.readlines() config.close() config_lines = [line.replace(old, new) for line in config_lines] - config = open(config_path, 'w') + config = open(config_path, "w") config.writelines(config_lines) config.close() diff --git a/tests/integration/runner b/tests/integration/runner index 3687ca4068c..737eaeef683 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -238,6 +238,8 @@ if __name__ == "__main__": env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag) elif image == "clickhouse/integration-test": env_tags += "-e {}={} ".format("DOCKER_BASE_TAG", tag) + elif image == "clickhouse/kerberized-hadoop": + env_tags += "-e {}={} ".format("DOCKER_KERBERIZED_HADOOP_TAG", tag) elif image == "clickhouse/kerberos-kdc": env_tags += "-e {}={} ".format("DOCKER_KERBEROS_KDC_TAG", tag) else: diff --git a/tests/integration/test_MemoryTracking/test.py b/tests/integration/test_MemoryTracking/test.py index 2ec5b2457af..517090988ee 100644 --- a/tests/integration/test_MemoryTracking/test.py +++ b/tests/integration/test_MemoryTracking/test.py @@ -19,14 +19,19 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=[ - 'configs/no_system_log.xml', - 'configs/asynchronous_metrics_update_period_s.xml', -], user_configs=[ - 'configs/users.d/overrides.xml', -]) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/no_system_log.xml", + "configs/asynchronous_metrics_update_period_s.xml", + ], + user_configs=[ + "configs/users.d/overrides.xml", + ], +) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -34,31 +39,39 @@ def start_cluster(): finally: cluster.shutdown() + query_settings = { - 'max_threads': 1, - 'log_queries': 0, + "max_threads": 1, + "log_queries": 0, } sample_query = "SELECT groupArray(repeat('a', 1000)) FROM numbers(10000) GROUP BY number%10 FORMAT JSON" + def query(*args, **kwargs): - if 'settings' not in kwargs: - kwargs['settings'] = query_settings + if "settings" not in kwargs: + kwargs["settings"] = query_settings else: - kwargs['settings'].update(query_settings) + kwargs["settings"].update(query_settings) return node.query(*args, **kwargs) + + def http_query(*args, **kwargs): - if 'params' not in kwargs: - kwargs['params'] = query_settings + if "params" not in kwargs: + kwargs["params"] = query_settings else: - kwargs['params'].update(query_settings) + kwargs["params"].update(query_settings) return node.http_query(*args, **kwargs) + def get_MemoryTracking(): - return int(http_query("SELECT value FROM system.metrics WHERE metric = 'MemoryTracking'")) + return int( + http_query("SELECT value FROM system.metrics WHERE metric = 'MemoryTracking'") + ) + def check_memory(memory): # bytes -> megabytes - memory = [*map(lambda x: int(int(x)/1024/1024), memory)] + memory = [*map(lambda x: int(int(x) / 1024 / 1024), memory)] # 3 changes to MemoryTracking is minimum, since: # - this is not that high to not detect inacuracy # - memory can go like X/X+N due to some background allocations @@ -66,14 +79,19 @@ def check_memory(memory): changes_allowed = 3 # if number of samples is large enough, use 10% from them # (actually most of the time there will be only few changes, it was made 10% to avoid flackiness) - changes_allowed_auto=int(len(memory) * 0.1) + changes_allowed_auto = int(len(memory) * 0.1) changes_allowed = max(changes_allowed_auto, changes_allowed) - changed=len(set(memory)) - logging.info('Changes: allowed=%s, actual=%s, sample=%s', - changes_allowed, changed, len(memory)) + changed = len(set(memory)) + logging.info( + "Changes: allowed=%s, actual=%s, sample=%s", + changes_allowed, + changed, + len(memory), + ) assert changed < changes_allowed + def test_http(): memory = [] memory.append(get_MemoryTracking()) @@ -82,6 +100,7 @@ def test_http(): memory.append(get_MemoryTracking()) check_memory(memory) + def test_tcp_multiple_sessions(): memory = [] memory.append(get_MemoryTracking()) @@ -90,6 +109,7 @@ def test_tcp_multiple_sessions(): memory.append(get_MemoryTracking()) check_memory(memory) + def test_tcp_single_session(): memory = [] memory.append(get_MemoryTracking()) @@ -97,9 +117,9 @@ def test_tcp_single_session(): sample_query, "SELECT metric, value FROM system.metrics WHERE metric = 'MemoryTracking'", ] * 100 - rows = query(';'.join(sample_queries)) - memory = rows.split('\n') - memory = filter(lambda x: x.startswith('MemoryTracking'), memory) - memory = map(lambda x: x.split('\t')[1], memory) + rows = query(";".join(sample_queries)) + memory = rows.split("\n") + memory = filter(lambda x: x.startswith("MemoryTracking"), memory) + memory = map(lambda x: x.split("\t")[1], memory) memory = [*memory] check_memory(memory) diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 6bcf67779ef..6c2331178e0 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -2,9 +2,15 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) -ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) -ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) +ch1 = cluster.add_instance( + "ch1", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True +) +ch2 = cluster.add_instance( + "ch2", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True +) +ch3 = cluster.add_instance( + "ch3", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True +) @pytest.fixture(scope="module", autouse=True) @@ -18,17 +24,23 @@ def started_cluster(): def test_access_control_on_cluster(): - ch1.query_with_retry("CREATE USER IF NOT EXISTS Alex ON CLUSTER 'cluster'", retry_count=5) + ch1.query_with_retry( + "CREATE USER IF NOT EXISTS Alex ON CLUSTER 'cluster'", retry_count=5 + ) assert ch1.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" assert ch2.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" assert ch3.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" - ch2.query_with_retry("GRANT ON CLUSTER 'cluster' SELECT ON *.* TO Alex", retry_count=3) + ch2.query_with_retry( + "GRANT ON CLUSTER 'cluster' SELECT ON *.* TO Alex", retry_count=3 + ) assert ch1.query("SHOW GRANTS FOR Alex") == "GRANT SELECT ON *.* TO Alex\n" assert ch2.query("SHOW GRANTS FOR Alex") == "GRANT SELECT ON *.* TO Alex\n" assert ch3.query("SHOW GRANTS FOR Alex") == "GRANT SELECT ON *.* TO Alex\n" - ch3.query_with_retry("REVOKE ON CLUSTER 'cluster' SELECT ON *.* FROM Alex", retry_count=3) + ch3.query_with_retry( + "REVOKE ON CLUSTER 'cluster' SELECT ON *.* FROM Alex", retry_count=3 + ) assert ch1.query("SHOW GRANTS FOR Alex") == "" assert ch2.query("SHOW GRANTS FOR Alex") == "" assert ch3.query("SHOW GRANTS FOR Alex") == "" diff --git a/tests/integration/test_access_for_functions/test.py b/tests/integration/test_access_for_functions/test.py index ebd0f6bd907..be4d71502d2 100644 --- a/tests/integration/test_access_for_functions/test.py +++ b/tests/integration/test_access_for_functions/test.py @@ -1,8 +1,9 @@ import pytest +import uuid from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance", stay_alive=True) @pytest.fixture(scope="module", autouse=True) @@ -14,26 +15,62 @@ def started_cluster(): finally: cluster.shutdown() -def test_access_rights_for_funtion(): + +def test_access_rights_for_function(): create_function_query = "CREATE FUNCTION MySum AS (a, b) -> a + b" instance.query("CREATE USER A") instance.query("CREATE USER B") - assert "it's necessary to have grant CREATE FUNCTION ON *.*" in instance.query_and_get_error(create_function_query, user = 'A') + assert ( + "it's necessary to have grant CREATE FUNCTION ON *.*" + in instance.query_and_get_error(create_function_query, user="A") + ) instance.query("GRANT CREATE FUNCTION on *.* TO A") - instance.query(create_function_query, user = 'A') + instance.query(create_function_query, user="A") assert instance.query("SELECT MySum(1, 2)") == "3\n" - assert "it's necessary to have grant DROP FUNCTION ON *.*" in instance.query_and_get_error("DROP FUNCTION MySum", user = 'B') + assert ( + "it's necessary to have grant DROP FUNCTION ON *.*" + in instance.query_and_get_error("DROP FUNCTION MySum", user="B") + ) instance.query("GRANT DROP FUNCTION ON *.* TO B") - instance.query("DROP FUNCTION MySum", user = 'B') - assert "Unknown function MySum" in instance.query_and_get_error("SELECT MySum(1, 2)") + instance.query("DROP FUNCTION MySum", user="B") + assert "Unknown function MySum" in instance.query_and_get_error( + "SELECT MySum(1, 2)" + ) instance.query("REVOKE CREATE FUNCTION ON *.* FROM A") - assert "it's necessary to have grant CREATE FUNCTION ON *.*" in instance.query_and_get_error(create_function_query, user = 'A') + assert ( + "it's necessary to have grant CREATE FUNCTION ON *.*" + in instance.query_and_get_error(create_function_query, user="A") + ) instance.query("DROP USER IF EXISTS A") instance.query("DROP USER IF EXISTS B") + + +def test_ignore_obsolete_grant_on_database(): + instance.stop_clickhouse() + + user_id = uuid.uuid4() + instance.exec_in_container( + [ + "bash", + "-c", + f""" + cat > /var/lib/clickhouse/access/{user_id}.sql << EOF +ATTACH USER X; +ATTACH GRANT CREATE FUNCTION, SELECT ON mydb.* TO X; +EOF""", + ] + ) + + instance.exec_in_container( + ["bash", "-c", "touch /var/lib/clickhouse/access/need_rebuild_lists.mark"] + ) + instance.start_clickhouse() + + assert instance.query("SHOW GRANTS FOR X") == "GRANT SELECT ON mydb.* TO X\n" diff --git a/tests/integration/test_aggregation_memory_efficient/test.py b/tests/integration/test_aggregation_memory_efficient/test.py index db0449173ca..8131fd9c1d7 100644 --- a/tests/integration/test_aggregation_memory_efficient/test.py +++ b/tests/integration/test_aggregation_memory_efficient/test.py @@ -3,8 +3,8 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1') -node2 = cluster.add_instance('node2') +node1 = cluster.add_instance("node1") +node2 = cluster.add_instance("node2") @pytest.fixture(scope="module") @@ -14,10 +14,15 @@ def start_cluster(): for node in [node1, node2]: node.query( - "create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;") + "create table da_memory_efficient_shard(A Int64, B Int64) Engine=MergeTree order by A partition by B % 2;" + ) - node1.query("insert into da_memory_efficient_shard select number, number from numbers(100000);") - node2.query("insert into da_memory_efficient_shard select number + 100000, number from numbers(100000);") + node1.query( + "insert into da_memory_efficient_shard select number, number from numbers(100000);" + ) + node2.query( + "insert into da_memory_efficient_shard select number + 100000, number from numbers(100000);" + ) yield cluster @@ -27,23 +32,29 @@ def start_cluster(): def test_remote(start_cluster): node1.query( - "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1" + ) res = node1.query( - "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") - assert res == '200000\n' + "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)" + ) + assert res == "200000\n" node1.query("set distributed_aggregation_memory_efficient = 0") res = node1.query( - "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") - assert res == '200000\n' + "select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)" + ) + assert res == "200000\n" node1.query( - "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + "set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1" + ) res = node1.query( - "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") - assert res == 'node1\t100000\nnode2\t100000\n' + "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;" + ) + assert res == "node1\t100000\nnode2\t100000\n" node1.query("set distributed_aggregation_memory_efficient = 0") res = node1.query( - "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") - assert res == 'node1\t100000\nnode2\t100000\n' + "SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;" + ) + assert res == "node1\t100000\nnode2\t100000\n" diff --git a/tests/integration/test_allowed_client_hosts/test.py b/tests/integration/test_allowed_client_hosts/test.py index 7b803fd50f3..db2ba464b38 100644 --- a/tests/integration/test_allowed_client_hosts/test.py +++ b/tests/integration/test_allowed_client_hosts/test.py @@ -2,31 +2,42 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -server = cluster.add_instance('server', user_configs=["configs/users.d/network.xml"]) +server = cluster.add_instance("server", user_configs=["configs/users.d/network.xml"]) -clientA1 = cluster.add_instance('clientA1', hostname='clientA1.com') -clientA2 = cluster.add_instance('clientA2', hostname='clientA2.com') -clientA3 = cluster.add_instance('clientA3', hostname='clientA3.com') -clientB1 = cluster.add_instance('clientB1', hostname='clientB001.ru') -clientB2 = cluster.add_instance('clientB2', hostname='clientB002.ru') -clientB3 = cluster.add_instance('clientB3', hostname='xxx.clientB003.rutracker.com') -clientC1 = cluster.add_instance('clientC1', hostname='clientC01.ru') -clientC2 = cluster.add_instance('clientC2', hostname='xxx.clientC02.ru') -clientC3 = cluster.add_instance('clientC3', hostname='xxx.clientC03.rutracker.com') -clientD1 = cluster.add_instance('clientD1', hostname='clientD0001.ru') -clientD2 = cluster.add_instance('clientD2', hostname='xxx.clientD0002.ru') -clientD3 = cluster.add_instance('clientD3', hostname='clientD0003.ru') +clientA1 = cluster.add_instance("clientA1", hostname="clientA1.com") +clientA2 = cluster.add_instance("clientA2", hostname="clientA2.com") +clientA3 = cluster.add_instance("clientA3", hostname="clientA3.com") +clientB1 = cluster.add_instance("clientB1", hostname="clientB001.ru") +clientB2 = cluster.add_instance("clientB2", hostname="clientB002.ru") +clientB3 = cluster.add_instance("clientB3", hostname="xxx.clientB003.rutracker.com") +clientC1 = cluster.add_instance("clientC1", hostname="clientC01.ru") +clientC2 = cluster.add_instance("clientC2", hostname="xxx.clientC02.ru") +clientC3 = cluster.add_instance("clientC3", hostname="xxx.clientC03.rutracker.com") +clientD1 = cluster.add_instance("clientD1", hostname="clientD0001.ru") +clientD2 = cluster.add_instance("clientD2", hostname="xxx.clientD0002.ru") +clientD3 = cluster.add_instance("clientD3", hostname="clientD0003.ru") def check_clickhouse_is_ok(client_node, server_node): - assert client_node.exec_in_container( - ["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)]) == "Ok.\n" + assert ( + client_node.exec_in_container( + ["bash", "-c", "/usr/bin/curl -s {}:8123 ".format(server_node.hostname)] + ) + == "Ok.\n" + ) def query_from_one_node_to_another(client_node, server_node, query): check_clickhouse_is_ok(client_node, server_node) return client_node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.hostname, query)]) + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host {} --query {!r}".format( + server_node.hostname, query + ), + ] + ) def query(node, query): @@ -38,7 +49,10 @@ def setup_nodes(): try: cluster.start() query(server, "DROP TABLE IF EXISTS test_allowed_client_hosts") - query(server, "CREATE TABLE test_allowed_client_hosts (x Int32) ENGINE = MergeTree() ORDER BY tuple()") + query( + server, + "CREATE TABLE test_allowed_client_hosts (x Int32) ENGINE = MergeTree() ORDER BY tuple()", + ) query(server, "INSERT INTO test_allowed_client_hosts VALUES (5)") yield cluster @@ -58,8 +72,15 @@ def test_allowed_host(): # expected_to_fail.extend([clientC3, clientD2]) for client_node in expected_to_pass: - assert query_from_one_node_to_another(client_node, server, "SELECT * FROM test_allowed_client_hosts") == "5\n" + assert ( + query_from_one_node_to_another( + client_node, server, "SELECT * FROM test_allowed_client_hosts" + ) + == "5\n" + ) for client_node in expected_to_fail: - with pytest.raises(Exception, match=r'default: Authentication failed'): - query_from_one_node_to_another(client_node, server, "SELECT * FROM test_allowed_client_hosts") + with pytest.raises(Exception, match=r"default: Authentication failed"): + query_from_one_node_to_another( + client_node, server, "SELECT * FROM test_allowed_client_hosts" + ) diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 71bcea482f8..4f4f02fffdc 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -2,13 +2,23 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/config_with_only_primary_hosts.xml']) -node3 = cluster.add_instance('node3', main_configs=['configs/config_with_only_regexp_hosts.xml']) -node4 = cluster.add_instance('node4', main_configs=[]) # No `remote_url_allow_hosts` at all. -node5 = cluster.add_instance('node5', main_configs=['configs/config_without_allowed_hosts.xml']) -node6 = cluster.add_instance('node6', main_configs=['configs/config_for_remote.xml']) -node7 = cluster.add_instance('node7', main_configs=['configs/config_for_redirect.xml'], with_hdfs=True) +node1 = cluster.add_instance("node1", main_configs=["configs/config_with_hosts.xml"]) +node2 = cluster.add_instance( + "node2", main_configs=["configs/config_with_only_primary_hosts.xml"] +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/config_with_only_regexp_hosts.xml"] +) +node4 = cluster.add_instance( + "node4", main_configs=[] +) # No `remote_url_allow_hosts` at all. +node5 = cluster.add_instance( + "node5", main_configs=["configs/config_without_allowed_hosts.xml"] +) +node6 = cluster.add_instance("node6", main_configs=["configs/config_for_remote.xml"]) +node7 = cluster.add_instance( + "node7", main_configs=["configs/config_for_redirect.xml"], with_hdfs=True +) @pytest.fixture(scope="module") @@ -21,97 +31,229 @@ def start_cluster(): def test_config_with_hosts(start_cluster): - assert node1.query("CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)") == "" - assert node1.query("CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert ( + node1.query( + "CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)" + ) + == "" + ) + assert ( + node1.query( + "CREATE TABLE table_test_1_2 (word String) Engine=URL('https://yandex.ru', CSV)" + ) + == "" + ) assert "not allowed" in node1.query_and_get_error( - "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)") + "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)" + ) assert "not allowed" in node1.query_and_get_error( - "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)") + "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)" + ) def test_config_with_only_primary_hosts(start_cluster): - assert node2.query("CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)") == "" - assert node2.query("CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)") == "" - assert node2.query("CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" - assert node2.query("CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)") == "" + assert ( + node2.query( + "CREATE TABLE table_test_2_1 (word String) Engine=URL('https://host:80', CSV)" + ) + == "" + ) + assert ( + node2.query( + "CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)" + ) + == "" + ) + assert ( + node2.query( + "CREATE TABLE table_test_2_3 (word String) Engine=URL('https://yandex.ru', CSV)" + ) + == "" + ) + assert ( + node2.query( + "CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)" + ) + == "" + ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)") + "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)" + ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)") + "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)" + ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)") + "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)" + ) def test_config_with_only_regexp_hosts(start_cluster): - assert node3.query("CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)") == "" - assert node3.query("CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert ( + node3.query( + "CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)" + ) + == "" + ) + assert ( + node3.query( + "CREATE TABLE table_test_3_2 (word String) Engine=URL('https://yandex.ru', CSV)" + ) + == "" + ) assert "not allowed" in node3.query_and_get_error( - "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)") + "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)" + ) assert "not allowed" in node3.query_and_get_error( - "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)") + "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)" + ) def test_config_without_allowed_hosts_section(start_cluster): - assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == "" - assert node4.query("CREATE TABLE table_test_4_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)") == "" - assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', HDFS)") == "" - assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('https://yandex.ru', CSV)") == "" - assert node4.query("CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', S3)") == "" + assert ( + node4.query( + "CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)" + ) + == "" + ) + assert ( + node4.query( + "CREATE TABLE table_test_4_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)" + ) + == "" + ) + assert ( + node4.query( + "CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', HDFS)" + ) + == "" + ) + assert ( + node4.query( + "CREATE TABLE table_test_4_4 (word String) Engine=URL('https://yandex.ru', CSV)" + ) + == "" + ) + assert ( + node4.query( + "CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', S3)" + ) + == "" + ) def test_config_without_allowed_hosts(start_cluster): assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_1 (word String) Engine=URL('https://host:80', CSV)") + "CREATE TABLE table_test_5_1 (word String) Engine=URL('https://host:80', CSV)" + ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)") + "CREATE TABLE table_test_5_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)" + ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', HDFS)") + "CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', HDFS)" + ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_4 (word String) Engine=URL('https://yandex.ru', CSV)") + "CREATE TABLE table_test_5_4 (word String) Engine=URL('https://yandex.ru', CSV)" + ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', S3)") + "CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', S3)" + ) def test_table_function_remote(start_cluster): assert "not allowed in configuration file" not in node6.query_and_get_error( "SELECT * FROM remoteSecure('example01-01-{1|2}', system, events)", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert "not allowed in configuration file" not in node6.query_and_get_error( "SELECT * FROM remoteSecure('example01-01-1,example01-02-1', system, events)", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert "not allowed in configuration file" not in node6.query_and_get_error( "SELECT * FROM remote('example01-0{1,2}-1', system, events", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert "not allowed in configuration file" not in node6.query_and_get_error( "SELECT * FROM remote('example01-0{1,2}-{1|2}', system, events)", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert "not allowed in configuration file" not in node6.query_and_get_error( "SELECT * FROM remoteSecure('example01-{01..02}-{1|2}', system, events)", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert "not allowed" in node6.query_and_get_error( "SELECT * FROM remoteSecure('example01-01-1,example01-03-1', system, events)", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) - assert "not allowed" in node6.query_and_get_error("SELECT * FROM remote('example01-01-{1|3}', system, events)", - settings={"connections_with_failover_max_tries": 1, - "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, - "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) + assert "not allowed" in node6.query_and_get_error( + "SELECT * FROM remote('example01-01-{1|3}', system, events)", + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert "not allowed" in node6.query_and_get_error( "SELECT * FROM remoteSecure('example01-0{1,3}-1', system, metrics)", - settings={"connections_with_failover_max_tries": 1, "connect_timeout_with_failover_ms": 1000, - "connect_timeout_with_failover_secure_ms": 1000, "connect_timeout": 1, "send_timeout": 1}) + settings={ + "connections_with_failover_max_tries": 1, + "connect_timeout_with_failover_ms": 1000, + "connect_timeout_with_failover_secure_ms": 1000, + "connect_timeout": 1, + "send_timeout": 1, + }, + ) assert node6.query("SELECT * FROM remote('localhost', system, events)") != "" assert node6.query("SELECT * FROM remoteSecure('localhost', system, metrics)") != "" - assert "URL \"localhost:800\" is not allowed in configuration file" in node6.query_and_get_error( - "SELECT * FROM remoteSecure('localhost:800', system, events)") - assert "URL \"localhost:800\" is not allowed in configuration file" in node6.query_and_get_error( - "SELECT * FROM remote('localhost:800', system, metrics)") + assert ( + 'URL "localhost:800" is not allowed in configuration file' + in node6.query_and_get_error( + "SELECT * FROM remoteSecure('localhost:800', system, events)" + ) + ) + assert ( + 'URL "localhost:800" is not allowed in configuration file' + in node6.query_and_get_error( + "SELECT * FROM remote('localhost:800', system, metrics)" + ) + ) def test_redirect(start_cluster): @@ -120,12 +262,17 @@ def test_redirect(start_cluster): hdfs_api.write_data("/simple_storage", "1\t\n") assert hdfs_api.read_data("/simple_storage") == "1\t\n" node7.query( - "CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)") - assert "not allowed" in node7.query_and_get_error("SET max_http_get_redirects=1; SELECT * from table_test_7_1") + "CREATE TABLE table_test_7_1 (word String) ENGINE=URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', CSV)" + ) + assert "not allowed" in node7.query_and_get_error( + "SET max_http_get_redirects=1; SELECT * from table_test_7_1" + ) def test_HDFS(start_cluster): assert "not allowed" in node7.query_and_get_error( - "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')") + "CREATE TABLE table_test_7_2 (word String) ENGINE=HDFS('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'CSV')" + ) assert "not allowed" in node7.query_and_get_error( - "SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')") + "SELECT * FROM hdfs('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'word String')" + ) diff --git a/tests/integration/test_alter_codec/test.py b/tests/integration/test_alter_codec/test.py index 2117893af5b..7c7ef4803e9 100644 --- a/tests/integration/test_alter_codec/test.py +++ b/tests/integration/test_alter_codec/test.py @@ -4,8 +4,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml']) +node1 = cluster.add_instance("node1", main_configs=["configs/logs_config.xml"]) @pytest.fixture(scope="module") @@ -21,30 +20,60 @@ def started_cluster(): def test_alter_codec_pk(started_cluster): try: name = "test_alter_codec_pk" - node1.query(""" + node1.query( + """ CREATE TABLE {name} (id UInt64, value UInt64) Engine=MergeTree() ORDER BY id - """.format(name=name)) + """.format( + name=name + ) + ) - node1.query("INSERT INTO {name} SELECT number, number * number from numbers(100)".format(name=name)) + node1.query( + "INSERT INTO {name} SELECT number, number * number from numbers(100)".format( + name=name + ) + ) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(NONE)".format(name=name)) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(NONE)".format(name=name) + ) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(Delta, LZ4)".format( + name=name + ) + ) assert node1.query("SELECT sum(id) FROM {name}".format(name=name)) == "4950\n" with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format( + name=name + ) + ) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format( + name=name + ) + ) node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name)) assert node1.query("SELECT sum(id) FROM {name}".format(name=name)) == "4953\n" with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 ALIAS 3 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 ALIAS 3 CODEC(Delta, LZ4)".format( + name=name + ) + ) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 MATERIALIZED 3 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 MATERIALIZED 3 CODEC(Delta, LZ4)".format( + name=name + ) + ) node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name)) @@ -61,28 +90,58 @@ def test_alter_codec_pk(started_cluster): def test_alter_codec_index(started_cluster): try: name = "test_alter_codec_index" - node1.query(""" + node1.query( + """ CREATE TABLE {name} (`id` UInt64, value UInt64, INDEX id_index id TYPE minmax GRANULARITY 1) Engine=MergeTree() ORDER BY tuple() - """.format(name=name)) + """.format( + name=name + ) + ) - node1.query("INSERT INTO {name} SELECT number, number * number from numbers(100)".format(name=name)) + node1.query( + "INSERT INTO {name} SELECT number, number * number from numbers(100)".format( + name=name + ) + ) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(NONE)".format(name=name)) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(NONE)".format(name=name) + ) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 CODEC(Delta, LZ4)".format( + name=name + ) + ) with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt32 CODEC(Delta, LZ4)".format( + name=name + ) + ) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 DEFAULT 3 CODEC(Delta, LZ4)".format( + name=name + ) + ) node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name)) assert node1.query("SELECT sum(id) FROM {name}".format(name=name)) == "4953\n" with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 ALIAS 3 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 ALIAS 3 CODEC(Delta, LZ4)".format( + name=name + ) + ) - node1.query("ALTER TABLE {name} MODIFY COLUMN id UInt64 MATERIALIZED 3 CODEC(Delta, LZ4)".format(name=name)) + node1.query( + "ALTER TABLE {name} MODIFY COLUMN id UInt64 MATERIALIZED 3 CODEC(Delta, LZ4)".format( + name=name + ) + ) node1.query("INSERT INTO {name} (value) VALUES (1)".format(name=name)) diff --git a/tests/integration/test_alter_on_mixed_type_cluster/test.py b/tests/integration/test_alter_on_mixed_type_cluster/test.py index c22626cb379..f21a97d40e1 100644 --- a/tests/integration/test_alter_on_mixed_type_cluster/test.py +++ b/tests/integration/test_alter_on_mixed_type_cluster/test.py @@ -4,11 +4,18 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) - +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node4 = cluster.add_instance( + "node4", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -17,19 +24,31 @@ def started_cluster(): cluster.start() for node in [node1, node2]: - node.query_with_retry(''' + node.query_with_retry( + """ CREATE TABLE IF NOT EXISTS test_table_replicated(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}') ORDER BY id; - '''.format(replica=node.name)) - node.query_with_retry('''CREATE TABLE IF NOT EXISTS test_table(date Date, id UInt32, value Int32) ENGINE=MergeTree ORDER BY id''') + """.format( + replica=node.name + ) + ) + node.query_with_retry( + """CREATE TABLE IF NOT EXISTS test_table(date Date, id UInt32, value Int32) ENGINE=MergeTree ORDER BY id""" + ) for node in [node3, node4]: - node.query_with_retry(''' + node.query_with_retry( + """ CREATE TABLE IF NOT EXISTS test_table_replicated(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/1/someotable', '{replica}') ORDER BY id; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) - node.query_with_retry('''CREATE TABLE IF NOT EXISTS test_table(date Date, id UInt32, value Int32) ENGINE=MergeTree ORDER BY id''') + node.query_with_retry( + """CREATE TABLE IF NOT EXISTS test_table(date Date, id UInt32, value Int32) ENGINE=MergeTree ORDER BY id""" + ) yield cluster @@ -46,17 +65,23 @@ def test_alter_on_cluter_non_replicated(started_cluster): assert node3.query("SELECT COUNT() FROM test_table") == "1\n" assert node4.query("SELECT COUNT() FROM test_table") == "1\n" - node1.query("ALTER TABLE test_table ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN date DateTime") + node1.query( + "ALTER TABLE test_table ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN date DateTime" + ) - assert node1.query("SELECT date FROM test_table") == '2019-10-01 00:00:00\n' - assert node2.query("SELECT date FROM test_table") == '2019-10-01 00:00:00\n' - assert node3.query("SELECT date FROM test_table") == '2019-10-01 00:00:00\n' - assert node4.query("SELECT date FROM test_table") == '2019-10-01 00:00:00\n' + assert node1.query("SELECT date FROM test_table") == "2019-10-01 00:00:00\n" + assert node2.query("SELECT date FROM test_table") == "2019-10-01 00:00:00\n" + assert node3.query("SELECT date FROM test_table") == "2019-10-01 00:00:00\n" + assert node4.query("SELECT date FROM test_table") == "2019-10-01 00:00:00\n" - node3.query("ALTER TABLE test_table ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN value String") + node3.query( + "ALTER TABLE test_table ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN value String" + ) for node in [node1, node2, node3, node4]: - node.query("INSERT INTO test_table VALUES(toDateTime('2019-10-02 00:00:00'), 2, 'Hello')") + node.query( + "INSERT INTO test_table VALUES(toDateTime('2019-10-02 00:00:00'), 2, 'Hello')" + ) assert node1.query("SELECT COUNT() FROM test_table") == "2\n" assert node2.query("SELECT COUNT() FROM test_table") == "2\n" @@ -66,22 +91,40 @@ def test_alter_on_cluter_non_replicated(started_cluster): def test_alter_replicated_on_cluster(started_cluster): for node in [node1, node3]: - node.query("INSERT INTO test_table_replicated VALUES(toDate('2019-10-01'), 1, 1)") + node.query( + "INSERT INTO test_table_replicated VALUES(toDate('2019-10-01'), 1, 1)" + ) for node in [node2, node4]: node.query("SYSTEM SYNC REPLICA test_table_replicated", timeout=20) - node1.query("ALTER TABLE test_table_replicated ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN date DateTime", settings={"replication_alter_partitions_sync": "2"}) + node1.query( + "ALTER TABLE test_table_replicated ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN date DateTime", + settings={"replication_alter_partitions_sync": "2"}, + ) - assert node1.query("SELECT date FROM test_table_replicated") == '2019-10-01 00:00:00\n' - assert node2.query("SELECT date FROM test_table_replicated") == '2019-10-01 00:00:00\n' - assert node3.query("SELECT date FROM test_table_replicated") == '2019-10-01 00:00:00\n' - assert node4.query("SELECT date FROM test_table_replicated") == '2019-10-01 00:00:00\n' + assert ( + node1.query("SELECT date FROM test_table_replicated") == "2019-10-01 00:00:00\n" + ) + assert ( + node2.query("SELECT date FROM test_table_replicated") == "2019-10-01 00:00:00\n" + ) + assert ( + node3.query("SELECT date FROM test_table_replicated") == "2019-10-01 00:00:00\n" + ) + assert ( + node4.query("SELECT date FROM test_table_replicated") == "2019-10-01 00:00:00\n" + ) - node3.query_with_retry("ALTER TABLE test_table_replicated ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN value String", settings={"replication_alter_partitions_sync": "2"}) + node3.query_with_retry( + "ALTER TABLE test_table_replicated ON CLUSTER 'test_cluster_mixed' MODIFY COLUMN value String", + settings={"replication_alter_partitions_sync": "2"}, + ) for node in [node2, node4]: - node.query("INSERT INTO test_table_replicated VALUES(toDateTime('2019-10-02 00:00:00'), 2, 'Hello')") + node.query( + "INSERT INTO test_table_replicated VALUES(toDateTime('2019-10-02 00:00:00'), 2, 'Hello')" + ) for node in [node1, node3]: node.query("SYSTEM SYNC REPLICA test_table_replicated", timeout=20) diff --git a/tests/integration/test_alter_update_cast_keep_nullable/test.py b/tests/integration/test_alter_update_cast_keep_nullable/test.py index 497a9e21d94..71735888d69 100644 --- a/tests/integration/test_alter_update_cast_keep_nullable/test.py +++ b/tests/integration/test_alter_update_cast_keep_nullable/test.py @@ -3,7 +3,10 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=['configs/users.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", user_configs=["configs/users.xml"], with_zookeeper=True +) + @pytest.fixture(scope="module") def started_cluster(): @@ -13,24 +16,30 @@ def started_cluster(): finally: cluster.shutdown() -def test_cast_keep_nullable(started_cluster): - setting = node1.query("SELECT value FROM system.settings WHERE name='cast_keep_nullable'") - assert(setting.strip() == "1") - result = node1.query(""" +def test_cast_keep_nullable(started_cluster): + setting = node1.query( + "SELECT value FROM system.settings WHERE name='cast_keep_nullable'" + ) + assert setting.strip() == "1" + + result = node1.query( + """ DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO t SELECT number FROM numbers(10); SELECT * FROM t; - """) - assert(result.strip() == "0\n1\n2\n3\n4\n5\n6\n7\n8\n9") + """ + ) + assert result.strip() == "0\n1\n2\n3\n4\n5\n6\n7\n8\n9" - error = node1.query_and_get_error(""" + error = node1.query_and_get_error( + """ SET mutations_sync = 1; ALTER TABLE t UPDATE x = x % 3 = 0 ? NULL : x WHERE x % 2 = 1;　 - """) - assert("DB::Exception: Cannot convert NULL value to non-Nullable type" in error) + """ + ) + assert "DB::Exception: Cannot convert NULL value to non-Nullable type" in error result = node1.query("SELECT * FROM t;") - assert(result.strip() == "0\n1\n2\n3\n4\n5\n6\n7\n8\n9") - + assert result.strip() == "0\n1\n2\n3\n4\n5\n6\n7\n8\n9" diff --git a/tests/integration/test_always_fetch_merged/test.py b/tests/integration/test_always_fetch_merged/test.py index e3b2d5ca392..ca8e775fb97 100644 --- a/tests/integration/test_always_fetch_merged/test.py +++ b/tests/integration/test_always_fetch_merged/test.py @@ -6,8 +6,8 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) @pytest.fixture(scope="module") @@ -22,21 +22,25 @@ def started_cluster(): def test_replica_always_download(started_cluster): - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS test_table( key UInt64, value String ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table/replicated', '1') ORDER BY tuple() - """) - node2.query_with_retry(""" + """ + ) + node2.query_with_retry( + """ CREATE TABLE IF NOT EXISTS test_table( key UInt64, value String ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table/replicated', '2') ORDER BY tuple() SETTINGS always_fetch_merged_part=1 - """) + """ + ) # Stop merges on single node node1.query("SYSTEM STOP MERGES") @@ -50,15 +54,29 @@ def test_replica_always_download(started_cluster): time.sleep(5) # Nothing is merged - assert node1.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1") == "10\n" - assert node2.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1") == "10\n" + assert ( + node1.query( + "SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1" + ) + == "10\n" + ) + assert ( + node2.query( + "SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1" + ) + == "10\n" + ) node1.query("SYSTEM START MERGES") node1.query("OPTIMIZE TABLE test_table") node2.query("SYSTEM SYNC REPLICA test_table") - node1_parts = node1.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() - node2_parts = node2.query("SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1").strip() + node1_parts = node1.query( + "SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1" + ).strip() + node2_parts = node2.query( + "SELECT COUNT() FROM system.parts WHERE table = 'test_table' and active=1" + ).strip() assert int(node1_parts) < 10 assert int(node2_parts) < 10 diff --git a/tests/integration/test_async_drain_connection/test.py b/tests/integration/test_async_drain_connection/test.py index 40d78ebbe7c..66786f4a8f9 100644 --- a/tests/integration/test_async_drain_connection/test.py +++ b/tests/integration/test_async_drain_connection/test.py @@ -5,17 +5,19 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/config.xml']) +node = cluster.add_instance("node", main_configs=["configs/config.xml"]) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node.query(""" + node.query( + """ create table t (number UInt64) engine = Distributed(test_cluster_two_shards, system, numbers) - """) + """ + ) yield cluster finally: @@ -25,12 +27,15 @@ def started_cluster(): def test_filled_async_drain_connection_pool(started_cluster): def execute_queries(_): for _ in range(100): - node.query('select * from t where number = 0 limit 2', settings={ - 'sleep_in_receive_cancel_ms': int(10e6), - 'max_execution_time': 5, - # decrease drain_timeout to make test more stable - # (another way is to increase max_execution_time, but this will make test slower) - 'drain_timeout': 1, - }) + node.query( + "select * from t where number = 0 limit 2", + settings={ + "sleep_in_receive_cancel_ms": int(10e6), + "max_execution_time": 5, + # decrease drain_timeout to make test more stable + # (another way is to increase max_execution_time, but this will make test slower) + "drain_timeout": 1, + }, + ) any(map(execute_queries, range(10))) diff --git a/tests/integration/test_asynchronous_metric_log_table/test.py b/tests/integration/test_asynchronous_metric_log_table/test.py index 0091832aa7c..96de7daf9e1 100644 --- a/tests/integration/test_asynchronous_metric_log_table/test.py +++ b/tests/integration/test_asynchronous_metric_log_table/test.py @@ -4,8 +4,11 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, - main_configs=['configs/asynchronous_metrics_update_period_s.xml']) +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + main_configs=["configs/asynchronous_metrics_update_period_s.xml"], +) @pytest.fixture(scope="module") @@ -27,20 +30,20 @@ def test_event_time_microseconds_field(started_cluster): cluster.start() node1.query("SET log_queries = 1;") node1.query("CREATE DATABASE replica;") - query_create = '''CREATE TABLE replica.test + query_create = """CREATE TABLE replica.test ( id Int64, event_time DateTime ) Engine=MergeTree() PARTITION BY toYYYYMMDD(event_time) - ORDER BY id;''' + ORDER BY id;""" time.sleep(2) node1.query(query_create) - node1.query('''INSERT INTO replica.test VALUES (1, now())''') + node1.query("""INSERT INTO replica.test VALUES (1, now())""") node1.query("SYSTEM FLUSH LOGS;") # query assumes that the event_time field is accurate - equals_query = '''WITH ( + equals_query = """WITH ( ( SELECT event_time_microseconds FROM system.asynchronous_metric_log @@ -53,7 +56,7 @@ def test_event_time_microseconds_field(started_cluster): ORDER BY event_time DESC LIMIT 1 ) AS time) - SELECT if(dateDiff('second', toDateTime(time_with_microseconds), toDateTime(time)) = 0, 'ok', 'fail')''' + SELECT if(dateDiff('second', toDateTime(time_with_microseconds), toDateTime(time)) = 0, 'ok', 'fail')""" assert "ok\n" in node1.query(equals_query) finally: cluster.shutdown() diff --git a/tests/integration/test_atomic_drop_table/test.py b/tests/integration/test_atomic_drop_table/test.py index dc1ad47aa75..1fe88dde099 100644 --- a/tests/integration/test_atomic_drop_table/test.py +++ b/tests/integration/test_atomic_drop_table/test.py @@ -5,21 +5,29 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/config.d/zookeeper_session_timeout.xml", - "configs/remote_servers.xml"], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/config.d/zookeeper_session_timeout.xml", + "configs/remote_servers.xml", + ], + with_zookeeper=True, +) @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() - node1.query("CREATE DATABASE zktest ENGINE=Ordinary;") # Different behaviour with Atomic node1.query( - ''' + "CREATE DATABASE zktest ENGINE=Ordinary;" + ) # Different behaviour with Atomic + node1.query( + """ CREATE TABLE zktest.atomic_drop_table (n UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/zktest/tables/atomic_drop_table', 'node1') PARTITION BY n ORDER BY n - ''' + """ ) yield cluster finally: @@ -31,8 +39,10 @@ def test_atomic_delete_with_stopped_zookeeper(start_cluster): with PartitionManager() as pm: pm.drop_instance_zk_connections(node1) - error = node1.query_and_get_error("DROP TABLE zktest.atomic_drop_table") # Table won't drop + error = node1.query_and_get_error( + "DROP TABLE zktest.atomic_drop_table" + ) # Table won't drop assert error != "" time.sleep(5) - assert '8192' in node1.query("select * from zktest.atomic_drop_table") + assert "8192" in node1.query("select * from zktest.atomic_drop_table") diff --git a/tests/integration/test_attach_partition_with_large_destination/test.py b/tests/integration/test_attach_partition_with_large_destination/test.py index 50f24f7a01e..0a4ab9fada1 100644 --- a/tests/integration/test_attach_partition_with_large_destination/test.py +++ b/tests/integration/test_attach_partition_with_large_destination/test.py @@ -3,7 +3,9 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.xml"], with_zookeeper=True) +node = cluster.add_instance( + "node", main_configs=["configs/config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -14,18 +16,35 @@ def started_cluster(): finally: cluster.shutdown() + def create_force_drop_flag(node): force_drop_flag_path = "/var/lib/clickhouse/flags/force_drop_table" - node.exec_in_container(["bash", "-c", "touch {} && chmod a=rw {}".format(force_drop_flag_path, force_drop_flag_path)], user="root") + node.exec_in_container( + [ + "bash", + "-c", + "touch {} && chmod a=rw {}".format( + force_drop_flag_path, force_drop_flag_path + ), + ], + user="root", + ) -@pytest.mark.parametrize("engine", ['Ordinary', 'Atomic']) + +@pytest.mark.parametrize("engine", ["Ordinary", "Atomic"]) def test_attach_partition_with_large_destination(started_cluster, engine): # Initialize node.query("CREATE DATABASE db ENGINE={}".format(engine)) - node.query("CREATE TABLE db.destination (n UInt64) ENGINE=ReplicatedMergeTree('/test/destination', 'r1') ORDER BY n PARTITION BY n % 2") - node.query("CREATE TABLE db.source_1 (n UInt64) ENGINE=ReplicatedMergeTree('/test/source_1', 'r1') ORDER BY n PARTITION BY n % 2") + node.query( + "CREATE TABLE db.destination (n UInt64) ENGINE=ReplicatedMergeTree('/test/destination', 'r1') ORDER BY n PARTITION BY n % 2" + ) + node.query( + "CREATE TABLE db.source_1 (n UInt64) ENGINE=ReplicatedMergeTree('/test/source_1', 'r1') ORDER BY n PARTITION BY n % 2" + ) node.query("INSERT INTO db.source_1 VALUES (1), (2), (3), (4)") - node.query("CREATE TABLE db.source_2 (n UInt64) ENGINE=ReplicatedMergeTree('/test/source_2', 'r1') ORDER BY n PARTITION BY n % 2") + node.query( + "CREATE TABLE db.source_2 (n UInt64) ENGINE=ReplicatedMergeTree('/test/source_2', 'r1') ORDER BY n PARTITION BY n % 2" + ) node.query("INSERT INTO db.source_2 VALUES (5), (6), (7), (8)") # Attach partition when destination partition is empty @@ -33,7 +52,9 @@ def test_attach_partition_with_large_destination(started_cluster, engine): assert node.query("SELECT n FROM db.destination ORDER BY n") == "2\n4\n" # REPLACE PARTITION should still respect max_partition_size_to_drop - assert node.query_and_get_error("ALTER TABLE db.destination REPLACE PARTITION 0 FROM db.source_2") + assert node.query_and_get_error( + "ALTER TABLE db.destination REPLACE PARTITION 0 FROM db.source_2" + ) assert node.query("SELECT n FROM db.destination ORDER BY n") == "2\n4\n" # Attach partition when destination partition is larger than max_partition_size_to_drop @@ -47,4 +68,4 @@ def test_attach_partition_with_large_destination(started_cluster, engine): node.query("DROP TABLE db.source_2 SYNC") create_force_drop_flag(node) node.query("DROP TABLE db.destination SYNC") - node.query("DROP DATABASE db") \ No newline at end of file + node.query("DROP DATABASE db") diff --git a/tests/integration/test_attach_without_checksums/test.py b/tests/integration/test_attach_without_checksums/test.py index ab55c5efb43..aee4b757efe 100644 --- a/tests/integration/test_attach_without_checksums/test.py +++ b/tests/integration/test_attach_without_checksums/test.py @@ -3,7 +3,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1') +node1 = cluster.add_instance("node1") + @pytest.fixture(scope="module") def start_cluster(): @@ -17,9 +18,12 @@ def start_cluster(): def test_attach_without_checksums(start_cluster): node1.query( - "CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date") + "CREATE TABLE test (date Date, key Int32, value String) Engine=MergeTree ORDER BY key PARTITION by date" + ) - node1.query("INSERT INTO test SELECT toDate('2019-10-01'), number, toString(number) FROM numbers(100)") + node1.query( + "INSERT INTO test SELECT toDate('2019-10-01'), number, toString(number) FROM numbers(100)" + ) assert node1.query("SELECT COUNT() FROM test WHERE key % 10 == 0") == "10\n" @@ -30,15 +34,27 @@ def test_attach_without_checksums(start_cluster): # to be sure output not empty node1.exec_in_container( - ['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" '], - privileged=True, user='root') + [ + "bash", + "-c", + 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" | grep -e ".*" ', + ], + privileged=True, + user="root", + ) node1.exec_in_container( - ['bash', '-c', 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete'], - privileged=True, user='root') + [ + "bash", + "-c", + 'find /var/lib/clickhouse/data/default/test/detached -name "checksums.txt" -delete', + ], + privileged=True, + user="root", + ) node1.query("ALTER TABLE test ATTACH PARTITION '2019-10-01'") assert node1.query("SELECT COUNT() FROM test WHERE key % 10 == 0") == "10\n" assert node1.query("SELECT COUNT() FROM test") == "100\n" - node1.query("DROP TABLE test") \ No newline at end of file + node1.query("DROP TABLE test") diff --git a/tests/integration/test_attach_without_fetching/test.py b/tests/integration/test_attach_without_fetching/test.py index 874f5b36ddc..60500380b31 100644 --- a/tests/integration/test_attach_without_fetching/test.py +++ b/tests/integration/test_attach_without_fetching/test.py @@ -7,19 +7,25 @@ from helpers.test_tools import assert_eq_with_retry from helpers.network import PartitionManager from helpers.corrupt_part_data_on_disk import corrupt_part_data_by_path + def fill_node(node): node.query_with_retry( - ''' + """ CREATE TABLE IF NOT EXISTS test(n UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test', '{replica}') ORDER BY n PARTITION BY n % 10; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) + cluster = ClickHouseCluster(__file__) -node_1 = cluster.add_instance('replica1', with_zookeeper=True) -node_2 = cluster.add_instance('replica2', with_zookeeper=True) -node_3 = cluster.add_instance('replica3', with_zookeeper=True) +node_1 = cluster.add_instance("replica1", with_zookeeper=True) +node_2 = cluster.add_instance("replica2", with_zookeeper=True) +node_3 = cluster.add_instance("replica3", with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -36,27 +42,42 @@ def start_cluster(): finally: cluster.shutdown() + def check_data(nodes, detached_parts): for node in nodes: - print("> Replication queue for", node.name, "\n> table\treplica_name\tsource_replica\ttype\tposition\n", - node.query_with_retry("SELECT table, replica_name, source_replica, type, position FROM system.replication_queue")) + print( + "> Replication queue for", + node.name, + "\n> table\treplica_name\tsource_replica\ttype\tposition\n", + node.query_with_retry( + "SELECT table, replica_name, source_replica, type, position FROM system.replication_queue" + ), + ) node.query_with_retry("SYSTEM SYNC REPLICA test") print("> Checking data integrity for", node.name) for i in range(10): - assert_eq_with_retry(node, "SELECT count() FROM test WHERE n % 10 == " + str(i), - "0\n" if i in detached_parts else "10\n") + assert_eq_with_retry( + node, + "SELECT count() FROM test WHERE n % 10 == " + str(i), + "0\n" if i in detached_parts else "10\n", + ) - assert_eq_with_retry(node, "SELECT count() FROM system.parts WHERE table='test'", - str(10 - len(detached_parts)) + "\n") + assert_eq_with_retry( + node, + "SELECT count() FROM system.parts WHERE table='test'", + str(10 - len(detached_parts)) + "\n", + ) res: str = node.query("SELECT * FROM test ORDER BY n") for other in nodes: if other != node: - logging.debug(f"> Checking data consistency, {other.name} vs {node.name}") + logging.debug( + f"> Checking data consistency, {other.name} vs {node.name}" + ) assert_eq_with_retry(other, "SELECT * FROM test ORDER BY n", res) @@ -83,7 +104,6 @@ def test_attach_without_fetching(start_cluster): # files missing. node_1.query("ALTER TABLE test DETACH PARTITION 2") - check_data([node_1, node_2], detached_parts=[0, 1, 2]) # 2. Create the third replica @@ -94,14 +114,28 @@ def test_attach_without_fetching(start_cluster): # Replica 2 should also download the data from 1 as the checksums won't match. logging.debug("Checking attach with corrupted part data with files missing") - to_delete = node_2.exec_in_container(['bash', '-c', - 'cd {p} && ls *.bin'.format( - p="/var/lib/clickhouse/data/default/test/detached/2_0_0_0")], privileged=True) + to_delete = node_2.exec_in_container( + [ + "bash", + "-c", + "cd {p} && ls *.bin".format( + p="/var/lib/clickhouse/data/default/test/detached/2_0_0_0" + ), + ], + privileged=True, + ) logging.debug(f"Before deleting: {to_delete}") - node_2.exec_in_container(['bash', '-c', - 'cd {p} && rm -fr *.bin'.format( - p="/var/lib/clickhouse/data/default/test/detached/2_0_0_0")], privileged=True) + node_2.exec_in_container( + [ + "bash", + "-c", + "cd {p} && rm -fr *.bin".format( + p="/var/lib/clickhouse/data/default/test/detached/2_0_0_0" + ), + ], + privileged=True, + ) node_1.query("ALTER TABLE test ATTACH PARTITION 2") check_data([node_1, node_2, node_3], detached_parts=[0, 1]) @@ -111,7 +145,9 @@ def test_attach_without_fetching(start_cluster): # Replica 2 should also download the data from 1 as the checksums won't match. print("Checking attach with corrupted part data with all of the files present") - corrupt_part_data_by_path(node_2, "/var/lib/clickhouse/data/default/test/detached/1_0_0_0") + corrupt_part_data_by_path( + node_2, "/var/lib/clickhouse/data/default/test/detached/1_0_0_0" + ) node_1.query("ALTER TABLE test ATTACH PARTITION 1") check_data([node_1, node_2, node_3], detached_parts=[0]) @@ -123,8 +159,8 @@ def test_attach_without_fetching(start_cluster): with PartitionManager() as pm: # If something goes wrong and replica 2 wants to fetch data, the test will fail. - pm.partition_instances(node_2, node_1, action='REJECT --reject-with tcp-reset') - pm.partition_instances(node_1, node_3, action='REJECT --reject-with tcp-reset') + pm.partition_instances(node_2, node_1, action="REJECT --reject-with tcp-reset") + pm.partition_instances(node_1, node_3, action="REJECT --reject-with tcp-reset") node_1.query("ALTER TABLE test ATTACH PART '0_0_0_0'") diff --git a/tests/integration/test_authentication/test.py b/tests/integration/test_authentication/test.py index 0651efa11b4..38be07eca49 100644 --- a/tests/integration/test_authentication/test.py +++ b/tests/integration/test_authentication/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -20,18 +20,30 @@ def setup_nodes(): def test_authentication_pass(): - assert instance.query("SELECT currentUser()", user='sasha') == 'sasha\n' - assert instance.query("SELECT currentUser()", user='masha', password='qwerty') == 'masha\n' + assert instance.query("SELECT currentUser()", user="sasha") == "sasha\n" + assert ( + instance.query("SELECT currentUser()", user="masha", password="qwerty") + == "masha\n" + ) # 'no_password' authentication type allows to login with any password. - assert instance.query("SELECT currentUser()", user='sasha', password='something') == 'sasha\n' - assert instance.query("SELECT currentUser()", user='sasha', password='something2') == 'sasha\n' + assert ( + instance.query("SELECT currentUser()", user="sasha", password="something") + == "sasha\n" + ) + assert ( + instance.query("SELECT currentUser()", user="sasha", password="something2") + == "sasha\n" + ) def test_authentication_fail(): # User doesn't exist. - assert "vasya: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='vasya') + assert "vasya: Authentication failed" in instance.query_and_get_error( + "SELECT currentUser()", user="vasya" + ) # Wrong password. - assert "masha: Authentication failed" in instance.query_and_get_error("SELECT currentUser()", user='masha', - password='123') + assert "masha: Authentication failed" in instance.query_and_get_error( + "SELECT currentUser()", user="masha", password="123" + ) diff --git a/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py b/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py index 08fb6e53e7b..c1d5cdc7ce5 100644 --- a/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py +++ b/tests/integration/test_azure_blob_storage_zero_copy_replication/test.py @@ -17,12 +17,20 @@ CLUSTER_NAME = "test_cluster" def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance(NODE1, main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '1'}, - with_azurite=True, - with_zookeeper=True) - cluster.add_instance(NODE2, main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '2'}, - with_azurite=True, - with_zookeeper=True) + cluster.add_instance( + NODE1, + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "1"}, + with_azurite=True, + with_zookeeper=True, + ) + cluster.add_instance( + NODE2, + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "2"}, + with_azurite=True, + with_zookeeper=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -53,7 +61,10 @@ def create_table(node, table_name, replica, **additional_settings): def get_large_objects_count(blob_container_client, large_size_threshold=100): - return sum(blob['size'] > large_size_threshold for blob in blob_container_client.list_blobs()) + return sum( + blob["size"] > large_size_threshold + for blob in blob_container_client.list_blobs() + ) def test_zero_copy_replication(cluster): @@ -61,15 +72,21 @@ def test_zero_copy_replication(cluster): node2 = cluster.instances[NODE2] create_table(node1, TABLE_NAME, 1) - blob_container_client = cluster.blob_service_client.get_container_client(CONTAINER_NAME) + blob_container_client = cluster.blob_service_client.get_container_client( + CONTAINER_NAME + ) values1 = "(0,'data'),(1,'data')" values2 = "(2,'data'),(3,'data')" node1.query(f"INSERT INTO {TABLE_NAME} VALUES {values1}") node2.query(f"SYSTEM SYNC REPLICA {TABLE_NAME}") - assert node1.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") == values1 - assert node2.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") == values1 + assert ( + node1.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") == values1 + ) + assert ( + node2.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") == values1 + ) # Based on version 21.x - should be only one file with size 100+ (checksums.txt), used by both nodes assert get_large_objects_count(blob_container_client) == 1 @@ -77,7 +94,13 @@ def test_zero_copy_replication(cluster): node2.query(f"INSERT INTO {TABLE_NAME} VALUES {values2}") node1.query(f"SYSTEM SYNC REPLICA {TABLE_NAME}") - assert node2.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") == values1 + "," + values2 - assert node1.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") == values1 + "," + values2 + assert ( + node2.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") + == values1 + "," + values2 + ) + assert ( + node1.query(f"SELECT * FROM {TABLE_NAME} order by id FORMAT Values") + == values1 + "," + values2 + ) assert get_large_objects_count(blob_container_client) == 2 diff --git a/tests/integration/test_backup_restore/test.py b/tests/integration/test_backup_restore/test.py index b990cec2364..905abef05b0 100644 --- a/tests/integration/test_backup_restore/test.py +++ b/tests/integration/test_backup_restore/test.py @@ -6,25 +6,35 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('node') -path_to_data = '/var/lib/clickhouse/' +instance = cluster.add_instance("node") +path_to_data = "/var/lib/clickhouse/" @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - instance.query('CREATE DATABASE test ENGINE = Ordinary') # Different path in shadow/ with Atomic + instance.query( + "CREATE DATABASE test ENGINE = Ordinary" + ) # Different path in shadow/ with Atomic instance.query("DROP TABLE IF EXISTS test.tbl") - instance.query("CREATE TABLE test.tbl (p Date, k Int8) ENGINE = MergeTree PARTITION BY toYYYYMM(p) ORDER BY p") + instance.query( + "CREATE TABLE test.tbl (p Date, k Int8) ENGINE = MergeTree PARTITION BY toYYYYMM(p) ORDER BY p" + ) for i in range(1, 4): - instance.query('INSERT INTO test.tbl (p, k) VALUES(toDate({}), {})'.format(i, i)) + instance.query( + "INSERT INTO test.tbl (p, k) VALUES(toDate({}), {})".format(i, i) + ) for i in range(31, 34): - instance.query('INSERT INTO test.tbl (p, k) VALUES(toDate({}), {})'.format(i, i)) + instance.query( + "INSERT INTO test.tbl (p, k) VALUES(toDate({}), {})".format(i, i) + ) - expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') + expected = TSV( + "1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33" + ) res = instance.query("SELECT * FROM test.tbl ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("ALTER TABLE test.tbl FREEZE") @@ -33,21 +43,24 @@ def started_cluster(): finally: cluster.shutdown() + def get_last_backup_path(instance, database, table): - fp_increment = os.path.join(path_to_data, 'shadow/increment.txt') - increment = instance.exec_in_container(['cat', fp_increment]).strip() - return os.path.join(path_to_data, 'shadow', increment, 'data', database, table) + fp_increment = os.path.join(path_to_data, "shadow/increment.txt") + increment = instance.exec_in_container(["cat", fp_increment]).strip() + return os.path.join(path_to_data, "shadow", increment, "data", database, table) + def copy_backup_to_detached(instance, database, src_table, dst_table): - fp_backup = os.path.join(path_to_data, 'shadow', '*', 'data', database, src_table) - fp_detached = os.path.join(path_to_data, 'data', database, dst_table, 'detached') - logging.debug(f'copy from {fp_backup} to {fp_detached}') - instance.exec_in_container(['bash', '-c', f'cp -r {fp_backup} -T {fp_detached}']) + fp_backup = os.path.join(path_to_data, "shadow", "*", "data", database, src_table) + fp_detached = os.path.join(path_to_data, "data", database, dst_table, "detached") + logging.debug(f"copy from {fp_backup} to {fp_detached}") + instance.exec_in_container(["bash", "-c", f"cp -r {fp_backup} -T {fp_detached}"]) + def test_restore(started_cluster): instance.query("CREATE TABLE test.tbl1 AS test.tbl") - copy_backup_to_detached(started_cluster.instances['node'], 'test', 'tbl', 'tbl1') + copy_backup_to_detached(started_cluster.instances["node"], "test", "tbl", "tbl1") # The data_version of parts to be attached are larger than the newly created table's data_version. instance.query("ALTER TABLE test.tbl1 ATTACH PARTITION 197001") @@ -55,17 +68,21 @@ def test_restore(started_cluster): instance.query("SELECT sleep(2)") # Validate the attached parts are identical to the backup. - expected = TSV('1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') + expected = TSV( + "1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33" + ) res = instance.query("SELECT * FROM test.tbl1 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("ALTER TABLE test.tbl1 UPDATE k=10 WHERE 1") instance.query("SELECT sleep(2)") # Validate mutation has been applied to all attached parts. - expected = TSV('1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') + expected = TSV( + "1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10" + ) res = instance.query("SELECT * FROM test.tbl1 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("DROP TABLE IF EXISTS test.tbl1") @@ -73,15 +90,19 @@ def test_restore(started_cluster): def test_attach_partition(started_cluster): instance.query("CREATE TABLE test.tbl2 AS test.tbl") for i in range(3, 5): - instance.query('INSERT INTO test.tbl2(p, k) VALUES(toDate({}), {})'.format(i, i)) + instance.query( + "INSERT INTO test.tbl2(p, k) VALUES(toDate({}), {})".format(i, i) + ) for i in range(33, 35): - instance.query('INSERT INTO test.tbl2(p, k) VALUES(toDate({}), {})'.format(i, i)) + instance.query( + "INSERT INTO test.tbl2(p, k) VALUES(toDate({}), {})".format(i, i) + ) - expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') + expected = TSV("1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34") res = instance.query("SELECT * FROM test.tbl2 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected - copy_backup_to_detached(started_cluster.instances['node'], 'test', 'tbl', 'tbl2') + copy_backup_to_detached(started_cluster.instances["node"], "test", "tbl", "tbl2") # The data_version of parts to be attached # - may be less than, equal to or larger than the current table's data_version. @@ -91,18 +112,20 @@ def test_attach_partition(started_cluster): instance.query("SELECT sleep(2)") expected = TSV( - '1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34') + "1970-01-02\t1\n1970-01-03\t2\n1970-01-04\t3\n1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33\n1970-02-03\t33\n1970-02-04\t34" + ) res = instance.query("SELECT * FROM test.tbl2 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("ALTER TABLE test.tbl2 UPDATE k=10 WHERE 1") instance.query("SELECT sleep(2)") # Validate mutation has been applied to all attached parts. expected = TSV( - '1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10') + "1970-01-02\t10\n1970-01-03\t10\n1970-01-04\t10\n1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10\n1970-02-03\t10\n1970-02-04\t10" + ) res = instance.query("SELECT * FROM test.tbl2 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("DROP TABLE IF EXISTS test.tbl2") @@ -110,15 +133,19 @@ def test_attach_partition(started_cluster): def test_replace_partition(started_cluster): instance.query("CREATE TABLE test.tbl3 AS test.tbl") for i in range(3, 5): - instance.query('INSERT INTO test.tbl3(p, k) VALUES(toDate({}), {})'.format(i, i)) + instance.query( + "INSERT INTO test.tbl3(p, k) VALUES(toDate({}), {})".format(i, i) + ) for i in range(33, 35): - instance.query('INSERT INTO test.tbl3(p, k) VALUES(toDate({}), {})'.format(i, i)) + instance.query( + "INSERT INTO test.tbl3(p, k) VALUES(toDate({}), {})".format(i, i) + ) - expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34') + expected = TSV("1970-01-04\t3\n1970-01-05\t4\n1970-02-03\t33\n1970-02-04\t34") res = instance.query("SELECT * FROM test.tbl3 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected - copy_backup_to_detached(started_cluster.instances['node'], 'test', 'tbl', 'tbl3') + copy_backup_to_detached(started_cluster.instances["node"], "test", "tbl", "tbl3") # The data_version of parts to be copied # - may be less than, equal to or larger than the current table data_version. @@ -126,35 +153,56 @@ def test_replace_partition(started_cluster): instance.query("ALTER TABLE test.tbl3 REPLACE PARTITION 197002 FROM test.tbl") instance.query("SELECT sleep(2)") - expected = TSV('1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33') + expected = TSV( + "1970-01-04\t3\n1970-01-05\t4\n1970-02-01\t31\n1970-02-02\t32\n1970-02-03\t33" + ) res = instance.query("SELECT * FROM test.tbl3 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("ALTER TABLE test.tbl3 UPDATE k=10 WHERE 1") instance.query("SELECT sleep(2)") # Validate mutation has been applied to all copied parts. - expected = TSV('1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10') + expected = TSV( + "1970-01-04\t10\n1970-01-05\t10\n1970-02-01\t10\n1970-02-02\t10\n1970-02-03\t10" + ) res = instance.query("SELECT * FROM test.tbl3 ORDER BY p") - assert (TSV(res) == expected) + assert TSV(res) == expected instance.query("DROP TABLE IF EXISTS test.tbl3") + def test_freeze_in_memory(started_cluster): - instance.query("CREATE TABLE test.t_in_memory(a UInt32, s String) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_compact_part = 1000") + instance.query( + "CREATE TABLE test.t_in_memory(a UInt32, s String) ENGINE = MergeTree ORDER BY a SETTINGS min_rows_for_compact_part = 1000" + ) instance.query("INSERT INTO test.t_in_memory VALUES (1, 'a')") instance.query("ALTER TABLE test.t_in_memory FREEZE") - fp_backup = get_last_backup_path(started_cluster.instances['node'], 'test', 't_in_memory') - part_path = fp_backup + '/all_1_1_0/' + fp_backup = get_last_backup_path( + started_cluster.instances["node"], "test", "t_in_memory" + ) + part_path = fp_backup + "/all_1_1_0/" - assert TSV(instance.query("SELECT part_type, is_frozen FROM system.parts WHERE database = 'test' AND table = 't_in_memory'")) == TSV("InMemory\t1\n") - instance.exec_in_container(['test', '-f', part_path + '/data.bin']) - assert instance.exec_in_container(['cat', part_path + '/count.txt']).strip() == '1' + assert TSV( + instance.query( + "SELECT part_type, is_frozen FROM system.parts WHERE database = 'test' AND table = 't_in_memory'" + ) + ) == TSV("InMemory\t1\n") + instance.exec_in_container(["test", "-f", part_path + "/data.bin"]) + assert instance.exec_in_container(["cat", part_path + "/count.txt"]).strip() == "1" - instance.query("CREATE TABLE test.t_in_memory_2(a UInt32, s String) ENGINE = MergeTree ORDER BY a") - copy_backup_to_detached(started_cluster.instances['node'], 'test', 't_in_memory', 't_in_memory_2') + instance.query( + "CREATE TABLE test.t_in_memory_2(a UInt32, s String) ENGINE = MergeTree ORDER BY a" + ) + copy_backup_to_detached( + started_cluster.instances["node"], "test", "t_in_memory", "t_in_memory_2" + ) instance.query("ALTER TABLE test.t_in_memory_2 ATTACH PARTITION ID 'all'") - assert TSV(instance.query("SELECT part_type FROM system.parts WHERE database = 'test' AND table = 't_in_memory_2'")) == TSV("Compact\n") + assert TSV( + instance.query( + "SELECT part_type FROM system.parts WHERE database = 'test' AND table = 't_in_memory_2'" + ) + ) == TSV("Compact\n") assert TSV(instance.query("SELECT a, s FROM test.t_in_memory_2")) == TSV("1\ta\n") diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index f9bfababadc..32ad0fbebbc 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1,16 +1,22 @@ import pytest import re +import os.path from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=["configs/backups_disk.xml"], external_dirs=["/backups/"]) +instance = cluster.add_instance( + "instance", main_configs=["configs/backups_disk.xml"], external_dirs=["/backups/"] +) + def create_and_fill_table(engine="MergeTree"): if engine == "MergeTree": engine = "MergeTree ORDER BY y PARTITION BY x%10" instance.query("CREATE DATABASE test") instance.query(f"CREATE TABLE test.table(x UInt32, y String) ENGINE={engine}") - instance.query("INSERT INTO test.table SELECT number, toString(number) FROM numbers(100)") + instance.query( + "INSERT INTO test.table SELECT number, toString(number) FROM numbers(100)" + ) @pytest.fixture(scope="module", autouse=True) @@ -31,13 +37,22 @@ def cleanup_after_test(): backup_id_counter = 0 + + def new_backup_name(): global backup_id_counter backup_id_counter += 1 return f"Disk('backups', '{backup_id_counter}/')" -@pytest.mark.parametrize("engine", ["MergeTree", "Log", "TinyLog", "StripeLog"]) +def get_backup_dir(backup_name): + counter = int(backup_name.split(",")[1].strip("')/ ")) + return os.path.join(instance.path, f"backups/{counter}") + + +@pytest.mark.parametrize( + "engine", ["MergeTree", "Log", "TinyLog", "StripeLog", "Memory"] +) def test_restore_table(engine): backup_name = new_backup_name() create_and_fill_table(engine=engine) @@ -52,7 +67,9 @@ def test_restore_table(engine): assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" -@pytest.mark.parametrize("engine", ["MergeTree", "Log", "TinyLog", "StripeLog"]) +@pytest.mark.parametrize( + "engine", ["MergeTree", "Log", "TinyLog", "StripeLog", "Memory"] +) def test_restore_table_into_existing_table(engine): backup_name = new_backup_name() create_and_fill_table(engine=engine) @@ -60,10 +77,14 @@ def test_restore_table_into_existing_table(engine): assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" instance.query(f"BACKUP TABLE test.table TO {backup_name}") - instance.query(f"RESTORE TABLE test.table INTO test.table FROM {backup_name}") + instance.query( + f"RESTORE TABLE test.table INTO test.table FROM {backup_name} SETTINGS throw_if_table_exists=0" + ) assert instance.query("SELECT count(), sum(x) FROM test.table") == "200\t9900\n" - instance.query(f"RESTORE TABLE test.table INTO test.table FROM {backup_name}") + instance.query( + f"RESTORE TABLE test.table INTO test.table FROM {backup_name} SETTINGS throw_if_table_exists=0" + ) assert instance.query("SELECT count(), sum(x) FROM test.table") == "300\t14850\n" @@ -93,6 +114,20 @@ def test_backup_table_under_another_name(): assert instance.query("SELECT count(), sum(x) FROM test.table2") == "100\t4950\n" +def test_materialized_view(): + backup_name = new_backup_name() + instance.query( + "CREATE MATERIALIZED VIEW mv_1(x UInt8) ENGINE=MergeTree ORDER BY tuple() POPULATE AS SELECT 1 AS x" + ) + + instance.query(f"BACKUP TABLE mv_1 TO {backup_name}") + instance.query("DROP TABLE mv_1") + instance.query(f"RESTORE TABLE mv_1 FROM {backup_name}") + + assert instance.query("SELECT * FROM mv_1") == "1\n" + instance.query("DROP TABLE mv_1") + + def test_incremental_backup(): backup_name = new_backup_name() incremental_backup_name = new_backup_name() @@ -104,23 +139,64 @@ def test_incremental_backup(): instance.query("INSERT INTO test.table VALUES (65, 'a'), (66, 'b')") assert instance.query("SELECT count(), sum(x) FROM test.table") == "102\t5081\n" - instance.query(f"BACKUP TABLE test.table TO {incremental_backup_name} SETTINGS base_backup = {backup_name}") + instance.query( + f"BACKUP TABLE test.table TO {incremental_backup_name} SETTINGS base_backup = {backup_name}" + ) - instance.query(f"RESTORE TABLE test.table AS test.table2 FROM {incremental_backup_name}") + instance.query( + f"RESTORE TABLE test.table AS test.table2 FROM {incremental_backup_name}" + ) assert instance.query("SELECT count(), sum(x) FROM test.table2") == "102\t5081\n" +def test_incremental_backup_after_renaming_table(): + backup_name = new_backup_name() + incremental_backup_name = new_backup_name() + create_and_fill_table() + + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + instance.query("RENAME TABLE test.table TO test.table2") + instance.query( + f"BACKUP TABLE test.table2 TO {incremental_backup_name} SETTINGS base_backup = {backup_name}" + ) + + # Files in a base backup can be searched by checksum, so an incremental backup with a renamed table actually + # contains only its changed metadata. + assert os.path.isdir(os.path.join(get_backup_dir(backup_name), "metadata")) == True + assert os.path.isdir(os.path.join(get_backup_dir(backup_name), "data")) == True + assert ( + os.path.isdir(os.path.join(get_backup_dir(incremental_backup_name), "metadata")) + == True + ) + assert ( + os.path.isdir(os.path.join(get_backup_dir(incremental_backup_name), "data")) + == False + ) + + instance.query("DROP TABLE test.table2") + instance.query(f"RESTORE TABLE test.table2 FROM {incremental_backup_name}") + assert instance.query("SELECT count(), sum(x) FROM test.table2") == "100\t4950\n" + + def test_backup_not_found_or_already_exists(): backup_name = new_backup_name() expected_error = "Backup .* not found" - assert re.search(expected_error, instance.query_and_get_error(f"RESTORE TABLE test.table AS test.table2 FROM {backup_name}")) + assert re.search( + expected_error, + instance.query_and_get_error( + f"RESTORE TABLE test.table AS test.table2 FROM {backup_name}" + ), + ) create_and_fill_table() instance.query(f"BACKUP TABLE test.table TO {backup_name}") expected_error = "Backup .* already exists" - assert re.search(expected_error, instance.query_and_get_error(f"BACKUP TABLE test.table TO {backup_name}")) + assert re.search( + expected_error, + instance.query_and_get_error(f"BACKUP TABLE test.table TO {backup_name}"), + ) def test_file_engine(): @@ -147,3 +223,38 @@ def test_database(): instance.query(f"RESTORE DATABASE test FROM {backup_name}") assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + + +def test_zip_archive(): + backup_name = f"File('/backups/archive.zip')" + create_and_fill_table() + + assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + assert os.path.isfile( + os.path.join(os.path.join(instance.path, "backups/archive.zip")) + ) + + instance.query("DROP TABLE test.table") + assert instance.query("EXISTS test.table") == "0\n" + + instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + + +def test_zip_archive_with_settings(): + backup_name = f"File('/backups/archive_with_settings.zip')" + create_and_fill_table() + + assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + instance.query( + f"BACKUP TABLE test.table TO {backup_name} SETTINGS compression_method='lzma', compression_level=3, password='qwerty'" + ) + + instance.query("DROP TABLE test.table") + assert instance.query("EXISTS test.table") == "0\n" + + instance.query( + f"RESTORE TABLE test.table FROM {backup_name} SETTINGS password='qwerty'" + ) + assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 0f35c0f849e..2fd5e65b123 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -4,13 +4,31 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', - stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', - stay_alive=True, with_installed_binary=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.4.5.35', - stay_alive=True, with_installed_binary=True) -node4 = cluster.add_instance('node4') +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="19.4.5.35", + stay_alive=True, + with_installed_binary=True, +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="19.4.5.35", + stay_alive=True, + with_installed_binary=True, +) +node3 = cluster.add_instance( + "node3", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="19.4.5.35", + stay_alive=True, + with_installed_binary=True, +) +node4 = cluster.add_instance("node4") @pytest.fixture(scope="module") @@ -24,7 +42,9 @@ def started_cluster(): def test_backup_from_old_version(started_cluster): - node1.query("CREATE TABLE source_table(A Int64, B String) Engine = MergeTree order by tuple()") + node1.query( + "CREATE TABLE source_table(A Int64, B String) Engine = MergeTree order by tuple()" + ) node1.query("INSERT INTO source_table VALUES(1, '1')") @@ -37,14 +57,24 @@ def test_backup_from_old_version(started_cluster): node1.restart_with_latest_version() node1.query( - "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()") + "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table1', '1') ORDER BY tuple()" + ) node1.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" - node1.exec_in_container(['find', '/var/lib/clickhouse/shadow/1/data/default/source_table']) - node1.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) + node1.exec_in_container( + ["find", "/var/lib/clickhouse/shadow/1/data/default/source_table"] + ) + node1.exec_in_container( + [ + "cp", + "-r", + "/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/", + "/var/lib/clickhouse/data/default/dest_table/detached", + ] + ) assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -62,7 +92,9 @@ def test_backup_from_old_version(started_cluster): def test_backup_from_old_version_setting(started_cluster): - node2.query("CREATE TABLE source_table(A Int64, B String) Engine = MergeTree order by tuple()") + node2.query( + "CREATE TABLE source_table(A Int64, B String) Engine = MergeTree order by tuple()" + ) node2.query("INSERT INTO source_table VALUES(1, '1')") @@ -75,13 +107,21 @@ def test_backup_from_old_version_setting(started_cluster): node2.restart_with_latest_version() node2.query( - "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") + "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table2', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" + ) node2.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" - node2.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) + node2.exec_in_container( + [ + "cp", + "-r", + "/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/", + "/var/lib/clickhouse/data/default/dest_table/detached", + ] + ) assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -99,7 +139,9 @@ def test_backup_from_old_version_setting(started_cluster): def test_backup_from_old_version_config(started_cluster): - node3.query("CREATE TABLE source_table(A Int64, B String) Engine = MergeTree order by tuple()") + node3.query( + "CREATE TABLE source_table(A Int64, B String) Engine = MergeTree order by tuple()" + ) node3.query("INSERT INTO source_table VALUES(1, '1')") @@ -110,19 +152,29 @@ def test_backup_from_old_version_config(started_cluster): node3.query("ALTER TABLE source_table FREEZE PARTITION tuple();") def callback(n): - n.replace_config("/etc/clickhouse-server/merge_tree_settings.xml", - "1") + n.replace_config( + "/etc/clickhouse-server/merge_tree_settings.xml", + "1", + ) node3.restart_with_latest_version(callback_onstop=callback) node3.query( - "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1") + "CREATE TABLE dest_table (A Int64, B String, Y String) ENGINE = ReplicatedMergeTree('/test/dest_table3', '1') ORDER BY tuple() SETTINGS enable_mixed_granularity_parts = 1" + ) node3.query("INSERT INTO dest_table VALUES(2, '2', 'Hello')") assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" - node3.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) + node3.exec_in_container( + [ + "cp", + "-r", + "/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/", + "/var/lib/clickhouse/data/default/dest_table/detached", + ] + ) assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -140,9 +192,13 @@ def test_backup_from_old_version_config(started_cluster): def test_backup_and_alter(started_cluster): - node4.query("CREATE DATABASE test ENGINE=Ordinary") # Different path in shadow/ with Atomic + node4.query( + "CREATE DATABASE test ENGINE=Ordinary" + ) # Different path in shadow/ with Atomic - node4.query("CREATE TABLE test.backup_table(A Int64, B String, C Date) Engine = MergeTree order by tuple()") + node4.query( + "CREATE TABLE test.backup_table(A Int64, B String, C Date) Engine = MergeTree order by tuple()" + ) node4.query("INSERT INTO test.backup_table VALUES(2, '2', toDate('2019-10-01'))") @@ -154,7 +210,14 @@ def test_backup_and_alter(started_cluster): node4.query("ALTER TABLE test.backup_table DROP PARTITION tuple()") - node4.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/', '/var/lib/clickhouse/data/test/backup_table/detached']) + node4.exec_in_container( + [ + "cp", + "-r", + "/var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/", + "/var/lib/clickhouse/data/test/backup_table/detached", + ] + ) node4.query("ALTER TABLE test.backup_table ATTACH PARTITION tuple()") diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index a8f4968956c..01ed02720f8 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -3,20 +3,29 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', main_configs=['configs/wide_parts_only.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="19.17.8.54", + stay_alive=True, + with_installed_binary=True, +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/wide_parts_only.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() - create_query = '''CREATE TABLE t(date Date, id UInt32) + create_query = """CREATE TABLE t(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/t', '{}') PARTITION BY toYYYYMM(date) - ORDER BY id''' + ORDER BY id""" node1.query(create_query.format(1)) - node1.query("DETACH TABLE t") # stop being leader + node1.query("DETACH TABLE t") # stop being leader node2.query(create_query.format(2)) node1.query("ATTACH TABLE t") yield cluster diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index fc8d27cfa16..35cdaeef9ac 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -3,9 +3,15 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="aggregate_fixed_key") -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='21.3', with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="21.3", + with_installed_binary=True, +) +node2 = cluster.add_instance("node2", with_zookeeper=True) +node3 = cluster.add_instance("node3", with_zookeeper=True) @pytest.fixture(scope="module") @@ -38,8 +44,9 @@ def test_two_level_merge(start_cluster): # covers only the keys64 method for node in start_cluster.instances.values(): - print(node.query( - """ + print( + node.query( + """ SELECT throwIf(uniqExact(date) != count(), 'group by is borked') FROM ( @@ -58,4 +65,5 @@ def test_two_level_merge(start_cluster): max_threads = 2, prefer_localhost_replica = 0 """ - )) + ) + ) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py index feaf96c439d..b3ad9011239 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py @@ -3,14 +3,24 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="aggregate_state") -node1 = cluster.add_instance('node1', - with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, - with_installed_binary=True) -node2 = cluster.add_instance('node2', - with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', stay_alive=True, - with_installed_binary=True) -node3 = cluster.add_instance('node3', with_zookeeper=False) -node4 = cluster.add_instance('node4', with_zookeeper=False) +node1 = cluster.add_instance( + "node1", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="19.16.9.37", + stay_alive=True, + with_installed_binary=True, +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="19.16.9.37", + stay_alive=True, + with_installed_binary=True, +) +node3 = cluster.add_instance("node3", with_zookeeper=False) +node4 = cluster.add_instance("node4", with_zookeeper=False) @pytest.fixture(scope="module") @@ -27,6 +37,7 @@ def start_cluster(): # TODO Implement versioning of serialization format for aggregate function states. # NOTE This test is too ad-hoc. + def test_backward_compatability(start_cluster): node1.query("create table tab (x UInt64) engine = Memory") node2.query("create table tab (x UInt64) engine = Memory") @@ -38,24 +49,34 @@ def test_backward_compatability(start_cluster): node3.query("INSERT INTO tab VALUES (3)") node4.query("INSERT INTO tab VALUES (4)") - assert (node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') - assert (node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') - assert (node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') - assert (node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == '2.5\n') + assert ( + node1.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == "2.5\n" + ) + assert ( + node2.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == "2.5\n" + ) + assert ( + node3.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == "2.5\n" + ) + assert ( + node4.query("SELECT avg(x) FROM remote('node{1..4}', default, tab)") == "2.5\n" + ) # Also check with persisted aggregate function state node1.query("create table state (x AggregateFunction(avg, UInt64)) engine = Log") - node1.query("INSERT INTO state SELECT avgState(arrayJoin(CAST([1, 2, 3, 4] AS Array(UInt64))))") + node1.query( + "INSERT INTO state SELECT avgState(arrayJoin(CAST([1, 2, 3, 4] AS Array(UInt64))))" + ) - assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') + assert node1.query("SELECT avgMerge(x) FROM state") == "2.5\n" node1.restart_with_latest_version() - assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') + assert node1.query("SELECT avgMerge(x) FROM state") == "2.5\n" node1.query("drop table tab") node1.query("drop table state") node2.query("drop table tab") node3.query("drop table tab") - node4.query("drop table tab") \ No newline at end of file + node4.query("drop table tab") diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index 3aec527524b..89a565b4b37 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -3,10 +3,15 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="cte_distributed") -node1 = cluster.add_instance('node1', with_zookeeper=False) -node2 = cluster.add_instance('node2', - with_zookeeper=False, image='yandex/clickhouse-server', tag='21.7.3.14', stay_alive=True, - with_installed_binary=True) +node1 = cluster.add_instance("node1", with_zookeeper=False) +node2 = cluster.add_instance( + "node2", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="21.7.3.14", + stay_alive=True, + with_installed_binary=True, +) @pytest.fixture(scope="module") @@ -19,9 +24,9 @@ def start_cluster(): cluster.shutdown() - def test_cte_distributed(start_cluster): - node2.query(""" + node2.query( + """ WITH quantile(0.05)(cnt) as p05, quantile(0.95)(cnt) as p95, @@ -35,9 +40,11 @@ FROM ( count() as cnt FROM remote('node{1,2}', numbers(10)) GROUP BY number -)""") +)""" + ) - node1.query(""" + node1.query( + """ WITH quantile(0.05)(cnt) as p05, quantile(0.95)(cnt) as p95, @@ -51,4 +58,5 @@ FROM ( count() as cnt FROM remote('node{1,2}', numbers(10)) GROUP BY number -)""") +)""" + ) diff --git a/tests/integration/test_backward_compatibility/test_data_skipping_indices.py b/tests/integration/test_backward_compatibility/test_data_skipping_indices.py index db6a3eb7a08..60d709c257f 100644 --- a/tests/integration/test_backward_compatibility/test_data_skipping_indices.py +++ b/tests/integration/test_backward_compatibility/test_data_skipping_indices.py @@ -6,7 +6,13 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="skipping_indices") -node = cluster.add_instance('node', image='yandex/clickhouse-server', tag='21.6', stay_alive=True, with_installed_binary=True) +node = cluster.add_instance( + "node", + image="yandex/clickhouse-server", + tag="21.6", + stay_alive=True, + with_installed_binary=True, +) @pytest.fixture(scope="module") @@ -23,7 +29,8 @@ def start_cluster(): # restart_with_tagged_version(), since right now it is not possible to # switch to old tagged clickhouse version. def test_index(start_cluster): - node.query(""" + node.query( + """ CREATE TABLE data ( key Int, @@ -36,9 +43,12 @@ def test_index(start_cluster): INSERT INTO data SELECT number, number FROM numbers(10000); SELECT * FROM data WHERE value = 20000 SETTINGS force_data_skipping_indices = 'value_index' SETTINGS force_data_skipping_indices = 'value_index', max_rows_to_read=1; - """) + """ + ) node.restart_with_latest_version() - node.query(""" + node.query( + """ SELECT * FROM data WHERE value = 20000 SETTINGS force_data_skipping_indices = 'value_index' SETTINGS force_data_skipping_indices = 'value_index', max_rows_to_read=1; DROP TABLE data; - """) \ No newline at end of file + """ + ) diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py index abebaaea8b8..cb9929db48b 100644 --- a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py +++ b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py @@ -4,7 +4,13 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="detach") # Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key. -node_21_6 = cluster.add_instance('node_21_6', image='yandex/clickhouse-server', tag='21.6.3.14', stay_alive=True, with_installed_binary=True) +node_21_6 = cluster.add_instance( + "node_21_6", + image="yandex/clickhouse-server", + tag="21.6.3.14", + stay_alive=True, + with_installed_binary=True, +) @pytest.fixture(scope="module") @@ -16,11 +22,16 @@ def start_cluster(): finally: cluster.shutdown() + def test_detach_part_wrong_partition_id(start_cluster): # Here we create table with partition by UUID. - node_21_6.query("create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()") - node_21_6.query("insert into tab values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 2)") + node_21_6.query( + "create table tab (id UUID, value UInt32) engine = MergeTree PARTITION BY (id) order by tuple()" + ) + node_21_6.query( + "insert into tab values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 2)" + ) # After restart, partition id will be different. # There is a single 0-level part, which will become broken. @@ -29,7 +40,7 @@ def test_detach_part_wrong_partition_id(start_cluster): node_21_6.restart_with_latest_version() num_detached = node_21_6.query("select count() from system.detached_parts") - assert num_detached == '1\n' + assert num_detached == "1\n" node_21_6.restart_with_original_version() diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index 9a7c7f73eb5..e98894d887a 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -3,10 +3,15 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="aggregate_alias_column") -node1 = cluster.add_instance('node1', with_zookeeper=False) -node2 = cluster.add_instance('node2', - with_zookeeper=False, image='yandex/clickhouse-server', tag='21.7.2.7', stay_alive=True, - with_installed_binary=True) +node1 = cluster.add_instance("node1", with_zookeeper=False) +node2 = cluster.add_instance( + "node2", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="21.7.2.7", + stay_alive=True, + with_installed_binary=True, +) @pytest.fixture(scope="module") @@ -22,11 +27,11 @@ def start_cluster(): def test_select_aggregate_alias_column(start_cluster): node1.query("create table tab (x UInt64, x_alias UInt64 ALIAS x) engine = Memory") node2.query("create table tab (x UInt64, x_alias UInt64 ALIAS x) engine = Memory") - node1.query('insert into tab values (1)') - node2.query('insert into tab values (1)') + node1.query("insert into tab values (1)") + node2.query("insert into tab values (1)") node1.query("select sum(x_alias) from remote('node{1,2}', default, tab)") node2.query("select sum(x_alias) from remote('node{1,2}', default, tab)") node1.query("drop table tab") - node2.query("drop table tab") \ No newline at end of file + node2.query("drop table tab") diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index 54dd53c344e..8053ad417ec 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -3,11 +3,23 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="short_strings") -node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', - stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server', tag='19.16.9.37', - stay_alive=True, with_installed_binary=True) -node3 = cluster.add_instance('node3', with_zookeeper=False) +node1 = cluster.add_instance( + "node1", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="19.16.9.37", + stay_alive=True, + with_installed_binary=True, +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=False, + image="yandex/clickhouse-server", + tag="19.16.9.37", + stay_alive=True, + with_installed_binary=True, +) +node3 = cluster.add_instance("node3", with_zookeeper=False) @pytest.fixture(scope="module") @@ -26,8 +38,9 @@ def test_backward_compatability(start_cluster): node1.query("insert into tab select number from numbers(50)") node2.query("insert into tab select number from numbers(1000000)") res = node3.query( - "select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") + "select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50" + ) print(res) - assert res == ''.join('{}\t2\n'.format(i) for i in range(50)) + assert res == "".join("{}\t2\n".format(i) for i in range(50)) node1.query("drop table tab") node2.query("drop table tab") diff --git a/tests/integration/test_block_structure_mismatch/test.py b/tests/integration/test_block_structure_mismatch/test.py index 12f9bd090a3..b04607fc9d6 100644 --- a/tests/integration/test_block_structure_mismatch/test.py +++ b/tests/integration/test_block_structure_mismatch/test.py @@ -4,8 +4,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) +node1 = cluster.add_instance("node1", main_configs=["configs/remote_servers.xml"]) +node2 = cluster.add_instance("node2", main_configs=["configs/remote_servers.xml"]) # test reproducing issue https://github.com/ClickHouse/ClickHouse/issues/3162 @@ -15,7 +15,8 @@ def started_cluster(): cluster.start() for node in (node1, node2): - node.query(''' + node.query( + """ CREATE TABLE local_test ( t UInt64, date Date DEFAULT toDate(t/1000), @@ -26,9 +27,11 @@ CREATE TABLE local_test ( PARTITION BY toRelativeDayNum(date) ORDER BY (t) SETTINGS index_granularity=8192 - ''') + """ + ) - node.query(''' + node.query( + """ CREATE TABLE dist_test ( t UInt64, shard UInt64, @@ -36,7 +39,8 @@ CREATE TABLE dist_test ( col1 String, col2 String ) Engine = Distributed(testcluster, default, local_test, shard) - ''') + """ + ) yield cluster @@ -45,7 +49,15 @@ CREATE TABLE dist_test ( def test(started_cluster): - node1.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 0, 'x', 'y')") - node2.query("INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 1, 'foo', 'bar')") - assert node1.query( - "SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC") == "foo\tbar\n" + node1.query( + "INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 0, 'x', 'y')" + ) + node2.query( + "INSERT INTO local_test (t, shard, col1, col2) VALUES (1000, 1, 'foo', 'bar')" + ) + assert ( + node1.query( + "SELECT col1, col2 FROM dist_test WHERE (t < 3600000) AND (col1 = 'foo') ORDER BY t ASC" + ) + == "foo\tbar\n" + ) diff --git a/tests/integration/test_broken_part_during_merge/test.py b/tests/integration/test_broken_part_during_merge/test.py index 1c03add49db..d7492be686b 100644 --- a/tests/integration/test_broken_part_during_merge/test.py +++ b/tests/integration/test_broken_part_during_merge/test.py @@ -8,7 +8,7 @@ import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) @pytest.fixture(scope="module") @@ -22,36 +22,55 @@ def started_cluster(): def test_merge_and_part_corruption(started_cluster): - node1.query(''' + node1.query( + """ CREATE TABLE replicated_mt(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_mt', '{replica}') ORDER BY id; - '''.format(replica=node1.name)) - + """.format( + replica=node1.name + ) + ) node1.query("SYSTEM STOP REPLICATION QUEUES replicated_mt") for i in range(4): - node1.query("INSERT INTO replicated_mt SELECT toDate('2019-10-01'), number, number * number FROM numbers ({f}, 100000)".format(f=i*100000)) + node1.query( + "INSERT INTO replicated_mt SELECT toDate('2019-10-01'), number, number * number FROM numbers ({f}, 100000)".format( + f=i * 100000 + ) + ) - assert node1.query("SELECT COUNT() FROM system.parts WHERE table='replicated_mt' AND active=1") == "4\n" + assert ( + node1.query( + "SELECT COUNT() FROM system.parts WHERE table='replicated_mt' AND active=1" + ) + == "4\n" + ) # Need to corrupt "border part" (left or right). If we will corrupt something in the middle # clickhouse will not consider merge as broken, because we have parts with the same min and max # block numbers. - corrupt_part_data_on_disk(node1, 'replicated_mt', 'all_3_3_0') + corrupt_part_data_on_disk(node1, "replicated_mt", "all_3_3_0") with Pool(1) as p: + def optimize_with_delay(x): node1.query("OPTIMIZE TABLE replicated_mt FINAL", timeout=30) # corrupt part after merge already assigned, but not started res_opt = p.apply_async(optimize_with_delay, (1,)) - node1.query("CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0}) + node1.query( + "CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0} + ) # start merge node1.query("SYSTEM START REPLICATION QUEUES replicated_mt") res_opt.get() # will hung if checked bug not fixed - node1.query("ALTER TABLE replicated_mt UPDATE value = 7 WHERE 1", settings={"mutations_sync": 2}, timeout=30) + node1.query( + "ALTER TABLE replicated_mt UPDATE value = 7 WHERE 1", + settings={"mutations_sync": 2}, + timeout=30, + ) assert node1.query("SELECT sum(value) FROM replicated_mt") == "2100000\n" - node1.query('DROP TABLE replicated_mt SYNC') + node1.query("DROP TABLE replicated_mt SYNC") diff --git a/tests/integration/test_buffer_profile/test.py b/tests/integration/test_buffer_profile/test.py index ae9220898ab..b1185493c47 100644 --- a/tests/integration/test_buffer_profile/test.py +++ b/tests/integration/test_buffer_profile/test.py @@ -9,12 +9,15 @@ from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) -node_default = cluster.add_instance('node_default') -node_buffer_profile = cluster.add_instance('node_buffer_profile', - main_configs=['configs/buffer_profile.xml'], - user_configs=['configs/users.d/buffer_profile.xml']) +node_default = cluster.add_instance("node_default") +node_buffer_profile = cluster.add_instance( + "node_buffer_profile", + main_configs=["configs/buffer_profile.xml"], + user_configs=["configs/users.d/buffer_profile.xml"], +) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -22,8 +25,10 @@ def start_cluster(): finally: cluster.shutdown() + def bootstrap(node): - node.query(""" + node.query( + """ CREATE TABLE data (key Int) Engine=MergeTree() ORDER BY key PARTITION BY key % 2; @@ -40,15 +45,20 @@ def bootstrap(node): ); INSERT INTO buffer SELECT * FROM numbers(100); - """) + """ + ) + def test_default_profile(): bootstrap(node_default) # flush the buffer - node_default.query('OPTIMIZE TABLE buffer') + node_default.query("OPTIMIZE TABLE buffer") + def test_buffer_profile(): bootstrap(node_buffer_profile) - with pytest.raises(QueryRuntimeException, match='Too many partitions for single INSERT block'): + with pytest.raises( + QueryRuntimeException, match="Too many partitions for single INSERT block" + ): # flush the buffer - node_buffer_profile.query('OPTIMIZE TABLE buffer') + node_buffer_profile.query("OPTIMIZE TABLE buffer") diff --git a/tests/integration/test_catboost_model_config_reload/test.py b/tests/integration/test_catboost_model_config_reload/test.py index 4059e739dc9..c12c28e2338 100644 --- a/tests/integration/test_catboost_model_config_reload/test.py +++ b/tests/integration/test_catboost_model_config_reload/test.py @@ -10,16 +10,24 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=['config/models_config.xml', 'config/catboost_lib.xml']) +node = cluster.add_instance( + "node", + stay_alive=True, + main_configs=["config/models_config.xml", "config/catboost_lib.xml"], +) def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) -config = ''' +config = """ /etc/clickhouse-server/model/{model_config} -''' +""" @pytest.fixture(scope="module") @@ -27,7 +35,11 @@ def started_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'model/.'), '/etc/clickhouse-server/model', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "model/."), + "/etc/clickhouse-server/model", + node.docker_id, + ) node.restart_clickhouse() yield cluster @@ -37,7 +49,10 @@ def started_cluster(): def change_config(model_config): - node.replace_config("/etc/clickhouse-server/config.d/models_config.xml", config.format(model_config=model_config)) + node.replace_config( + "/etc/clickhouse-server/config.d/models_config.xml", + config.format(model_config=model_config), + ) node.query("SYSTEM RELOAD CONFIG;") @@ -57,5 +72,6 @@ def test(started_cluster): node.query("SELECT modelEvaluate('model2', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") # Check that the old model was unloaded. - node.query_and_get_error("SELECT modelEvaluate('model1', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") - + node.query_and_get_error( + "SELECT modelEvaluate('model1', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);" + ) diff --git a/tests/integration/test_catboost_model_first_evaluate/test.py b/tests/integration/test_catboost_model_first_evaluate/test.py index 7e498ccfe21..b15f481c0e9 100644 --- a/tests/integration/test_catboost_model_first_evaluate/test.py +++ b/tests/integration/test_catboost_model_first_evaluate/test.py @@ -10,11 +10,17 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=['config/models_config.xml']) +node = cluster.add_instance( + "node", stay_alive=True, main_configs=["config/models_config.xml"] +) def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) @pytest.fixture(scope="module") @@ -22,7 +28,11 @@ def started_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'model/.'), '/etc/clickhouse-server/model', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "model/."), + "/etc/clickhouse-server/model", + node.docker_id, + ) node.restart_clickhouse() yield cluster @@ -36,4 +46,3 @@ def test(started_cluster): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") node.query("select modelEvaluate('titanic', 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);") - diff --git a/tests/integration/test_catboost_model_reload/test.py b/tests/integration/test_catboost_model_reload/test.py index 3d88c19cd2c..3bf7ca18cdd 100644 --- a/tests/integration/test_catboost_model_reload/test.py +++ b/tests/integration/test_catboost_model_reload/test.py @@ -10,17 +10,31 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=['config/models_config.xml', 'config/catboost_lib.xml']) +node = cluster.add_instance( + "node", + stay_alive=True, + main_configs=["config/models_config.xml", "config/catboost_lib.xml"], +) + def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'model/.'), '/etc/clickhouse-server/model', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "model/."), + "/etc/clickhouse-server/model", + node.docker_id, + ) node.query("CREATE TABLE binary (x UInt64, y UInt64) ENGINE = TinyLog()") node.query("INSERT INTO binary VALUES (1, 1), (1, 0), (0, 1), (0, 0)") @@ -31,50 +45,88 @@ def started_cluster(): finally: cluster.shutdown() + def test_model_reload(started_cluster): if node.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - node.exec_in_container(["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container(["bash", "-c", "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm"]) + node.exec_in_container( + ["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"] + ) + node.exec_in_container( + [ + "bash", + "-c", + "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm", + ] + ) node.query("SYSTEM RELOAD MODEL model") - result = node.query(""" + result = node.query( + """ WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability SELECT if(probability > 0.5, 1, 0) FROM binary; - """) - assert result == '1\n0\n0\n0\n' + """ + ) + assert result == "1\n0\n0\n0\n" node.exec_in_container(["bash", "-c", "rm /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container(["bash", "-c", "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm"]) + node.exec_in_container( + [ + "bash", + "-c", + "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm", + ] + ) node.query("SYSTEM RELOAD MODEL model") - result = node.query(""" + result = node.query( + """ WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability SELECT if(probability > 0.5, 1, 0) FROM binary; - """) - assert result == '1\n1\n1\n0\n' + """ + ) + assert result == "1\n1\n1\n0\n" + def test_models_reload(started_cluster): if node.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - node.exec_in_container(["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container(["bash", "-c", "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm"]) + node.exec_in_container( + ["bash", "-c", "rm -f /etc/clickhouse-server/model/model.cbm"] + ) + node.exec_in_container( + [ + "bash", + "-c", + "ln /etc/clickhouse-server/model/conjunction.cbm /etc/clickhouse-server/model/model.cbm", + ] + ) node.query("SYSTEM RELOAD MODELS") - result = node.query(""" + result = node.query( + """ WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability SELECT if(probability > 0.5, 1, 0) FROM binary; - """) - assert result == '1\n0\n0\n0\n' + """ + ) + assert result == "1\n0\n0\n0\n" node.exec_in_container(["bash", "-c", "rm /etc/clickhouse-server/model/model.cbm"]) - node.exec_in_container(["bash", "-c", "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm"]) + node.exec_in_container( + [ + "bash", + "-c", + "ln /etc/clickhouse-server/model/disjunction.cbm /etc/clickhouse-server/model/model.cbm", + ] + ) node.query("SYSTEM RELOAD MODELS") - result = node.query(""" + result = node.query( + """ WITH modelEvaluate('model', toFloat64(x), toFloat64(y)) as prediction, exp(prediction) / (1 + exp(prediction)) as probability SELECT if(probability > 0.5, 1, 0) FROM binary; - """) - assert result == '1\n1\n1\n0\n' + """ + ) + assert result == "1\n1\n1\n0\n" diff --git a/tests/integration/test_cgroup_limit/test.py b/tests/integration/test_cgroup_limit/test.py index c3a92bee032..f6392eca4d7 100644 --- a/tests/integration/test_cgroup_limit/test.py +++ b/tests/integration/test_cgroup_limit/test.py @@ -6,32 +6,50 @@ import subprocess from tempfile import NamedTemporaryFile import pytest + def run_command_in_container(cmd, *args): # /clickhouse is mounted by integration tests runner - alternative_binary = os.getenv('CLICKHOUSE_BINARY', '/clickhouse') + alternative_binary = os.getenv("CLICKHOUSE_BINARY", "/clickhouse") if alternative_binary: args += ( - '--volume', f'{alternative_binary}:/usr/bin/clickhouse', + "--volume", + f"{alternative_binary}:/usr/bin/clickhouse", ) - return subprocess.check_output(['docker', 'run', '--rm', - *args, - 'ubuntu:20.04', - 'sh', '-c', cmd, - ]) + return subprocess.check_output( + [ + "docker", + "run", + "--rm", + *args, + "ubuntu:20.04", + "sh", + "-c", + cmd, + ] + ) + def run_with_cpu_limit(cmd, num_cpus, *args): args += ( - '--cpus', f'{num_cpus}', + "--cpus", + f"{num_cpus}", ) return run_command_in_container(cmd, *args) + def test_cgroup_cpu_limit(): for num_cpus in (1, 2, 4, 2.8): - result = run_with_cpu_limit('clickhouse local -q "select value from system.settings where name=\'max_threads\'"', num_cpus) + result = run_with_cpu_limit( + "clickhouse local -q \"select value from system.settings where name='max_threads'\"", + num_cpus, + ) expect_output = (r"\'auto({})\'".format(math.ceil(num_cpus))).encode() - assert result.strip() == expect_output, f"fail for cpu limit={num_cpus}, result={result.strip()}, expect={expect_output}" + assert ( + result.strip() == expect_output + ), f"fail for cpu limit={num_cpus}, result={result.strip()}, expect={expect_output}" + # For manual run -if __name__ == '__main__': +if __name__ == "__main__": test_cgroup_cpu_limit() diff --git a/tests/integration/test_check_table/test.py b/tests/integration/test_check_table/test.py index b184813d24f..613ac3fb35f 100644 --- a/tests/integration/test_check_table/test.py +++ b/tests/integration/test_check_table/test.py @@ -4,8 +4,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) @pytest.fixture(scope="module") @@ -21,140 +21,246 @@ def started_cluster(): def corrupt_data_part_on_disk(node, table, part_name): part_path = node.query( - "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() - node.exec_in_container(['bash', '-c', - 'cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c \'echo "1" >> $1\' -- {{}}'.format( - p=part_path)], privileged=True) + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() + node.exec_in_container( + [ + "bash", + "-c", + "cd {p} && ls *.bin | head -n 1 | xargs -I{{}} sh -c 'echo \"1\" >> $1' -- {{}}".format( + p=part_path + ), + ], + privileged=True, + ) def remove_checksums_on_disk(node, table, part_name): part_path = node.query( - "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() - node.exec_in_container(['bash', '-c', 'rm -r {p}/checksums.txt'.format(p=part_path)], privileged=True) + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() + node.exec_in_container( + ["bash", "-c", "rm -r {p}/checksums.txt".format(p=part_path)], privileged=True + ) def remove_part_from_disk(node, table, part_name): part_path = node.query( - "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() if not part_path: raise Exception("Part " + part_name + "doesn't exist") - node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True) + node.exec_in_container( + ["bash", "-c", "rm -r {p}/*".format(p=part_path)], privileged=True + ) def test_check_normal_table_corruption(started_cluster): node1.query("DROP TABLE IF EXISTS non_replicated_mt") - node1.query(''' + node1.query( + """ CREATE TABLE non_replicated_mt(date Date, id UInt32, value Int32) ENGINE = MergeTree() PARTITION BY toYYYYMM(date) ORDER BY id SETTINGS min_bytes_for_wide_part=0; - ''') + """ + ) - node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", - settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n" + node1.query( + "INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)" + ) + assert ( + node1.query( + "CHECK TABLE non_replicated_mt PARTITION 201902", + settings={"check_query_single_value_result": 0}, + ) + == "201902_1_1_0\t1\t\n" + ) remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") - assert node1.query("CHECK TABLE non_replicated_mt", settings={ - "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." + assert ( + node1.query( + "CHECK TABLE non_replicated_mt", + settings={"check_query_single_value_result": 0}, + ).strip() + == "201902_1_1_0\t1\tChecksums recounted and written to disk." + ) assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={ - "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk." + assert ( + node1.query( + "CHECK TABLE non_replicated_mt PARTITION 201902", + settings={"check_query_single_value_result": 0}, + ).strip() + == "201902_1_1_0\t1\tChecksums recounted and written to disk." + ) assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n" corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0") - assert node1.query("CHECK TABLE non_replicated_mt", settings={ - "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 25." + assert ( + node1.query( + "CHECK TABLE non_replicated_mt", + settings={"check_query_single_value_result": 0}, + ).strip() + == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 25." + ) - assert node1.query("CHECK TABLE non_replicated_mt", settings={ - "check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 25." + assert ( + node1.query( + "CHECK TABLE non_replicated_mt", + settings={"check_query_single_value_result": 0}, + ).strip() + == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 25." + ) - node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)") + node1.query( + "INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)" + ) - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", - settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n" + assert ( + node1.query( + "CHECK TABLE non_replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) + == "201901_2_2_0\t1\t\n" + ) corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0") remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0") - assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={ - "check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 25.\\'\n" + assert ( + node1.query( + "CHECK TABLE non_replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) + == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 25.\\'\n" + ) def test_check_replicated_table_simple(started_cluster): for node in [node1, node2]: node.query("DROP TABLE IF EXISTS replicated_mt") - node.query(''' + node.query( + """ CREATE TABLE replicated_mt(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_mt', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) - node1.query("INSERT INTO replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") + node1.query( + "INSERT INTO replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)" + ) node2.query("SYSTEM SYNC REPLICA replicated_mt") assert node1.query("SELECT count() from replicated_mt") == "2\n" assert node2.query("SELECT count() from replicated_mt") == "2\n" - assert node1.query("CHECK TABLE replicated_mt", - settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" - assert node2.query("CHECK TABLE replicated_mt", - settings={"check_query_single_value_result": 0}) == "201902_0_0_0\t1\t\n" + assert ( + node1.query( + "CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0} + ) + == "201902_0_0_0\t1\t\n" + ) + assert ( + node2.query( + "CHECK TABLE replicated_mt", settings={"check_query_single_value_result": 0} + ) + == "201902_0_0_0\t1\t\n" + ) - node2.query("INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)") + node2.query( + "INSERT INTO replicated_mt VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)" + ) node1.query("SYSTEM SYNC REPLICA replicated_mt") assert node1.query("SELECT count() from replicated_mt") == "4\n" assert node2.query("SELECT count() from replicated_mt") == "4\n" - assert node1.query("CHECK TABLE replicated_mt PARTITION 201901", - settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" - assert node2.query("CHECK TABLE replicated_mt PARTITION 201901", - settings={"check_query_single_value_result": 0}) == "201901_0_0_0\t1\t\n" + assert ( + node1.query( + "CHECK TABLE replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) + == "201901_0_0_0\t1\t\n" + ) + assert ( + node2.query( + "CHECK TABLE replicated_mt PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) + == "201901_0_0_0\t1\t\n" + ) def test_check_replicated_table_corruption(started_cluster): for node in [node1, node2]: node.query_with_retry("DROP TABLE IF EXISTS replicated_mt_1") - node.query_with_retry(''' + node.query_with_retry( + """ CREATE TABLE replicated_mt_1(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/replicated_mt_1', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY id; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) - node1.query("INSERT INTO replicated_mt_1 VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)") - node1.query("INSERT INTO replicated_mt_1 VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)") + node1.query( + "INSERT INTO replicated_mt_1 VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)" + ) + node1.query( + "INSERT INTO replicated_mt_1 VALUES (toDate('2019-01-02'), 3, 10), (toDate('2019-01-02'), 4, 12)" + ) node2.query("SYSTEM SYNC REPLICA replicated_mt_1") assert node1.query("SELECT count() from replicated_mt_1") == "4\n" assert node2.query("SELECT count() from replicated_mt_1") == "4\n" part_name = node1.query_with_retry( - "SELECT name from system.parts where table = 'replicated_mt_1' and partition_id = '201901' and active = 1").strip() + "SELECT name from system.parts where table = 'replicated_mt_1' and partition_id = '201901' and active = 1" + ).strip() corrupt_data_part_on_disk(node1, "replicated_mt_1", part_name) - assert node1.query("CHECK TABLE replicated_mt_1 PARTITION 201901", settings={ - "check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and will try to fetch.\n".format( - p=part_name) + assert node1.query( + "CHECK TABLE replicated_mt_1 PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) == "{p}\t0\tPart {p} looks broken. Removing it and will try to fetch.\n".format( + p=part_name + ) node1.query_with_retry("SYSTEM SYNC REPLICA replicated_mt_1") - assert node1.query("CHECK TABLE replicated_mt_1 PARTITION 201901", - settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) + assert node1.query( + "CHECK TABLE replicated_mt_1 PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) == "{}\t1\t\n".format(part_name) assert node1.query("SELECT count() from replicated_mt_1") == "4\n" remove_part_from_disk(node2, "replicated_mt_1", part_name) - assert node2.query("CHECK TABLE replicated_mt_1 PARTITION 201901", settings={ - "check_query_single_value_result": 0}) == "{p}\t0\tPart {p} looks broken. Removing it and will try to fetch.\n".format( - p=part_name) + assert node2.query( + "CHECK TABLE replicated_mt_1 PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) == "{p}\t0\tPart {p} looks broken. Removing it and will try to fetch.\n".format( + p=part_name + ) node1.query("SYSTEM SYNC REPLICA replicated_mt_1") - assert node1.query("CHECK TABLE replicated_mt_1 PARTITION 201901", - settings={"check_query_single_value_result": 0}) == "{}\t1\t\n".format(part_name) + assert node1.query( + "CHECK TABLE replicated_mt_1 PARTITION 201901", + settings={"check_query_single_value_result": 0}, + ) == "{}\t1\t\n".format(part_name) assert node1.query("SELECT count() from replicated_mt_1") == "4\n" diff --git a/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py b/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py index a79015835db..3c59d99b7fc 100644 --- a/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py +++ b/tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py @@ -5,7 +5,7 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) @pytest.fixture(scope="module") @@ -29,14 +29,14 @@ def start_cluster(): # the table creation works. def test_cleanup_dir_after_bad_zk_conn(start_cluster): node1.query("CREATE DATABASE replica;") - query_create = '''CREATE TABLE replica.test + query_create = """CREATE TABLE replica.test ( id Int64, event_time DateTime ) Engine=ReplicatedMergeTree('/clickhouse/tables/replica/test', 'node1') PARTITION BY toYYYYMMDD(event_time) - ORDER BY id;''' + ORDER BY id;""" with PartitionManager() as pm: pm.drop_instance_zk_connections(node1) time.sleep(3) @@ -45,38 +45,54 @@ def test_cleanup_dir_after_bad_zk_conn(start_cluster): error = node1.query_and_get_error(query_create) assert "Directory for table data data/replica/test/ already exists" not in error node1.query_with_retry(query_create) - node1.query_with_retry('''INSERT INTO replica.test VALUES (1, now())''') - assert "1\n" in node1.query('''SELECT count() from replica.test FORMAT TSV''') + node1.query_with_retry("""INSERT INTO replica.test VALUES (1, now())""") + assert "1\n" in node1.query("""SELECT count() from replica.test FORMAT TSV""") node1.query("DROP TABLE replica.test SYNC") node1.query("DROP DATABASE replica") + def test_cleanup_dir_after_wrong_replica_name(start_cluster): node1.query_with_retry( - "CREATE TABLE IF NOT EXISTS test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") + "CREATE TABLE IF NOT EXISTS test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n" + ) error = node1.query_and_get_error( - "CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n") + "CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n" + ) assert "already exists" in error node1.query_with_retry( - "CREATE TABLE IF NOT EXISTS test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n") + "CREATE TABLE IF NOT EXISTS test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n" + ) def test_cleanup_dir_after_wrong_zk_path(start_cluster): node1.query( - "CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n") + "CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n" + ) error = node1.query_and_get_error( - "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n") + "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n" + ) assert "Cannot create" in error node1.query( - "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n") + "CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n" + ) node1.query("DROP TABLE test3_r1 SYNC") node1.query("DROP TABLE test3_r2 SYNC") + def test_attach_without_zk(start_cluster): node1.query_with_retry( - "CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n") + "CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n" + ) node1.query("DETACH TABLE test4_r1") with PartitionManager() as pm: - pm._add_rule({'probability': 0.5, 'source': node1.ip_address, 'destination_port': 2181, 'action': 'DROP'}) + pm._add_rule( + { + "probability": 0.5, + "source": node1.ip_address, + "destination_port": 2181, + "action": "DROP", + } + ) try: node1.query("ATTACH TABLE test4_r1") except: diff --git a/tests/integration/test_cluster_all_replicas/test.py b/tests/integration/test_cluster_all_replicas/test.py index 7cb170ce52a..445eef64fcb 100644 --- a/tests/integration/test_cluster_all_replicas/test.py +++ b/tests/integration/test_cluster_all_replicas/test.py @@ -4,8 +4,12 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -18,5 +22,13 @@ def start_cluster(): def test_remote(start_cluster): - assert node1.query('''SELECT hostName() FROM clusterAllReplicas("two_shards", system.one)''') == 'node1\nnode2\n' - assert node1.query('''SELECT hostName() FROM cluster("two_shards", system.one)''') == 'node1\n' + assert ( + node1.query( + """SELECT hostName() FROM clusterAllReplicas("two_shards", system.one)""" + ) + == "node1\nnode2\n" + ) + assert ( + node1.query("""SELECT hostName() FROM cluster("two_shards", system.one)""") + == "node1\n" + ) diff --git a/tests/integration/test_cluster_copier/test.py b/tests/integration/test_cluster_copier/test.py index 3d28295d40e..14417f151ee 100644 --- a/tests/integration/test_cluster_copier/test.py +++ b/tests/integration/test_cluster_copier/test.py @@ -18,10 +18,13 @@ sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) COPYING_FAIL_PROBABILITY = 0.2 MOVING_FAIL_PROBABILITY = 0.2 -cluster = ClickHouseCluster(__file__, name='copier_test') +cluster = ClickHouseCluster(__file__, name="copier_test") + def generateRandomString(count): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(count)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(count) + ) def check_all_hosts_sucesfully_executed(tsv_content, num_hosts): @@ -45,26 +48,29 @@ def started_cluster(): global cluster try: clusters_schema = { - "0": { - "0": ["0", "1"], - "1": ["0"] - }, - "1": { - "0": ["0", "1"], - "1": ["0"] - } + "0": {"0": ["0", "1"], "1": ["0"]}, + "1": {"0": ["0", "1"], "1": ["0"]}, } for cluster_name, shards in clusters_schema.items(): for shard_name, replicas in shards.items(): for replica_name in replicas: name = "s{}_{}_{}".format(cluster_name, shard_name, replica_name) - cluster.add_instance(name, - main_configs=["configs/conf.d/query_log.xml", "configs/conf.d/ddl.xml", - "configs/conf.d/clusters.xml"], - user_configs=["configs/users.xml"], - macros={"cluster": cluster_name, "shard": shard_name, "replica": replica_name}, - with_zookeeper=True) + cluster.add_instance( + name, + main_configs=[ + "configs/conf.d/query_log.xml", + "configs/conf.d/ddl.xml", + "configs/conf.d/clusters.xml", + ], + user_configs=["configs/users.xml"], + macros={ + "cluster": cluster_name, + "shard": shard_name, + "replica": replica_name, + }, + with_zookeeper=True, + ) cluster.start() yield cluster @@ -74,7 +80,6 @@ def started_cluster(): class Task1: - def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_simple_" + generateRandomString(10) @@ -82,36 +87,78 @@ class Task1: for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task0_description.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task0_description.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] for cluster_num in ["0", "1"]: - ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{} SYNC".format(cluster_num)) - ddl_check_query(instance, - "CREATE DATABASE default ON CLUSTER cluster{} ".format( - cluster_num)) + ddl_check_query( + instance, + "DROP DATABASE IF EXISTS default ON CLUSTER cluster{} SYNC".format( + cluster_num + ), + ) + ddl_check_query( + instance, + "CREATE DATABASE default ON CLUSTER cluster{} ".format(cluster_num), + ) - ddl_check_query(instance, "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + - "ENGINE=ReplicatedMergeTree " + - "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") - ddl_check_query(instance, - "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)") - ddl_check_query(instance, - "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)") - instance.query("INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", - settings={"insert_distributed_sync": 1}) + ddl_check_query( + instance, + "CREATE TABLE hits ON CLUSTER cluster0 (d UInt64, d1 UInt64 MATERIALIZED d+1) " + + "ENGINE=ReplicatedMergeTree " + + "PARTITION BY d % 3 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16", + ) + ddl_check_query( + instance, + "CREATE TABLE hits_all ON CLUSTER cluster0 (d UInt64) ENGINE=Distributed(cluster0, default, hits, d)", + ) + ddl_check_query( + instance, + "CREATE TABLE hits_all ON CLUSTER cluster1 (d UInt64) ENGINE=Distributed(cluster1, default, hits, d + 1)", + ) + instance.query( + "INSERT INTO hits_all SELECT * FROM system.numbers LIMIT 1002", + settings={"insert_distributed_sync": 1}, + ) def check(self): - assert self.cluster.instances['s0_0_0'].query("SELECT count() FROM hits_all").strip() == "1002" - assert self.cluster.instances['s1_0_0'].query("SELECT count() FROM hits_all").strip() == "1002" + assert ( + self.cluster.instances["s0_0_0"] + .query("SELECT count() FROM hits_all") + .strip() + == "1002" + ) + assert ( + self.cluster.instances["s1_0_0"] + .query("SELECT count() FROM hits_all") + .strip() + == "1002" + ) - assert self.cluster.instances['s1_0_0'].query("SELECT DISTINCT d % 2 FROM hits").strip() == "1" - assert self.cluster.instances['s1_1_0'].query("SELECT DISTINCT d % 2 FROM hits").strip() == "0" + assert ( + self.cluster.instances["s1_0_0"] + .query("SELECT DISTINCT d % 2 FROM hits") + .strip() + == "1" + ) + assert ( + self.cluster.instances["s1_1_0"] + .query("SELECT DISTINCT d % 2 FROM hits") + .strip() + == "0" + ) - instance = self.cluster.instances['s0_0_0'] + instance = self.cluster.instances["s0_0_0"] ddl_check_query(instance, "DROP TABLE hits_all ON CLUSTER cluster0") ddl_check_query(instance, "DROP TABLE hits_all ON CLUSTER cluster1") ddl_check_query(instance, "DROP TABLE hits ON CLUSTER cluster0") @@ -119,124 +166,193 @@ class Task1: class Task2: - def __init__(self, cluster, unique_zk_path): self.cluster = cluster - self.zk_task_path = "/clickhouse-copier/task_month_to_week_partition_" + generateRandomString(5) + self.zk_task_path = ( + "/clickhouse-copier/task_month_to_week_partition_" + generateRandomString(5) + ) self.unique_zk_path = generateRandomString(10) self.container_task_file = "/task_month_to_week_description.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_month_to_week_description.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_month_to_week_description.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] for cluster_num in ["0", "1"]: - ddl_check_query(instance, "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format(cluster_num)) - ddl_check_query(instance, - "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( - cluster_num)) + ddl_check_query( + instance, + "DROP DATABASE IF EXISTS default ON CLUSTER cluster{}".format( + cluster_num + ), + ) + ddl_check_query( + instance, + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER cluster{}".format( + cluster_num + ), + ) - ddl_check_query(instance, - "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) " - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/" + self.unique_zk_path + "', " - "'{replica}', date, intHash64(d), (date, intHash64(d)), 8192)") - ddl_check_query(instance, - "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)") + ddl_check_query( + instance, + "CREATE TABLE a ON CLUSTER cluster0 (date Date, d UInt64, d1 UInt64 ALIAS d+1) " + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/cluster_{cluster}/{shard}/" + + self.unique_zk_path + + "', " + "'{replica}', date, intHash64(d), (date, intHash64(d)), 8192)", + ) + ddl_check_query( + instance, + "CREATE TABLE a_all ON CLUSTER cluster0 (date Date, d UInt64) ENGINE=Distributed(cluster0, default, a, d)", + ) instance.query( "INSERT INTO a_all SELECT toDate(17581 + number) AS date, number AS d FROM system.numbers LIMIT 85", - settings={"insert_distributed_sync": 1}) + settings={"insert_distributed_sync": 1}, + ) def check(self): - assert TSV(self.cluster.instances['s0_0_0'].query("SELECT count() FROM cluster(cluster0, default, a)")) == TSV( - "85\n") - assert TSV(self.cluster.instances['s1_0_0'].query( - "SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)")) == TSV("85\t85\n") + assert TSV( + self.cluster.instances["s0_0_0"].query( + "SELECT count() FROM cluster(cluster0, default, a)" + ) + ) == TSV("85\n") + assert TSV( + self.cluster.instances["s1_0_0"].query( + "SELECT count(), uniqExact(date) FROM cluster(cluster1, default, b)" + ) + ) == TSV("85\t85\n") - assert TSV(self.cluster.instances['s1_0_0'].query( - "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("0\n") - assert TSV(self.cluster.instances['s1_1_0'].query( - "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b")) == TSV("1\n") + assert TSV( + self.cluster.instances["s1_0_0"].query( + "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b" + ) + ) == TSV("0\n") + assert TSV( + self.cluster.instances["s1_1_0"].query( + "SELECT DISTINCT jumpConsistentHash(intHash64(d), 2) FROM b" + ) + ) == TSV("1\n") - assert TSV(self.cluster.instances['s1_0_0'].query( - "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV( - "1\n") - assert TSV(self.cluster.instances['s1_1_0'].query( - "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'")) == TSV( - "1\n") + assert TSV( + self.cluster.instances["s1_0_0"].query( + "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'" + ) + ) == TSV("1\n") + assert TSV( + self.cluster.instances["s1_1_0"].query( + "SELECT uniqExact(partition) IN (12, 13) FROM system.parts WHERE active AND database='default' AND table='b'" + ) + ) == TSV("1\n") - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] ddl_check_query(instance, "DROP TABLE a ON CLUSTER cluster0") ddl_check_query(instance, "DROP TABLE b ON CLUSTER cluster1") class Task_test_block_size: - def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = "/clickhouse-copier/task_test_block_size_" + generateRandomString(5) + self.zk_task_path = ( + "/clickhouse-copier/task_test_block_size_" + generateRandomString(5) + ) self.rows = 1000000 self.container_task_file = "/task_test_block_size.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_test_block_size.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_test_block_size.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] - ddl_check_query(instance, """ + ddl_check_query( + instance, + """ CREATE TABLE test_block_size ON CLUSTER shard_0_0 (partition Date, d UInt64) ENGINE=ReplicatedMergeTree - ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", 2) + ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d)""", + 2, + ) instance.query( "INSERT INTO test_block_size SELECT toDate(0) AS partition, number as d FROM system.numbers LIMIT {}".format( - self.rows)) + self.rows + ) + ) def check(self): - assert TSV(self.cluster.instances['s1_0_0'].query( - "SELECT count() FROM cluster(cluster1, default, test_block_size)")) == TSV("{}\n".format(self.rows)) + assert TSV( + self.cluster.instances["s1_0_0"].query( + "SELECT count() FROM cluster(cluster1, default, test_block_size)" + ) + ) == TSV("{}\n".format(self.rows)) - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] ddl_check_query(instance, "DROP TABLE test_block_size ON CLUSTER shard_0_0", 2) ddl_check_query(instance, "DROP TABLE test_block_size ON CLUSTER cluster1") class Task_no_index: - def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = "/clickhouse-copier/task_no_index_" + generateRandomString(5) + self.zk_task_path = "/clickhouse-copier/task_no_index_" + generateRandomString( + 5 + ) self.rows = 1000000 self.container_task_file = "/task_no_index.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_no_index.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_no_index.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] instance.query("DROP TABLE IF EXISTS ontime SYNC") - instance.query("create table IF NOT EXISTS ontime (Year UInt16, FlightDate String) ENGINE = Memory") - instance.query("insert into ontime values (2016, 'test6'), (2017, 'test7'), (2018, 'test8')") + instance.query( + "create table IF NOT EXISTS ontime (Year UInt16, FlightDate String) ENGINE = Memory" + ) + instance.query( + "insert into ontime values (2016, 'test6'), (2017, 'test7'), (2018, 'test8')" + ) def check(self): - assert TSV(self.cluster.instances['s1_1_0'].query("SELECT Year FROM ontime22")) == TSV("2017\n") - instance = cluster.instances['s0_0_0'] + assert TSV( + self.cluster.instances["s1_1_0"].query("SELECT Year FROM ontime22") + ) == TSV("2017\n") + instance = cluster.instances["s0_0_0"] instance.query("DROP TABLE ontime") - instance = cluster.instances['s1_1_0'] + instance = cluster.instances["s1_1_0"] instance.query("DROP TABLE ontime22") class Task_no_arg: - def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_no_arg" @@ -245,25 +361,35 @@ class Task_no_arg: for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_no_arg.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_no_arg.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] instance.query("DROP TABLE IF EXISTS copier_test1 SYNC") instance.query( - "create table if not exists copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192") + "create table if not exists copier_test1 (date Date, id UInt32) engine = MergeTree PARTITION BY date ORDER BY date SETTINGS index_granularity = 8192" + ) instance.query("insert into copier_test1 values ('2016-01-01', 10);") def check(self): - assert TSV(self.cluster.instances['s1_1_0'].query("SELECT date FROM copier_test1_1")) == TSV("2016-01-01\n") - instance = cluster.instances['s0_0_0'] + assert TSV( + self.cluster.instances["s1_1_0"].query("SELECT date FROM copier_test1_1") + ) == TSV("2016-01-01\n") + instance = cluster.instances["s0_0_0"] instance.query("DROP TABLE copier_test1 SYNC") - instance = cluster.instances['s1_1_0'] + instance = cluster.instances["s1_1_0"] instance.query("DROP TABLE copier_test1_1 SYNC") -class Task_non_partitioned_table: +class Task_non_partitioned_table: def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_non_partitoned_table" @@ -272,25 +398,35 @@ class Task_non_partitioned_table: for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_non_partitioned_table.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_non_partitioned_table.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] instance.query("DROP TABLE IF EXISTS copier_test1 SYNC") instance.query( - "create table copier_test1 (date Date, id UInt32) engine = MergeTree ORDER BY date SETTINGS index_granularity = 8192") + "create table copier_test1 (date Date, id UInt32) engine = MergeTree ORDER BY date SETTINGS index_granularity = 8192" + ) instance.query("insert into copier_test1 values ('2016-01-01', 10);") def check(self): - assert TSV(self.cluster.instances['s1_1_0'].query("SELECT date FROM copier_test1_1")) == TSV("2016-01-01\n") - instance = cluster.instances['s0_0_0'] + assert TSV( + self.cluster.instances["s1_1_0"].query("SELECT date FROM copier_test1_1") + ) == TSV("2016-01-01\n") + instance = cluster.instances["s0_0_0"] instance.query("DROP TABLE copier_test1") - instance = cluster.instances['s1_1_0'] + instance = cluster.instances["s1_1_0"] instance.query("DROP TABLE copier_test1_1") -class Task_self_copy: +class Task_self_copy: def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_self_copy" @@ -298,26 +434,37 @@ class Task_self_copy: for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_self_copy.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_self_copy.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] instance.query("DROP DATABASE IF EXISTS db1 SYNC") instance.query("DROP DATABASE IF EXISTS db2 SYNC") instance.query("CREATE DATABASE IF NOT EXISTS db1;") instance.query( - "CREATE TABLE IF NOT EXISTS db1.source_table (`a` Int8, `b` String, `c` Int8) ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity = 8192") + "CREATE TABLE IF NOT EXISTS db1.source_table (`a` Int8, `b` String, `c` Int8) ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity = 8192" + ) instance.query("CREATE DATABASE IF NOT EXISTS db2;") instance.query( - "CREATE TABLE IF NOT EXISTS db2.destination_table (`a` Int8, `b` String, `c` Int8) ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity = 8192") + "CREATE TABLE IF NOT EXISTS db2.destination_table (`a` Int8, `b` String, `c` Int8) ENGINE = MergeTree PARTITION BY a ORDER BY a SETTINGS index_granularity = 8192" + ) instance.query("INSERT INTO db1.source_table VALUES (1, 'ClickHouse', 1);") instance.query("INSERT INTO db1.source_table VALUES (2, 'Copier', 2);") def check(self): - instance = cluster.instances['s0_0_0'] - assert TSV(instance.query("SELECT * FROM db2.destination_table ORDER BY a")) == TSV(instance.query("SELECT * FROM db1.source_table ORDER BY a")) - instance = cluster.instances['s0_0_0'] + instance = cluster.instances["s0_0_0"] + assert TSV( + instance.query("SELECT * FROM db2.destination_table ORDER BY a") + ) == TSV(instance.query("SELECT * FROM db1.source_table ORDER BY a")) + instance = cluster.instances["s0_0_0"] instance.query("DROP DATABASE IF EXISTS db1 SYNC") instance.query("DROP DATABASE IF EXISTS db2 SYNC") @@ -325,10 +472,9 @@ class Task_self_copy: def execute_task(started_cluster, task, cmd_options): task.start() - zk = started_cluster.get_kazoo_client('zoo1') + zk = started_cluster.get_kazoo_client("zoo1") print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) - try: zk.delete("/clickhouse-copier", recursive=True) except kazoo.exceptions.NoNodeError: @@ -338,12 +484,20 @@ def execute_task(started_cluster, task, cmd_options): docker_api = started_cluster.docker_client.api copiers_exec_ids = [] - cmd = ['/usr/bin/clickhouse', 'copier', - '--config', '/etc/clickhouse-server/config-copier.xml', - '--task-path', task.zk_task_path, - '--task-file', task.container_task_file, - '--task-upload-force', 'true', - '--base-dir', '/var/log/clickhouse-server/copier'] + cmd = [ + "/usr/bin/clickhouse", + "copier", + "--config", + "/etc/clickhouse-server/config-copier.xml", + "--task-path", + task.zk_task_path, + "--task-file", + task.container_task_file, + "--task-upload-force", + "true", + "--base-dir", + "/var/log/clickhouse-server/copier", + ] cmd += cmd_options print(cmd) @@ -353,25 +507,31 @@ def execute_task(started_cluster, task, cmd_options): for instance_name in copiers: instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), - "/etc/clickhouse-server/config-copier.xml") + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), + "/etc/clickhouse-server/config-copier.xml", + ) print("Copied copier config to {}".format(instance.name)) exec_id = docker_api.exec_create(container.id, cmd, stderr=True) - output = docker_api.exec_start(exec_id).decode('utf8') + output = docker_api.exec_start(exec_id).decode("utf8") print(output) copiers_exec_ids.append(exec_id) - print("Copier for {} ({}) has started".format(instance.name, instance.ip_address)) + print( + "Copier for {} ({}) has started".format(instance.name, instance.ip_address) + ) # Wait for copiers stopping and check their return codes for exec_id, instance_name in zip(copiers_exec_ids, copiers): instance = started_cluster.instances[instance_name] while True: res = docker_api.exec_inspect(exec_id) - if not res['Running']: + if not res["Running"]: break time.sleep(0.5) - assert res['ExitCode'] == 0, "Instance: {} ({}). Info: {}".format(instance.name, instance.ip_address, repr(res)) + assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( + instance.name, instance.ip_address, repr(res) + ) try: task.check() @@ -381,30 +541,59 @@ def execute_task(started_cluster, task, cmd_options): # Tests -@pytest.mark.parametrize(('use_sample_offset'), [False, True]) + +@pytest.mark.parametrize(("use_sample_offset"), [False, True]) def test_copy_simple(started_cluster, use_sample_offset): if use_sample_offset: - execute_task(started_cluster, Task1(started_cluster), ['--experimental-use-sample-offset', '1']) + execute_task( + started_cluster, + Task1(started_cluster), + ["--experimental-use-sample-offset", "1"], + ) else: execute_task(started_cluster, Task1(started_cluster), []) -@pytest.mark.parametrize(('use_sample_offset'),[False, True]) +@pytest.mark.parametrize(("use_sample_offset"), [False, True]) def test_copy_with_recovering(started_cluster, use_sample_offset): if use_sample_offset: - execute_task(started_cluster, Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY), - '--experimental-use-sample-offset', '1']) + execute_task( + started_cluster, + Task1(started_cluster), + [ + "--copy-fault-probability", + str(COPYING_FAIL_PROBABILITY), + "--experimental-use-sample-offset", + "1", + ], + ) else: - execute_task(started_cluster, Task1(started_cluster), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + execute_task( + started_cluster, + Task1(started_cluster), + ["--copy-fault-probability", str(COPYING_FAIL_PROBABILITY)], + ) -@pytest.mark.parametrize(('use_sample_offset'),[False, True]) +@pytest.mark.parametrize(("use_sample_offset"), [False, True]) def test_copy_with_recovering_after_move_faults(started_cluster, use_sample_offset): if use_sample_offset: - execute_task(started_cluster, Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY), - '--experimental-use-sample-offset', '1']) + execute_task( + started_cluster, + Task1(started_cluster), + [ + "--move-fault-probability", + str(MOVING_FAIL_PROBABILITY), + "--experimental-use-sample-offset", + "1", + ], + ) else: - execute_task(started_cluster, Task1(started_cluster), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) + execute_task( + started_cluster, + Task1(started_cluster), + ["--move-fault-probability", str(MOVING_FAIL_PROBABILITY)], + ) @pytest.mark.timeout(600) @@ -414,12 +603,22 @@ def test_copy_month_to_week_partition(started_cluster): @pytest.mark.timeout(600) def test_copy_month_to_week_partition_with_recovering(started_cluster): - execute_task(started_cluster, Task2(started_cluster, "test2"), ['--copy-fault-probability', str(COPYING_FAIL_PROBABILITY)]) + execute_task( + started_cluster, + Task2(started_cluster, "test2"), + ["--copy-fault-probability", str(COPYING_FAIL_PROBABILITY)], + ) @pytest.mark.timeout(600) -def test_copy_month_to_week_partition_with_recovering_after_move_faults(started_cluster): - execute_task(started_cluster, Task2(started_cluster, "test3"), ['--move-fault-probability', str(MOVING_FAIL_PROBABILITY)]) +def test_copy_month_to_week_partition_with_recovering_after_move_faults( + started_cluster, +): + execute_task( + started_cluster, + Task2(started_cluster, "test3"), + ["--move-fault-probability", str(MOVING_FAIL_PROBABILITY)], + ) def test_block_size(started_cluster): diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index 63b0bcc6679..c8039792fe8 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -12,7 +12,8 @@ import docker CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) -cluster = ClickHouseCluster(__file__, name='copier_test_three_nodes') +cluster = ClickHouseCluster(__file__, name="copier_test_three_nodes") + @pytest.fixture(scope="module") def started_cluster(): @@ -20,9 +21,15 @@ def started_cluster(): try: for name in ["first", "second", "third"]: - cluster.add_instance(name, - main_configs=["configs_three_nodes/conf.d/clusters.xml", "configs_three_nodes/conf.d/ddl.xml"], user_configs=["configs_three_nodes/users.xml"], - with_zookeeper=True) + cluster.add_instance( + name, + main_configs=[ + "configs_three_nodes/conf.d/clusters.xml", + "configs_three_nodes/conf.d/ddl.xml", + ], + user_configs=["configs_three_nodes/users.xml"], + with_zookeeper=True, + ) cluster.start() yield cluster @@ -30,17 +37,22 @@ def started_cluster(): finally: cluster.shutdown() + class Task: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = '/clickhouse-copier/task' + self.zk_task_path = "/clickhouse-copier/task" self.container_task_file = "/task_taxi_data.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_taxi_data.xml'), self.container_task_file) - logging.debug(f"Copied task file to container of '{instance_name}' instance. Path {self.container_task_file}") - + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_taxi_data.xml"), + self.container_task_file, + ) + logging.debug( + f"Copied task file to container of '{instance_name}' instance. Path {self.container_task_file}" + ) def start(self): for name in ["first", "second", "third"]: @@ -48,11 +60,12 @@ class Task: node.query("DROP DATABASE IF EXISTS dailyhistory SYNC;") node.query("DROP DATABASE IF EXISTS monthlyhistory SYNC;") - first = cluster.instances['first'] + first = cluster.instances["first"] # daily partition database first.query("CREATE DATABASE IF NOT EXISTS dailyhistory on cluster events;") - first.query("""CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events + first.query( + """CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events ( id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -82,14 +95,18 @@ class Task: Engine = ReplacingMergeTree() PRIMARY KEY (tpep_pickup_datetime, id) ORDER BY (tpep_pickup_datetime, id) - PARTITION BY (toYYYYMMDD(tpep_pickup_datetime))""") + PARTITION BY (toYYYYMMDD(tpep_pickup_datetime))""" + ) - first.query("""CREATE TABLE dailyhistory.yellow_tripdata + first.query( + """CREATE TABLE dailyhistory.yellow_tripdata ON CLUSTER events AS dailyhistory.yellow_tripdata_staging - ENGINE = Distributed('events', 'dailyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""") + ENGINE = Distributed('events', 'dailyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""" + ) - first.query("""INSERT INTO dailyhistory.yellow_tripdata + first.query( + """INSERT INTO dailyhistory.yellow_tripdata SELECT * FROM generateRandom( 'id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -116,11 +133,13 @@ class Task: congestion_surcharge String, junk1 String, junk2 String', - 1, 10, 2) LIMIT 50;""") + 1, 10, 2) LIMIT 50;""" + ) # monthly partition database first.query("create database IF NOT EXISTS monthlyhistory on cluster events;") - first.query("""CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events + first.query( + """CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events ( id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -151,13 +170,15 @@ class Task: Engine = ReplacingMergeTree() PRIMARY KEY (tpep_pickup_datetime, id) ORDER BY (tpep_pickup_datetime, id) - PARTITION BY (pickup_location_id, toYYYYMM(tpep_pickup_datetime))""") + PARTITION BY (pickup_location_id, toYYYYMM(tpep_pickup_datetime))""" + ) - first.query("""CREATE TABLE monthlyhistory.yellow_tripdata + first.query( + """CREATE TABLE monthlyhistory.yellow_tripdata ON CLUSTER events AS monthlyhistory.yellow_tripdata_staging - ENGINE = Distributed('events', 'monthlyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""") - + ENGINE = Distributed('events', 'monthlyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""" + ) def check(self): first = cluster.instances["first"] @@ -167,12 +188,24 @@ class Task: for instance_name, instance in cluster.instances.items(): instance = cluster.instances[instance_name] - a = instance.query("SELECT count() from dailyhistory.yellow_tripdata_staging") - b = instance.query("SELECT count() from monthlyhistory.yellow_tripdata_staging") + a = instance.query( + "SELECT count() from dailyhistory.yellow_tripdata_staging" + ) + b = instance.query( + "SELECT count() from monthlyhistory.yellow_tripdata_staging" + ) assert a == b, "MergeTree tables on each shard" - a = TSV(instance.query("SELECT sipHash64(*) from dailyhistory.yellow_tripdata_staging ORDER BY id")) - b = TSV(instance.query("SELECT sipHash64(*) from monthlyhistory.yellow_tripdata_staging ORDER BY id")) + a = TSV( + instance.query( + "SELECT sipHash64(*) from dailyhistory.yellow_tripdata_staging ORDER BY id" + ) + ) + b = TSV( + instance.query( + "SELECT sipHash64(*) from monthlyhistory.yellow_tripdata_staging ORDER BY id" + ) + ) assert a == b, "Data on each shard" @@ -182,23 +215,30 @@ class Task: node.query("DROP DATABASE IF EXISTS monthlyhistory SYNC;") - def execute_task(started_cluster, task, cmd_options): task.start() - zk = started_cluster.get_kazoo_client('zoo1') + zk = started_cluster.get_kazoo_client("zoo1") logging.debug("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) # Run cluster-copier processes on each node docker_api = started_cluster.docker_client.api copiers_exec_ids = [] - cmd = ['/usr/bin/clickhouse', 'copier', - '--config', '/etc/clickhouse-server/config-copier.xml', - '--task-path', task.zk_task_path, - '--task-file', task.container_task_file, - '--task-upload-force', 'true', - '--base-dir', '/var/log/clickhouse-server/copier'] + cmd = [ + "/usr/bin/clickhouse", + "copier", + "--config", + "/etc/clickhouse-server/config-copier.xml", + "--task-path", + task.zk_task_path, + "--task-file", + task.container_task_file, + "--task-upload-force", + "true", + "--base-dir", + "/var/log/clickhouse-server/copier", + ] cmd += cmd_options logging.debug(f"execute_task cmd: {cmd}") @@ -206,25 +246,34 @@ def execute_task(started_cluster, task, cmd_options): for instance_name in started_cluster.instances.keys(): instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs_three_nodes/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs_three_nodes/config-copier.xml"), + "/etc/clickhouse-server/config-copier.xml", + ) logging.info("Copied copier config to {}".format(instance.name)) exec_id = docker_api.exec_create(container.id, cmd, stderr=True) - output = docker_api.exec_start(exec_id).decode('utf8') + output = docker_api.exec_start(exec_id).decode("utf8") logging.info(output) copiers_exec_ids.append(exec_id) - logging.info("Copier for {} ({}) has started".format(instance.name, instance.ip_address)) + logging.info( + "Copier for {} ({}) has started".format(instance.name, instance.ip_address) + ) # time.sleep(1000) # Wait for copiers stopping and check their return codes - for exec_id, instance in zip(copiers_exec_ids, iter(started_cluster.instances.values())): + for exec_id, instance in zip( + copiers_exec_ids, iter(started_cluster.instances.values()) + ): while True: res = docker_api.exec_inspect(exec_id) - if not res['Running']: + if not res["Running"]: break time.sleep(1) - assert res['ExitCode'] == 0, "Instance: {} ({}). Info: {}".format(instance.name, instance.ip_address, repr(res)) + assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( + instance.name, instance.ip_address, repr(res) + ) try: task.check() diff --git a/tests/integration/test_cluster_copier/test_trivial.py b/tests/integration/test_cluster_copier/test_trivial.py index e58c6edcb4d..84bf39f0d76 100644 --- a/tests/integration/test_cluster_copier/test_trivial.py +++ b/tests/integration/test_cluster_copier/test_trivial.py @@ -19,11 +19,13 @@ sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) COPYING_FAIL_PROBABILITY = 0.1 MOVING_FAIL_PROBABILITY = 0.1 -cluster = ClickHouseCluster(__file__, name='copier_test_trivial') +cluster = ClickHouseCluster(__file__, name="copier_test_trivial") def generateRandomString(count): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(count)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(count) + ) @pytest.fixture(scope="module") @@ -31,11 +33,17 @@ def started_cluster(): global cluster try: for name in ["first_trivial", "second_trivial"]: - instance = cluster.add_instance(name, + instance = cluster.add_instance( + name, main_configs=["configs/conf.d/clusters_trivial.xml"], user_configs=["configs_two_nodes/users.xml"], - macros={"cluster" : name, "shard" : "the_only_shard", "replica" : "the_only_replica"}, - with_zookeeper=True) + macros={ + "cluster": name, + "shard": "the_only_shard", + "replica": "the_only_replica", + }, + with_zookeeper=True, + ) cluster.start() yield cluster @@ -48,30 +56,41 @@ class TaskTrivial: def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_trivial" - self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial.xml'), 'r').read() + self.copier_task_config = open( + os.path.join(CURRENT_TEST_DIR, "task_trivial.xml"), "r" + ).read() def start(self): - source = cluster.instances['first_trivial'] - destination = cluster.instances['second_trivial'] + source = cluster.instances["first_trivial"] + destination = cluster.instances["second_trivial"] for node in [source, destination]: node.query("DROP DATABASE IF EXISTS default") node.query("CREATE DATABASE IF NOT EXISTS default") - source.query("CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1)" - "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial/{}', '1') " - "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16".format(generateRandomString(10))) + source.query( + "CREATE TABLE trivial (d UInt64, d1 UInt64 MATERIALIZED d+1)" + "ENGINE=ReplicatedMergeTree('/clickhouse/tables/source_trivial_cluster/1/trivial/{}', '1') " + "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16".format( + generateRandomString(10) + ) + ) - source.query("INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", - settings={"insert_distributed_sync": 1}) + source.query( + "INSERT INTO trivial SELECT * FROM system.numbers LIMIT 1002", + settings={"insert_distributed_sync": 1}, + ) def check(self): - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") status_data, _ = zk.get(self.zk_task_path + "/status") - assert status_data == b'{"hits":{"all_partitions_count":5,"processed_partitions_count":5}}' + assert ( + status_data + == b'{"hits":{"all_partitions_count":5,"processed_partitions_count":5}}' + ) - source = cluster.instances['first_trivial'] - destination = cluster.instances['second_trivial'] + source = cluster.instances["first_trivial"] + destination = cluster.instances["second_trivial"] assert TSV(source.query("SELECT count() FROM trivial")) == TSV("1002\n") assert TSV(destination.query("SELECT count() FROM trivial")) == TSV("1002\n") @@ -84,33 +103,46 @@ class TaskReplicatedWithoutArguments: def __init__(self, cluster): self.cluster = cluster self.zk_task_path = "/clickhouse-copier/task_trivial_without_arguments" - self.copier_task_config = open(os.path.join(CURRENT_TEST_DIR, 'task_trivial_without_arguments.xml'), 'r').read() + self.copier_task_config = open( + os.path.join(CURRENT_TEST_DIR, "task_trivial_without_arguments.xml"), "r" + ).read() def start(self): - source = cluster.instances['first_trivial'] - destination = cluster.instances['second_trivial'] + source = cluster.instances["first_trivial"] + destination = cluster.instances["second_trivial"] for node in [source, destination]: node.query("DROP DATABASE IF EXISTS default") node.query("CREATE DATABASE IF NOT EXISTS default") - source.query("CREATE TABLE trivial_without_arguments ON CLUSTER source_trivial_cluster (d UInt64, d1 UInt64 MATERIALIZED d+1) " - "ENGINE=ReplicatedMergeTree() " - "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16") + source.query( + "CREATE TABLE trivial_without_arguments ON CLUSTER source_trivial_cluster (d UInt64, d1 UInt64 MATERIALIZED d+1) " + "ENGINE=ReplicatedMergeTree() " + "PARTITION BY d % 5 ORDER BY (d, sipHash64(d)) SAMPLE BY sipHash64(d) SETTINGS index_granularity = 16" + ) - source.query("INSERT INTO trivial_without_arguments SELECT * FROM system.numbers LIMIT 1002", - settings={"insert_distributed_sync": 1}) + source.query( + "INSERT INTO trivial_without_arguments SELECT * FROM system.numbers LIMIT 1002", + settings={"insert_distributed_sync": 1}, + ) def check(self): - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") status_data, _ = zk.get(self.zk_task_path + "/status") - assert status_data == b'{"hits":{"all_partitions_count":5,"processed_partitions_count":5}}' + assert ( + status_data + == b'{"hits":{"all_partitions_count":5,"processed_partitions_count":5}}' + ) - source = cluster.instances['first_trivial'] - destination = cluster.instances['second_trivial'] + source = cluster.instances["first_trivial"] + destination = cluster.instances["second_trivial"] - assert TSV(source.query("SELECT count() FROM trivial_without_arguments")) == TSV("1002\n") - assert TSV(destination.query("SELECT count() FROM trivial_without_arguments")) == TSV("1002\n") + assert TSV( + source.query("SELECT count() FROM trivial_without_arguments") + ) == TSV("1002\n") + assert TSV( + destination.query("SELECT count() FROM trivial_without_arguments") + ) == TSV("1002\n") for node in [source, destination]: node.query("DROP TABLE trivial_without_arguments") @@ -119,7 +151,7 @@ class TaskReplicatedWithoutArguments: def execute_task(started_cluster, task, cmd_options): task.start() - zk = started_cluster.get_kazoo_client('zoo1') + zk = started_cluster.get_kazoo_client("zoo1") print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) try: @@ -135,10 +167,16 @@ def execute_task(started_cluster, task, cmd_options): docker_api = started_cluster.docker_client.api copiers_exec_ids = [] - cmd = ['/usr/bin/clickhouse', 'copier', - '--config', '/etc/clickhouse-server/config-copier.xml', - '--task-path', zk_task_path, - '--base-dir', '/var/log/clickhouse-server/copier'] + cmd = [ + "/usr/bin/clickhouse", + "copier", + "--config", + "/etc/clickhouse-server/config-copier.xml", + "--task-path", + zk_task_path, + "--base-dir", + "/var/log/clickhouse-server/copier", + ] cmd += cmd_options copiers = list(started_cluster.instances.keys()) @@ -146,25 +184,31 @@ def execute_task(started_cluster, task, cmd_options): for instance_name in copiers: instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), - "/etc/clickhouse-server/config-copier.xml") + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/config-copier.xml"), + "/etc/clickhouse-server/config-copier.xml", + ) print("Copied copier config to {}".format(instance.name)) exec_id = docker_api.exec_create(container.id, cmd, stderr=True) - output = docker_api.exec_start(exec_id).decode('utf8') + output = docker_api.exec_start(exec_id).decode("utf8") print(output) copiers_exec_ids.append(exec_id) - print("Copier for {} ({}) has started".format(instance.name, instance.ip_address)) + print( + "Copier for {} ({}) has started".format(instance.name, instance.ip_address) + ) # Wait for copiers stopping and check their return codes for exec_id, instance_name in zip(copiers_exec_ids, copiers): instance = started_cluster.instances[instance_name] while True: res = docker_api.exec_inspect(exec_id) - if not res['Running']: + if not res["Running"]: break time.sleep(0.5) - assert res['ExitCode'] == 0, "Instance: {} ({}). Info: {}".format(instance.name, instance.ip_address, repr(res)) + assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( + instance.name, instance.ip_address, repr(res) + ) try: task.check() @@ -174,6 +218,7 @@ def execute_task(started_cluster, task, cmd_options): # Tests + def test_trivial_copy(started_cluster): execute_task(started_cluster, TaskTrivial(started_cluster), []) diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py index 255af13213a..6fdaaeea720 100644 --- a/tests/integration/test_cluster_copier/test_two_nodes.py +++ b/tests/integration/test_cluster_copier/test_two_nodes.py @@ -12,7 +12,7 @@ import docker CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(CURRENT_TEST_DIR)) -cluster = ClickHouseCluster(__file__, name='copier_test_two_nodes') +cluster = ClickHouseCluster(__file__, name="copier_test_two_nodes") @pytest.fixture(scope="module") @@ -21,38 +21,49 @@ def started_cluster(): try: for name in ["first_of_two", "second_of_two"]: - instance = cluster.add_instance(name, + instance = cluster.add_instance( + name, main_configs=[ "configs_two_nodes/conf.d/clusters.xml", "configs_two_nodes/conf.d/ddl.xml", - "configs_two_nodes/conf.d/storage_configuration.xml"], + "configs_two_nodes/conf.d/storage_configuration.xml", + ], user_configs=["configs_two_nodes/users.xml"], - with_zookeeper=True) + with_zookeeper=True, + ) cluster.start() for name in ["first_of_two", "second_of_two"]: instance = cluster.instances[name] - instance.exec_in_container(['bash', '-c', 'mkdir /jbod1']) - instance.exec_in_container(['bash', '-c', 'mkdir /jbod2']) - instance.exec_in_container(['bash', '-c', 'mkdir /external']) + instance.exec_in_container(["bash", "-c", "mkdir /jbod1"]) + instance.exec_in_container(["bash", "-c", "mkdir /jbod2"]) + instance.exec_in_container(["bash", "-c", "mkdir /external"]) yield cluster finally: cluster.shutdown() + # Will copy table from `first` node to `second` class TaskWithDifferentSchema: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = '/clickhouse-copier/task_with_different_schema' + self.zk_task_path = "/clickhouse-copier/task_with_different_schema" self.container_task_file = "/task_with_different_schema.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_with_different_schema.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_with_different_schema.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): first = cluster.instances["first_of_two"] @@ -62,7 +73,8 @@ class TaskWithDifferentSchema: second.query("DROP DATABASE IF EXISTS db_different_schema SYNC") first.query("CREATE DATABASE IF NOT EXISTS db_different_schema;") - first.query("""CREATE TABLE db_different_schema.source + first.query( + """CREATE TABLE db_different_schema.source ( Column1 String, Column2 UInt32, @@ -83,16 +95,19 @@ class TaskWithDifferentSchema: PARTITION BY (toYYYYMMDD(Column3), Column3) PRIMARY KEY (Column1, Column2, Column3, Column4, Column6, Column7, Column8, Column9) ORDER BY (Column1, Column2, Column3, Column4, Column6, Column7, Column8, Column9) - SETTINGS index_granularity = 8192""") + SETTINGS index_granularity = 8192""" + ) - first.query("""INSERT INTO db_different_schema.source SELECT * FROM generateRandom( + first.query( + """INSERT INTO db_different_schema.source SELECT * FROM generateRandom( 'Column1 String, Column2 UInt32, Column3 Date, Column4 DateTime, Column5 UInt16, Column6 String, Column7 String, Column8 String, Column9 String, Column10 String, - Column11 String, Column12 Decimal(3, 1), Column13 DateTime, Column14 UInt16', 1, 10, 2) LIMIT 50;""") - + Column11 String, Column12 Decimal(3, 1), Column13 DateTime, Column14 UInt16', 1, 10, 2) LIMIT 50;""" + ) second.query("CREATE DATABASE IF NOT EXISTS db_different_schema;") - second.query("""CREATE TABLE db_different_schema.destination + second.query( + """CREATE TABLE db_different_schema.destination ( Column1 LowCardinality(String) CODEC(LZ4), Column2 UInt32 CODEC(LZ4), @@ -110,7 +125,8 @@ class TaskWithDifferentSchema: Column14 UInt16 CODEC(LZ4) ) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) - ORDER BY (Column9, Column1, Column2, Column3, Column4);""") + ORDER BY (Column9, Column1, Column2, Column3, Column4);""" + ) print("Preparation completed") @@ -122,10 +138,18 @@ class TaskWithDifferentSchema: b = second.query("SELECT count() from db_different_schema.destination") assert a == b, "Count" - a = TSV(first.query("""SELECT sipHash64(*) from db_different_schema.source - ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8, Column9, Column10, Column11, Column12, Column13, Column14)""")) - b = TSV(second.query("""SELECT sipHash64(*) from db_different_schema.destination - ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8, Column9, Column10, Column11, Column12, Column13, Column14)""")) + a = TSV( + first.query( + """SELECT sipHash64(*) from db_different_schema.source + ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8, Column9, Column10, Column11, Column12, Column13, Column14)""" + ) + ) + b = TSV( + second.query( + """SELECT sipHash64(*) from db_different_schema.destination + ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8, Column9, Column10, Column11, Column12, Column13, Column14)""" + ) + ) assert a == b, "Data" first.query("DROP DATABASE IF EXISTS db_different_schema SYNC") @@ -137,13 +161,20 @@ class TaskWithDifferentSchema: class TaskTTL: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = '/clickhouse-copier/task_ttl_columns' + self.zk_task_path = "/clickhouse-copier/task_ttl_columns" self.container_task_file = "/task_ttl_columns.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_ttl_columns.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_ttl_columns.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): first = cluster.instances["first_of_two"] @@ -153,7 +184,8 @@ class TaskTTL: second.query("DROP DATABASE IF EXISTS db_ttl_columns SYNC") first.query("CREATE DATABASE IF NOT EXISTS db_ttl_columns;") - first.query("""CREATE TABLE db_ttl_columns.source + first.query( + """CREATE TABLE db_ttl_columns.source ( Column1 String, Column2 UInt32, @@ -168,14 +200,18 @@ class TaskTTL: PARTITION BY (toYYYYMMDD(Column3), Column3) PRIMARY KEY (Column1, Column2, Column3) ORDER BY (Column1, Column2, Column3) - SETTINGS index_granularity = 8192""") + SETTINGS index_granularity = 8192""" + ) - first.query("""INSERT INTO db_ttl_columns.source SELECT * FROM generateRandom( + first.query( + """INSERT INTO db_ttl_columns.source SELECT * FROM generateRandom( 'Column1 String, Column2 UInt32, Column3 Date, Column4 DateTime, Column5 UInt16, - Column6 String, Column7 Decimal(3, 1), Column8 Tuple(Float64, Float64)', 1, 10, 2) LIMIT 50;""") + Column6 String, Column7 Decimal(3, 1), Column8 Tuple(Float64, Float64)', 1, 10, 2) LIMIT 50;""" + ) second.query("CREATE DATABASE IF NOT EXISTS db_ttl_columns;") - second.query("""CREATE TABLE db_ttl_columns.destination + second.query( + """CREATE TABLE db_ttl_columns.destination ( Column1 String, Column2 UInt32, @@ -187,7 +223,8 @@ class TaskTTL: Column8 Tuple(Float64, Float64) ) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) - ORDER BY (Column3, Column2, Column1);""") + ORDER BY (Column3, Column2, Column1);""" + ) print("Preparation completed") @@ -199,10 +236,18 @@ class TaskTTL: b = second.query("SELECT count() from db_ttl_columns.destination") assert a == b, "Count" - a = TSV(first.query("""SELECT sipHash64(*) from db_ttl_columns.source - ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8)""")) - b = TSV(second.query("""SELECT sipHash64(*) from db_ttl_columns.destination - ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8)""")) + a = TSV( + first.query( + """SELECT sipHash64(*) from db_ttl_columns.source + ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8)""" + ) + ) + b = TSV( + second.query( + """SELECT sipHash64(*) from db_ttl_columns.destination + ORDER BY (Column1, Column2, Column3, Column4, Column5, Column6, Column7, Column8)""" + ) + ) assert a == b, "Data" first.query("DROP DATABASE IF EXISTS db_ttl_columns SYNC") @@ -212,13 +257,20 @@ class TaskTTL: class TaskSkipIndex: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = '/clickhouse-copier/task_skip_index' + self.zk_task_path = "/clickhouse-copier/task_skip_index" self.container_task_file = "/task_skip_index.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_skip_index.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_skip_index.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): first = cluster.instances["first_of_two"] @@ -228,7 +280,8 @@ class TaskSkipIndex: second.query("DROP DATABASE IF EXISTS db_skip_index SYNC") first.query("CREATE DATABASE IF NOT EXISTS db_skip_index;") - first.query("""CREATE TABLE db_skip_index.source + first.query( + """CREATE TABLE db_skip_index.source ( Column1 UInt64, Column2 Int32, @@ -242,13 +295,17 @@ class TaskSkipIndex: PARTITION BY (toYYYYMMDD(Column3), Column3) PRIMARY KEY (Column1, Column2, Column3) ORDER BY (Column1, Column2, Column3) - SETTINGS index_granularity = 8192""") + SETTINGS index_granularity = 8192""" + ) - first.query("""INSERT INTO db_skip_index.source SELECT * FROM generateRandom( - 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""") + first.query( + """INSERT INTO db_skip_index.source SELECT * FROM generateRandom( + 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""" + ) second.query("CREATE DATABASE IF NOT EXISTS db_skip_index;") - second.query("""CREATE TABLE db_skip_index.destination + second.query( + """CREATE TABLE db_skip_index.destination ( Column1 UInt64, Column2 Int32, @@ -259,7 +316,8 @@ class TaskSkipIndex: INDEX b (Column1 * length(Column5)) TYPE set(1000) GRANULARITY 4 ) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) - ORDER BY (Column3, Column2, Column1);""") + ORDER BY (Column3, Column2, Column1);""" + ) print("Preparation completed") @@ -271,10 +329,18 @@ class TaskSkipIndex: b = second.query("SELECT count() from db_skip_index.destination") assert a == b, "Count" - a = TSV(first.query("""SELECT sipHash64(*) from db_skip_index.source - ORDER BY (Column1, Column2, Column3, Column4, Column5)""")) - b = TSV(second.query("""SELECT sipHash64(*) from db_skip_index.destination - ORDER BY (Column1, Column2, Column3, Column4, Column5)""")) + a = TSV( + first.query( + """SELECT sipHash64(*) from db_skip_index.source + ORDER BY (Column1, Column2, Column3, Column4, Column5)""" + ) + ) + b = TSV( + second.query( + """SELECT sipHash64(*) from db_skip_index.destination + ORDER BY (Column1, Column2, Column3, Column4, Column5)""" + ) + ) assert a == b, "Data" first.query("DROP DATABASE IF EXISTS db_skip_index SYNC") @@ -284,13 +350,20 @@ class TaskSkipIndex: class TaskTTLMoveToVolume: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = '/clickhouse-copier/task_ttl_move_to_volume' + self.zk_task_path = "/clickhouse-copier/task_ttl_move_to_volume" self.container_task_file = "/task_ttl_move_to_volume.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_ttl_move_to_volume.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_ttl_move_to_volume.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): first = cluster.instances["first_of_two"] @@ -300,7 +373,8 @@ class TaskTTLMoveToVolume: second.query("DROP DATABASE IF EXISTS db_move_to_volume SYNC") first.query("CREATE DATABASE IF NOT EXISTS db_move_to_volume;") - first.query("""CREATE TABLE db_move_to_volume.source + first.query( + """CREATE TABLE db_move_to_volume.source ( Column1 UInt64, Column2 Int32, @@ -313,13 +387,17 @@ class TaskTTLMoveToVolume: PRIMARY KEY (Column1, Column2, Column3) ORDER BY (Column1, Column2, Column3) TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external' - SETTINGS storage_policy = 'external_with_jbods';""") + SETTINGS storage_policy = 'external_with_jbods';""" + ) - first.query("""INSERT INTO db_move_to_volume.source SELECT * FROM generateRandom( - 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""") + first.query( + """INSERT INTO db_move_to_volume.source SELECT * FROM generateRandom( + 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""" + ) second.query("CREATE DATABASE IF NOT EXISTS db_move_to_volume;") - second.query("""CREATE TABLE db_move_to_volume.destination + second.query( + """CREATE TABLE db_move_to_volume.destination ( Column1 UInt64, Column2 Int32, @@ -330,7 +408,8 @@ class TaskTTLMoveToVolume: PARTITION BY toYYYYMMDD(Column3) ORDER BY (Column3, Column2, Column1) TTL Column3 + INTERVAL 1 MONTH TO VOLUME 'external' - SETTINGS storage_policy = 'external_with_jbods';""") + SETTINGS storage_policy = 'external_with_jbods';""" + ) print("Preparation completed") @@ -342,10 +421,18 @@ class TaskTTLMoveToVolume: b = second.query("SELECT count() from db_move_to_volume.destination") assert a == b, "Count" - a = TSV(first.query("""SELECT sipHash64(*) from db_move_to_volume.source - ORDER BY (Column1, Column2, Column3, Column4, Column5)""")) - b = TSV(second.query("""SELECT sipHash64(*) from db_move_to_volume.destination - ORDER BY (Column1, Column2, Column3, Column4, Column5)""")) + a = TSV( + first.query( + """SELECT sipHash64(*) from db_move_to_volume.source + ORDER BY (Column1, Column2, Column3, Column4, Column5)""" + ) + ) + b = TSV( + second.query( + """SELECT sipHash64(*) from db_move_to_volume.destination + ORDER BY (Column1, Column2, Column3, Column4, Column5)""" + ) + ) assert a == b, "Data" first.query("DROP DATABASE IF EXISTS db_move_to_volume SYNC") @@ -355,13 +442,20 @@ class TaskTTLMoveToVolume: class TaskDropTargetPartition: def __init__(self, cluster): self.cluster = cluster - self.zk_task_path = '/clickhouse-copier/task_drop_target_partition' + self.zk_task_path = "/clickhouse-copier/task_drop_target_partition" self.container_task_file = "/task_drop_target_partition.xml" for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_drop_target_partition.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "./task_drop_target_partition.xml"), + self.container_task_file, + ) + print( + "Copied task file to container of '{}' instance. Path {}".format( + instance_name, self.container_task_file + ) + ) def start(self): first = cluster.instances["first_of_two"] @@ -371,7 +465,8 @@ class TaskDropTargetPartition: second.query("DROP DATABASE IF EXISTS db_drop_target_partition SYNC") first.query("CREATE DATABASE IF NOT EXISTS db_drop_target_partition;") - first.query("""CREATE TABLE db_drop_target_partition.source + first.query( + """CREATE TABLE db_drop_target_partition.source ( Column1 UInt64, Column2 Int32, @@ -382,14 +477,17 @@ class TaskDropTargetPartition: ENGINE = MergeTree() PARTITION BY (toYYYYMMDD(Column3), Column3) PRIMARY KEY (Column1, Column2, Column3) - ORDER BY (Column1, Column2, Column3);""") - - first.query("""INSERT INTO db_drop_target_partition.source SELECT * FROM generateRandom( - 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""") + ORDER BY (Column1, Column2, Column3);""" + ) + first.query( + """INSERT INTO db_drop_target_partition.source SELECT * FROM generateRandom( + 'Column1 UInt64, Column2 Int32, Column3 Date, Column4 DateTime, Column5 String', 1, 10, 2) LIMIT 100;""" + ) second.query("CREATE DATABASE IF NOT EXISTS db_drop_target_partition;") - second.query("""CREATE TABLE db_drop_target_partition.destination + second.query( + """CREATE TABLE db_drop_target_partition.destination ( Column1 UInt64, Column2 Int32, @@ -398,10 +496,13 @@ class TaskDropTargetPartition: Column5 String ) ENGINE = MergeTree() PARTITION BY toYYYYMMDD(Column3) - ORDER BY (Column3, Column2, Column1);""") + ORDER BY (Column3, Column2, Column1);""" + ) # Insert data in target too. It has to be dropped. - first.query("""INSERT INTO db_drop_target_partition.destination SELECT * FROM db_drop_target_partition.source;""") + first.query( + """INSERT INTO db_drop_target_partition.destination SELECT * FROM db_drop_target_partition.source;""" + ) print("Preparation completed") @@ -413,10 +514,18 @@ class TaskDropTargetPartition: b = second.query("SELECT count() from db_drop_target_partition.destination") assert a == b, "Count" - a = TSV(first.query("""SELECT sipHash64(*) from db_drop_target_partition.source - ORDER BY (Column1, Column2, Column3, Column4, Column5)""")) - b = TSV(second.query("""SELECT sipHash64(*) from db_drop_target_partition.destination - ORDER BY (Column1, Column2, Column3, Column4, Column5)""")) + a = TSV( + first.query( + """SELECT sipHash64(*) from db_drop_target_partition.source + ORDER BY (Column1, Column2, Column3, Column4, Column5)""" + ) + ) + b = TSV( + second.query( + """SELECT sipHash64(*) from db_drop_target_partition.destination + ORDER BY (Column1, Column2, Column3, Column4, Column5)""" + ) + ) assert a == b, "Data" first.query("DROP DATABASE IF EXISTS db_drop_target_partition SYNC") @@ -426,19 +535,27 @@ class TaskDropTargetPartition: def execute_task(started_cluster, task, cmd_options): task.start() - zk = started_cluster.get_kazoo_client('zoo1') + zk = started_cluster.get_kazoo_client("zoo1") print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) # Run cluster-copier processes on each node docker_api = started_cluster.docker_client.api copiers_exec_ids = [] - cmd = ['/usr/bin/clickhouse', 'copier', - '--config', '/etc/clickhouse-server/config-copier.xml', - '--task-path', task.zk_task_path, - '--task-file', task.container_task_file, - '--task-upload-force', 'true', - '--base-dir', '/var/log/clickhouse-server/copier'] + cmd = [ + "/usr/bin/clickhouse", + "copier", + "--config", + "/etc/clickhouse-server/config-copier.xml", + "--task-path", + task.zk_task_path, + "--task-file", + task.container_task_file, + "--task-upload-force", + "true", + "--base-dir", + "/var/log/clickhouse-server/copier", + ] cmd += cmd_options print(cmd) @@ -446,25 +563,34 @@ def execute_task(started_cluster, task, cmd_options): for instance_name in started_cluster.instances.keys(): instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() - instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs_two_nodes/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") + instance.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs_two_nodes/config-copier.xml"), + "/etc/clickhouse-server/config-copier.xml", + ) logging.info("Copied copier config to {}".format(instance.name)) exec_id = docker_api.exec_create(container.id, cmd, stderr=True) - output = docker_api.exec_start(exec_id).decode('utf8') + output = docker_api.exec_start(exec_id).decode("utf8") logging.info(output) copiers_exec_ids.append(exec_id) - logging.info("Copier for {} ({}) has started".format(instance.name, instance.ip_address)) + logging.info( + "Copier for {} ({}) has started".format(instance.name, instance.ip_address) + ) # time.sleep(1000) # Wait for copiers stopping and check their return codes - for exec_id, instance in zip(copiers_exec_ids, iter(started_cluster.instances.values())): + for exec_id, instance in zip( + copiers_exec_ids, iter(started_cluster.instances.values()) + ): while True: res = docker_api.exec_inspect(exec_id) - if not res['Running']: + if not res["Running"]: break time.sleep(1) - assert res['ExitCode'] == 0, "Instance: {} ({}). Info: {}".format(instance.name, instance.ip_address, repr(res)) + assert res["ExitCode"] == 0, "Instance: {} ({}). Info: {}".format( + instance.name, instance.ip_address, repr(res) + ) try: task.check() diff --git a/tests/integration/test_cluster_discovery/test.py b/tests/integration/test_cluster_discovery/test.py index acddd855040..311c955d1e3 100644 --- a/tests/integration/test_cluster_discovery/test.py +++ b/tests/integration/test_cluster_discovery/test.py @@ -7,18 +7,16 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -shard_configs = { - i: f'config/config_shard{i}.xml' - for i in [1, 3] -} +shard_configs = {i: f"config/config_shard{i}.xml" for i in [1, 3]} nodes = [ cluster.add_instance( - f'node{i}', - main_configs=[shard_configs.get(i, 'config/config.xml')], + f"node{i}", + main_configs=[shard_configs.get(i, "config/config.xml")], stay_alive=True, - with_zookeeper=True - ) for i in range(5) + with_zookeeper=True, + ) + for i in range(5) ] @@ -31,7 +29,9 @@ def start_cluster(): cluster.shutdown() -def check_on_cluster(nodes, expected, *, what, cluster_name='test_auto_cluster', msg=None, retries=5): +def check_on_cluster( + nodes, expected, *, what, cluster_name="test_auto_cluster", msg=None, retries=5 +): """ Select data from `system.clusters` on specified nodes and check the result """ @@ -39,17 +39,23 @@ def check_on_cluster(nodes, expected, *, what, cluster_name='test_auto_cluster', for retry in range(1, retries + 1): nodes_res = { - node.name: int(node.query(f"SELECT {what} FROM system.clusters WHERE cluster = '{cluster_name}'")) + node.name: int( + node.query( + f"SELECT {what} FROM system.clusters WHERE cluster = '{cluster_name}'" + ) + ) for node in nodes } if all(actual == expected for actual in nodes_res.values()): break if retry != retries: - time.sleep(2 ** retry) + time.sleep(2**retry) else: msg = msg or f"Wrong '{what}' result" - raise Exception(f'{msg}: {nodes_res}, expected: {expected} (after {retries} retries)') + raise Exception( + f"{msg}: {nodes_res}, expected: {expected} (after {retries} retries)" + ) def test_cluster_discovery_startup_and_stop(start_cluster): @@ -58,8 +64,14 @@ def test_cluster_discovery_startup_and_stop(start_cluster): then stop/start some nodes and check that it (dis)appeared in cluster. """ - check_nodes_count = functools.partial(check_on_cluster, what='count()', msg='Wrong nodes count in cluster') - check_shard_num = functools.partial(check_on_cluster, what='count(DISTINCT shard_num)', msg='Wrong shard_num count in cluster') + check_nodes_count = functools.partial( + check_on_cluster, what="count()", msg="Wrong nodes count in cluster" + ) + check_shard_num = functools.partial( + check_on_cluster, + what="count(DISTINCT shard_num)", + msg="Wrong shard_num count in cluster", + ) total_shards = len(shard_configs) + 1 check_nodes_count([nodes[0], nodes[2]], len(nodes)) @@ -78,4 +90,4 @@ def test_cluster_discovery_startup_and_stop(start_cluster): nodes[3].start_clickhouse() check_nodes_count([nodes[0], nodes[2]], len(nodes)) - check_nodes_count([nodes[1], nodes[2]], 2, cluster_name='two_shards', retries=1) + check_nodes_count([nodes[1], nodes[2]], 2, cluster_name="two_shards", retries=1) diff --git a/tests/integration/test_codec_encrypted/test.py b/tests/integration/test_codec_encrypted/test.py index 439fcdd8ef8..ebe837c9e3a 100644 --- a/tests/integration/test_codec_encrypted/test.py +++ b/tests/integration/test_codec_encrypted/test.py @@ -5,7 +5,8 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') +node = cluster.add_instance("node") + @pytest.fixture(scope="module") def start_cluster(): @@ -16,8 +17,13 @@ def start_cluster(): finally: cluster.shutdown() + def make_storage_with_key(id): - node.exec_in_container(["bash", "-c" , """cat > /etc/clickhouse-server/config.d/storage_keys_config.xml << EOF + node.exec_in_container( + [ + "bash", + "-c", + """cat > /etc/clickhouse-server/config.d/storage_keys_config.xml << EOF @@ -33,27 +39,36 @@ def make_storage_with_key(id): -EOF""".format(cur_id=id)]) - node.query("SYSTEM RELOAD CONFIG") +EOF""".format( + cur_id=id + ), + ] + ) + node.query("SYSTEM RELOAD CONFIG") + def test_different_keys(start_cluster): make_storage_with_key(0) - node.query(""" + node.query( + """ CREATE TABLE encrypted_test_128 ( id Int64, data String Codec(AES_128_GCM_SIV) ) ENGINE=MergeTree() ORDER BY id - """) + """ + ) - node.query(""" + node.query( + """ CREATE TABLE encrypted_test_256 ( id Int64, data String Codec(AES_256_GCM_SIV) ) ENGINE=MergeTree() ORDER BY id - """) - + """ + ) + node.query("INSERT INTO encrypted_test_128 VALUES (0,'data'),(1,'data')") select_query = "SELECT * FROM encrypted_test_128 ORDER BY id FORMAT Values" assert node.query(select_query) == "(0,'data'),(1,'data')" diff --git a/tests/integration/test_compression_codec_read/test.py b/tests/integration/test_compression_codec_read/test.py index 35ae60f05ea..38cd61e241d 100644 --- a/tests/integration/test_compression_codec_read/test.py +++ b/tests/integration/test_compression_codec_read/test.py @@ -5,7 +5,14 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', image='yandex/clickhouse-server', tag='20.8.11.17', with_installed_binary=True, stay_alive=True) +node1 = cluster.add_instance( + "node1", + image="yandex/clickhouse-server", + tag="20.8.11.17", + with_installed_binary=True, + stay_alive=True, +) + @pytest.fixture(scope="module") def start_cluster(): @@ -16,10 +23,12 @@ def start_cluster(): finally: cluster.shutdown() + def test_default_codec_read(start_cluster): node1.query("DROP TABLE IF EXISTS test_18340") - node1.query(""" + node1.query( + """ CREATE TABLE test_18340 ( `lns` LowCardinality(Nullable(String)), @@ -36,10 +45,12 @@ def test_default_codec_read(start_cluster): PARTITION BY i32 ORDER BY (s, farmHash64(s)) SAMPLE BY farmHash64(s) - """) - - node1.query("insert into test_18340 values ('test', 'test', 'test', 0, 0, ['a'], ['a'], now(), 0)") + """ + ) + node1.query( + "insert into test_18340 values ('test', 'test', 'test', 0, 0, ['a'], ['a'], now(), 0)" + ) assert node1.query("SELECT COUNT() FROM test_18340") == "1\n" diff --git a/tests/integration/test_compression_nested_columns/test.py b/tests/integration/test_compression_nested_columns/test.py index f73adadd770..55d88174287 100644 --- a/tests/integration/test_compression_nested_columns/test.py +++ b/tests/integration/test_compression_nested_columns/test.py @@ -6,8 +6,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) @pytest.fixture(scope="module") @@ -19,24 +19,29 @@ def start_cluster(): finally: cluster.shutdown() + def get_compression_codec_byte(node, table_name, part_name, filename): cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/{}.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format( - table_name, part_name, filename) + table_name, part_name, filename + ) return node.exec_in_container(["bash", "-c", cmd]).strip() + CODECS_MAPPING = { - 'NONE' : '0002', - 'LZ4': '0082', - 'LZ4HC': '0082', # not an error, same byte - 'ZSTD': '0090', - 'Multiple': '0091', - 'Delta': '0092', - 'T64': '0093', + "NONE": "0002", + "LZ4": "0082", + "LZ4HC": "0082", # not an error, same byte + "ZSTD": "0090", + "Multiple": "0091", + "Delta": "0092", + "T64": "0093", } + def test_nested_compression_codec(start_cluster): for i, node in enumerate([node1, node2]): - node.query(""" + node.query( + """ CREATE TABLE compression_table ( key UInt64, column_ok Nullable(UInt64) CODEC(Delta, LZ4), @@ -44,7 +49,14 @@ def test_nested_compression_codec(start_cluster): column_bad LowCardinality(Int64) CODEC(Delta) ) ENGINE = ReplicatedMergeTree('/t', '{}') ORDER BY tuple() PARTITION BY key SETTINGS min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; - """.format(i), settings={"allow_suspicious_codecs" : "1", "allow_suspicious_low_cardinality_types" : "1"}) + """.format( + i + ), + settings={ + "allow_suspicious_codecs": "1", + "allow_suspicious_low_cardinality_types": "1", + }, + ) node1.query("INSERT INTO compression_table VALUES (1, 1, [[77]], 32)") @@ -57,12 +69,47 @@ def test_nested_compression_codec(start_cluster): node2.query("ATTACH TABLE compression_table") for node in [node1, node2]: - assert get_compression_codec_byte(node, "compression_table", "1_0_0_0", "column_ok") == CODECS_MAPPING['Multiple'] - assert get_compression_codec_byte(node, "compression_table", "1_0_0_0", "column_ok.null") == CODECS_MAPPING['LZ4'] + assert ( + get_compression_codec_byte( + node, "compression_table", "1_0_0_0", "column_ok" + ) + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_compression_codec_byte( + node, "compression_table", "1_0_0_0", "column_ok.null" + ) + == CODECS_MAPPING["LZ4"] + ) - assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0", "column_array") == CODECS_MAPPING['Multiple'] - assert get_compression_codec_byte(node2, "compression_table", "1_0_0_0", "column_array.size0") == CODECS_MAPPING['LZ4'] - assert get_compression_codec_byte(node2, "compression_table", "1_0_0_0", "column_array.size1") == CODECS_MAPPING['LZ4'] + assert ( + get_compression_codec_byte( + node1, "compression_table", "1_0_0_0", "column_array" + ) + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_compression_codec_byte( + node2, "compression_table", "1_0_0_0", "column_array.size0" + ) + == CODECS_MAPPING["LZ4"] + ) + assert ( + get_compression_codec_byte( + node2, "compression_table", "1_0_0_0", "column_array.size1" + ) + == CODECS_MAPPING["LZ4"] + ) - assert get_compression_codec_byte(node2, "compression_table", "1_0_0_0", "column_bad.dict") == CODECS_MAPPING['Delta'] - assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0", "column_bad") == CODECS_MAPPING['NONE'] + assert ( + get_compression_codec_byte( + node2, "compression_table", "1_0_0_0", "column_bad.dict" + ) + == CODECS_MAPPING["Delta"] + ) + assert ( + get_compression_codec_byte( + node1, "compression_table", "1_0_0_0", "column_bad" + ) + == CODECS_MAPPING["NONE"] + ) diff --git a/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py b/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py index ac6e87cdee5..166724a7f8c 100644 --- a/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py +++ b/tests/integration/test_concurrent_queries_for_all_users_restriction/test.py @@ -6,14 +6,16 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml']) +node1 = cluster.add_instance("node1", user_configs=["configs/user_restrictions.xml"]) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node1.query("create table nums (number UInt64) ENGINE = MergeTree() order by tuple()") + node1.query( + "create table nums (number UInt64) ENGINE = MergeTree() order by tuple()" + ) node1.query("insert into nums values (0), (1)") yield cluster finally: @@ -25,7 +27,7 @@ def test_exception_message(started_cluster): def node_busy(_): for i in range(10): - node1.query("select sleep(2)", user='someuser', ignore_error=True) + node1.query("select sleep(2)", user="someuser", ignore_error=True) busy_pool = Pool(3) busy_pool.map_async(node_busy, range(3)) @@ -33,9 +35,21 @@ def test_exception_message(started_cluster): with pytest.raises(Exception) as exc_info: for i in range(3): - assert node1.query("select number from remote('node1', 'default', 'nums')", user='default') == "0\n1\n" + assert ( + node1.query( + "select number from remote('node1', 'default', 'nums')", + user="default", + ) + == "0\n1\n" + ) exc_info.match("Too many simultaneous queries for all users") for i in range(3): - assert node1.query("select number from remote('node1', 'default', 'nums')", user='default', - settings={'max_concurrent_queries_for_all_users': 0}) == "0\n1\n" + assert ( + node1.query( + "select number from remote('node1', 'default', 'nums')", + user="default", + settings={"max_concurrent_queries_for_all_users": 0}, + ) + == "0\n1\n" + ) diff --git a/tests/integration/test_concurrent_queries_for_user_restriction/test.py b/tests/integration/test_concurrent_queries_for_user_restriction/test.py index 279e0dfe439..c4afdf99685 100644 --- a/tests/integration/test_concurrent_queries_for_user_restriction/test.py +++ b/tests/integration/test_concurrent_queries_for_user_restriction/test.py @@ -6,15 +6,17 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=['configs/user_restrictions.xml']) -node2 = cluster.add_instance('node2', user_configs=['configs/user_restrictions.xml']) +node1 = cluster.add_instance("node1", user_configs=["configs/user_restrictions.xml"]) +node2 = cluster.add_instance("node2", user_configs=["configs/user_restrictions.xml"]) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node1.query("create table nums (number UInt64) ENGINE = MergeTree() order by tuple()") + node1.query( + "create table nums (number UInt64) ENGINE = MergeTree() order by tuple()" + ) node1.query("insert into nums values(0),(1)") yield cluster @@ -27,13 +29,20 @@ def test_exception_message(started_cluster): def node_busy(_): for i in range(10): - node1.query("select sleep(2)", user='default') + node1.query("select sleep(2)", user="default") busy_pool = Pool(3) busy_pool.map_async(node_busy, range(3)) time.sleep(1) # wait a little until polling starts try: - assert node2.query("select number from remote('node1', 'default', 'nums')", user='good') == "0\n1\n" + assert ( + node2.query( + "select number from remote('node1', 'default', 'nums')", user="good" + ) + == "0\n1\n" + ) except Exception as ex: print(ex.message) - assert False, "Exception thrown while max_concurrent_queries_for_user is not exceeded" + assert ( + False + ), "Exception thrown while max_concurrent_queries_for_user is not exceeded" diff --git a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py index 2d16d9157f6..6bda1df147c 100644 --- a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py +++ b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py @@ -6,16 +6,24 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node_insert = cluster.add_instance('node_insert', main_configs=['configs/concurrent_insert_restriction.xml']) -node_select = cluster.add_instance('node_select', main_configs=['configs/concurrent_select_restriction.xml']) +node_insert = cluster.add_instance( + "node_insert", main_configs=["configs/concurrent_insert_restriction.xml"] +) +node_select = cluster.add_instance( + "node_select", main_configs=["configs/concurrent_select_restriction.xml"] +) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node_select.query("create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()") - node_insert.query("create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()") + node_select.query( + "create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()" + ) + node_insert.query( + "create table test_concurrent_insert (x UInt64) ENGINE = MergeTree() order by tuple()" + ) yield cluster finally: cluster.shutdown() @@ -24,54 +32,58 @@ def started_cluster(): def execute_with_background(node, sql, background_sql, background_times, wait_times=3): r = None for _ in range(wait_times): - r = node.query('show processlist', stdin='') + r = node.query("show processlist", stdin="") if not r.strip(): break time.sleep(1) else: assert False, "there are unknown background queries: {}".format(r) for _ in range(background_times): - node.get_query_request(background_sql, stdin='') - time.sleep(0.5) # wait background to start. - return node.query(sql, stdin='') + node.get_query_request(background_sql, stdin="") + time.sleep(0.5) # wait background to start. + return node.query(sql, stdin="") def common_pattern(node, query_kind, restricted_sql, normal_sql, limit, wait_times): # restriction is working - with pytest.raises(Exception, match=r".*Too many simultaneous {} queries.*".format(query_kind)): + with pytest.raises( + Exception, match=r".*Too many simultaneous {} queries.*".format(query_kind) + ): execute_with_background(node, restricted_sql, restricted_sql, limit, wait_times) # different query kind is independent execute_with_background(node, normal_sql, restricted_sql, limit, wait_times) # normal - execute_with_background(node, restricted_sql, '', 0, wait_times) + execute_with_background(node, restricted_sql, "", 0, wait_times) def test_select(started_cluster): common_pattern( - node_select, 'select', - 'select sleep(3)', - 'insert into test_concurrent_insert values (0)', + node_select, + "select", + "select sleep(3)", + "insert into test_concurrent_insert values (0)", 2, - 10 + 10, ) # subquery is not counted execute_with_background( node_select, - 'select sleep(3)', - 'insert into test_concurrent_insert select sleep(3)', + "select sleep(3)", + "insert into test_concurrent_insert select sleep(3)", 2, - 10 + 10, ) def test_insert(started_cluster): common_pattern( - node_insert, 'insert', - 'insert into test_concurrent_insert select sleep(3)', - 'select 1', + node_insert, + "insert", + "insert into test_concurrent_insert select sleep(3)", + "select 1", 2, - 10 + 10, ) diff --git a/tests/integration/test_concurrent_ttl_merges/test.py b/tests/integration/test_concurrent_ttl_merges/test.py index 8c3c490d055..07e91dcbc9f 100644 --- a/tests/integration/test_concurrent_ttl_merges/test.py +++ b/tests/integration/test_concurrent_ttl_merges/test.py @@ -6,8 +6,12 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, TSV cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/fast_background_pool.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/fast_background_pool.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -22,30 +26,40 @@ def started_cluster(): def count_ttl_merges_in_queue(node, table): result = node.query( - f"SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{table}'") + f"SELECT count() FROM system.replication_queue WHERE merge_type = 'TTL_DELETE' and table = '{table}'" + ) if not result: return 0 return int(result.strip()) def count_ttl_merges_in_background_pool(node, table, level): - result = TSV(node.query( - f"SELECT * FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{table}'")) + result = TSV( + node.query( + f"SELECT * FROM system.merges WHERE merge_type = 'TTL_DELETE' and table = '{table}'" + ) + ) count = len(result) if count >= level: - logging.debug(f"count_ttl_merges_in_background_pool: merges more than warn level:\n{result}") + logging.debug( + f"count_ttl_merges_in_background_pool: merges more than warn level:\n{result}" + ) return count def count_regular_merges_in_background_pool(node, table): - result = node.query(f"SELECT count() FROM system.merges WHERE merge_type = 'REGULAR' and table = '{table}'") + result = node.query( + f"SELECT count() FROM system.merges WHERE merge_type = 'REGULAR' and table = '{table}'" + ) if not result: return 0 return int(result.strip()) def count_running_mutations(node, table): - result = node.query(f"SELECT count() FROM system.merges WHERE table = '{table}' and is_mutation=1") + result = node.query( + f"SELECT count() FROM system.merges WHERE table = '{table}' and is_mutation=1" + ) if not result: return 0 return int(result.strip()) @@ -56,13 +70,15 @@ def count_running_mutations(node, table): # on the borders of partitions. def test_no_ttl_merges_in_busy_pool(started_cluster): node1.query( - "CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0") + "CREATE TABLE test_ttl (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0, number_of_free_entries_in_pool_to_execute_mutation = 0" + ) node1.query("SYSTEM STOP TTL MERGES") for i in range(1, 7): node1.query( - f"INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {i}, number FROM numbers(5)") + f"INSERT INTO test_ttl SELECT now() - INTERVAL 1 MONTH + number - 1, {i}, number FROM numbers(5)" + ) node1.query("ALTER TABLE test_ttl UPDATE data = data + 1 WHERE sleepEachRow(1) = 0") @@ -75,7 +91,9 @@ def test_no_ttl_merges_in_busy_pool(started_cluster): rows_count = [] while count_running_mutations(node1, "test_ttl") == 6: - logging.debug(f"Mutations count after start TTL{count_running_mutations(node1, 'test_ttl')}") + logging.debug( + f"Mutations count after start TTL{count_running_mutations(node1, 'test_ttl')}" + ) rows_count.append(int(node1.query("SELECT count() FROM test_ttl").strip())) time.sleep(0.5) @@ -89,12 +107,15 @@ def test_no_ttl_merges_in_busy_pool(started_cluster): def test_limited_ttl_merges_in_empty_pool(started_cluster): node1.query( - "CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + "CREATE TABLE test_ttl_v2 (d DateTime, key UInt64, data UInt64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0" + ) node1.query("SYSTEM STOP TTL MERGES") for i in range(100): - node1.query(f"INSERT INTO test_ttl_v2 SELECT now() - INTERVAL 1 MONTH, {i}, number FROM numbers(1)") + node1.query( + f"INSERT INTO test_ttl_v2 SELECT now() - INTERVAL 1 MONTH, {i}, number FROM numbers(1)" + ) assert node1.query("SELECT COUNT() FROM test_ttl_v2") == "100\n" @@ -102,7 +123,9 @@ def test_limited_ttl_merges_in_empty_pool(started_cluster): merges_with_ttl_count = set({}) while True: - merges_with_ttl_count.add(count_ttl_merges_in_background_pool(node1, "test_ttl_v2", 3)) + merges_with_ttl_count.add( + count_ttl_merges_in_background_pool(node1, "test_ttl_v2", 3) + ) time.sleep(0.01) if node1.query("SELECT COUNT() FROM test_ttl_v2") == "0\n": break @@ -113,12 +136,15 @@ def test_limited_ttl_merges_in_empty_pool(started_cluster): def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster): node1.query( - "CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + "CREATE TABLE replicated_ttl (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0" + ) node1.query("SYSTEM STOP TTL MERGES") for i in range(100): - node1.query_with_retry(f"INSERT INTO replicated_ttl SELECT now() - INTERVAL 1 MONTH, {i}, number FROM numbers(1)") + node1.query_with_retry( + f"INSERT INTO replicated_ttl SELECT now() - INTERVAL 1 MONTH, {i}, number FROM numbers(1)" + ) assert node1.query("SELECT COUNT() FROM replicated_ttl") == "100\n" @@ -127,7 +153,9 @@ def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster): merges_with_ttl_count = set({}) entries_with_ttl_count = set({}) while True: - merges_with_ttl_count.add(count_ttl_merges_in_background_pool(node1, "replicated_ttl", 3)) + merges_with_ttl_count.add( + count_ttl_merges_in_background_pool(node1, "replicated_ttl", 3) + ) entries_with_ttl_count.add(count_ttl_merges_in_queue(node1, "replicated_ttl")) time.sleep(0.01) if node1.query("SELECT COUNT() FROM replicated_ttl") == "0\n": @@ -142,16 +170,19 @@ def test_limited_ttl_merges_in_empty_pool_replicated(started_cluster): def test_limited_ttl_merges_two_replicas(started_cluster): # Actually this test quite fast and often we cannot catch any merges. node1.query( - "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '1') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0" + ) node2.query( - "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0") + "CREATE TABLE replicated_ttl_2 (d DateTime, key UInt64, data UInt64) ENGINE = ReplicatedMergeTree('/test/t2', '2') ORDER BY tuple() PARTITION BY key TTL d + INTERVAL 1 MONTH SETTINGS merge_with_ttl_timeout = 0" + ) node1.query("SYSTEM STOP TTL MERGES") node2.query("SYSTEM STOP TTL MERGES") for i in range(100): node1.query_with_retry( - f"INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {i}, number FROM numbers(10000)") + f"INSERT INTO replicated_ttl_2 SELECT now() - INTERVAL 1 MONTH, {i}, number FROM numbers(10000)" + ) node2.query("SYSTEM SYNC REPLICA replicated_ttl_2", timeout=10) assert node1.query("SELECT COUNT() FROM replicated_ttl_2") == "1000000\n" @@ -163,10 +194,16 @@ def test_limited_ttl_merges_two_replicas(started_cluster): merges_with_ttl_count_node1 = set({}) merges_with_ttl_count_node2 = set({}) while True: - merges_with_ttl_count_node1.add(count_ttl_merges_in_background_pool(node1, "replicated_ttl_2", 3)) - merges_with_ttl_count_node2.add(count_ttl_merges_in_background_pool(node2, "replicated_ttl_2", 3)) - if node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" and node2.query( - "SELECT COUNT() FROM replicated_ttl_2") == "0\n": + merges_with_ttl_count_node1.add( + count_ttl_merges_in_background_pool(node1, "replicated_ttl_2", 3) + ) + merges_with_ttl_count_node2.add( + count_ttl_merges_in_background_pool(node2, "replicated_ttl_2", 3) + ) + if ( + node1.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" + and node2.query("SELECT COUNT() FROM replicated_ttl_2") == "0\n" + ): break # Both replicas can assign merges with TTL. If one will perform better than diff --git a/tests/integration/test_config_corresponding_root/configs/config.xml b/tests/integration/test_config_corresponding_root/configs/config.xml index 5a0179fa25f..e1a1c1c75df 100644 --- a/tests/integration/test_config_corresponding_root/configs/config.xml +++ b/tests/integration/test_config_corresponding_root/configs/config.xml @@ -129,20 +129,6 @@ default - - - diff --git a/tests/integration/test_config_corresponding_root/test.py b/tests/integration/test_config_corresponding_root/test.py index da6af7d11ef..f4ec1f1e658 100644 --- a/tests/integration/test_config_corresponding_root/test.py +++ b/tests/integration/test_config_corresponding_root/test.py @@ -6,7 +6,7 @@ from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.d/bad.xml"]) +node = cluster.add_instance("node", main_configs=["configs/config.d/bad.xml"]) caught_exception = "" @@ -21,4 +21,9 @@ def start_cluster(): def test_work(start_cluster): print(caught_exception) - assert caught_exception.find("Root element doesn't have the corresponding root element as the config file.") != -1 + assert ( + caught_exception.find( + "Root element doesn't have the corresponding root element as the config file." + ) + != -1 + ) diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index aec3f1d3635..692b36f1fae 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -3,25 +3,51 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=['configs/config_no_substs.xml']) # hardcoded value 33333 -node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], - env_variables={"MAX_QUERY_SIZE": "55555"}) -node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], - main_configs=['configs/include_from_source.xml']) # include value 77777 -node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml']) -node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], - env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/include_from_source.xml"}, - main_configs=['configs/include_from_source.xml']) +node1 = cluster.add_instance( + "node1", user_configs=["configs/config_no_substs.xml"] +) # hardcoded value 33333 +node2 = cluster.add_instance( + "node2", + user_configs=["configs/config_env.xml"], + env_variables={"MAX_QUERY_SIZE": "55555"}, +) +node3 = cluster.add_instance( + "node3", user_configs=["configs/config_zk.xml"], with_zookeeper=True +) +node4 = cluster.add_instance( + "node4", + user_configs=["configs/config_incl.xml"], + main_configs=["configs/include_from_source.xml"], +) # include value 77777 +node5 = cluster.add_instance( + "node5", user_configs=["configs/config_allow_databases.xml"] +) +node6 = cluster.add_instance( + "node6", + user_configs=["configs/config_include_from_env.xml"], + env_variables={ + "INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/include_from_source.xml" + }, + main_configs=["configs/include_from_source.xml"], +) @pytest.fixture(scope="module") def start_cluster(): try: + def create_zk_roots(zk): zk.create(path="/setting/max_query_size", value=b"77777", makepath=True) - zk.create(path="/users_from_zk_1", value=b"default", makepath=True) - zk.create(path="/users_from_zk_2", value=b"default", makepath=True) + zk.create( + path="/users_from_zk_1", + value=b"default", + makepath=True, + ) + zk.create( + path="/users_from_zk_2", + value=b"default", + makepath=True, + ) cluster.add_zookeeper_startup_command(create_zk_roots) @@ -32,11 +58,26 @@ def start_cluster(): def test_config(start_cluster): - assert node1.query("select value from system.settings where name = 'max_query_size'") == "33333\n" - assert node2.query("select value from system.settings where name = 'max_query_size'") == "55555\n" - assert node3.query("select value from system.settings where name = 'max_query_size'") == "77777\n" - assert node4.query("select value from system.settings where name = 'max_query_size'") == "99999\n" - assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" + assert ( + node1.query("select value from system.settings where name = 'max_query_size'") + == "33333\n" + ) + assert ( + node2.query("select value from system.settings where name = 'max_query_size'") + == "55555\n" + ) + assert ( + node3.query("select value from system.settings where name = 'max_query_size'") + == "77777\n" + ) + assert ( + node4.query("select value from system.settings where name = 'max_query_size'") + == "99999\n" + ) + assert ( + node6.query("select value from system.settings where name = 'max_query_size'") + == "99999\n" + ) def test_include_config(start_cluster): @@ -54,24 +95,68 @@ def test_include_config(start_cluster): def test_allow_databases(start_cluster): node5.query("CREATE DATABASE db1") node5.query( - "CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)") + "CREATE TABLE db1.test_table(date Date, k1 String, v1 Int32) ENGINE = MergeTree(date, (k1, date), 8192)" + ) node5.query("INSERT INTO db1.test_table VALUES('2000-01-01', 'test_key', 1)") - assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n" - assert node5.query( - "SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n" - assert node5.query( - "SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n" - assert node5.query( - "SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n" - assert node5.query( - "SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n" + assert ( + node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n" + ) + assert ( + node5.query( + "SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' " + ) + == "test_table\n" + ) + assert ( + node5.query( + "SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'" + ) + == "date\nk1\nv1\n" + ) + assert ( + node5.query( + "SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'" + ) + == "20000101_20000101_1_1_0\n" + ) + assert ( + node5.query( + "SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'" + ) + == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n" + ) - assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", - user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'", - user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'", - user="test_allow").strip() == "" - assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'", - user="test_allow").strip() == "" + assert ( + node5.query( + "SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow" + ).strip() + == "" + ) + assert ( + node5.query( + "SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", + user="test_allow", + ).strip() + == "" + ) + assert ( + node5.query( + "SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'", + user="test_allow", + ).strip() + == "" + ) + assert ( + node5.query( + "SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'", + user="test_allow", + ).strip() + == "" + ) + assert ( + node5.query( + "SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'", + user="test_allow", + ).strip() + == "" + ) diff --git a/tests/integration/test_config_xml_full/configs/config.xml b/tests/integration/test_config_xml_full/configs/config.xml index 76eceedbcea..4c0b8275869 100644 --- a/tests/integration/test_config_xml_full/configs/config.xml +++ b/tests/integration/test_config_xml_full/configs/config.xml @@ -490,20 +490,6 @@ default - - - diff --git a/tests/integration/test_config_xml_full/test.py b/tests/integration/test_config_xml_full/test.py index a8650a0dc55..ada3dc3f027 100644 --- a/tests/integration/test_config_xml_full/test.py +++ b/tests/integration/test_config_xml_full/test.py @@ -10,31 +10,53 @@ from helpers.cluster import ClickHouseCluster def test_xml_full_conf(): # all configs are in XML - cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml') + cluster = ClickHouseCluster( + __file__, zookeeper_config_path="configs/config.d/zookeeper.xml" + ) - all_confd = ['configs/config.d/access_control.xml', - 'configs/config.d/keeper_port.xml', - 'configs/config.d/logging_no_rotate.xml', - 'configs/config.d/log_to_console.xml', - 'configs/config.d/macros.xml', - 'configs/config.d/metric_log.xml', - 'configs/config.d/more_clusters.xml', - 'configs/config.d/part_log.xml', - 'configs/config.d/path.xml', - 'configs/config.d/query_masking_rules.xml', - 'configs/config.d/tcp_with_proxy.xml', - 'configs/config.d/text_log.xml', - 'configs/config.d/zookeeper.xml'] + all_confd = [ + "configs/config.d/access_control.xml", + "configs/config.d/keeper_port.xml", + "configs/config.d/logging_no_rotate.xml", + "configs/config.d/log_to_console.xml", + "configs/config.d/macros.xml", + "configs/config.d/metric_log.xml", + "configs/config.d/more_clusters.xml", + "configs/config.d/part_log.xml", + "configs/config.d/path.xml", + "configs/config.d/query_masking_rules.xml", + "configs/config.d/tcp_with_proxy.xml", + "configs/config.d/text_log.xml", + "configs/config.d/zookeeper.xml", + ] - all_userd = ['configs/users.d/allow_introspection_functions.xml', - 'configs/users.d/log_queries.xml'] + all_userd = [ + "configs/users.d/allow_introspection_functions.xml", + "configs/users.d/log_queries.xml", + ] - node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False) + node = cluster.add_instance( + "node", + base_config_dir="configs", + main_configs=all_confd, + user_configs=all_userd, + with_zookeeper=False, + ) try: cluster.start() - assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") - assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + assert ( + node.query( + "select value from system.settings where name = 'max_memory_usage'" + ) + == "10000000000\n" + ) + assert ( + node.query( + "select value from system.settings where name = 'max_block_size'" + ) + == "64999\n" + ) finally: cluster.shutdown() diff --git a/tests/integration/test_config_xml_main/test.py b/tests/integration/test_config_xml_main/test.py index 11efb5e283c..e6c2cf2973e 100644 --- a/tests/integration/test_config_xml_main/test.py +++ b/tests/integration/test_config_xml_main/test.py @@ -1,5 +1,3 @@ - - import time import threading from os import path as p, unlink @@ -12,32 +10,55 @@ from helpers.cluster import ClickHouseCluster def test_xml_main_conf(): # main configs are in XML; config.d and users.d are in YAML - cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml') + cluster = ClickHouseCluster( + __file__, zookeeper_config_path="configs/config.d/zookeeper.yaml" + ) - all_confd = ['configs/config.d/access_control.yaml', - 'configs/config.d/keeper_port.yaml', - 'configs/config.d/logging_no_rotate.yaml', - 'configs/config.d/log_to_console.yaml', - 'configs/config.d/macros.yaml', - 'configs/config.d/metric_log.yaml', - 'configs/config.d/more_clusters.yaml', - 'configs/config.d/part_log.yaml', - 'configs/config.d/path.yaml', - 'configs/config.d/query_masking_rules.yaml', - 'configs/config.d/tcp_with_proxy.yaml', - 'configs/config.d/test_cluster_with_incorrect_pw.yaml', - 'configs/config.d/text_log.yaml', - 'configs/config.d/zookeeper.yaml'] + all_confd = [ + "configs/config.d/access_control.yaml", + "configs/config.d/keeper_port.yaml", + "configs/config.d/logging_no_rotate.yaml", + "configs/config.d/log_to_console.yaml", + "configs/config.d/macros.yaml", + "configs/config.d/metric_log.yaml", + "configs/config.d/more_clusters.yaml", + "configs/config.d/part_log.yaml", + "configs/config.d/path.yaml", + "configs/config.d/query_masking_rules.yaml", + "configs/config.d/tcp_with_proxy.yaml", + "configs/config.d/test_cluster_with_incorrect_pw.yaml", + "configs/config.d/text_log.yaml", + "configs/config.d/zookeeper.yaml", + ] - all_userd = ['configs/users.d/allow_introspection_functions.yaml', - 'configs/users.d/log_queries.yaml'] + all_userd = [ + "configs/users.d/allow_introspection_functions.yaml", + "configs/users.d/log_queries.yaml", + ] - node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False, config_root_name='clickhouse') + node = cluster.add_instance( + "node", + base_config_dir="configs", + main_configs=all_confd, + user_configs=all_userd, + with_zookeeper=False, + config_root_name="clickhouse", + ) try: cluster.start() - assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") - assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + assert ( + node.query( + "select value from system.settings where name = 'max_memory_usage'" + ) + == "10000000000\n" + ) + assert ( + node.query( + "select value from system.settings where name = 'max_block_size'" + ) + == "64999\n" + ) finally: cluster.shutdown() diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml index 62e4ba8c744..8603c85e940 100644 --- a/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml @@ -1,4 +1,4 @@ -timezone: Europe/Moscow +timezone: Etc/UTC listen_host: 0.0.0.0 custom_settings_prefixes: custom_ path: /var/lib/clickhouse/ diff --git a/tests/integration/test_config_xml_yaml_mix/test.py b/tests/integration/test_config_xml_yaml_mix/test.py index 86cd68b3378..4138441b881 100644 --- a/tests/integration/test_config_xml_yaml_mix/test.py +++ b/tests/integration/test_config_xml_yaml_mix/test.py @@ -10,34 +10,58 @@ from helpers.cluster import ClickHouseCluster def test_extra_yaml_mix(): # some configs are written in XML, others are written in YAML - cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml') + cluster = ClickHouseCluster( + __file__, zookeeper_config_path="configs/config.d/zookeeper.xml" + ) - all_confd = ['configs/config.d/0_common_instance_config.yaml', - 'configs/config.d/access_control.yaml', - 'configs/config.d/keeper_port.xml', - 'configs/config.d/logging_no_rotate.xml', - 'configs/config.d/log_to_console.yaml', - 'configs/config.d/macros.yaml', - 'configs/config.d/metric_log.xml', - 'configs/config.d/more_clusters.yaml', - 'configs/config.d/part_log.xml', - 'configs/config.d/path.yaml', - 'configs/config.d/query_masking_rules.xml', - 'configs/config.d/tcp_with_proxy.yaml', - 'configs/config.d/test_cluster_with_incorrect_pw.xml', - 'configs/config.d/text_log.yaml', - 'configs/config.d/zookeeper.xml'] + all_confd = [ + "configs/config.d/0_common_instance_config.yaml", + "configs/config.d/access_control.yaml", + "configs/config.d/keeper_port.xml", + "configs/config.d/logging_no_rotate.xml", + "configs/config.d/log_to_console.yaml", + "configs/config.d/macros.yaml", + "configs/config.d/metric_log.xml", + "configs/config.d/more_clusters.yaml", + "configs/config.d/part_log.xml", + "configs/config.d/path.yaml", + "configs/config.d/query_masking_rules.xml", + "configs/config.d/tcp_with_proxy.yaml", + "configs/config.d/test_cluster_with_incorrect_pw.xml", + "configs/config.d/text_log.yaml", + "configs/config.d/zookeeper.xml", + ] - all_userd = ['configs/users.d/allow_introspection_functions.xml', - 'configs/users.d/log_queries.yaml'] + all_userd = [ + "configs/users.d/allow_introspection_functions.xml", + "configs/users.d/log_queries.yaml", + ] - node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False, - users_config_name="users.yaml", copy_common_configs=False, config_root_name="clickhouse") + node = cluster.add_instance( + "node", + base_config_dir="configs", + main_configs=all_confd, + user_configs=all_userd, + with_zookeeper=False, + users_config_name="users.yaml", + copy_common_configs=False, + config_root_name="clickhouse", + ) try: cluster.start() - assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") - assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + assert ( + node.query( + "select value from system.settings where name = 'max_memory_usage'" + ) + == "10000000000\n" + ) + assert ( + node.query( + "select value from system.settings where name = 'max_block_size'" + ) + == "64999\n" + ) finally: cluster.shutdown() diff --git a/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml index 62e4ba8c744..8603c85e940 100644 --- a/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml +++ b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml @@ -1,4 +1,4 @@ -timezone: Europe/Moscow +timezone: Etc/UTC listen_host: 0.0.0.0 custom_settings_prefixes: custom_ path: /var/lib/clickhouse/ diff --git a/tests/integration/test_config_yaml_full/test.py b/tests/integration/test_config_yaml_full/test.py index e8bf21754e0..ea0fd8c130c 100644 --- a/tests/integration/test_config_yaml_full/test.py +++ b/tests/integration/test_config_yaml_full/test.py @@ -7,36 +7,62 @@ import helpers import pytest from helpers.cluster import ClickHouseCluster + def test_yaml_full_conf(): # all configs are in YAML - cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml') + cluster = ClickHouseCluster( + __file__, zookeeper_config_path="configs/config.d/zookeeper.yaml" + ) - all_confd = ['configs/config.d/0_common_instance_config.yaml', - 'configs/config.d/access_control.yaml', - 'configs/config.d/keeper_port.yaml', - 'configs/config.d/logging_no_rotate.yaml', - 'configs/config.d/log_to_console.yaml', - 'configs/config.d/macros.yaml', - 'configs/config.d/metric_log.yaml', - 'configs/config.d/more_clusters.yaml', - 'configs/config.d/part_log.yaml', - 'configs/config.d/path.yaml', - 'configs/config.d/query_masking_rules.yaml', - 'configs/config.d/tcp_with_proxy.yaml', - 'configs/config.d/test_cluster_with_incorrect_pw.yaml', - 'configs/config.d/text_log.yaml', - 'configs/config.d/zookeeper.yaml'] + all_confd = [ + "configs/config.d/0_common_instance_config.yaml", + "configs/config.d/access_control.yaml", + "configs/config.d/keeper_port.yaml", + "configs/config.d/logging_no_rotate.yaml", + "configs/config.d/log_to_console.yaml", + "configs/config.d/macros.yaml", + "configs/config.d/metric_log.yaml", + "configs/config.d/more_clusters.yaml", + "configs/config.d/part_log.yaml", + "configs/config.d/path.yaml", + "configs/config.d/query_masking_rules.yaml", + "configs/config.d/tcp_with_proxy.yaml", + "configs/config.d/test_cluster_with_incorrect_pw.yaml", + "configs/config.d/text_log.yaml", + "configs/config.d/zookeeper.yaml", + ] - all_userd = ['configs/users.d/allow_introspection_functions.yaml', - 'configs/users.d/log_queries.yaml'] + all_userd = [ + "configs/users.d/allow_introspection_functions.yaml", + "configs/users.d/log_queries.yaml", + ] - node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, - with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False, config_root_name="clickhouse") + node = cluster.add_instance( + "node", + base_config_dir="configs", + main_configs=all_confd, + user_configs=all_userd, + with_zookeeper=False, + main_config_name="config.yaml", + users_config_name="users.yaml", + copy_common_configs=False, + config_root_name="clickhouse", + ) try: cluster.start() - assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") - assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + assert ( + node.query( + "select value from system.settings where name = 'max_memory_usage'" + ) + == "10000000000\n" + ) + assert ( + node.query( + "select value from system.settings where name = 'max_block_size'" + ) + == "64999\n" + ) finally: cluster.shutdown() diff --git a/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml index 62e4ba8c744..8603c85e940 100644 --- a/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml +++ b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml @@ -1,4 +1,4 @@ -timezone: Europe/Moscow +timezone: Etc/UTC listen_host: 0.0.0.0 custom_settings_prefixes: custom_ path: /var/lib/clickhouse/ diff --git a/tests/integration/test_config_yaml_main/test.py b/tests/integration/test_config_yaml_main/test.py index bb4c8eb8f9f..468a63359e3 100644 --- a/tests/integration/test_config_yaml_main/test.py +++ b/tests/integration/test_config_yaml_main/test.py @@ -10,35 +10,59 @@ from helpers.cluster import ClickHouseCluster def test_yaml_main_conf(): # main configs are in YAML; config.d and users.d are in XML - cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml') + cluster = ClickHouseCluster( + __file__, zookeeper_config_path="configs/config.d/zookeeper.xml" + ) - all_confd = ['configs/config.d/0_common_instance_config.yaml', - 'configs/config.d/access_control.xml', - 'configs/config.d/keeper_port.xml', - 'configs/config.d/logging_no_rotate.xml', - 'configs/config.d/log_to_console.xml', - 'configs/config.d/macros.xml', - 'configs/config.d/metric_log.xml', - 'configs/config.d/more_clusters.xml', - 'configs/config.d/part_log.xml', - 'configs/config.d/path.xml', - 'configs/config.d/query_masking_rules.xml', - 'configs/config.d/tcp_with_proxy.xml', - 'configs/config.d/test_cluster_with_incorrect_pw.xml', - 'configs/config.d/text_log.xml', - 'configs/config.d/zookeeper.xml'] + all_confd = [ + "configs/config.d/0_common_instance_config.yaml", + "configs/config.d/access_control.xml", + "configs/config.d/keeper_port.xml", + "configs/config.d/logging_no_rotate.xml", + "configs/config.d/log_to_console.xml", + "configs/config.d/macros.xml", + "configs/config.d/metric_log.xml", + "configs/config.d/more_clusters.xml", + "configs/config.d/part_log.xml", + "configs/config.d/path.xml", + "configs/config.d/query_masking_rules.xml", + "configs/config.d/tcp_with_proxy.xml", + "configs/config.d/test_cluster_with_incorrect_pw.xml", + "configs/config.d/text_log.xml", + "configs/config.d/zookeeper.xml", + ] - all_userd = ['configs/users.d/allow_introspection_functions.xml', - 'configs/users.d/log_queries.xml'] + all_userd = [ + "configs/users.d/allow_introspection_functions.xml", + "configs/users.d/log_queries.xml", + ] - node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, - with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", - copy_common_configs=False, config_root_name="clickhouse") + node = cluster.add_instance( + "node", + base_config_dir="configs", + main_configs=all_confd, + user_configs=all_userd, + with_zookeeper=False, + main_config_name="config.yaml", + users_config_name="users.yaml", + copy_common_configs=False, + config_root_name="clickhouse", + ) try: cluster.start() - assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") - assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + assert ( + node.query( + "select value from system.settings where name = 'max_memory_usage'" + ) + == "10000000000\n" + ) + assert ( + node.query( + "select value from system.settings where name = 'max_block_size'" + ) + == "64999\n" + ) finally: cluster.shutdown() diff --git a/tests/integration/test_consistant_parts_after_move_partition/test.py b/tests/integration/test_consistant_parts_after_move_partition/test.py index 2070c8cb3f8..63a51472773 100644 --- a/tests/integration/test_consistant_parts_after_move_partition/test.py +++ b/tests/integration/test_consistant_parts_after_move_partition/test.py @@ -3,12 +3,13 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry -CLICKHOUSE_DATABASE = 'test' +CLICKHOUSE_DATABASE = "test" def initialize_database(nodes, shard): for node in nodes: - node.query(''' + node.query( + """ CREATE DATABASE {database}; CREATE TABLE `{database}`.src (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/{database}/tables/test_consistent_shard1{shard}/replicated', '{replica}') @@ -18,12 +19,19 @@ def initialize_database(nodes, shard): ENGINE = ReplicatedMergeTree('/clickhouse/{database}/tables/test_consistent_shard2{shard}/replicated', '{replica}') ORDER BY d PARTITION BY p SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name, database=CLICKHOUSE_DATABASE)) + """.format( + shard=shard, replica=node.name, database=CLICKHOUSE_DATABASE + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -41,15 +49,25 @@ def start_cluster(): def test_consistent_part_after_move_partition(start_cluster): # insert into all replicas for i in range(100): - node1.query('INSERT INTO `{database}`.src VALUES ({value} % 2, {value})'.format(database=CLICKHOUSE_DATABASE, - value=i)) - query_source = 'SELECT COUNT(*) FROM `{database}`.src'.format(database=CLICKHOUSE_DATABASE) - query_dest = 'SELECT COUNT(*) FROM `{database}`.dest'.format(database=CLICKHOUSE_DATABASE) + node1.query( + "INSERT INTO `{database}`.src VALUES ({value} % 2, {value})".format( + database=CLICKHOUSE_DATABASE, value=i + ) + ) + query_source = "SELECT COUNT(*) FROM `{database}`.src".format( + database=CLICKHOUSE_DATABASE + ) + query_dest = "SELECT COUNT(*) FROM `{database}`.dest".format( + database=CLICKHOUSE_DATABASE + ) assert_eq_with_retry(node2, query_source, node1.query(query_source)) assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) node1.query( - 'ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest'.format(database=CLICKHOUSE_DATABASE)) + "ALTER TABLE `{database}`.src MOVE PARTITION 1 TO TABLE `{database}`.dest".format( + database=CLICKHOUSE_DATABASE + ) + ) assert_eq_with_retry(node2, query_source, node1.query(query_source)) assert_eq_with_retry(node2, query_dest, node1.query(query_dest)) diff --git a/tests/integration/test_consistent_parts_after_clone_replica/test.py b/tests/integration/test_consistent_parts_after_clone_replica/test.py index b0b69da0902..d3f8b22ef57 100644 --- a/tests/integration/test_consistent_parts_after_clone_replica/test.py +++ b/tests/integration/test_consistent_parts_after_clone_replica/test.py @@ -4,21 +4,29 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry + def fill_nodes(nodes, shard): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -38,14 +46,20 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster): # insert into all replicas for i in range(10): node1.query("INSERT INTO test_table VALUES ('2019-08-16', {})".format(i)) - assert_eq_with_retry(node2, "SELECT count(*) FROM test_table", node1.query("SELECT count(*) FROM test_table")) + assert_eq_with_retry( + node2, + "SELECT count(*) FROM test_table", + node1.query("SELECT count(*) FROM test_table"), + ) # partition the first replica from the second one and (later) from zk pm.partition_instances(node1, node2) # insert some parts on the second replica only, we will drop these parts for i in range(10): - node2.query("INSERT INTO test_table VALUES ('2019-08-16', {})".format(10 + i)) + node2.query( + "INSERT INTO test_table VALUES ('2019-08-16', {})".format(10 + i) + ) pm.drop_instance_zk_connections(node1) @@ -56,16 +70,26 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster): # insert into the second replica # DROP_RANGE will be removed from the replication log and the first replica will be lost for i in range(20): - node2.query("INSERT INTO test_table VALUES ('2019-08-16', {})".format(20 + i)) + node2.query( + "INSERT INTO test_table VALUES ('2019-08-16', {})".format(20 + i) + ) - assert_eq_with_retry(node2, "SELECT value FROM system.zookeeper WHERE path='/clickhouse/tables/test1/replicated/replicas/node1' AND name='is_lost'", "1") + assert_eq_with_retry( + node2, + "SELECT value FROM system.zookeeper WHERE path='/clickhouse/tables/test1/replicated/replicas/node1' AND name='is_lost'", + "1", + ) node2.wait_for_log_line("Will mark replica node1 as lost") # the first replica will be cloned from the second pm.heal_all() node2.wait_for_log_line("Sending part") - assert_eq_with_retry(node1, "SELECT count(*) FROM test_table", node2.query("SELECT count(*) FROM test_table")) + assert_eq_with_retry( + node1, + "SELECT count(*) FROM test_table", + node2.query("SELECT count(*) FROM test_table"), + ) # ensure replica was cloned assert node1.contains_in_log("Will mimic node2") @@ -77,5 +101,13 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster): # `Skipping action for part 201908_40_40_0 because part 201908_21_40_4 already exists.` # # In any case after a short while the replication queue should be empty - assert_eq_with_retry(node1, "SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'", "0") - assert_eq_with_retry(node2, "SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'", "0") + assert_eq_with_retry( + node1, + "SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'", + "0", + ) + assert_eq_with_retry( + node2, + "SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'", + "0", + ) diff --git a/tests/integration/test_create_user_and_login/test.py b/tests/integration/test_create_user_and_login/test.py index 2ce134fea1a..fd052ba9716 100644 --- a/tests/integration/test_create_user_and_login/test.py +++ b/tests/integration/test_create_user_and_login/test.py @@ -5,7 +5,7 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -28,34 +28,38 @@ def cleanup_after_test(): def test_login(): instance.query("CREATE USER A") instance.query("CREATE USER B") - assert instance.query("SELECT 1", user='A') == "1\n" - assert instance.query("SELECT 1", user='B') == "1\n" + assert instance.query("SELECT 1", user="A") == "1\n" + assert instance.query("SELECT 1", user="B") == "1\n" def test_grant_create_user(): instance.query("CREATE USER A") expected_error = "Not enough privileges" - assert expected_error in instance.query_and_get_error("CREATE USER B", user='A') + assert expected_error in instance.query_and_get_error("CREATE USER B", user="A") instance.query("GRANT CREATE USER ON *.* TO A") - instance.query("CREATE USER B", user='A') - assert instance.query("SELECT 1", user='B') == "1\n" + instance.query("CREATE USER B", user="A") + assert instance.query("SELECT 1", user="B") == "1\n" def test_login_as_dropped_user(): for _ in range(0, 2): instance.query("CREATE USER A") - assert instance.query("SELECT 1", user='A') == "1\n" + assert instance.query("SELECT 1", user="A") == "1\n" instance.query("DROP USER A") expected_error = "no user with such name" - assert expected_error in instance.query_and_get_error("SELECT 1", user='A') + assert expected_error in instance.query_and_get_error("SELECT 1", user="A") def test_login_as_dropped_user_xml(): for _ in range(0, 2): - instance.exec_in_container(["bash", "-c" , """ + instance.exec_in_container( + [ + "bash", + "-c", + """ cat > /etc/clickhouse-server/users.d/user_c.xml << EOF @@ -65,15 +69,21 @@ def test_login_as_dropped_user_xml(): -EOF"""]) +EOF""", + ] + ) - assert_eq_with_retry(instance, "SELECT name FROM system.users WHERE name='C'", "C") + assert_eq_with_retry( + instance, "SELECT name FROM system.users WHERE name='C'", "C" + ) - instance.exec_in_container(["bash", "-c" , "rm /etc/clickhouse-server/users.d/user_c.xml"]) + instance.exec_in_container( + ["bash", "-c", "rm /etc/clickhouse-server/users.d/user_c.xml"] + ) expected_error = "no user with such name" while True: - out, err = instance.query_and_get_answer_with_error("SELECT 1", user='C') + out, err = instance.query_and_get_answer_with_error("SELECT 1", user="C") if expected_error in err: logging.debug(f"Got error '{expected_error}' just as expected") break @@ -81,6 +91,8 @@ EOF"""]) logging.debug(f"Got output '1', retrying...") time.sleep(0.5) continue - raise Exception(f"Expected either output '1' or error '{expected_error}', got output={out} and error={err}") - + raise Exception( + f"Expected either output '1' or error '{expected_error}', got output={out} and error={err}" + ) + assert instance.query("SELECT name FROM system.users WHERE name='C'") == "" diff --git a/tests/integration/test_cross_replication/test.py b/tests/integration/test_cross_replication/test.py index cc5618e04e6..143b8823bf2 100644 --- a/tests/integration/test_cross_replication/test.py +++ b/tests/integration/test_cross_replication/test.py @@ -8,9 +8,15 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -26,24 +32,30 @@ def started_cluster(): for node, shards in node_to_shards: for shard in shards: - node.query(''' + node.query( + """ CREATE DATABASE shard_{shard}; CREATE TABLE shard_{shard}.replicated(date Date, id UInt32, shard_id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) - node.query(''' + node.query( + """ CREATE TABLE distributed(date Date, id UInt32, shard_id UInt32) ENGINE = Distributed(test_cluster, '', replicated, shard_id); -''') +""" + ) # Insert some data onto different shards using the Distributed table - to_insert = '''\ + to_insert = """\ 2017-06-16 111 0 2017-06-16 222 1 2017-06-16 333 2 -''' +""" node1.query("INSERT INTO distributed FORMAT TSV", stdin=to_insert) time.sleep(5) @@ -55,38 +67,52 @@ CREATE TABLE distributed(date Date, id UInt32, shard_id UInt32) def test(started_cluster): # Check that the data has been inserted into correct tables. - assert_eq_with_retry(node1, "SELECT id FROM shard_0.replicated", '111') - assert_eq_with_retry(node1, "SELECT id FROM shard_2.replicated", '333') + assert_eq_with_retry(node1, "SELECT id FROM shard_0.replicated", "111") + assert_eq_with_retry(node1, "SELECT id FROM shard_2.replicated", "333") - assert_eq_with_retry(node2, "SELECT id FROM shard_0.replicated", '111') - assert_eq_with_retry(node2, "SELECT id FROM shard_1.replicated", '222') + assert_eq_with_retry(node2, "SELECT id FROM shard_0.replicated", "111") + assert_eq_with_retry(node2, "SELECT id FROM shard_1.replicated", "222") - assert_eq_with_retry(node3, "SELECT id FROM shard_1.replicated", '222') - assert_eq_with_retry(node3, "SELECT id FROM shard_2.replicated", '333') + assert_eq_with_retry(node3, "SELECT id FROM shard_1.replicated", "222") + assert_eq_with_retry(node3, "SELECT id FROM shard_2.replicated", "333") # Check that SELECT from the Distributed table works. - expected_from_distributed = '''\ + expected_from_distributed = """\ 2017-06-16 111 0 2017-06-16 222 1 2017-06-16 333 2 -''' - assert_eq_with_retry(node1, "SELECT * FROM distributed ORDER BY id", expected_from_distributed) - assert_eq_with_retry(node2, "SELECT * FROM distributed ORDER BY id", expected_from_distributed) - assert_eq_with_retry(node3, "SELECT * FROM distributed ORDER BY id", expected_from_distributed) +""" + assert_eq_with_retry( + node1, "SELECT * FROM distributed ORDER BY id", expected_from_distributed + ) + assert_eq_with_retry( + node2, "SELECT * FROM distributed ORDER BY id", expected_from_distributed + ) + assert_eq_with_retry( + node3, "SELECT * FROM distributed ORDER BY id", expected_from_distributed + ) # Now isolate node3 from other nodes and check that SELECTs on other nodes still work. with PartitionManager() as pm: - pm.partition_instances(node3, node1, action='REJECT --reject-with tcp-reset') - pm.partition_instances(node3, node2, action='REJECT --reject-with tcp-reset') + pm.partition_instances(node3, node1, action="REJECT --reject-with tcp-reset") + pm.partition_instances(node3, node2, action="REJECT --reject-with tcp-reset") - assert_eq_with_retry(node1, "SELECT * FROM distributed ORDER BY id", expected_from_distributed) - assert_eq_with_retry(node2, "SELECT * FROM distributed ORDER BY id", expected_from_distributed) + assert_eq_with_retry( + node1, "SELECT * FROM distributed ORDER BY id", expected_from_distributed + ) + assert_eq_with_retry( + node2, "SELECT * FROM distributed ORDER BY id", expected_from_distributed + ) with pytest.raises(Exception): - print(node3.query_with_retry("SELECT * FROM distributed ORDER BY id", retry_count=5)) + print( + node3.query_with_retry( + "SELECT * FROM distributed ORDER BY id", retry_count=5 + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": with contextmanager(started_cluster)() as cluster: for name, instance in list(cluster.instances.items()): print(name, instance.ip_address) diff --git a/tests/integration/test_custom_settings/test.py b/tests/integration/test_custom_settings/test.py index 7e147f999a9..2dd4a7dafef 100644 --- a/tests/integration/test_custom_settings/test.py +++ b/tests/integration/test_custom_settings/test.py @@ -4,7 +4,7 @@ from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') +node = cluster.add_instance("node") @pytest.fixture(scope="module", autouse=True) @@ -18,7 +18,10 @@ def started_cluster(): def test_custom_settings(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/custom_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/custom_settings.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) node.query("SYSTEM RELOAD CONFIG") assert node.query("SELECT getSetting('custom_a')") == "-5\n" @@ -28,6 +31,9 @@ def test_custom_settings(): def test_illformed_setting(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/illformed_setting.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/illformed_setting.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) error_message = "Couldn't restore Field from dump: 1" assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") diff --git a/tests/integration/test_ddl_alter_query/test.py b/tests/integration/test_ddl_alter_query/test.py index d65e40084f6..f87d943622c 100644 --- a/tests/integration/test_ddl_alter_query/test.py +++ b/tests/integration/test_ddl_alter_query/test.py @@ -4,10 +4,18 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node4 = cluster.add_instance( + "node4", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -18,13 +26,17 @@ def started_cluster(): for i, node in enumerate([node1, node2]): node.query("CREATE DATABASE testdb") node.query( - '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format( - i)) + """CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;""".format( + i + ) + ) for i, node in enumerate([node3, node4]): node.query("CREATE DATABASE testdb") node.query( - '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format( - i)) + """CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;""".format( + i + ) + ) yield cluster finally: @@ -32,13 +44,19 @@ def started_cluster(): def test_alter(started_cluster): - node1.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") - node3.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node1.query( + "INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)" + ) + node3.query( + "INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)" + ) node2.query("SYSTEM SYNC REPLICA testdb.test_table") node4.query("SYSTEM SYNC REPLICA testdb.test_table") - node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", - settings={"replication_alter_partitions_sync": "2"}) + node1.query( + "ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", + settings={"replication_alter_partitions_sync": "2"}, + ) node1.query("SYSTEM SYNC REPLICA testdb.test_table") node2.query("SYSTEM SYNC REPLICA testdb.test_table") diff --git a/tests/integration/test_ddl_worker_non_leader/test.py b/tests/integration/test_ddl_worker_non_leader/test.py index 172fc03c005..e7b0efa54f1 100644 --- a/tests/integration/test_ddl_worker_non_leader/test.py +++ b/tests/integration/test_ddl_worker_non_leader/test.py @@ -5,8 +5,13 @@ from helpers.network import PartitionManager from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) + @pytest.fixture(scope="module") def started_cluster(): @@ -20,40 +25,58 @@ def started_cluster(): def test_non_leader_replica(started_cluster): - node1.query_with_retry('''CREATE TABLE IF NOT EXISTS sometable(id UInt32, value String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '1') ORDER BY tuple()''') + node1.query_with_retry( + """CREATE TABLE IF NOT EXISTS sometable(id UInt32, value String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '1') ORDER BY tuple()""" + ) - node2.query_with_retry('''CREATE TABLE IF NOT EXISTS sometable(id UInt32, value String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '2') ORDER BY tuple() SETTINGS replicated_can_become_leader = 0''') + node2.query_with_retry( + """CREATE TABLE IF NOT EXISTS sometable(id UInt32, value String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '2') ORDER BY tuple() SETTINGS replicated_can_become_leader = 0""" + ) - node1.query("INSERT INTO sometable SELECT number, toString(number) FROM numbers(100)") + node1.query( + "INSERT INTO sometable SELECT number, toString(number) FROM numbers(100)" + ) node2.query_with_retry("SYSTEM SYNC REPLICA sometable", timeout=10) assert node1.query("SELECT COUNT() FROM sometable") == "100\n" assert node2.query("SELECT COUNT() FROM sometable") == "100\n" - with PartitionManager() as pm: pm.drop_instance_zk_connections(node1) # this query should be executed by leader, but leader partitioned from zookeeper with pytest.raises(Exception): - node2.query("ALTER TABLE sometable ON CLUSTER 'test_cluster' MODIFY COLUMN value UInt64 SETTINGS distributed_ddl_task_timeout=5") + node2.query( + "ALTER TABLE sometable ON CLUSTER 'test_cluster' MODIFY COLUMN value UInt64 SETTINGS distributed_ddl_task_timeout=5" + ) for _ in range(100): - if 'UInt64' in node1.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'"): + if "UInt64" in node1.query( + "SELECT type FROM system.columns WHERE name='value' and table = 'sometable'" + ): break time.sleep(0.1) for _ in range(100): - if 'UInt64' in node2.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'"): + if "UInt64" in node2.query( + "SELECT type FROM system.columns WHERE name='value' and table = 'sometable'" + ): break time.sleep(0.1) - assert 'UInt64' in node1.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'") - assert 'UInt64' in node2.query("SELECT type FROM system.columns WHERE name='value' and table = 'sometable'") + assert "UInt64" in node1.query( + "SELECT type FROM system.columns WHERE name='value' and table = 'sometable'" + ) + assert "UInt64" in node2.query( + "SELECT type FROM system.columns WHERE name='value' and table = 'sometable'" + ) # Checking that DDLWorker doesn't hung and still able to execute DDL queries - node1.query("CREATE TABLE new_table_with_ddl ON CLUSTER 'test_cluster' (key UInt32) ENGINE=MergeTree() ORDER BY tuple()", settings={"distributed_ddl_task_timeout": "10"}) + node1.query( + "CREATE TABLE new_table_with_ddl ON CLUSTER 'test_cluster' (key UInt32) ENGINE=MergeTree() ORDER BY tuple()", + settings={"distributed_ddl_task_timeout": "10"}, + ) assert node1.query("EXISTS new_table_with_ddl") == "1\n" assert node2.query("EXISTS new_table_with_ddl") == "1\n" diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index d114954d739..4af276b9728 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -7,10 +7,26 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/default_compression.xml', 'configs/wide_parts_only.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/default_compression.xml', 'configs/wide_parts_only.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/default_compression.xml', 'configs/wide_parts_only.xml'], image='yandex/clickhouse-server', tag='20.3.16', stay_alive=True, with_installed_binary=True) -node4 = cluster.add_instance('node4') +node1 = cluster.add_instance( + "node1", + main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], + with_zookeeper=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], + image="yandex/clickhouse-server", + tag="20.3.16", + stay_alive=True, + with_installed_binary=True, +) +node4 = cluster.add_instance("node4") + @pytest.fixture(scope="module") def start_cluster(): @@ -24,45 +40,59 @@ def start_cluster(): def get_compression_codec_byte(node, table_name, part_name): cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -N 1 | head -n 1 | awk '{{print $2}}'".format( - table_name, part_name) + table_name, part_name + ) return node.exec_in_container(["bash", "-c", cmd]).strip() def get_second_multiple_codec_byte(node, table_name, part_name): cmd = "tail -c +17 /var/lib/clickhouse/data/default/{}/{}/data1.bin | od -x -j 11 -N 1 | head -n 1 | awk '{{print $2}}'".format( - table_name, part_name) + table_name, part_name + ) return node.exec_in_container(["bash", "-c", cmd]).strip() def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) CODECS_MAPPING = { - 'LZ4': '0082', - 'LZ4HC': '0082', # not an error, same byte - 'ZSTD': '0090', - 'Multiple': '0091', + "LZ4": "0082", + "LZ4HC": "0082", # not an error, same byte + "ZSTD": "0090", + "Multiple": "0091", } def test_default_codec_single(start_cluster): for i, node in enumerate([node1, node2]): - node.query(""" + node.query( + """ CREATE TABLE compression_table ( key UInt64, data1 String CODEC(Default) ) ENGINE = ReplicatedMergeTree('/t', '{}') ORDER BY tuple() PARTITION BY key; - """.format(i)) + """.format( + i + ) + ) # ZSTD(10) and ZSTD(10) after merge node1.query("INSERT INTO compression_table VALUES (1, 'x')") # ZSTD(10) and LZ4HC(10) after merge - node1.query("INSERT INTO compression_table VALUES (2, '{}')".format(get_random_string(2048))) + node1.query( + "INSERT INTO compression_table VALUES (2, '{}')".format(get_random_string(2048)) + ) # ZSTD(10) and LZ4 after merge - node1.query("INSERT INTO compression_table VALUES (3, '{}')".format(get_random_string(22048))) + node1.query( + "INSERT INTO compression_table VALUES (3, '{}')".format( + get_random_string(22048) + ) + ) node2.query("SYSTEM SYNC REPLICA compression_table", timeout=15) @@ -77,23 +107,56 @@ def test_default_codec_single(start_cluster): node2.query("SYSTEM FLUSH LOGS") # Same codec for all - assert get_compression_codec_byte(node1, "compression_table", "1_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table", "1_0_0_0") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_0'" + ) + == "ZSTD(10)\n" + ) - assert get_compression_codec_byte(node1, "compression_table", "2_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table", "2_0_0_0") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_0'" + ) + == "ZSTD(10)\n" + ) - assert get_compression_codec_byte(node1, "compression_table", "3_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table", "3_0_0_0") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_0'" + ) + == "ZSTD(10)\n" + ) # just to be sure that replication works node1.query("OPTIMIZE TABLE compression_table FINAL") @@ -110,23 +173,56 @@ def test_default_codec_single(start_cluster): node1.query("SYSTEM FLUSH LOGS") node2.query("SYSTEM FLUSH LOGS") - assert get_compression_codec_byte(node1, "compression_table", "1_0_0_1") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table", "1_0_0_1") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_0_0_1'" + ) + == "ZSTD(10)\n" + ) - assert get_compression_codec_byte(node1, "compression_table", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'") == "LZ4HC(5)\n" + assert ( + get_compression_codec_byte(node1, "compression_table", "2_0_0_1") + == CODECS_MAPPING["LZ4HC"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'" + ) + == "LZ4HC(5)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_0_0_1'" + ) + == "LZ4HC(5)\n" + ) - assert get_compression_codec_byte(node1, "compression_table", "3_0_0_1") == CODECS_MAPPING['LZ4'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'") == "LZ4\n" + assert ( + get_compression_codec_byte(node1, "compression_table", "3_0_0_1") + == CODECS_MAPPING["LZ4"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'" + ) + == "LZ4\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_0_0_1'" + ) + == "LZ4\n" + ) assert node1.query("SELECT COUNT() FROM compression_table") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table") == "3\n" @@ -137,68 +233,165 @@ def test_default_codec_single(start_cluster): def test_default_codec_multiple(start_cluster): for i, node in enumerate([node1, node2]): - node.query(""" + node.query( + """ CREATE TABLE compression_table_multiple ( key UInt64, data1 String CODEC(NONE, Default) ) ENGINE = ReplicatedMergeTree('/d', '{}') ORDER BY tuple() PARTITION BY key; - """.format(i), settings={"allow_suspicious_codecs": 1}) + """.format( + i + ), + settings={"allow_suspicious_codecs": 1}, + ) # ZSTD(10) and ZSTD(10) after merge node1.query("INSERT INTO compression_table_multiple VALUES (1, 'x')") # ZSTD(10) and LZ4HC(10) after merge - node1.query("INSERT INTO compression_table_multiple VALUES (2, '{}')".format(get_random_string(2048))) + node1.query( + "INSERT INTO compression_table_multiple VALUES (2, '{}')".format( + get_random_string(2048) + ) + ) # ZSTD(10) and LZ4 after merge - node1.query("INSERT INTO compression_table_multiple VALUES (3, '{}')".format(get_random_string(22048))) + node1.query( + "INSERT INTO compression_table_multiple VALUES (3, '{}')".format( + get_random_string(22048) + ) + ) # Same codec for all - assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['Multiple'] - assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_0") + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_0") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_0'" + ) + == "ZSTD(10)\n" + ) - assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['Multiple'] - assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_0") + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_0") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_0'" + ) + == "ZSTD(10)\n" + ) - assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['Multiple'] - assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_0") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_0") + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_0") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_0'" + ) + == "ZSTD(10)\n" + ) node2.query("SYSTEM SYNC REPLICA compression_table_multiple", timeout=15) node1.query("OPTIMIZE TABLE compression_table_multiple FINAL") - assert get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['Multiple'] - assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_1") == CODECS_MAPPING['ZSTD'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'") == "ZSTD(10)\n" + assert ( + get_compression_codec_byte(node1, "compression_table_multiple", "1_0_0_1") + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_second_multiple_codec_byte(node1, "compression_table_multiple", "1_0_0_1") + == CODECS_MAPPING["ZSTD"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'" + ) + == "ZSTD(10)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '1_0_0_1'" + ) + == "ZSTD(10)\n" + ) - assert get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['Multiple'] - assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") == CODECS_MAPPING['LZ4HC'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'") == "LZ4HC(5)\n" + assert ( + get_compression_codec_byte(node1, "compression_table_multiple", "2_0_0_1") + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_second_multiple_codec_byte(node1, "compression_table_multiple", "2_0_0_1") + == CODECS_MAPPING["LZ4HC"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'" + ) + == "LZ4HC(5)\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '2_0_0_1'" + ) + == "LZ4HC(5)\n" + ) - assert get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['Multiple'] - assert get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_1") == CODECS_MAPPING['LZ4'] - assert node1.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'") == "LZ4\n" + assert ( + get_compression_codec_byte(node1, "compression_table_multiple", "3_0_0_1") + == CODECS_MAPPING["Multiple"] + ) + assert ( + get_second_multiple_codec_byte(node1, "compression_table_multiple", "3_0_0_1") + == CODECS_MAPPING["LZ4"] + ) + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'" + ) + == "LZ4\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table_multiple' and name = '3_0_0_1'" + ) + == "LZ4\n" + ) assert node1.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" assert node2.query("SELECT COUNT() FROM compression_table_multiple") == "3\n" @@ -208,41 +401,81 @@ def test_default_codec_multiple(start_cluster): def test_default_codec_version_update(start_cluster): - node3.query(""" + node3.query( + """ CREATE TABLE compression_table ( key UInt64 CODEC(LZ4HC(7)), data1 String ) ENGINE = MergeTree ORDER BY tuple() PARTITION BY key; - """) + """ + ) node3.query("INSERT INTO compression_table VALUES (1, 'x')") - node3.query("INSERT INTO compression_table VALUES (2, '{}')".format(get_random_string(2048))) - node3.query("INSERT INTO compression_table VALUES (3, '{}')".format(get_random_string(22048))) + node3.query( + "INSERT INTO compression_table VALUES (2, '{}')".format(get_random_string(2048)) + ) + node3.query( + "INSERT INTO compression_table VALUES (3, '{}')".format( + get_random_string(22048) + ) + ) old_version = node3.query("SELECT version()") node3.restart_with_latest_version() new_version = node3.query("SELECT version()") logging.debug(f"Updated from {old_version} to {new_version}") - assert node3.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'") == "ZSTD(1)\n" - assert node3.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'") == "ZSTD(1)\n" - assert node3.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'") == "ZSTD(1)\n" + assert ( + node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_0'" + ) + == "ZSTD(1)\n" + ) + assert ( + node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_0'" + ) + == "ZSTD(1)\n" + ) + assert ( + node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_0'" + ) + == "ZSTD(1)\n" + ) node3.query("OPTIMIZE TABLE compression_table FINAL") - assert node3.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'") == "ZSTD(10)\n" - assert node3.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'") == "LZ4HC(5)\n" - assert node3.query( - "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'") == "LZ4\n" + assert ( + node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '1_1_1_1'" + ) + == "ZSTD(10)\n" + ) + assert ( + node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '2_2_2_1'" + ) + == "LZ4HC(5)\n" + ) + assert ( + node3.query( + "SELECT default_compression_codec FROM system.parts WHERE table = 'compression_table' and name = '3_3_3_1'" + ) + == "LZ4\n" + ) node3.query("DROP TABLE compression_table SYNC") def callback(n): - n.exec_in_container(['bash', '-c', 'rm -rf /var/lib/clickhouse/metadata/system /var/lib/clickhouse/data/system '], user='root') + n.exec_in_container( + [ + "bash", + "-c", + "rm -rf /var/lib/clickhouse/metadata/system /var/lib/clickhouse/data/system ", + ], + user="root", + ) + node3.restart_with_original_version(callback_onstop=callback) cur_version = node3.query("SELECT version()") @@ -250,20 +483,28 @@ def test_default_codec_version_update(start_cluster): def test_default_codec_for_compact_parts(start_cluster): - node4.query(""" + node4.query( + """ CREATE TABLE compact_parts_table ( key UInt64, data String ) ENGINE MergeTree ORDER BY tuple() - """) + """ + ) node4.query("INSERT INTO compact_parts_table VALUES (1, 'Hello world')") assert node4.query("SELECT COUNT() FROM compact_parts_table") == "1\n" node4.query("ALTER TABLE compact_parts_table DETACH PART 'all_1_1_0'") - node4.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/data/default/compact_parts_table/detached/all_1_1_0/default_compression_codec.txt"]) + node4.exec_in_container( + [ + "bash", + "-c", + "rm /var/lib/clickhouse/data/default/compact_parts_table/detached/all_1_1_0/default_compression_codec.txt", + ] + ) node4.query("ALTER TABLE compact_parts_table ATTACH PART 'all_1_1_0'") diff --git a/tests/integration/test_default_database_on_cluster/test.py b/tests/integration/test_default_database_on_cluster/test.py index 28a3cfad1d1..f0f1360434f 100644 --- a/tests/integration/test_default_database_on_cluster/test.py +++ b/tests/integration/test_default_database_on_cluster/test.py @@ -2,18 +2,38 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -ch1 = cluster.add_instance('ch1', - main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], - with_zookeeper=True) -ch2 = cluster.add_instance('ch2', - main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], - with_zookeeper=True) -ch3 = cluster.add_instance('ch3', - main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], - with_zookeeper=True) -ch4 = cluster.add_instance('ch4', - main_configs=["configs/config.d/clusters.xml", "configs/config.d/distributed_ddl.xml"], - with_zookeeper=True) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) +ch2 = cluster.add_instance( + "ch2", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) +ch3 = cluster.add_instance( + "ch3", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) +ch4 = cluster.add_instance( + "ch4", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -28,17 +48,30 @@ def started_cluster(): def test_default_database_on_cluster(started_cluster): - ch1.query(database='test_default_database', - sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;") + ch1.query( + database="test_default_database", + sql="CREATE TABLE test_local_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Memory;", + ) for node in [ch1, ch2, ch3, ch4]: - assert node.query("SHOW TABLES FROM test_default_database FORMAT TSV") == "test_local_table\n" + assert ( + node.query("SHOW TABLES FROM test_default_database FORMAT TSV") + == "test_local_table\n" + ) - ch1.query(database='test_default_database', - sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');") + ch1.query( + database="test_default_database", + sql="CREATE TABLE test_distributed_table ON CLUSTER 'cluster' (column UInt8) ENGINE = Distributed(cluster, currentDatabase(), 'test_local_table');", + ) for node in [ch1, ch2, ch3, ch4]: - assert node.query( - "SHOW TABLES FROM test_default_database FORMAT TSV") == "test_distributed_table\ntest_local_table\n" - assert node.query( - "SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV") == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n" + assert ( + node.query("SHOW TABLES FROM test_default_database FORMAT TSV") + == "test_distributed_table\ntest_local_table\n" + ) + assert ( + node.query( + "SHOW CREATE TABLE test_default_database.test_distributed_table FORMAT TSV" + ) + == "CREATE TABLE test_default_database.test_distributed_table\\n(\\n `column` UInt8\\n)\\nENGINE = Distributed(\\'cluster\\', \\'test_default_database\\', \\'test_local_table\\')\n" + ) diff --git a/tests/integration/test_default_role/test.py b/tests/integration/test_default_role/test.py index 2f00fb603a8..1a321a8269a 100644 --- a/tests/integration/test_default_role/test.py +++ b/tests/integration/test_default_role/test.py @@ -3,7 +3,7 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -31,46 +31,64 @@ def test_set_default_roles(): assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("GRANT rx, ry TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( + [["rx", 0, 1], ["ry", 0, 1]] + ) instance.query("SET DEFAULT ROLE NONE TO john") assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("SET DEFAULT ROLE rx TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([["rx", 0, 1]]) instance.query("SET DEFAULT ROLE ry TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([["ry", 0, 1]]) instance.query("SET DEFAULT ROLE ALL TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( + [["rx", 0, 1], ["ry", 0, 1]] + ) instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([["ry", 0, 1]]) def test_alter_user(): assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("GRANT rx, ry TO john") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( + [["rx", 0, 1], ["ry", 0, 1]] + ) instance.query("ALTER USER john DEFAULT ROLE NONE") assert instance.query("SHOW CURRENT ROLES", user="john") == "" instance.query("ALTER USER john DEFAULT ROLE rx") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([["rx", 0, 1]]) instance.query("ALTER USER john DEFAULT ROLE ALL") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['rx', 0, 1], ['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( + [["rx", 0, 1], ["ry", 0, 1]] + ) instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx") - assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([['ry', 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV([["ry", 0, 1]]) def test_wrong_set_default_role(): - assert "There is no user `rx`" in instance.query_and_get_error("SET DEFAULT ROLE NONE TO rx") - assert "There is no user `ry`" in instance.query_and_get_error("SET DEFAULT ROLE rx TO ry") - assert "There is no role `john`" in instance.query_and_get_error("SET DEFAULT ROLE john TO john") - assert "There is no role `john`" in instance.query_and_get_error("ALTER USER john DEFAULT ROLE john") - assert "There is no role `john`" in instance.query_and_get_error("ALTER USER john DEFAULT ROLE ALL EXCEPT john") + assert "There is no user `rx`" in instance.query_and_get_error( + "SET DEFAULT ROLE NONE TO rx" + ) + assert "There is no user `ry`" in instance.query_and_get_error( + "SET DEFAULT ROLE rx TO ry" + ) + assert "There is no role `john`" in instance.query_and_get_error( + "SET DEFAULT ROLE john TO john" + ) + assert "There is no role `john`" in instance.query_and_get_error( + "ALTER USER john DEFAULT ROLE john" + ) + assert "There is no role `john`" in instance.query_and_get_error( + "ALTER USER john DEFAULT ROLE ALL EXCEPT john" + ) diff --git a/tests/integration/test_delayed_replica_failover/test.py b/tests/integration/test_delayed_replica_failover/test.py index 404848e155b..387d6a12f48 100644 --- a/tests/integration/test_delayed_replica_failover/test.py +++ b/tests/integration/test_delayed_replica_failover/test.py @@ -14,10 +14,11 @@ cluster = ClickHouseCluster(__file__) # Cluster with 2 shards of 2 replicas each. node_1_1 is the instance with Distributed table. # Thus we have a shard with a local replica and a shard with remote replicas. node_1_1 = instance_with_dist_table = cluster.add_instance( - 'node_1_1', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) -node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True) -node_2_1 = cluster.add_instance('node_2_1', with_zookeeper=True) -node_2_2 = cluster.add_instance('node_2_2', with_zookeeper=True) + "node_1_1", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) +node_1_2 = cluster.add_instance("node_1_2", with_zookeeper=True) +node_2_1 = cluster.add_instance("node_2_1", with_zookeeper=True) +node_2_2 = cluster.add_instance("node_2_2", with_zookeeper=True) @pytest.fixture(scope="module") @@ -27,15 +28,19 @@ def started_cluster(): for shard in (1, 2): for replica in (1, 2): - node = cluster.instances['node_{}_{}'.format(shard, replica)] - node.query(''' + node = cluster.instances["node_{}_{}".format(shard, replica)] + node.query( + """ CREATE TABLE replicated (d Date, x UInt32) ENGINE = - ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)''' - .format(shard=shard, instance=node.name)) + ReplicatedMergeTree('/clickhouse/tables/{shard}/replicated', '{instance}', d, d, 8192)""".format( + shard=shard, instance=node.name + ) + ) node_1_1.query( "CREATE TABLE distributed (d Date, x UInt32) ENGINE = " - "Distributed('test_cluster', 'default', 'replicated')") + "Distributed('test_cluster', 'default', 'replicated')" + ) yield cluster @@ -54,27 +59,41 @@ def test(started_cluster): time.sleep(1) # accrue replica delay - assert node_1_1.query("SELECT sum(x) FROM replicated").strip() == '0' - assert node_1_2.query("SELECT sum(x) FROM replicated").strip() == '1' - assert node_2_1.query("SELECT sum(x) FROM replicated").strip() == '0' - assert node_2_2.query("SELECT sum(x) FROM replicated").strip() == '2' + assert node_1_1.query("SELECT sum(x) FROM replicated").strip() == "0" + assert node_1_2.query("SELECT sum(x) FROM replicated").strip() == "1" + assert node_2_1.query("SELECT sum(x) FROM replicated").strip() == "0" + assert node_2_2.query("SELECT sum(x) FROM replicated").strip() == "2" # With in_order balancing first replicas are chosen. - assert instance_with_dist_table.query( - "SELECT count() FROM distributed SETTINGS load_balancing='in_order'").strip() == '0' + assert ( + instance_with_dist_table.query( + "SELECT count() FROM distributed SETTINGS load_balancing='in_order'" + ).strip() + == "0" + ) # When we set max_replica_delay, first replicas must be excluded. - assert instance_with_dist_table.query(''' + assert ( + instance_with_dist_table.query( + """ SELECT sum(x) FROM distributed SETTINGS load_balancing='in_order', max_replica_delay_for_distributed_queries=1 -''').strip() == '3' +""" + ).strip() + == "3" + ) - assert instance_with_dist_table.query(''' + assert ( + instance_with_dist_table.query( + """ SELECT sum(x) FROM distributed WITH TOTALS SETTINGS load_balancing='in_order', max_replica_delay_for_distributed_queries=1 -''').strip() == '3\n\n3' +""" + ).strip() + == "3\n\n3" + ) pm.drop_instance_zk_connections(node_1_2) pm.drop_instance_zk_connections(node_2_2) @@ -90,29 +109,41 @@ SELECT sum(x) FROM distributed WITH TOTALS SETTINGS raise Exception("Connection with zookeeper was not lost") # At this point all replicas are stale, but the query must still go to second replicas which are the least stale ones. - assert instance_with_dist_table.query(''' + assert ( + instance_with_dist_table.query( + """ SELECT sum(x) FROM distributed SETTINGS load_balancing='in_order', max_replica_delay_for_distributed_queries=1 -''').strip() == '3' +""" + ).strip() + == "3" + ) # Regression for skip_unavailable_shards in conjunction with skip_unavailable_shards - assert instance_with_dist_table.query(''' + assert ( + instance_with_dist_table.query( + """ SELECT sum(x) FROM distributed SETTINGS load_balancing='in_order', skip_unavailable_shards=1, max_replica_delay_for_distributed_queries=1 -''').strip() == '3' +""" + ).strip() + == "3" + ) # If we forbid stale replicas, the query must fail. But sometimes we must have bigger timeouts. for _ in range(20): try: - instance_with_dist_table.query(''' + instance_with_dist_table.query( + """ SELECT count() FROM distributed SETTINGS load_balancing='in_order', max_replica_delay_for_distributed_queries=1, fallback_to_stale_replicas_for_distributed_queries=0 -''') +""" + ) time.sleep(0.5) except: break @@ -122,8 +153,13 @@ SELECT count() FROM distributed SETTINGS # Now partition off the remote replica of the local shard and test that failover still works. pm.partition_instances(node_1_1, node_1_2, port=9000) - assert instance_with_dist_table.query(''' + assert ( + instance_with_dist_table.query( + """ SELECT sum(x) FROM distributed SETTINGS load_balancing='in_order', max_replica_delay_for_distributed_queries=1 -''').strip() == '2' +""" + ).strip() + == "2" + ) diff --git a/tests/integration/test_dictionaries_access/test.py b/tests/integration/test_dictionaries_access/test.py index 1b64b0de1fb..993c8259f32 100644 --- a/tests/integration/test_dictionaries_access/test.py +++ b/tests/integration/test_dictionaries_access/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -41,7 +41,9 @@ drop_query = "DROP DICTIONARY test_dict" def test_create(): assert instance.query("SHOW GRANTS FOR mira") == "" - assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + assert "Not enough privileges" in instance.query_and_get_error( + create_query, user="mira" + ) instance.query("GRANT CREATE DICTIONARY ON *.* TO mira") instance.query(create_query, user="mira") @@ -49,7 +51,9 @@ def test_create(): instance.query("REVOKE CREATE DICTIONARY ON *.* FROM mira") assert instance.query("SHOW GRANTS FOR mira") == "" - assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + assert "Not enough privileges" in instance.query_and_get_error( + create_query, user="mira" + ) instance.query("GRANT CREATE DICTIONARY ON default.* TO mira") instance.query(create_query, user="mira") @@ -57,7 +61,9 @@ def test_create(): instance.query("REVOKE CREATE DICTIONARY ON default.* FROM mira") assert instance.query("SHOW GRANTS FOR mira") == "" - assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + assert "Not enough privileges" in instance.query_and_get_error( + create_query, user="mira" + ) instance.query("GRANT CREATE DICTIONARY ON default.test_dict TO mira") instance.query(create_query, user="mira") @@ -67,7 +73,9 @@ def test_drop(): instance.query(create_query) assert instance.query("SHOW GRANTS FOR mira") == "" - assert "Not enough privileges" in instance.query_and_get_error(drop_query, user="mira") + assert "Not enough privileges" in instance.query_and_get_error( + drop_query, user="mira" + ) instance.query("GRANT DROP DICTIONARY ON *.* TO mira") instance.query(drop_query, user="mira") @@ -79,14 +87,18 @@ def test_dictget(): dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(5))" instance.query(dictget_query) == "6\n" - assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') + assert "Not enough privileges" in instance.query_and_get_error( + dictget_query, user="mira" + ) instance.query("GRANT dictGet ON default.test_dict TO mira") - instance.query(dictget_query, user='mira') == "6\n" + instance.query(dictget_query, user="mira") == "6\n" dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(1))" instance.query(dictget_query) == "0\n" - instance.query(dictget_query, user='mira') == "0\n" + instance.query(dictget_query, user="mira") == "0\n" instance.query("REVOKE dictGet ON *.* FROM mira") - assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') + assert "Not enough privileges" in instance.query_and_get_error( + dictget_query, user="mira" + ) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py index a3d0e8a019b..b38e81b0227 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/common.py @@ -4,51 +4,51 @@ import shutil from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout KEY_FIELDS = { - "simple": [ - Field("KeyField", 'UInt64', is_key=True, default_value_for_get=9999999) - ], + "simple": [Field("KeyField", "UInt64", is_key=True, default_value_for_get=9999999)], "complex": [ - Field("KeyField1", 'UInt64', is_key=True, default_value_for_get=9999999), - Field("KeyField2", 'String', is_key=True, default_value_for_get='xxxxxxxxx') + Field("KeyField1", "UInt64", is_key=True, default_value_for_get=9999999), + Field("KeyField2", "String", is_key=True, default_value_for_get="xxxxxxxxx"), ], "ranged": [ - Field("KeyField1", 'UInt64', is_key=True), - Field("KeyField2", 'Date', is_range_key=True) - ] + Field("KeyField1", "UInt64", is_key=True), + Field("KeyField2", "Date", is_range_key=True), + ], } START_FIELDS = { "simple": [], "complex": [], - "ranged" : [ - Field("StartDate", 'Date', range_hash_type='min'), - Field("EndDate", 'Date', range_hash_type='max') - ] + "ranged": [ + Field("StartDate", "Date", range_hash_type="min"), + Field("EndDate", "Date", range_hash_type="max"), + ], } MIDDLE_FIELDS = [ - Field("UInt8_", 'UInt8', default_value_for_get=55), - Field("UInt16_", 'UInt16', default_value_for_get=66), - Field("UInt32_", 'UInt32', default_value_for_get=77), - Field("UInt64_", 'UInt64', default_value_for_get=88), - Field("Int8_", 'Int8', default_value_for_get=-55), - Field("Int16_", 'Int16', default_value_for_get=-66), - Field("Int32_", 'Int32', default_value_for_get=-77), - Field("Int64_", 'Int64', default_value_for_get=-88), - Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'), - Field("Date_", 'Date', default_value_for_get='2018-12-30'), - Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'), - Field("String_", 'String', default_value_for_get='hi'), - Field("Float32_", 'Float32', default_value_for_get=555.11), - Field("Float64_", 'Float64', default_value_for_get=777.11), + Field("UInt8_", "UInt8", default_value_for_get=55), + Field("UInt16_", "UInt16", default_value_for_get=66), + Field("UInt32_", "UInt32", default_value_for_get=77), + Field("UInt64_", "UInt64", default_value_for_get=88), + Field("Int8_", "Int8", default_value_for_get=-55), + Field("Int16_", "Int16", default_value_for_get=-66), + Field("Int32_", "Int32", default_value_for_get=-77), + Field("Int64_", "Int64", default_value_for_get=-88), + Field( + "UUID_", "UUID", default_value_for_get="550e8400-0000-0000-0000-000000000000" + ), + Field("Date_", "Date", default_value_for_get="2018-12-30"), + Field("DateTime_", "DateTime", default_value_for_get="2018-12-30 00:00:00"), + Field("String_", "String", default_value_for_get="hi"), + Field("Float32_", "Float32", default_value_for_get=555.11), + Field("Float64_", "Float64", default_value_for_get=777.11), ] END_FIELDS = { - "simple" : [ + "simple": [ Field("ParentKeyField", "UInt64", default_value_for_get=444, hierarchical=True) ], - "complex" : [], - "ranged" : [] + "complex": [], + "ranged": [], } LAYOUTS_SIMPLE = ["flat", "hashed", "cache", "direct"] @@ -57,40 +57,129 @@ LAYOUTS_RANGED = ["range_hashed"] VALUES = { "simple": [ - [1, 22, 333, 4444, 55555, -6, -77, - -888, -999, '550e8400-e29b-41d4-a716-446655440003', - '1973-06-28', '1985-02-28 23:43:25', 'hello', 22.543, 3332154213.4, 0], - [2, 3, 4, 5, 6, -7, -8, - -9, -10, '550e8400-e29b-41d4-a716-446655440002', - '1978-06-28', '1986-02-28 23:42:25', 'hello', 21.543, 3222154213.4, 1] + [ + 1, + 22, + 333, + 4444, + 55555, + -6, + -77, + -888, + -999, + "550e8400-e29b-41d4-a716-446655440003", + "1973-06-28", + "1985-02-28 23:43:25", + "hello", + 22.543, + 3332154213.4, + 0, + ], + [ + 2, + 3, + 4, + 5, + 6, + -7, + -8, + -9, + -10, + "550e8400-e29b-41d4-a716-446655440002", + "1978-06-28", + "1986-02-28 23:42:25", + "hello", + 21.543, + 3222154213.4, + 1, + ], ], "complex": [ - [1, 'world', 22, 333, 4444, 55555, -6, - -77, -888, -999, '550e8400-e29b-41d4-a716-446655440003', - '1973-06-28', '1985-02-28 23:43:25', - 'hello', 22.543, 3332154213.4], - [2, 'qwerty2', 52, 2345, 6544, 9191991, -2, - -717, -81818, -92929, '550e8400-e29b-41d4-a716-446655440007', - '1975-09-28', '2000-02-28 23:33:24', - 'my', 255.543, 3332221.44] + [ + 1, + "world", + 22, + 333, + 4444, + 55555, + -6, + -77, + -888, + -999, + "550e8400-e29b-41d4-a716-446655440003", + "1973-06-28", + "1985-02-28 23:43:25", + "hello", + 22.543, + 3332154213.4, + ], + [ + 2, + "qwerty2", + 52, + 2345, + 6544, + 9191991, + -2, + -717, + -81818, + -92929, + "550e8400-e29b-41d4-a716-446655440007", + "1975-09-28", + "2000-02-28 23:33:24", + "my", + 255.543, + 3332221.44, + ], ], "ranged": [ - [1, '2019-02-10', '2019-02-01', '2019-02-28', - 22, 333, 4444, 55555, -6, -77, -888, -999, - '550e8400-e29b-41d4-a716-446655440003', - '1973-06-28', '1985-02-28 23:43:25', 'hello', - 22.543, 3332154213.4], - [2, '2019-04-10', '2019-04-01', '2019-04-28', - 11, 3223, 41444, 52515, -65, -747, -8388, -9099, - '550e8400-e29b-41d4-a716-446655440004', - '1973-06-29', '2002-02-28 23:23:25', '!!!!', - 32.543, 3332543.4] - ] + [ + 1, + "2019-02-10", + "2019-02-01", + "2019-02-28", + 22, + 333, + 4444, + 55555, + -6, + -77, + -888, + -999, + "550e8400-e29b-41d4-a716-446655440003", + "1973-06-28", + "1985-02-28 23:43:25", + "hello", + 22.543, + 3332154213.4, + ], + [ + 2, + "2019-04-10", + "2019-04-01", + "2019-04-28", + 11, + 3223, + 41444, + 52515, + -65, + -747, + -8388, + -9099, + "550e8400-e29b-41d4-a716-446655440004", + "1973-06-29", + "2002-02-28 23:23:25", + "!!!!", + 32.543, + 3332543.4, + ], + ], } SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -DICT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'configs', 'dictionaries') +DICT_CONFIG_PATH = os.path.join(SCRIPT_DIR, "configs", "dictionaries") + class BaseLayoutTester: def __init__(self, test_name): @@ -109,29 +198,39 @@ class BaseLayoutTester: directory = self.get_dict_directory() for fname in os.listdir(directory): dictionaries.append(os.path.join(directory, fname)) - return dictionaries + return dictionaries def create_dictionaries(self, source_): for layout in self.layouts: if source_.compatible_with_layout(Layout(layout)): - self.layout_to_dictionary[layout] = self.get_dict(source_, Layout(layout), self.fields) + self.layout_to_dictionary[layout] = self.get_dict( + source_, Layout(layout), self.fields + ) def prepare(self, cluster_): for _, dictionary in list(self.layout_to_dictionary.items()): dictionary.prepare_source(cluster_) dictionary.load_data(self.data) - def get_dict(self, source, layout, fields, suffix_name=''): + def get_dict(self, source, layout, fields, suffix_name=""): structure = DictionaryStructure(layout, fields) - dict_name = source.name + "_" + layout.name + '_' + suffix_name - dict_path = os.path.join(self.get_dict_directory(), dict_name + '.xml') - dictionary = Dictionary(dict_name, structure, source, dict_path, "table_" + dict_name, fields) + dict_name = source.name + "_" + layout.name + "_" + suffix_name + dict_path = os.path.join(self.get_dict_directory(), dict_name + ".xml") + dictionary = Dictionary( + dict_name, structure, source, dict_path, "table_" + dict_name, fields + ) dictionary.generate_config() return dictionary + class SimpleLayoutTester(BaseLayoutTester): def __init__(self, test_name): - self.fields = KEY_FIELDS["simple"] + START_FIELDS["simple"] + MIDDLE_FIELDS + END_FIELDS["simple"] + self.fields = ( + KEY_FIELDS["simple"] + + START_FIELDS["simple"] + + MIDDLE_FIELDS + + END_FIELDS["simple"] + ) self.values = VALUES["simple"] self.data = [Row(self.fields, vals) for vals in self.values] self.layout_to_dictionary = dict() @@ -151,13 +250,17 @@ class SimpleLayoutTester(BaseLayoutTester): for field in self.fields: if not field.is_key: for query in dct.get_select_get_queries(field, row): - queries_with_answers.append((query, row.get_value_by_name(field.name))) + queries_with_answers.append( + (query, row.get_value_by_name(field.name)) + ) for query in dct.get_select_has_queries(field, row): queries_with_answers.append((query, 1)) for query in dct.get_select_get_or_default_queries(field, row): - queries_with_answers.append((query, field.default_value_for_get)) + queries_with_answers.append( + (query, field.default_value_for_get) + ) for query in dct.get_hierarchical_queries(self.data[0]): queries_with_answers.append((query, [1])) @@ -174,13 +277,22 @@ class SimpleLayoutTester(BaseLayoutTester): for query, answer in queries_with_answers: # print query if isinstance(answer, list): - answer = str(answer).replace(' ', '') - assert node.query(query) == str(answer) + '\n' + answer = str(answer).replace(" ", "") + answer = str(answer) + "\n" + node_answer = node.query(query) + assert ( + str(node_answer).strip() == answer.strip() + ), f"Expected '{answer.strip()}', got '{node_answer.strip()}' in query '{query}'" class ComplexLayoutTester(BaseLayoutTester): def __init__(self, test_name): - self.fields = KEY_FIELDS["complex"] + START_FIELDS["complex"] + MIDDLE_FIELDS + END_FIELDS["complex"] + self.fields = ( + KEY_FIELDS["complex"] + + START_FIELDS["complex"] + + MIDDLE_FIELDS + + END_FIELDS["complex"] + ) self.values = VALUES["complex"] self.data = [Row(self.fields, vals) for vals in self.values] self.layout_to_dictionary = dict() @@ -200,22 +312,35 @@ class ComplexLayoutTester(BaseLayoutTester): for field in self.fields: if not field.is_key: for query in dct.get_select_get_queries(field, row): - queries_with_answers.append((query, row.get_value_by_name(field.name))) + queries_with_answers.append( + (query, row.get_value_by_name(field.name)) + ) for query in dct.get_select_has_queries(field, row): queries_with_answers.append((query, 1)) for query in dct.get_select_get_or_default_queries(field, row): - queries_with_answers.append((query, field.default_value_for_get)) + queries_with_answers.append( + (query, field.default_value_for_get) + ) for query, answer in queries_with_answers: # print query - assert node.query(query) == str(answer) + '\n' + node_answer = node.query(query) + answer = str(answer) + "\n" + assert ( + node_answer == answer + ), f"Expected '{answer.strip()}', got '{node_answer.strip()}' in query '{query}'" class RangedLayoutTester(BaseLayoutTester): def __init__(self, test_name): - self.fields = KEY_FIELDS["ranged"] + START_FIELDS["ranged"] + MIDDLE_FIELDS + END_FIELDS["ranged"] + self.fields = ( + KEY_FIELDS["ranged"] + + START_FIELDS["ranged"] + + MIDDLE_FIELDS + + END_FIELDS["ranged"] + ) self.values = VALUES["ranged"] self.data = [Row(self.fields, vals) for vals in self.values] self.layout_to_dictionary = dict() @@ -236,9 +361,14 @@ class RangedLayoutTester(BaseLayoutTester): for field in self.fields: if not field.is_key and not field.is_range: for query in dct.get_select_get_queries(field, row): - queries_with_answers.append((query, row.get_value_by_name(field.name))) + queries_with_answers.append( + (query, row.get_value_by_name(field.name)) + ) for query, answer in queries_with_answers: # print query - assert node.query(query) == str(answer) + '\n' - + node_answer = node.query(query) + answer = str(answer) + "\n" + assert ( + node_answer == answer + ), f"Expected '{answer.strip()}', got '{node_answer.strip()}' in query '{query}'" diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py index 78715bd17cf..aa1eb614dd5 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_cassandra.py @@ -16,6 +16,7 @@ complex_tester = None ranged_tester = None test_name = "cassandra" + def setup_module(module): global cluster global node @@ -25,7 +26,15 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) - SOURCE = SourceCassandra("Cassandra", None, cluster.cassandra_port, cluster.cassandra_host, cluster.cassandra_port, "", "") + SOURCE = SourceCassandra( + "Cassandra", + None, + cluster.cassandra_port, + cluster.cassandra_host, + cluster.cassandra_port, + "", + "", + ) simple_tester = SimpleLayoutTester(test_name) simple_tester.cleanup() @@ -39,16 +48,22 @@ def setup_module(module): # Since that all .xml configs were created main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - node = cluster.add_instance('cass_node', main_configs=main_configs, dictionaries=dictionaries, with_cassandra=True) + node = cluster.add_instance( + "cass_node", + main_configs=main_configs, + dictionaries=dictionaries, + with_cassandra=True, + ) def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -63,14 +78,17 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py index 051b4ff3086..b7f8226960f 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py @@ -1,4 +1,4 @@ -import os +import os import math import pytest @@ -8,7 +8,9 @@ from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceClickHouse -SOURCE = SourceClickHouse("LocalClickHouse", "localhost", "9000", "local_node", "9000", "default", "") +SOURCE = SourceClickHouse( + "LocalClickHouse", "localhost", "9000", "local_node", "9000", "default", "" +) cluster = None node = None @@ -17,6 +19,7 @@ complex_tester = None ranged_tester = None test_name = "local" + def setup_module(module): global cluster global node @@ -38,13 +41,15 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - - node = cluster.add_instance('local_node', main_configs=main_configs, dictionaries=dictionaries) - + node = cluster.add_instance( + "local_node", main_configs=main_configs, dictionaries=dictionaries + ) + + def teardown_module(module): simple_tester.cleanup() @@ -63,14 +68,17 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) - + + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py index 3ed335a1987..6790d11ed1a 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py @@ -8,7 +8,9 @@ from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceClickHouse -SOURCE = SourceClickHouse("RemoteClickHouse", "localhost", "9000", "clickhouse_remote", "9000", "default", "") +SOURCE = SourceClickHouse( + "RemoteClickHouse", "localhost", "9000", "clickhouse_remote", "9000", "default", "" +) cluster = None node = None @@ -17,6 +19,7 @@ complex_tester = None ranged_tester = None test_name = "remote" + def setup_module(module): global cluster global node @@ -38,13 +41,15 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - cluster.add_instance('clickhouse_remote', main_configs=main_configs) + cluster.add_instance("clickhouse_remote", main_configs=main_configs) - node = cluster.add_instance('remote_node', main_configs=main_configs, dictionaries=dictionaries) + node = cluster.add_instance( + "remote_node", main_configs=main_configs, dictionaries=dictionaries + ) def teardown_module(module): @@ -65,14 +70,22 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", sorted(list(set(LAYOUTS_SIMPLE).difference(set("cache"))) )) + +@pytest.mark.parametrize( + "layout_name", sorted(list(set(LAYOUTS_SIMPLE).difference(set("cache")))) +) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", sorted(list(set(LAYOUTS_COMPLEX).difference(set("complex_key_cache"))))) + +@pytest.mark.parametrize( + "layout_name", + sorted(list(set(LAYOUTS_COMPLEX).difference(set("complex_key_cache")))), +) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py index 5d694bc09a2..5186139ddf6 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_cache.py @@ -1,4 +1,4 @@ -import os +import os import math import pytest @@ -8,7 +8,9 @@ from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceExecutableCache -SOURCE = SourceExecutableCache("ExecutableCache", "localhost", "9000", "cache_node", "9000", "", "") +SOURCE = SourceExecutableCache( + "ExecutableCache", "localhost", "9000", "cache_node", "9000", "", "" +) cluster = None node = None @@ -17,6 +19,7 @@ complex_tester = None ranged_tester = None test_name = "cache" + def setup_module(module): global cluster global node @@ -38,16 +41,19 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - - node = cluster.add_instance('cache_node', main_configs=main_configs, dictionaries=dictionaries) - + node = cluster.add_instance( + "cache_node", main_configs=main_configs, dictionaries=dictionaries + ) + + def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -62,10 +68,12 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", ['cache']) + +@pytest.mark.parametrize("layout_name", ["cache"]) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", ['complex_key_cache']) + +@pytest.mark.parametrize("layout_name", ["complex_key_cache"]) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py index 8c0e6f8b878..63f4ff87cce 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py @@ -1,4 +1,4 @@ -import os +import os import math import pytest @@ -8,7 +8,9 @@ from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceExecutableHashed -SOURCE = SourceExecutableHashed("ExecutableHashed", "localhost", "9000", "hashed_node", "9000", "", "") +SOURCE = SourceExecutableHashed( + "ExecutableHashed", "localhost", "9000", "hashed_node", "9000", "", "" +) cluster = None node = None @@ -39,16 +41,19 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - - node = cluster.add_instance('hashed_node', main_configs=main_configs, dictionaries=dictionaries) - + node = cluster.add_instance( + "hashed_node", main_configs=main_configs, dictionaries=dictionaries + ) + + def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -63,14 +68,17 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", ['hashed']) + +@pytest.mark.parametrize("layout_name", ["hashed"]) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", ['complex_key_hashed']) + +@pytest.mark.parametrize("layout_name", ["complex_key_hashed"]) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py index 97a06fadc5e..0147b95c786 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_file.py @@ -17,6 +17,7 @@ complex_tester = None ranged_tester = None test_name = "file" + def setup_module(module): global cluster global node @@ -38,16 +39,19 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - node = cluster.add_instance('file_node', main_configs=main_configs, dictionaries=dictionaries) + node = cluster.add_instance( + "file_node", main_configs=main_configs, dictionaries=dictionaries + ) def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -62,14 +66,26 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("layout_name", sorted(set(LAYOUTS_SIMPLE).difference({'cache', 'direct'})) ) + +@pytest.mark.parametrize( + "layout_name", sorted(set(LAYOUTS_SIMPLE).difference({"cache", "direct"})) +) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) -@pytest.mark.parametrize("layout_name", sorted(list(set(LAYOUTS_COMPLEX).difference({'complex_key_cache', 'complex_key_direct'})))) + +@pytest.mark.parametrize( + "layout_name", + sorted( + list( + set(LAYOUTS_COMPLEX).difference({"complex_key_cache", "complex_key_direct"}) + ) + ), +) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py index c8c73011f61..96d17508880 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_http.py @@ -17,6 +17,7 @@ complex_tester = None ranged_tester = None test_name = "http" + def setup_module(module): global cluster global node @@ -38,18 +39,21 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - cluster.add_instance('clickhouse_h', main_configs=main_configs) + cluster.add_instance("clickhouse_h", main_configs=main_configs) - node = cluster.add_instance('http_node', main_configs=main_configs, dictionaries=dictionaries) + node = cluster.add_instance( + "http_node", main_configs=main_configs, dictionaries=dictionaries + ) def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -64,14 +68,17 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py index 42f33e3da3c..007e318e037 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_https.py @@ -8,7 +8,9 @@ from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceHTTPS -SOURCE = SourceHTTPS("SourceHTTPS", "localhost", "9000", "clickhouse_hs", "9000", "", "") +SOURCE = SourceHTTPS( + "SourceHTTPS", "localhost", "9000", "clickhouse_hs", "9000", "", "" +) cluster = None node = None @@ -17,6 +19,7 @@ complex_tester = None ranged_tester = None test_name = "https" + def setup_module(module): global cluster global node @@ -38,18 +41,21 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - cluster.add_instance('clickhouse_hs', main_configs=main_configs) + cluster.add_instance("clickhouse_hs", main_configs=main_configs) - node = cluster.add_instance('https_node', main_configs=main_configs, dictionaries=dictionaries) + node = cluster.add_instance( + "https_node", main_configs=main_configs, dictionaries=dictionaries + ) def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -64,14 +70,17 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py index deaaf044bce..4a9d054b08f 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py @@ -16,6 +16,7 @@ complex_tester = None ranged_tester = None test_name = "mongo" + def setup_module(module): global cluster global node @@ -24,7 +25,15 @@ def setup_module(module): global ranged_tester cluster = ClickHouseCluster(__file__, name=test_name) - SOURCE = SourceMongo("MongoDB", "localhost", cluster.mongo_port, cluster.mongo_host, "27017", "root", "clickhouse") + SOURCE = SourceMongo( + "MongoDB", + "localhost", + cluster.mongo_port, + cluster.mongo_host, + "27017", + "root", + "clickhouse", + ) simple_tester = SimpleLayoutTester(test_name) simple_tester.cleanup() @@ -38,16 +47,19 @@ def setup_module(module): # Since that all .xml configs were created main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True) + node = cluster.add_instance( + "node", main_configs=main_configs, dictionaries=dictionaries, with_mongo=True + ) def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -62,14 +74,17 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py index fdf4826cb63..c6551e0eb70 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -1,4 +1,4 @@ -import os +import os import math import pytest @@ -26,7 +26,15 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) - SOURCE = SourceMongoURI("MongoDB", "localhost", cluster.mongo_port, cluster.mongo_host, "27017", "root", "clickhouse") + SOURCE = SourceMongoURI( + "MongoDB", + "localhost", + cluster.mongo_port, + cluster.mongo_host, + "27017", + "root", + "clickhouse", + ) simple_tester = SimpleLayoutTester(test_name) simple_tester.cleanup() @@ -40,16 +48,22 @@ def setup_module(module): # Since that all .xml configs were created main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) - + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) + dictionaries = simple_tester.list_dictionaries() - node = cluster.add_instance('uri_node', main_configs=main_configs, dictionaries=dictionaries, with_mongo=True) + node = cluster.add_instance( + "uri_node", + main_configs=main_configs, + dictionaries=dictionaries, + with_mongo=True, + ) + - def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -64,6 +78,7 @@ def started_cluster(): finally: cluster.shutdown() + # See comment in SourceMongoURI @pytest.mark.parametrize("layout_name", ["flat"]) def test_simple(started_cluster, layout_name): diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py index 7cd7460b8cb..96757c58e0c 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mysql.py @@ -26,7 +26,15 @@ def setup_module(module): cluster = ClickHouseCluster(__file__, name=test_name) - SOURCE = SourceMySQL("MySQL", None, cluster.mysql_port, cluster.mysql_host, cluster.mysql_port, "root", "clickhouse") + SOURCE = SourceMySQL( + "MySQL", + None, + cluster.mysql_port, + cluster.mysql_host, + cluster.mysql_port, + "root", + "clickhouse", + ) simple_tester = SimpleLayoutTester(test_name) simple_tester.cleanup() @@ -40,21 +48,24 @@ def setup_module(module): # Since that all .xml configs were created main_configs = [] - main_configs.append(os.path.join('configs', 'disable_ssl_verification.xml')) + main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) dictionaries = simple_tester.list_dictionaries() - - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_mysql=True) + + node = cluster.add_instance( + "node", main_configs=main_configs, dictionaries=dictionaries, with_mysql=True + ) def teardown_module(module): simple_tester.cleanup() + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - + simple_tester.prepare(cluster) complex_tester.prepare(cluster) ranged_tester.prepare(cluster) @@ -64,14 +75,17 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) def test_simple(started_cluster, layout_name): simple_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) def test_complex(started_cluster, layout_name): complex_tester.execute(layout_name, node) + @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) def test_ranged(started_cluster, layout_name): ranged_tester.execute(layout_name, node) diff --git a/tests/integration/test_dictionaries_complex_key_cache_string/test.py b/tests/integration/test_dictionaries_complex_key_cache_string/test.py index f5a1be7daf4..ae9cd4e7c91 100644 --- a/tests/integration/test_dictionaries_complex_key_cache_string/test.py +++ b/tests/integration/test_dictionaries_complex_key_cache_string/test.py @@ -5,54 +5,92 @@ from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node_memory = cluster.add_instance('node_memory', dictionaries=['configs/dictionaries/complex_key_cache_string.xml']) -node_ssd = cluster.add_instance('node_ssd', dictionaries=['configs/dictionaries/ssd_complex_key_cache_string.xml']) +node_memory = cluster.add_instance( + "node_memory", dictionaries=["configs/dictionaries/complex_key_cache_string.xml"] +) +node_ssd = cluster.add_instance( + "node_ssd", dictionaries=["configs/dictionaries/ssd_complex_key_cache_string.xml"] +) + @pytest.fixture() def started_cluster(): try: cluster.start() node_memory.query( - "create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id") + "create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id" + ) node_ssd.query( - "create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id") + "create table radars_table (radar_id String, radar_ip String, client_id String) engine=MergeTree() order by radar_id" + ) yield cluster finally: cluster.shutdown() + @pytest.mark.skip(reason="SSD cache test can run on disk only") @pytest.mark.parametrize("type", ["memory", "ssd"]) def test_memory_consumption(started_cluster, type): - node = started_cluster.instances[f'node_{type}'] + node = started_cluster.instances[f"node_{type}"] node.query( - "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('w' * 8)) + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format( + "w" * 8 + ) + ) node.query( - "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('x' * 16)) + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format( + "x" * 16 + ) + ) node.query( - "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('y' * 32)) + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format( + "y" * 32 + ) + ) node.query( - "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format('z' * 64)) + "insert into radars_table select toString(rand() % 5000), '{0}', '{0}' from numbers(1000)".format( + "z" * 64 + ) + ) # Fill dictionary - node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)") + node.query( + "select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)" + ) - allocated_first = int(node.query("select bytes_allocated from system.dictionaries where name = 'radars'").strip()) + allocated_first = int( + node.query( + "select bytes_allocated from system.dictionaries where name = 'radars'" + ).strip() + ) alloc_array = [] for i in range(5): - node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)") + node.query( + "select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)" + ) - allocated = int(node.query("select bytes_allocated from system.dictionaries where name = 'radars'").strip()) + allocated = int( + node.query( + "select bytes_allocated from system.dictionaries where name = 'radars'" + ).strip() + ) alloc_array.append(allocated) # size doesn't grow assert all(allocated_first >= a for a in alloc_array) for i in range(5): - node.query("select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)") + node.query( + "select dictGetString('radars', 'client_id', tuple(toString(number))) from numbers(0, 5000)" + ) - allocated = int(node.query("select bytes_allocated from system.dictionaries where name = 'radars'").strip()) + allocated = int( + node.query( + "select bytes_allocated from system.dictionaries where name = 'radars'" + ).strip() + ) alloc_array.append(allocated) # size doesn't grow diff --git a/tests/integration/test_dictionaries_config_reload/test.py b/tests/integration/test_dictionaries_config_reload/test.py index 4f338767304..7be179f854b 100644 --- a/tests/integration/test_dictionaries_config_reload/test.py +++ b/tests/integration/test_dictionaries_config_reload/test.py @@ -10,16 +10,22 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=['config/dictionaries_config.xml']) +node = cluster.add_instance( + "node", stay_alive=True, main_configs=["config/dictionaries_config.xml"] +) def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) -config = ''' +config = """ /etc/clickhouse-server/dictionaries/{dictionaries_config} -''' +""" @pytest.fixture(scope="module") @@ -27,9 +33,15 @@ def started_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'dictionaries/.'), '/etc/clickhouse-server/dictionaries', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "dictionaries/."), + "/etc/clickhouse-server/dictionaries", + node.docker_id, + ) - node.query("CREATE TABLE dictionary_values (id UInt64, value_1 String, value_2 String) ENGINE=TinyLog;") + node.query( + "CREATE TABLE dictionary_values (id UInt64, value_1 String, value_2 String) ENGINE=TinyLog;" + ) node.query("INSERT INTO dictionary_values VALUES (0, 'Value_1', 'Value_2')") node.restart_clickhouse() @@ -41,7 +53,10 @@ def started_cluster(): def change_config(dictionaries_config): - node.replace_config("/etc/clickhouse-server/config.d/dictionaries_config.xml", config.format(dictionaries_config=dictionaries_config)) + node.replace_config( + "/etc/clickhouse-server/config.d/dictionaries_config.xml", + config.format(dictionaries_config=dictionaries_config), + ) node.query("SYSTEM RELOAD CONFIG;") @@ -51,7 +66,10 @@ def test(started_cluster): time.sleep(10) - assert node.query("SELECT dictGet('test_dictionary_1', 'value_1', toUInt64(0));") == 'Value_1\n' + assert ( + node.query("SELECT dictGet('test_dictionary_1', 'value_1', toUInt64(0));") + == "Value_1\n" + ) # Change path to the second dictionary in config. change_config("dictionary_config2.xml") @@ -59,7 +77,12 @@ def test(started_cluster): time.sleep(10) # Check that the new dictionary is loaded. - assert node.query("SELECT dictGet('test_dictionary_2', 'value_2', toUInt64(0));") == 'Value_2\n' + assert ( + node.query("SELECT dictGet('test_dictionary_2', 'value_2', toUInt64(0));") + == "Value_2\n" + ) # Check that the previous dictionary was unloaded. - node.query_and_get_error("SELECT dictGet('test_dictionary_1', 'value', toUInt64(0));") + node.query_and_get_error( + "SELECT dictGet('test_dictionary_1', 'value', toUInt64(0));" + ) diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 6e4d9958fac..cb70deef72b 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -11,25 +11,43 @@ from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], - user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) -node2 = cluster.add_instance('node2', with_mysql=True, dictionaries=['configs/dictionaries/simple_dictionary.xml'], - main_configs=['configs/dictionaries/lazy_load.xml', 'configs/allow_remote_node.xml'], - user_configs=['configs/user_admin.xml', 'configs/user_default.xml']) -node3 = cluster.add_instance('node3', main_configs=['configs/allow_remote_node.xml'], - dictionaries=['configs/dictionaries/dictionary_with_conflict_name.xml', - 'configs/dictionaries/conflict_name_dictionary.xml'], - user_configs=['configs/user_admin.xml']) -node4 = cluster.add_instance('node4', user_configs=['configs/user_admin.xml', 'configs/config_password.xml']) +node1 = cluster.add_instance( + "node1", + with_mysql=True, + dictionaries=["configs/dictionaries/simple_dictionary.xml"], + user_configs=["configs/user_admin.xml", "configs/user_default.xml"], +) +node2 = cluster.add_instance( + "node2", + with_mysql=True, + dictionaries=["configs/dictionaries/simple_dictionary.xml"], + main_configs=[ + "configs/dictionaries/lazy_load.xml", + "configs/allow_remote_node.xml", + ], + user_configs=["configs/user_admin.xml", "configs/user_default.xml"], +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/allow_remote_node.xml"], + dictionaries=[ + "configs/dictionaries/dictionary_with_conflict_name.xml", + "configs/dictionaries/conflict_name_dictionary.xml", + ], + user_configs=["configs/user_admin.xml"], +) +node4 = cluster.add_instance( + "node4", user_configs=["configs/user_admin.xml", "configs/config_password.xml"] +) def create_mysql_conn(user, password, hostname, port): - logging.debug("Created MySQL connection user:{}, password:{}, host:{}, port{}".format(user, password, hostname, port)) - return pymysql.connect( - user=user, - password=password, - host=hostname, - port=port) + logging.debug( + "Created MySQL connection user:{}, password:{}, host:{}, port{}".format( + user, password, hostname, port + ) + ) + return pymysql.connect(user=user, password=password, host=hostname, port=port) def execute_mysql_query(connection, query): @@ -49,35 +67,71 @@ def started_cluster(): clickhouse.query("CREATE DATABASE test", user="admin") clickhouse.query( "CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id", - user="admin") + user="admin", + ) clickhouse.query( "INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)", - user="admin") + user="admin", + ) yield cluster finally: cluster.shutdown() -@pytest.mark.parametrize("clickhouse,name,layout", [ - pytest.param(node1, 'complex_node1_hashed', 'LAYOUT(COMPLEX_KEY_HASHED())', id="complex_node1_hashed"), - pytest.param(node1, 'complex_node1_cache', 'LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))', id="complex_node1_cache"), - pytest.param(node2, 'complex_node2_hashed', 'LAYOUT(COMPLEX_KEY_HASHED())', id="complex_node2_hashed"), - pytest.param(node2, 'complex_node2_cache', 'LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))', id="complex_node2_cache"), -]) +@pytest.mark.parametrize( + "clickhouse,name,layout", + [ + pytest.param( + node1, + "complex_node1_hashed", + "LAYOUT(COMPLEX_KEY_HASHED())", + id="complex_node1_hashed", + ), + pytest.param( + node1, + "complex_node1_cache", + "LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))", + id="complex_node1_cache", + ), + pytest.param( + node2, + "complex_node2_hashed", + "LAYOUT(COMPLEX_KEY_HASHED())", + id="complex_node2_hashed", + ), + pytest.param( + node2, + "complex_node2_cache", + "LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10))", + id="complex_node2_cache", + ), + ], +) def test_create_and_select_mysql(started_cluster, clickhouse, name, layout): - mysql_conn = create_mysql_conn("root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port) + mysql_conn = create_mysql_conn( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) execute_mysql_query(mysql_conn, "DROP DATABASE IF EXISTS create_and_select") execute_mysql_query(mysql_conn, "CREATE DATABASE create_and_select") - execute_mysql_query(mysql_conn, - "CREATE TABLE create_and_select.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format( - name)) + execute_mysql_query( + mysql_conn, + "CREATE TABLE create_and_select.{} (key_field1 int, key_field2 bigint, value1 text, value2 float, PRIMARY KEY (key_field1, key_field2))".format( + name + ), + ) values = [] for i in range(1000): - values.append('(' + ','.join([str(i), str(i * i), str(i) * 5, str(i * 3.14)]) + ')') - execute_mysql_query(mysql_conn, "INSERT INTO create_and_select.{} VALUES ".format(name) + ','.join(values)) + values.append( + "(" + ",".join([str(i), str(i * i), str(i) * 5, str(i * 3.14)]) + ")" + ) + execute_mysql_query( + mysql_conn, + "INSERT INTO create_and_select.{} VALUES ".format(name) + ",".join(values), + ) - clickhouse.query(""" + clickhouse.query( + """ CREATE DICTIONARY default.{} ( key_field1 Int32, key_field2 Int64, @@ -95,48 +149,76 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout): )) {} LIFETIME(MIN 1 MAX 3) - """.format(name, name, layout)) + """.format( + name, name, layout + ) + ) for i in range(172, 200): - assert clickhouse.query( - "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, - i * i)) == str( - i) * 5 + '\n' + assert ( + clickhouse.query( + "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format( + name, i, i * i + ) + ) + == str(i) * 5 + "\n" + ) stroka = clickhouse.query( - "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, - i * i)).strip() + "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format( + name, i, i * i + ) + ).strip() value = float(stroka) assert int(value) == int(i * 3.14) for i in range(1000): - values.append('(' + ','.join([str(i), str(i * i), str(i) * 3, str(i * 2.718)]) + ')') - execute_mysql_query(mysql_conn, "REPLACE INTO create_and_select.{} VALUES ".format(name) + ','.join(values)) + values.append( + "(" + ",".join([str(i), str(i * i), str(i) * 3, str(i * 2.718)]) + ")" + ) + execute_mysql_query( + mysql_conn, + "REPLACE INTO create_and_select.{} VALUES ".format(name) + ",".join(values), + ) clickhouse.query("SYSTEM RELOAD DICTIONARY 'default.{}'".format(name)) for i in range(172, 200): - assert clickhouse.query( - "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format(name, i, - i * i)) == str( - i) * 3 + '\n' + assert ( + clickhouse.query( + "SELECT dictGetString('default.{}', 'value1', tuple(toInt32({}), toInt64({})))".format( + name, i, i * i + ) + ) + == str(i) * 3 + "\n" + ) string = clickhouse.query( - "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format(name, i, - i * i)).strip() + "SELECT dictGetFloat32('default.{}', 'value2', tuple(toInt32({}), toInt64({})))".format( + name, i, i * i + ) + ).strip() value = float(string) assert int(value) == int(i * 2.718) - clickhouse.query("select dictGetUInt8('xml_dictionary', 'SomeValue1', toUInt64(17))") == "17\n" - clickhouse.query("select dictGetString('xml_dictionary', 'SomeValue2', toUInt64(977))") == str(hex(977))[2:] + '\n' + clickhouse.query( + "select dictGetUInt8('xml_dictionary', 'SomeValue1', toUInt64(17))" + ) == "17\n" + clickhouse.query( + "select dictGetString('xml_dictionary', 'SomeValue2', toUInt64(977))" + ) == str(hex(977))[2:] + "\n" clickhouse.query(f"drop dictionary default.{name}") def test_restricted_database(started_cluster): for node in [node1, node2]: node.query("CREATE DATABASE IF NOT EXISTS restricted_db", user="admin") - node.query("CREATE TABLE restricted_db.table_in_restricted_db AS test.xml_dictionary_table", user="admin") + node.query( + "CREATE TABLE restricted_db.table_in_restricted_db AS test.xml_dictionary_table", + user="admin", + ) with pytest.raises(QueryRuntimeException): - node1.query(""" + node1.query( + """ CREATE DICTIONARY restricted_db.some_dict( id UInt64, SomeValue1 UInt8, @@ -146,10 +228,12 @@ def test_restricted_database(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_in_restricted_db' DB 'restricted_db')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) with pytest.raises(QueryRuntimeException): - node1.query(""" + node1.query( + """ CREATE DICTIONARY default.some_dict( id UInt64, SomeValue1 UInt8, @@ -159,13 +243,17 @@ def test_restricted_database(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_in_restricted_db' DB 'restricted_db')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) - node1.query("SELECT dictGetUInt8('default.some_dict', 'SomeValue1', toUInt64(17))") == "17\n" + node1.query( + "SELECT dictGetUInt8('default.some_dict', 'SomeValue1', toUInt64(17))" + ) == "17\n" # with lazy load we don't need query to get exception with pytest.raises(QueryRuntimeException): - node2.query(""" + node2.query( + """ CREATE DICTIONARY restricted_db.some_dict( id UInt64, SomeValue1 UInt8, @@ -175,10 +263,12 @@ def test_restricted_database(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_in_restricted_db' DB 'restricted_db')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) with pytest.raises(QueryRuntimeException): - node2.query(""" + node2.query( + """ CREATE DICTIONARY default.some_dict( id UInt64, SomeValue1 UInt8, @@ -188,17 +278,24 @@ def test_restricted_database(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_in_restricted_db' DB 'restricted_db')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) for node in [node1, node2]: node.query("DROP DICTIONARY IF EXISTS default.some_dict", user="admin") node.query("DROP DATABASE restricted_db", user="admin") def test_conflicting_name(started_cluster): - assert node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n' + assert ( + node3.query( + "select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))" + ) + == "17\n" + ) with pytest.raises(QueryRuntimeException): - node3.query(""" + node3.query( + """ CREATE DICTIONARY test.conflicting_dictionary( id UInt64, SomeValue1 UInt8, @@ -208,15 +305,19 @@ def test_conflicting_name(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'xml_dictionary_table' DB 'test')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) # old version still works - node3.query("select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))") == '17\n' + node3.query( + "select dictGetUInt8('test.conflicting_dictionary', 'SomeValue1', toUInt64(17))" + ) == "17\n" def test_http_dictionary_restrictions(started_cluster): try: - node3.query(""" + node3.query( + """ CREATE DICTIONARY test.restricted_http_dictionary ( id UInt64, value String @@ -225,16 +326,20 @@ def test_http_dictionary_restrictions(started_cluster): LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://somehost.net' FORMAT TabSeparated)) LIFETIME(1) - """) - node3.query("SELECT dictGetString('test.restricted_http_dictionary', 'value', toUInt64(1))") + """ + ) + node3.query( + "SELECT dictGetString('test.restricted_http_dictionary', 'value', toUInt64(1))" + ) except QueryRuntimeException as ex: - assert 'is not allowed in configuration file' in str(ex) + assert "is not allowed in configuration file" in str(ex) node3.query("DROP DICTIONARY test.restricted_http_dictionary") def test_file_dictionary_restrictions(started_cluster): try: - node3.query(""" + node3.query( + """ CREATE DICTIONARY test.restricted_file_dictionary ( id UInt64, value String @@ -243,21 +348,34 @@ def test_file_dictionary_restrictions(started_cluster): LAYOUT(FLAT()) SOURCE(FILE(PATH '/usr/bin/cat' FORMAT TabSeparated)) LIFETIME(1) - """) - node3.query("SELECT dictGetString('test.restricted_file_dictionary', 'value', toUInt64(1))") + """ + ) + node3.query( + "SELECT dictGetString('test.restricted_file_dictionary', 'value', toUInt64(1))" + ) except QueryRuntimeException as ex: - assert 'is not inside' in str(ex) + assert "is not inside" in str(ex) node3.query("DROP DICTIONARY test.restricted_file_dictionary") def test_dictionary_with_where(started_cluster): - mysql_conn = create_mysql_conn("root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port) - execute_mysql_query(mysql_conn, "CREATE DATABASE IF NOT EXISTS dictionary_with_where") - execute_mysql_query(mysql_conn, - "CREATE TABLE dictionary_with_where.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))") - execute_mysql_query(mysql_conn, "INSERT INTO dictionary_with_where.special_table VALUES (1, 'abcabc'), (2, 'qweqwe')") + mysql_conn = create_mysql_conn( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + execute_mysql_query( + mysql_conn, "CREATE DATABASE IF NOT EXISTS dictionary_with_where" + ) + execute_mysql_query( + mysql_conn, + "CREATE TABLE dictionary_with_where.special_table (key_field1 int, value1 text, PRIMARY KEY (key_field1))", + ) + execute_mysql_query( + mysql_conn, + "INSERT INTO dictionary_with_where.special_table VALUES (1, 'abcabc'), (2, 'qweqwe')", + ) - node1.query(""" + node1.query( + """ CREATE DICTIONARY default.special_dict ( key_field1 Int32, value1 String DEFAULT 'xxx' @@ -273,11 +391,17 @@ def test_dictionary_with_where(started_cluster): )) LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 3) - """) + """ + ) node1.query("SYSTEM RELOAD DICTIONARY default.special_dict") - assert node1.query("SELECT dictGetString('default.special_dict', 'value1', toUInt64(2))") == 'qweqwe\n' + assert ( + node1.query( + "SELECT dictGetString('default.special_dict', 'value1', toUInt64(2))" + ) + == "qweqwe\n" + ) node1.query("DROP DICTIONARY default.special_dict") execute_mysql_query(mysql_conn, "DROP TABLE dictionary_with_where.special_table") execute_mysql_query(mysql_conn, "DROP DATABASE dictionary_with_where") @@ -285,7 +409,8 @@ def test_dictionary_with_where(started_cluster): def test_clickhouse_remote(started_cluster): with pytest.raises(QueryRuntimeException): - node3.query(""" + node3.query( + """ CREATE DICTIONARY test.clickhouse_remote( id UInt64, SomeValue1 UInt8, @@ -295,7 +420,8 @@ def test_clickhouse_remote(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'node4' PORT 9000 USER 'default' TABLE 'xml_dictionary_table' DB 'test')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) for i in range(5): node3.query("system reload dictionary test.clickhouse_remote") time.sleep(0.5) @@ -303,7 +429,8 @@ def test_clickhouse_remote(started_cluster): node3.query("detach dictionary if exists test.clickhouse_remote") with pytest.raises(QueryRuntimeException): - node3.query(""" + node3.query( + """ CREATE DICTIONARY test.clickhouse_remote( id UInt64, SomeValue1 UInt8, @@ -313,12 +440,14 @@ def test_clickhouse_remote(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'node4' PORT 9000 USER 'default' PASSWORD 'default' TABLE 'xml_dictionary_table' DB 'test')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) node3.query("attach dictionary test.clickhouse_remote") node3.query("drop dictionary test.clickhouse_remote") - node3.query(""" + node3.query( + """ CREATE DICTIONARY test.clickhouse_remote( id UInt64, SomeValue1 UInt8, @@ -328,6 +457,9 @@ def test_clickhouse_remote(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'node4' PORT 9000 USER 'default' PASSWORD 'default' TABLE 'xml_dictionary_table' DB 'test')) LIFETIME(MIN 1 MAX 10) - """) + """ + ) - node3.query("select dictGetUInt8('test.clickhouse_remote', 'SomeValue1', toUInt64(17))") == '17\n' + node3.query( + "select dictGetUInt8('test.clickhouse_remote', 'SomeValue1', toUInt64(17))" + ) == "17\n" diff --git a/tests/integration/test_dictionaries_dependency/test.py b/tests/integration/test_dictionaries_dependency/test.py index 7dc7f84d50b..f57d4e42813 100644 --- a/tests/integration/test_dictionaries_dependency/test.py +++ b/tests/integration/test_dictionaries_dependency/test.py @@ -2,8 +2,10 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', stay_alive=True) -node2 = cluster.add_instance('node2', stay_alive=True, main_configs=['configs/disable_lazy_load.xml']) +node1 = cluster.add_instance("node1", stay_alive=True) +node2 = cluster.add_instance( + "node2", stay_alive=True, main_configs=["configs/disable_lazy_load.xml"] +) nodes = [node1, node2] @@ -21,9 +23,11 @@ def start_cluster(): node.query("INSERT INTO test.source VALUES (5,6)") for db in ("test", "test_ordinary"): - node.query("CREATE DICTIONARY {}.dict(x UInt64, y UInt64) PRIMARY KEY x " \ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " \ - "LAYOUT(FLAT()) LIFETIME(0)".format(db)) + node.query( + "CREATE DICTIONARY {}.dict(x UInt64, y UInt64) PRIMARY KEY x " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'source' DB 'test')) " + "LAYOUT(FLAT()) LIFETIME(0)".format(db) + ) yield cluster finally: @@ -53,13 +57,18 @@ def cleanup_after_test(): def test_dependency_via_implicit_table(node): d_names = ["test.adict", "test.zdict", "atest.dict", "ztest.dict"] for d_name in d_names: - node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) " \ - "LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) + node.query( + "CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'dict' DB 'test')) " + "LAYOUT(FLAT()) LIFETIME(0)".format(d_name) + ) def check(): for d_name in d_names: - assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" + assert ( + node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) + == "6\n" + ) check() @@ -74,16 +83,25 @@ def test_dependency_via_explicit_table(node): d_names = ["test.other_{}".format(i) for i in range(0, len(tbl_names))] for i in range(0, len(tbl_names)): tbl_name = tbl_names[i] - tbl_database, tbl_shortname = tbl_name.split('.') + tbl_database, tbl_shortname = tbl_name.split(".") d_name = d_names[i] - node.query("CREATE TABLE {}(x UInt64, y UInt64) ENGINE=Dictionary('test.dict')".format(tbl_name)) - node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) " \ - "LAYOUT(FLAT()) LIFETIME(0)".format(d_name, tbl_shortname, tbl_database)) + node.query( + "CREATE TABLE {}(x UInt64, y UInt64) ENGINE=Dictionary('test.dict')".format( + tbl_name + ) + ) + node.query( + "CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE '{}' DB '{}')) " + "LAYOUT(FLAT()) LIFETIME(0)".format(d_name, tbl_shortname, tbl_database) + ) def check(): for d_name in d_names: - assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" + assert ( + node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) + == "6\n" + ) check() @@ -95,30 +113,40 @@ def test_dependency_via_explicit_table(node): for tbl in tbl_names: node.query(f"DROP TABLE {tbl}") + @pytest.mark.parametrize("node", nodes) def test_dependency_via_dictionary_database(node): node.query("CREATE DATABASE dict_db ENGINE=Dictionary") d_names = ["test_ordinary.adict", "test_ordinary.zdict", "atest.dict", "ztest.dict"] for d_name in d_names: - node.query("CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " \ - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_ordinary.dict' DB 'dict_db')) " \ - "LAYOUT(FLAT()) LIFETIME(0)".format(d_name)) + node.query( + "CREATE DICTIONARY {}(x UInt64, y UInt64) PRIMARY KEY x " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_ordinary.dict' DB 'dict_db')) " + "LAYOUT(FLAT()) LIFETIME(0)".format(d_name) + ) def check(): for d_name in d_names: - assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" - + assert ( + node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) + == "6\n" + ) for d_name in d_names: - assert node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" + assert ( + node.query("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" + ) # Restart must not break anything. node.restart_clickhouse() for d_name in d_names: - assert node.query_with_retry("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) == "6\n" + assert ( + node.query_with_retry("SELECT dictGet({}, 'y', toUInt64(5))".format(d_name)) + == "6\n" + ) - # cleanup + # cleanup for d_name in d_names: node.query(f"DROP DICTIONARY IF EXISTS {d_name} SYNC") node.query("DROP DATABASE dict_db SYNC") diff --git a/tests/integration/test_dictionaries_dependency_xml/test.py b/tests/integration/test_dictionaries_dependency_xml/test.py index 13635c7b969..3f4c3320920 100644 --- a/tests/integration/test_dictionaries_dependency_xml/test.py +++ b/tests/integration/test_dictionaries_dependency_xml/test.py @@ -2,11 +2,17 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry -DICTIONARY_FILES = ['configs/dictionaries/dep_x.xml', 'configs/dictionaries/dep_y.xml', - 'configs/dictionaries/dep_z.xml', 'configs/dictionaries/node.xml'] +DICTIONARY_FILES = [ + "configs/dictionaries/dep_x.xml", + "configs/dictionaries/dep_y.xml", + "configs/dictionaries/dep_z.xml", + "configs/dictionaries/node.xml", +] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', dictionaries=DICTIONARY_FILES, stay_alive=True) +instance = cluster.add_instance( + "instance", dictionaries=DICTIONARY_FILES, stay_alive=True +) @pytest.fixture(scope="module") @@ -14,13 +20,15 @@ def started_cluster(): try: cluster.start() - instance.query(''' + instance.query( + """ CREATE DATABASE IF NOT EXISTS dict ENGINE=Dictionary; CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.elements; CREATE TABLE test.elements (id UInt64, a String, b Int32, c Float64) ENGINE=Log; INSERT INTO test.elements VALUES (0, 'water', 10, 1), (1, 'air', 40, 0.01), (2, 'earth', 100, 1.7); - ''') + """ + ) yield cluster @@ -29,23 +37,25 @@ def started_cluster(): def get_status(dictionary_name): - return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + return instance.query( + "SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'" + ).rstrip("\n") def test_get_data(started_cluster): query = instance.query # dictionaries_lazy_load == false, so these dictionary are not loaded. - assert get_status('dep_x') == 'NOT_LOADED' - assert get_status('dep_y') == 'NOT_LOADED' - assert get_status('dep_z') == 'NOT_LOADED' + assert get_status("dep_x") == "NOT_LOADED" + assert get_status("dep_y") == "NOT_LOADED" + assert get_status("dep_z") == "NOT_LOADED" # Dictionary 'dep_x' depends on 'dep_z', which depends on 'dep_y'. # So they all should be loaded at once. assert query("SELECT dictGetString('dep_x', 'a', toUInt64(1))") == "air\n" - assert get_status('dep_x') == 'LOADED' - assert get_status('dep_y') == 'LOADED' - assert get_status('dep_z') == 'LOADED' + assert get_status("dep_x") == "LOADED" + assert get_status("dep_y") == "LOADED" + assert get_status("dep_z") == "LOADED" # Other dictionaries should work too. assert query("SELECT dictGetString('dep_y', 'a', toUInt64(1))") == "air\n" @@ -59,7 +69,13 @@ def test_get_data(started_cluster): query("INSERT INTO test.elements VALUES (3, 'fire', 30, 8)") # Wait for dictionaries to be reloaded. - assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(3))", "1", sleep_time=2, retry_count=10) + assert_eq_with_retry( + instance, + "SELECT dictHas('dep_x', toUInt64(3))", + "1", + sleep_time=2, + retry_count=10, + ) assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" @@ -67,7 +83,13 @@ def test_get_data(started_cluster): # dep_z (and hence dep_x) are updated only when there `intDiv(count(), 4)` is changed, now `count()==4`, # so dep_x and dep_z are not going to be updated after the following INSERT. query("INSERT INTO test.elements VALUES (4, 'ether', 404, 0.001)") - assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(4))", "1", sleep_time=2, retry_count=10) + assert_eq_with_retry( + instance, + "SELECT dictHas('dep_y', toUInt64(4))", + "1", + sleep_time=2, + retry_count=10, + ) assert query("SELECT dictGetString('dep_x', 'a', toUInt64(4))") == "XX\n" assert query("SELECT dictGetString('dep_y', 'a', toUInt64(4))") == "ether\n" assert query("SELECT dictGetString('dep_z', 'a', toUInt64(4))") == "ZZ\n" @@ -83,28 +105,41 @@ def dependent_tables_assert(): assert "default.join" in res assert "a.t" in res + def test_dependent_tables(started_cluster): query = instance.query query("create database lazy engine=Lazy(10)") query("create database a") query("create table lazy.src (n int, m int) engine=Log") - query("create dictionary a.d (n int default 0, m int default 42) primary key n " - "source(clickhouse(host 'localhost' port tcpPort() user 'default' table 'src' password '' db 'lazy'))" - "lifetime(min 1 max 10) layout(flat())") + query( + "create dictionary a.d (n int default 0, m int default 42) primary key n " + "source(clickhouse(host 'localhost' port tcpPort() user 'default' table 'src' password '' db 'lazy'))" + "lifetime(min 1 max 10) layout(flat())" + ) query("create table system.join (n int, m int) engine=Join(any, left, n)") query("insert into system.join values (1, 1)") - query("create table src (n int, m default joinGet('system.join', 'm', 1::int)," - "t default dictGetOrNull('a.d', 'm', toUInt64(3))," - "k default dictGet('a.d', 'm', toUInt64(4))) engine=MergeTree order by n") - query("create dictionary test.d (n int default 0, m int default 42) primary key n " - "source(clickhouse(host 'localhost' port tcpPort() user 'default' table 'src' password '' db 'default'))" - "lifetime(min 1 max 10) layout(flat())") - query("create table join (n int, m default dictGet('a.d', 'm', toUInt64(3))," - "k default dictGet('test.d', 'm', toUInt64(0))) engine=Join(any, left, n)") - query("create table lazy.log (n default dictGet(test.d, 'm', toUInt64(0))) engine=Log") - query("create table a.t (n default joinGet('system.join', 'm', 1::int)," - "m default dictGet('test.d', 'm', toUInt64(3))," - "k default joinGet(join, 'm', 1::int)) engine=MergeTree order by n") + query( + "create table src (n int, m default joinGet('system.join', 'm', 1::int)," + "t default dictGetOrNull('a.d', 'm', toUInt64(3))," + "k default dictGet('a.d', 'm', toUInt64(4))) engine=MergeTree order by n" + ) + query( + "create dictionary test.d (n int default 0, m int default 42) primary key n " + "source(clickhouse(host 'localhost' port tcpPort() user 'default' table 'src' password '' db 'default'))" + "lifetime(min 1 max 10) layout(flat())" + ) + query( + "create table join (n int, m default dictGet('a.d', 'm', toUInt64(3))," + "k default dictGet('test.d', 'm', toUInt64(0))) engine=Join(any, left, n)" + ) + query( + "create table lazy.log (n default dictGet(test.d, 'm', toUInt64(0))) engine=Log" + ) + query( + "create table a.t (n default joinGet('system.join', 'm', 1::int)," + "m default dictGet('test.d', 'm', toUInt64(3))," + "k default joinGet(join, 'm', 1::int)) engine=MergeTree order by n" + ) dependent_tables_assert() instance.restart_clickhouse() @@ -120,7 +155,9 @@ def test_dependent_tables(started_cluster): def test_xml_dict_same_name(started_cluster): - instance.query("create table default.node ( key UInt64, name String ) Engine=Dictionary(node);") + instance.query( + "create table default.node ( key UInt64, name String ) Engine=Dictionary(node);" + ) instance.restart_clickhouse() assert "node" in instance.query("show tables from default") instance.query("drop table default.node") diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index 664fde2baa8..5c67a4c434a 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -6,10 +6,12 @@ from helpers.cluster import ClickHouseCluster import time import logging -DICTS = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml'] -CONFIG_FILES = ['configs/remote_servers.xml', 'configs/named_collections.xml'] +DICTS = ["configs/dictionaries/mysql_dict1.xml", "configs/dictionaries/mysql_dict2.xml"] +CONFIG_FILES = ["configs/remote_servers.xml", "configs/named_collections.xml"] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=CONFIG_FILES, with_mysql=True, dictionaries=DICTS) +instance = cluster.add_instance( + "instance", main_configs=CONFIG_FILES, with_mysql=True, dictionaries=DICTS +) create_table_mysql_template = """ CREATE TABLE IF NOT EXISTS `test`.`{}` ( @@ -32,14 +34,16 @@ def started_cluster(): # Create a MySQL database mysql_connection = get_mysql_conn(cluster) - create_mysql_db(mysql_connection, 'test') + create_mysql_db(mysql_connection, "test") mysql_connection.close() # Create database in ClickHouse instance.query("CREATE DATABASE IF NOT EXISTS test") # Create database in ClickChouse using MySQL protocol (will be used for data insertion) - instance.query("CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql57:3306', 'test', 'root', 'clickhouse')") + instance.query( + "CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql57:3306', 'test', 'root', 'clickhouse')" + ) yield cluster @@ -50,13 +54,24 @@ def started_cluster(): def test_mysql_dictionaries_custom_query_full_load(started_cluster): mysql_connection = get_mysql_conn(started_cluster) - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table_1 (id Integer, value_1 Text);") - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table_2 (id Integer, value_2 Text);") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table_1 VALUES (1, 'Value_1');") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table_2 VALUES (1, 'Value_2');") + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table_1 (id Integer, value_1 Text);", + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table_2 (id Integer, value_2 Text);", + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table_1 VALUES (1, 'Value_1');" + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table_2 VALUES (1, 'Value_2');" + ) query = instance.query - query(""" + query( + """ CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -72,11 +87,12 @@ def test_mysql_dictionaries_custom_query_full_load(started_cluster): PASSWORD 'clickhouse' QUERY $doc$SELECT id, value_1, value_2 FROM test.test_table_1 INNER JOIN test.test_table_2 USING (id);$doc$)) LIFETIME(0) - """) + """ + ) result = query("SELECT id, value_1, value_2 FROM test_dictionary_custom_query") - assert result == '1\tValue_1\tValue_2\n' + assert result == "1\tValue_1\tValue_2\n" query("DROP DICTIONARY test_dictionary_custom_query;") @@ -87,13 +103,24 @@ def test_mysql_dictionaries_custom_query_full_load(started_cluster): def test_mysql_dictionaries_custom_query_partial_load_simple_key(started_cluster): mysql_connection = get_mysql_conn(started_cluster) - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table_1 (id Integer, value_1 Text);") - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table_2 (id Integer, value_2 Text);") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table_1 VALUES (1, 'Value_1');") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table_2 VALUES (1, 'Value_2');") + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table_1 (id Integer, value_1 Text);", + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table_2 (id Integer, value_2 Text);", + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table_1 VALUES (1, 'Value_1');" + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table_2 VALUES (1, 'Value_2');" + ) query = instance.query - query(""" + query( + """ CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -108,9 +135,12 @@ def test_mysql_dictionaries_custom_query_partial_load_simple_key(started_cluster USER 'root' PASSWORD 'clickhouse' QUERY $doc$SELECT id, value_1, value_2 FROM test.test_table_1 INNER JOIN test.test_table_2 USING (id) WHERE {condition};$doc$)) - """) + """ + ) - result = query("SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), toUInt64(1))") + result = query( + "SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), toUInt64(1))" + ) assert result == "('Value_1','Value_2')\n" @@ -123,13 +153,24 @@ def test_mysql_dictionaries_custom_query_partial_load_simple_key(started_cluster def test_mysql_dictionaries_custom_query_partial_load_complex_key(started_cluster): mysql_connection = get_mysql_conn(started_cluster) - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table_1 (id Integer, id_key Text, value_1 Text);") - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table_2 (id Integer, id_key Text, value_2 Text);") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table_1 VALUES (1, 'Key', 'Value_1');") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table_2 VALUES (1, 'Key', 'Value_2');") + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table_1 (id Integer, id_key Text, value_1 Text);", + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table_2 (id Integer, id_key Text, value_2 Text);", + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table_1 VALUES (1, 'Key', 'Value_1');" + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table_2 VALUES (1, 'Key', 'Value_2');" + ) query = instance.query - query(""" + query( + """ CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -145,9 +186,12 @@ def test_mysql_dictionaries_custom_query_partial_load_complex_key(started_cluste USER 'root' PASSWORD 'clickhouse' QUERY $doc$SELECT id, id_key, value_1, value_2 FROM test.test_table_1 INNER JOIN test.test_table_2 USING (id, id_key) WHERE {condition};$doc$)) - """) + """ + ) - result = query("SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), (toUInt64(1), 'Key'))") + result = query( + "SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), (toUInt64(1), 'Key'))" + ) assert result == "('Value_1','Value_2')\n" @@ -161,82 +205,109 @@ def test_predefined_connection_configuration(started_cluster): mysql_connection = get_mysql_conn(started_cluster) execute_mysql_query(mysql_connection, "DROP TABLE IF EXISTS test.test_table") - execute_mysql_query(mysql_connection, "CREATE TABLE IF NOT EXISTS test.test_table (id Integer, value Integer);") - execute_mysql_query(mysql_connection, "INSERT INTO test.test_table VALUES (100, 200);") + execute_mysql_query( + mysql_connection, + "CREATE TABLE IF NOT EXISTS test.test_table (id Integer, value Integer);", + ) + execute_mysql_query( + mysql_connection, "INSERT INTO test.test_table VALUES (100, 200);" + ) - instance.query(''' + instance.query( + """ DROP DICTIONARY IF EXISTS dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME mysql1)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = instance.query("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") - assert(int(result) == 200) + assert int(result) == 200 - instance.query(''' + instance.query( + """ DROP DICTIONARY dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME mysql2)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') - result = instance.query_and_get_error("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") - instance.query(''' + """ + ) + result = instance.query_and_get_error( + "SELECT dictGetUInt32(dict, 'value', toUInt64(100))" + ) + instance.query( + """ DROP DICTIONARY dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME unknown_collection)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') - result = instance.query_and_get_error("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") + """ + ) + result = instance.query_and_get_error( + "SELECT dictGetUInt32(dict, 'value', toUInt64(100))" + ) - instance.query(''' + instance.query( + """ DROP DICTIONARY dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME mysql3 PORT 3306)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = instance.query("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") - assert(int(result) == 200) + assert int(result) == 200 - instance.query(''' + instance.query( + """ DROP DICTIONARY IF EXISTS dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME mysql1 connection_pool_size 0)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') - result = instance.query_and_get_error("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") - assert 'Connection pool cannot have zero size' in result + """ + ) + result = instance.query_and_get_error( + "SELECT dictGetUInt32(dict, 'value', toUInt64(100))" + ) + assert "Connection pool cannot have zero size" in result - instance.query(''' + instance.query( + """ DROP DICTIONARY IF EXISTS dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME mysql4)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') - result = instance.query_and_get_error("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") - assert 'Connection pool cannot have zero size' in result + """ + ) + result = instance.query_and_get_error( + "SELECT dictGetUInt32(dict, 'value', toUInt64(100))" + ) + assert "Connection pool cannot have zero size" in result - instance.query(''' + instance.query( + """ DROP DICTIONARY IF EXISTS dict; CREATE DICTIONARY dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(MYSQL(NAME mysql4 connection_pool_size 1)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = instance.query("SELECT dictGetUInt32(dict, 'value', toUInt64(100))") - assert(int(result) == 200) + assert int(result) == 200 def create_mysql_db(mysql_connection, name): @@ -255,12 +326,24 @@ def prepare_mysql_table(started_cluster, table_name, index): query = instance.query query( "INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format( - table_name + str(index), table_name + str(index))) - assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index))).rstrip() == '10000' + table_name + str(index), table_name + str(index) + ) + ) + assert ( + query( + "SELECT count() FROM `clickhouse_mysql`.{}".format(table_name + str(index)) + ).rstrip() + == "10000" + ) mysql_connection.close() # Create CH Dictionary tables based on MySQL tables - query(create_clickhouse_dictionary_table_template.format(table_name + str(index), 'dict' + str(index))) + query( + create_clickhouse_dictionary_table_template.format( + table_name + str(index), "dict" + str(index) + ) + ) + def get_mysql_conn(started_cluster): errors = [] @@ -268,10 +351,17 @@ def get_mysql_conn(started_cluster): for _ in range(5): try: if conn is None: - conn = pymysql.connect(user='root', password='clickhouse', host=started_cluster.mysql_ip, port=started_cluster.mysql_port) + conn = pymysql.connect( + user="root", + password="clickhouse", + host=started_cluster.mysql_ip, + port=started_cluster.mysql_port, + ) else: conn.ping(reconnect=True) - logging.debug(f"MySQL Connection establised: {started_cluster.mysql_ip}:{started_cluster.mysql_port}") + logging.debug( + f"MySQL Connection establised: {started_cluster.mysql_ip}:{started_cluster.mysql_port}" + ) return conn except Exception as e: errors += [str(e)] @@ -279,6 +369,7 @@ def get_mysql_conn(started_cluster): raise Exception("Connection not establised, {}".format(errors)) + def execute_mysql_query(connection, query): logging.debug("Execute MySQL query:{}".format(query)) with warnings.catch_warnings(): @@ -287,6 +378,7 @@ def execute_mysql_query(connection, query): cursor.execute(query) connection.commit() + def create_mysql_table(conn, table_name): with conn.cursor() as cursor: cursor.execute(create_table_mysql_template.format(table_name)) diff --git a/tests/integration/test_dictionaries_null_value/test.py b/tests/integration/test_dictionaries_null_value/test.py index 96ca76f594e..d62b1e6fc49 100644 --- a/tests/integration/test_dictionaries_null_value/test.py +++ b/tests/integration/test_dictionaries_null_value/test.py @@ -1,10 +1,10 @@ import pytest from helpers.cluster import ClickHouseCluster -DICTIONARY_FILES = ['configs/dictionaries/cache.xml'] +DICTIONARY_FILES = ["configs/dictionaries/cache.xml"] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', dictionaries=DICTIONARY_FILES) +instance = cluster.add_instance("instance", dictionaries=DICTIONARY_FILES) @pytest.fixture(scope="module") @@ -12,7 +12,8 @@ def started_cluster(): try: cluster.start() - instance.query(''' + instance.query( + """ CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.source; CREATE TABLE test.source (id UInt64, key0 UInt8, key0_str String, key1 UInt8, @@ -22,7 +23,8 @@ def started_cluster(): Float32_ Float32, Float64_ Float64, String_ String, Date_ Date, DateTime_ DateTime, Parent UInt64) ENGINE=Log; - ''') + """ + ) yield cluster @@ -34,10 +36,22 @@ def test_null_value(started_cluster): query = instance.query assert query("select dictGetUInt8('cache', 'UInt8_', toUInt64(12121212))") == "1\n" - assert query("select dictGetString('cache', 'String_', toUInt64(12121212))") == "implicit-default\n" - assert query("select dictGetDate('cache', 'Date_', toUInt64(12121212))") == "2015-11-25\n" + assert ( + query("select dictGetString('cache', 'String_', toUInt64(12121212))") + == "implicit-default\n" + ) + assert ( + query("select dictGetDate('cache', 'Date_', toUInt64(12121212))") + == "2015-11-25\n" + ) # Check, that empty null_value interprets as default value - assert query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n" - assert query( - "select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')") == "1970-01-01 00:00:00\n" + assert ( + query("select dictGetUInt64('cache', 'UInt64_', toUInt64(12121212))") == "0\n" + ) + assert ( + query( + "select toTimeZone(dictGetDateTime('cache', 'DateTime_', toUInt64(12121212)), 'UTC')" + ) + == "1970-01-01 00:00:00\n" + ) diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 53333fe2012..49a75a09e4e 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -7,9 +7,16 @@ from helpers.cluster import ClickHouseCluster from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/config.xml', 'configs/dictionaries/postgres_dict.xml', 'configs/named_collections.xml'], - with_postgres=True, with_postgres_cluster=True) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/config.xml", + "configs/dictionaries/postgres_dict.xml", + "configs/named_collections.xml", + ], + with_postgres=True, + with_postgres_cluster=True, +) postgres_dict_table_template = """ CREATE TABLE IF NOT EXISTS {} ( @@ -21,35 +28,52 @@ click_dict_table_template = """ ) ENGINE = Dictionary({}) """ + def get_postgres_conn(ip, port, database=False): if database == True: - conn_string = "host={} port={} dbname='clickhouse' user='postgres' password='mysecretpassword'".format(ip, port) + conn_string = "host={} port={} dbname='clickhouse' user='postgres' password='mysecretpassword'".format( + ip, port + ) else: - conn_string = "host={} port={} user='postgres' password='mysecretpassword'".format(ip, port) + conn_string = ( + "host={} port={} user='postgres' password='mysecretpassword'".format( + ip, port + ) + ) conn = psycopg2.connect(conn_string) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) conn.autocommit = True return conn + def create_postgres_db(conn, name): cursor = conn.cursor() cursor.execute("CREATE DATABASE {}".format(name)) + def create_postgres_table(cursor, table_name): cursor.execute(postgres_dict_table_template.format(table_name)) + def create_and_fill_postgres_table(cursor, table_name, port, host): create_postgres_table(cursor, table_name) # Fill postgres table using clickhouse postgres table function and check - table_func = '''postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(host, port, table_name) - node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number, number from numbers(10000) - '''.format(table_func, table_name)) + table_func = """postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')""".format( + host, port, table_name + ) + node1.query( + """INSERT INTO TABLE FUNCTION {} SELECT number, number, number from numbers(10000) + """.format( + table_func, table_name + ) + ) result = node1.query("SELECT count() FROM {}".format(table_func)) - assert result.rstrip() == '10000' + assert result.rstrip() == "10000" + def create_dict(table_name, index=0): - node1.query(click_dict_table_template.format(table_name, 'dict' + str(index))) + node1.query(click_dict_table_template.format(table_name, "dict" + str(index))) @pytest.fixture(scope="module") @@ -58,13 +82,17 @@ def started_cluster(): cluster.start() node1.query("CREATE DATABASE IF NOT EXISTS test") - postgres_conn = get_postgres_conn(ip=cluster.postgres_ip, port=cluster.postgres_port) + postgres_conn = get_postgres_conn( + ip=cluster.postgres_ip, port=cluster.postgres_port + ) print("postgres1 connected") - create_postgres_db(postgres_conn, 'clickhouse') + create_postgres_db(postgres_conn, "clickhouse") - postgres_conn = get_postgres_conn(ip=cluster.postgres2_ip, port=cluster.postgres_port) + postgres_conn = get_postgres_conn( + ip=cluster.postgres2_ip, port=cluster.postgres_port + ) print("postgres2 connected") - create_postgres_db(postgres_conn, 'clickhouse') + create_postgres_db(postgres_conn, "clickhouse") yield cluster @@ -73,17 +101,39 @@ def started_cluster(): def test_load_dictionaries(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, database=True, port=started_cluster.postgres_port) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + database=True, + port=started_cluster.postgres_port, + ) cursor = conn.cursor() - table_name = 'test0' - create_and_fill_postgres_table(cursor, table_name, port=started_cluster.postgres_port, host=started_cluster.postgres_ip) + table_name = "test0" + create_and_fill_postgres_table( + cursor, + table_name, + port=started_cluster.postgres_port, + host=started_cluster.postgres_ip, + ) create_dict(table_name) - dict_name = 'dict0' + dict_name = "dict0" node1.query("SYSTEM RELOAD DICTIONARY {}".format(dict_name)) - assert node1.query("SELECT count() FROM `test`.`dict_table_{}`".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT dictGetUInt32('{}', 'key', toUInt64(0))".format(dict_name)) == '0\n' - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name)) == '9999\n' + assert ( + node1.query( + "SELECT count() FROM `test`.`dict_table_{}`".format(table_name) + ).rstrip() + == "10000" + ) + assert ( + node1.query("SELECT dictGetUInt32('{}', 'key', toUInt64(0))".format(dict_name)) + == "0\n" + ) + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name) + ) + == "9999\n" + ) cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) node1.query("DROP TABLE IF EXISTS {}".format(table_name)) @@ -91,16 +141,25 @@ def test_load_dictionaries(started_cluster): def test_postgres_dictionaries_custom_query_full_load(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, database=True, port=started_cluster.postgres_port) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + database=True, + port=started_cluster.postgres_port, + ) cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS test_table_1 (id Integer, value_1 Text);") - cursor.execute("CREATE TABLE IF NOT EXISTS test_table_2 (id Integer, value_2 Text);") + cursor.execute( + "CREATE TABLE IF NOT EXISTS test_table_1 (id Integer, value_1 Text);" + ) + cursor.execute( + "CREATE TABLE IF NOT EXISTS test_table_2 (id Integer, value_2 Text);" + ) cursor.execute("INSERT INTO test_table_1 VALUES (1, 'Value_1');") cursor.execute("INSERT INTO test_table_2 VALUES (1, 'Value_2');") query = node1.query - query(""" + query( + """ CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -117,11 +176,14 @@ def test_postgres_dictionaries_custom_query_full_load(started_cluster): PASSWORD 'mysecretpassword' QUERY $doc$SELECT id, value_1, value_2 FROM test_table_1 INNER JOIN test_table_2 USING (id);$doc$)) LIFETIME(0) - """.format(started_cluster.postgres_ip, started_cluster.postgres_port)) + """.format( + started_cluster.postgres_ip, started_cluster.postgres_port + ) + ) result = query("SELECT id, value_1, value_2 FROM test_dictionary_custom_query") - assert result == '1\tValue_1\tValue_2\n' + assert result == "1\tValue_1\tValue_2\n" query("DROP DICTIONARY test_dictionary_custom_query;") @@ -130,16 +192,25 @@ def test_postgres_dictionaries_custom_query_full_load(started_cluster): def test_postgres_dictionaries_custom_query_partial_load_simple_key(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, database=True, port=started_cluster.postgres_port) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + database=True, + port=started_cluster.postgres_port, + ) cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS test_table_1 (id Integer, value_1 Text);") - cursor.execute("CREATE TABLE IF NOT EXISTS test_table_2 (id Integer, value_2 Text);") + cursor.execute( + "CREATE TABLE IF NOT EXISTS test_table_1 (id Integer, value_1 Text);" + ) + cursor.execute( + "CREATE TABLE IF NOT EXISTS test_table_2 (id Integer, value_2 Text);" + ) cursor.execute("INSERT INTO test_table_1 VALUES (1, 'Value_1');") cursor.execute("INSERT INTO test_table_2 VALUES (1, 'Value_2');") query = node1.query - query(""" + query( + """ CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -155,11 +226,16 @@ def test_postgres_dictionaries_custom_query_partial_load_simple_key(started_clus USER 'postgres' PASSWORD 'mysecretpassword' QUERY $doc$SELECT id, value_1, value_2 FROM test_table_1 INNER JOIN test_table_2 USING (id) WHERE {{condition}};$doc$)) - """.format(started_cluster.postgres_ip, started_cluster.postgres_port)) + """.format( + started_cluster.postgres_ip, started_cluster.postgres_port + ) + ) - result = query("SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), toUInt64(1))") + result = query( + "SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), toUInt64(1))" + ) - assert result == '(\'Value_1\',\'Value_2\')\n' + assert result == "('Value_1','Value_2')\n" query("DROP DICTIONARY test_dictionary_custom_query;") @@ -168,16 +244,25 @@ def test_postgres_dictionaries_custom_query_partial_load_simple_key(started_clus def test_postgres_dictionaries_custom_query_partial_load_complex_key(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, database=True, port=started_cluster.postgres_port) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + database=True, + port=started_cluster.postgres_port, + ) cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS test_table_1 (id Integer, key Text, value_1 Text);") - cursor.execute("CREATE TABLE IF NOT EXISTS test_table_2 (id Integer, key Text, value_2 Text);") + cursor.execute( + "CREATE TABLE IF NOT EXISTS test_table_1 (id Integer, key Text, value_1 Text);" + ) + cursor.execute( + "CREATE TABLE IF NOT EXISTS test_table_2 (id Integer, key Text, value_2 Text);" + ) cursor.execute("INSERT INTO test_table_1 VALUES (1, 'Key', 'Value_1');") cursor.execute("INSERT INTO test_table_2 VALUES (1, 'Key', 'Value_2');") query = node1.query - query(""" + query( + """ CREATE DICTIONARY test_dictionary_custom_query ( id UInt64, @@ -194,49 +279,101 @@ def test_postgres_dictionaries_custom_query_partial_load_complex_key(started_clu USER 'postgres' PASSWORD 'mysecretpassword' QUERY $doc$SELECT id, key, value_1, value_2 FROM test_table_1 INNER JOIN test_table_2 USING (id, key) WHERE {{condition}};$doc$)) - """.format(started_cluster.postgres_ip, started_cluster.postgres_port)) + """.format( + started_cluster.postgres_ip, started_cluster.postgres_port + ) + ) - result = query("SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), (toUInt64(1), 'Key'))") + result = query( + "SELECT dictGet('test_dictionary_custom_query', ('value_1', 'value_2'), (toUInt64(1), 'Key'))" + ) - assert result == '(\'Value_1\',\'Value_2\')\n' + assert result == "('Value_1','Value_2')\n" query("DROP DICTIONARY test_dictionary_custom_query;") cursor.execute("DROP TABLE test_table_2;") cursor.execute("DROP TABLE test_table_1;") + def test_invalidate_query(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, database=True, port=started_cluster.postgres_port) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + database=True, + port=started_cluster.postgres_port, + ) cursor = conn.cursor() - table_name = 'test0' - create_and_fill_postgres_table(cursor, table_name, port=started_cluster.postgres_port, host=started_cluster.postgres_ip) + table_name = "test0" + create_and_fill_postgres_table( + cursor, + table_name, + port=started_cluster.postgres_port, + host=started_cluster.postgres_ip, + ) # invalidate query: SELECT value FROM test0 WHERE id = 0 - dict_name = 'dict0' + dict_name = "dict0" create_dict(table_name) node1.query("SYSTEM RELOAD DICTIONARY {}".format(dict_name)) - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) == "0\n" - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name)) == "1\n" + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + ) + == "0\n" + ) + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name) + ) + == "1\n" + ) # update should happen cursor.execute("UPDATE {} SET value=value+1 WHERE id = 0".format(table_name)) while True: - result = node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) - if result != '0\n': + result = node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + ) + if result != "0\n": break - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) == '1\n' + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + ) + == "1\n" + ) # no update should happen cursor.execute("UPDATE {} SET value=value*2 WHERE id != 0".format(table_name)) time.sleep(5) - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) == '1\n' - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name)) == '1\n' + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + ) + == "1\n" + ) + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name) + ) + == "1\n" + ) # update should happen cursor.execute("UPDATE {} SET value=value+1 WHERE id = 0".format(table_name)) time.sleep(5) - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) == '2\n' - assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name)) == '2\n' + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name) + ) + == "2\n" + ) + assert ( + node1.query( + "SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name) + ) + == "2\n" + ) node1.query("DROP TABLE IF EXISTS {}".format(table_name)) node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) @@ -244,27 +381,39 @@ def test_invalidate_query(started_cluster): def test_dictionary_with_replicas(started_cluster): - conn1 = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, database=True) + conn1 = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor1 = conn1.cursor() - conn2 = get_postgres_conn(ip=started_cluster.postgres2_ip, port=started_cluster.postgres_port, database=True) + conn2 = get_postgres_conn( + ip=started_cluster.postgres2_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor2 = conn2.cursor() - create_postgres_table(cursor1, 'test1') - create_postgres_table(cursor2, 'test1') + create_postgres_table(cursor1, "test1") + create_postgres_table(cursor2, "test1") - cursor1.execute('INSERT INTO test1 select i, i, i from generate_series(0, 99) as t(i);') - cursor2.execute('INSERT INTO test1 select i, i, i from generate_series(100, 199) as t(i);') + cursor1.execute( + "INSERT INTO test1 select i, i, i from generate_series(0, 99) as t(i);" + ) + cursor2.execute( + "INSERT INTO test1 select i, i, i from generate_series(100, 199) as t(i);" + ) - create_dict('test1', 1) + create_dict("test1", 1) result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY key") # priority 0 - non running port - assert node1.contains_in_log('PostgreSQLConnectionPool: Connection error*') + assert node1.contains_in_log("PostgreSQLConnectionPool: Connection error*") # priority 1 - postgres2, table contains rows with values 100-200 # priority 2 - postgres1, table contains rows with values 0-100 expected = node1.query("SELECT number, number FROM numbers(100, 100)") - assert(result == expected) + assert result == expected cursor1.execute("DROP TABLE IF EXISTS test1") cursor2.execute("DROP TABLE IF EXISTS test1") @@ -274,14 +423,21 @@ def test_dictionary_with_replicas(started_cluster): def test_postgres_schema(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - cursor.execute('CREATE SCHEMA test_schema') - cursor.execute('CREATE TABLE test_schema.test_table (id integer, value integer)') - cursor.execute('INSERT INTO test_schema.test_table SELECT i, i FROM generate_series(0, 99) as t(i)') + cursor.execute("CREATE SCHEMA test_schema") + cursor.execute("CREATE TABLE test_schema.test_table (id integer, value integer)") + cursor.execute( + "INSERT INTO test_schema.test_table SELECT i, i FROM generate_series(0, 99) as t(i)" + ) - node1.query(''' + node1.query( + """ DROP DICTIONARY IF EXISTS postgres_dict; CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id @@ -294,88 +450,114 @@ def test_postgres_schema(started_cluster): table 'test_schema.test_table')) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(1))") - assert(int(result.strip()) == 1) + assert int(result.strip()) == 1 result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") - assert(int(result.strip()) == 99) + assert int(result.strip()) == 99 node1.query("DROP DICTIONARY IF EXISTS postgres_dict") def test_predefined_connection_configuration(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - cursor.execute('DROP TABLE IF EXISTS test_table') - cursor.execute('CREATE TABLE test_table (id integer, value integer)') - cursor.execute('INSERT INTO test_table SELECT i, i FROM generate_series(0, 99) as t(i)') + cursor.execute("DROP TABLE IF EXISTS test_table") + cursor.execute("CREATE TABLE test_table (id integer, value integer)") + cursor.execute( + "INSERT INTO test_table SELECT i, i FROM generate_series(0, 99) as t(i)" + ) - node1.query(''' + node1.query( + """ DROP DICTIONARY IF EXISTS postgres_dict; CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(POSTGRESQL(NAME postgres1)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") - assert(int(result.strip()) == 99) + assert int(result.strip()) == 99 - cursor.execute('DROP SCHEMA IF EXISTS test_schema CASCADE') - cursor.execute('CREATE SCHEMA test_schema') - cursor.execute('CREATE TABLE test_schema.test_table (id integer, value integer)') - cursor.execute('INSERT INTO test_schema.test_table SELECT i, 100 FROM generate_series(0, 99) as t(i)') + cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") + cursor.execute("CREATE SCHEMA test_schema") + cursor.execute("CREATE TABLE test_schema.test_table (id integer, value integer)") + cursor.execute( + "INSERT INTO test_schema.test_table SELECT i, 100 FROM generate_series(0, 99) as t(i)" + ) - node1.query(''' + node1.query( + """ DROP DICTIONARY postgres_dict; CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(POSTGRESQL(NAME postgres1 SCHEMA test_schema)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") - assert(int(result.strip()) == 100) + assert int(result.strip()) == 100 - node1.query(''' + node1.query( + """ DROP DICTIONARY postgres_dict; CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(POSTGRESQL(NAME postgres2)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") - assert(int(result.strip()) == 100) + assert int(result.strip()) == 100 - node1.query('DROP DICTIONARY postgres_dict') - node1.query(''' + node1.query("DROP DICTIONARY postgres_dict") + node1.query( + """ CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(POSTGRESQL(NAME postgres4)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') - result = node1.query_and_get_error("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") + """ + ) + result = node1.query_and_get_error( + "SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))" + ) - node1.query(''' + node1.query( + """ DROP DICTIONARY postgres_dict; CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id SOURCE(POSTGRESQL(NAME postgres1 PORT 5432)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) result = node1.query("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(99))") - assert(int(result.strip()) == 99) + assert int(result.strip()) == 99 def test_bad_configuration(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - node1.query(''' + node1.query( + """ DROP DICTIONARY IF EXISTS postgres_dict; CREATE DICTIONARY postgres_dict (id UInt32, value UInt32) PRIMARY KEY id @@ -388,13 +570,16 @@ def test_bad_configuration(started_cluster): table 'test_schema.test_table')) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); - ''') + """ + ) - node1.query_and_get_error("SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(1))") - assert node1.contains_in_log('Unexpected key `dbbb`') + node1.query_and_get_error( + "SELECT dictGetUInt32(postgres_dict, 'value', toUInt64(1))" + ) + assert node1.contains_in_log("Unexpected key `dbbb`") -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_dictionaries_redis/test.py b/tests/integration/test_dictionaries_redis/test.py index e5a51bcb88a..bc8170ab08d 100644 --- a/tests/integration/test_dictionaries_redis/test.py +++ b/tests/integration/test_dictionaries_redis/test.py @@ -7,43 +7,36 @@ from helpers.external_sources import SourceRedis cluster = None SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') +dict_configs_path = os.path.join(SCRIPT_DIR, "configs/dictionaries") node = None KEY_FIELDS = { - "simple": [ - Field("KeyField", 'UInt64', is_key=True, default_value_for_get=9999999) - ], + "simple": [Field("KeyField", "UInt64", is_key=True, default_value_for_get=9999999)], "complex": [ - Field("KeyField1", 'UInt64', is_key=True, default_value_for_get=9999999), - Field("KeyField2", 'String', is_key=True, default_value_for_get='xxxxxxxxx'), - ] + Field("KeyField1", "UInt64", is_key=True, default_value_for_get=9999999), + Field("KeyField2", "String", is_key=True, default_value_for_get="xxxxxxxxx"), + ], } -KEY_VALUES = { - "simple": [ - [1], [2] - ], - "complex": [ - [1, 'world'], [2, 'qwerty2'] - ] -} +KEY_VALUES = {"simple": [[1], [2]], "complex": [[1, "world"], [2, "qwerty2"]]} FIELDS = [ - Field("UInt8_", 'UInt8', default_value_for_get=55), - Field("UInt16_", 'UInt16', default_value_for_get=66), - Field("UInt32_", 'UInt32', default_value_for_get=77), - Field("UInt64_", 'UInt64', default_value_for_get=88), - Field("Int8_", 'Int8', default_value_for_get=-55), - Field("Int16_", 'Int16', default_value_for_get=-66), - Field("Int32_", 'Int32', default_value_for_get=-77), - Field("Int64_", 'Int64', default_value_for_get=-88), - Field("UUID_", 'UUID', default_value_for_get='550e8400-0000-0000-0000-000000000000'), - Field("Date_", 'Date', default_value_for_get='2018-12-30'), - Field("DateTime_", 'DateTime', default_value_for_get='2018-12-30 00:00:00'), - Field("String_", 'String', default_value_for_get='hi'), - Field("Float32_", 'Float32', default_value_for_get=555.11), - Field("Float64_", 'Float64', default_value_for_get=777.11), + Field("UInt8_", "UInt8", default_value_for_get=55), + Field("UInt16_", "UInt16", default_value_for_get=66), + Field("UInt32_", "UInt32", default_value_for_get=77), + Field("UInt64_", "UInt64", default_value_for_get=88), + Field("Int8_", "Int8", default_value_for_get=-55), + Field("Int16_", "Int16", default_value_for_get=-66), + Field("Int32_", "Int32", default_value_for_get=-77), + Field("Int64_", "Int64", default_value_for_get=-88), + Field( + "UUID_", "UUID", default_value_for_get="550e8400-0000-0000-0000-000000000000" + ), + Field("Date_", "Date", default_value_for_get="2018-12-30"), + Field("DateTime_", "DateTime", default_value_for_get="2018-12-30 00:00:00"), + Field("String_", "String", default_value_for_get="hi"), + Field("Float32_", "Float32", default_value_for_get=555.11), + Field("Float64_", "Float64", default_value_for_get=777.11), ] VALUES = [ @@ -55,10 +48,10 @@ VALUES = [ [-77, -8], [-888, -9], [-999, -10], - ['550e8400-e29b-41d4-a716-446655440003', '550e8400-e29b-41d4-a716-446655440002'], - ['1973-06-28', '1978-06-28'], - ['1985-02-28 23:43:25', '1986-02-28 23:42:25'], - ['hello', 'hello'], + ["550e8400-e29b-41d4-a716-446655440003", "550e8400-e29b-41d4-a716-446655440002"], + ["1973-06-28", "1978-06-28"], + ["1985-02-28 23:43:25", "1986-02-28 23:42:25"], + ["hello", "hello"], [22.543, 21.543], [3332154213.4, 3222154213.4], ] @@ -70,19 +63,21 @@ LAYOUTS = [ Layout("complex_key_hashed"), Layout("complex_key_cache"), Layout("direct"), - Layout("complex_key_direct") + Layout("complex_key_direct"), ] DICTIONARIES = [] -def get_dict(source, layout, fields, suffix_name=''): +def get_dict(source, layout, fields, suffix_name=""): global dict_configs_path structure = DictionaryStructure(layout, fields) - dict_name = source.name + "_" + layout.name + '_' + suffix_name - dict_path = os.path.join(dict_configs_path, dict_name + '.xml') - dictionary = Dictionary(dict_name, structure, source, dict_path, "table_" + dict_name, fields) + dict_name = source.name + "_" + layout.name + "_" + suffix_name + dict_path = os.path.join(dict_configs_path, dict_name + ".xml") + dictionary = Dictionary( + dict_name, structure, source, dict_path, "table_" + dict_name, fields + ) dictionary.generate_config() return dictionary @@ -102,14 +97,38 @@ def setup_module(module): for i, field in enumerate(FIELDS): DICTIONARIES.append([]) sources = [] - sources.append(SourceRedis("RedisSimple", "localhost", cluster.redis_port, cluster.redis_host, "6379", "", "clickhouse", i * 2, - storage_type="simple")) - sources.append(SourceRedis("RedisHash", "localhost", cluster.redis_port, cluster.redis_host, "6379", "", "clickhouse", i * 2 + 1, - storage_type="hash_map")) + sources.append( + SourceRedis( + "RedisSimple", + "localhost", + cluster.redis_port, + cluster.redis_host, + "6379", + "", + "clickhouse", + i * 2, + storage_type="simple", + ) + ) + sources.append( + SourceRedis( + "RedisHash", + "localhost", + cluster.redis_port, + cluster.redis_host, + "6379", + "", + "clickhouse", + i * 2 + 1, + storage_type="hash_map", + ) + ) for source in sources: for layout in LAYOUTS: if not source.compatible_with_layout(layout): - print("Source", source.name, "incompatible with layout", layout.name) + print( + "Source", source.name, "incompatible with layout", layout.name + ) continue fields = KEY_FIELDS[layout.layout_type] + [field] @@ -120,7 +139,9 @@ def setup_module(module): for fname in os.listdir(dict_configs_path): dictionaries.append(os.path.join(dict_configs_path, fname)) - node = cluster.add_instance('node', main_configs=main_configs, dictionaries=dictionaries, with_redis=True) + node = cluster.add_instance( + "node", main_configs=main_configs, dictionaries=dictionaries, with_redis=True + ) @pytest.fixture(scope="module", autouse=True) @@ -142,7 +163,7 @@ def started_cluster(): @pytest.mark.parametrize("id", list(range(len(FIELDS)))) def test_redis_dictionaries(started_cluster, id): - print('id:', id) + print("id:", id) dicts = DICTIONARIES[id] values = VALUES[id] @@ -176,7 +197,7 @@ def test_redis_dictionaries(started_cluster, id): for query, answer in queries_with_answers: print(query) - assert node.query(query) == str(answer) + '\n' + assert node.query(query) == str(answer) + "\n" # Checks, that dictionaries can be reloaded. node.query("system reload dictionaries") diff --git a/tests/integration/test_dictionaries_redis/test_long.py b/tests/integration/test_dictionaries_redis/test_long.py index 3f29403df62..094df789704 100644 --- a/tests/integration/test_dictionaries_redis/test_long.py +++ b/tests/integration/test_dictionaries_redis/test_long.py @@ -4,7 +4,8 @@ import redis cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', with_redis=True) +node = cluster.add_instance("node", with_redis=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -12,12 +13,15 @@ def start_cluster(): cluster.start() N = 1000 - client = redis.Redis(host='localhost', port=cluster.redis_port, password='clickhouse', db=0) + client = redis.Redis( + host="localhost", port=cluster.redis_port, password="clickhouse", db=0 + ) client.flushdb() for i in range(N): - client.hset('2020-10-10', i, i) + client.hset("2020-10-10", i, i) - node.query(""" + node.query( + """ CREATE DICTIONARY redis_dict ( date String, @@ -27,10 +31,13 @@ def start_cluster(): PRIMARY KEY date, id SOURCE(REDIS(HOST '{}' PORT 6379 STORAGE_TYPE 'hash_map' DB_INDEX 0 PASSWORD 'clickhouse')) LAYOUT(COMPLEX_KEY_DIRECT()) - """.format(cluster.redis_host) + """.format( + cluster.redis_host + ) ) - node.query(""" + node.query( + """ CREATE TABLE redis_dictionary_test ( date Date, @@ -39,13 +46,24 @@ def start_cluster(): ENGINE = MergeTree ORDER BY id""" ) - node.query("INSERT INTO default.redis_dictionary_test SELECT '2020-10-10', number FROM numbers(1000000)") + node.query( + "INSERT INTO default.redis_dictionary_test SELECT '2020-10-10', number FROM numbers(1000000)" + ) yield cluster finally: cluster.shutdown() + def test_redis_dict_long(start_cluster): - assert node.query("SELECT count(), uniqExact(date), uniqExact(id) FROM redis_dict") == "1000\t1\t1000\n" - assert node.query("SELECT count(DISTINCT dictGet('redis_dict', 'value', tuple(date, id % 1000))) FROM redis_dictionary_test") == "1000\n" + assert ( + node.query("SELECT count(), uniqExact(date), uniqExact(id) FROM redis_dict") + == "1000\t1\t1000\n" + ) + assert ( + node.query( + "SELECT count(DISTINCT dictGet('redis_dict', 'value', tuple(date, id % 1000))) FROM redis_dictionary_test" + ) + == "1000\n" + ) diff --git a/tests/integration/test_dictionaries_select_all/generate_dictionaries.py b/tests/integration/test_dictionaries_select_all/generate_dictionaries.py index 4208615bdc3..31480974d86 100644 --- a/tests/integration/test_dictionaries_select_all/generate_dictionaries.py +++ b/tests/integration/test_dictionaries_select_all/generate_dictionaries.py @@ -2,22 +2,38 @@ import difflib import os from functools import reduce -files = ['key_simple.tsv', 'key_complex_integers.tsv', 'key_complex_mixed.tsv'] +files = ["key_simple.tsv", "key_complex_integers.tsv", "key_complex_mixed.tsv"] types = [ - 'UInt8', 'UInt16', 'UInt32', 'UInt64', - 'Int8', 'Int16', 'Int32', 'Int64', - 'Float32', 'Float64', - 'String', - 'Date', 'DateTime' + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "Int8", + "Int16", + "Int32", + "Int64", + "Float32", + "Float64", + "String", + "Date", + "DateTime", ] implicit_defaults = [ - '1', '1', '1', '', - '-1', '-1', '-1', '-1', - '2.71828', '2.71828', - 'implicit-default', - '2015-11-25', '' + "1", + "1", + "1", + "", + "-1", + "-1", + "-1", + "-1", + "2.71828", + "2.71828", + "implicit-default", + "2015-11-25", + "", ] @@ -25,25 +41,22 @@ def generate_structure(): # [ name, key_type, has_parent ] return [ # Simple key dictionaries - ['clickhouse_flat', 0, True], - ['clickhouse_hashed', 0, True], - ['clickhouse_cache', 0, True], - + ["clickhouse_flat", 0, True], + ["clickhouse_hashed", 0, True], + ["clickhouse_cache", 0, True], # Complex key dictionaries with (UInt8, UInt8) key - ['clickhouse_complex_integers_key_hashed', 1, False], - ['clickhouse_complex_integers_key_cache', 1, False], - + ["clickhouse_complex_integers_key_hashed", 1, False], + ["clickhouse_complex_integers_key_cache", 1, False], # Complex key dictionaries with (String, UInt8) key - ['clickhouse_complex_mixed_key_hashed', 2, False], - ['clickhouse_complex_mixed_key_cache', 2, False], - + ["clickhouse_complex_mixed_key_hashed", 2, False], + ["clickhouse_complex_mixed_key_cache", 2, False], # Range hashed dictionary - ['clickhouse_range_hashed', 3, False], + ["clickhouse_range_hashed", 3, False], ] def generate_dictionaries(path, structure): - dictionary_skeleton = ''' + dictionary_skeleton = """ {name} @@ -69,21 +82,23 @@ def generate_dictionaries(path, structure): {parent} - ''' - attribute_skeleton = ''' + """ + attribute_skeleton = """ %s_ %s %s - ''' + """ - dictionary_skeleton = \ - dictionary_skeleton % reduce( - lambda xml, type_default: xml + attribute_skeleton % (type_default[0], type_default[0], type_default[1]), - list(zip(types, implicit_defaults)), '') + dictionary_skeleton = dictionary_skeleton % reduce( + lambda xml, type_default: xml + + attribute_skeleton % (type_default[0], type_default[0], type_default[1]), + list(zip(types, implicit_defaults)), + "", + ) - source_clickhouse = ''' + source_clickhouse = """ localhost 9000 @@ -92,21 +107,23 @@ def generate_dictionaries(path, structure): test dictionary_source
 - ''' + """ - layout_flat = '' - layout_hashed = '' - layout_cache = '128' - layout_complex_key_hashed = '' - layout_complex_key_cache = '128' - layout_range_hashed = '' + layout_flat = "" + layout_hashed = "" + layout_cache = "128" + layout_complex_key_hashed = "" + layout_complex_key_cache = ( + "128" + ) + layout_range_hashed = "" - key_simple = ''' + key_simple = """ id - ''' - key_complex_integers = ''' + """ + key_complex_integers = """ key0 @@ -118,8 +135,8 @@ def generate_dictionaries(path, structure): UInt8 - ''' - key_complex_mixed = ''' + """ + key_complex_mixed = """ key0_str @@ -131,9 +148,9 @@ def generate_dictionaries(path, structure): UInt8 - ''' + """ - key_range_hashed = ''' + key_range_hashed = """ id @@ -143,32 +160,29 @@ def generate_dictionaries(path, structure): EndDate - ''' + """ keys = [key_simple, key_complex_integers, key_complex_mixed, key_range_hashed] - parent_attribute = ''' + parent_attribute = """ Parent UInt64 true 0 - ''' + """ sources_and_layouts = [ # Simple key dictionaries [source_clickhouse, layout_flat], [source_clickhouse, layout_hashed], [source_clickhouse, layout_cache], - # Complex key dictionaries with (UInt8, UInt8) key [source_clickhouse, layout_complex_key_hashed], [source_clickhouse, layout_complex_key_cache], - # Complex key dictionaries with (String, UInt8) key [source_clickhouse, layout_complex_key_hashed], [source_clickhouse, layout_complex_key_cache], - # Range hashed dictionary [source_clickhouse, layout_range_hashed], ] @@ -176,12 +190,17 @@ def generate_dictionaries(path, structure): file_names = [] # Generate dictionaries. - for (name, key_idx, has_parent), (source, layout) in zip(structure, sources_and_layouts): - filename = os.path.join(path, 'dictionary_%s.xml' % name) + for (name, key_idx, has_parent), (source, layout) in zip( + structure, sources_and_layouts + ): + filename = os.path.join(path, "dictionary_%s.xml" % name) file_names.append(filename) - with open(filename, 'w') as file: + with open(filename, "w") as file: dictionary_xml = dictionary_skeleton.format( - key=keys[key_idx], parent=parent_attribute if has_parent else '', **locals()) + key=keys[key_idx], + parent=parent_attribute if has_parent else "", + **locals() + ) file.write(dictionary_xml) return file_names @@ -189,77 +208,99 @@ def generate_dictionaries(path, structure): class DictionaryTestTable: def __init__(self, source_file_name): - self.structure = '''id UInt64, key0 UInt8, key0_str String, key1 UInt8, + self.structure = """id UInt64, key0 UInt8, key0_str String, key1 UInt8, StartDate Date, EndDate Date, UInt8_ UInt8, UInt16_ UInt16, UInt32_ UInt32, UInt64_ UInt64, Int8_ Int8, Int16_ Int16, Int32_ Int32, Int64_ Int64, Float32_ Float32, Float64_ Float64, String_ String, - Date_ Date, DateTime_ DateTime, Parent UInt64''' + Date_ Date, DateTime_ DateTime, Parent UInt64""" - self.names_and_types = list(map(str.split, self.structure.split(','))) + self.names_and_types = list(map(str.split, self.structure.split(","))) self.keys_names_and_types = self.names_and_types[:6] self.values_names_and_types = self.names_and_types[6:] self.source_file_name = source_file_name self.rows = None def create_clickhouse_source(self, instance): - query = ''' + query = """ create database if not exists test; drop table if exists test.dictionary_source; create table test.dictionary_source (%s) engine=Log; insert into test.dictionary_source values %s ; - ''' + """ types = tuple(pair[1] for pair in self.names_and_types) with open(self.source_file_name) as source_file: - lines = source_file.read().split('\n') + lines = source_file.read().split("\n") lines = tuple(filter(len, lines)) self.rows = [] def wrap_value(pair): value, type = pair - return "'" + value + "'" if type in ('String', 'Date', 'DateTime') else value + return ( + "'" + value + "'" if type in ("String", "Date", "DateTime") else value + ) def make_tuple(line): - row = tuple(line.split('\t')) + row = tuple(line.split("\t")) self.rows.append(row) - return '(' + ','.join(map(wrap_value, list(zip(row, types)))) + ')' + return "(" + ",".join(map(wrap_value, list(zip(row, types)))) + ")" - values = ','.join(map(make_tuple, lines)) + values = ",".join(map(make_tuple, lines)) print(query % (self.structure, values)) instance.query(query % (self.structure, values)) def get_structure_for_keys(self, keys, enable_parent=True): - structure = ','.join(name + ' ' + type for name, type in self.keys_names_and_types if name in keys) - return structure + ', ' + ','.join(name + ' ' + type for name, type in self.values_names_and_types - if enable_parent or name != 'Parent') + structure = ",".join( + name + " " + type + for name, type in self.keys_names_and_types + if name in keys + ) + return ( + structure + + ", " + + ",".join( + name + " " + type + for name, type in self.values_names_and_types + if enable_parent or name != "Parent" + ) + ) def _build_line_from_row(self, row, names): - return '\t'.join((value for value, (name, type) in zip(row, self.names_and_types) if name in set(names))) + return "\t".join( + ( + value + for value, (name, type) in zip(row, self.names_and_types) + if name in set(names) + ) + ) def compare_rows_by_keys(self, keys, values, lines, add_not_found_rows=True): - rows = [line.rstrip('\n').split('\t') for line in lines] + rows = [line.rstrip("\n").split("\t") for line in lines] diff = [] matched = [] - lines_map = {self._build_line_from_row(row, keys): self._build_line_from_row(row, values) for row in self.rows} + lines_map = { + self._build_line_from_row(row, keys): self._build_line_from_row(row, values) + for row in self.rows + } for row in rows: - key = '\t'.join(row[:len(keys)]) - value = '\t'.join(row[len(keys):]) + key = "\t".join(row[: len(keys)]) + value = "\t".join(row[len(keys) :]) if key in list(lines_map.keys()): pattern_value = lines_map[key] del lines_map[key] if not value == pattern_value: - diff.append((key + '\t' + value, key + '\t' + pattern_value)) + diff.append((key + "\t" + value, key + "\t" + pattern_value)) else: - matched.append((key + '\t' + value, key + '\t' + pattern_value)) + matched.append((key + "\t" + value, key + "\t" + pattern_value)) else: - diff.append((key + '\t' + value, '')) + diff.append((key + "\t" + value, "")) if add_not_found_rows: for key, value in list(lines_map.items()): - diff.append(('', key + '\t' + value)) + diff.append(("", key + "\t" + value)) if not diff: return None @@ -269,13 +310,21 @@ class DictionaryTestTable: right_lines = tuple(pair[1] for pair in diff) return left_lines, right_lines - def compare_by_keys(self, keys, lines, with_parent_column=True, add_not_found_rows=True): - values = [name for name, type in self.values_names_and_types if with_parent_column or name != 'Parent'] + def compare_by_keys( + self, keys, lines, with_parent_column=True, add_not_found_rows=True + ): + values = [ + name + for name, type in self.values_names_and_types + if with_parent_column or name != "Parent" + ] return self.compare_rows_by_keys(keys, values, lines, add_not_found_rows) def process_diff(self, diff): if not diff: - return '' + return "" left_lines, right_lines = diff - args = {'fromfile': 'received', 'tofile': 'expected', 'lineterm': ''} - return '\n'.join(tuple(difflib.context_diff(left_lines, right_lines, **args))[:]) + args = {"fromfile": "received", "tofile": "expected", "lineterm": ""} + return "\n".join( + tuple(difflib.context_diff(left_lines, right_lines, **args))[:] + ) diff --git a/tests/integration/test_dictionaries_select_all/test.py b/tests/integration/test_dictionaries_select_all/test.py index b1bf2e98b25..0a740394129 100644 --- a/tests/integration/test_dictionaries_select_all/test.py +++ b/tests/integration/test_dictionaries_select_all/test.py @@ -4,7 +4,11 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV -from .generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable +from .generate_dictionaries import ( + generate_structure, + generate_dictionaries, + DictionaryTestTable, +) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -19,11 +23,15 @@ def setup_module(module): global test_table structure = generate_structure() - dictionary_files = generate_dictionaries(os.path.join(SCRIPT_DIR, 'configs/dictionaries'), structure) + dictionary_files = generate_dictionaries( + os.path.join(SCRIPT_DIR, "configs/dictionaries"), structure + ) cluster = ClickHouseCluster(__file__) - instance = cluster.add_instance('instance', dictionaries=dictionary_files) - test_table = DictionaryTestTable(os.path.join(SCRIPT_DIR, 'configs/dictionaries/source.tsv')) + instance = cluster.add_instance("instance", dictionaries=dictionary_files) + test_table = DictionaryTestTable( + os.path.join(SCRIPT_DIR, "configs/dictionaries/source.tsv") + ) @pytest.fixture(scope="module", autouse=True) @@ -31,8 +39,8 @@ def started_cluster(): try: cluster.start() test_table.create_clickhouse_source(instance) - for line in TSV(instance.query('select name from system.dictionaries')).lines: - print(line, end=' ') + for line in TSV(instance.query("select name from system.dictionaries")).lines: + print(line, end=" ") yield cluster @@ -40,18 +48,22 @@ def started_cluster(): cluster.shutdown() -@pytest.fixture(params=[ - # name, keys, use_parent - ('clickhouse_hashed', ('id',), True), - ('clickhouse_flat', ('id',), True), - ('clickhouse_complex_integers_key_hashed', ('key0', 'key1'), False), - ('clickhouse_complex_mixed_key_hashed', ('key0_str', 'key1'), False), - ('clickhouse_range_hashed', ('id', 'StartDate', 'EndDate'), False), -], - ids=['clickhouse_hashed', 'clickhouse_flat', - 'clickhouse_complex_integers_key_hashed', - 'clickhouse_complex_mixed_key_hashed', - 'clickhouse_range_hashed'] +@pytest.fixture( + params=[ + # name, keys, use_parent + ("clickhouse_hashed", ("id",), True), + ("clickhouse_flat", ("id",), True), + ("clickhouse_complex_integers_key_hashed", ("key0", "key1"), False), + ("clickhouse_complex_mixed_key_hashed", ("key0_str", "key1"), False), + ("clickhouse_range_hashed", ("id", "StartDate", "EndDate"), False), + ], + ids=[ + "clickhouse_hashed", + "clickhouse_flat", + "clickhouse_complex_integers_key_hashed", + "clickhouse_complex_mixed_key_hashed", + "clickhouse_range_hashed", + ], ) def dictionary_structure(started_cluster, request): return request.param @@ -62,27 +74,40 @@ def test_select_all(dictionary_structure): query = instance.query structure = test_table.get_structure_for_keys(keys, use_parent) - query(''' + query( + """ DROP TABLE IF EXISTS test.{0} - '''.format(name)) + """.format( + name + ) + ) - create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format(name, structure) + create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format( + name, structure + ) TSV(query(create_query)) - result = TSV(query('select * from test.{0}'.format(name))) + result = TSV(query("select * from test.{0}".format(name))) - diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True) + diff = test_table.compare_by_keys( + keys, result.lines, use_parent, add_not_found_rows=True + ) print(test_table.process_diff(diff)) assert not diff -@pytest.fixture(params=[ - # name, keys, use_parent - ('clickhouse_cache', ('id',), True), - ('clickhouse_complex_integers_key_cache', ('key0', 'key1'), False), - ('clickhouse_complex_mixed_key_cache', ('key0_str', 'key1'), False) -], - ids=['clickhouse_cache', 'clickhouse_complex_integers_key_cache', 'clickhouse_complex_mixed_key_cache'] +@pytest.fixture( + params=[ + # name, keys, use_parent + ("clickhouse_cache", ("id",), True), + ("clickhouse_complex_integers_key_cache", ("key0", "key1"), False), + ("clickhouse_complex_mixed_key_cache", ("key0_str", "key1"), False), + ], + ids=[ + "clickhouse_cache", + "clickhouse_complex_integers_key_cache", + "clickhouse_complex_mixed_key_cache", + ], ) def cached_dictionary_structure(started_cluster, request): return request.param @@ -93,32 +118,42 @@ def test_select_all_from_cached(cached_dictionary_structure): query = instance.query structure = test_table.get_structure_for_keys(keys, use_parent) - query(''' + query( + """ DROP TABLE IF EXISTS test.{0} - '''.format(name)) + """.format( + name + ) + ) - create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format(name, structure) + create_query = "CREATE TABLE test.{0} ({1}) engine = Dictionary({0})".format( + name, structure + ) TSV(query(create_query)) for i in range(4): - result = TSV(query('select * from test.{0}'.format(name))) - diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=False) + result = TSV(query("select * from test.{0}".format(name))) + diff = test_table.compare_by_keys( + keys, result.lines, use_parent, add_not_found_rows=False + ) print(test_table.process_diff(diff)) assert not diff key = [] for key_name in keys: - if key_name.endswith('str'): + if key_name.endswith("str"): key.append("'" + str(i) + "'") else: key.append(str(i)) if len(key) == 1: - key = 'toUInt64(' + str(i) + ')' + key = "toUInt64(" + str(i) + ")" else: - key = str('(' + ','.join(key) + ')') + key = str("(" + ",".join(key) + ")") query("select dictGetUInt8('{0}', 'UInt8_', {1})".format(name, key)) - result = TSV(query('select * from test.{0}'.format(name))) - diff = test_table.compare_by_keys(keys, result.lines, use_parent, add_not_found_rows=True) + result = TSV(query("select * from test.{0}".format(name))) + diff = test_table.compare_by_keys( + keys, result.lines, use_parent, add_not_found_rows=True + ) print(test_table.process_diff(diff)) assert not diff diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py index 9bee5db8ce1..a973b697d0d 100644 --- a/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/tests/integration/test_dictionaries_update_and_reload/test.py @@ -7,11 +7,16 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -DICTIONARY_FILES = ['configs/dictionaries/cache_xypairs.xml', 'configs/dictionaries/executable.xml', - 'configs/dictionaries/file.xml', 'configs/dictionaries/file.txt', 'configs/dictionaries/slow.xml'] +DICTIONARY_FILES = [ + "configs/dictionaries/cache_xypairs.xml", + "configs/dictionaries/executable.xml", + "configs/dictionaries/file.xml", + "configs/dictionaries/file.txt", + "configs/dictionaries/slow.xml", +] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', dictionaries=DICTIONARY_FILES) +instance = cluster.add_instance("instance", dictionaries=DICTIONARY_FILES) @pytest.fixture(scope="module") @@ -27,18 +32,29 @@ def started_cluster(): def get_status(dictionary_name): - return instance.query("SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip("\n") + return instance.query( + "SELECT status FROM system.dictionaries WHERE name='" + dictionary_name + "'" + ).rstrip("\n") def get_last_exception(dictionary_name): - return instance.query("SELECT last_exception FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip( - "\n").replace("\\'", "'") + return ( + instance.query( + "SELECT last_exception FROM system.dictionaries WHERE name='" + + dictionary_name + + "'" + ) + .rstrip("\n") + .replace("\\'", "'") + ) def get_loading_start_time(dictionary_name): s = instance.query( - "SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip( - "\n") + "SELECT toTimeZone(loading_start_time, 'UTC') FROM system.dictionaries WHERE name='" + + dictionary_name + + "'" + ).rstrip("\n") if s == "1970-01-01 00:00:00": return None return time.strptime(s, "%Y-%m-%d %H:%M:%S") @@ -46,8 +62,10 @@ def get_loading_start_time(dictionary_name): def get_last_successful_update_time(dictionary_name): s = instance.query( - "SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + dictionary_name + "'").rstrip( - "\n") + "SELECT toTimeZone(last_successful_update_time, 'UTC') FROM system.dictionaries WHERE name='" + + dictionary_name + + "'" + ).rstrip("\n") if s == "1970-01-01 00:00:00": return None return time.strptime(s, "%Y-%m-%d %H:%M:%S") @@ -55,60 +73,67 @@ def get_last_successful_update_time(dictionary_name): def get_loading_duration(dictionary_name): return float( - instance.query("SELECT loading_duration FROM system.dictionaries WHERE name='" + dictionary_name + "'")) + instance.query( + "SELECT loading_duration FROM system.dictionaries WHERE name='" + + dictionary_name + + "'" + ) + ) def replace_in_file_in_container(file_name, what, replace_with): - instance.exec_in_container(['sed', '-i', f's/{what}/{replace_with}/g', file_name]) + instance.exec_in_container(["sed", "-i", f"s/{what}/{replace_with}/g", file_name]) def test_reload_while_loading(started_cluster): query = instance.query # dictionaries_lazy_load == false, so this dictionary is not loaded. - assert get_status('slow') == "NOT_LOADED" - assert get_loading_duration('slow') == 0 + assert get_status("slow") == "NOT_LOADED" + assert get_loading_duration("slow") == 0 # It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout. with pytest.raises(QueryTimeoutExceedException): query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout=0.5) # The dictionary is now loading. - assert get_status('slow') == "LOADING" - start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + assert get_status("slow") == "LOADING" + start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow") assert duration > 0 time.sleep(0.5) # Still loading. - assert get_status('slow') == "LOADING" + assert get_status("slow") == "LOADING" prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow") assert start_time == prev_start_time assert duration >= prev_duration # SYSTEM RELOAD DICTIONARY should restart loading. with pytest.raises(QueryTimeoutExceedException): query("SYSTEM RELOAD DICTIONARY 'slow'", timeout=0.5) - assert get_status('slow') == "LOADING" + assert get_status("slow") == "LOADING" prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow") assert start_time > prev_start_time assert duration < prev_duration time.sleep(0.5) # Still loading. - assert get_status('slow') == "LOADING" + assert get_status("slow") == "LOADING" prev_start_time, prev_duration = start_time, duration - start_time, duration = get_loading_start_time('slow'), get_loading_duration('slow') + start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow") assert start_time == prev_start_time assert duration >= prev_duration # Changing the configuration file should restart loading again. - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/slow.xml', 'sleep 100', 'sleep 0') + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/slow.xml", "sleep 100", "sleep 0" + ) time.sleep(5) # Configuration files are reloaded once in 5 seconds. # This time loading should finish quickly. - assert get_status('slow') == "LOADED" + assert get_status("slow") == "LOADED" - last_successful_update_time = get_last_successful_update_time('slow') + last_successful_update_time = get_last_successful_update_time("slow") assert last_successful_update_time > start_time assert query("SELECT dictGetInt32('slow', 'a', toUInt64(5))") == "6\n" @@ -124,8 +149,12 @@ def test_reload_after_loading(started_cluster): # for mtime, and clickhouse will miss the update if we change the file too # soon. Should probably be fixed by switching to use std::filesystem. time.sleep(1) - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/executable.xml', '8', '81') - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/file.txt', '10', '101') + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/executable.xml", "8", "81" + ) + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/file.txt", "10", "101" + ) # SYSTEM RELOAD 'name' reloads only the specified dictionary. query("SYSTEM RELOAD DICTIONARY 'executable'") @@ -138,8 +167,12 @@ def test_reload_after_loading(started_cluster): # SYSTEM RELOAD DICTIONARIES reloads all loaded dictionaries. time.sleep(1) # see the comment above - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/executable.xml', '81', '82') - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/file.txt', '101', '102') + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/executable.xml", "81", "82" + ) + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/file.txt", "101", "102" + ) query("SYSTEM RELOAD DICTIONARY 'file'") query("SYSTEM RELOAD DICTIONARY 'executable'") assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "82\n" @@ -148,8 +181,12 @@ def test_reload_after_loading(started_cluster): # Configuration files are reloaded and lifetimes are checked automatically once in 5 seconds. # Wait slightly more, to be sure it did reload. time.sleep(1) # see the comment above - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/executable.xml', '82', '83') - replace_in_file_in_container('/etc/clickhouse-server/dictionaries/file.txt', '102', '103') + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/executable.xml", "82", "83" + ) + replace_in_file_in_container( + "/etc/clickhouse-server/dictionaries/file.txt", "102", "103" + ) time.sleep(10) assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "103\n" assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "83\n" @@ -163,24 +200,36 @@ def test_reload_after_fail_by_system_reload(started_cluster): # We expect an error because the file source doesn't exist. no_such_file_error = "No such file" - assert no_such_file_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") + assert no_such_file_error in instance.query_and_get_error( + "SELECT dictGetInt32('no_file', 'a', toUInt64(9))" + ) assert get_status("no_file") == "FAILED" # SYSTEM RELOAD should not change anything now, the status is still FAILED. - assert no_such_file_error in instance.query_and_get_error("SYSTEM RELOAD DICTIONARY 'no_file'") - assert no_such_file_error in instance.query_and_get_error("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") + assert no_such_file_error in instance.query_and_get_error( + "SYSTEM RELOAD DICTIONARY 'no_file'" + ) + assert no_such_file_error in instance.query_and_get_error( + "SELECT dictGetInt32('no_file', 'a', toUInt64(9))" + ) assert get_status("no_file") == "FAILED" # Creating the file source makes the dictionary able to load. - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), - "/etc/clickhouse-server/dictionaries/no_file.txt") + instance.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), + "/etc/clickhouse-server/dictionaries/no_file.txt", + ) query("SYSTEM RELOAD DICTIONARY 'no_file'") query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" assert get_status("no_file") == "LOADED" # Removing the file source should not spoil the loaded dictionary. - instance.exec_in_container(["rm", "/etc/clickhouse-server/dictionaries/no_file.txt"]) - assert no_such_file_error in instance.query_and_get_error("SYSTEM RELOAD DICTIONARY 'no_file'") + instance.exec_in_container( + ["rm", "/etc/clickhouse-server/dictionaries/no_file.txt"] + ) + assert no_such_file_error in instance.query_and_get_error( + "SYSTEM RELOAD DICTIONARY 'no_file'" + ) query("SELECT dictGetInt32('no_file', 'a', toUInt64(9))") == "10\n" assert get_status("no_file") == "LOADED" @@ -191,28 +240,38 @@ def test_reload_after_fail_by_timer(started_cluster): # We expect an error because the file source doesn't exist. expected_error = "No such file" - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") + assert expected_error in instance.query_and_get_error( + "SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))" + ) assert get_status("no_file_2") == "FAILED" # Passed time should not change anything now, the status is still FAILED. - time.sleep(6); - assert expected_error in instance.query_and_get_error("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") + time.sleep(6) + assert expected_error in instance.query_and_get_error( + "SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))" + ) assert get_status("no_file_2") == "FAILED" # Creating the file source makes the dictionary able to load. - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), - "/etc/clickhouse-server/dictionaries/no_file_2.txt") + instance.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/dictionaries/file.txt"), + "/etc/clickhouse-server/dictionaries/no_file_2.txt", + ) # Check that file appears in container and wait if needed. while not instance.path_exists("/etc/clickhouse-server/dictionaries/no_file_2.txt"): time.sleep(1) - assert("9\t10\n" == instance.exec_in_container(["cat", "/etc/clickhouse-server/dictionaries/no_file_2.txt"])) + assert "9\t10\n" == instance.exec_in_container( + ["cat", "/etc/clickhouse-server/dictionaries/no_file_2.txt"] + ) instance.query("SYSTEM RELOAD DICTIONARY no_file_2") instance.query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" assert get_status("no_file_2") == "LOADED" # Removing the file source should not spoil the loaded dictionary. - instance.exec_in_container(["rm", "/etc/clickhouse-server/dictionaries/no_file_2.txt"]) - time.sleep(6); + instance.exec_in_container( + ["rm", "/etc/clickhouse-server/dictionaries/no_file_2.txt"] + ) + time.sleep(6) instance.query("SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))") == "10\n" assert get_status("no_file_2") == "LOADED" @@ -224,24 +283,33 @@ def test_reload_after_fail_in_cache_dictionary(started_cluster): # Can't get a value from the cache dictionary because the source (table `test.xypairs`) doesn't respond. expected_error = "Table test.xypairs doesn't exist" update_error = "Could not update cache dictionary cache_xypairs now" - assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))") + assert expected_error in query_and_get_error( + "SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(1))" + ) assert get_status("cache_xypairs") == "LOADED" assert expected_error in get_last_exception("cache_xypairs") # Create table `test.xypairs`. - query(''' + query( + """ DROP TABLE IF EXISTS test.xypairs; CREATE TABLE test.xypairs (x UInt64, y UInt64) ENGINE=Log; INSERT INTO test.xypairs VALUES (1, 56), (3, 78); - ''') + """ + ) # Cache dictionary now works. - assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "56", ignore_error=True) + assert_eq_with_retry( + instance, + "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", + "56", + ignore_error=True, + ) query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" assert get_last_exception("cache_xypairs") == "" # Drop table `test.xypairs`. - query('DROP TABLE test.xypairs') + query("DROP TABLE test.xypairs") # Values are cached so we can get them. query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" @@ -249,28 +317,36 @@ def test_reload_after_fail_in_cache_dictionary(started_cluster): assert get_last_exception("cache_xypairs") == "" # But we can't get a value from the source table which isn't cached. - assert expected_error in query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + assert expected_error in query_and_get_error( + "SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))" + ) assert expected_error in get_last_exception("cache_xypairs") # Passed time should not spoil the cache. time.sleep(5) query("SELECT dictGet('cache_xypairs', 'y', toUInt64(1))") == "56" query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" - error = query_and_get_error("SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))") + error = query_and_get_error( + "SELECT dictGetUInt64('cache_xypairs', 'y', toUInt64(3))" + ) assert (expected_error in error) or (update_error in error) last_exception = get_last_exception("cache_xypairs") assert (expected_error in last_exception) or (update_error in last_exception) # Create table `test.xypairs` again with changed values. - query(''' + query( + """ CREATE TABLE test.xypairs (x UInt64, y UInt64) ENGINE=Log; INSERT INTO test.xypairs VALUES (1, 57), (3, 79); - ''') + """ + ) - query('SYSTEM RELOAD DICTIONARY cache_xypairs') + query("SYSTEM RELOAD DICTIONARY cache_xypairs") # The cache dictionary returns new values now. - assert_eq_with_retry(instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57") + assert_eq_with_retry( + instance, "SELECT dictGet('cache_xypairs', 'y', toUInt64(1))", "57" + ) query("SELECT dictGet('cache_xypairs', 'y', toUInt64(2))") == "0" query("SELECT dictGet('cache_xypairs', 'y', toUInt64(3))") == "79" assert get_last_exception("cache_xypairs") == "" diff --git a/tests/integration/test_dictionaries_update_field/test.py b/tests/integration/test_dictionaries_update_field/test.py index 8fb0d67e8b8..a98239e3a40 100644 --- a/tests/integration/test_dictionaries_update_field/test.py +++ b/tests/integration/test_dictionaries_update_field/test.py @@ -7,7 +7,8 @@ from helpers.cluster import ClickHouseKiller cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('main_node', main_configs=[]) +node = cluster.add_instance("main_node", main_configs=[]) + @pytest.fixture(scope="module") def started_cluster(): @@ -31,11 +32,15 @@ def started_cluster(): finally: cluster.shutdown() -@pytest.mark.parametrize("dictionary_name,dictionary_type", [ - ("flat_update_field_dictionary", "FLAT"), - ("simple_key_hashed_update_field_dictionary", "HASHED"), - ("complex_key_hashed_update_field_dictionary", "COMPLEX_KEY_HASHED") -]) + +@pytest.mark.parametrize( + "dictionary_name,dictionary_type", + [ + ("flat_update_field_dictionary", "FLAT"), + ("simple_key_hashed_update_field_dictionary", "HASHED"), + ("complex_key_hashed_update_field_dictionary", "COMPLEX_KEY_HASHED"), + ], +) def test_update_field(started_cluster, dictionary_name, dictionary_type): create_dictionary_query = """ CREATE DICTIONARY {dictionary_name} @@ -48,29 +53,53 @@ def test_update_field(started_cluster, dictionary_name, dictionary_type): SOURCE(CLICKHOUSE(table 'table_for_update_field_dictionary' update_field 'last_insert_time')) LAYOUT({dictionary_type}()) LIFETIME(1); - """.format(dictionary_name=dictionary_name, dictionary_type=dictionary_type) + """.format( + dictionary_name=dictionary_name, dictionary_type=dictionary_type + ) node.query(create_dictionary_query) - node.query("INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now());") - query_result = node.query("SELECT key, value FROM {dictionary_name} ORDER BY key ASC".format(dictionary_name=dictionary_name)) - assert query_result == '1\tFirst\n' + node.query( + "INSERT INTO table_for_update_field_dictionary VALUES (1, 'First', now());" + ) + query_result = node.query( + "SELECT key, value FROM {dictionary_name} ORDER BY key ASC".format( + dictionary_name=dictionary_name + ) + ) + assert query_result == "1\tFirst\n" - node.query("INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now());") + node.query( + "INSERT INTO table_for_update_field_dictionary VALUES (2, 'Second', now());" + ) time.sleep(10) - query_result = node.query("SELECT key, value FROM {dictionary_name} ORDER BY key ASC".format(dictionary_name=dictionary_name)) + query_result = node.query( + "SELECT key, value FROM {dictionary_name} ORDER BY key ASC".format( + dictionary_name=dictionary_name + ) + ) - assert query_result == '1\tFirst\n2\tSecond\n' + assert query_result == "1\tFirst\n2\tSecond\n" - node.query("INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now());") - node.query("INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now());") + node.query( + "INSERT INTO table_for_update_field_dictionary VALUES (2, 'SecondUpdated', now());" + ) + node.query( + "INSERT INTO table_for_update_field_dictionary VALUES (3, 'Third', now());" + ) time.sleep(10) - query_result = node.query("SELECT key, value FROM {dictionary_name} ORDER BY key ASC".format(dictionary_name=dictionary_name)) + query_result = node.query( + "SELECT key, value FROM {dictionary_name} ORDER BY key ASC".format( + dictionary_name=dictionary_name + ) + ) - assert query_result == '1\tFirst\n2\tSecondUpdated\n3\tThird\n' + assert query_result == "1\tFirst\n2\tSecondUpdated\n3\tThird\n" node.query("TRUNCATE TABLE table_for_update_field_dictionary") - node.query("DROP DICTIONARY {dictionary_name}".format(dictionary_name=dictionary_name)) + node.query( + "DROP DICTIONARY {dictionary_name}".format(dictionary_name=dictionary_name) + ) diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py index cfd5f4d5607..bb587efa7e9 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_reading.py @@ -1,5 +1,3 @@ - - import time import pytest @@ -9,8 +7,10 @@ from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__, name="reading") -dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', dictionaries=['configs/dictionaries/cache_ints_dictionary.xml']) +dictionary_node = cluster.add_instance("dictionary_node", stay_alive=True) +main_node = cluster.add_instance( + "main_node", dictionaries=["configs/dictionaries/cache_ints_dictionary.xml"] +) @pytest.fixture(scope="module") @@ -19,13 +19,19 @@ def started_cluster(): cluster.start() dictionary_node.query("create database if not exists test;") dictionary_node.query("drop table if exists test.ints;") - dictionary_node.query("create table test.ints " - "(key UInt64, " - "i8 Int8, i16 Int16, i32 Int32, i64 Int64, " - "u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) " - "Engine = Memory;") - dictionary_node.query("insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);") - dictionary_node.query("insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);") + dictionary_node.query( + "create table test.ints " + "(key UInt64, " + "i8 Int8, i16 Int16, i32 Int32, i64 Int64, " + "u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) " + "Engine = Memory;" + ) + dictionary_node.query( + "insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);" + ) + dictionary_node.query( + "insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);" + ) yield cluster finally: @@ -34,24 +40,68 @@ def started_cluster(): # @pytest.mark.skip(reason="debugging") def test_default_reading(started_cluster): - assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + assert None != dictionary_node.get_process_pid( + "clickhouse" + ), "ClickHouse must be alive" # Key 0 is not in dictionary, so default value will be returned def test_helper(): - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'i8', toUInt64(13), toInt8(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'i16', toUInt64(13), toInt16(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'i32', toUInt64(13), toInt32(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'i64', toUInt64(13), toInt64(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'u8', toUInt64(13), toUInt8(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'u16', toUInt64(13), toUInt16(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'u32', toUInt64(13), toUInt32(42));").rstrip() - assert '42' == main_node.query("select dictGetOrDefault('experimental_dict', 'u64', toUInt64(13), toUInt64(42));").rstrip() + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i8', toUInt64(13), toInt8(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i16', toUInt64(13), toInt16(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i32', toUInt64(13), toInt32(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i64', toUInt64(13), toInt64(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u8', toUInt64(13), toUInt8(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u16', toUInt64(13), toUInt16(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u32', toUInt64(13), toUInt32(42));" + ).rstrip() + ) + assert ( + "42" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u64', toUInt64(13), toUInt64(42));" + ).rstrip() + ) test_helper() with PartitionManager() as pm, ClickHouseKiller(dictionary_node): - assert None == dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + assert None == dictionary_node.get_process_pid( + "clickhouse" + ), "ClickHouse must be alive" # Remove connection between main_node and dictionary for sure pm.heal_all() diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py index 3611d382b12..7acc26a66e0 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py @@ -1,5 +1,3 @@ - - import os import random import string @@ -11,14 +9,19 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__, name="string") -dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', dictionaries=['configs/dictionaries/cache_ints_dictionary.xml', - 'configs/dictionaries/cache_strings_default_settings.xml']) +dictionary_node = cluster.add_instance("dictionary_node", stay_alive=True) +main_node = cluster.add_instance( + "main_node", + dictionaries=[ + "configs/dictionaries/cache_ints_dictionary.xml", + "configs/dictionaries/cache_strings_default_settings.xml", + ], +) def get_random_string(string_length=8): alphabet = string.ascii_letters + string.digits - return ''.join((random.choice(alphabet) for _ in range(string_length))) + return "".join((random.choice(alphabet) for _ in range(string_length))) @pytest.fixture(scope="module") @@ -27,15 +30,23 @@ def started_cluster(): cluster.start() dictionary_node.query("CREATE DATABASE IF NOT EXISTS test;") dictionary_node.query("DROP TABLE IF EXISTS test.strings;") - dictionary_node.query(""" + dictionary_node.query( + """ CREATE TABLE test.strings (key UInt64, value String) ENGINE = Memory; - """) + """ + ) values_to_insert = ", ".join( - ["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)]) - dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert)) + [ + "({}, '{}')".format(1000000 + number, get_random_string()) + for number in range(100) + ] + ) + dictionary_node.query( + "INSERT INTO test.strings VALUES {}".format(values_to_insert) + ) yield cluster finally: @@ -44,7 +55,9 @@ def started_cluster(): # @pytest.mark.skip(reason="debugging") def test_return_real_values(started_cluster): - assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + assert None != dictionary_node.get_process_pid( + "clickhouse" + ), "ClickHouse must be alive" first_batch = """ SELECT count(*) diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py index cf2234c0601..05f638ec337 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get.py @@ -1,5 +1,3 @@ - - import time import pytest @@ -9,8 +7,10 @@ from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', dictionaries=['configs/dictionaries/cache_ints_dictionary.xml']) +dictionary_node = cluster.add_instance("dictionary_node", stay_alive=True) +main_node = cluster.add_instance( + "main_node", dictionaries=["configs/dictionaries/cache_ints_dictionary.xml"] +) @pytest.fixture(scope="module") @@ -19,13 +19,19 @@ def started_cluster(): cluster.start() dictionary_node.query("create database if not exists test;") dictionary_node.query("drop table if exists test.ints;") - dictionary_node.query("create table test.ints " - "(key UInt64, " - "i8 Int8, i16 Int16, i32 Int32, i64 Int64, " - "u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) " - "Engine = Memory;") - dictionary_node.query("insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);") - dictionary_node.query("insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);") + dictionary_node.query( + "create table test.ints " + "(key UInt64, " + "i8 Int8, i16 Int16, i32 Int32, i64 Int64, " + "u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) " + "Engine = Memory;" + ) + dictionary_node.query( + "insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);" + ) + dictionary_node.query( + "insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);" + ) yield cluster finally: @@ -34,17 +40,59 @@ def started_cluster(): # @pytest.mark.skip(reason="debugging") def test_simple_dict_get(started_cluster): - assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + assert None != dictionary_node.get_process_pid( + "clickhouse" + ), "ClickHouse must be alive" def test_helper(): - assert '7' == main_node.query("select dictGet('experimental_dict', 'i8', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'i16', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'i32', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'i64', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'u8', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'u16', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'u32', toUInt64(7));").rstrip(), "Wrong answer." - assert '7' == main_node.query("select dictGet('experimental_dict', 'u64', toUInt64(7));").rstrip(), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'i8', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'i16', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'i32', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'i64', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'u8', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'u16', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'u32', toUInt64(7));" + ).rstrip() + ), "Wrong answer." + assert ( + "7" + == main_node.query( + "select dictGet('experimental_dict', 'u64', toUInt64(7));" + ).rstrip() + ), "Wrong answer." test_helper() diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py index df36218fc7b..54c5976f295 100644 --- a/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py @@ -1,5 +1,3 @@ - - import time import pytest @@ -9,8 +7,10 @@ from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__, name="default") -dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) -main_node = cluster.add_instance('main_node', dictionaries=['configs/dictionaries/cache_ints_dictionary.xml']) +dictionary_node = cluster.add_instance("dictionary_node", stay_alive=True) +main_node = cluster.add_instance( + "main_node", dictionaries=["configs/dictionaries/cache_ints_dictionary.xml"] +) @pytest.fixture(scope="module") @@ -19,13 +19,19 @@ def started_cluster(): cluster.start() dictionary_node.query("create database if not exists test;") dictionary_node.query("drop table if exists test.ints;") - dictionary_node.query("create table test.ints " - "(key UInt64, " - "i8 Int8, i16 Int16, i32 Int32, i64 Int64, " - "u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) " - "Engine = Memory;") - dictionary_node.query("insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);") - dictionary_node.query("insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);") + dictionary_node.query( + "create table test.ints " + "(key UInt64, " + "i8 Int8, i16 Int16, i32 Int32, i64 Int64, " + "u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64) " + "Engine = Memory;" + ) + dictionary_node.query( + "insert into test.ints values (7, 7, 7, 7, 7, 7, 7, 7, 7);" + ) + dictionary_node.query( + "insert into test.ints values (5, 5, 5, 5, 5, 5, 5, 5, 5);" + ) yield cluster finally: @@ -34,17 +40,59 @@ def started_cluster(): # @pytest.mark.skip(reason="debugging") def test_simple_dict_get_or_default(started_cluster): - assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + assert None != dictionary_node.get_process_pid( + "clickhouse" + ), "ClickHouse must be alive" def test_helper(): - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'i8', toUInt64(5), toInt8(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'i16', toUInt64(5), toInt16(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'i32', toUInt64(5), toInt32(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'i64', toUInt64(5), toInt64(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'u8', toUInt64(5), toUInt8(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'u16', toUInt64(5), toUInt16(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'u32', toUInt64(5), toUInt32(42));").rstrip() - assert '5' == main_node.query("select dictGetOrDefault('experimental_dict', 'u64', toUInt64(5), toUInt64(42));").rstrip() + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i8', toUInt64(5), toInt8(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i16', toUInt64(5), toInt16(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i32', toUInt64(5), toInt32(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'i64', toUInt64(5), toInt64(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u8', toUInt64(5), toUInt8(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u16', toUInt64(5), toUInt16(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u32', toUInt64(5), toUInt32(42));" + ).rstrip() + ) + assert ( + "5" + == main_node.query( + "select dictGetOrDefault('experimental_dict', 'u64', toUInt64(5), toUInt64(42));" + ).rstrip() + ) test_helper() diff --git a/tests/integration/test_dictionary_custom_settings/http_server.py b/tests/integration/test_dictionary_custom_settings/http_server.py index bd5ce22dbac..8683e98af9c 100644 --- a/tests/integration/test_dictionary_custom_settings/http_server.py +++ b/tests/integration/test_dictionary_custom_settings/http_server.py @@ -9,9 +9,14 @@ from http.server import BaseHTTPRequestHandler, HTTPServer # Decorator used to see if authentication works for external dictionary who use a HTTP source. def check_auth(fn): def wrapper(req): - auth_header = req.headers.get('authorization', None) - api_key = req.headers.get('api-key', None) - if not auth_header or auth_header != 'Basic Zm9vOmJhcg==' or not api_key or api_key != 'secret': + auth_header = req.headers.get("authorization", None) + api_key = req.headers.get("api-key", None) + if ( + not auth_header + or auth_header != "Basic Zm9vOmJhcg==" + or not api_key + or api_key != "secret" + ): req.send_response(401) else: fn(req) @@ -35,15 +40,15 @@ def start_server(server_address, data_path, schema, cert_path, address_family): def __send_headers(self): self.send_response(200) - self.send_header('Content-type', 'text/csv') + self.send_header("Content-type", "text/csv") self.end_headers() def __send_data(self, only_ids=None): - with open(data_path, 'r') as fl: - reader = csv.reader(fl, delimiter='\t') + with open(data_path, "r") as fl: + reader = csv.reader(fl, delimiter="\t") for row in reader: if not only_ids or (row[0] in only_ids): - self.wfile.write(('\t'.join(row) + '\n').encode()) + self.wfile.write(("\t".join(row) + "\n").encode()) def __read_and_decode_post_ids(self): data = self.__read_and_decode_post_data() @@ -69,19 +74,29 @@ def start_server(server_address, data_path, schema, cert_path, address_family): HTTPServer.address_family = socket.AF_INET6 httpd = HTTPServer(server_address, TSVHTTPHandler) if schema == "https": - httpd.socket = ssl.wrap_socket(httpd.socket, certfile=cert_path, server_side=True) + httpd.socket = ssl.wrap_socket( + httpd.socket, certfile=cert_path, server_side=True + ) httpd.serve_forever() if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple HTTP server returns data from file") + parser = argparse.ArgumentParser( + description="Simple HTTP server returns data from file" + ) parser.add_argument("--host", default="localhost") parser.add_argument("--port", default=5555, type=int) parser.add_argument("--data-path", required=True) parser.add_argument("--schema", choices=("http", "https"), required=True) parser.add_argument("--cert-path", default="./fake_cert.pem") - parser.add_argument('--address-family', choices=("ipv4", "ipv6"), default="ipv4") + parser.add_argument("--address-family", choices=("ipv4", "ipv6"), default="ipv4") args = parser.parse_args() - start_server((args.host, args.port), args.data_path, args.schema, args.cert_path, args.address_family) + start_server( + (args.host, args.port), + args.data_path, + args.schema, + args.cert_path, + args.address_family, + ) diff --git a/tests/integration/test_dictionary_custom_settings/test.py b/tests/integration/test_dictionary_custom_settings/test.py index 0d337e8c00e..715219ceb87 100644 --- a/tests/integration/test_dictionary_custom_settings/test.py +++ b/tests/integration/test_dictionary_custom_settings/test.py @@ -4,15 +4,15 @@ import pytest from helpers.cluster import ClickHouseCluster DICTIONARY_FILES = [ - 'configs/dictionaries/FileSourceConfig.xml', - 'configs/dictionaries/ExecutableSourceConfig.xml', - 'configs/dictionaries/source.csv', - 'configs/dictionaries/HTTPSourceConfig.xml', - 'configs/dictionaries/ClickHouseSourceConfig.xml' + "configs/dictionaries/FileSourceConfig.xml", + "configs/dictionaries/ExecutableSourceConfig.xml", + "configs/dictionaries/source.csv", + "configs/dictionaries/HTTPSourceConfig.xml", + "configs/dictionaries/ClickHouseSourceConfig.xml", ] cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('node', dictionaries=DICTIONARY_FILES) +instance = cluster.add_instance("node", dictionaries=DICTIONARY_FILES) def prepare(): @@ -20,14 +20,22 @@ def prepare(): path = "/source.csv" script_dir = os.path.dirname(os.path.realpath(__file__)) - node.copy_file_to_container(os.path.join(script_dir, './http_server.py'), '/http_server.py') - node.copy_file_to_container(os.path.join(script_dir, 'configs/dictionaries/source.csv'), './source.csv') - node.exec_in_container([ - "bash", - "-c", - "python3 /http_server.py --data-path={tbl} --schema=http --host=localhost --port=5555".format( - tbl=path) - ], detach=True) + node.copy_file_to_container( + os.path.join(script_dir, "./http_server.py"), "/http_server.py" + ) + node.copy_file_to_container( + os.path.join(script_dir, "configs/dictionaries/source.csv"), "./source.csv" + ) + node.exec_in_container( + [ + "bash", + "-c", + "python3 /http_server.py --data-path={tbl} --schema=http --host=localhost --port=5555".format( + tbl=path + ), + ], + detach=True, + ) @pytest.fixture(scope="module") @@ -45,20 +53,30 @@ def test_work(start_cluster): instance.query("SYSTEM RELOAD DICTIONARIES") - assert query("SELECT dictGetString('test_file', 'first', toUInt64(1))") == "\\\'a\n" - assert query("SELECT dictGetString('test_file', 'second', toUInt64(1))") == "\"b\n" - assert query("SELECT dictGetString('test_executable', 'first', toUInt64(1))") == "\\\'a\n" - assert query("SELECT dictGetString('test_executable', 'second', toUInt64(1))") == "\"b\n" + assert query("SELECT dictGetString('test_file', 'first', toUInt64(1))") == "\\'a\n" + assert query("SELECT dictGetString('test_file', 'second', toUInt64(1))") == '"b\n' + assert ( + query("SELECT dictGetString('test_executable', 'first', toUInt64(1))") + == "\\'a\n" + ) + assert ( + query("SELECT dictGetString('test_executable', 'second', toUInt64(1))") + == '"b\n' + ) - caught_exception = '' + caught_exception = "" try: - instance.query("CREATE TABLE source (id UInt64, first String, second String, third String) ENGINE=TinyLog;") - instance.query("INSERT INTO default.source VALUES (1, 'aaa', 'bbb', 'cccc'), (2, 'ddd', 'eee', 'fff')") + instance.query( + "CREATE TABLE source (id UInt64, first String, second String, third String) ENGINE=TinyLog;" + ) + instance.query( + "INSERT INTO default.source VALUES (1, 'aaa', 'bbb', 'cccc'), (2, 'ddd', 'eee', 'fff')" + ) instance.query("SELECT dictGetString('test_clickhouse', 'second', toUInt64(1))") except Exception as e: caught_exception = str(e) assert caught_exception.find("Limit for result exceeded") != -1 - assert query("SELECT dictGetString('test_http', 'first', toUInt64(1))") == "\\\'a\n" - assert query("SELECT dictGetString('test_http', 'second', toUInt64(1))") == "\"b\n" + assert query("SELECT dictGetString('test_http', 'first', toUInt64(1))") == "\\'a\n" + assert query("SELECT dictGetString('test_http', 'second', toUInt64(1))") == '"b\n' diff --git a/tests/integration/test_dictionary_ddl_on_cluster/test.py b/tests/integration/test_dictionary_ddl_on_cluster/test.py index feca1532974..dc9d31d75bd 100644 --- a/tests/integration/test_dictionary_ddl_on_cluster/test.py +++ b/tests/integration/test_dictionary_ddl_on_cluster/test.py @@ -3,14 +3,26 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], - with_zookeeper=True) -ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], - with_zookeeper=True) -ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], - with_zookeeper=True) -ch4 = cluster.add_instance('ch4', main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], - with_zookeeper=True) +ch1 = cluster.add_instance( + "ch1", + main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True, +) +ch2 = cluster.add_instance( + "ch2", + main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True, +) +ch3 = cluster.add_instance( + "ch3", + main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True, +) +ch4 = cluster.add_instance( + "ch4", + main_configs=["configs/config.d/clusters.xml", "configs/config.d/ddl.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -18,7 +30,8 @@ def started_cluster(): try: cluster.start() ch1.query( - "CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key") + "CREATE TABLE sometbl ON CLUSTER 'cluster' (key UInt64, value String) ENGINE = MergeTree ORDER by key" + ) yield cluster finally: @@ -42,12 +55,19 @@ def test_dictionary_ddl_on_cluster(started_cluster): LAYOUT(FLAT()) SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'sometbl' DB 'default')) LIFETIME(10) - """) + """ + ) for num, node in enumerate([ch1, ch2, ch3, ch4]): assert node.query("SELECT count() from sometbl") == "1\n" - assert node.query( - "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' + assert ( + node.query( + "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format( + num + ) + ) + == node.name + "\n" + ) ch1.query("DETACH DICTIONARY default.somedict ON CLUSTER 'cluster'") @@ -59,8 +79,14 @@ def test_dictionary_ddl_on_cluster(started_cluster): for num, node in enumerate([ch1, ch2, ch3, ch4]): assert node.query("SELECT count() from sometbl") == "1\n" - assert node.query( - "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == node.name + '\n' + assert ( + node.query( + "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format( + num + ) + ) + == node.name + "\n" + ) for num, node in enumerate([ch1, ch2, ch3, ch4]): node.query("ALTER TABLE sometbl UPDATE value = 'new_key' WHERE 1") @@ -68,8 +94,14 @@ def test_dictionary_ddl_on_cluster(started_cluster): ch1.query("SYSTEM RELOAD DICTIONARY ON CLUSTER 'cluster' `default.somedict`") for num, node in enumerate([ch1, ch2, ch3, ch4]): - assert node.query( - "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format(num)) == 'new_key' + '\n' + assert ( + node.query( + "SELECT dictGetString('default.somedict', 'value', toUInt64({}))".format( + num + ) + ) + == "new_key" + "\n" + ) ch1.query("DROP DICTIONARY default.somedict ON CLUSTER 'cluster'") diff --git a/tests/integration/test_disabled_mysql_server/test.py b/tests/integration/test_disabled_mysql_server/test.py index d7977404c73..6a4df3fc0b4 100644 --- a/tests/integration/test_disabled_mysql_server/test.py +++ b/tests/integration/test_disabled_mysql_server/test.py @@ -10,7 +10,10 @@ from helpers.cluster import ClickHouseCluster, get_docker_compose_path from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_mysql=True) +clickhouse_node = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_mysql=True +) + @pytest.fixture(scope="module") def started_cluster(): @@ -22,17 +25,22 @@ def started_cluster(): class MySQLNodeInstance: - def __init__(self, started_cluster, user='root', password='clickhouse'): + def __init__(self, started_cluster, user="root", password="clickhouse"): self.user = user self.port = cluster.mysql_port self.hostname = cluster.mysql_ip self.password = password - self.mysql_connection = None # lazy init + self.mysql_connection = None # lazy init def alloc_connection(self): if self.mysql_connection is None: - self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, - port=self.port, autocommit=True) + self.mysql_connection = pymysql.connect( + user=self.user, + password=self.password, + host=self.hostname, + port=self.port, + autocommit=True, + ) return self.mysql_connection def query(self, execution_query): @@ -48,12 +56,22 @@ def test_disabled_mysql_server(started_cluster): with contextlib.closing(MySQLNodeInstance(started_cluster)) as mysql_node: mysql_node.query("DROP DATABASE IF EXISTS test_db_disabled;") mysql_node.query("CREATE DATABASE test_db_disabled;") - mysql_node.query("CREATE TABLE test_db_disabled.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + mysql_node.query( + "CREATE TABLE test_db_disabled.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) with PartitionManager() as pm: - clickhouse_node.query("CREATE DATABASE test_db_disabled ENGINE = MySQL('mysql57:3306', 'test_db_disabled', 'root', 'clickhouse')") - - pm._add_rule({'source': clickhouse_node.ip_address, 'destination_port': 3306, 'action': 'DROP'}) + clickhouse_node.query( + "CREATE DATABASE test_db_disabled ENGINE = MySQL('mysql57:3306', 'test_db_disabled', 'root', 'clickhouse')" + ) + + pm._add_rule( + { + "source": clickhouse_node.ip_address, + "destination_port": 3306, + "action": "DROP", + } + ) clickhouse_node.query("SELECT * FROM system.parts") clickhouse_node.query("SELECT * FROM system.mutations") clickhouse_node.query("SELECT * FROM system.graphite_retentions") diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index ad31be4284a..273a00adffe 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', stay_alive=True) +instance = cluster.add_instance("instance", stay_alive=True) @pytest.fixture(scope="module", autouse=True) @@ -16,13 +16,19 @@ def started_cluster(): def create_entities(): - instance.query("CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000") + instance.query( + "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000" + ) instance.query("CREATE USER u1 SETTINGS PROFILE s1") instance.query("CREATE ROLE rx SETTINGS PROFILE s1") instance.query("CREATE USER u2 IDENTIFIED BY 'qwerty' HOST LOCAL DEFAULT ROLE rx") instance.query("CREATE SETTINGS PROFILE s2 SETTINGS PROFILE s1 TO u2") - instance.query("CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a<1000 TO u1, u2") - instance.query("CREATE QUOTA q FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx") + instance.query( + "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a<1000 TO u1, u2" + ) + instance.query( + "CREATE QUOTA q FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx" + ) @pytest.fixture(autouse=True) @@ -38,21 +44,37 @@ def test_create(): create_entities() def check(): - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query( - "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" - assert instance.query( - "SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" - assert instance.query( - "SHOW CREATE QUOTA q") == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n" + assert ( + instance.query("SHOW CREATE USER u1") + == "CREATE USER u1 SETTINGS PROFILE s1\n" + ) + assert ( + instance.query("SHOW CREATE USER u2") + == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" + ) + assert ( + instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") + == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" + ) + assert ( + instance.query("SHOW CREATE QUOTA q") + == "CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 100 TO ALL EXCEPT rx\n" + ) assert instance.query("SHOW GRANTS FOR u1") == "" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx TO u2\n" - assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s1\n" + assert ( + instance.query("SHOW CREATE ROLE rx") + == "CREATE ROLE rx SETTINGS PROFILE s1\n" + ) assert instance.query("SHOW GRANTS FOR rx") == "" - assert instance.query( - "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n" - assert instance.query( - "SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE s1") + == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 123456789 MIN 100000000 MAX 200000000\n" + ) + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE s2") + == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" + ) check() instance.restart_clickhouse() # Check persistency @@ -70,22 +92,44 @@ def test_alter(): instance.query("ALTER ROLE rx SETTINGS PROFILE s2") instance.query("GRANT SELECT ON mydb.mytable TO u1") instance.query("GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION") - instance.query("ALTER SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY") + instance.query( + "ALTER SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY" + ) def check(): - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query( - "SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" - assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n" + assert ( + instance.query("SHOW CREATE USER u1") + == "CREATE USER u1 SETTINGS PROFILE s1\n" + ) + assert ( + instance.query("SHOW CREATE USER u2") + == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" + ) + assert ( + instance.query("SHOW GRANTS FOR u1") + == "GRANT SELECT ON mydb.mytable TO u1\n" + ) assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n" - assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n" + assert ( + instance.query("SHOW CREATE ROLE rx") + == "CREATE ROLE rx SETTINGS PROFILE s2\n" + ) assert instance.query("SHOW CREATE ROLE ry") == "CREATE ROLE ry\n" - assert instance.query("SHOW GRANTS FOR rx") == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n" - assert instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n" - assert instance.query( - "SHOW CREATE SETTINGS PROFILE s1") == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" - assert instance.query( - "SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" + assert ( + instance.query("SHOW GRANTS FOR rx") + == "GRANT SELECT ON mydb.* TO rx WITH GRANT OPTION\n" + ) + assert ( + instance.query("SHOW GRANTS FOR ry") == "GRANT rx TO ry WITH ADMIN OPTION\n" + ) + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE s1") + == "CREATE SETTINGS PROFILE s1 SETTINGS max_memory_usage = 987654321 READONLY\n" + ) + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE s2") + == "CREATE SETTINGS PROFILE s2 SETTINGS INHERIT s1 TO u2\n" + ) check() instance.restart_clickhouse() # Check persistency @@ -104,11 +148,20 @@ def test_drop(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" - assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n" - assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2") - assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error( - "SHOW CREATE ROW POLICY p ON mydb.mytable") - assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q") + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE s2") + == "CREATE SETTINGS PROFILE s2\n" + ) + assert "There is no user `u2`" in instance.query_and_get_error( + "SHOW CREATE USER u2" + ) + assert ( + "There is no row policy `p ON mydb.mytable`" + in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") + ) + assert "There is no quota `q`" in instance.query_and_get_error( + "SHOW CREATE QUOTA q" + ) check() instance.restart_clickhouse() # Check persistency diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index f80cccac1be..b82c35e617f 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -4,32 +4,59 @@ from helpers.cluster import ClickHouseCluster uuids = [] + @pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/storage_conf.xml"], with_nginx=True) - cluster.add_instance("node2", main_configs=["configs/storage_conf_web.xml"], with_nginx=True) - cluster.add_instance("node3", main_configs=["configs/storage_conf_web.xml"], with_nginx=True) + cluster.add_instance( + "node1", main_configs=["configs/storage_conf.xml"], with_nginx=True + ) + cluster.add_instance( + "node2", main_configs=["configs/storage_conf_web.xml"], with_nginx=True + ) + cluster.add_instance( + "node3", main_configs=["configs/storage_conf_web.xml"], with_nginx=True + ) cluster.start() node1 = cluster.instances["node1"] expected = "" global uuids for i in range(3): - node1.query(""" CREATE TABLE data{} (id Int32) ENGINE = MergeTree() ORDER BY id SETTINGS storage_policy = 'def';""".format(i)) - node1.query("INSERT INTO data{} SELECT number FROM numbers(500000 * {})".format(i, i + 1)) + node1.query( + """ CREATE TABLE data{} (id Int32) ENGINE = MergeTree() ORDER BY id SETTINGS storage_policy = 'def';""".format( + i + ) + ) + node1.query( + "INSERT INTO data{} SELECT number FROM numbers(500000 * {})".format( + i, i + 1 + ) + ) expected = node1.query("SELECT * FROM data{} ORDER BY id".format(i)) - metadata_path = node1.query("SELECT data_paths FROM system.tables WHERE name='data{}'".format(i)) - metadata_path = metadata_path[metadata_path.find('/'):metadata_path.rfind('/')+1] - print(f'Metadata: {metadata_path}') + metadata_path = node1.query( + "SELECT data_paths FROM system.tables WHERE name='data{}'".format(i) + ) + metadata_path = metadata_path[ + metadata_path.find("/") : metadata_path.rfind("/") + 1 + ] + print(f"Metadata: {metadata_path}") - node1.exec_in_container(['bash', '-c', - '/usr/bin/clickhouse static-files-disk-uploader --test-mode --url http://nginx:80/test1 --metadata-path {}'.format(metadata_path)], user='root') - parts = metadata_path.split('/') + node1.exec_in_container( + [ + "bash", + "-c", + "/usr/bin/clickhouse static-files-disk-uploader --test-mode --url http://nginx:80/test1 --metadata-path {}".format( + metadata_path + ), + ], + user="root", + ) + parts = metadata_path.split("/") uuids.append(parts[3]) - print(f'UUID: {parts[3]}') + print(f"UUID: {parts[3]}") yield cluster @@ -42,24 +69,40 @@ def test_usage(cluster, node_name): node1 = cluster.instances["node1"] node2 = cluster.instances[node_name] global uuids - assert(len(uuids) == 3) + assert len(uuids) == 3 for i in range(3): - node2.query(""" + node2.query( + """ ATTACH TABLE test{} UUID '{}' (id Int32) ENGINE = MergeTree() ORDER BY id SETTINGS storage_policy = 'web'; - """.format(i, uuids[i], i, i)) + """.format( + i, uuids[i], i, i + ) + ) result = node2.query("SELECT * FROM test{} settings max_threads=20".format(i)) result = node2.query("SELECT count() FROM test{}".format(i)) - assert(int(result) == 500000 * (i+1)) + assert int(result) == 500000 * (i + 1) - result = node2.query("SELECT id FROM test{} WHERE id % 56 = 3 ORDER BY id".format(i)) - assert(result == node1.query("SELECT id FROM data{} WHERE id % 56 = 3 ORDER BY id".format(i))) + result = node2.query( + "SELECT id FROM test{} WHERE id % 56 = 3 ORDER BY id".format(i) + ) + assert result == node1.query( + "SELECT id FROM data{} WHERE id % 56 = 3 ORDER BY id".format(i) + ) - result = node2.query("SELECT id FROM test{} WHERE id > 789999 AND id < 999999 ORDER BY id".format(i)) - assert(result == node1.query("SELECT id FROM data{} WHERE id > 789999 AND id < 999999 ORDER BY id".format(i))) + result = node2.query( + "SELECT id FROM test{} WHERE id > 789999 AND id < 999999 ORDER BY id".format( + i + ) + ) + assert result == node1.query( + "SELECT id FROM data{} WHERE id > 789999 AND id < 999999 ORDER BY id".format( + i + ) + ) node2.query("DROP TABLE test{}".format(i)) print(f"Ok {i}") @@ -69,19 +112,23 @@ def test_incorrect_usage(cluster): node1 = cluster.instances["node1"] node2 = cluster.instances["node3"] global uuids - node2.query(""" + node2.query( + """ ATTACH TABLE test0 UUID '{}' (id Int32) ENGINE = MergeTree() ORDER BY id SETTINGS storage_policy = 'web'; - """.format(uuids[0])) + """.format( + uuids[0] + ) + ) result = node2.query("SELECT count() FROM test0") - assert(int(result) == 500000) + assert int(result) == 500000 result = node2.query_and_get_error("ALTER TABLE test0 ADD COLUMN col1 Int32 first") - assert("Table is read-only" in result) + assert "Table is read-only" in result result = node2.query_and_get_error("TRUNCATE TABLE test0") - assert("Table is read-only" in result) + assert "Table is read-only" in result node2.query("DROP TABLE test0") diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index 35e900c3c9f..a26f80165e8 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -14,7 +14,12 @@ disk_types = { def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/storage.xml"], with_minio=True, with_hdfs=True) + cluster.add_instance( + "node", + main_configs=["configs/storage.xml"], + with_minio=True, + with_hdfs=True, + ) cluster.start() yield cluster finally: @@ -26,7 +31,7 @@ def test_different_types(cluster): response = node.query("SELECT * FROM system.disks") disks = response.split("\n") for disk in disks: - if disk == '': # skip empty line (after split at last position) + if disk == "": # skip empty line (after split at last position) continue fields = disk.split("\t") assert len(fields) >= 6 @@ -36,5 +41,7 @@ def test_different_types(cluster): def test_select_by_type(cluster): node = cluster.instances["node"] for name, disk_type in list(disk_types.items()): - assert node.query("SELECT name FROM system.disks WHERE type='" + disk_type + "'") == name + "\n" - + assert ( + node.query("SELECT name FROM system.disks WHERE type='" + disk_type + "'") + == name + "\n" + ) diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py index 0d36aaa23f4..cb51142d249 100644 --- a/tests/integration/test_distributed_backward_compatability/test.py +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -4,9 +4,19 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', - tag='20.8.9.6', stay_alive=True, with_installed_binary=True) -node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], user_configs=['configs/legacy.xml']) +node_old = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + image="yandex/clickhouse-server", + tag="20.8.9.6", + stay_alive=True, + with_installed_binary=True, +) +node_new = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/legacy.xml"], +) @pytest.fixture(scope="module") @@ -15,15 +25,19 @@ def started_cluster(): cluster.start() for node in (node_old, node_new): - node.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") + node.query( + "CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id" + ) node_old.query("INSERT INTO local_table VALUES (1, 'node1')") node_new.query("INSERT INTO local_table VALUES (2, 'node2')") node_old.query( - "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)" + ) node_new.query( - "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + "CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)" + ) yield cluster @@ -33,13 +47,15 @@ def started_cluster(): def test_distributed_in_tuple(started_cluster): query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))" - query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed" + query2 = ( + "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed" + ) assert node_old.query(query1) == "1\n" assert node_old.query(query2) == "1\n" assert node_new.query(query1) == "1\n" assert node_new.query(query2) == "1\n" - large_set = '(' + ','.join([str(i) for i in range(1000)]) + ')' + large_set = "(" + ",".join([str(i) for i in range(1000)]) + ")" query3 = "SELECT count() FROM distributed WHERE id IN " + large_set query4 = "SELECT sum(id IN {}) FROM distributed".format(large_set) assert node_old.query(query3) == "2\n" diff --git a/tests/integration/test_distributed_ddl/cluster.py b/tests/integration/test_distributed_ddl/cluster.py index f90d15ebd08..9b2f6622ada 100644 --- a/tests/integration/test_distributed_ddl/cluster.py +++ b/tests/integration/test_distributed_ddl/cluster.py @@ -8,6 +8,7 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV + class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): def __init__(self, base_path, config_dir, testcase_name): ClickHouseCluster.__init__(self, base_path, name=testcase_name) @@ -16,49 +17,82 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): def prepare(self, replace_hostnames_with_ips=True): try: - main_configs_files = ["clusters.xml", "zookeeper_session_timeout.xml", "macro.xml", "query_log.xml", - "ddl.xml"] - main_configs = [os.path.join(self.test_config_dir, "config.d", f) for f in main_configs_files] - user_configs = [os.path.join(self.test_config_dir, "users.d", f) for f in - ["restricted_user.xml", "query_log.xml"]] + main_configs_files = [ + "clusters.xml", + "zookeeper_session_timeout.xml", + "macro.xml", + "query_log.xml", + "ddl.xml", + ] + main_configs = [ + os.path.join(self.test_config_dir, "config.d", f) + for f in main_configs_files + ] + user_configs = [ + os.path.join(self.test_config_dir, "users.d", f) + for f in ["restricted_user.xml", "query_log.xml"] + ] if self.test_config_dir == "configs_secure": - main_configs += [os.path.join(self.test_config_dir, f) for f in - ["server.crt", "server.key", "dhparam.pem", "config.d/ssl_conf.xml"]] + main_configs += [ + os.path.join(self.test_config_dir, f) + for f in [ + "server.crt", + "server.key", + "dhparam.pem", + "config.d/ssl_conf.xml", + ] + ] for i in range(4): self.add_instance( - 'ch{}'.format(i + 1), + "ch{}".format(i + 1), main_configs=main_configs, user_configs=user_configs, macros={"layer": 0, "shard": i // 2 + 1, "replica": i % 2 + 1}, - with_zookeeper=True) + with_zookeeper=True, + ) self.start() # Replace config files for testing ability to set host in DNS and IP formats if replace_hostnames_with_ips: - self.replace_domains_to_ip_addresses_in_cluster_config(['ch1', 'ch3']) + self.replace_domains_to_ip_addresses_in_cluster_config(["ch1", "ch3"]) # Select sacrifice instance to test CONNECTION_LOSS and server fail on it - sacrifice = self.instances['ch4'] + sacrifice = self.instances["ch4"] self.pm_random_drops = PartitionManager() self.pm_random_drops._add_rule( - {'probability': 0.01, 'destination': sacrifice.ip_address, 'source_port': 2181, - 'action': 'REJECT --reject-with tcp-reset'}) + { + "probability": 0.01, + "destination": sacrifice.ip_address, + "source_port": 2181, + "action": "REJECT --reject-with tcp-reset", + } + ) self.pm_random_drops._add_rule( - {'probability': 0.01, 'source': sacrifice.ip_address, 'destination_port': 2181, - 'action': 'REJECT --reject-with tcp-reset'}) + { + "probability": 0.01, + "source": sacrifice.ip_address, + "destination_port": 2181, + "action": "REJECT --reject-with tcp-reset", + } + ) # Initialize databases and service tables - instance = self.instances['ch1'] + instance = self.instances["ch1"] - self.ddl_check_query(instance, """ + self.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS all_tables ON CLUSTER 'cluster_no_replicas' (database String, name String, engine String, metadata_modification_time DateTime) ENGINE = Distributed('cluster_no_replicas', 'system', 'tables') - """) + """, + ) - self.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test ON CLUSTER 'cluster'") + self.ddl_check_query( + instance, "CREATE DATABASE IF NOT EXISTS test ON CLUSTER 'cluster'" + ) except Exception as e: print(e) @@ -77,7 +111,9 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): codes = [l[2] for l in M] messages = [l[3] for l in M] - assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, "\n" + tsv_content + assert len(hosts) == num_hosts and len(set(hosts)) == num_hosts, ( + "\n" + tsv_content + ) assert len(set(codes)) == 1, "\n" + tsv_content assert codes[0] == "0", "\n" + tsv_content @@ -87,7 +123,11 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): return contents def replace_domains_to_ip_addresses_in_cluster_config(self, instances_to_replace): - clusters_config = open(p.join(self.base_dir, '{}/config.d/clusters.xml'.format(self.test_config_dir))).read() + clusters_config = open( + p.join( + self.base_dir, "{}/config.d/clusters.xml".format(self.test_config_dir) + ) + ).read() for inst_name, inst in list(self.instances.items()): clusters_config = clusters_config.replace(inst_name, str(inst.ip_address)) @@ -95,16 +135,23 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): for inst_name in instances_to_replace: inst = self.instances[inst_name] self.instances[inst_name].exec_in_container( - ['bash', '-c', 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml'], - environment={"NEW_CONFIG": clusters_config}, privileged=True) + [ + "bash", + "-c", + 'echo "$NEW_CONFIG" > /etc/clickhouse-server/config.d/clusters.xml', + ], + environment={"NEW_CONFIG": clusters_config}, + privileged=True, + ) # print cluster.instances[inst_name].exec_in_container(['cat', "/etc/clickhouse-server/config.d/clusters.xml"]) @staticmethod def ddl_check_there_are_no_dublicates(instance): query = "SELECT max(c), argMax(q, c) FROM (SELECT lower(query) AS q, count() AS c FROM system.query_log WHERE type=2 AND q LIKE '/* ddl_entry=query-%' GROUP BY query)" rows = instance.query(query) - assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}: {}".format(instance.name, - instance.ip_address, rows) + assert len(rows) > 0 and rows[0][0] == "1", "dublicates on {} {}: {}".format( + instance.name, instance.ip_address, rows + ) @staticmethod def insert_reliable(instance, query_insert): @@ -119,7 +166,10 @@ class ClickHouseClusterWithDDLHelpers(ClickHouseCluster): except Exception as e: last_exception = e s = str(e) - if not (s.find('Unknown status, client must retry') >= 0 or s.find('zkutil::KeeperException')): + if not ( + s.find("Unknown status, client must retry") >= 0 + or s.find("zkutil::KeeperException") + ): raise e raise last_exception diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 18e091de1ec..9270efdd29b 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -21,9 +21,11 @@ def test_cluster(request): yield cluster - instance = cluster.instances['ch1'] + instance = cluster.instances["ch1"] cluster.ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") - cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") + cluster.ddl_check_query( + instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'" + ) # Check query log to ensure that DDL queries are not executed twice time.sleep(1.5) @@ -37,233 +39,379 @@ def test_cluster(request): def test_default_database(test_cluster): - instance = test_cluster.instances['ch3'] + instance = test_cluster.instances["ch3"] - test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, - "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null") + test_cluster.ddl_check_query( + instance, "CREATE DATABASE IF NOT EXISTS test2 ON CLUSTER 'cluster' FORMAT TSV" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS null ON CLUSTER 'cluster' FORMAT TSV" + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE null ON CLUSTER 'cluster2' (s String DEFAULT 'escape\t\nme') ENGINE = Null", + ) - contents = instance.query("SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h") + contents = instance.query( + "SELECT hostName() AS h, database FROM all_tables WHERE name = 'null' ORDER BY h" + ) assert TSV(contents) == TSV("ch1\tdefault\nch2\ttest2\nch3\tdefault\nch4\ttest2\n") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS null ON CLUSTER cluster2") - test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS null ON CLUSTER cluster2" + ) + test_cluster.ddl_check_query( + instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'" + ) def test_create_view(test_cluster): - instance = test_cluster.instances['ch3'] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view ON CLUSTER 'cluster'") - test_cluster.ddl_check_query(instance, - "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") - test_cluster.ddl_check_query(instance, - "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV") - test_cluster.ddl_check_query(instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, - "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV") + instance = test_cluster.instances["ch3"] + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test.super_simple_view ON CLUSTER 'cluster'" + ) + test_cluster.ddl_check_query( + instance, + "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV", + ) + test_cluster.ddl_check_query( + instance, + "CREATE MATERIALIZED VIEW test.simple_mat_view ON CLUSTER 'cluster' ENGINE = Memory AS SELECT * FROM system.numbers FORMAT TSV", + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE test.simple_mat_view ON CLUSTER 'cluster' FORMAT TSV" + ) + test_cluster.ddl_check_query( + instance, + "DROP TABLE IF EXISTS test.super_simple_view2 ON CLUSTER 'cluster' FORMAT TSV", + ) - test_cluster.ddl_check_query(instance, - "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory") - test_cluster.ddl_check_query(instance, - "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV") - test_cluster.ddl_check_query(instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'") + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test.super_simple ON CLUSTER 'cluster' (i Int8) ENGINE = Memory", + ) + test_cluster.ddl_check_query( + instance, + "RENAME TABLE test.super_simple TO test.super_simple2 ON CLUSTER 'cluster' FORMAT TSV", + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE test.super_simple2 ON CLUSTER 'cluster'" + ) def test_on_server_fail(test_cluster): - instance = test_cluster.instances['ch1'] - kill_instance = test_cluster.instances['ch2'] + instance = test_cluster.instances["ch1"] + kill_instance = test_cluster.instances["ch2"] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test.test_server_fail ON CLUSTER 'cluster'" + ) kill_instance.get_docker_handle().stop() - request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", - timeout=180) + request = instance.get_query_request( + "CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", + timeout=180, + ) kill_instance.get_docker_handle().start() - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'" + ) # Check query itself test_cluster.check_all_hosts_successfully_executed(request.get_answer()) # And check query artefacts contents = instance.query( - "SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h") + "SELECT hostName() AS h FROM all_tables WHERE database='test' AND name='test_server_fail' ORDER BY h" + ) assert TSV(contents) == TSV("ch1\nch2\nch3\nch4\n") - test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") + test_cluster.ddl_check_query( + instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'" + ) def test_simple_alters(test_cluster): - instance = test_cluster.instances['ch2'] + instance = test_cluster.instances["ch2"] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge ON CLUSTER '{cluster}'") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER '{cluster}'") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS merge ON CLUSTER '{cluster}'" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER '{cluster}'" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER '{cluster}'" + ) - test_cluster.ddl_check_query(instance, """ + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS merge ON CLUSTER '{cluster}' (p Date, i Int32) ENGINE = MergeTree(p, p, 1) -""") - test_cluster.ddl_check_query(instance, """ +""", + ) + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER '{cluster}' (p Date, i Int32) ENGINE = Distributed('{cluster}', default, merge, i) -""") - test_cluster.ddl_check_query(instance, """ +""", + ) + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER '{cluster}' (p Date, i Int64, s String) ENGINE = Distributed('{cluster}', default, merge, i) -""") +""", + ) for i in range(0, 4, 2): k = (i / 2) * 2 - test_cluster.instances['ch{}'.format(i + 1)].query("INSERT INTO merge (i) VALUES ({})({})".format(k, k + 1)) + test_cluster.instances["ch{}".format(i + 1)].query( + "INSERT INTO merge (i) VALUES ({})({})".format(k, k + 1) + ) assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV( - ''.join(['{}\n'.format(x) for x in range(4)])) + "".join(["{}\n".format(x) for x in range(4)]) + ) time.sleep(5) - test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' MODIFY COLUMN i Int64") + test_cluster.ddl_check_query( + instance, "ALTER TABLE merge ON CLUSTER '{cluster}' MODIFY COLUMN i Int64" + ) time.sleep(5) - test_cluster.ddl_check_query(instance, - "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV") + test_cluster.ddl_check_query( + instance, + "ALTER TABLE merge ON CLUSTER '{cluster}' ADD COLUMN s String DEFAULT toString(i) FORMAT TSV", + ) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( - ''.join(['{}\t{}\n'.format(x, x) for x in range(4)])) + "".join(["{}\t{}\n".format(x, x) for x in range(4)]) + ) for i in range(0, 4, 2): k = (i / 2) * 2 + 4 - test_cluster.instances['ch{}'.format(i + 1)].query( - "INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k + 1)) + test_cluster.instances["ch{}".format(i + 1)].query( + "INSERT INTO merge (p, i) VALUES (31, {})(31, {})".format(k, k + 1) + ) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( - ''.join(['{}\t{}\n'.format(x, x) for x in range(8)])) + "".join(["{}\t{}\n".format(x, x) for x in range(8)]) + ) - test_cluster.ddl_check_query(instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002") + test_cluster.ddl_check_query( + instance, "ALTER TABLE merge ON CLUSTER '{cluster}' DETACH PARTITION 197002" + ) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( - ''.join(['{}\t{}\n'.format(x, x) for x in range(4)])) + "".join(["{}\t{}\n".format(x, x) for x in range(4)]) + ) test_cluster.ddl_check_query(instance, "DROP TABLE merge ON CLUSTER '{cluster}'") - test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'") - test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query( + instance, "DROP TABLE all_merge_32 ON CLUSTER '{cluster}'" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE all_merge_64 ON CLUSTER '{cluster}'" + ) def test_macro(test_cluster): - instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory") + instance = test_cluster.instances["ch2"] + test_cluster.ddl_check_query( + instance, + "CREATE TABLE tab ON CLUSTER '{cluster}' (value UInt8) ENGINE = Memory", + ) for i in range(4): - test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], - "INSERT INTO tab VALUES ({})".format(i)) + test_cluster.insert_reliable( + test_cluster.instances["ch{}".format(i + 1)], + "INSERT INTO tab VALUES ({})".format(i), + ) - test_cluster.ddl_check_query(instance, - "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)") + test_cluster.ddl_check_query( + instance, + "CREATE TABLE distr ON CLUSTER '{cluster}' (value UInt8) ENGINE = Distributed('{cluster}', 'default', 'tab', value % 4)", + ) - assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') - assert TSV(test_cluster.instances['ch3'].query("SELECT value FROM distr ORDER BY value")) == TSV('0\n1\n2\n3\n') + assert TSV(instance.query("SELECT value FROM distr ORDER BY value")) == TSV( + "0\n1\n2\n3\n" + ) + assert TSV( + test_cluster.instances["ch3"].query("SELECT value FROM distr ORDER BY value") + ) == TSV("0\n1\n2\n3\n") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS distr ON CLUSTER '{cluster}'") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS tab ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS distr ON CLUSTER '{cluster}'" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS tab ON CLUSTER '{cluster}'" + ) def test_implicit_macros(test_cluster): # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas firewall_drops_rules = test_cluster.pm_random_drops.pop_rules() - instance = test_cluster.instances['ch2'] + instance = test_cluster.instances["ch2"] - test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}' SYNC") - test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query( + instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}' SYNC" + ) + test_cluster.ddl_check_query( + instance, "CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER '{cluster}'" + ) - test_cluster.ddl_check_query(instance, """ + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS test_db.test_macro ON CLUSTER '{cluster}' (p Date, i Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/{layer}-{shard}/{table}', '{replica}', p, p, 1) -""") +""", + ) # Check that table was created at correct path in zookeeper - assert test_cluster.get_kazoo_client('zoo1').exists('/clickhouse/tables/test_db/0-1/test_macro') is not None + assert ( + test_cluster.get_kazoo_client("zoo1").exists( + "/clickhouse/tables/test_db/0-1/test_macro" + ) + is not None + ) # Enable random ZK packet drops test_cluster.pm_random_drops.push_rules(firewall_drops_rules) def test_allowed_databases(test_cluster): - instance = test_cluster.instances['ch2'] + instance = test_cluster.instances["ch2"] instance.query("CREATE DATABASE IF NOT EXISTS db1 ON CLUSTER cluster") instance.query("CREATE DATABASE IF NOT EXISTS db2 ON CLUSTER cluster") - instance.query("CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", - settings={"user": "restricted_user"}) + instance.query( + "CREATE TABLE db1.t1 ON CLUSTER cluster (i Int8) ENGINE = Memory", + settings={"user": "restricted_user"}, + ) with pytest.raises(Exception): - instance.query("CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", - settings={"user": "restricted_user"}) + instance.query( + "CREATE TABLE db2.t2 ON CLUSTER cluster (i Int8) ENGINE = Memory", + settings={"user": "restricted_user"}, + ) with pytest.raises(Exception): - instance.query("CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", - settings={"user": "restricted_user"}) + instance.query( + "CREATE TABLE t3 ON CLUSTER cluster (i Int8) ENGINE = Memory", + settings={"user": "restricted_user"}, + ) with pytest.raises(Exception): - instance.query("DROP DATABASE db2 ON CLUSTER cluster", settings={"user": "restricted_user"}) + instance.query( + "DROP DATABASE db2 ON CLUSTER cluster", settings={"user": "restricted_user"} + ) - instance.query("DROP DATABASE db1 ON CLUSTER cluster", settings={"user": "restricted_user"}) + instance.query( + "DROP DATABASE db1 ON CLUSTER cluster", settings={"user": "restricted_user"} + ) def test_kill_query(test_cluster): - instance = test_cluster.instances['ch3'] + instance = test_cluster.instances["ch3"] - test_cluster.ddl_check_query(instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV") + test_cluster.ddl_check_query( + instance, "KILL QUERY ON CLUSTER 'cluster' WHERE NOT elapsed FORMAT TSV" + ) def test_detach_query(test_cluster): - instance = test_cluster.instances['ch3'] + instance = test_cluster.instances["ch3"] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_attach ON CLUSTER cluster FORMAT TSV") - test_cluster.ddl_check_query(instance, "CREATE TABLE test_attach ON CLUSTER cluster (i Int8)ENGINE = Log") - test_cluster.ddl_check_query(instance, "DETACH TABLE test_attach ON CLUSTER cluster FORMAT TSV") - test_cluster.ddl_check_query(instance, "ATTACH TABLE test_attach ON CLUSTER cluster") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test_attach ON CLUSTER cluster FORMAT TSV" + ) + test_cluster.ddl_check_query( + instance, "CREATE TABLE test_attach ON CLUSTER cluster (i Int8)ENGINE = Log" + ) + test_cluster.ddl_check_query( + instance, "DETACH TABLE test_attach ON CLUSTER cluster FORMAT TSV" + ) + test_cluster.ddl_check_query( + instance, "ATTACH TABLE test_attach ON CLUSTER cluster" + ) def test_optimize_query(test_cluster): - instance = test_cluster.instances['ch3'] + instance = test_cluster.instances["ch3"] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)") - test_cluster.ddl_check_query(instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test_optimize ON CLUSTER cluster FORMAT TSV" + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_optimize ON CLUSTER cluster (p Date, i Int32) ENGINE = MergeTree(p, p, 8192)", + ) + test_cluster.ddl_check_query( + instance, "OPTIMIZE TABLE test_optimize ON CLUSTER cluster FORMAT TSV" + ) def test_create_as_select(test_cluster): - instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)") - assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV("1\n2\n") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster") + instance = test_cluster.instances["ch2"] + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_as_select ON CLUSTER cluster ENGINE = Memory AS (SELECT 1 AS x UNION ALL SELECT 2 AS x)", + ) + assert TSV(instance.query("SELECT x FROM test_as_select ORDER BY x")) == TSV( + "1\n2\n" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test_as_select ON CLUSTER cluster" + ) def test_create_reserved(test_cluster): - instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster") + instance = test_cluster.instances["ch2"] + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_reserved ON CLUSTER cluster (`p` Date, `image` Nullable(String), `index` Nullable(Float64), `invalidate` Nullable(Int64)) ENGINE = MergeTree(`p`, `p`, 8192)", + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_as_reserved ON CLUSTER cluster ENGINE = Memory AS (SELECT * from test_reserved)", + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test_reserved ON CLUSTER cluster" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test_as_reserved ON CLUSTER cluster" + ) def test_rename(test_cluster): - instance = test_cluster.instances['ch1'] + instance = test_cluster.instances["ch1"] rules = test_cluster.pm_random_drops.pop_rules() - test_cluster.ddl_check_query(instance, - "DROP TABLE IF EXISTS rename_shard ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, - "DROP TABLE IF EXISTS rename_new ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, - "DROP TABLE IF EXISTS rename_old ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, - "DROP TABLE IF EXISTS rename ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS rename_shard ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS rename_new ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS rename_old ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS rename ON CLUSTER cluster SYNC" + ) - test_cluster.ddl_check_query(instance, - "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)") - test_cluster.ddl_check_query(instance, - "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") - test_cluster.ddl_check_query(instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;") + test_cluster.ddl_check_query( + instance, + "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)", + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)", + ) + test_cluster.ddl_check_query( + instance, "RENAME TABLE rename_new TO rename ON CLUSTER cluster;" + ) for i in range(10): instance.query("insert into rename (id) values ({})".format(i)) @@ -275,87 +423,165 @@ def test_rename(test_cluster): # because path of lock in zk contains shard name, which is list of host names of replicas instance.query( "ALTER TABLE rename_shard ON CLUSTER cluster MODIFY COLUMN sid String DEFAULT concat('new', toString(id))", - ignore_error=True) + ignore_error=True, + ) time.sleep(1) - test_cluster.ddl_check_query(instance, - "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)") + test_cluster.ddl_check_query( + instance, + "CREATE TABLE rename_new ON CLUSTER cluster AS rename_shard ENGINE = Distributed(cluster, default, rename_shard, id % 2)", + ) instance.query("system stop distributed sends rename") for i in range(10, 20): instance.query("insert into rename (id) values ({})".format(i)) - test_cluster.ddl_check_query(instance, "RENAME TABLE rename TO rename_old, rename_new TO rename ON CLUSTER cluster") + test_cluster.ddl_check_query( + instance, + "RENAME TABLE rename TO rename_old, rename_new TO rename ON CLUSTER cluster", + ) for i in range(20, 30): instance.query("insert into rename (id) values ({})".format(i)) instance.query("system flush distributed rename") - for name in ['ch1', 'ch2', 'ch3', 'ch4']: + for name in ["ch1", "ch2", "ch3", "ch4"]: test_cluster.instances[name].query("system sync replica rename_shard") # system stop distributed sends does not affect inserts into local shard, # so some ids in range (10, 20) will be inserted into rename_shard assert instance.query("select count(id), sum(id) from rename").rstrip() == "25\t360" # assert instance.query("select count(id), sum(id) from rename").rstrip() == "20\t290" - assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "15\t115" + assert ( + instance.query( + "select count(id), sum(id) from rename where sid like 'old%'" + ).rstrip() + == "15\t115" + ) # assert instance.query("select count(id), sum(id) from rename where sid like 'old%'").rstrip() == "10\t45" - assert instance.query("select count(id), sum(id) from rename where sid like 'new%'").rstrip() == "10\t245" + assert ( + instance.query( + "select count(id), sum(id) from rename where sid like 'new%'" + ).rstrip() + == "10\t245" + ) test_cluster.pm_random_drops.push_rules(rules) def test_socket_timeout(test_cluster): - instance = test_cluster.instances['ch1'] + instance = test_cluster.instances["ch1"] # queries should not fail with "Timeout exceeded while reading from socket" in case of EINTR caused by query profiler for i in range(0, 100): - instance.query("select hostName() as host, count() from cluster('cluster', 'system', 'settings') group by host") + instance.query( + "select hostName() as host, count() from cluster('cluster', 'system', 'settings') group by host" + ) def test_replicated_without_arguments(test_cluster): rules = test_cluster.pm_random_drops.pop_rules() - instance = test_cluster.instances['ch1'] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_atomic.rmt ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test_atomic ON CLUSTER cluster SYNC") + instance = test_cluster.instances["ch1"] + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS test_atomic.rmt ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP DATABASE IF EXISTS test_atomic ON CLUSTER cluster SYNC" + ) - test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic") - assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ - instance.query_and_get_error("CREATE TABLE test_atomic.rmt (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree() ORDER BY n") - test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt UUID '12345678-0000-4000-8000-000000000001' ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") - assert instance.query("SHOW CREATE test_atomic.rmt FORMAT TSVRaw") == \ - "CREATE TABLE test_atomic.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/clickhouse/tables/12345678-0000-4000-8000-000000000001/{shard}', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n" - test_cluster.ddl_check_query(instance, "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY n") - test_cluster.ddl_check_query(instance, - "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster") - assert instance.query("SELECT countDistinct(uuid) from clusterAllReplicas('cluster', 'system', 'databases') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='test_atomic'") == "1\n" - assert instance.query("SELECT countDistinct(uuid) from clusterAllReplicas('cluster', 'system', 'tables') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='rmt'") == "1\n" - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rrmt ON CLUSTER cluster (n UInt64, m UInt64) ENGINE=ReplicatedReplacingMergeTree(m) ORDER BY n") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rsmt ON CLUSTER cluster (n UInt64, m UInt64, k UInt64) ENGINE=ReplicatedSummingMergeTree((m, k)) ORDER BY n") - test_cluster.ddl_check_query(instance, - "CREATE TABLE test_atomic.rvcmt ON CLUSTER cluster (n UInt64, m Int8, k UInt64) ENGINE=ReplicatedVersionedCollapsingMergeTree(m, k) ORDER BY n") - test_cluster.ddl_check_query(instance, "DROP DATABASE test_atomic ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query( + instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic" + ) + assert ( + "are supported only for ON CLUSTER queries with Atomic database engine" + in instance.query_and_get_error( + "CREATE TABLE test_atomic.rmt (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n" + ) + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree() ORDER BY n", + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_atomic.rmt UUID '12345678-0000-4000-8000-000000000001' ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n", + ) + assert ( + instance.query("SHOW CREATE test_atomic.rmt FORMAT TSVRaw") + == "CREATE TABLE test_atomic.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/clickhouse/tables/12345678-0000-4000-8000-000000000001/{shard}', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n" + ) + test_cluster.ddl_check_query( + instance, + "RENAME TABLE test_atomic.rmt TO test_atomic.rmt_renamed ON CLUSTER cluster", + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY n", + ) + test_cluster.ddl_check_query( + instance, + "EXCHANGE TABLES test_atomic.rmt AND test_atomic.rmt_renamed ON CLUSTER cluster", + ) + assert ( + instance.query( + "SELECT countDistinct(uuid) from clusterAllReplicas('cluster', 'system', 'databases') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='test_atomic'" + ) + == "1\n" + ) + assert ( + instance.query( + "SELECT countDistinct(uuid) from clusterAllReplicas('cluster', 'system', 'tables') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='rmt'" + ) + == "1\n" + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_atomic.rrmt ON CLUSTER cluster (n UInt64, m UInt64) ENGINE=ReplicatedReplacingMergeTree(m) ORDER BY n", + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_atomic.rsmt ON CLUSTER cluster (n UInt64, m UInt64, k UInt64) ENGINE=ReplicatedSummingMergeTree((m, k)) ORDER BY n", + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_atomic.rvcmt ON CLUSTER cluster (n UInt64, m Int8, k UInt64) ENGINE=ReplicatedVersionedCollapsingMergeTree(m, k) ORDER BY n", + ) + test_cluster.ddl_check_query( + instance, "DROP DATABASE test_atomic ON CLUSTER cluster SYNC" + ) - test_cluster.ddl_check_query(instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary") - assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ - instance.query_and_get_error("CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") - assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ - instance.query_and_get_error("CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n") - test_cluster.ddl_check_query(instance, "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n") - assert instance.query("SHOW CREATE test_ordinary.rmt FORMAT TSVRaw") == \ - "CREATE TABLE test_ordinary.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/{shard}/rmt/', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n" - test_cluster.ddl_check_query(instance, "DROP DATABASE test_ordinary ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query( + instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary" + ) + assert ( + "are supported only for ON CLUSTER queries with Atomic database engine" + in instance.query_and_get_error( + "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n" + ) + ) + assert ( + "are supported only for ON CLUSTER queries with Atomic database engine" + in instance.query_and_get_error( + "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n" + ) + ) + test_cluster.ddl_check_query( + instance, + "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n", + ) + assert ( + instance.query("SHOW CREATE test_ordinary.rmt FORMAT TSVRaw") + == "CREATE TABLE test_ordinary.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/{shard}/rmt/', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n" + ) + test_cluster.ddl_check_query( + instance, "DROP DATABASE test_ordinary ON CLUSTER cluster SYNC" + ) test_cluster.pm_random_drops.push_rules(rules) -if __name__ == '__main__': +if __name__ == "__main__": with contextmanager(test_cluster)() as ctx_cluster: for name, instance in list(ctx_cluster.instances.items()): print(name, instance.ip_address) diff --git a/tests/integration/test_distributed_ddl/test_replicated_alter.py b/tests/integration/test_distributed_ddl/test_replicated_alter.py index 5e7989cb256..08d2c1da278 100644 --- a/tests/integration/test_distributed_ddl/test_replicated_alter.py +++ b/tests/integration/test_distributed_ddl/test_replicated_alter.py @@ -12,7 +12,9 @@ from .cluster import ClickHouseClusterWithDDLHelpers @pytest.fixture(scope="module", params=["configs", "configs_secure"]) def test_cluster(request): - cluster = ClickHouseClusterWithDDLHelpers(__file__, request.param, "alters_" + request.param) + cluster = ClickHouseClusterWithDDLHelpers( + __file__, request.param, "alters_" + request.param + ) try: # TODO: Fix ON CLUSTER alters when nodes have different configs. Need to canonicalize node identity. @@ -20,9 +22,11 @@ def test_cluster(request): yield cluster - instance = cluster.instances['ch1'] + instance = cluster.instances["ch1"] cluster.ddl_check_query(instance, "DROP DATABASE test ON CLUSTER 'cluster'") - cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'") + cluster.ddl_check_query( + instance, "DROP DATABASE IF EXISTS test2 ON CLUSTER 'cluster'" + ) # Check query log to ensure that DDL queries are not executed twice time.sleep(1.5) @@ -36,64 +40,102 @@ def test_cluster(request): def test_replicated_alters(test_cluster): - instance = test_cluster.instances['ch2'] + instance = test_cluster.instances["ch2"] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster SYNC" + ) # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas firewall_drops_rules = test_cluster.pm_random_drops.pop_rules() - test_cluster.ddl_check_query(instance, """ + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS merge_for_alter ON CLUSTER cluster (p Date, i Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/hits', '{replica}', p, p, 1) -""") +""", + ) - test_cluster.ddl_check_query(instance, """ + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS all_merge_32 ON CLUSTER cluster (p Date, i Int32) ENGINE = Distributed(cluster, default, merge_for_alter, i) -""") - test_cluster.ddl_check_query(instance, """ +""", + ) + test_cluster.ddl_check_query( + instance, + """ CREATE TABLE IF NOT EXISTS all_merge_64 ON CLUSTER cluster (p Date, i Int64, s String) ENGINE = Distributed(cluster, default, merge_for_alter, i) -""") +""", + ) for i in range(4): k = (i // 2) * 2 - test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], - "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k + 1)) + test_cluster.insert_reliable( + test_cluster.instances["ch{}".format(i + 1)], + "INSERT INTO merge_for_alter (i) VALUES ({})({})".format(k, k + 1), + ) test_cluster.sync_replicas("merge_for_alter") assert TSV(instance.query("SELECT i FROM all_merge_32 ORDER BY i")) == TSV( - ''.join(['{}\n'.format(x) for x in range(4)])) + "".join(["{}\n".format(x) for x in range(4)]) + ) - test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64") - test_cluster.ddl_check_query(instance, - "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)") + test_cluster.ddl_check_query( + instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster MODIFY COLUMN i Int64" + ) + test_cluster.ddl_check_query( + instance, + "ALTER TABLE merge_for_alter ON CLUSTER cluster ADD COLUMN s String DEFAULT toString(i)", + ) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( - ''.join(['{}\t{}\n'.format(x, x) for x in range(4)])) + "".join(["{}\t{}\n".format(x, x) for x in range(4)]) + ) for i in range(4): k = (i // 2) * 2 + 4 - test_cluster.insert_reliable(test_cluster.instances['ch{}'.format(i + 1)], - "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format(k, k + 1)) + test_cluster.insert_reliable( + test_cluster.instances["ch{}".format(i + 1)], + "INSERT INTO merge_for_alter (p, i) VALUES (31, {})(31, {})".format( + k, k + 1 + ), + ) test_cluster.sync_replicas("merge_for_alter") assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( - ''.join(['{}\t{}\n'.format(x, x) for x in range(8)])) + "".join(["{}\t{}\n".format(x, x) for x in range(8)]) + ) - test_cluster.ddl_check_query(instance, "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002") + test_cluster.ddl_check_query( + instance, + "ALTER TABLE merge_for_alter ON CLUSTER cluster DETACH PARTITION 197002", + ) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( - ''.join(['{}\t{}\n'.format(x, x) for x in range(4)])) + "".join(["{}\t{}\n".format(x, x) for x in range(4)]) + ) - test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query( + instance, "DROP TABLE merge_for_alter ON CLUSTER cluster SYNC" + ) # Enable random ZK packet drops test_cluster.pm_random_drops.push_rules(firewall_drops_rules) - test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster SYNC") - test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query( + instance, "DROP TABLE all_merge_32 ON CLUSTER cluster SYNC" + ) + test_cluster.ddl_check_query( + instance, "DROP TABLE all_merge_64 ON CLUSTER cluster SYNC" + ) diff --git a/tests/integration/test_distributed_ddl_on_cross_replication/test.py b/tests/integration/test_distributed_ddl_on_cross_replication/test.py index 833a3fb1f04..b89091d4034 100644 --- a/tests/integration/test_distributed_ddl_on_cross_replication/test.py +++ b/tests/integration/test_distributed_ddl_on_cross_replication/test.py @@ -4,12 +4,24 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, - macros={"shard": 1, "replica": 1, "shard_bk": 3, "replica_bk": 2}) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, - macros={"shard": 2, "replica": 1, "shard_bk": 1, "replica_bk": 2}) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, - macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2}) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, + macros={"shard": 1, "replica": 1, "shard_bk": 3, "replica_bk": 2}, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, + macros={"shard": 2, "replica": 1, "shard_bk": 1, "replica_bk": 2}, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, + macros={"shard": 3, "replica": 1, "shard_bk": 2, "replica_bk": 2}, +) @pytest.fixture(scope="module") @@ -17,7 +29,8 @@ def started_cluster(): try: cluster.start() - node1.query(''' + node1.query( + """ CREATE DATABASE replica_1 ON CLUSTER cross_3shards_2replicas; CREATE DATABASE replica_2 ON CLUSTER cross_3shards_2replicas; @@ -38,19 +51,23 @@ def started_cluster(): CREATE TABLE replica_2.replicated ON CLUSTER cross_3shards_2replicas as replica_2.replicated_local ENGINE = Distributed(cross_3shards_2replicas, '', replicated_local, shard_id); - ''') + """ + ) - to_insert = '''\ + to_insert = """\ 2017-06-16 10 0 2017-06-17 11 0 2017-06-16 20 1 2017-06-17 21 1 2017-06-16 30 2 2017-06-17 31 2 -''' +""" - node1.query("INSERT INTO replica_1.replicated FORMAT TSV", stdin=to_insert, - settings={"insert_distributed_sync": 1}) + node1.query( + "INSERT INTO replica_1.replicated FORMAT TSV", + stdin=to_insert, + settings={"insert_distributed_sync": 1}, + ) yield cluster finally: @@ -59,48 +76,93 @@ def started_cluster(): def test_alter_ddl(started_cluster): - node1.query("ALTER TABLE replica_1.replicated_local \ + node1.query( + "ALTER TABLE replica_1.replicated_local \ ON CLUSTER cross_3shards_2replicas \ UPDATE shard_id=shard_id+3 \ - WHERE part_key='2017-06-16'") + WHERE part_key='2017-06-16'" + ) node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) - assert_eq_with_retry(node1, - "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", '3') + assert_eq_with_retry( + node1, + "SELECT count(*) FROM replica_2.replicated where shard_id >= 3 and part_key='2017-06-16'", + "3", + ) - node1.query("ALTER TABLE replica_1.replicated_local \ - ON CLUSTER cross_3shards_2replicas DELETE WHERE shard_id >=3;") + node1.query( + "ALTER TABLE replica_1.replicated_local \ + ON CLUSTER cross_3shards_2replicas DELETE WHERE shard_id >=3;" + ) node1.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) - assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3", '0') + assert_eq_with_retry( + node1, "SELECT count(*) FROM replica_2.replicated where shard_id >= 3", "0" + ) - node2.query("ALTER TABLE replica_1.replicated_local ON CLUSTER cross_3shards_2replicas DROP PARTITION '2017-06-17'") + node2.query( + "ALTER TABLE replica_1.replicated_local ON CLUSTER cross_3shards_2replicas DROP PARTITION '2017-06-17'" + ) node2.query("SYSTEM SYNC REPLICA replica_2.replicated_local;", timeout=5) - assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", '0') + assert_eq_with_retry(node1, "SELECT count(*) FROM replica_2.replicated", "0") + def test_atomic_database(started_cluster): - node1.query('''DROP DATABASE IF EXISTS replica_1 ON CLUSTER cross_3shards_2replicas; + node1.query( + """DROP DATABASE IF EXISTS replica_1 ON CLUSTER cross_3shards_2replicas; DROP DATABASE IF EXISTS replica_2 ON CLUSTER cross_3shards_2replicas; CREATE DATABASE replica_1 ON CLUSTER cross_3shards_2replicas ENGINE=Atomic; - CREATE DATABASE replica_2 ON CLUSTER cross_3shards_2replicas ENGINE=Atomic;''') + CREATE DATABASE replica_2 ON CLUSTER cross_3shards_2replicas ENGINE=Atomic;""" + ) - assert "It's not supported for cross replication" in \ - node1.query_and_get_error("CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") - assert "It's not supported for cross replication" in \ - node1.query_and_get_error("CREATE TABLE replica_1.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") - assert "It's not supported for cross replication" in \ - node1.query_and_get_error("CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n") - assert "It's not supported for cross replication" in \ - node1.query_and_get_error("CREATE TABLE replica_2.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n") - assert "For a distributed DDL on circular replicated cluster its table name must be qualified by database name" in \ - node1.query_and_get_error("CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard}/rmt/', '{replica}') ORDER BY n") + assert "It's not supported for cross replication" in node1.query_and_get_error( + "CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n" + ) + assert "It's not supported for cross replication" in node1.query_and_get_error( + "CREATE TABLE replica_1.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n" + ) + assert "It's not supported for cross replication" in node1.query_and_get_error( + "CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n" + ) + assert "It's not supported for cross replication" in node1.query_and_get_error( + "CREATE TABLE replica_2.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{uuid}/', '{replica}') ORDER BY n" + ) + assert ( + "For a distributed DDL on circular replicated cluster its table name must be qualified by database name" + in node1.query_and_get_error( + "CREATE TABLE rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard}/rmt/', '{replica}') ORDER BY n" + ) + ) - node1.query("CREATE TABLE replica_1.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard}/rmt/', '{replica}') ORDER BY n") - node1.query("CREATE TABLE replica_2.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard_bk}/rmt/', '{replica_bk}') ORDER BY n") + node1.query( + "CREATE TABLE replica_1.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard}/rmt/', '{replica}') ORDER BY n" + ) + node1.query( + "CREATE TABLE replica_2.rmt ON CLUSTER cross_3shards_2replicas (n UInt64, s String) ENGINE=ReplicatedMergeTree('/tables/{shard_bk}/rmt/', '{replica_bk}') ORDER BY n" + ) - assert node1.query("SELECT countDistinct(uuid) from remote('node1,node2,node3', 'system', 'databases') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='replica_1'") == "1\n" - assert node1.query("SELECT countDistinct(uuid) from remote('node1,node2,node3', 'system', 'tables') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='rmt'") == "2\n" + assert ( + node1.query( + "SELECT countDistinct(uuid) from remote('node1,node2,node3', 'system', 'databases') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='replica_1'" + ) + == "1\n" + ) + assert ( + node1.query( + "SELECT countDistinct(uuid) from remote('node1,node2,node3', 'system', 'tables') WHERE uuid != '00000000-0000-0000-0000-000000000000' AND name='rmt'" + ) + == "2\n" + ) node1.query("INSERT INTO replica_1.rmt VALUES (1, 'test')") node2.query("SYSTEM SYNC REPLICA replica_2.rmt", timeout=5) - assert_eq_with_retry(node2, "SELECT * FROM replica_2.rmt", '1\ttest') + assert_eq_with_retry(node2, "SELECT * FROM replica_2.rmt", "1\ttest") + + +def test_non_query_with_table_ddl(started_cluster): + node1.query("CREATE USER A ON CLUSTER cross_3shards_2replicas") + + assert node1.query("SELECT 1", user="A") == "1\n" + assert node2.query("SELECT 1", user="A") == "1\n" + + node2.query("DROP USER A ON CLUSTER cross_3shards_2replicas") diff --git a/tests/integration/test_distributed_ddl_parallel/test.py b/tests/integration/test_distributed_ddl_parallel/test.py index 44971ca3d9e..a3fe00623ca 100644 --- a/tests/integration/test_distributed_ddl_parallel/test.py +++ b/tests/integration/test_distributed_ddl_parallel/test.py @@ -19,39 +19,43 @@ class SafeThread(threading.Thread): super().__init__() self.target = target self.exception = None + def run(self): try: self.target() - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except self.exception = e + def join(self, timeout=None): super().join(timeout) if self.exception: raise self.exception + def add_instance(name, ddl_config=None): - main_configs=[ - 'configs/remote_servers.xml', + main_configs = [ + "configs/remote_servers.xml", ] if ddl_config: main_configs.append(ddl_config) - dictionaries=[ - 'configs/dict.xml', + dictionaries = [ + "configs/dict.xml", ] - return cluster.add_instance(name, - main_configs=main_configs, - dictionaries=dictionaries, - with_zookeeper=True) + return cluster.add_instance( + name, main_configs=main_configs, dictionaries=dictionaries, with_zookeeper=True + ) -initiator = add_instance('initiator') + +initiator = add_instance("initiator") # distributed_ddl.pool_size = 2 -n1 = add_instance('n1', 'configs/ddl_a.xml') -n2 = add_instance('n2', 'configs/ddl_a.xml') +n1 = add_instance("n1", "configs/ddl_a.xml") +n2 = add_instance("n2", "configs/ddl_a.xml") # distributed_ddl.pool_size = 20 -n3 = add_instance('n3', 'configs/ddl_b.xml') -n4 = add_instance('n4', 'configs/ddl_b.xml') +n3 = add_instance("n3", "configs/ddl_b.xml") +n4 = add_instance("n4", "configs/ddl_b.xml") -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -59,6 +63,7 @@ def start_cluster(): finally: cluster.shutdown() + # verifies that functions executes longer then `sec` def longer_then(sec): def wrapper(func): @@ -67,40 +72,61 @@ def longer_then(sec): ts = time.time() result = func(*args, **kwargs) te = time.time() - took = te-ts + took = te - ts assert took >= sec return result + return inner + return wrapper + # It takes 7 seconds to load slow_dict_7. def execute_reload_dictionary_slow_dict_7(): - initiator.query('SYSTEM RELOAD DICTIONARY ON CLUSTER cluster_a slow_dict_7', settings={ - 'distributed_ddl_task_timeout': 60, - }) + initiator.query( + "SYSTEM RELOAD DICTIONARY ON CLUSTER cluster_a slow_dict_7", + settings={ + "distributed_ddl_task_timeout": 60, + }, + ) + + def execute_reload_dictionary_slow_dict_3(): - initiator.query('SYSTEM RELOAD DICTIONARY ON CLUSTER cluster_b slow_dict_3', settings={ - 'distributed_ddl_task_timeout': 60, - }) + initiator.query( + "SYSTEM RELOAD DICTIONARY ON CLUSTER cluster_b slow_dict_3", + settings={ + "distributed_ddl_task_timeout": 60, + }, + ) + + def execute_smoke_query(): - initiator.query('DROP DATABASE IF EXISTS foo ON CLUSTER cluster_b', settings={ - 'distributed_ddl_task_timeout': 60, - }) + initiator.query( + "DROP DATABASE IF EXISTS foo ON CLUSTER cluster_b", + settings={ + "distributed_ddl_task_timeout": 60, + }, + ) + def check_log(): # ensure that none of tasks processed multiple times for _, instance in list(cluster.instances.items()): - assert not instance.contains_in_log('Coordination::Exception: Node exists') + assert not instance.contains_in_log("Coordination::Exception: Node exists") + # NOTE: uses inner function to exclude slow start_cluster() from timeout. + def test_slow_dict_load_7(): @pytest.mark.timeout(10) @longer_then(7) def inner_test(): - initiator.query('SYSTEM RELOAD DICTIONARY slow_dict_7') + initiator.query("SYSTEM RELOAD DICTIONARY slow_dict_7") + inner_test() + def test_all_in_parallel(): @pytest.mark.timeout(10) @longer_then(7) @@ -112,9 +138,11 @@ def test_all_in_parallel(): thread.start() for thread in threads: thread.join(70) + inner_test() check_log() + def test_two_in_parallel_two_queued(): @pytest.mark.timeout(19) @longer_then(14) @@ -126,14 +154,17 @@ def test_two_in_parallel_two_queued(): thread.start() for thread in threads: thread.join(70) + inner_test() check_log() + def test_smoke(): for _ in range(100): execute_smoke_query() check_log() + def test_smoke_parallel(): threads = [] for _ in range(100): @@ -144,6 +175,7 @@ def test_smoke_parallel(): thread.join(70) check_log() + def test_smoke_parallel_dict_reload(): threads = [] for _ in range(100): diff --git a/tests/integration/test_distributed_ddl_password/test.py b/tests/integration/test_distributed_ddl_password/test.py index 0c061914497..bf2b7979c3c 100644 --- a/tests/integration/test_distributed_ddl_password/test.py +++ b/tests/integration/test_distributed_ddl_password/test.py @@ -4,18 +4,42 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/config.d/clusters.xml"], - user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=["configs/config.d/clusters.xml"], - user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=["configs/config.d/clusters.xml"], - user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=["configs/config.d/clusters.xml"], - user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node5 = cluster.add_instance('node5', main_configs=["configs/config.d/clusters.xml"], - user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=["configs/config.d/clusters.xml"], - user_configs=["configs/users.d/default_with_password.xml"], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], + with_zookeeper=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], + with_zookeeper=True, +) +node5 = cluster.add_instance( + "node5", + main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], + with_zookeeper=True, +) +node6 = cluster.add_instance( + "node6", + main_configs=["configs/config.d/clusters.xml"], + user_configs=["configs/users.d/default_with_password.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -23,14 +47,25 @@ def start_cluster(): try: cluster.start() - for node, shard in [(node1, 1), (node2, 1), (node3, 2), (node4, 2), (node5, 3), (node6, 3)]: + for node, shard in [ + (node1, 1), + (node2, 1), + (node3, 2), + (node4, 2), + (node5, 3), + (node6, 3), + ]: node.query( - ''' + """ CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') PARTITION BY date ORDER BY id - '''.format(shard=shard, replica=node.name), settings={"password": "clickhouse"}) + """.format( + shard=shard, replica=node.name + ), + settings={"password": "clickhouse"}, + ) yield cluster @@ -39,76 +74,191 @@ def start_cluster(): def test_truncate(start_cluster): - node1.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", - settings={"password": "clickhouse"}) + node1.query( + "insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", + settings={"password": "clickhouse"}, + ) - assert node1.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" + assert ( + node1.query( + "select count(*) from test_table", settings={"password": "clickhouse"} + ) + == "3\n" + ) node2.query("system sync replica test_table", settings={"password": "clickhouse"}) - assert node2.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" + assert ( + node2.query( + "select count(*) from test_table", settings={"password": "clickhouse"} + ) + == "3\n" + ) - node3.query("insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)", - settings={"password": "clickhouse"}) + node3.query( + "insert into test_table values ('2019-02-16', 1, 2), ('2019-02-16', 2, 3), ('2019-02-16', 3, 4)", + settings={"password": "clickhouse"}, + ) - assert node3.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" + assert ( + node3.query( + "select count(*) from test_table", settings={"password": "clickhouse"} + ) + == "3\n" + ) node4.query("system sync replica test_table", settings={"password": "clickhouse"}) - assert node4.query("select count(*) from test_table", settings={"password": "clickhouse"}) == "3\n" + assert ( + node4.query( + "select count(*) from test_table", settings={"password": "clickhouse"} + ) + == "3\n" + ) - node3.query("truncate table test_table on cluster 'awesome_cluster'", settings={"password": "clickhouse"}) + node3.query( + "truncate table test_table on cluster 'awesome_cluster'", + settings={"password": "clickhouse"}, + ) for node in [node1, node2, node3, node4]: - assert_eq_with_retry(node, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) + assert_eq_with_retry( + node, + "select count(*) from test_table", + "0", + settings={"password": "clickhouse"}, + ) - node2.query("drop table test_table on cluster 'awesome_cluster'", settings={"password": "clickhouse"}) + node2.query( + "drop table test_table on cluster 'awesome_cluster'", + settings={"password": "clickhouse"}, + ) for node in [node1, node2, node3, node4]: - assert_eq_with_retry(node, "select count(*) from system.tables where name='test_table'", "0", - settings={"password": "clickhouse"}) + assert_eq_with_retry( + node, + "select count(*) from system.tables where name='test_table'", + "0", + settings={"password": "clickhouse"}, + ) def test_alter(start_cluster): - node5.query("insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", - settings={"password": "clickhouse"}) - node6.query("insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)", - settings={"password": "clickhouse"}) + node5.query( + "insert into test_table values ('2019-02-15', 1, 2), ('2019-02-15', 2, 3), ('2019-02-15', 3, 4)", + settings={"password": "clickhouse"}, + ) + node6.query( + "insert into test_table values ('2019-02-15', 4, 2), ('2019-02-15', 5, 3), ('2019-02-15', 6, 4)", + settings={"password": "clickhouse"}, + ) node5.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) node6.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) - assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) + assert_eq_with_retry( + node5, + "select count(*) from test_table", + "6", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node6, + "select count(*) from test_table", + "6", + settings={"password": "clickhouse"}, + ) - node6.query("OPTIMIZE TABLE test_table ON CLUSTER 'simple_cluster' FINAL", settings={"password": "clickhouse"}) + node6.query( + "OPTIMIZE TABLE test_table ON CLUSTER 'simple_cluster' FINAL", + settings={"password": "clickhouse"}, + ) node5.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) node6.query("SYSTEM SYNC REPLICA test_table", settings={"password": "clickhouse"}) - assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) + assert_eq_with_retry( + node5, + "select count(*) from test_table", + "6", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node6, + "select count(*) from test_table", + "6", + settings={"password": "clickhouse"}, + ) - node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'", - settings={"password": "clickhouse"}) - assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) + node6.query( + "ALTER TABLE test_table ON CLUSTER 'simple_cluster' DETACH PARTITION '2019-02-15'", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node5, + "select count(*) from test_table", + "0", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node6, + "select count(*) from test_table", + "0", + settings={"password": "clickhouse"}, + ) with pytest.raises(QueryRuntimeException): - node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'", - settings={"password": "clickhouse"}) + node6.query( + "ALTER TABLE test_table ON CLUSTER 'simple_cluster' ATTACH PARTITION '2019-02-15'", + settings={"password": "clickhouse"}, + ) - node5.query("ALTER TABLE test_table ATTACH PARTITION '2019-02-15'", settings={"password": "clickhouse"}) + node5.query( + "ALTER TABLE test_table ATTACH PARTITION '2019-02-15'", + settings={"password": "clickhouse"}, + ) - assert_eq_with_retry(node5, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select count(*) from test_table", "6", settings={"password": "clickhouse"}) + assert_eq_with_retry( + node5, + "select count(*) from test_table", + "6", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node6, + "select count(*) from test_table", + "6", + settings={"password": "clickhouse"}, + ) - node5.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String", - settings={"password": "clickhouse"}) + node5.query( + "ALTER TABLE test_table ON CLUSTER 'simple_cluster' MODIFY COLUMN dummy String", + settings={"password": "clickhouse"}, + ) - assert_eq_with_retry(node5, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", - settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select length(dummy) from test_table ORDER BY dummy LIMIT 1", "1", - settings={"password": "clickhouse"}) + assert_eq_with_retry( + node5, + "select length(dummy) from test_table ORDER BY dummy LIMIT 1", + "1", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node6, + "select length(dummy) from test_table ORDER BY dummy LIMIT 1", + "1", + settings={"password": "clickhouse"}, + ) - node6.query("ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'", - settings={"password": "clickhouse"}) + node6.query( + "ALTER TABLE test_table ON CLUSTER 'simple_cluster' DROP PARTITION '2019-02-15'", + settings={"password": "clickhouse"}, + ) - assert_eq_with_retry(node5, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) - assert_eq_with_retry(node6, "select count(*) from test_table", "0", settings={"password": "clickhouse"}) + assert_eq_with_retry( + node5, + "select count(*) from test_table", + "0", + settings={"password": "clickhouse"}, + ) + assert_eq_with_retry( + node6, + "select count(*) from test_table", + "0", + settings={"password": "clickhouse"}, + ) diff --git a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py index b0b89fde41f..a47268b06fd 100644 --- a/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py +++ b/tests/integration/test_distributed_directory_monitor_split_batch_on_failure/test.py @@ -5,23 +5,27 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) # node1 -- distributed_directory_monitor_split_batch_on_failure=on -node1 = cluster.add_instance('node1', - main_configs=['configs/remote_servers.xml'], - user_configs=['configs/overrides_1.xml'], +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/overrides_1.xml"], ) # node2 -- distributed_directory_monitor_split_batch_on_failure=off -node2 = cluster.add_instance('node2', - main_configs=['configs/remote_servers.xml'], - user_configs=['configs/overrides_2.xml'], +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/overrides_2.xml"], ) -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def started_cluster(): try: cluster.start() for _, node in cluster.instances.items(): - node.query(""" + node.query( + """ create table null_ (key Int, value Int) engine=Null(); create table dist as null_ engine=Distributed(test_cluster, currentDatabase(), null_, key); create table data (key Int, uniq_values Int) engine=Memory(); @@ -29,34 +33,46 @@ def started_cluster(): system stop distributed sends dist; create table dist_data as data engine=Distributed(test_cluster, currentDatabase(), data); - """) + """ + ) yield cluster finally: cluster.shutdown() + def test_distributed_directory_monitor_split_batch_on_failure_OFF(started_cluster): for i in range(0, 100): limit = 100e3 - node2.query(f'insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}', settings={ - # max_memory_usage is the limit for the batch on the remote node - # (local query should not be affected since 30MB is enough for 100K rows) - 'max_memory_usage': '30Mi', - 'max_untracked_memory': '0' - }) + node2.query( + f"insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}", + settings={ + # max_memory_usage is the limit for the batch on the remote node + # (local query should not be affected since 30MB is enough for 100K rows) + "max_memory_usage": "30Mi", + "max_untracked_memory": "0", + }, + ) # "Received from" is mandatory, since the exception should be thrown on the remote node. - with pytest.raises(QueryRuntimeException, match=r'DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*while pushing to view default\.mv'): - node2.query('system flush distributed dist') - assert int(node2.query('select count() from dist_data')) == 0 + with pytest.raises( + QueryRuntimeException, + match=r"DB::Exception: Received from.*Memory limit \(for query\) exceeded: .*while pushing to view default\.mv", + ): + node2.query("system flush distributed dist") + assert int(node2.query("select count() from dist_data")) == 0 + def test_distributed_directory_monitor_split_batch_on_failure_ON(started_cluster): for i in range(0, 100): limit = 100e3 - node1.query(f'insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}', settings={ - # max_memory_usage is the limit for the batch on the remote node - # (local query should not be affected since 30MB is enough for 100K rows) - 'max_memory_usage': '30Mi', - 'max_untracked_memory': '0' - }) - node1.query('system flush distributed dist') - assert int(node1.query('select count() from dist_data')) == 100000 + node1.query( + f"insert into dist select number/100, number from system.numbers limit {limit} offset {limit*i}", + settings={ + # max_memory_usage is the limit for the batch on the remote node + # (local query should not be affected since 30MB is enough for 100K rows) + "max_memory_usage": "30Mi", + "max_untracked_memory": "0", + }, + ) + node1.query("system flush distributed dist") + assert int(node1.query("select count() from dist_data")) == 100000 diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py index d6e1cc03fa8..415141be021 100644 --- a/tests/integration/test_distributed_format/test.py +++ b/tests/integration/test_distributed_format/test.py @@ -6,20 +6,23 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/remote_servers.xml']) +node = cluster.add_instance("node", main_configs=["configs/remote_servers.xml"]) -cluster_param = pytest.mark.parametrize("cluster", [ - ('test_cluster_internal_replication'), - ('test_cluster_no_internal_replication'), -]) +cluster_param = pytest.mark.parametrize( + "cluster", + [ + ("test_cluster_internal_replication"), + ("test_cluster_no_internal_replication"), + ], +) def get_dist_path(cluster, table, dist_format): if dist_format == 0: - return f'/var/lib/clickhouse/data/test/{table}/default@not_existing:9000' - if cluster == 'test_cluster_internal_replication': - return f'/var/lib/clickhouse/data/test/{table}/shard1_all_replicas' - return f'/var/lib/clickhouse/data/test/{table}/shard1_replica1' + return f"/var/lib/clickhouse/data/test/{table}/default@not_existing:9000" + if cluster == "test_cluster_internal_replication": + return f"/var/lib/clickhouse/data/test/{table}/shard1_all_replicas" + return f"/var/lib/clickhouse/data/test/{table}/shard1_replica1" @pytest.fixture(scope="module") @@ -36,23 +39,32 @@ def started_cluster(): @cluster_param def test_single_file(started_cluster, cluster): node.query( - "create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) - node.query("insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", - settings={"use_compact_format_in_distributed_parts_names": "1"}) + "create table test.distr_1 (x UInt64, s String) engine = Distributed('{}', database, table)".format( + cluster + ) + ) + node.query( + "insert into test.distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')", + settings={"use_compact_format_in_distributed_parts_names": "1"}, + ) - path = get_dist_path(cluster, 'distr_1', 1) + path = get_dist_path(cluster, "distr_1", 1) query = f"select * from file('{path}/1.bin', 'Distributed')" - out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) + out = node.exec_in_container( + ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query] + ) - assert out == '1\ta\n2\tbb\n3\tccc\n' + assert out == "1\ta\n2\tbb\n3\tccc\n" query = f""" create table t (x UInt64, s String) engine = File('Distributed', '{path}/1.bin'); select * from t; """ - out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) + out = node.exec_in_container( + ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query] + ) - assert out == '1\ta\n2\tbb\n3\tccc\n' + assert out == "1\ta\n2\tbb\n3\tccc\n" node.query("drop table test.distr_1") @@ -60,27 +72,40 @@ def test_single_file(started_cluster, cluster): @cluster_param def test_two_files(started_cluster, cluster): node.query( - "create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) - node.query("insert into test.distr_2 values (0, '_'), (1, 'a')", settings={ - "use_compact_format_in_distributed_parts_names": "1", - }) - node.query("insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", settings={ - "use_compact_format_in_distributed_parts_names": "1", - }) + "create table test.distr_2 (x UInt64, s String) engine = Distributed('{}', database, table)".format( + cluster + ) + ) + node.query( + "insert into test.distr_2 values (0, '_'), (1, 'a')", + settings={ + "use_compact_format_in_distributed_parts_names": "1", + }, + ) + node.query( + "insert into test.distr_2 values (2, 'bb'), (3, 'ccc')", + settings={ + "use_compact_format_in_distributed_parts_names": "1", + }, + ) - path = get_dist_path(cluster, 'distr_2', 1) + path = get_dist_path(cluster, "distr_2", 1) query = f"select * from file('{path}/{{1,2,3,4}}.bin', 'Distributed') order by x" - out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) + out = node.exec_in_container( + ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query] + ) - assert out == '0\t_\n1\ta\n2\tbb\n3\tccc\n' + assert out == "0\t_\n1\ta\n2\tbb\n3\tccc\n" query = f""" create table t (x UInt64, s String) engine = File('Distributed', '{path}/{{1,2,3,4}}.bin'); select * from t order by x; """ - out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) + out = node.exec_in_container( + ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query] + ) - assert out == '0\t_\n1\ta\n2\tbb\n3\tccc\n' + assert out == "0\t_\n1\ta\n2\tbb\n3\tccc\n" node.query("drop table test.distr_2") @@ -88,23 +113,33 @@ def test_two_files(started_cluster, cluster): @cluster_param def test_single_file_old(started_cluster, cluster): node.query( - "create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format(cluster)) - node.query("insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')", settings={ - "use_compact_format_in_distributed_parts_names": "0", - }) + "create table test.distr_3 (x UInt64, s String) engine = Distributed('{}', database, table)".format( + cluster + ) + ) + node.query( + "insert into test.distr_3 values (1, 'a'), (2, 'bb'), (3, 'ccc')", + settings={ + "use_compact_format_in_distributed_parts_names": "0", + }, + ) - path = get_dist_path(cluster, 'distr_3', 0) + path = get_dist_path(cluster, "distr_3", 0) query = f"select * from file('{path}/1.bin', 'Distributed')" - out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) + out = node.exec_in_container( + ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query] + ) - assert out == '1\ta\n2\tbb\n3\tccc\n' + assert out == "1\ta\n2\tbb\n3\tccc\n" query = f""" create table t (x UInt64, s String) engine = File('Distributed', '{path}/1.bin'); select * from t; """ - out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) + out = node.exec_in_container( + ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query] + ) - assert out == '1\ta\n2\tbb\n3\tccc\n' + assert out == "1\ta\n2\tbb\n3\tccc\n" node.query("drop table test.distr_3") diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index ba7d8e0a25d..ad61a2ad6f5 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -5,19 +5,32 @@ from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) -node_shard = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) +node_shard = cluster.add_instance("node1", main_configs=["configs/remote_servers.xml"]) + +node_dist = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + image="yandex/clickhouse-server", + tag="21.11.9.1", + stay_alive=True, + with_installed_binary=True, +) -node_dist = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', - tag='21.11.9.1', stay_alive=True, with_installed_binary=True) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node_shard.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") - node_dist.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") - node_dist.query("CREATE TABLE dist_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, rand())") + node_shard.query( + "CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id" + ) + node_dist.query( + "CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id" + ) + node_dist.query( + "CREATE TABLE dist_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, rand())" + ) yield cluster diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 2601163d790..8d344834c50 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -11,45 +11,63 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -def make_instance(name, cfg): - return cluster.add_instance(name, - with_zookeeper=True, - main_configs=['configs/remote_servers.xml', cfg], - user_configs=['configs/users.xml']) -# _n1/_n2 contains cluster with different -- should fail -n1 = make_instance('n1', 'configs/remote_servers_n1.xml') -n2 = make_instance('n2', 'configs/remote_servers_n2.xml') -users = pytest.mark.parametrize('user,password', [ - ('default', '' ), - ('nopass', '' ), - ('pass', 'foo'), -]) +def make_instance(name, cfg): + return cluster.add_instance( + name, + with_zookeeper=True, + main_configs=["configs/remote_servers.xml", cfg], + user_configs=["configs/users.xml"], + ) + + +# _n1/_n2 contains cluster with different -- should fail +n1 = make_instance("n1", "configs/remote_servers_n1.xml") +n2 = make_instance("n2", "configs/remote_servers_n2.xml") + +users = pytest.mark.parametrize( + "user,password", + [ + ("default", ""), + ("nopass", ""), + ("pass", "foo"), + ], +) + def bootstrap(): for n in list(cluster.instances.values()): - n.query('DROP TABLE IF EXISTS data') - n.query('DROP TABLE IF EXISTS data_from_buffer') - n.query('DROP TABLE IF EXISTS dist') - n.query('CREATE TABLE data (key Int) Engine=Memory()') - n.query('CREATE TABLE data_from_buffer (key Int) Engine=Memory()') - n.query(""" + n.query("DROP TABLE IF EXISTS data") + n.query("DROP TABLE IF EXISTS data_from_buffer") + n.query("DROP TABLE IF EXISTS dist") + n.query("CREATE TABLE data (key Int) Engine=Memory()") + n.query("CREATE TABLE data_from_buffer (key Int) Engine=Memory()") + n.query( + """ CREATE TABLE dist_insecure AS data Engine=Distributed(insecure, currentDatabase(), data, key) - """) - n.query(""" + """ + ) + n.query( + """ CREATE TABLE dist_secure AS data Engine=Distributed(secure, currentDatabase(), data, key) - """) - n.query(""" + """ + ) + n.query( + """ CREATE TABLE dist_secure_from_buffer AS data_from_buffer Engine=Distributed(secure, currentDatabase(), data_from_buffer, key) - """) - n.query(""" + """ + ) + n.query( + """ CREATE TABLE dist_secure_disagree AS data Engine=Distributed(secure_disagree, currentDatabase(), data, key) - """) - n.query(""" + """ + ) + n.query( + """ CREATE TABLE dist_secure_buffer AS dist_secure_from_buffer Engine=Buffer(currentDatabase(), dist_secure_from_buffer, /* settings for manual flush only */ @@ -61,9 +79,11 @@ def bootstrap(): 0, /* min_bytes */ 0 /* max_bytes */ ) - """) + """ + ) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -72,36 +92,57 @@ def start_cluster(): finally: cluster.shutdown() + def query_with_id(node, id_, query, **kwargs): return node.query("WITH '{}' AS __id {}".format(id_, query), **kwargs) + # @return -- [user, initial_user] def get_query_user_info(node, query_pattern): node.query("SYSTEM FLUSH LOGS") - return node.query(""" + return ( + node.query( + """ SELECT user, initial_user FROM system.query_log WHERE query LIKE '%{}%' AND query NOT LIKE '%system.query_log%' AND type = 'QueryFinish' - """.format(query_pattern)).strip().split('\t') + """.format( + query_pattern + ) + ) + .strip() + .split("\t") + ) + # @return -- [user, initial_user] def get_query_user_info_by_id(node, query_id): node.query("SYSTEM FLUSH LOGS") - return node.query(""" + return ( + node.query( + """ SELECT user, initial_user FROM system.query_log WHERE query_id = '{}' AND type = 'QueryFinish' - """.format(query_id)).strip().split('\t') + """.format( + query_id + ) + ) + .strip() + .split("\t") + ) + # @return -- settings def get_query_setting_on_shard(node, query_pattern, setting): node.query("SYSTEM FLUSH LOGS") - return node.query(""" + return node.query( + """ SELECT Settings['{}'] FROM system.query_log WHERE @@ -110,39 +151,55 @@ def get_query_setting_on_shard(node, query_pattern, setting): query NOT LIKE '%system.query_log%' AND type = 'QueryFinish' LIMIT 1 - """.format(setting, query_pattern)).strip() + """.format( + setting, query_pattern + ) + ).strip() + def test_insecure(): - n1.query('SELECT * FROM dist_insecure') + n1.query("SELECT * FROM dist_insecure") + def test_insecure_insert_async(): n1.query("TRUNCATE TABLE data") - n1.query('INSERT INTO dist_insecure SELECT * FROM numbers(2)') - n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER insecure dist_insecure') - assert int(n1.query('SELECT count() FROM dist_insecure')) == 2 - n1.query('TRUNCATE TABLE data ON CLUSTER insecure') + n1.query("INSERT INTO dist_insecure SELECT * FROM numbers(2)") + n1.query("SYSTEM FLUSH DISTRIBUTED ON CLUSTER insecure dist_insecure") + assert int(n1.query("SELECT count() FROM dist_insecure")) == 2 + n1.query("TRUNCATE TABLE data ON CLUSTER insecure") + def test_insecure_insert_sync(): n1.query("TRUNCATE TABLE data") - n1.query('INSERT INTO dist_insecure SELECT * FROM numbers(2)', settings={'insert_distributed_sync': 1}) - assert int(n1.query('SELECT count() FROM dist_insecure')) == 2 - n1.query('TRUNCATE TABLE data ON CLUSTER secure') + n1.query( + "INSERT INTO dist_insecure SELECT * FROM numbers(2)", + settings={"insert_distributed_sync": 1}, + ) + assert int(n1.query("SELECT count() FROM dist_insecure")) == 2 + n1.query("TRUNCATE TABLE data ON CLUSTER secure") + def test_secure(): - n1.query('SELECT * FROM dist_secure') + n1.query("SELECT * FROM dist_secure") + def test_secure_insert_async(): n1.query("TRUNCATE TABLE data") - n1.query('INSERT INTO dist_secure SELECT * FROM numbers(2)') - n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure') - assert int(n1.query('SELECT count() FROM dist_secure')) == 2 - n1.query('TRUNCATE TABLE data ON CLUSTER secure') + n1.query("INSERT INTO dist_secure SELECT * FROM numbers(2)") + n1.query("SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure") + assert int(n1.query("SELECT count() FROM dist_secure")) == 2 + n1.query("TRUNCATE TABLE data ON CLUSTER secure") + def test_secure_insert_sync(): n1.query("TRUNCATE TABLE data") - n1.query('INSERT INTO dist_secure SELECT * FROM numbers(2)', settings={'insert_distributed_sync': 1}) - assert int(n1.query('SELECT count() FROM dist_secure')) == 2 - n1.query('TRUNCATE TABLE data ON CLUSTER secure') + n1.query( + "INSERT INTO dist_secure SELECT * FROM numbers(2)", + settings={"insert_distributed_sync": 1}, + ) + assert int(n1.query("SELECT count() FROM dist_secure")) == 2 + n1.query("TRUNCATE TABLE data ON CLUSTER secure") + # INSERT w/o initial_user # @@ -180,28 +237,40 @@ def test_secure_insert_sync(): def test_secure_insert_buffer_async(): # Change cluster definition so that the SELECT will always creates new connection priority = int(time.time()) - n1.exec_in_container(['bash', '-c', f'sed -i "s#.*#{priority}#" /etc/clickhouse-server/config.d/remote_servers.xml']) - n1.query('SYSTEM RELOAD CONFIG') + n1.exec_in_container( + [ + "bash", + "-c", + f'sed -i "s#.*#{priority}#" /etc/clickhouse-server/config.d/remote_servers.xml', + ] + ) + n1.query("SYSTEM RELOAD CONFIG") # ensure that SELECT creates new connection (we need separate table for # this, so that separate distributed pool will be used) query_id = uuid.uuid4().hex - n1.query('SELECT * FROM dist_secure_from_buffer', user='ro', query_id=query_id) - assert n1.contains_in_log('{' + query_id + '} Connection (n2:9000): Connecting.') + n1.query("SELECT * FROM dist_secure_from_buffer", user="ro", query_id=query_id) + assert n1.contains_in_log( + "{" + query_id + "} Connection (n2:9000): Connecting." + ) query_id = uuid.uuid4().hex - n1.query('INSERT INTO dist_secure_buffer SELECT * FROM numbers(2)', query_id=query_id) + n1.query( + "INSERT INTO dist_secure_buffer SELECT * FROM numbers(2)", query_id=query_id + ) # ensure that INSERT does not creates new connection, so that it will use # previous connection that was instantiated with "ro" user (using # interserver secret) - assert not n1.contains_in_log('{' + query_id + '} Connection (n2:9000): Connecting.') - assert get_query_user_info_by_id(n1, query_id) == ['default', 'default'] + assert not n1.contains_in_log( + "{" + query_id + "} Connection (n2:9000): Connecting." + ) + assert get_query_user_info_by_id(n1, query_id) == ["default", "default"] # And before the bug was fixed this query will fail with the following error: # # Code: 164. DB::Exception: Received from 172.16.2.5:9000. DB::Exception: There was an error on [n1:9000]: Code: 164. DB::Exception: Received from n2:9000. DB::Exception: ro: Cannot execute query in readonly mode. (READONLY) - n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure_from_buffer') - n1.query('OPTIMIZE TABLE dist_secure_buffer') - n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure_from_buffer') + n1.query("SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure_from_buffer") + n1.query("OPTIMIZE TABLE dist_secure_buffer") + n1.query("SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure_from_buffer") # Check user from which the INSERT on the remote node will be executed # @@ -213,76 +282,124 @@ def test_secure_insert_buffer_async(): # # {2c55669f-71ad-48fe-98fa-7b475b80718e} executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES # - assert n2.contains_in_log('executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES') + assert n2.contains_in_log( + "executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES" + ) + + assert int(n1.query("SELECT count() FROM dist_secure_from_buffer")) == 2 + n1.query("TRUNCATE TABLE data_from_buffer ON CLUSTER secure") - assert int(n1.query('SELECT count() FROM dist_secure_from_buffer')) == 2 - n1.query('TRUNCATE TABLE data_from_buffer ON CLUSTER secure') def test_secure_disagree(): - with pytest.raises(QueryRuntimeException, match='.*Hash mismatch.*'): - n1.query('SELECT * FROM dist_secure_disagree') + with pytest.raises(QueryRuntimeException, match=".*Hash mismatch.*"): + n1.query("SELECT * FROM dist_secure_disagree") + def test_secure_disagree_insert(): n1.query("TRUNCATE TABLE data") - n1.query('INSERT INTO dist_secure_disagree SELECT * FROM numbers(2)') - with pytest.raises(QueryRuntimeException, match='.*Hash mismatch.*'): - n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure_disagree dist_secure_disagree') + n1.query("INSERT INTO dist_secure_disagree SELECT * FROM numbers(2)") + with pytest.raises(QueryRuntimeException, match=".*Hash mismatch.*"): + n1.query( + "SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure_disagree dist_secure_disagree" + ) # check the the connection will be re-established # IOW that we will not get "Unknown BlockInfo field" - with pytest.raises(QueryRuntimeException, match='.*Hash mismatch.*'): - assert int(n1.query('SELECT count() FROM dist_secure_disagree')) == 0 + with pytest.raises(QueryRuntimeException, match=".*Hash mismatch.*"): + assert int(n1.query("SELECT count() FROM dist_secure_disagree")) == 0 + @users def test_user_insecure_cluster(user, password): - id_ = 'query-dist_insecure-' + user - query_with_id(n1, id_, 'SELECT * FROM dist_insecure', user=user, password=password) - assert get_query_user_info(n1, id_) == [user, user] # due to prefer_localhost_replica - assert get_query_user_info(n2, id_) == ['default', user] + id_ = "query-dist_insecure-" + user + query_with_id(n1, id_, "SELECT * FROM dist_insecure", user=user, password=password) + assert get_query_user_info(n1, id_) == [ + user, + user, + ] # due to prefer_localhost_replica + assert get_query_user_info(n2, id_) == ["default", user] + @users def test_user_secure_cluster(user, password): - id_ = 'query-dist_secure-' + user - query_with_id(n1, id_, 'SELECT * FROM dist_secure', user=user, password=password) + id_ = "query-dist_secure-" + user + query_with_id(n1, id_, "SELECT * FROM dist_secure", user=user, password=password) assert get_query_user_info(n1, id_) == [user, user] assert get_query_user_info(n2, id_) == [user, user] + @users def test_per_user_inline_settings_insecure_cluster(user, password): - id_ = 'query-ddl-settings-dist_insecure-' + user - query_with_id(n1, id_, """ + id_ = "query-ddl-settings-dist_insecure-" + user + query_with_id( + n1, + id_, + """ SELECT * FROM dist_insecure SETTINGS prefer_localhost_replica=0, max_memory_usage_for_user=1e9, max_untracked_memory=0 - """, user=user, password=password) - assert get_query_setting_on_shard(n1, id_, 'max_memory_usage_for_user') == '' + """, + user=user, + password=password, + ) + assert get_query_setting_on_shard(n1, id_, "max_memory_usage_for_user") == "" + + @users def test_per_user_inline_settings_secure_cluster(user, password): - id_ = 'query-ddl-settings-dist_secure-' + user - query_with_id(n1, id_, """ + id_ = "query-ddl-settings-dist_secure-" + user + query_with_id( + n1, + id_, + """ SELECT * FROM dist_secure SETTINGS prefer_localhost_replica=0, max_memory_usage_for_user=1e9, max_untracked_memory=0 - """, user=user, password=password) - assert int(get_query_setting_on_shard(n1, id_, 'max_memory_usage_for_user')) == int(1e9) + """, + user=user, + password=password, + ) + assert int(get_query_setting_on_shard(n1, id_, "max_memory_usage_for_user")) == int( + 1e9 + ) + + @users def test_per_user_protocol_settings_insecure_cluster(user, password): - id_ = 'query-protocol-settings-dist_insecure-' + user - query_with_id(n1, id_, 'SELECT * FROM dist_insecure', user=user, password=password, settings={ - 'prefer_localhost_replica': 0, - 'max_memory_usage_for_user': int(1e9), - 'max_untracked_memory': 0, - }) - assert get_query_setting_on_shard(n1, id_, 'max_memory_usage_for_user') == '' + id_ = "query-protocol-settings-dist_insecure-" + user + query_with_id( + n1, + id_, + "SELECT * FROM dist_insecure", + user=user, + password=password, + settings={ + "prefer_localhost_replica": 0, + "max_memory_usage_for_user": int(1e9), + "max_untracked_memory": 0, + }, + ) + assert get_query_setting_on_shard(n1, id_, "max_memory_usage_for_user") == "" + + @users def test_per_user_protocol_settings_secure_cluster(user, password): - id_ = 'query-protocol-settings-dist_secure-' + user - query_with_id(n1, id_, 'SELECT * FROM dist_secure', user=user, password=password, settings={ - 'prefer_localhost_replica': 0, - 'max_memory_usage_for_user': int(1e9), - 'max_untracked_memory': 0, - }) - assert int(get_query_setting_on_shard(n1, id_, 'max_memory_usage_for_user')) == int(1e9) + id_ = "query-protocol-settings-dist_secure-" + user + query_with_id( + n1, + id_, + "SELECT * FROM dist_secure", + user=user, + password=password, + settings={ + "prefer_localhost_replica": 0, + "max_memory_usage_for_user": int(1e9), + "max_untracked_memory": 0, + }, + ) + assert int(get_query_setting_on_shard(n1, id_, "max_memory_usage_for_user")) == int( + 1e9 + ) diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py index 8a1c282eff2..90771c027dc 100644 --- a/tests/integration/test_distributed_load_balancing/test.py +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -9,9 +9,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -n1 = cluster.add_instance('n1', main_configs=['configs/remote_servers.xml']) -n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml']) -n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers.xml']) +n1 = cluster.add_instance("n1", main_configs=["configs/remote_servers.xml"]) +n2 = cluster.add_instance("n2", main_configs=["configs/remote_servers.xml"]) +n3 = cluster.add_instance("n3", main_configs=["configs/remote_servers.xml"]) nodes = len(cluster.instances) queries = nodes * 10 @@ -33,38 +33,44 @@ def bootstrap(): # And if the reload will happen during round_robin test it will start # querying from the beginning, so let's issue config reload just after # start to avoid reload in the middle of the test execution. - n.query('SYSTEM RELOAD CONFIG') - n.query('DROP TABLE IF EXISTS data') - n.query('DROP TABLE IF EXISTS dist') - n.query('CREATE TABLE data (key Int) Engine=Memory()') - n.query(""" + n.query("SYSTEM RELOAD CONFIG") + n.query("DROP TABLE IF EXISTS data") + n.query("DROP TABLE IF EXISTS dist") + n.query("CREATE TABLE data (key Int) Engine=Memory()") + n.query( + """ CREATE TABLE dist AS data Engine=Distributed( replicas_cluster, currentDatabase(), data) - """) - n.query(""" + """ + ) + n.query( + """ CREATE TABLE dist_priority AS data Engine=Distributed( replicas_priority_cluster, currentDatabase(), data) - """) - n.query(""" + """ + ) + n.query( + """ CREATE TABLE dist_priority_negative AS data Engine=Distributed( replicas_priority_negative_cluster, currentDatabase(), data) - """) + """ + ) def make_uuid(): return uuid.uuid4().hex -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -74,26 +80,27 @@ def start_cluster(): cluster.shutdown() -def get_node(query_node, table='dist', *args, **kwargs): +def get_node(query_node, table="dist", *args, **kwargs): query_id = make_uuid() settings = { - 'query_id': query_id, - 'log_queries': 1, - 'log_queries_min_type': 'QUERY_START', - 'prefer_localhost_replica': 0, + "query_id": query_id, + "log_queries": 1, + "log_queries_min_type": "QUERY_START", + "prefer_localhost_replica": 0, } - if 'settings' not in kwargs: - kwargs['settings'] = settings + if "settings" not in kwargs: + kwargs["settings"] = settings else: - kwargs['settings'].update(settings) + kwargs["settings"].update(settings) - query_node.query('SELECT * FROM ' + table, *args, **kwargs) + query_node.query("SELECT * FROM " + table, *args, **kwargs) for n in list(cluster.instances.values()): - n.query('SYSTEM FLUSH LOGS') + n.query("SYSTEM FLUSH LOGS") - rows = query_node.query(""" + rows = query_node.query( + """ SELECT c.host_name FROM ( SELECT _shard_num @@ -107,7 +114,10 @@ def get_node(query_node, table='dist', *args, **kwargs): ) a JOIN system.clusters c ON a._shard_num = c.shard_num WHERE cluster = 'shards_cluster' - """.format(query_id=query_id)) + """.format( + query_id=query_id + ) + ) return rows.strip() @@ -115,88 +125,100 @@ def get_node(query_node, table='dist', *args, **kwargs): def test_load_balancing_default(): unique_nodes = set() for _ in range(0, queries): - unique_nodes.add(get_node(n1, settings={'load_balancing': 'random'})) + unique_nodes.add(get_node(n1, settings={"load_balancing": "random"})) assert len(unique_nodes) == nodes, unique_nodes def test_load_balancing_nearest_hostname(): unique_nodes = set() for _ in range(0, queries): - unique_nodes.add(get_node(n1, settings={'load_balancing': 'nearest_hostname'})) + unique_nodes.add(get_node(n1, settings={"load_balancing": "nearest_hostname"})) assert len(unique_nodes) == 1, unique_nodes - assert unique_nodes == set(['n1']) + assert unique_nodes == set(["n1"]) def test_load_balancing_in_order(): unique_nodes = set() for _ in range(0, queries): - unique_nodes.add(get_node(n1, settings={'load_balancing': 'in_order'})) + unique_nodes.add(get_node(n1, settings={"load_balancing": "in_order"})) assert len(unique_nodes) == 1, unique_nodes - assert unique_nodes == set(['n1']) + assert unique_nodes == set(["n1"]) def test_load_balancing_first_or_random(): unique_nodes = set() for _ in range(0, queries): - unique_nodes.add(get_node(n1, settings={'load_balancing': 'first_or_random'})) + unique_nodes.add(get_node(n1, settings={"load_balancing": "first_or_random"})) assert len(unique_nodes) == 1, unique_nodes - assert unique_nodes == set(['n1']) + assert unique_nodes == set(["n1"]) def test_load_balancing_round_robin(): unique_nodes = set() for _ in range(0, nodes): - unique_nodes.add(get_node(n1, settings={'load_balancing': 'round_robin'})) + unique_nodes.add(get_node(n1, settings={"load_balancing": "round_robin"})) assert len(unique_nodes) == nodes, unique_nodes - assert unique_nodes == set(['n1', 'n2', 'n3']) + assert unique_nodes == set(["n1", "n2", "n3"]) -@pytest.mark.parametrize('dist_table', [ - ('dist_priority'), - ('dist_priority_negative'), -]) +@pytest.mark.parametrize( + "dist_table", + [ + ("dist_priority"), + ("dist_priority_negative"), + ], +) def test_load_balancing_priority_round_robin(dist_table): unique_nodes = set() for _ in range(0, nodes): - unique_nodes.add(get_node(n1, dist_table, settings={'load_balancing': 'round_robin'})) + unique_nodes.add( + get_node(n1, dist_table, settings={"load_balancing": "round_robin"}) + ) assert len(unique_nodes) == 2, unique_nodes # n2 has bigger priority in config - assert unique_nodes == set(['n1', 'n3']) + assert unique_nodes == set(["n1", "n3"]) def test_distributed_replica_max_ignored_errors(): settings = { - 'use_hedged_requests' : 0, - 'load_balancing': 'in_order', - 'prefer_localhost_replica': 0, - 'connect_timeout': 2, - 'receive_timeout': 2, - 'send_timeout': 2, - 'idle_connection_timeout': 2, - 'tcp_keep_alive_timeout': 2, - - 'distributed_replica_max_ignored_errors': 0, - 'distributed_replica_error_half_life': 60, + "use_hedged_requests": 0, + "load_balancing": "in_order", + "prefer_localhost_replica": 0, + "connect_timeout": 2, + "receive_timeout": 2, + "send_timeout": 2, + "idle_connection_timeout": 2, + "tcp_keep_alive_timeout": 2, + "distributed_replica_max_ignored_errors": 0, + "distributed_replica_error_half_life": 60, } # initiate connection (if started only this test) - n2.query('SELECT * FROM dist', settings=settings) - cluster.pause_container('n1') + n2.query("SELECT * FROM dist", settings=settings) + cluster.pause_container("n1") # n1 paused -- skipping, and increment error_count for n1 # but the query succeeds, no need in query_and_get_error() - n2.query('SELECT * FROM dist', settings=settings) + n2.query("SELECT * FROM dist", settings=settings) # XXX: due to config reloading we need second time (sigh) - n2.query('SELECT * FROM dist', settings=settings) + n2.query("SELECT * FROM dist", settings=settings) # check error_count for n1 - assert int(n2.query(""" + assert ( + int( + n2.query( + """ SELECT errors_count FROM system.clusters WHERE cluster = 'replicas_cluster' AND host_name = 'n1' - """, settings=settings)) == 1 + """, + settings=settings, + ) + ) + == 1 + ) - cluster.unpause_container('n1') + cluster.unpause_container("n1") # still n2 - assert get_node(n2, settings=settings) == 'n2' + assert get_node(n2, settings=settings) == "n2" # now n1 - settings['distributed_replica_max_ignored_errors'] = 1 - assert get_node(n2, settings=settings) == 'n1' + settings["distributed_replica_max_ignored_errors"] = 1 + assert get_node(n2, settings=settings) == "n1" diff --git a/tests/integration/test_distributed_over_distributed/test.py b/tests/integration/test_distributed_over_distributed/test.py index ae86a70f31b..c000005e55a 100644 --- a/tests/integration/test_distributed_over_distributed/test.py +++ b/tests/integration/test_distributed_over_distributed/test.py @@ -7,13 +7,17 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -NODES = {'node' + str(i): cluster.add_instance( - 'node' + str(i), - main_configs=['configs/remote_servers.xml'], - user_configs=['configs/set_distributed_defaults.xml'], -) for i in (1, 2)} +NODES = { + "node" + + str(i): cluster.add_instance( + "node" + str(i), + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/set_distributed_defaults.xml"], + ) + for i in (1, 2) +} -CREATE_TABLES_SQL = ''' +CREATE_TABLES_SQL = """ CREATE TABLE base_table( node String, @@ -31,7 +35,7 @@ CREATE TABLE distributed_over_distributed_table AS distributed_table ENGINE = Distributed('test_cluster', default, distributed_table); -''' +""" INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" @@ -43,24 +47,45 @@ def started_cluster(): for node_index, (node_name, node) in enumerate(NODES.items()): node.query(CREATE_TABLES_SQL) for i in range(0, 2): - node.query(INSERT_SQL_TEMPLATE.format(node_id=node_name, key=i, value=i + (node_index * 10))) + node.query( + INSERT_SQL_TEMPLATE.format( + node_id=node_name, key=i, value=i + (node_index * 10) + ) + ) yield cluster finally: cluster.shutdown() -@pytest.mark.parametrize("node,source", [ - pytest.param(NODES["node1"], "distributed_over_distributed_table", id="dod_node1"), - pytest.param(NODES["node1"], "cluster('test_cluster', default, distributed_table)", id="cluster_node1"), - pytest.param(NODES["node2"], "distributed_over_distributed_table", id="dod_node2"), - pytest.param(NODES["node2"], "cluster('test_cluster', default, distributed_table)", id="cluster_node2"), -] +@pytest.mark.parametrize( + "node,source", + [ + pytest.param( + NODES["node1"], "distributed_over_distributed_table", id="dod_node1" + ), + pytest.param( + NODES["node1"], + "cluster('test_cluster', default, distributed_table)", + id="cluster_node1", + ), + pytest.param( + NODES["node2"], "distributed_over_distributed_table", id="dod_node2" + ), + pytest.param( + NODES["node2"], + "cluster('test_cluster', default, distributed_table)", + id="cluster_node2", + ), + ], ) class TestDistributedOverDistributedSuite: def test_select_with_order_by_node(self, started_cluster, node, source): - assert node.query("SELECT * FROM {source} ORDER BY node, key".format(source=source)) \ - == """node1 0 0 + assert ( + node.query( + "SELECT * FROM {source} ORDER BY node, key".format(source=source) + ) + == """node1 0 0 node1 0 0 node1 1 1 node1 1 1 @@ -69,10 +94,14 @@ node2 0 10 node2 1 11 node2 1 11 """ + ) def test_select_with_order_by_key(self, started_cluster, node, source): - assert node.query("SELECT * FROM {source} ORDER BY key, node".format(source=source)) \ - == """node1 0 0 + assert ( + node.query( + "SELECT * FROM {source} ORDER BY key, node".format(source=source) + ) + == """node1 0 0 node1 0 0 node2 0 10 node2 0 10 @@ -81,15 +110,30 @@ node1 1 1 node2 1 11 node2 1 11 """ + ) def test_select_with_group_by_node(self, started_cluster, node, source): - assert node.query("SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format(source=source)) \ - == "node1 2\nnode2 42\n" + assert ( + node.query( + "SELECT node, SUM(value) FROM {source} GROUP BY node ORDER BY node".format( + source=source + ) + ) + == "node1 2\nnode2 42\n" + ) def test_select_with_group_by_key(self, started_cluster, node, source): - assert node.query("SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format(source=source)) \ - == "0 20\n1 24\n" + assert ( + node.query( + "SELECT key, SUM(value) FROM {source} GROUP BY key ORDER BY key".format( + source=source + ) + ) + == "0 20\n1 24\n" + ) def test_select_sum(self, started_cluster, node, source): - assert node.query("SELECT SUM(value) FROM {source}".format(source=source)) \ - == "44\n" + assert ( + node.query("SELECT SUM(value) FROM {source}".format(source=source)) + == "44\n" + ) diff --git a/tests/integration/test_distributed_queries_stress/test.py b/tests/integration/test_distributed_queries_stress/test.py index 45a1b714cc4..a5df8562676 100644 --- a/tests/integration/test_distributed_queries_stress/test.py +++ b/tests/integration/test_distributed_queries_stress/test.py @@ -8,35 +8,46 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1_r1 = cluster.add_instance('node1_r1', main_configs=['configs/remote_servers.xml']) -node2_r1 = cluster.add_instance('node2_r1', main_configs=['configs/remote_servers.xml']) -node1_r2 = cluster.add_instance('node1_r2', main_configs=['configs/remote_servers.xml']) -node2_r2 = cluster.add_instance('node2_r2', main_configs=['configs/remote_servers.xml']) +node1_r1 = cluster.add_instance("node1_r1", main_configs=["configs/remote_servers.xml"]) +node2_r1 = cluster.add_instance("node2_r1", main_configs=["configs/remote_servers.xml"]) +node1_r2 = cluster.add_instance("node1_r2", main_configs=["configs/remote_servers.xml"]) +node2_r2 = cluster.add_instance("node2_r2", main_configs=["configs/remote_servers.xml"]) + def run_benchmark(payload, settings): - node1_r1.exec_in_container([ - 'bash', '-c', 'echo {} | '.format(shlex.quote(payload.strip())) + ' '.join([ - 'clickhouse', 'benchmark', - '--concurrency=100', - '--cumulative', - '--delay=0', - # NOTE: with current matrix even 3 seconds it huge... - '--timelimit=3', - # tune some basic timeouts - '--hedged_connection_timeout_ms=200', - '--connect_timeout_with_failover_ms=200', - '--connections_with_failover_max_tries=5', - *settings, - ]) - ]) + node1_r1.exec_in_container( + [ + "bash", + "-c", + "echo {} | ".format(shlex.quote(payload.strip())) + + " ".join( + [ + "clickhouse", + "benchmark", + "--concurrency=100", + "--cumulative", + "--delay=0", + # NOTE: with current matrix even 3 seconds it huge... + "--timelimit=3", + # tune some basic timeouts + "--hedged_connection_timeout_ms=200", + "--connect_timeout_with_failover_ms=200", + "--connections_with_failover_max_tries=5", + *settings, + ] + ), + ] + ) -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def started_cluster(): try: cluster.start() for _, instance in cluster.instances.items(): - instance.query(""" + instance.query( + """ create table if not exists data ( key Int, /* just to increase block size */ @@ -60,42 +71,51 @@ def started_cluster(): create table if not exists dist_two as data engine=Distributed(two_shards, currentDatabase(), data, key); create table if not exists dist_two_over_dist as data engine=Distributed(two_shards, currentDatabase(), dist_two, yandexConsistentHash(key, 2)); - """) + """ + ) yield cluster finally: cluster.shutdown() -@pytest.mark.parametrize('table,settings', itertools.product( - [ # tables - 'dist_one', - 'dist_one_over_dist', - 'dist_two', - 'dist_two_over_dist', - ], - [ # settings - *list(itertools.combinations([ - '', # defaults - '--prefer_localhost_replica=0', - '--async_socket_for_remote=0', - '--use_hedged_requests=0', - '--optimize_skip_unused_shards=1', - '--distributed_group_by_no_merge=2', - '--optimize_distributed_group_by_sharding_key=1', - # TODO: enlarge test matrix (but first those values to accept ms): - # - # - sleep_in_send_tables_status - # - sleep_in_send_data - ], 2)) - # TODO: more combinations that just 2 - ], -)) +@pytest.mark.parametrize( + "table,settings", + itertools.product( + [ # tables + "dist_one", + "dist_one_over_dist", + "dist_two", + "dist_two_over_dist", + ], + [ # settings + *list( + itertools.combinations( + [ + "", # defaults + "--prefer_localhost_replica=0", + "--async_socket_for_remote=0", + "--use_hedged_requests=0", + "--optimize_skip_unused_shards=1", + "--distributed_group_by_no_merge=2", + "--optimize_distributed_group_by_sharding_key=1", + # TODO: enlarge test matrix (but first those values to accept ms): + # + # - sleep_in_send_tables_status + # - sleep_in_send_data + ], + 2, + ) + ) + # TODO: more combinations that just 2 + ], + ), +) def test_stress_distributed(table, settings, started_cluster): - payload = f''' + payload = f""" select * from {table} where key = 0; select * from {table} where key = 1; select * from {table} where key = 2; select * from {table} where key = 3; select * from {table}; - ''' + """ run_benchmark(payload, settings) diff --git a/tests/integration/test_distributed_respect_user_timeouts/test.py b/tests/integration/test_distributed_respect_user_timeouts/test.py index a774a01e3c2..9cf7082d63a 100644 --- a/tests/integration/test_distributed_respect_user_timeouts/test.py +++ b/tests/integration/test_distributed_respect_user_timeouts/test.py @@ -10,11 +10,11 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -NODES = {'node' + str(i): None for i in (1, 2)} +NODES = {"node" + str(i): None for i in (1, 2)} IS_DEBUG = False -CREATE_TABLES_SQL = ''' +CREATE_TABLES_SQL = """ CREATE DATABASE test; CREATE TABLE base_table( @@ -26,68 +26,70 @@ ORDER BY node; CREATE TABLE distributed_table ENGINE = Distributed(test_cluster, default, base_table) AS base_table; -''' +""" INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}')" SELECTS_SQL = { - 'distributed': 'SELECT node FROM distributed_table ORDER BY node', - 'remote': ("SELECT node FROM remote('node1,node2', default.base_table) " - "ORDER BY node"), + "distributed": "SELECT node FROM distributed_table ORDER BY node", + "remote": ( + "SELECT node FROM remote('node1,node2', default.base_table) " "ORDER BY node" + ), } -EXCEPTION_NETWORK = 'DB::NetException: ' -EXCEPTION_TIMEOUT = 'Timeout exceeded while reading from socket (' -EXCEPTION_CONNECT = 'Timeout: connect timed out: ' +EXCEPTION_NETWORK = "DB::NetException: " +EXCEPTION_TIMEOUT = "Timeout exceeded while reading from socket (" +EXCEPTION_CONNECT = "Timeout: connect timed out: " TIMEOUT_MEASUREMENT_EPS = 0.01 EXPECTED_BEHAVIOR = { - 'default': { - 'times': 3, - 'timeout': 1, + "default": { + "times": 3, + "timeout": 1, }, - 'ready_to_wait': { - 'times': 5, - 'timeout': 3, + "ready_to_wait": { + "times": 5, + "timeout": 3, }, } TIMEOUT_DIFF_UPPER_BOUND = { - 'default': { - 'distributed': 5.5, - 'remote': 2.5, + "default": { + "distributed": 5.5, + "remote": 2.5, }, - 'ready_to_wait': { - 'distributed': 3, - 'remote': 2.0, + "ready_to_wait": { + "distributed": 3, + "remote": 2.0, }, } def _check_exception(exception, expected_tries=3): - lines = exception.split('\n') + lines = exception.split("\n") assert len(lines) > 4, "Unexpected exception (expected: timeout info)" - assert lines[0].startswith('Received exception from server (version') + assert lines[0].startswith("Received exception from server (version") - assert lines[1].startswith('Code: 279') - assert lines[1].endswith('All connection tries failed. Log: ') + assert lines[1].startswith("Code: 279") + assert lines[1].endswith("All connection tries failed. Log: ") - assert lines[2] == '', "Unexpected exception text (expected: empty line)" + assert lines[2] == "", "Unexpected exception text (expected: empty line)" - for i, line in enumerate(lines[3:3 + expected_tries]): + for i, line in enumerate(lines[3 : 3 + expected_tries]): expected_lines = ( - 'Code: 209. ' + EXCEPTION_NETWORK + EXCEPTION_TIMEOUT, - 'Code: 209. ' + EXCEPTION_NETWORK + EXCEPTION_CONNECT, + "Code: 209. " + EXCEPTION_NETWORK + EXCEPTION_TIMEOUT, + "Code: 209. " + EXCEPTION_NETWORK + EXCEPTION_CONNECT, EXCEPTION_TIMEOUT, ) - assert any(line.startswith(expected) for expected in expected_lines), \ - 'Unexpected exception "{}" at one of the connection attempts'.format(line) + assert any( + line.startswith(expected) for expected in expected_lines + ), 'Unexpected exception "{}" at one of the connection attempts'.format(line) - assert lines[3 + expected_tries] == '', 'Wrong number of connect attempts' + assert lines[3 + expected_tries] == "", "Wrong number of connect attempts" @pytest.fixture(scope="module", params=["configs", "configs_secure"]) @@ -103,14 +105,18 @@ def started_cluster(request): main_configs += [os.path.join(request.param, "config.d/ssl_conf.xml")] user_configs = [os.path.join(request.param, "users.d/set_distributed_defaults.xml")] for name in NODES: - NODES[name] = cluster.add_instance(name, main_configs=main_configs, user_configs=user_configs) + NODES[name] = cluster.add_instance( + name, main_configs=main_configs, user_configs=user_configs + ) try: cluster.start() if cluster.instances["node1"].is_debug_build(): global IS_DEBUG IS_DEBUG = True - logging.warning("Debug build is too slow to show difference in timings. We disable checks.") + logging.warning( + "Debug build is too slow to show difference in timings. We disable checks." + ) for node_id, node in list(NODES.items()): node.query(CREATE_TABLES_SQL) @@ -123,17 +129,17 @@ def started_cluster(request): def _check_timeout_and_exception(node, user, query_base, query): - repeats = EXPECTED_BEHAVIOR[user]['times'] + repeats = EXPECTED_BEHAVIOR[user]["times"] extra_repeats = 1 # Table function remote() are executed two times. # It tries to get table structure from remote shards. # On 'node2' it will firstly try to get structure from 'node1' (which is not available), # so there are 1 extra connection attempts for 'node2' and 'remote' - if node.name == 'node2' and query_base == 'remote': + if node.name == "node2" and query_base == "remote": extra_repeats = 2 - expected_timeout = EXPECTED_BEHAVIOR[user]['timeout'] * repeats * extra_repeats + expected_timeout = EXPECTED_BEHAVIOR[user]["timeout"] * repeats * extra_repeats start = timeit.default_timer() exception = node.query_and_get_error(query, user=user) @@ -143,25 +149,27 @@ def _check_timeout_and_exception(node, user, query_base, query): if not IS_DEBUG: assert expected_timeout - measured_timeout <= TIMEOUT_MEASUREMENT_EPS - assert measured_timeout - expected_timeout <= TIMEOUT_DIFF_UPPER_BOUND[user][query_base] + assert ( + measured_timeout - expected_timeout + <= TIMEOUT_DIFF_UPPER_BOUND[user][query_base] + ) # And exception should reflect connection attempts: _check_exception(exception, repeats) @pytest.mark.parametrize( - ('first_user', 'node_name', 'query_base'), + ("first_user", "node_name", "query_base"), tuple(itertools.product(EXPECTED_BEHAVIOR, NODES, SELECTS_SQL)), ) def test_reconnect(started_cluster, node_name, first_user, query_base): node = NODES[node_name] query = SELECTS_SQL[query_base] if started_cluster.__with_ssl_config: - query = query.replace('remote(', 'remoteSecure(') + query = query.replace("remote(", "remoteSecure(") # Everything is up, select should work: - assert TSV(node.query(query, - user=first_user)) == TSV('node1\nnode2') + assert TSV(node.query(query, user=first_user)) == TSV("node1\nnode2") with PartitionManager() as pm: # Break the connection. @@ -173,11 +181,10 @@ def test_reconnect(started_cluster, node_name, first_user, query_base): # Other user should have different timeout and exception _check_timeout_and_exception( node, - 'default' if first_user != 'default' else 'ready_to_wait', + "default" if first_user != "default" else "ready_to_wait", query_base, query, ) # select should work again: - assert TSV(node.query(query, - user=first_user)) == TSV('node1\nnode2') + assert TSV(node.query(query, user=first_user)) == TSV("node1\nnode2") diff --git a/tests/integration/test_distributed_storage_configuration/test.py b/tests/integration/test_distributed_storage_configuration/test.py index 94beb7b57ca..fa4e01bb7b3 100644 --- a/tests/integration/test_distributed_storage_configuration/test.py +++ b/tests/integration/test_distributed_storage_configuration/test.py @@ -8,34 +8,44 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', - main_configs=["configs/config.d/storage_configuration.xml"], - tmpfs=['/disk1:size=100M', '/disk2:size=100M']) +node = cluster.add_instance( + "node", + main_configs=["configs/config.d/storage_configuration.xml"], + tmpfs=["/disk1:size=100M", "/disk2:size=100M"], +) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def start_cluster(): try: cluster.start() - node.query('CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary') # Different paths with Atomic + node.query( + "CREATE DATABASE IF NOT EXISTS test ENGINE=Ordinary" + ) # Different paths with Atomic yield cluster finally: cluster.shutdown() def _files_in_dist_mon(node, root, table): - return int(node.exec_in_container([ - 'bash', - '-c', - # `-maxdepth 1` to avoid /tmp/ subdirectory - 'find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l'.format( - root=root, table=table) - ]).split('\n')[0]) + return int( + node.exec_in_container( + [ + "bash", + "-c", + # `-maxdepth 1` to avoid /tmp/ subdirectory + "find /{root}/data/test/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f 2>/dev/null | wc -l".format( + root=root, table=table + ), + ] + ).split("\n")[0] + ) def test_insert(start_cluster): - node.query('CREATE TABLE test.foo (key Int) Engine=Memory()') - node.query(""" + node.query("CREATE TABLE test.foo (key Int) Engine=Memory()") + node.query( + """ CREATE TABLE test.dist_foo (key Int) Engine=Distributed( test_cluster_two_shards, @@ -44,41 +54,47 @@ def test_insert(start_cluster): key%2, 'default' ) - """) + """ + ) # manual only (but only for remote node) - node.query('SYSTEM STOP DISTRIBUTED SENDS test.dist_foo') + node.query("SYSTEM STOP DISTRIBUTED SENDS test.dist_foo") - node.query('INSERT INTO test.dist_foo SELECT * FROM numbers(100)', settings={ - 'use_compact_format_in_distributed_parts_names': '0', - }) - assert _files_in_dist_mon(node, 'disk1', 'dist_foo') == 1 - assert _files_in_dist_mon(node, 'disk2', 'dist_foo') == 0 + node.query( + "INSERT INTO test.dist_foo SELECT * FROM numbers(100)", + settings={ + "use_compact_format_in_distributed_parts_names": "0", + }, + ) + assert _files_in_dist_mon(node, "disk1", "dist_foo") == 1 + assert _files_in_dist_mon(node, "disk2", "dist_foo") == 0 - assert node.query('SELECT count() FROM test.dist_foo') == '100\n' - node.query('SYSTEM FLUSH DISTRIBUTED test.dist_foo') - assert node.query('SELECT count() FROM test.dist_foo') == '200\n' + assert node.query("SELECT count() FROM test.dist_foo") == "100\n" + node.query("SYSTEM FLUSH DISTRIBUTED test.dist_foo") + assert node.query("SELECT count() FROM test.dist_foo") == "200\n" # # RENAME # - node.query('RENAME TABLE test.dist_foo TO test.dist2_foo') + node.query("RENAME TABLE test.dist_foo TO test.dist2_foo") - node.query('INSERT INTO test.dist2_foo SELECT * FROM numbers(100)', settings={ - 'use_compact_format_in_distributed_parts_names': '0', - }) - assert _files_in_dist_mon(node, 'disk1', 'dist2_foo') == 0 - assert _files_in_dist_mon(node, 'disk2', 'dist2_foo') == 1 + node.query( + "INSERT INTO test.dist2_foo SELECT * FROM numbers(100)", + settings={ + "use_compact_format_in_distributed_parts_names": "0", + }, + ) + assert _files_in_dist_mon(node, "disk1", "dist2_foo") == 0 + assert _files_in_dist_mon(node, "disk2", "dist2_foo") == 1 - assert node.query('SELECT count() FROM test.dist2_foo') == '300\n' - node.query('SYSTEM FLUSH DISTRIBUTED test.dist2_foo') - assert node.query('SELECT count() FROM test.dist2_foo') == '400\n' + assert node.query("SELECT count() FROM test.dist2_foo") == "300\n" + node.query("SYSTEM FLUSH DISTRIBUTED test.dist2_foo") + assert node.query("SELECT count() FROM test.dist2_foo") == "400\n" # # DROP # - node.query('DROP TABLE test.dist2_foo') - for disk in ['disk1', 'disk2']: - node.exec_in_container([ - 'bash', '-c', - 'test ! -e /{}/data/test/dist2_foo'.format(disk) - ]) + node.query("DROP TABLE test.dist2_foo") + for disk in ["disk1", "disk2"]: + node.exec_in_container( + ["bash", "-c", "test ! -e /{}/data/test/dist2_foo".format(disk)] + ) diff --git a/tests/integration/test_distributed_system_query/test.py b/tests/integration/test_distributed_system_query/test.py index bf643fabf86..d221aa90dcb 100644 --- a/tests/integration/test_distributed_system_query/test.py +++ b/tests/integration/test_distributed_system_query/test.py @@ -4,8 +4,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) +node1 = cluster.add_instance("node1", main_configs=["configs/remote_servers.xml"]) +node2 = cluster.add_instance("node2", main_configs=["configs/remote_servers.xml"]) @pytest.fixture(scope="module") @@ -14,10 +14,13 @@ def started_cluster(): cluster.start() for node in (node1, node2): - node.query('''CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id;''') + node.query( + """CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id;""" + ) node1.query( - '''CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);''') + """CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, id);""" + ) yield cluster @@ -32,8 +35,8 @@ def test_start_and_stop_replica_send(started_cluster): node1.query("INSERT INTO distributed_table VALUES (1, 'node2')") # Write only to this node when stop distributed sends - assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == '1' + assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == "1" node1.query("SYSTEM START DISTRIBUTED SENDS distributed_table;") node1.query("SYSTEM FLUSH DISTRIBUTED distributed_table;") - assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == '2' + assert node1.query("SELECT COUNT() FROM distributed_table").rstrip() == "2" diff --git a/tests/integration/test_distributed_type_object/__init__.py b/tests/integration/test_distributed_type_object/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_query_deduplication/configs/remote_servers.xml b/tests/integration/test_distributed_type_object/configs/remote_servers.xml similarity index 68% rename from tests/integration/test_query_deduplication/configs/remote_servers.xml rename to tests/integration/test_distributed_type_object/configs/remote_servers.xml index 0caa824c3a1..ebce4697529 100644 --- a/tests/integration/test_query_deduplication/configs/remote_servers.xml +++ b/tests/integration/test_distributed_type_object/configs/remote_servers.xml @@ -1,4 +1,4 @@ - + @@ -13,12 +13,6 @@ 9000 - - - node3 - 9000 - - - + diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py new file mode 100644 index 00000000000..b2179af8a3f --- /dev/null +++ b/tests/integration/test_distributed_type_object/test.py @@ -0,0 +1,88 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance("node1", main_configs=["configs/remote_servers.xml"]) +node2 = cluster.add_instance("node2", main_configs=["configs/remote_servers.xml"]) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in (node1, node2): + node.query( + "CREATE TABLE local_table(id UInt32, data JSON) ENGINE = MergeTree ORDER BY id", + settings={"allow_experimental_object_type": 1}, + ) + node.query( + "CREATE TABLE dist_table AS local_table ENGINE = Distributed(test_cluster, default, local_table)", + settings={"allow_experimental_object_type": 1}, + ) + + yield cluster + + finally: + cluster.shutdown() + + +def test_distributed_type_object(started_cluster): + node1.query( + 'INSERT INTO local_table FORMAT JSONEachRow {"id": 1, "data": {"k1": 10}}' + ) + node2.query( + 'INSERT INTO local_table FORMAT JSONEachRow {"id": 2, "data": {"k1": 20}}' + ) + + expected = TSV("10\n20\n") + assert TSV(node1.query("SELECT data.k1 FROM dist_table ORDER BY id")) == expected + + node1.query( + 'INSERT INTO local_table FORMAT JSONEachRow {"id": 3, "data": {"k1": "str1"}}' + ) + + expected = TSV("10\n20\nstr1\n") + assert TSV(node1.query("SELECT data.k1 FROM dist_table ORDER BY id")) == expected + + node1.query( + 'INSERT INTO local_table FORMAT JSONEachRow {"id": 4, "data": {"k2": 30}}' + ) + + expected = TSV("10\t0\n20\t0\nstr1\t0\n\t30") + assert ( + TSV(node1.query("SELECT data.k1, data.k2 FROM dist_table ORDER BY id")) + == expected + ) + + expected = TSV("120\n") + assert TSV(node1.query("SELECT sum(data.k2 * id) FROM dist_table")) == expected + + node1.query("TRUNCATE TABLE local_table") + node2.query("TRUNCATE TABLE local_table") + + node1.query( + 'INSERT INTO local_table FORMAT JSONEachRow {"id": 1, "data": {"k1": "aa", "k2": {"k3": "bb", "k4": "c"}}} {"id": 2, "data": {"k1": "ee", "k5": "ff"}};' + ) + node2.query( + 'INSERT INTO local_table FORMAT JSONEachRow {"id": 3, "data": {"k5":"foo"}};' + ) + + expected = TSV( + """ +1\taa\tbb\tc\t +2\tee\t\t\tff +3\t\t\t\tfoo""" + ) + + assert ( + TSV( + node1.query( + "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id" + ) + ) + == expected + ) diff --git a/tests/integration/test_dotnet_client/configs/config.xml b/tests/integration/test_dotnet_client/configs/config.xml index 9bcadc43f10..fe64e47d384 100644 --- a/tests/integration/test_dotnet_client/configs/config.xml +++ b/tests/integration/test_dotnet_client/configs/config.xml @@ -1,5 +1,6 @@ + trace /var/log/clickhouse-server/clickhouse-server.log diff --git a/tests/integration/test_dotnet_client/dotnet.reference b/tests/integration/test_dotnet_client/dotnet.reference index a3d6e1d5ba8..8f1d786a237 100644 Binary files a/tests/integration/test_dotnet_client/dotnet.reference and b/tests/integration/test_dotnet_client/dotnet.reference differ diff --git a/tests/integration/test_dotnet_client/test.py b/tests/integration/test_dotnet_client/test.py index 4cc16ac826e..b147688c099 100644 --- a/tests/integration/test_dotnet_client/test.py +++ b/tests/integration/test_dotnet_client/test.py @@ -15,8 +15,12 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', - user_configs=["configs/users.xml"], env_variables={'UBSAN_OPTIONS': 'print_stacktrace=1'}) +node = cluster.add_instance( + "node", + user_configs=["configs/users.xml"], + env_variables={"UBSAN_OPTIONS": "print_stacktrace=1"}, +) + @pytest.fixture(scope="module") def started_cluster(): @@ -27,21 +31,37 @@ def started_cluster(): cluster.shutdown() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def dotnet_container(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_dotnet_client.yml') + docker_compose = os.path.join( + DOCKER_COMPOSE_PATH, "docker_compose_dotnet_client.yml" + ) run_and_check( - ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) - yield docker.from_env().containers.get(cluster.project_name + '_dotnet1_1') + [ + "docker-compose", + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--no-build", + ] + ) + yield docker.from_env().containers.get(cluster.project_name + "_dotnet1_1") def test_dotnet_client(started_cluster, dotnet_container): - with open(os.path.join(SCRIPT_DIR, 'dotnet.reference'), 'rb') as fp: + with open(os.path.join(SCRIPT_DIR, "dotnet.reference"), "rb") as fp: reference = fp.read() code, (stdout, stderr) = dotnet_container.exec_run( - 'dotnet run --host {host} --port {port} --user default --password 123 --database default' - .format(host=started_cluster.get_instance_ip('node'), port=8123), demux=True) + "dotnet run --host {host} --port {port} --user default --password 123 --database default".format( + host=started_cluster.get_instance_ip("node"), port=8123 + ), + demux=True, + ) assert code == 0 assert stdout == reference diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py index eb67a25f9f5..1fa086a4217 100644 --- a/tests/integration/test_drop_replica/test.py +++ b/tests/integration/test_drop_replica/test.py @@ -8,50 +8,71 @@ from helpers.network import PartitionManager def fill_nodes(nodes, shard): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test.test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) node.query( - ''' + """ CREATE DATABASE test1; CREATE TABLE test1.test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test1/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) node.query( - ''' + """ CREATE DATABASE test2; CREATE TABLE test2.test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test2/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) node.query( - ''' + """ CREATE DATABASE test3; CREATE TABLE test3.test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test3/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) node.query( - ''' + """ CREATE DATABASE test4; CREATE TABLE test4.test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test4/{shard}/replicated/test_table', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) cluster = ClickHouseCluster(__file__) -node_1_1 = cluster.add_instance('node_1_1', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) -node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) -node_1_3 = cluster.add_instance('node_1_3', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) +node_1_1 = cluster.add_instance( + "node_1_1", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) +node_1_2 = cluster.add_instance( + "node_1_2", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) +node_1_3 = cluster.add_instance( + "node_1_3", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) @pytest.fixture(scope="module") @@ -71,82 +92,125 @@ def start_cluster(): def test_drop_replica(start_cluster): - node_1_1.query("INSERT INTO test.test_table SELECT number, toString(number) FROM numbers(100)") - node_1_1.query("INSERT INTO test1.test_table SELECT number, toString(number) FROM numbers(100)") - node_1_1.query("INSERT INTO test2.test_table SELECT number, toString(number) FROM numbers(100)") - node_1_1.query("INSERT INTO test3.test_table SELECT number, toString(number) FROM numbers(100)") - node_1_1.query("INSERT INTO test4.test_table SELECT number, toString(number) FROM numbers(100)") + node_1_1.query( + "INSERT INTO test.test_table SELECT number, toString(number) FROM numbers(100)" + ) + node_1_1.query( + "INSERT INTO test1.test_table SELECT number, toString(number) FROM numbers(100)" + ) + node_1_1.query( + "INSERT INTO test2.test_table SELECT number, toString(number) FROM numbers(100)" + ) + node_1_1.query( + "INSERT INTO test3.test_table SELECT number, toString(number) FROM numbers(100)" + ) + node_1_1.query( + "INSERT INTO test4.test_table SELECT number, toString(number) FROM numbers(100)" + ) - zk = cluster.get_kazoo_client('zoo1') - assert "can't drop local replica" in node_1_1.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'") + zk = cluster.get_kazoo_client("zoo1") assert "can't drop local replica" in node_1_1.query_and_get_error( - "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") + "SYSTEM DROP REPLICA 'node_1_1'" + ) assert "can't drop local replica" in node_1_1.query_and_get_error( - "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") - assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1'") - assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test") - assert "it's active" in node_1_2.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") - assert "it's active" in \ - node_1_3.query_and_get_error( - "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( - shard=1)) - assert "There is a local table" in \ - node_1_2.query_and_get_error( - "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( - shard=1)) - assert "There is a local table" in \ - node_1_1.query_and_get_error( - "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( - shard=1)) - assert "does not look like a table path" in \ - node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'") + "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test" + ) + assert "can't drop local replica" in node_1_1.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table" + ) + assert "it's active" in node_1_2.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1'" + ) + assert "it's active" in node_1_2.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test" + ) + assert "it's active" in node_1_2.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table" + ) + assert "it's active" in node_1_3.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( + shard=1 + ) + ) + assert "There is a local table" in node_1_2.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( + shard=1 + ) + ) + assert "There is a local table" in node_1_1.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test/{shard}/replicated/test_table'".format( + shard=1 + ) + ) + assert "does not look like a table path" in node_1_3.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'" + ) node_1_1.query("DETACH DATABASE test") for i in range(1, 5): node_1_1.query("DETACH DATABASE test{}".format(i)) assert "doesn't exist" in node_1_3.query_and_get_error( - "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") + "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table" + ) - assert "doesn't exist" in node_1_3.query_and_get_error("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1") + assert "doesn't exist" in node_1_3.query_and_get_error( + "SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1" + ) node_1_3.query("SYSTEM DROP REPLICA 'node_1_1'") exists_replica_1_1 = zk.exists( - "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, - replica='node_1_1')) - assert (exists_replica_1_1 != None) + "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format( + shard=1, replica="node_1_1" + ) + ) + assert exists_replica_1_1 != None ## If you want to drop a inactive/stale replicate table that does not have a local replica, you can following syntax(ZKPATH): node_1_3.query( "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test2/{shard}/replicated/test_table'".format( - shard=1)) + shard=1 + ) + ) exists_replica_1_1 = zk.exists( - "/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, - replica='node_1_1')) - assert (exists_replica_1_1 == None) + "/clickhouse/tables/test2/{shard}/replicated/test_table/replicas/{replica}".format( + shard=1, replica="node_1_1" + ) + ) + assert exists_replica_1_1 == None node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table") exists_replica_1_1 = zk.exists( - "/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, - replica='node_1_1')) - assert (exists_replica_1_1 == None) + "/clickhouse/tables/test/{shard}/replicated/test_table/replicas/{replica}".format( + shard=1, replica="node_1_1" + ) + ) + assert exists_replica_1_1 == None node_1_2.query("SYSTEM DROP REPLICA 'node_1_1' FROM DATABASE test1") exists_replica_1_1 = zk.exists( - "/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, - replica='node_1_1')) - assert (exists_replica_1_1 == None) + "/clickhouse/tables/test1/{shard}/replicated/test_table/replicas/{replica}".format( + shard=1, replica="node_1_1" + ) + ) + assert exists_replica_1_1 == None node_1_3.query( "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test3/{shard}/replicated/test_table'".format( - shard=1)) + shard=1 + ) + ) exists_replica_1_1 = zk.exists( - "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, - replica='node_1_1')) - assert (exists_replica_1_1 == None) + "/clickhouse/tables/test3/{shard}/replicated/test_table/replicas/{replica}".format( + shard=1, replica="node_1_1" + ) + ) + assert exists_replica_1_1 == None node_1_2.query("SYSTEM DROP REPLICA 'node_1_1'") exists_replica_1_1 = zk.exists( - "/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format(shard=1, - replica='node_1_1')) - assert (exists_replica_1_1 == None) + "/clickhouse/tables/test4/{shard}/replicated/test_table/replicas/{replica}".format( + shard=1, replica="node_1_1" + ) + ) + assert exists_replica_1_1 == None diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index e93a643cd16..0b8c1771a40 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -2,7 +2,9 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', user_configs=["configs/users.d/extra_users.xml"]) +instance = cluster.add_instance( + "instance", user_configs=["configs/users.d/extra_users.xml"] +) @pytest.fixture(scope="module", autouse=True) @@ -16,10 +18,20 @@ def started_cluster(): def test_enabling_access_management(): - instance.query("CREATE USER Alex", user='default') - assert instance.query("SHOW CREATE USER Alex", user='default') == "CREATE USER Alex\n" - assert instance.query("SHOW CREATE USER Alex", user='readonly') == "CREATE USER Alex\n" - assert "Not enough privileges" in instance.query_and_get_error("SHOW CREATE USER Alex", user='xyz') + instance.query("CREATE USER Alex", user="default") + assert ( + instance.query("SHOW CREATE USER Alex", user="default") == "CREATE USER Alex\n" + ) + assert ( + instance.query("SHOW CREATE USER Alex", user="readonly") == "CREATE USER Alex\n" + ) + assert "Not enough privileges" in instance.query_and_get_error( + "SHOW CREATE USER Alex", user="xyz" + ) - assert "Cannot execute query in readonly mode" in instance.query_and_get_error("CREATE USER Robin", user='readonly') - assert "Not enough privileges" in instance.query_and_get_error("CREATE USER Robin", user='xyz') + assert "Cannot execute query in readonly mode" in instance.query_and_get_error( + "CREATE USER Robin", user="readonly" + ) + assert "Not enough privileges" in instance.query_and_get_error( + "CREATE USER Robin", user="xyz" + ) diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index 7d94f7ccdc5..4e6d1db9e99 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -7,10 +7,12 @@ from helpers.test_tools import assert_eq_with_retry FIRST_PART_NAME = "all_1_1_0" cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", - main_configs=["configs/storage.xml"], - tmpfs=["/disk:size=100M"], - with_minio=True) +node = cluster.add_instance( + "node", + main_configs=["configs/storage.xml"], + tmpfs=["/disk:size=100M"], + with_minio=True, +) @pytest.fixture(scope="module", autouse=True) @@ -30,7 +32,10 @@ def cleanup_after_test(): node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY") -@pytest.mark.parametrize("policy", ["encrypted_policy", "encrypted_policy_key192b", "local_policy", "s3_policy"]) +@pytest.mark.parametrize( + "policy", + ["encrypted_policy", "encrypted_policy_key192b", "local_policy", "s3_policy"], +) def test_encrypted_disk(policy): node.query( """ @@ -40,7 +45,9 @@ def test_encrypted_disk(policy): ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='{}' - """.format(policy) + """.format( + policy + ) ) node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") @@ -52,7 +59,21 @@ def test_encrypted_disk(policy): assert node.query(select_query) == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" -@pytest.mark.parametrize("policy, destination_disks", [("local_policy", ["disk_local_encrypted", "disk_local_encrypted2", "disk_local_encrypted_key192b", "disk_local"]), ("s3_policy", ["disk_s3_encrypted", "disk_s3"])]) +@pytest.mark.parametrize( + "policy, destination_disks", + [ + ( + "local_policy", + [ + "disk_local_encrypted", + "disk_local_encrypted2", + "disk_local_encrypted_key192b", + "disk_local", + ], + ), + ("s3_policy", ["disk_s3_encrypted", "disk_s3"]), + ], +) def test_part_move(policy, destination_disks): node.query( """ @@ -62,7 +83,9 @@ def test_part_move(policy, destination_disks): ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='{}' - """.format(policy) + """.format( + policy + ) ) node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") @@ -70,16 +93,29 @@ def test_part_move(policy, destination_disks): assert node.query(select_query) == "(0,'data'),(1,'data')" for destination_disk in destination_disks: - node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, destination_disk)) + node.query( + "ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format( + FIRST_PART_NAME, destination_disk + ) + ) assert node.query(select_query) == "(0,'data'),(1,'data')" with pytest.raises(QueryRuntimeException) as exc: - node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, destination_disk)) - assert("Part '{}' is already on disk '{}'".format(FIRST_PART_NAME, destination_disk) in str(exc.value)) + node.query( + "ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format( + FIRST_PART_NAME, destination_disk + ) + ) + assert "Part '{}' is already on disk '{}'".format( + FIRST_PART_NAME, destination_disk + ) in str(exc.value) assert node.query(select_query) == "(0,'data'),(1,'data')" -@pytest.mark.parametrize("policy,encrypted_disk", [("local_policy", "disk_local_encrypted"), ("s3_policy", "disk_s3_encrypted")]) +@pytest.mark.parametrize( + "policy,encrypted_disk", + [("local_policy", "disk_local_encrypted"), ("s3_policy", "disk_s3_encrypted")], +) def test_optimize_table(policy, encrypted_disk): node.query( """ @@ -89,23 +125,35 @@ def test_optimize_table(policy, encrypted_disk): ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='{}' - """.format(policy) + """.format( + policy + ) ) node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" assert node.query(select_query) == "(0,'data'),(1,'data')" - node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk)) + node.query( + "ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format( + FIRST_PART_NAME, encrypted_disk + ) + ) assert node.query(select_query) == "(0,'data'),(1,'data')" node.query("INSERT INTO encrypted_test VALUES (2,'data'),(3,'data')") node.query("OPTIMIZE TABLE encrypted_test FINAL") with pytest.raises(QueryRuntimeException) as exc: - node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk)) + node.query( + "ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format( + FIRST_PART_NAME, encrypted_disk + ) + ) - assert("Part {} is not exists or not active".format(FIRST_PART_NAME) in str(exc.value)) + assert "Part {} is not exists or not active".format(FIRST_PART_NAME) in str( + exc.value + ) assert node.query(select_query) == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" @@ -113,7 +161,11 @@ def test_optimize_table(policy, encrypted_disk): # Test adding encryption key on the fly. def test_add_key(): def make_storage_policy_with_keys(policy_name, keys): - node.exec_in_container(["bash", "-c" , """cat > /etc/clickhouse-server/config.d/storage_policy_{policy_name}.xml << EOF + node.exec_in_container( + [ + "bash", + "-c", + """cat > /etc/clickhouse-server/config.d/storage_policy_{policy_name}.xml << EOF @@ -136,33 +188,48 @@ def test_add_key(): -EOF""".format(policy_name=policy_name, keys=keys)]) +EOF""".format( + policy_name=policy_name, keys=keys + ), + ] + ) node.query("SYSTEM RELOAD CONFIG") # Add some data to an encrypted disk. node.query("SELECT policy_name FROM system.storage_policies") - make_storage_policy_with_keys("encrypted_policy_multikeys", "firstfirstfirstf") - assert_eq_with_retry(node, "SELECT policy_name FROM system.storage_policies WHERE policy_name='encrypted_policy_multikeys'", "encrypted_policy_multikeys") - - node.query(""" + make_storage_policy_with_keys( + "encrypted_policy_multikeys", "firstfirstfirstf" + ) + assert_eq_with_retry( + node, + "SELECT policy_name FROM system.storage_policies WHERE policy_name='encrypted_policy_multikeys'", + "encrypted_policy_multikeys", + ) + + node.query( + """ CREATE TABLE encrypted_test ( id Int64, data String ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='encrypted_policy_multikeys' - """) + """ + ) node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" assert node.query(select_query) == "(0,'data'),(1,'data')" # Add a second key and start using it. - make_storage_policy_with_keys("encrypted_policy_multikeys", """ + make_storage_policy_with_keys( + "encrypted_policy_multikeys", + """ firstfirstfirstf secondsecondseco 1 - """) + """, + ) node.query("INSERT INTO encrypted_test VALUES (2,'data'),(3,'data')") # Now "(0,'data'),(1,'data')" is encrypted with the first key and "(2,'data'),(3,'data')" is encrypted with the second key. @@ -170,11 +237,14 @@ EOF""".format(policy_name=policy_name, keys=keys)]) assert node.query(select_query) == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" # Try to replace the first key with something wrong, and check that "(0,'data'),(1,'data')" cannot be read. - make_storage_policy_with_keys("encrypted_policy_multikeys", """ + make_storage_policy_with_keys( + "encrypted_policy_multikeys", + """ wrongwrongwrongw secondsecondseco 1 - """) + """, + ) expected_error = "Wrong key" assert expected_error in node.query_and_get_error(select_query) diff --git a/tests/integration/test_executable_dictionary/test.py b/tests/integration/test_executable_dictionary/test.py index 5e50a092a29..43e6ec0a800 100644 --- a/tests/integration/test_executable_dictionary/test.py +++ b/tests/integration/test_executable_dictionary/test.py @@ -10,29 +10,46 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=[]) +node = cluster.add_instance("node", stay_alive=True, main_configs=[]) def skip_test_msan(instance): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with vfork") -def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) -config = ''' +def copy_file_to_container(local_path, dist_path, container_id): + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) + + +config = """ /etc/clickhouse-server/dictionaries/*_dictionary.xml -''' +""" + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node.replace_config("/etc/clickhouse-server/config.d/dictionaries_config.xml", config) + node.replace_config( + "/etc/clickhouse-server/config.d/dictionaries_config.xml", config + ) - copy_file_to_container(os.path.join(SCRIPT_DIR, 'dictionaries/.'), '/etc/clickhouse-server/dictionaries', node.docker_id) - copy_file_to_container(os.path.join(SCRIPT_DIR, 'user_scripts/.'), '/var/lib/clickhouse/user_scripts', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "dictionaries/."), + "/etc/clickhouse-server/dictionaries", + node.docker_id, + ) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "user_scripts/."), + "/var/lib/clickhouse/user_scripts", + node.docker_id, + ) node.restart_clickhouse() @@ -41,135 +58,427 @@ def started_cluster(): finally: cluster.shutdown() + def test_executable_input_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_bash', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_input_pool_bash', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query("SELECT dictGet('executable_input_bash', 'result', toUInt64(1))") + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_pool_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_implicit_input_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_implicit_input_bash', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_implicit_input_pool_bash', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_pool_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_input_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_python', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_input_pool_python', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query("SELECT dictGet('executable_input_python', 'result', toUInt64(1))") + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_pool_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_implicit_input_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_implicit_input_python', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_implicit_input_pool_python', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_pool_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_input_send_chunk_header_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_send_chunk_header_python', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_input_send_chunk_header_pool_python', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query( + "SELECT dictGet('executable_input_send_chunk_header_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_send_chunk_header_pool_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_implicit_input_send_chunk_header_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_implicit_input_send_chunk_header_python', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_implicit_input_send_chunk_header_pool_python', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_send_chunk_header_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_send_chunk_header_pool_python', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_input_sum_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_sum_python', 'result', tuple(toUInt64(1), toUInt64(1)))") == '2\n' - assert node.query("SELECT dictGet('executable_input_sum_pool_python', 'result', tuple(toUInt64(1), toUInt64(1)))") == '2\n' + assert ( + node.query( + "SELECT dictGet('executable_input_sum_python', 'result', tuple(toUInt64(1), toUInt64(1)))" + ) + == "2\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_sum_pool_python', 'result', tuple(toUInt64(1), toUInt64(1)))" + ) + == "2\n" + ) + def test_executable_implicit_input_sum_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_implicit_input_sum_python', 'result', tuple(toUInt64(1), toUInt64(1)))") == '2\n' - assert node.query("SELECT dictGet('executable_implicit_input_sum_pool_python', 'result', tuple(toUInt64(1), toUInt64(1)))") == '2\n' + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_sum_python', 'result', tuple(toUInt64(1), toUInt64(1)))" + ) + == "2\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_sum_pool_python', 'result', tuple(toUInt64(1), toUInt64(1)))" + ) + == "2\n" + ) + def test_executable_input_argument_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_argument_python', 'result', toUInt64(1))") == 'Key 1 1\n' - assert node.query("SELECT dictGet('executable_input_argument_pool_python', 'result', toUInt64(1))") == 'Key 1 1\n' + assert ( + node.query( + "SELECT dictGet('executable_input_argument_python', 'result', toUInt64(1))" + ) + == "Key 1 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_argument_pool_python', 'result', toUInt64(1))" + ) + == "Key 1 1\n" + ) + def test_executable_implicit_input_argument_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_implicit_input_argument_python', 'result', toUInt64(1))") == 'Key 1 1\n' - assert node.query("SELECT dictGet('executable_implicit_input_argument_pool_python', 'result', toUInt64(1))") == 'Key 1 1\n' + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_argument_python', 'result', toUInt64(1))" + ) + == "Key 1 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_argument_pool_python', 'result', toUInt64(1))" + ) + == "Key 1 1\n" + ) + def test_executable_input_signalled_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_signalled_python', 'result', toUInt64(1))") == 'Default result\n' - assert node.query("SELECT dictGet('executable_input_signalled_pool_python', 'result', toUInt64(1))") == 'Default result\n' + assert ( + node.query( + "SELECT dictGet('executable_input_signalled_python', 'result', toUInt64(1))" + ) + == "Default result\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_signalled_pool_python', 'result', toUInt64(1))" + ) + == "Default result\n" + ) + def test_executable_implicit_input_signalled_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_implicit_input_signalled_python', 'result', toUInt64(1))") == 'Default result\n' - assert node.query("SELECT dictGet('executable_implicit_input_signalled_pool_python', 'result', toUInt64(1))") == 'Default result\n' + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_signalled_python', 'result', toUInt64(1))" + ) + == "Default result\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_implicit_input_signalled_pool_python', 'result', toUInt64(1))" + ) + == "Default result\n" + ) + def test_executable_input_slow_python(started_cluster): skip_test_msan(node) - assert node.query_and_get_error("SELECT dictGet('executable_input_slow_python', 'result', toUInt64(1))") - assert node.query_and_get_error("SELECT dictGet('executable_input_slow_pool_python', 'result', toUInt64(1))") + assert node.query_and_get_error( + "SELECT dictGet('executable_input_slow_python', 'result', toUInt64(1))" + ) + assert node.query_and_get_error( + "SELECT dictGet('executable_input_slow_pool_python', 'result', toUInt64(1))" + ) + def test_executable_implicit_input_slow_python(started_cluster): skip_test_msan(node) - assert node.query_and_get_error("SELECT dictGet('executable_implicit_input_slow_python', 'result', toUInt64(1))") - assert node.query_and_get_error("SELECT dictGet('executable_implicit_input_slow_pool_python', 'result', toUInt64(1))") + assert node.query_and_get_error( + "SELECT dictGet('executable_implicit_input_slow_python', 'result', toUInt64(1))" + ) + assert node.query_and_get_error( + "SELECT dictGet('executable_implicit_input_slow_pool_python', 'result', toUInt64(1))" + ) + def test_executable_input_slow_python(started_cluster): skip_test_msan(node) - assert node.query_and_get_error("SELECT dictGet('executable_input_slow_python', 'result', toUInt64(1))") - assert node.query_and_get_error("SELECT dictGet('executable_input_slow_pool_python', 'result', toUInt64(1))") + assert node.query_and_get_error( + "SELECT dictGet('executable_input_slow_python', 'result', toUInt64(1))" + ) + assert node.query_and_get_error( + "SELECT dictGet('executable_input_slow_pool_python', 'result', toUInt64(1))" + ) + def test_executable_implicit_input_slow_python(started_cluster): skip_test_msan(node) - assert node.query_and_get_error("SELECT dictGet('executable_implicit_input_slow_python', 'result', toUInt64(1))") - assert node.query_and_get_error("SELECT dictGet('executable_implicit_input_slow_pool_python', 'result', toUInt64(1))") + assert node.query_and_get_error( + "SELECT dictGet('executable_implicit_input_slow_python', 'result', toUInt64(1))" + ) + assert node.query_and_get_error( + "SELECT dictGet('executable_implicit_input_slow_pool_python', 'result', toUInt64(1))" + ) + def test_executable_non_direct_input_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_non_direct_bash', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_input_non_direct_pool_bash', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query( + "SELECT dictGet('executable_input_non_direct_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_non_direct_pool_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_implicit_non_direct_input_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT dictGet('executable_input_implicit_non_direct_bash', 'result', toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT dictGet('executable_input_implicit_non_direct_pool_bash', 'result', toUInt64(1))") == 'Key 1\n' + assert ( + node.query( + "SELECT dictGet('executable_input_implicit_non_direct_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_input_implicit_non_direct_pool_bash', 'result', toUInt64(1))" + ) + == "Key 1\n" + ) + def test_executable_source_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT * FROM dictionary(executable_source_simple_key_python) ORDER BY input") == '1\tValue 1\n2\tValue 2\n3\tValue 3\n' - assert node.query("SELECT dictGet('executable_source_simple_key_python', 'result', toUInt64(1))") == 'Value 1\n' - assert node.query("SELECT dictGet('executable_source_simple_key_python', 'result', toUInt64(2))") == 'Value 2\n' - assert node.query("SELECT dictGet('executable_source_simple_key_python', 'result', toUInt64(3))") == 'Value 3\n' + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_simple_key_python) ORDER BY input" + ) + == "1\tValue 1\n2\tValue 2\n3\tValue 3\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_python', 'result', toUInt64(1))" + ) + == "Value 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_python', 'result', toUInt64(2))" + ) + == "Value 2\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_python', 'result', toUInt64(3))" + ) + == "Value 3\n" + ) + + assert ( + node.query( + "SELECT * FROM dictionary('executable_source_complex_key_python') ORDER BY input" + ) + == "1\tValue 1\n2\tValue 2\n3\tValue 3\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_python', 'result', tuple(toUInt64(1)))" + ) + == "Value 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_python', 'result', tuple(toUInt64(2)))" + ) + == "Value 2\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_python', 'result', tuple(toUInt64(3)))" + ) + == "Value 3\n" + ) - assert node.query("SELECT * FROM dictionary('executable_source_complex_key_python') ORDER BY input") == '1\tValue 1\n2\tValue 2\n3\tValue 3\n' - assert node.query("SELECT dictGet('executable_source_complex_key_python', 'result', tuple(toUInt64(1)))") == 'Value 1\n' - assert node.query("SELECT dictGet('executable_source_complex_key_python', 'result', tuple(toUInt64(2)))") == 'Value 2\n' - assert node.query("SELECT dictGet('executable_source_complex_key_python', 'result', tuple(toUInt64(3)))") == 'Value 3\n' def test_executable_source_argument_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT * FROM dictionary(executable_source_simple_key_argument_python) ORDER BY input") == '1\tValue 1 1\n2\tValue 1 2\n3\tValue 1 3\n' - assert node.query("SELECT dictGet('executable_source_simple_key_argument_python', 'result', toUInt64(1))") == 'Value 1 1\n' - assert node.query("SELECT dictGet('executable_source_simple_key_argument_python', 'result', toUInt64(2))") == 'Value 1 2\n' - assert node.query("SELECT dictGet('executable_source_simple_key_argument_python', 'result', toUInt64(3))") == 'Value 1 3\n' + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_simple_key_argument_python) ORDER BY input" + ) + == "1\tValue 1 1\n2\tValue 1 2\n3\tValue 1 3\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_argument_python', 'result', toUInt64(1))" + ) + == "Value 1 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_argument_python', 'result', toUInt64(2))" + ) + == "Value 1 2\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_argument_python', 'result', toUInt64(3))" + ) + == "Value 1 3\n" + ) + + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_complex_key_argument_python) ORDER BY input" + ) + == "1\tValue 1 1\n2\tValue 1 2\n3\tValue 1 3\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_argument_python', 'result', toUInt64(1))" + ) + == "Value 1 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_argument_python', 'result', toUInt64(2))" + ) + == "Value 1 2\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_argument_python', 'result', toUInt64(3))" + ) + == "Value 1 3\n" + ) - assert node.query("SELECT * FROM dictionary(executable_source_complex_key_argument_python) ORDER BY input") == '1\tValue 1 1\n2\tValue 1 2\n3\tValue 1 3\n' - assert node.query("SELECT dictGet('executable_source_complex_key_argument_python', 'result', toUInt64(1))") == 'Value 1 1\n' - assert node.query("SELECT dictGet('executable_source_complex_key_argument_python', 'result', toUInt64(2))") == 'Value 1 2\n' - assert node.query("SELECT dictGet('executable_source_complex_key_argument_python', 'result', toUInt64(3))") == 'Value 1 3\n' def test_executable_source_updated_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT * FROM dictionary(executable_source_simple_key_update_python) ORDER BY input") == '1\tValue 0 1\n' - assert node.query("SELECT dictGet('executable_source_simple_key_update_python', 'result', toUInt64(1))") == 'Value 0 1\n' + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_simple_key_update_python) ORDER BY input" + ) + == "1\tValue 0 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_update_python', 'result', toUInt64(1))" + ) + == "Value 0 1\n" + ) time.sleep(10) - assert node.query("SELECT * FROM dictionary(executable_source_simple_key_update_python) ORDER BY input") == '1\tValue 1 1\n' - assert node.query("SELECT dictGet('executable_source_simple_key_update_python', 'result', toUInt64(1))") == 'Value 1 1\n' + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_simple_key_update_python) ORDER BY input" + ) + == "1\tValue 1 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_simple_key_update_python', 'result', toUInt64(1))" + ) + == "Value 1 1\n" + ) - assert node.query("SELECT * FROM dictionary(executable_source_complex_key_update_python) ORDER BY input") == '1\tValue 0 1\n' - assert node.query("SELECT dictGet('executable_source_complex_key_update_python', 'result', toUInt64(1))") == 'Value 0 1\n' + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_complex_key_update_python) ORDER BY input" + ) + == "1\tValue 0 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_update_python', 'result', toUInt64(1))" + ) + == "Value 0 1\n" + ) time.sleep(10) - assert node.query("SELECT * FROM dictionary(executable_source_complex_key_update_python) ORDER BY input") == '1\tValue 1 1\n' - assert node.query("SELECT dictGet('executable_source_complex_key_update_python', 'result', toUInt64(1))") == 'Value 1 1\n' - + assert ( + node.query( + "SELECT * FROM dictionary(executable_source_complex_key_update_python) ORDER BY input" + ) + == "1\tValue 1 1\n" + ) + assert ( + node.query( + "SELECT dictGet('executable_source_complex_key_update_python', 'result', toUInt64(1))" + ) + == "Value 1 1\n" + ) diff --git a/tests/integration/test_executable_dictionary/user_scripts/input.py b/tests/integration/test_executable_dictionary/user_scripts/input.py index e711dd8e306..75a3ccac52c 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input.py @@ -4,8 +4,8 @@ import sys import os import signal -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - updated_line = line.replace('\n', '') - print(updated_line + '\t' + "Key " + updated_line, end='\n') + updated_line = line.replace("\n", "") + print(updated_line + "\t" + "Key " + updated_line, end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_argument.py b/tests/integration/test_executable_dictionary/user_scripts/input_argument.py index 163f9c4183f..349650fad6e 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_argument.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_argument.py @@ -2,10 +2,10 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": arg = int(sys.argv[1]) for line in sys.stdin: - updated_line = line.replace('\n', '') - print(updated_line + '\t' + "Key " + str(arg) + " " + updated_line, end='\n') + updated_line = line.replace("\n", "") + print(updated_line + "\t" + "Key " + str(arg) + " " + updated_line, end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_chunk_header.py b/tests/integration/test_executable_dictionary/user_scripts/input_chunk_header.py index 4eb00f64eb3..f8a60a771ea 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_chunk_header.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_chunk_header.py @@ -2,14 +2,14 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) while chunk_length != 0: line = sys.stdin.readline() - updated_line = line.replace('\n', '') + updated_line = line.replace("\n", "") chunk_length -= 1 - print(updated_line + '\t' + "Key " + updated_line, end='\n') + print(updated_line + "\t" + "Key " + updated_line, end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit.py index 835ab1f441a..3ace4f73611 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit.py @@ -2,7 +2,7 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_argument.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_argument.py index c1b2e5966d7..b9b7f5065b2 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_argument.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_argument.py @@ -2,9 +2,9 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": arg = int(sys.argv[1]) for line in sys.stdin: - print("Key " + str(arg) + " " + line, end='') + print("Key " + str(arg) + " " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_chunk_header.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_chunk_header.py index 5dc03e1c507..90c8bfd9a2f 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_chunk_header.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_chunk_header.py @@ -2,13 +2,13 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py index 27c8bc4840e..11a86737966 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_signalled.py @@ -5,9 +5,9 @@ import os import signal import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: os.signal(os.getpid(), signal.SIGTERM) - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py index 648a9eac918..cbe47041712 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_slow.py @@ -5,8 +5,8 @@ import os import signal import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: time.sleep(5) - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py index 432d7a13a2f..b8297cc42bc 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_implicit_sum.py @@ -3,8 +3,8 @@ import sys import re -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - line_split = re.split(r'\t+', line) - print(int(line_split[0]) + int(line_split[1]), end='\n') + line_split = re.split(r"\t+", line) + print(int(line_split[0]) + int(line_split[1]), end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py b/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py index a3a99f1e71e..4c131ddffd0 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_signalled.py @@ -5,9 +5,9 @@ import os import signal import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: os.signal(os.getpid(), signal.SIGTERM) - updated_line = line.replace('\n', '') - print(updated_line + '\t' + "Key " + updated_line, end='\n') + updated_line = line.replace("\n", "") + print(updated_line + "\t" + "Key " + updated_line, end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_slow.py b/tests/integration/test_executable_dictionary/user_scripts/input_slow.py index a3b8c484b29..aa8ec0101e2 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_slow.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_slow.py @@ -5,9 +5,9 @@ import os import signal import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: time.sleep(5) - updated_line = line.replace('\n', '') - print(updated_line + '\t' + "Key " + updated_line, end='\n') + updated_line = line.replace("\n", "") + print(updated_line + "\t" + "Key " + updated_line, end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/input_sum.py b/tests/integration/test_executable_dictionary/user_scripts/input_sum.py index e9ec5028701..ffdf599c886 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/input_sum.py +++ b/tests/integration/test_executable_dictionary/user_scripts/input_sum.py @@ -3,10 +3,10 @@ import sys import re -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - updated_line = line.replace('\n', '') - line_split = re.split(r'\t+', line) + updated_line = line.replace("\n", "") + line_split = re.split(r"\t+", line) sum = int(line_split[0]) + int(line_split[1]) - print(updated_line + '\t' + str(sum), end='\n') + print(updated_line + "\t" + str(sum), end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/source.py b/tests/integration/test_executable_dictionary/user_scripts/source.py index e105773c467..7af4d950f44 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/source.py +++ b/tests/integration/test_executable_dictionary/user_scripts/source.py @@ -2,9 +2,9 @@ import sys -if __name__ == '__main__': - print('1' + '\t' + 'Value 1', end='\n') - print('2' + '\t' + 'Value 2', end='\n') - print('3' + '\t' + 'Value 3', end='\n') +if __name__ == "__main__": + print("1" + "\t" + "Value 1", end="\n") + print("2" + "\t" + "Value 2", end="\n") + print("3" + "\t" + "Value 3", end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/source_argument.py b/tests/integration/test_executable_dictionary/user_scripts/source_argument.py index 881e73adc97..decb0482fac 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/source_argument.py +++ b/tests/integration/test_executable_dictionary/user_scripts/source_argument.py @@ -2,11 +2,11 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": arg = int(sys.argv[1]) - print('1' + '\t' + 'Value ' + str(arg) + ' 1', end='\n') - print('2' + '\t' + 'Value ' + str(arg) + ' 2', end='\n') - print('3' + '\t' + 'Value ' + str(arg) + ' 3', end='\n') + print("1" + "\t" + "Value " + str(arg) + " 1", end="\n") + print("2" + "\t" + "Value " + str(arg) + " 2", end="\n") + print("3" + "\t" + "Value " + str(arg) + " 3", end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_dictionary/user_scripts/source_update.py b/tests/integration/test_executable_dictionary/user_scripts/source_update.py index 99388f9ada3..1090dac85b9 100755 --- a/tests/integration/test_executable_dictionary/user_scripts/source_update.py +++ b/tests/integration/test_executable_dictionary/user_scripts/source_update.py @@ -2,11 +2,11 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": update_field_value = 0 if len(sys.argv) >= 2: update_field_value = int(sys.argv[1]) - print('1' + '\t' + 'Value ' + str(update_field_value) + ' 1', end='\n') + print("1" + "\t" + "Value " + str(update_field_value) + " 1", end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/test.py b/tests/integration/test_executable_table_function/test.py index 7820396d20f..868e056993b 100644 --- a/tests/integration/test_executable_table_function/test.py +++ b/tests/integration/test_executable_table_function/test.py @@ -9,7 +9,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=[]) +node = cluster.add_instance("node", stay_alive=True, main_configs=[]) # Something like https://reviews.llvm.org/D33325 @@ -19,14 +19,23 @@ def skip_test_msan(instance): def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'user_scripts/.'), '/var/lib/clickhouse/user_scripts', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "user_scripts/."), + "/var/lib/clickhouse/user_scripts", + node.docker_id, + ) node.restart_clickhouse() node.query("CREATE TABLE test_data_table (id UInt64) ENGINE=TinyLog;") @@ -37,266 +46,331 @@ def started_cluster(): finally: cluster.shutdown() + def test_executable_function_no_input_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT * FROM executable('no_input.sh', 'TabSeparated', 'value String')") == 'Key 0\nKey 1\nKey 2\n' + assert ( + node.query( + "SELECT * FROM executable('no_input.sh', 'TabSeparated', 'value String')" + ) + == "Key 0\nKey 1\nKey 2\n" + ) + def test_executable_function_no_input_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT * FROM executable('no_input.py', 'TabSeparated', 'value String')") == 'Key 0\nKey 1\nKey 2\n' + assert ( + node.query( + "SELECT * FROM executable('no_input.py', 'TabSeparated', 'value String')" + ) + == "Key 0\nKey 1\nKey 2\n" + ) + def test_executable_function_input_bash(started_cluster): skip_test_msan(node) - query = "SELECT * FROM executable('input.sh', 'TabSeparated', 'value String', {source})" - assert node.query(query.format(source='(SELECT 1)')) == 'Key 1\n' - assert node.query(query.format(source='(SELECT id FROM test_data_table)')) == 'Key 0\nKey 1\nKey 2\n' + query = ( + "SELECT * FROM executable('input.sh', 'TabSeparated', 'value String', {source})" + ) + assert node.query(query.format(source="(SELECT 1)")) == "Key 1\n" + assert ( + node.query(query.format(source="(SELECT id FROM test_data_table)")) + == "Key 0\nKey 1\nKey 2\n" + ) + def test_executable_function_input_python(started_cluster): skip_test_msan(node) - query = "SELECT * FROM executable('input.py', 'TabSeparated', 'value String', {source})" - assert node.query(query.format(source='(SELECT 1)')) == 'Key 1\n' - assert node.query(query.format(source='(SELECT id FROM test_data_table)')) == 'Key 0\nKey 1\nKey 2\n' + query = ( + "SELECT * FROM executable('input.py', 'TabSeparated', 'value String', {source})" + ) + assert node.query(query.format(source="(SELECT 1)")) == "Key 1\n" + assert ( + node.query(query.format(source="(SELECT id FROM test_data_table)")) + == "Key 0\nKey 1\nKey 2\n" + ) + def test_executable_function_input_sum_python(started_cluster): skip_test_msan(node) query = "SELECT * FROM executable('input_sum.py', 'TabSeparated', 'value UInt64', {source})" - assert node.query(query.format(source='(SELECT 1, 1)')) == '2\n' - assert node.query(query.format(source='(SELECT id, id FROM test_data_table)')) == '0\n2\n4\n' + assert node.query(query.format(source="(SELECT 1, 1)")) == "2\n" + assert ( + node.query(query.format(source="(SELECT id, id FROM test_data_table)")) + == "0\n2\n4\n" + ) + def test_executable_function_input_argument_python(started_cluster): skip_test_msan(node) query = "SELECT * FROM executable('input_argument.py 1', 'TabSeparated', 'value String', {source})" - assert node.query(query.format(source='(SELECT 1)')) == 'Key 1 1\n' - assert node.query(query.format(source='(SELECT id FROM test_data_table)')) == 'Key 1 0\nKey 1 1\nKey 1 2\n' + assert node.query(query.format(source="(SELECT 1)")) == "Key 1 1\n" + assert ( + node.query(query.format(source="(SELECT id FROM test_data_table)")) + == "Key 1 0\nKey 1 1\nKey 1 2\n" + ) + def test_executable_function_input_signalled_python(started_cluster): skip_test_msan(node) query = "SELECT * FROM executable('input_signalled.py', 'TabSeparated', 'value String', {source})" - assert node.query(query.format(source='(SELECT 1)')) == '' - assert node.query(query.format(source='(SELECT id FROM test_data_table)')) == '' + assert node.query(query.format(source="(SELECT 1)")) == "" + assert node.query(query.format(source="(SELECT id FROM test_data_table)")) == "" + def test_executable_function_input_slow_python(started_cluster): skip_test_msan(node) query = "SELECT * FROM executable('input_slow.py', 'TabSeparated', 'value String', {source})" - assert node.query_and_get_error(query.format(source='(SELECT 1)')) - assert node.query_and_get_error(query.format(source='(SELECT id FROM test_data_table)')) + assert node.query_and_get_error(query.format(source="(SELECT 1)")) + assert node.query_and_get_error( + query.format(source="(SELECT id FROM test_data_table)") + ) + def test_executable_function_input_multiple_pipes_python(started_cluster): skip_test_msan(node) query = "SELECT * FROM executable('input_multiple_pipes.py', 'TabSeparated', 'value String', {source})" - actual = node.query(query.format(source='(SELECT 1), (SELECT 2), (SELECT 3)')) - expected = 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n' + actual = node.query(query.format(source="(SELECT 1), (SELECT 2), (SELECT 3)")) + expected = "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n" assert actual == expected - actual = node.query(query.format(source='(SELECT id FROM test_data_table), (SELECT 2), (SELECT 3)')) - expected = 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n' + actual = node.query( + query.format(source="(SELECT id FROM test_data_table), (SELECT 2), (SELECT 3)") + ) + expected = "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n" assert actual == expected + def test_executable_storage_no_input_bash(started_cluster): skip_test_msan(node) node.query("DROP TABLE IF EXISTS test_table") - node.query("CREATE TABLE test_table (value String) ENGINE=Executable('no_input.sh', 'TabSeparated')") - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' + node.query( + "CREATE TABLE test_table (value String) ENGINE=Executable('no_input.sh', 'TabSeparated')" + ) + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" node.query("DROP TABLE test_table") + def test_executable_storage_no_input_python(started_cluster): skip_test_msan(node) node.query("DROP TABLE IF EXISTS test_table") - node.query("CREATE TABLE test_table (value String) ENGINE=Executable('no_input.py', 'TabSeparated')") - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' + node.query( + "CREATE TABLE test_table (value String) ENGINE=Executable('no_input.py', 'TabSeparated')" + ) + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" node.query("DROP TABLE test_table") + def test_executable_storage_input_bash(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input.sh', 'TabSeparated', {source})" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) - assert node.query("SELECT * FROM test_table") == 'Key 1\n' + node.query(query.format(source="(SELECT 1)")) + assert node.query("SELECT * FROM test_table") == "Key 1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' + node.query(query.format(source="(SELECT id FROM test_data_table)")) + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" node.query("DROP TABLE test_table") + def test_executable_storage_input_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input.py', 'TabSeparated', {source})" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) - assert node.query("SELECT * FROM test_table") == 'Key 1\n' + node.query(query.format(source="(SELECT 1)")) + assert node.query("SELECT * FROM test_table") == "Key 1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' + node.query(query.format(source="(SELECT id FROM test_data_table)")) + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" node.query("DROP TABLE test_table") + def test_executable_storage_input_send_chunk_header_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input_chunk_header.py', 'TabSeparated', {source}) SETTINGS send_chunk_header=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) - assert node.query("SELECT * FROM test_table") == 'Key 1\n' + node.query(query.format(source="(SELECT 1)")) + assert node.query("SELECT * FROM test_table") == "Key 1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' + node.query(query.format(source="(SELECT id FROM test_data_table)")) + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" node.query("DROP TABLE test_table") + def test_executable_storage_input_sum_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value UInt64) ENGINE=Executable('input_sum.py', 'TabSeparated', {source})" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1, 1)')) - assert node.query("SELECT * FROM test_table") == '2\n' + node.query(query.format(source="(SELECT 1, 1)")) + assert node.query("SELECT * FROM test_table") == "2\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id, id FROM test_data_table)')) - assert node.query("SELECT * FROM test_table") == '0\n2\n4\n' + node.query(query.format(source="(SELECT id, id FROM test_data_table)")) + assert node.query("SELECT * FROM test_table") == "0\n2\n4\n" node.query("DROP TABLE test_table") + def test_executable_storage_input_argument_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input_argument.py 1', 'TabSeparated', {source})" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) - assert node.query("SELECT * FROM test_table") == 'Key 1 1\n' + node.query(query.format(source="(SELECT 1)")) + assert node.query("SELECT * FROM test_table") == "Key 1 1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) - assert node.query("SELECT * FROM test_table") == 'Key 1 0\nKey 1 1\nKey 1 2\n' + node.query(query.format(source="(SELECT id FROM test_data_table)")) + assert node.query("SELECT * FROM test_table") == "Key 1 0\nKey 1 1\nKey 1 2\n" node.query("DROP TABLE test_table") + def test_executable_storage_input_signalled_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input_signalled.py', 'TabSeparated', {source})" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) - assert node.query("SELECT * FROM test_table") == '' + node.query(query.format(source="(SELECT 1)")) + assert node.query("SELECT * FROM test_table") == "" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) - assert node.query("SELECT * FROM test_table") == '' + node.query(query.format(source="(SELECT id FROM test_data_table)")) + assert node.query("SELECT * FROM test_table") == "" node.query("DROP TABLE test_table") + def test_executable_storage_input_slow_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input_slow.py', 'TabSeparated', {source}) SETTINGS command_read_timeout=2500" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) + node.query(query.format(source="(SELECT 1)")) assert node.query_and_get_error("SELECT * FROM test_table") node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) + node.query(query.format(source="(SELECT id FROM test_data_table)")) assert node.query_and_get_error("SELECT * FROM test_table") node.query("DROP TABLE test_table") + def test_executable_function_input_multiple_pipes_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=Executable('input_multiple_pipes.py', 'TabSeparated', {source})" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1), (SELECT 2), (SELECT 3)')) - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n' + node.query(query.format(source="(SELECT 1), (SELECT 2), (SELECT 3)")) + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n" + ) node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table), (SELECT 2), (SELECT 3)')) - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n' + node.query( + query.format(source="(SELECT id FROM test_data_table), (SELECT 2), (SELECT 3)") + ) + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n" + ) node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=ExecutablePool('input_pool.py', 'TabSeparated', {source}) SETTINGS send_chunk_header=1, pool_size=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) + node.query(query.format(source="(SELECT 1)")) - assert node.query("SELECT * FROM test_table") == 'Key 1\n' - assert node.query("SELECT * FROM test_table") == 'Key 1\n' - assert node.query("SELECT * FROM test_table") == 'Key 1\n' + assert node.query("SELECT * FROM test_table") == "Key 1\n" + assert node.query("SELECT * FROM test_table") == "Key 1\n" + assert node.query("SELECT * FROM test_table") == "Key 1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) + node.query(query.format(source="(SELECT id FROM test_data_table)")) - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' - assert node.query("SELECT * FROM test_table") == 'Key 0\nKey 1\nKey 2\n' + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" + assert node.query("SELECT * FROM test_table") == "Key 0\nKey 1\nKey 2\n" node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_sum_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value UInt64) ENGINE=ExecutablePool('input_sum_pool.py', 'TabSeparated', {source}) SETTINGS send_chunk_header=1, pool_size=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1, 1)')) + node.query(query.format(source="(SELECT 1, 1)")) - assert node.query("SELECT * FROM test_table") == '2\n' - assert node.query("SELECT * FROM test_table") == '2\n' - assert node.query("SELECT * FROM test_table") == '2\n' + assert node.query("SELECT * FROM test_table") == "2\n" + assert node.query("SELECT * FROM test_table") == "2\n" + assert node.query("SELECT * FROM test_table") == "2\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id, id FROM test_data_table)')) + node.query(query.format(source="(SELECT id, id FROM test_data_table)")) - assert node.query("SELECT * FROM test_table") == '0\n2\n4\n' - assert node.query("SELECT * FROM test_table") == '0\n2\n4\n' - assert node.query("SELECT * FROM test_table") == '0\n2\n4\n' + assert node.query("SELECT * FROM test_table") == "0\n2\n4\n" + assert node.query("SELECT * FROM test_table") == "0\n2\n4\n" + assert node.query("SELECT * FROM test_table") == "0\n2\n4\n" node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_argument_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=ExecutablePool('input_argument_pool.py 1', 'TabSeparated', {source}) SETTINGS send_chunk_header=1, pool_size=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) + node.query(query.format(source="(SELECT 1)")) - assert node.query("SELECT * FROM test_table") == 'Key 1 1\n' - assert node.query("SELECT * FROM test_table") == 'Key 1 1\n' - assert node.query("SELECT * FROM test_table") == 'Key 1 1\n' + assert node.query("SELECT * FROM test_table") == "Key 1 1\n" + assert node.query("SELECT * FROM test_table") == "Key 1 1\n" + assert node.query("SELECT * FROM test_table") == "Key 1 1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) + node.query(query.format(source="(SELECT id FROM test_data_table)")) - assert node.query("SELECT * FROM test_table") == 'Key 1 0\nKey 1 1\nKey 1 2\n' - assert node.query("SELECT * FROM test_table") == 'Key 1 0\nKey 1 1\nKey 1 2\n' - assert node.query("SELECT * FROM test_table") == 'Key 1 0\nKey 1 1\nKey 1 2\n' + assert node.query("SELECT * FROM test_table") == "Key 1 0\nKey 1 1\nKey 1 2\n" + assert node.query("SELECT * FROM test_table") == "Key 1 0\nKey 1 1\nKey 1 2\n" + assert node.query("SELECT * FROM test_table") == "Key 1 0\nKey 1 1\nKey 1 2\n" node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_signalled_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=ExecutablePool('input_signalled_pool.py', 'TabSeparated', {source}) SETTINGS send_chunk_header=1, pool_size=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) + node.query(query.format(source="(SELECT 1)")) assert node.query_and_get_error("SELECT * FROM test_table") assert node.query_and_get_error("SELECT * FROM test_table") @@ -304,7 +378,7 @@ def test_executable_pool_storage_input_signalled_python(started_cluster): node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) + node.query(query.format(source="(SELECT id FROM test_data_table)")) assert node.query_and_get_error("SELECT * FROM test_table") assert node.query_and_get_error("SELECT * FROM test_table") @@ -312,6 +386,7 @@ def test_executable_pool_storage_input_signalled_python(started_cluster): node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_slow_python(started_cluster): skip_test_msan(node) @@ -320,7 +395,7 @@ def test_executable_pool_storage_input_slow_python(started_cluster): SETTINGS send_chunk_header=1, pool_size=1, command_read_timeout=2500""" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) + node.query(query.format(source="(SELECT 1)")) assert node.query_and_get_error("SELECT * FROM test_table") assert node.query_and_get_error("SELECT * FROM test_table") @@ -328,7 +403,7 @@ def test_executable_pool_storage_input_slow_python(started_cluster): node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table)')) + node.query(query.format(source="(SELECT id FROM test_data_table)")) assert node.query_and_get_error("SELECT * FROM test_table") assert node.query_and_get_error("SELECT * FROM test_table") @@ -336,46 +411,68 @@ def test_executable_pool_storage_input_slow_python(started_cluster): node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_multiple_pipes_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=ExecutablePool('input_multiple_pipes_pool.py', 'TabSeparated', {source}) SETTINGS send_chunk_header=1, pool_size=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1), (SELECT 2), (SELECT 3)')) + node.query(query.format(source="(SELECT 1), (SELECT 2), (SELECT 3)")) - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n' - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n' - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n' + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n" + ) + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n" + ) + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 1\n" + ) node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT id FROM test_data_table), (SELECT 2), (SELECT 3)')) + node.query( + query.format(source="(SELECT id FROM test_data_table), (SELECT 2), (SELECT 3)") + ) - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n' - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n' - assert node.query("SELECT * FROM test_table") == 'Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n' + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n" + ) + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n" + ) + assert ( + node.query("SELECT * FROM test_table") + == "Key from 4 fd 3\nKey from 3 fd 2\nKey from 0 fd 0\nKey from 0 fd 1\nKey from 0 fd 2\n" + ) node.query("DROP TABLE test_table") + def test_executable_pool_storage_input_count_python(started_cluster): skip_test_msan(node) query = "CREATE TABLE test_table (value String) ENGINE=ExecutablePool('input_count_pool.py', 'TabSeparated', {source}) SETTINGS send_chunk_header=1, pool_size=1" node.query("DROP TABLE IF EXISTS test_table") - node.query(query.format(source='(SELECT 1)')) + node.query(query.format(source="(SELECT 1)")) - assert node.query("SELECT * FROM test_table") == '1\n' - assert node.query("SELECT * FROM test_table") == '1\n' - assert node.query("SELECT * FROM test_table") == '1\n' + assert node.query("SELECT * FROM test_table") == "1\n" + assert node.query("SELECT * FROM test_table") == "1\n" + assert node.query("SELECT * FROM test_table") == "1\n" node.query("DROP TABLE test_table") - node.query(query.format(source='(SELECT number FROM system.numbers LIMIT 250000)')) + node.query(query.format(source="(SELECT number FROM system.numbers LIMIT 250000)")) - assert node.query("SELECT * FROM test_table") == '250000\n' - assert node.query("SELECT * FROM test_table") == '250000\n' - assert node.query("SELECT * FROM test_table") == '250000\n' + assert node.query("SELECT * FROM test_table") == "250000\n" + assert node.query("SELECT * FROM test_table") == "250000\n" + assert node.query("SELECT * FROM test_table") == "250000\n" node.query("DROP TABLE test_table") diff --git a/tests/integration/test_executable_table_function/user_scripts/input.py b/tests/integration/test_executable_table_function/user_scripts/input.py index 835ab1f441a..3ace4f73611 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input.py +++ b/tests/integration/test_executable_table_function/user_scripts/input.py @@ -2,7 +2,7 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_argument.py b/tests/integration/test_executable_table_function/user_scripts/input_argument.py index c1b2e5966d7..b9b7f5065b2 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_argument.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_argument.py @@ -2,9 +2,9 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": arg = int(sys.argv[1]) for line in sys.stdin: - print("Key " + str(arg) + " " + line, end='') + print("Key " + str(arg) + " " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_argument_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_argument_pool.py index 378a6ef4391..13cad8e01d4 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_argument_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_argument_pool.py @@ -2,16 +2,16 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": arg = int(sys.argv[1]) for chunk_header in sys.stdin: chunk_length = int(chunk_header) - print(str(chunk_length), end='\n') + print(str(chunk_length), end="\n") while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + str(arg) + " " + line, end='') + print("Key " + str(arg) + " " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_chunk_header.py b/tests/integration/test_executable_table_function/user_scripts/input_chunk_header.py index 5dc03e1c507..90c8bfd9a2f 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_chunk_header.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_chunk_header.py @@ -2,13 +2,13 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_count_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_count_pool.py index 8b744168a82..b80c4832ab1 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_count_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_count_pool.py @@ -2,11 +2,11 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) - print(1, end='\n') - print(str(chunk_length), end='\n') + print(1, end="\n") + print(str(chunk_length), end="\n") while chunk_length != 0: line = sys.stdin.readline() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py index 64590cbc16a..4c7a03eee80 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes.py @@ -3,17 +3,17 @@ import sys import os -if __name__ == '__main__': +if __name__ == "__main__": fd3 = os.fdopen(3) fd4 = os.fdopen(4) for line in fd4: - print("Key from 4 fd " + line, end='') + print("Key from 4 fd " + line, end="") for line in fd3: - print("Key from 3 fd " + line, end='') + print("Key from 3 fd " + line, end="") for line in sys.stdin: - print("Key from 0 fd " + line, end='') + print("Key from 0 fd " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py index a3a515899f9..412e7d95299 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_multiple_pipes_pool.py @@ -3,7 +3,7 @@ import sys import os -if __name__ == '__main__': +if __name__ == "__main__": fd3 = os.fdopen(3) fd4 = os.fdopen(4) @@ -36,10 +36,10 @@ if __name__ == '__main__': break break - print(str(len(lines)), end='\n') + print(str(len(lines)), end="\n") for line in lines: - print(line, end='') + print(line, end="") lines.clear() - sys.stdout.flush() \ No newline at end of file + sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_pool.py index ec4e9af23cd..fe991be1417 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_pool.py @@ -2,14 +2,14 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) - print(str(chunk_length), end='\n') + print(str(chunk_length), end="\n") while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_signalled.py b/tests/integration/test_executable_table_function/user_scripts/input_signalled.py index 93ce20fa8e7..fd3ad19039d 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_signalled.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_signalled.py @@ -4,9 +4,9 @@ import sys import os import signal -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: os.signal(os.getpid(), signal.SIGTERM) - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py index 1ea0eddbd8d..79813c2e9c7 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_signalled_pool.py @@ -4,16 +4,16 @@ import sys import os import signal -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: os.signal(os.getpid(), signal.SIGTERM) chunk_length = int(chunk_header) - print(str(chunk_length), end='\n') + print(str(chunk_length), end="\n") while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_slow.py b/tests/integration/test_executable_table_function/user_scripts/input_slow.py index 4c2abe89e33..e007a58dfb4 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_slow.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_slow.py @@ -3,8 +3,8 @@ import sys import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: time.sleep(25) - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_slow_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_slow_pool.py index c8df7e18c4c..7cbf8950826 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_slow_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_slow_pool.py @@ -3,16 +3,16 @@ import sys import time -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: time.sleep(25) chunk_length = int(chunk_header) - print(str(chunk_length), end='\n') + print(str(chunk_length), end="\n") while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_sum.py b/tests/integration/test_executable_table_function/user_scripts/input_sum.py index 432d7a13a2f..b8297cc42bc 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_sum.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_sum.py @@ -3,8 +3,8 @@ import sys import re -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - line_split = re.split(r'\t+', line) - print(int(line_split[0]) + int(line_split[1]), end='\n') + line_split = re.split(r"\t+", line) + print(int(line_split[0]) + int(line_split[1]), end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py b/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py index cd0de25fe87..a04dc9a1b26 100755 --- a/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py +++ b/tests/integration/test_executable_table_function/user_scripts/input_sum_pool.py @@ -3,15 +3,15 @@ import sys import re -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) - print(str(chunk_length), end='\n') + print(str(chunk_length), end="\n") while chunk_length != 0: line = sys.stdin.readline() - line_split = re.split(r'\t+', line) - print(int(line_split[0]) + int(line_split[1]), end='\n') + line_split = re.split(r"\t+", line) + print(int(line_split[0]) + int(line_split[1]), end="\n") chunk_length -= 1 sys.stdout.flush() diff --git a/tests/integration/test_executable_table_function/user_scripts/no_input.py b/tests/integration/test_executable_table_function/user_scripts/no_input.py index 65b78f3d755..062032924ac 100755 --- a/tests/integration/test_executable_table_function/user_scripts/no_input.py +++ b/tests/integration/test_executable_table_function/user_scripts/no_input.py @@ -2,7 +2,7 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": print("Key 0") print("Key 1") print("Key 2") diff --git a/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml b/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml index d8f81a588a2..b2b7db83fbc 100644 --- a/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml +++ b/tests/integration/test_executable_user_defined_function/functions/test_function_config.xml @@ -193,4 +193,100 @@ 0 + + executable + test_function_sum_json_unnamed_args_python + UInt64 + result_name + + UInt64 + + + UInt64 + + JSONEachRow + input_sum_json_unnamed_args.py + + + + executable_pool + test_function_sum_json_unnamed_args_pool_python + UInt64 + result_name + + UInt64 + + + UInt64 + + JSONEachRow + input_sum_json_unnamed_args.py + + + + executable + test_function_sum_json_partially_named_args_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + + JSONEachRow + input_sum_json_partially_named_args.py + + + + executable_pool + test_function_sum_json_partially_named_args_pool_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + + JSONEachRow + input_sum_json_partially_named_args.py + + + + executable + test_function_sum_json_named_args_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + input_sum_json_named_args.py + + + + executable_pool + test_function_sum_json_named_args_pool_python + UInt64 + result_name + + UInt64 + argument_1 + + + UInt64 + argument_2 + + JSONEachRow + input_sum_json_named_args.py + + diff --git a/tests/integration/test_executable_user_defined_function/test.py b/tests/integration/test_executable_user_defined_function/test.py index 94afdf8d8a9..10993e9c5dd 100644 --- a/tests/integration/test_executable_user_defined_function/test.py +++ b/tests/integration/test_executable_user_defined_function/test.py @@ -10,29 +10,47 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=[]) +node = cluster.add_instance("node", stay_alive=True, main_configs=[]) def skip_test_msan(instance): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with vfork") -def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) -config = ''' +def copy_file_to_container(local_path, dist_path, container_id): + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) + + +config = """ /etc/clickhouse-server/functions/test_function_config.xml -''' +""" + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node.replace_config("/etc/clickhouse-server/config.d/executable_user_defined_functions_config.xml", config) + node.replace_config( + "/etc/clickhouse-server/config.d/executable_user_defined_functions_config.xml", + config, + ) - copy_file_to_container(os.path.join(SCRIPT_DIR, 'functions/.'), '/etc/clickhouse-server/functions', node.docker_id) - copy_file_to_container(os.path.join(SCRIPT_DIR, 'user_scripts/.'), '/var/lib/clickhouse/user_scripts', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "functions/."), + "/etc/clickhouse-server/functions", + node.docker_id, + ) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "user_scripts/."), + "/var/lib/clickhouse/user_scripts", + node.docker_id, + ) node.restart_clickhouse() @@ -41,66 +59,172 @@ def started_cluster(): finally: cluster.shutdown() + def test_executable_function_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT test_function_bash(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_bash(1)") == 'Key 1\n' + assert node.query("SELECT test_function_bash(toUInt64(1))") == "Key 1\n" + assert node.query("SELECT test_function_bash(1)") == "Key 1\n" + + assert node.query("SELECT test_function_pool_bash(toUInt64(1))") == "Key 1\n" + assert node.query("SELECT test_function_pool_bash(1)") == "Key 1\n" - assert node.query("SELECT test_function_pool_bash(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_pool_bash(1)") == 'Key 1\n' def test_executable_function_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT test_function_python(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_python(1)") == 'Key 1\n' + assert node.query("SELECT test_function_python(toUInt64(1))") == "Key 1\n" + assert node.query("SELECT test_function_python(1)") == "Key 1\n" + + assert node.query("SELECT test_function_pool_python(toUInt64(1))") == "Key 1\n" + assert node.query("SELECT test_function_pool_python(1)") == "Key 1\n" - assert node.query("SELECT test_function_pool_python(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_pool_python(1)") == 'Key 1\n' def test_executable_function_send_chunk_header_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT test_function_send_chunk_header_python(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_send_chunk_header_python(1)") == 'Key 1\n' + assert ( + node.query("SELECT test_function_send_chunk_header_python(toUInt64(1))") + == "Key 1\n" + ) + assert node.query("SELECT test_function_send_chunk_header_python(1)") == "Key 1\n" + + assert ( + node.query("SELECT test_function_send_chunk_header_pool_python(toUInt64(1))") + == "Key 1\n" + ) + assert ( + node.query("SELECT test_function_send_chunk_header_pool_python(1)") == "Key 1\n" + ) - assert node.query("SELECT test_function_send_chunk_header_pool_python(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_send_chunk_header_pool_python(1)") == 'Key 1\n' def test_executable_function_sum_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT test_function_sum_python(toUInt64(1), toUInt64(1))") == '2\n' - assert node.query("SELECT test_function_sum_python(1, 1)") == '2\n' + assert ( + node.query("SELECT test_function_sum_python(toUInt64(1), toUInt64(1))") == "2\n" + ) + assert node.query("SELECT test_function_sum_python(1, 1)") == "2\n" + + assert ( + node.query("SELECT test_function_sum_pool_python(toUInt64(1), toUInt64(1))") + == "2\n" + ) + assert node.query("SELECT test_function_sum_pool_python(1, 1)") == "2\n" - assert node.query("SELECT test_function_sum_pool_python(toUInt64(1), toUInt64(1))") == '2\n' - assert node.query("SELECT test_function_sum_pool_python(1, 1)") == '2\n' def test_executable_function_argument_python(started_cluster): skip_test_msan(node) - assert node.query("SELECT test_function_argument_python(toUInt64(1))") == 'Key 1 1\n' - assert node.query("SELECT test_function_argument_python(1)") == 'Key 1 1\n' + assert ( + node.query("SELECT test_function_argument_python(toUInt64(1))") == "Key 1 1\n" + ) + assert node.query("SELECT test_function_argument_python(1)") == "Key 1 1\n" + + assert ( + node.query("SELECT test_function_argument_pool_python(toUInt64(1))") + == "Key 1 1\n" + ) + assert node.query("SELECT test_function_argument_pool_python(1)") == "Key 1 1\n" - assert node.query("SELECT test_function_argument_pool_python(toUInt64(1))") == 'Key 1 1\n' - assert node.query("SELECT test_function_argument_pool_python(1)") == 'Key 1 1\n' def test_executable_function_signalled_python(started_cluster): skip_test_msan(node) - assert node.query_and_get_error("SELECT test_function_signalled_python(toUInt64(1))") + assert node.query_and_get_error( + "SELECT test_function_signalled_python(toUInt64(1))" + ) assert node.query_and_get_error("SELECT test_function_signalled_python(1)") - assert node.query_and_get_error("SELECT test_function_signalled_pool_python(toUInt64(1))") + assert node.query_and_get_error( + "SELECT test_function_signalled_pool_python(toUInt64(1))" + ) assert node.query_and_get_error("SELECT test_function_signalled_pool_python(1)") + def test_executable_function_slow_python(started_cluster): skip_test_msan(node) assert node.query_and_get_error("SELECT test_function_slow_python(toUInt64(1))") assert node.query_and_get_error("SELECT test_function_slow_python(1)") - assert node.query_and_get_error("SELECT test_function_slow_pool_python(toUInt64(1))") + assert node.query_and_get_error( + "SELECT test_function_slow_pool_python(toUInt64(1))" + ) assert node.query_and_get_error("SELECT test_function_slow_pool_python(1)") + def test_executable_function_non_direct_bash(started_cluster): skip_test_msan(node) - assert node.query("SELECT test_function_non_direct_bash(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_non_direct_bash(1)") == 'Key 1\n' + assert node.query("SELECT test_function_non_direct_bash(toUInt64(1))") == "Key 1\n" + assert node.query("SELECT test_function_non_direct_bash(1)") == "Key 1\n" - assert node.query("SELECT test_function_non_direct_pool_bash(toUInt64(1))") == 'Key 1\n' - assert node.query("SELECT test_function_non_direct_pool_bash(1)") == 'Key 1\n' + assert ( + node.query("SELECT test_function_non_direct_pool_bash(toUInt64(1))") + == "Key 1\n" + ) + assert node.query("SELECT test_function_non_direct_pool_bash(1)") == "Key 1\n" + + +def test_executable_function_sum_json_python(started_cluster): + skip_test_msan(node) + + node.query("CREATE TABLE test_table (lhs UInt64, rhs UInt64) ENGINE=TinyLog;") + node.query("INSERT INTO test_table VALUES (0, 0), (1, 1), (2, 2);") + + assert ( + node.query("SELECT test_function_sum_json_unnamed_args_python(1, 2);") == "3\n" + ) + assert ( + node.query( + "SELECT test_function_sum_json_unnamed_args_python(lhs, rhs) FROM test_table;" + ) + == "0\n2\n4\n" + ) + + assert ( + node.query("SELECT test_function_sum_json_partially_named_args_python(1, 2);") + == "3\n" + ) + assert ( + node.query( + "SELECT test_function_sum_json_partially_named_args_python(lhs, rhs) FROM test_table;" + ) + == "0\n2\n4\n" + ) + + assert node.query("SELECT test_function_sum_json_named_args_python(1, 2);") == "3\n" + assert ( + node.query( + "SELECT test_function_sum_json_named_args_python(lhs, rhs) FROM test_table;" + ) + == "0\n2\n4\n" + ) + + assert ( + node.query("SELECT test_function_sum_json_unnamed_args_pool_python(1, 2);") + == "3\n" + ) + assert ( + node.query( + "SELECT test_function_sum_json_unnamed_args_pool_python(lhs, rhs) FROM test_table;" + ) + == "0\n2\n4\n" + ) + + assert ( + node.query("SELECT test_function_sum_json_partially_named_args_python(1, 2);") + == "3\n" + ) + assert ( + node.query( + "SELECT test_function_sum_json_partially_named_args_python(lhs, rhs) FROM test_table;" + ) + == "0\n2\n4\n" + ) + + assert ( + node.query("SELECT test_function_sum_json_named_args_pool_python(1, 2);") + == "3\n" + ) + assert ( + node.query( + "SELECT test_function_sum_json_named_args_pool_python(lhs, rhs) FROM test_table;" + ) + == "0\n2\n4\n" + ) + + node.query("DROP TABLE test_table;") diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input.py b/tests/integration/test_executable_user_defined_function/user_scripts/input.py index 835ab1f441a..3ace4f73611 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input.py @@ -2,7 +2,7 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_argument.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_argument.py index c1b2e5966d7..b9b7f5065b2 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_argument.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_argument.py @@ -2,9 +2,9 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": arg = int(sys.argv[1]) for line in sys.stdin: - print("Key " + str(arg) + " " + line, end='') + print("Key " + str(arg) + " " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_chunk_header.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_chunk_header.py index 5dc03e1c507..90c8bfd9a2f 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_chunk_header.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_chunk_header.py @@ -2,13 +2,13 @@ import sys -if __name__ == '__main__': +if __name__ == "__main__": for chunk_header in sys.stdin: chunk_length = int(chunk_header) while chunk_length != 0: line = sys.stdin.readline() chunk_length -= 1 - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py index 27c8bc4840e..11a86737966 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_signalled.py @@ -5,9 +5,9 @@ import os import signal import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: os.signal(os.getpid(), signal.SIGTERM) - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py index 648a9eac918..cbe47041712 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_slow.py @@ -5,8 +5,8 @@ import os import signal import time -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: time.sleep(5) - print("Key " + line, end='') + print("Key " + line, end="") sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py index 432d7a13a2f..b8297cc42bc 100755 --- a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum.py @@ -3,8 +3,8 @@ import sys import re -if __name__ == '__main__': +if __name__ == "__main__": for line in sys.stdin: - line_split = re.split(r'\t+', line) - print(int(line_split[0]) + int(line_split[1]), end='\n') + line_split = re.split(r"\t+", line) + print(int(line_split[0]) + int(line_split[1]), end="\n") sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py new file mode 100755 index 00000000000..955196397d3 --- /dev/null +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_named_args.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import sys +import json + +if __name__ == "__main__": + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value["argument_1"]) + second_arg = int(value["argument_2"]) + result = {"result_name": first_arg + second_arg} + print(json.dumps(result), end="\n") + sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py new file mode 100755 index 00000000000..9f3e3c091c2 --- /dev/null +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_partially_named_args.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import sys +import json + +if __name__ == "__main__": + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value["argument_1"]) + second_arg = int(value["c2"]) + result = {"result_name": first_arg + second_arg} + print(json.dumps(result), end="\n") + sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py new file mode 100755 index 00000000000..0aad7b1b435 --- /dev/null +++ b/tests/integration/test_executable_user_defined_function/user_scripts/input_sum_json_unnamed_args.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import sys +import json + +if __name__ == "__main__": + for line in sys.stdin: + value = json.loads(line) + first_arg = int(value["c1"]) + second_arg = int(value["c2"]) + result = {"result_name": first_arg + second_arg} + print(json.dumps(result), end="\n") + sys.stdout.flush() diff --git a/tests/integration/test_executable_user_defined_functions_config_reload/test.py b/tests/integration/test_executable_user_defined_functions_config_reload/test.py index 629c426a28c..91c93c4593b 100644 --- a/tests/integration/test_executable_user_defined_functions_config_reload/test.py +++ b/tests/integration/test_executable_user_defined_functions_config_reload/test.py @@ -10,16 +10,24 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True, main_configs=['config/executable_user_defined_functions_config.xml']) +node = cluster.add_instance( + "node", + stay_alive=True, + main_configs=["config/executable_user_defined_functions_config.xml"], +) def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) -config = ''' +config = """ /etc/clickhouse-server/functions/{user_defined_executable_functions_config} -''' +""" @pytest.fixture(scope="module") @@ -27,8 +35,16 @@ def started_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'functions/.'), '/etc/clickhouse-server/functions', node.docker_id) - copy_file_to_container(os.path.join(SCRIPT_DIR, 'user_scripts/.'), '/var/lib/clickhouse/user_scripts', node.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "functions/."), + "/etc/clickhouse-server/functions", + node.docker_id, + ) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "user_scripts/."), + "/var/lib/clickhouse/user_scripts", + node.docker_id, + ) node.restart_clickhouse() @@ -39,7 +55,12 @@ def started_cluster(): def change_config(user_defined_executable_functions_config): - node.replace_config("/etc/clickhouse-server/config.d/executable_user_defined_functions_config.xml", config.format(user_defined_executable_functions_config=user_defined_executable_functions_config)) + node.replace_config( + "/etc/clickhouse-server/config.d/executable_user_defined_functions_config.xml", + config.format( + user_defined_executable_functions_config=user_defined_executable_functions_config + ), + ) node.query("SYSTEM RELOAD CONFIG;") @@ -49,7 +70,7 @@ def test(started_cluster): time.sleep(10) - assert node.query("SELECT test_function_1(toUInt64(1));") == 'Key_1 1\n' + assert node.query("SELECT test_function_1(toUInt64(1));") == "Key_1 1\n" # Change path to the second executable user defined function in config. change_config("test_function_config2.xml") @@ -57,7 +78,7 @@ def test(started_cluster): time.sleep(10) # Check that the new executable user defined function is loaded. - assert node.query("SELECT test_function_2(toUInt64(1))") == 'Key_2 1\n' + assert node.query("SELECT test_function_2(toUInt64(1))") == "Key_2 1\n" # Check that the previous executable user defined function was unloaded. node.query_and_get_error("SELECT test_function_1(toUInt64(1));") diff --git a/tests/integration/test_explain_estimates/test.py b/tests/integration/test_explain_estimates/test.py index 7bccfb11a37..9ccce61cf68 100644 --- a/tests/integration/test_explain_estimates/test.py +++ b/tests/integration/test_explain_estimates/test.py @@ -3,7 +3,8 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('instance') +node1 = cluster.add_instance("instance") + @pytest.fixture(scope="module") def start_cluster(): @@ -14,12 +15,16 @@ def start_cluster(): finally: cluster.shutdown() + def test_explain_estimates(start_cluster): - node1.query("CREATE TABLE test (i Int64) ENGINE = MergeTree() ORDER BY i SETTINGS index_granularity = 16") + node1.query( + "CREATE TABLE test (i Int64) ENGINE = MergeTree() ORDER BY i SETTINGS index_granularity = 16" + ) node1.query("INSERT INTO test SELECT number FROM numbers(128)") node1.query("OPTIMIZE TABLE test") # sum(marks) - 1 because EXPLAIN ESIMATES does not include final mark. - system_parts_result = node1.query("SELECT any(database), any(table), count() as parts, sum(rows) as rows, sum(marks)-1 as marks FROM system.parts WHERE database = 'default' AND table = 'test' and active = 1 GROUP BY (database, table)") + system_parts_result = node1.query( + "SELECT any(database), any(table), count() as parts, sum(rows) as rows, sum(marks)-1 as marks FROM system.parts WHERE database = 'default' AND table = 'test' and active = 1 GROUP BY (database, table)" + ) explain_estimates_result = node1.query("EXPLAIN ESTIMATE SELECT * FROM test") - assert(system_parts_result == explain_estimates_result) - + assert system_parts_result == explain_estimates_result diff --git a/tests/integration/test_extreme_deduplication/test.py b/tests/integration/test_extreme_deduplication/test.py index d0d4b83d10f..2c8772aad4e 100644 --- a/tests/integration/test_extreme_deduplication/test.py +++ b/tests/integration/test_extreme_deduplication/test.py @@ -8,12 +8,18 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], - with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) -node2 = cluster.add_instance('node2', - main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], - with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, + macros={"layer": 0, "shard": 0, "replica": 1}, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, + macros={"layer": 0, "shard": 0, "replica": 2}, +) nodes = [node1, node2] @@ -31,9 +37,11 @@ def started_cluster(): def test_deduplication_window_in_seconds(started_cluster): node = node1 - node1.query(""" + node1.query( + """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""") + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""" + ) node.query("INSERT INTO simple VALUES (0, 0)") time.sleep(1) @@ -44,11 +52,17 @@ def test_deduplication_window_in_seconds(started_cluster): # wait clean thread time.sleep(2) - assert \ - TSV.toMat(node.query("SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'"))[ - 0][ - 0] == "1" - node.query("INSERT INTO simple VALUES (0, 0)") # deduplication doesn't works here, the first hash node was deleted + assert ( + TSV.toMat( + node.query( + "SELECT count() FROM system.zookeeper WHERE path='/clickhouse/tables/0/simple/blocks'" + ) + )[0][0] + == "1" + ) + node.query( + "INSERT INTO simple VALUES (0, 0)" + ) # deduplication doesn't works here, the first hash node was deleted assert TSV.toMat(node.query("SELECT count() FROM simple"))[0][0] == "3" node1.query("""DROP TABLE simple ON CLUSTER test_cluster""") @@ -60,23 +74,37 @@ def test_deduplication_works_in_case_of_intensive_inserts(started_cluster): inserters = [] fetchers = [] - node1.query(""" + node1.query( + """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""") + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, id, 8192)""" + ) node1.query("INSERT INTO simple VALUES (0, 0)") for node in nodes: host = node.ip_address - inserters.append(CommandRequest(['/bin/bash'], timeout=10, stdin=""" + inserters.append( + CommandRequest( + ["/bin/bash"], + timeout=10, + stdin=""" set -e for i in `seq 1000`; do {} --host {} -q "INSERT INTO simple VALUES (0, 0)" done -""".format(cluster.get_client_cmd(), host))) +""".format( + cluster.get_client_cmd(), host + ), + ) + ) - fetchers.append(CommandRequest(['/bin/bash'], timeout=10, stdin=""" + fetchers.append( + CommandRequest( + ["/bin/bash"], + timeout=10, + stdin=""" set -e for i in `seq 1000`; do res=`{} --host {} -q "SELECT count() FROM simple"` @@ -85,7 +113,11 @@ for i in `seq 1000`; do exit -1 fi; done -""".format(cluster.get_client_cmd(), host, node.name))) +""".format( + cluster.get_client_cmd(), host, node.name + ), + ) + ) # There were not errors during INSERTs for inserter in inserters: diff --git a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py index 7bce2d50011..582748046f9 100644 --- a/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py +++ b/tests/integration/test_fetch_partition_from_auxiliary_zookeeper/test.py @@ -3,7 +3,9 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) +node = cluster.add_instance( + "node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -17,11 +19,11 @@ def start_cluster(): @pytest.mark.parametrize( - ('part', 'date', 'part_name'), + ("part", "date", "part_name"), [ - ('PARTITION', '2020-08-27', '2020-08-27'), - ('PART', '2020-08-28', '20200828_0_0_0'), - ] + ("PARTITION", "2020-08-27", "2020-08-27"), + ("PART", "2020-08-28", "20200828_0_0_0"), + ], ) def test_fetch_part_from_allowed_zookeeper(start_cluster, part, date, part_name): node.query( @@ -36,13 +38,26 @@ def test_fetch_part_from_allowed_zookeeper(start_cluster, part, date, part_name) node.query( """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper2:/clickhouse/tables/0/simple';""".format( - part=part, part_name=part_name)) + part=part, part_name=part_name + ) + ) - node.query("""ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format(part=part, part_name=part_name)) + node.query( + """ALTER TABLE simple2 ATTACH {part} '{part_name}';""".format( + part=part, part_name=part_name + ) + ) with pytest.raises(QueryRuntimeException): node.query( """ALTER TABLE simple2 FETCH {part} '{part_name}' FROM 'zookeeper:/clickhouse/tables/0/simple';""".format( - part=part, part_name=part_name)) + part=part, part_name=part_name + ) + ) - assert node.query("""SELECT id FROM simple2 where date = '{date}'""".format(date=date)).strip() == "1" + assert ( + node.query( + """SELECT id FROM simple2 where date = '{date}'""".format(date=date) + ).strip() + == "1" + ) diff --git a/tests/integration/test_fetch_partition_should_reset_mutation/test.py b/tests/integration/test_fetch_partition_should_reset_mutation/test.py index 14a91a42031..7037393a3d2 100644 --- a/tests/integration/test_fetch_partition_should_reset_mutation/test.py +++ b/tests/integration/test_fetch_partition_should_reset_mutation/test.py @@ -4,7 +4,9 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) +node = cluster.add_instance( + "node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -25,36 +27,45 @@ def test_part_should_reset_mutation(start_cluster): node.query("optimize table test final") node.query("optimize table test final") + expected = TSV("""all_0_0_2\t1\ta""") + assert TSV(node.query("SELECT _part, * FROM test")) == expected - expected = TSV('''all_0_0_2\t1\ta''') - assert TSV(node.query('SELECT _part, * FROM test')) == expected + node.query( + "ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"} + ) + node.query( + "ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"} + ) + node.query( + "ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"} + ) + node.query( + "ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"} + ) - node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) - node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) - node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) - node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) - - expected = TSV('''all_0_0_2_4\t1\txxx''') - assert TSV(node.query('SELECT _part, * FROM test')) == expected + expected = TSV("""all_0_0_2_4\t1\txxx""") + assert TSV(node.query("SELECT _part, * FROM test")) == expected node.query( "CREATE TABLE restore (i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/restore', 'node') ORDER BY i;" ) - node.query("ALTER TABLE restore FETCH PARTITION tuple() FROM '/clickhouse/tables/test/'") + node.query( + "ALTER TABLE restore FETCH PARTITION tuple() FROM '/clickhouse/tables/test/'" + ) node.query("ALTER TABLE restore ATTACH PART 'all_0_0_2_4'") node.query("INSERT INTO restore select 2, 'a'") - print(TSV(node.query('SELECT _part, * FROM restore'))) - expected = TSV('''all_0_0_0\t1\txxx\nall_1_1_0\t2\ta''') - assert TSV(node.query('SELECT _part, * FROM restore ORDER BY i')) == expected + print(TSV(node.query("SELECT _part, * FROM restore"))) + expected = TSV("""all_0_0_0\t1\txxx\nall_1_1_0\t2\ta""") + assert TSV(node.query("SELECT _part, * FROM restore ORDER BY i")) == expected - node.query("ALTER TABLE restore UPDATE s='yyy' WHERE 1", settings={"mutations_sync": "2"}) + node.query( + "ALTER TABLE restore UPDATE s='yyy' WHERE 1", settings={"mutations_sync": "2"} + ) - - expected = TSV('''all_0_0_0_2\t1\tyyy\nall_1_1_0_2\t2\tyyy''') - assert TSV(node.query('SELECT _part, * FROM restore ORDER BY i')) == expected + expected = TSV("""all_0_0_0_2\t1\tyyy\nall_1_1_0_2\t2\tyyy""") + assert TSV(node.query("SELECT _part, * FROM restore ORDER BY i")) == expected node.query("ALTER TABLE restore DELETE WHERE 1", settings={"mutations_sync": "2"}) - assert node.query("SELECT count() FROM restore").strip() == "0" diff --git a/tests/integration/test_fetch_partition_with_outdated_parts/test.py b/tests/integration/test_fetch_partition_with_outdated_parts/test.py index 08d5e53e41e..b78d09b0316 100644 --- a/tests/integration/test_fetch_partition_with_outdated_parts/test.py +++ b/tests/integration/test_fetch_partition_with_outdated_parts/test.py @@ -6,8 +6,9 @@ import pytest cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) - +node = cluster.add_instance( + "node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 2519d0e5ac3..34e377e0ae4 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -16,30 +16,65 @@ def started_cluster(): def test_file_path_escaping(started_cluster): - node.query('CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary') - node.query(''' + node.query("CREATE DATABASE IF NOT EXISTS test ENGINE = Ordinary") + node.query( + """ CREATE TABLE test.`T.a_b,l-e!` (`~Id` UInt32) ENGINE = MergeTree() PARTITION BY `~Id` ORDER BY `~Id` SETTINGS min_bytes_for_wide_part = 0; - ''') - node.query('''INSERT INTO test.`T.a_b,l-e!` VALUES (1);''') - node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''') + """ + ) + node.query("""INSERT INTO test.`T.a_b,l-e!` VALUES (1);""") + node.query("""ALTER TABLE test.`T.a_b,l-e!` FREEZE;""") - node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) node.exec_in_container( - ["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) + [ + "bash", + "-c", + "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin", + ] + ) + node.exec_in_container( + [ + "bash", + "-c", + "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin", + ] + ) + def test_file_path_escaping_atomic_db(started_cluster): - node.query('CREATE DATABASE IF NOT EXISTS `test 2` ENGINE = Atomic') - node.query(''' + node.query("CREATE DATABASE IF NOT EXISTS `test 2` ENGINE = Atomic") + node.query( + """ CREATE TABLE `test 2`.`T.a_b,l-e!` UUID '12345678-1000-4000-8000-000000000001' (`~Id` UInt32) ENGINE = MergeTree() PARTITION BY `~Id` ORDER BY `~Id` SETTINGS min_bytes_for_wide_part = 0; - ''') - node.query('''INSERT INTO `test 2`.`T.a_b,l-e!` VALUES (1);''') - node.query('''ALTER TABLE `test 2`.`T.a_b,l-e!` FREEZE;''') + """ + ) + node.query("""INSERT INTO `test 2`.`T.a_b,l-e!` VALUES (1);""") + node.query("""ALTER TABLE `test 2`.`T.a_b,l-e!` FREEZE;""") - node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/store/123/12345678-1000-4000-8000-000000000001/1_1_1_0/%7EId.bin"]) - # Check symlink - node.exec_in_container(["bash", "-c", "test -L /var/lib/clickhouse/data/test%202/T%2Ea_b%2Cl%2De%21"]) - node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test%202/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"]) node.exec_in_container( - ["bash", "-c", "test -f /var/lib/clickhouse/shadow/2/store/123/12345678-1000-4000-8000-000000000001/1_1_1_0/%7EId.bin"]) + [ + "bash", + "-c", + "test -f /var/lib/clickhouse/store/123/12345678-1000-4000-8000-000000000001/1_1_1_0/%7EId.bin", + ] + ) + # Check symlink + node.exec_in_container( + ["bash", "-c", "test -L /var/lib/clickhouse/data/test%202/T%2Ea_b%2Cl%2De%21"] + ) + node.exec_in_container( + [ + "bash", + "-c", + "test -f /var/lib/clickhouse/data/test%202/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin", + ] + ) + node.exec_in_container( + [ + "bash", + "-c", + "test -f /var/lib/clickhouse/shadow/2/store/123/12345678-1000-4000-8000-000000000001/1_1_1_0/%7EId.bin", + ] + ) diff --git a/tests/integration/test_force_deduplication/test.py b/tests/integration/test_force_deduplication/test.py index 991e289f912..87b2c45bbc5 100644 --- a/tests/integration/test_force_deduplication/test.py +++ b/tests/integration/test_force_deduplication/test.py @@ -7,10 +7,10 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', with_zookeeper=True) +node = cluster.add_instance("node", with_zookeeper=True) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def start_cluster(): try: cluster.start() @@ -30,21 +30,21 @@ def get_counts(): def test_basic(start_cluster): node.query( - ''' + """ CREATE TABLE test (A Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple(); CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') order by tuple() AS SELECT A FROM test; CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') partition by A order by tuple() AS SELECT A FROM test; CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') order by tuple() AS SELECT A FROM test; INSERT INTO test values(999); INSERT INTO test values(999); - ''' + """ ) with pytest.raises(QueryRuntimeException): node.query( - ''' + """ SET max_partitions_per_insert_block = 3; INSERT INTO test SELECT number FROM numbers(10); - ''' + """ ) old_src, old_a, old_b, old_c = get_counts() @@ -63,10 +63,10 @@ def test_basic(start_cluster): assert c == old_c node.query( - ''' + """ SET deduplicate_blocks_in_dependent_materialized_views = 1; INSERT INTO test SELECT number FROM numbers(10); - ''' + """ ) src, a, b, c = get_counts() assert src == 11 @@ -76,18 +76,18 @@ def test_basic(start_cluster): with pytest.raises(QueryRuntimeException): node.query( - ''' + """ SET max_partitions_per_insert_block = 3; SET deduplicate_blocks_in_dependent_materialized_views = 1; INSERT INTO test SELECT number FROM numbers(100,10); - ''' + """ ) node.query( - ''' + """ SET deduplicate_blocks_in_dependent_materialized_views = 1; INSERT INTO test SELECT number FROM numbers(100,10); - ''' + """ ) src, a, b, c = get_counts() diff --git a/tests/integration/test_force_drop_table/test.py b/tests/integration/test_force_drop_table/test.py index ad8316493e4..c1eec1cd277 100644 --- a/tests/integration/test_force_drop_table/test.py +++ b/tests/integration/test_force_drop_table/test.py @@ -3,7 +3,9 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.xml"], with_zookeeper=True) +node = cluster.add_instance( + "node", main_configs=["configs/config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -14,28 +16,47 @@ def started_cluster(): finally: cluster.shutdown() + def create_force_drop_flag(node): force_drop_flag_path = "/var/lib/clickhouse/flags/force_drop_table" - node.exec_in_container(["bash", "-c", "touch {} && chmod a=rw {}".format(force_drop_flag_path, force_drop_flag_path)], user="root") + node.exec_in_container( + [ + "bash", + "-c", + "touch {} && chmod a=rw {}".format( + force_drop_flag_path, force_drop_flag_path + ), + ], + user="root", + ) -@pytest.mark.parametrize("engine", ['Ordinary', 'Atomic']) + +@pytest.mark.parametrize("engine", ["Ordinary", "Atomic"]) def test_drop_materialized_view(started_cluster, engine): node.query("CREATE DATABASE d ENGINE={}".format(engine)) - node.query("CREATE TABLE d.rmt (n UInt64) ENGINE=ReplicatedMergeTree('/test/rmt', 'r1') ORDER BY n PARTITION BY n % 2") - node.query("CREATE MATERIALIZED VIEW d.mv (n UInt64, s String) ENGINE=MergeTree ORDER BY n PARTITION BY n % 2 AS SELECT n, toString(n) AS s FROM d.rmt") + node.query( + "CREATE TABLE d.rmt (n UInt64) ENGINE=ReplicatedMergeTree('/test/rmt', 'r1') ORDER BY n PARTITION BY n % 2" + ) + node.query( + "CREATE MATERIALIZED VIEW d.mv (n UInt64, s String) ENGINE=MergeTree ORDER BY n PARTITION BY n % 2 AS SELECT n, toString(n) AS s FROM d.rmt" + ) node.query("INSERT INTO d.rmt VALUES (1), (2)") assert "is greater than max" in node.query_and_get_error("DROP TABLE d.rmt") assert "is greater than max" in node.query_and_get_error("DROP TABLE d.mv") assert "is greater than max" in node.query_and_get_error("TRUNCATE TABLE d.rmt") assert "is greater than max" in node.query_and_get_error("TRUNCATE TABLE d.mv") - assert "is greater than max" in node.query_and_get_error("ALTER TABLE d.rmt DROP PARTITION '0'") + assert "is greater than max" in node.query_and_get_error( + "ALTER TABLE d.rmt DROP PARTITION '0'" + ) assert node.query("SELECT * FROM d.rmt ORDER BY n") == "1\n2\n" assert node.query("SELECT * FROM d.mv ORDER BY n") == "1\t1\n2\t2\n" create_force_drop_flag(node) node.query("ALTER TABLE d.rmt DROP PARTITION '0'") assert node.query("SELECT * FROM d.rmt ORDER BY n") == "1\n" - assert "is greater than max" in node.query_and_get_error("ALTER TABLE d.mv DROP PARTITION '0'") + assert "is greater than max" in node.query_and_get_error( + "ALTER TABLE d.mv DROP PARTITION '0'" + ) create_force_drop_flag(node) node.query("ALTER TABLE d.mv DROP PARTITION '0'") assert node.query("SELECT * FROM d.mv ORDER BY n") == "1\t1\n" @@ -46,4 +67,3 @@ def test_drop_materialized_view(started_cluster, engine): create_force_drop_flag(node) node.query("DROP TABLE d.mv SYNC") node.query("DROP DATABASE d") - diff --git a/tests/integration/test_format_avro_confluent/test.py b/tests/integration/test_format_avro_confluent/test.py index 23e2d8d8c47..42b7ddce193 100644 --- a/tests/integration/test_format_avro_confluent/test.py +++ b/tests/integration/test_format_avro_confluent/test.py @@ -3,10 +3,13 @@ import logging import avro.schema import pytest -from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient +from confluent_kafka.avro.cached_schema_registry_client import ( + CachedSchemaRegistryClient, +) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer from helpers.cluster import ClickHouseCluster, ClickHouseInstance + @pytest.fixture(scope="module") def started_cluster(): try: @@ -37,36 +40,34 @@ def run_query(instance, query, data=None, settings=None): def test_select(started_cluster): # type: (ClickHouseCluster) -> None - schema_registry_client = CachedSchemaRegistryClient('http://localhost:{}'.format(started_cluster.schema_registry_port)) + schema_registry_client = CachedSchemaRegistryClient( + "http://localhost:{}".format(started_cluster.schema_registry_port) + ) serializer = MessageSerializer(schema_registry_client) - schema = avro.schema.make_avsc_object({ - 'name': 'test_record', - 'type': 'record', - 'fields': [ - { - 'name': 'value', - 'type': 'long' - } - ] - }) + schema = avro.schema.make_avsc_object( + { + "name": "test_record", + "type": "record", + "fields": [{"name": "value", "type": "long"}], + } + ) buf = io.BytesIO() for x in range(0, 3): message = serializer.encode_record_with_schema( - 'test_subject', schema, {'value': x} + "test_subject", schema, {"value": x} ) buf.write(message) data = buf.getvalue() instance = started_cluster.instances["dummy"] # type: ClickHouseInstance schema_registry_url = "http://{}:{}".format( - started_cluster.schema_registry_host, - 8081 + started_cluster.schema_registry_host, 8081 ) run_query(instance, "create table avro_data(value Int64) engine = Memory()") - settings = {'format_avro_schema_registry_url': schema_registry_url} + settings = {"format_avro_schema_registry_url": schema_registry_url} run_query(instance, "insert into avro_data format AvroConfluent", data, settings) stdout = run_query(instance, "select * from avro_data") assert list(map(str.split, stdout.splitlines())) == [ diff --git a/tests/integration/test_format_schema_on_server/test.py b/tests/integration/test_format_schema_on_server/test.py index 3b53a897dc0..7001d53ccf2 100644 --- a/tests/integration/test_format_schema_on_server/test.py +++ b/tests/integration/test_format_schema_on_server/test.py @@ -2,15 +2,14 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - clickhouse_path_dir='clickhouse_path') +instance = cluster.add_instance("instance", clickhouse_path_dir="clickhouse_path") @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - instance.query('CREATE DATABASE test') + instance.query("CREATE DATABASE test") yield cluster finally: @@ -19,23 +18,29 @@ def started_cluster(): def create_simple_table(): instance.query("DROP TABLE IF EXISTS test.simple") - instance.query(''' + instance.query( + """ CREATE TABLE test.simple (key UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); - ''') + """ + ) def test_protobuf_format_input(started_cluster): create_simple_table() instance.http_query( "INSERT INTO test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'", - "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def") + "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def", + ) assert instance.query("SELECT * from test.simple") == "1\tabc\n2\tdef\n" def test_protobuf_format_output(started_cluster): create_simple_table() - instance.query("INSERT INTO test.simple VALUES (1, 'abc'), (2, 'def')"); - assert instance.http_query( - "SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'") == \ - "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def" + instance.query("INSERT INTO test.simple VALUES (1, 'abc'), (2, 'def')") + assert ( + instance.http_query( + "SELECT * FROM test.simple FORMAT Protobuf SETTINGS format_schema='simple:KeyValuePair'" + ) + == "\x07\x08\x01\x12\x03abc\x07\x08\x02\x12\x03def" + ) diff --git a/tests/integration/test_freeze_table/test.py b/tests/integration/test_freeze_table/test.py index 4d4aa22d4e2..8b9d1b58360 100644 --- a/tests/integration/test_freeze_table/test.py +++ b/tests/integration/test_freeze_table/test.py @@ -17,7 +17,8 @@ def started_cluster(): def test_freeze_table(started_cluster): - node.query(''' + node.query( + """ CREATE TABLE table_for_freeze ( key UInt64, @@ -26,38 +27,45 @@ def test_freeze_table(started_cluster): ENGINE = MergeTree() ORDER BY key PARTITION BY key % 10; - ''') - node.query(''' + """ + ) + node.query( + """ INSERT INTO table_for_freeze SELECT number, toString(number) from numbers(10); - ''') + """ + ) - freeze_result = TSV.toMat(node.query(''' + freeze_result = TSV.toMat( + node.query( + """ ALTER TABLE table_for_freeze FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - ''')) + """ + ) + ) assert 11 == len(freeze_result) - path_col_ix = freeze_result[0].index('part_backup_path') + path_col_ix = freeze_result[0].index("part_backup_path") for row in freeze_result[1:]: # skip header part_backup_path = row[path_col_ix] - node.exec_in_container( - ["bash", "-c", "test -d {}".format(part_backup_path)] - ) + node.exec_in_container(["bash", "-c", "test -d {}".format(part_backup_path)]) - freeze_result = TSV.toMat(node.query(''' + freeze_result = TSV.toMat( + node.query( + """ ALTER TABLE table_for_freeze FREEZE PARTITION '3' WITH NAME 'test_01417_single_part' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1; - ''')) + """ + ) + ) assert 2 == len(freeze_result) - path_col_ix = freeze_result[0].index('part_backup_path') + path_col_ix = freeze_result[0].index("part_backup_path") for row in freeze_result[1:]: # skip header part_backup_path = row[path_col_ix] - assert 'test_01417_single_part' in part_backup_path - node.exec_in_container( - ["bash", "-c", "test -d {}".format(part_backup_path)] - ) + assert "test_01417_single_part" in part_backup_path + node.exec_in_container(["bash", "-c", "test -d {}".format(part_backup_path)]) diff --git a/tests/integration/test_global_overcommit_tracker/__init__.py b/tests/integration/test_global_overcommit_tracker/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml new file mode 100644 index 00000000000..590759bd15d --- /dev/null +++ b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml @@ -0,0 +1,4 @@ + + 50000000 + 500 + \ No newline at end of file diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py new file mode 100644 index 00000000000..c2f3a22915f --- /dev/null +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -0,0 +1,53 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", main_configs=["configs/global_overcommit_tracker.xml"] +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +TEST_QUERY_A = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage_for_user=1" +TEST_QUERY_B = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS max_guaranteed_memory_usage_for_user=2" + + +def test_overcommited_is_killed(): + node.query("CREATE USER A") + node.query("GRANT ALL ON *.* TO A") + node.query("CREATE USER B") + node.query("GRANT ALL ON *.* TO B") + + responses_A = list() + responses_B = list() + for _ in range(100): + responses_A.append(node.get_query_request(TEST_QUERY_A, user="A")) + responses_B.append(node.get_query_request(TEST_QUERY_B, user="B")) + + overcommited_killed = False + for response in responses_A: + _, err = response.get_answer_and_error() + if "MEMORY_LIMIT_EXCEEDED" in err: + overcommited_killed = True + finished = False + for response in responses_B: + _, err = response.get_answer_and_error() + if err == "": + finished = True + + assert ( + overcommited_killed and finished + ), "no overcommited task was killed or all tasks are killed" + + node.query("DROP USER IF EXISTS A") + node.query("DROP USER IF EXISTS B") diff --git a/tests/integration/test_globs_in_filepath/test.py b/tests/integration/test_globs_in_filepath/test.py index 7e534dd69bc..6f5368e0243 100644 --- a/tests/integration/test_globs_in_filepath/test.py +++ b/tests/integration/test_globs_in_filepath/test.py @@ -3,8 +3,10 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') -path_to_userfiles_from_defaut_config = "/var/lib/clickhouse/user_files/" # should be the same as in config file +node = cluster.add_instance("node") +path_to_userfiles_from_defaut_config = ( + "/var/lib/clickhouse/user_files/" # should be the same as in config file +) @pytest.fixture(scope="module") @@ -25,85 +27,174 @@ def test_strange_filenames(start_cluster): # 2 rows data some_data = "\t111.222\nData\t333.444" - node.exec_in_container(['bash', '-c', 'mkdir {}strange_names/'.format(path_to_userfiles_from_defaut_config)], - privileged=True, user='root') + node.exec_in_container( + [ + "bash", + "-c", + "mkdir {}strange_names/".format(path_to_userfiles_from_defaut_config), + ], + privileged=True, + user="root", + ) - files = ["p.o.i.n.t.s", - "b}{ra{ces", - "b}.o{t.h"] + files = ["p.o.i.n.t.s", "b}{ra{ces", "b}.o{t.h"] # filename inside testing data for debug simplicity for filename in files: - node.exec_in_container(['bash', '-c', 'echo "{}{}" > {}strange_names/{}'.format(filename, some_data, - path_to_userfiles_from_defaut_config, - filename)], privileged=True, - user='root') + node.exec_in_container( + [ + "bash", + "-c", + 'echo "{}{}" > {}strange_names/{}'.format( + filename, some_data, path_to_userfiles_from_defaut_config, filename + ), + ], + privileged=True, + user="root", + ) - test_requests = [("p.o.??n.t.s", "2"), - ("p.o.*t.s", "2"), - ("b}{r?{ces", "2"), - ("b}*ces", "2"), - ("b}.?{t.h", "2")] + test_requests = [ + ("p.o.??n.t.s", "2"), + ("p.o.*t.s", "2"), + ("b}{r?{ces", "2"), + ("b}*ces", "2"), + ("b}.?{t.h", "2"), + ] for pattern, value in test_requests: - assert node.query(''' + assert ( + node.query( + """ select count(*) from file('strange_names/{}', 'TSV', 'text String, number Float64') - '''.format(pattern)) == '{}\n'.format(value) - assert node.query(''' + """.format( + pattern + ) + ) + == "{}\n".format(value) + ) + assert ( + node.query( + """ select count(*) from file('{}strange_names/{}', 'TSV', 'text String, number Float64') - '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + """.format( + path_to_userfiles_from_defaut_config, pattern + ) + ) + == "{}\n".format(value) + ) def test_linear_structure(start_cluster): # 2 rows data some_data = "\t123.456\nData\t789.012" - files = ["file1", "file2", "file3", "file4", "file5", - "file000", "file111", "file222", "file333", "file444", - "a_file", "b_file", "c_file", "d_file", "e_file", - "a_data", "b_data", "c_data", "d_data", "e_data"] + files = [ + "file1", + "file2", + "file3", + "file4", + "file5", + "file000", + "file111", + "file222", + "file333", + "file444", + "a_file", + "b_file", + "c_file", + "d_file", + "e_file", + "a_data", + "b_data", + "c_data", + "d_data", + "e_data", + ] # filename inside testing data for debug simplicity for filename in files: - node.exec_in_container(['bash', '-c', - 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, - filename)], privileged=True, user='root') + node.exec_in_container( + [ + "bash", + "-c", + 'echo "{}{}" > {}{}'.format( + filename, some_data, path_to_userfiles_from_defaut_config, filename + ), + ], + privileged=True, + user="root", + ) - test_requests = [("file{0..9}", "10"), - ("file?", "10"), - ("nothing*", "0"), - ("file{0..9}{0..9}{0..9}", "10"), - ("file{000..999}", "10"), - ("file???", "10"), - ("file*", "20"), - ("a_{file,data}", "4"), - ("?_{file,data}", "20"), - ("{a,b,c,d,e}_{file,data}", "20"), - ("{a,b,c,d,e}?{file,data}", "20"), - ("*", "40")] + test_requests = [ + ("file{0..9}", "10"), + ("file?", "10"), + ("nothing*", "0"), + ("file{0..9}{0..9}{0..9}", "10"), + ("file{000..999}", "10"), + ("file???", "10"), + ("file*", "20"), + ("a_{file,data}", "4"), + ("?_{file,data}", "20"), + ("{a,b,c,d,e}_{file,data}", "20"), + ("{a,b,c,d,e}?{file,data}", "20"), + ("*", "40"), + ] for pattern, value in test_requests: - assert node.query(''' + assert ( + node.query( + """ select count(*) from file('{}', 'TSV', 'text String, number Float64') - '''.format(pattern)) == '{}\n'.format(value) - assert node.query(''' + """.format( + pattern + ) + ) + == "{}\n".format(value) + ) + assert ( + node.query( + """ select count(*) from file('{}{}', 'TSV', 'text String, number Float64') - '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + """.format( + path_to_userfiles_from_defaut_config, pattern + ) + ) + == "{}\n".format(value) + ) def test_deep_structure(start_cluster): # 2 rows data some_data = "\t135.791\nData\t246.802" - dirs = ["directory1/", "directory2/", "some_more_dir/", "we/", - "directory1/big_dir/", - "directory1/dir1/", "directory1/dir2/", "directory1/dir3/", - "directory2/dir1/", "directory2/dir2/", "directory2/one_more_dir/", - "some_more_dir/yet_another_dir/", - "we/need/", "we/need/to/", "we/need/to/go/", "we/need/to/go/deeper/"] + dirs = [ + "directory1/", + "directory2/", + "some_more_dir/", + "we/", + "directory1/big_dir/", + "directory1/dir1/", + "directory1/dir2/", + "directory1/dir3/", + "directory2/dir1/", + "directory2/dir2/", + "directory2/one_more_dir/", + "some_more_dir/yet_another_dir/", + "we/need/", + "we/need/to/", + "we/need/to/go/", + "we/need/to/go/deeper/", + ] for dir in dirs: - node.exec_in_container(['bash', '-c', 'mkdir {}{}'.format(path_to_userfiles_from_defaut_config, dir)], - privileged=True, user='root') + node.exec_in_container( + [ + "bash", + "-c", + "mkdir {}{}".format(path_to_userfiles_from_defaut_config, dir), + ], + privileged=True, + user="root", + ) # all directories appeared in files must be listed in dirs files = [] @@ -117,34 +208,106 @@ def test_deep_structure(start_cluster): # filename inside testing data for debug simplicity for filename in files: - node.exec_in_container(['bash', '-c', - 'echo "{}{}" > {}{}'.format(filename, some_data, path_to_userfiles_from_defaut_config, - filename)], privileged=True, user='root') + node.exec_in_container( + [ + "bash", + "-c", + 'echo "{}{}" > {}{}'.format( + filename, some_data, path_to_userfiles_from_defaut_config, filename + ), + ], + privileged=True, + user="root", + ) - test_requests = [("directory{1..5}/big_dir/*", "2002"), ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"), - ("?", "0"), - ("directory{0..5}/dir{1..3}/file", "10"), ("directory{0..5}/dir?/file", "10"), - ("we/need/to/go/deeper/file", "2"), ("*/*/*/*/*/*", "2"), ("we/need/??/go/deeper/*?*?*?*?*", "2")] + test_requests = [ + ("directory{1..5}/big_dir/*", "2002"), + ("directory{0..6}/big_dir/*{0..9}{0..9}{0..9}", "2000"), + ("?", "0"), + ("directory{0..5}/dir{1..3}/file", "10"), + ("directory{0..5}/dir?/file", "10"), + ("we/need/to/go/deeper/file", "2"), + ("*/*/*/*/*/*", "2"), + ("we/need/??/go/deeper/*?*?*?*?*", "2"), + ] for pattern, value in test_requests: - assert node.query(''' + assert ( + node.query( + """ select count(*) from file('{}', 'TSV', 'text String, number Float64') - '''.format(pattern)) == '{}\n'.format(value) - assert node.query(''' + """.format( + pattern + ) + ) + == "{}\n".format(value) + ) + assert ( + node.query( + """ select count(*) from file('{}{}', 'TSV', 'text String, number Float64') - '''.format(path_to_userfiles_from_defaut_config, pattern)) == '{}\n'.format(value) + """.format( + path_to_userfiles_from_defaut_config, pattern + ) + ) + == "{}\n".format(value) + ) def test_table_function_and_virtual_columns(start_cluster): - node.exec_in_container(['bash', '-c', 'mkdir -p {}some/path/to/'.format(path_to_userfiles_from_defaut_config)]) - node.exec_in_container(['bash', '-c', 'touch {}some/path/to/data.CSV'.format(path_to_userfiles_from_defaut_config)]) + node.exec_in_container( + [ + "bash", + "-c", + "mkdir -p {}some/path/to/".format(path_to_userfiles_from_defaut_config), + ] + ) + node.exec_in_container( + [ + "bash", + "-c", + "touch {}some/path/to/data.CSV".format( + path_to_userfiles_from_defaut_config + ), + ] + ) node.query( - "insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)") - assert node.query( - "select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')").rstrip() == '100000' - node.query("insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)") - assert node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() == '1' - assert "nonexist.csv" in node.query("select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() - assert "nonexist.csv" in node.query("select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() - assert "nonexist.csv" == node.query("select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')").rstrip() - assert "nonexist.csv" == node.query("select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() + "insert into table function file('some/path/to/data.CSV', CSV, 'n UInt8, s String') select number, concat('str_', toString(number)) from numbers(100000)" + ) + assert ( + node.query( + "select count() from file('some/path/to/data.CSV', CSV, 'n UInt8, s String')" + ).rstrip() + == "100000" + ) + node.query( + "insert into table function file('nonexist.csv', 'CSV', 'val1 UInt32') values (1)" + ) + assert ( + node.query("select * from file('nonexist.csv', 'CSV', 'val1 UInt32')").rstrip() + == "1" + ) + assert ( + "nonexist.csv" + in node.query( + "select _path from file('nonexis?.csv', 'CSV', 'val1 UInt32')" + ).rstrip() + ) + assert ( + "nonexist.csv" + in node.query( + "select _path from file('nonexist.csv', 'CSV', 'val1 UInt32')" + ).rstrip() + ) + assert ( + "nonexist.csv" + == node.query( + "select _file from file('nonexis?.csv', 'CSV', 'val1 UInt32')" + ).rstrip() + ) + assert ( + "nonexist.csv" + == node.query( + "select _file from file('nonexist.csv', 'CSV', 'val1 UInt32')" + ).rstrip() + ) diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 89e07fecb0a..2988db24d74 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -3,7 +3,7 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -12,7 +12,9 @@ def start_cluster(): cluster.start() instance.query("CREATE DATABASE test") - instance.query("CREATE TABLE test.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE test.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) instance.query("INSERT INTO test.table VALUES (1,5), (2,10)") yield cluster @@ -32,28 +34,34 @@ def cleanup_after_test(): def test_smoke(): instance.query("CREATE USER A") - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test.table", user='A') + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test.table", user="A" + ) - instance.query('GRANT SELECT ON test.table TO A') - assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" + instance.query("GRANT SELECT ON test.table TO A") + assert instance.query("SELECT * FROM test.table", user="A") == "1\t5\n2\t10\n" - instance.query('REVOKE SELECT ON test.table FROM A') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test.table", user='A') + instance.query("REVOKE SELECT ON test.table FROM A") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test.table", user="A" + ) def test_grant_option(): instance.query("CREATE USER A") instance.query("CREATE USER B") - instance.query('GRANT SELECT ON test.table TO A') - assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" - assert "Not enough privileges" in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A') + instance.query("GRANT SELECT ON test.table TO A") + assert instance.query("SELECT * FROM test.table", user="A") == "1\t5\n2\t10\n" + assert "Not enough privileges" in instance.query_and_get_error( + "GRANT SELECT ON test.table TO B", user="A" + ) - instance.query('GRANT SELECT ON test.table TO A WITH GRANT OPTION') - instance.query("GRANT SELECT ON test.table TO B", user='A') - assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" + instance.query("GRANT SELECT ON test.table TO A WITH GRANT OPTION") + instance.query("GRANT SELECT ON test.table TO B", user="A") + assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" - instance.query('REVOKE SELECT ON test.table FROM A, B') + instance.query("REVOKE SELECT ON test.table FROM A, B") def test_revoke_requires_grant_option(): @@ -64,45 +72,51 @@ def test_revoke_requires_grant_option(): assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" expected_error = "Not enough privileges" - assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM B", user='A') + assert expected_error in instance.query_and_get_error( + "REVOKE SELECT ON test.table FROM B", user="A" + ) assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" instance.query("GRANT SELECT ON test.table TO A") expected_error = "privileges have been granted, but without grant option" - assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM B", user='A') + assert expected_error in instance.query_and_get_error( + "REVOKE SELECT ON test.table FROM B", user="A" + ) assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" instance.query("GRANT SELECT ON test.table TO A WITH GRANT OPTION") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" - instance.query("REVOKE SELECT ON test.table FROM B", user='A') + instance.query("REVOKE SELECT ON test.table FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" - instance.query("REVOKE SELECT ON test.* FROM B", user='A') + instance.query("REVOKE SELECT ON test.* FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" - instance.query("REVOKE ALL ON test.* FROM B", user='A') + instance.query("REVOKE ALL ON test.* FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" - instance.query("REVOKE ALL ON *.* FROM B", user='A') + instance.query("REVOKE ALL ON *.* FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("REVOKE GRANT OPTION FOR ALL ON *.* FROM A") instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" expected_error = "privileges have been granted, but without grant option" - assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM B", user='A') + assert expected_error in instance.query_and_get_error( + "REVOKE SELECT ON test.table FROM B", user="A" + ) assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" instance.query("GRANT SELECT ON test.* TO A WITH GRANT OPTION") instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT SELECT ON test.table TO B\n" - instance.query("REVOKE SELECT ON test.table FROM B", user='A') + instance.query("REVOKE SELECT ON test.table FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" @@ -110,98 +124,213 @@ def test_allowed_grantees(): instance.query("CREATE USER A") instance.query("CREATE USER B") - instance.query('GRANT SELECT ON test.table TO A WITH GRANT OPTION') - instance.query("GRANT SELECT ON test.table TO B", user='A') - assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" - instance.query("REVOKE SELECT ON test.table FROM B", user='A') + instance.query("GRANT SELECT ON test.table TO A WITH GRANT OPTION") + instance.query("GRANT SELECT ON test.table TO B", user="A") + assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" + instance.query("REVOKE SELECT ON test.table FROM B", user="A") - instance.query('ALTER USER A GRANTEES NONE') + instance.query("ALTER USER A GRANTEES NONE") expected_error = "user `B` is not allowed as grantee" - assert expected_error in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A') + assert expected_error in instance.query_and_get_error( + "GRANT SELECT ON test.table TO B", user="A" + ) - instance.query('ALTER USER A GRANTEES ANY EXCEPT B') - assert instance.query('SHOW CREATE USER A') == "CREATE USER A GRANTEES ANY EXCEPT B\n" + instance.query("ALTER USER A GRANTEES ANY EXCEPT B") + assert ( + instance.query("SHOW CREATE USER A") == "CREATE USER A GRANTEES ANY EXCEPT B\n" + ) expected_error = "user `B` is not allowed as grantee" - assert expected_error in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A') + assert expected_error in instance.query_and_get_error( + "GRANT SELECT ON test.table TO B", user="A" + ) - instance.query('ALTER USER A GRANTEES B') - instance.query("GRANT SELECT ON test.table TO B", user='A') - assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" - instance.query("REVOKE SELECT ON test.table FROM B", user='A') + instance.query("ALTER USER A GRANTEES B") + instance.query("GRANT SELECT ON test.table TO B", user="A") + assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" + instance.query("REVOKE SELECT ON test.table FROM B", user="A") - instance.query('ALTER USER A GRANTEES ANY') - assert instance.query('SHOW CREATE USER A') == "CREATE USER A\n" - instance.query("GRANT SELECT ON test.table TO B", user='A') - assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" + instance.query("ALTER USER A GRANTEES ANY") + assert instance.query("SHOW CREATE USER A") == "CREATE USER A\n" + instance.query("GRANT SELECT ON test.table TO B", user="A") + assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" - instance.query('ALTER USER A GRANTEES NONE') + instance.query("ALTER USER A GRANTEES NONE") expected_error = "user `B` is not allowed as grantee" - assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM B", user='A') + assert expected_error in instance.query_and_get_error( + "REVOKE SELECT ON test.table FROM B", user="A" + ) instance.query("CREATE USER C GRANTEES ANY EXCEPT C") - assert instance.query('SHOW CREATE USER C') == "CREATE USER C GRANTEES ANY EXCEPT C\n" - instance.query('GRANT SELECT ON test.table TO C WITH GRANT OPTION') - assert instance.query("SELECT * FROM test.table", user='C') == "1\t5\n2\t10\n" + assert ( + instance.query("SHOW CREATE USER C") == "CREATE USER C GRANTEES ANY EXCEPT C\n" + ) + instance.query("GRANT SELECT ON test.table TO C WITH GRANT OPTION") + assert instance.query("SELECT * FROM test.table", user="C") == "1\t5\n2\t10\n" expected_error = "user `C` is not allowed as grantee" - assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM C", user='C') + assert expected_error in instance.query_and_get_error( + "REVOKE SELECT ON test.table FROM C", user="C" + ) def test_grant_all_on_table(): instance.query("CREATE USER A, B") instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION") - instance.query("GRANT ALL ON test.table TO B", user='A') - assert instance.query( - "SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" - instance.query("REVOKE ALL ON test.table FROM B", user='A') + instance.query("GRANT ALL ON test.table TO B", user="A") + assert ( + instance.query("SHOW GRANTS FOR B") + == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, " + "DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, " + "SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, " + "SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" + ) + instance.query("REVOKE ALL ON test.table FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" def test_implicit_show_grants(): instance.query("CREATE USER A") - assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n" - assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", - user="A") == "0\n" + assert ( + instance.query( + "select count() FROM system.databases WHERE name='test'", user="A" + ) + == "0\n" + ) + assert ( + instance.query( + "select count() FROM system.tables WHERE database='test' AND name='table'", + user="A", + ) + == "0\n" + ) + assert ( + instance.query( + "select count() FROM system.columns WHERE database='test' AND table='table'", + user="A", + ) + == "0\n" + ) instance.query("GRANT SELECT(x) ON test.table TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT(x) ON test.table TO A\n" - assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" - assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", - user="A") == "1\n" + assert ( + instance.query( + "select count() FROM system.databases WHERE name='test'", user="A" + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.tables WHERE database='test' AND name='table'", + user="A", + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.columns WHERE database='test' AND table='table'", + user="A", + ) + == "1\n" + ) instance.query("GRANT SELECT ON test.table TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.table TO A\n" - assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" - assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", - user="A") == "2\n" + assert ( + instance.query( + "select count() FROM system.databases WHERE name='test'", user="A" + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.tables WHERE database='test' AND name='table'", + user="A", + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.columns WHERE database='test' AND table='table'", + user="A", + ) + == "2\n" + ) instance.query("GRANT SELECT ON test.* TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON test.* TO A\n" - assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" - assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", - user="A") == "2\n" + assert ( + instance.query( + "select count() FROM system.databases WHERE name='test'", user="A" + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.tables WHERE database='test' AND name='table'", + user="A", + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.columns WHERE database='test' AND table='table'", + user="A", + ) + == "2\n" + ) instance.query("GRANT SELECT ON *.* TO A") assert instance.query("SHOW GRANTS FOR A") == "GRANT SELECT ON *.* TO A\n" - assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "1\n" - assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "1\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", - user="A") == "2\n" + assert ( + instance.query( + "select count() FROM system.databases WHERE name='test'", user="A" + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.tables WHERE database='test' AND name='table'", + user="A", + ) + == "1\n" + ) + assert ( + instance.query( + "select count() FROM system.columns WHERE database='test' AND table='table'", + user="A", + ) + == "2\n" + ) instance.query("REVOKE ALL ON *.* FROM A") - assert instance.query("select count() FROM system.databases WHERE name='test'", user="A") == "0\n" - assert instance.query("select count() FROM system.tables WHERE database='test' AND name='table'", user="A") == "0\n" - assert instance.query("select count() FROM system.columns WHERE database='test' AND table='table'", - user="A") == "0\n" + assert ( + instance.query( + "select count() FROM system.databases WHERE name='test'", user="A" + ) + == "0\n" + ) + assert ( + instance.query( + "select count() FROM system.tables WHERE database='test' AND name='table'", + user="A", + ) + == "0\n" + ) + assert ( + instance.query( + "select count() FROM system.columns WHERE database='test' AND table='table'", + user="A", + ) + == "0\n" + ) def test_implicit_create_view_grant(): instance.query("CREATE USER A") expected_error = "Not enough privileges" - assert expected_error in instance.query_and_get_error("CREATE VIEW test.view_1 AS SELECT 1", user="A") + assert expected_error in instance.query_and_get_error( + "CREATE VIEW test.view_1 AS SELECT 1", user="A" + ) instance.query("GRANT CREATE TABLE ON test.* TO A") instance.query("CREATE VIEW test.view_1 AS SELECT 1", user="A") @@ -209,118 +338,230 @@ def test_implicit_create_view_grant(): instance.query("REVOKE CREATE TABLE ON test.* FROM A") instance.query("DROP TABLE test.view_1") - assert expected_error in instance.query_and_get_error("CREATE VIEW test.view_1 AS SELECT 1", user="A") + assert expected_error in instance.query_and_get_error( + "CREATE VIEW test.view_1 AS SELECT 1", user="A" + ) def test_implicit_create_temporary_table_grant(): instance.query("CREATE USER A") expected_error = "Not enough privileges" - assert expected_error in instance.query_and_get_error("CREATE TEMPORARY TABLE tmp(name String)", user="A") + assert expected_error in instance.query_and_get_error( + "CREATE TEMPORARY TABLE tmp(name String)", user="A" + ) instance.query("GRANT CREATE TABLE ON test.* TO A") instance.query("CREATE TEMPORARY TABLE tmp(name String)", user="A") instance.query("REVOKE CREATE TABLE ON *.* FROM A") - assert expected_error in instance.query_and_get_error("CREATE TEMPORARY TABLE tmp(name String)", user="A") + assert expected_error in instance.query_and_get_error( + "CREATE TEMPORARY TABLE tmp(name String)", user="A" + ) def test_introspection(): instance.query("CREATE USER A") instance.query("CREATE USER B") - instance.query('GRANT SELECT ON test.table TO A') - instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') + instance.query("GRANT SELECT ON test.table TO A") + instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"]) assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"]) - assert instance.query("SHOW CREATE USERS A,B") == TSV(["CREATE USER A", "CREATE USER B"]) - assert instance.query("SHOW CREATE USERS") == TSV(["CREATE USER A", "CREATE USER B", - "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default"]) + assert instance.query("SHOW CREATE USERS A,B") == TSV( + ["CREATE USER A", "CREATE USER B"] + ) + assert instance.query("SHOW CREATE USERS") == TSV( + [ + "CREATE USER A", + "CREATE USER B", + "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default", + ] + ) - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"]) - assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT SELECT ON test.table TO A"] + ) + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT CREATE ON *.* TO B WITH GRANT OPTION"] + ) assert instance.query("SHOW GRANTS FOR A,B") == TSV( - ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + [ + "GRANT SELECT ON test.table TO A", + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + ] + ) assert instance.query("SHOW GRANTS FOR B,A") == TSV( - ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + [ + "GRANT SELECT ON test.table TO A", + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + ] + ) assert instance.query("SHOW GRANTS FOR ALL") == TSV( - ["GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", - "GRANT ALL ON *.* TO default WITH GRANT OPTION"]) + [ + "GRANT SELECT ON test.table TO A", + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION", + ] + ) - assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) - assert instance.query("SHOW GRANTS", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS", user="A") == TSV( + ["GRANT SELECT ON test.table TO A"] + ) + assert instance.query("SHOW GRANTS", user="B") == TSV( + ["GRANT CREATE ON *.* TO B WITH GRANT OPTION"] + ) + + assert instance.query("SHOW GRANTS FOR ALL", user="A") == TSV( + ["GRANT SELECT ON test.table TO A"] + ) + assert instance.query("SHOW GRANTS FOR ALL", user="B") == TSV( + ["GRANT CREATE ON *.* TO B WITH GRANT OPTION"] + ) + assert instance.query("SHOW GRANTS FOR ALL") == TSV( + [ + "GRANT SELECT ON test.table TO A", + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION", + ] + ) - assert instance.query("SHOW GRANTS FOR ALL", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) - assert instance.query("SHOW GRANTS FOR ALL", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) - assert instance.query("SHOW GRANTS FOR ALL") == TSV(["GRANT SELECT ON test.table TO A", - "GRANT CREATE ON *.* TO B WITH GRANT OPTION", - "GRANT ALL ON *.* TO default WITH GRANT OPTION"]) - expected_error = "necessary to have grant SHOW USERS" - assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user='A') - - expected_access1 = "CREATE USER A\n" \ - "CREATE USER B\n" \ - "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" - expected_access2 = "GRANT SELECT ON test.table TO A\n" \ - "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" \ - "GRANT ALL ON *.* TO default WITH GRANT OPTION\n" + assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user="A") + + expected_access1 = ( + "CREATE USER A\n" + "CREATE USER B\n" + "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" + ) + expected_access2 = ( + "GRANT SELECT ON test.table TO A\n" + "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" + "GRANT ALL ON *.* TO default WITH GRANT OPTION\n" + ) assert expected_access1 in instance.query("SHOW ACCESS") assert expected_access2 in instance.query("SHOW ACCESS") assert instance.query( - "SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") == \ - TSV([["A", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"], - ["B", "local directory", "no_password", "{}", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]"]]) + "SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name" + ) == TSV( + [ + [ + "A", + "local directory", + "no_password", + "{}", + "['::/0']", + "[]", + "[]", + "[]", + 1, + "[]", + "[]", + ], + [ + "B", + "local directory", + "no_password", + "{}", + "['::/0']", + "[]", + "[]", + "[]", + 1, + "[]", + "[]", + ], + ] + ) assert instance.query( - "SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option") == \ - TSV([["A", "\\N", "SELECT", "test", "table", "\\N", 0, 0], - ["B", "\\N", "CREATE", "\\N", "\\N", "\\N", 0, 1]]) + "SELECT * from system.grants WHERE user_name IN ('A', 'B') ORDER BY user_name, access_type, grant_option" + ) == TSV( + [ + ["A", "\\N", "SELECT", "test", "table", "\\N", 0, 0], + ["B", "\\N", "CREATE", "\\N", "\\N", "\\N", 0, 1], + ] + ) def test_current_database(): instance.query("CREATE USER A") instance.query("GRANT SELECT ON table TO A", database="test") - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"]) - assert instance.query("SHOW GRANTS FOR A", database="test") == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT SELECT ON test.table TO A"] + ) + assert instance.query("SHOW GRANTS FOR A", database="test") == TSV( + ["GRANT SELECT ON test.table TO A"] + ) - assert instance.query("SELECT * FROM test.table", user='A') == "1\t5\n2\t10\n" - assert instance.query("SELECT * FROM table", user='A', database='test') == "1\t5\n2\t10\n" + assert instance.query("SELECT * FROM test.table", user="A") == "1\t5\n2\t10\n" + assert ( + instance.query("SELECT * FROM table", user="A", database="test") + == "1\t5\n2\t10\n" + ) - instance.query("CREATE TABLE default.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()") - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM table", user='A') + instance.query( + "CREATE TABLE default.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM table", user="A" + ) def test_grant_with_replace_option(): instance.query("CREATE USER A") - instance.query('GRANT SELECT ON test.table TO A') - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A"]) + instance.query("GRANT SELECT ON test.table TO A") + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT SELECT ON test.table TO A"] + ) - instance.query('GRANT INSERT ON test.table TO A WITH REPLACE OPTION') - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT INSERT ON test.table TO A"]) + instance.query("GRANT INSERT ON test.table TO A WITH REPLACE OPTION") + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT INSERT ON test.table TO A"] + ) - instance.query('GRANT NONE ON *.* TO A WITH REPLACE OPTION') + instance.query("GRANT NONE ON *.* TO A WITH REPLACE OPTION") assert instance.query("SHOW GRANTS FOR A") == TSV([]) - instance.query('CREATE USER B') - instance.query('GRANT SELECT ON test.table TO B') + instance.query("CREATE USER B") + instance.query("GRANT SELECT ON test.table TO B") assert instance.query("SHOW GRANTS FOR A") == TSV([]) - assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT SELECT ON test.table TO B"]) + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT SELECT ON test.table TO B"] + ) - expected_error = "it's necessary to have grant INSERT ON test.table WITH GRANT OPTION" - assert expected_error in instance.query_and_get_error("GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user='A') + expected_error = ( + "it's necessary to have grant INSERT ON test.table WITH GRANT OPTION" + ) + assert expected_error in instance.query_and_get_error( + "GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user="A" + ) assert instance.query("SHOW GRANTS FOR A") == TSV([]) - assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT SELECT ON test.table TO B"]) + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT SELECT ON test.table TO B"] + ) instance.query("GRANT INSERT ON test.table TO A WITH GRANT OPTION") - expected_error = "it's necessary to have grant SELECT ON test.table WITH GRANT OPTION" - assert expected_error in instance.query_and_get_error("GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user='A') - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT INSERT ON test.table TO A WITH GRANT OPTION"]) - assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT SELECT ON test.table TO B"]) + expected_error = ( + "it's necessary to have grant SELECT ON test.table WITH GRANT OPTION" + ) + assert expected_error in instance.query_and_get_error( + "GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user="A" + ) + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT INSERT ON test.table TO A WITH GRANT OPTION"] + ) + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT SELECT ON test.table TO B"] + ) instance.query("GRANT SELECT ON test.table TO A WITH GRANT OPTION") - instance.query("GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user='A') - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT, INSERT ON test.table TO A WITH GRANT OPTION"]) - assert instance.query("SHOW GRANTS FOR B") == TSV(["GRANT INSERT ON test.table TO B"]) + instance.query("GRANT INSERT ON test.table TO B WITH REPLACE OPTION", user="A") + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT SELECT, INSERT ON test.table TO A WITH GRANT OPTION"] + ) + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT INSERT ON test.table TO B"] + ) diff --git a/tests/integration/test_graphite_merge_tree/configs/graphite_rollup.xml b/tests/integration/test_graphite_merge_tree/configs/graphite_rollup.xml index 9373aed9b14..4ccdfa477c3 100644 --- a/tests/integration/test_graphite_merge_tree/configs/graphite_rollup.xml +++ b/tests/integration/test_graphite_merge_tree/configs/graphite_rollup.xml @@ -1,4 +1,6 @@ + + Asia/Istanbul metric diff --git a/tests/integration/test_graphite_merge_tree/test.py b/tests/integration/test_graphite_merge_tree/test.py index 9e48f12f007..c4364a03fd9 100644 --- a/tests/integration/test_graphite_merge_tree/test.py +++ b/tests/integration/test_graphite_merge_tree/test.py @@ -9,9 +9,11 @@ from helpers.test_tools import TSV from helpers.test_tools import csv_compare cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs=['configs/graphite_rollup.xml'], - user_configs=["configs/users.xml"]) +instance = cluster.add_instance( + "instance", + main_configs=["configs/graphite_rollup.xml"], + user_configs=["configs/users.xml"], +) q = instance.query @@ -19,7 +21,7 @@ q = instance.query def started_cluster(): try: cluster.start() - q('CREATE DATABASE test') + q("CREATE DATABASE test") yield cluster @@ -29,7 +31,8 @@ def started_cluster(): @pytest.fixture def graphite_table(started_cluster): - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite; CREATE TABLE test.graphite (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -37,11 +40,12 @@ CREATE TABLE test.graphite PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=8192; -''') +""" + ) yield - q('DROP TABLE test.graphite') + q("DROP TABLE test.graphite") def test_rollup_versions(graphite_table): @@ -52,35 +56,42 @@ def test_rollup_versions(graphite_table): # Insert rows with timestamps relative to the current time so that the # first retention clause is active. # Two parts are created. - q(''' + q( + """ INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('one_min.x1', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('one_min.x1', 200, {timestamp}, '{date}', 2); -'''.format(timestamp=timestamp, date=date)) +""".format( + timestamp=timestamp, date=date + ) + ) - expected1 = '''\ + expected1 = """\ one_min.x1 100 {timestamp} {date} 1 one_min.x1 200 {timestamp} {date} 2 -'''.format(timestamp=timestamp, date=date) +""".format( + timestamp=timestamp, date=date + ) - assert TSV( - q('SELECT * FROM test.graphite ORDER BY updated') - ) == TSV(expected1) + assert TSV(q("SELECT * FROM test.graphite ORDER BY updated")) == TSV(expected1) - q('OPTIMIZE TABLE test.graphite') + q("OPTIMIZE TABLE test.graphite") # After rollup only the row with max version is retained. - expected2 = '''\ + expected2 = """\ one_min.x1 200 {timestamp} {date} 2 -'''.format(timestamp=rounded_timestamp, date=date) +""".format( + timestamp=rounded_timestamp, date=date + ) - assert TSV(q('SELECT * FROM test.graphite')) == TSV(expected2) + assert TSV(q("SELECT * FROM test.graphite")) == TSV(expected2) def test_rollup_aggregation(graphite_table): # This query essentially emulates what rollup does. - result1 = q(''' + result1 = q( + """ SELECT avg(v), max(upd) FROM (SELECT timestamp, argMax(value, (updated, number)) AS v, @@ -94,16 +105,18 @@ FROM (SELECT timestamp, FROM system.numbers LIMIT 1000000) WHERE intDiv(timestamp, 600) * 600 = 1111444200 GROUP BY timestamp) -''') +""" + ) - expected1 = '''\ + expected1 = """\ 999634.9918367347 499999 -''' +""" assert TSV(result1) == TSV(expected1) # Timestamp 1111111111 is in sufficiently distant past # so that the last retention clause is active. - result2 = q(''' + result2 = q( + """ INSERT INTO test.graphite SELECT 'one_min.x' AS metric, toFloat64(number) AS value, @@ -115,17 +128,19 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - expected2 = '''\ + expected2 = """\ one_min.x 999634.9918367347 1111444200 2017-02-02 499999 -''' +""" assert TSV(result2) == TSV(expected2) def test_rollup_aggregation_2(graphite_table): - result = q(''' + result = q( + """ INSERT INTO test.graphite SELECT 'one_min.x' AS metric, toFloat64(number) AS value, @@ -137,17 +152,19 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - expected = '''\ + expected = """\ one_min.x 24 1111110600 2017-02-02 100 -''' +""" assert TSV(result) == TSV(expected) def test_multiple_paths_and_versions(graphite_table): - result = q(''' + result = q( + """ INSERT INTO test.graphite SELECT 'one_min.x' AS metric, toFloat64(number) AS value, @@ -172,69 +189,72 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - with open(p.join(p.dirname(__file__), - 'test_multiple_paths_and_versions.reference') - ) as reference: + with open( + p.join(p.dirname(__file__), "test_multiple_paths_and_versions.reference") + ) as reference: assert TSV(result) == TSV(reference) def test_multiple_output_blocks(graphite_table): MERGED_BLOCK_SIZE = 8192 - to_insert = '' - expected = '' + to_insert = "" + expected = "" for i in range(2 * MERGED_BLOCK_SIZE + 1): rolled_up_time = 1000000200 + 600 * i for j in range(3): cur_time = rolled_up_time + 100 * j - to_insert += 'one_min.x1 {} {} 2001-09-09 1\n'.format( - 10 * j, cur_time - ) - to_insert += 'one_min.x1 {} {} 2001-09-09 2\n'.format( + to_insert += "one_min.x1 {} {} 2001-09-09 1\n".format(10 * j, cur_time) + to_insert += "one_min.x1 {} {} 2001-09-09 2\n".format( 10 * (j + 1), cur_time ) - expected += 'one_min.x1 20 {} 2001-09-09 2\n'.format(rolled_up_time) + expected += "one_min.x1 20 {} 2001-09-09 2\n".format(rolled_up_time) - q('INSERT INTO test.graphite FORMAT TSV', to_insert) + q("INSERT INTO test.graphite FORMAT TSV", to_insert) - result = q(''' + result = q( + """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; -''') +""" + ) assert TSV(result) == TSV(expected) def test_paths_not_matching_any_pattern(graphite_table): - to_insert = '''\ + to_insert = """\ one_min.x1 100 1000000000 2001-09-09 1 zzzzzzzz 100 1000000001 2001-09-09 1 zzzzzzzz 200 1000000001 2001-09-09 2 -''' +""" - q('INSERT INTO test.graphite FORMAT TSV', to_insert) + q("INSERT INTO test.graphite FORMAT TSV", to_insert) - expected = '''\ + expected = """\ one_min.x1 100 999999600 2001-09-09 1 zzzzzzzz 200 1000000001 2001-09-09 2 -''' +""" - result = q(''' + result = q( + """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; -''') +""" + ) assert TSV(result) == TSV(expected) def test_system_graphite_retentions(graphite_table): - expected = ''' + expected = """ graphite_rollup all \\\\.count$ sum 0 0 1 0 ['test'] ['graphite'] graphite_rollup all \\\\.max$ max 0 0 2 0 ['test'] ['graphite'] graphite_rollup all ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite'] @@ -243,13 +263,14 @@ graphite_rollup all ^five_min\\\\. 0 300 3 0 ['test'] ['graphite'] graphite_rollup all ^one_min avg 31536000 600 4 0 ['test'] ['graphite'] graphite_rollup all ^one_min avg 7776000 300 4 0 ['test'] ['graphite'] graphite_rollup all ^one_min avg 0 60 4 0 ['test'] ['graphite'] - ''' - result = q('SELECT * from system.graphite_retentions') + """ + result = q("SELECT * from system.graphite_retentions") mismatch = csv_compare(result, expected) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n" - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite2; CREATE TABLE test.graphite2 (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -257,8 +278,9 @@ CREATE TABLE test.graphite2 PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=8192; - ''') - expected = ''' + """ + ) + expected = """ graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] @@ -267,19 +289,22 @@ graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] - ''' - result = q(''' + """ + result = q( + """ SELECT config_name, Tables.database, Tables.table FROM system.graphite_retentions - ''') + """ + ) assert TSV(result) == TSV(expected) def test_path_dangling_pointer(graphite_table): - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite2; CREATE TABLE test.graphite2 (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -287,37 +312,48 @@ CREATE TABLE test.graphite2 PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=1; - ''') + """ + ) - path = 'abcd' * 4000000 # 16MB - q('INSERT INTO test.graphite2 FORMAT TSV', - "{}\t0.0\t0\t2018-01-01\t100\n".format(path)) - q('INSERT INTO test.graphite2 FORMAT TSV', - "{}\t0.0\t0\t2018-01-01\t101\n".format(path)) + path = "abcd" * 4000000 # 16MB + q( + "INSERT INTO test.graphite2 FORMAT TSV", + "{}\t0.0\t0\t2018-01-01\t100\n".format(path), + ) + q( + "INSERT INTO test.graphite2 FORMAT TSV", + "{}\t0.0\t0\t2018-01-01\t101\n".format(path), + ) for version in range(10): - q('INSERT INTO test.graphite2 FORMAT TSV', - "{}\t0.0\t0\t2018-01-01\t{}\n".format(path, version)) + q( + "INSERT INTO test.graphite2 FORMAT TSV", + "{}\t0.0\t0\t2018-01-01\t{}\n".format(path, version), + ) while True: - q('OPTIMIZE TABLE test.graphite2 PARTITION 201801 FINAL') - parts = int(q("SELECT count() FROM system.parts " - "WHERE active AND database='test' " - "AND table='graphite2'")) + q("OPTIMIZE TABLE test.graphite2 PARTITION 201801 FINAL") + parts = int( + q( + "SELECT count() FROM system.parts " + "WHERE active AND database='test' " + "AND table='graphite2'" + ) + ) if parts == 1: break - print(('Parts', parts)) + print(("Parts", parts)) - assert TSV( - q("SELECT value, timestamp, date, updated FROM test.graphite2") - ) == TSV("0\t0\t2018-01-01\t101\n") + assert TSV(q("SELECT value, timestamp, date, updated FROM test.graphite2")) == TSV( + "0\t0\t2018-01-01\t101\n" + ) - q('DROP TABLE test.graphite2') + q("DROP TABLE test.graphite2") def test_combined_rules(graphite_table): # 1487970000 ~ Sat 25 Feb 00:00:00 MSK 2017 - to_insert = 'INSERT INTO test.graphite VALUES ' - expected_unmerged = '' + to_insert = "INSERT INTO test.graphite VALUES " + expected_unmerged = "" for i in range(384): to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format( v=1, t=1487970000 + (i * 300) @@ -325,18 +361,20 @@ def test_combined_rules(graphite_table): to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format( v=i, t=1487970000 + (i * 300) ) - expected_unmerged += ("five_min.count\t{v1}\t{t}\n" - "five_min.max\t{v2}\t{t}\n").format( - v1=1, v2=i, - t=1487970000 + (i * 300) - ) + expected_unmerged += ( + "five_min.count\t{v1}\t{t}\n" "five_min.max\t{v2}\t{t}\n" + ).format(v1=1, v2=i, t=1487970000 + (i * 300)) q(to_insert) - assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' - ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged) + assert TSV( + q( + "SELECT metric, value, timestamp FROM test.graphite" + " ORDER BY (timestamp, metric)" + ) + ) == TSV(expected_unmerged) - q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL') - expected_merged = ''' + q("OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL") + expected_merged = """ five_min.count 48 1487970000 2017-02-25 1 five_min.count 48 1487984400 2017-02-25 1 five_min.count 48 1487998800 2017-02-25 1 @@ -353,13 +391,15 @@ def test_combined_rules(graphite_table): five_min.max 287 1488042000 2017-02-25 1 five_min.max 335 1488056400 2017-02-26 1 five_min.max 383 1488070800 2017-02-26 1 - ''' - assert TSV(q('SELECT * FROM test.graphite' - ' ORDER BY (metric, timestamp)')) == TSV(expected_merged) + """ + assert TSV(q("SELECT * FROM test.graphite" " ORDER BY (metric, timestamp)")) == TSV( + expected_merged + ) def test_combined_rules_with_default(graphite_table): - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite; CREATE TABLE test.graphite (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -367,10 +407,11 @@ CREATE TABLE test.graphite PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=1; - ''') + """ + ) # 1487970000 ~ Sat 25 Feb 00:00:00 MSK 2017 - to_insert = 'INSERT INTO test.graphite VALUES ' - expected_unmerged = '' + to_insert = "INSERT INTO test.graphite VALUES " + expected_unmerged = "" for i in range(100): to_insert += "('top_level.count', {v}, {t}, toDate({t}), 1), ".format( v=1, t=1487970000 + (i * 60) @@ -378,18 +419,20 @@ CREATE TABLE test.graphite to_insert += "('top_level.max', {v}, {t}, toDate({t}), 1), ".format( v=i, t=1487970000 + (i * 60) ) - expected_unmerged += ("top_level.count\t{v1}\t{t}\n" - "top_level.max\t{v2}\t{t}\n").format( - v1=1, v2=i, - t=1487970000 + (i * 60) - ) + expected_unmerged += ( + "top_level.count\t{v1}\t{t}\n" "top_level.max\t{v2}\t{t}\n" + ).format(v1=1, v2=i, t=1487970000 + (i * 60)) q(to_insert) - assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' - ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged) + assert TSV( + q( + "SELECT metric, value, timestamp FROM test.graphite" + " ORDER BY (timestamp, metric)" + ) + ) == TSV(expected_unmerged) - q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL') - expected_merged = ''' + q("OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL") + expected_merged = """ top_level.count 10 1487970000 2017-02-25 1 top_level.count 10 1487970600 2017-02-25 1 top_level.count 10 1487971200 2017-02-25 1 @@ -410,13 +453,15 @@ CREATE TABLE test.graphite top_level.max 79 1487974200 2017-02-25 1 top_level.max 89 1487974800 2017-02-25 1 top_level.max 99 1487975400 2017-02-25 1 - ''' - assert TSV(q('SELECT * FROM test.graphite' - ' ORDER BY (metric, timestamp)')) == TSV(expected_merged) + """ + assert TSV(q("SELECT * FROM test.graphite" " ORDER BY (metric, timestamp)")) == TSV( + expected_merged + ) def test_broken_partial_rollup(graphite_table): - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite; CREATE TABLE test.graphite (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -424,41 +469,46 @@ CREATE TABLE test.graphite PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=1; - ''') - to_insert = '''\ + """ + ) + to_insert = """\ one_min.x1 100 1000000000 2001-09-09 1 zzzzzzzz 100 1000000001 2001-09-09 1 zzzzzzzz 200 1000000001 2001-09-09 2 -''' +""" - q('INSERT INTO test.graphite FORMAT TSV', to_insert) + q("INSERT INTO test.graphite FORMAT TSV", to_insert) - expected = '''\ + expected = """\ one_min.x1 100 1000000000 2001-09-09 1 zzzzzzzz 200 1000000001 2001-09-09 2 -''' +""" - result = q(''' + result = q( + """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; -''') +""" + ) assert TSV(result) == TSV(expected) def test_wrong_rollup_config(graphite_table): with pytest.raises(QueryRuntimeException) as exc: - q(''' + q( + """ CREATE TABLE test.graphite_not_created (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) ENGINE = GraphiteMergeTree('graphite_rollup_wrong_age_precision') PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=1; - ''') + """ + ) # The order of retentions is not guaranteed - assert ("age and precision should only grow up: " in str(exc.value)) - assert ("36000:600" in str(exc.value)) - assert ("72000:300" in str(exc.value)) + assert "age and precision should only grow up: " in str(exc.value) + assert "36000:600" in str(exc.value) + assert "72000:300" in str(exc.value) diff --git a/tests/integration/test_graphite_merge_tree_typed/configs/graphite_rollup.xml b/tests/integration/test_graphite_merge_tree_typed/configs/graphite_rollup.xml index c716540a61c..3369c9621e4 100644 --- a/tests/integration/test_graphite_merge_tree_typed/configs/graphite_rollup.xml +++ b/tests/integration/test_graphite_merge_tree_typed/configs/graphite_rollup.xml @@ -1,4 +1,6 @@ + + Asia/Istanbul metric diff --git a/tests/integration/test_graphite_merge_tree_typed/test.py b/tests/integration/test_graphite_merge_tree_typed/test.py index e26fd0d2e77..5647489f64f 100644 --- a/tests/integration/test_graphite_merge_tree_typed/test.py +++ b/tests/integration/test_graphite_merge_tree_typed/test.py @@ -10,9 +10,11 @@ from helpers.test_tools import TSV from helpers.test_tools import csv_compare cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs=['configs/graphite_rollup.xml'], - user_configs=["configs/users.xml"]) +instance = cluster.add_instance( + "instance", + main_configs=["configs/graphite_rollup.xml"], + user_configs=["configs/users.xml"], +) q = instance.query @@ -20,7 +22,7 @@ q = instance.query def started_cluster(): try: cluster.start() - q('CREATE DATABASE test') + q("CREATE DATABASE test") yield cluster @@ -30,7 +32,8 @@ def started_cluster(): @pytest.fixture def graphite_table(started_cluster): - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite; CREATE TABLE test.graphite (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -38,11 +41,12 @@ CREATE TABLE test.graphite PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=8192; -''') +""" + ) yield - q('DROP TABLE test.graphite') + q("DROP TABLE test.graphite") def test_rollup_versions_plain(graphite_table): @@ -53,30 +57,36 @@ def test_rollup_versions_plain(graphite_table): # Insert rows with timestamps relative to the current time so that the # first retention clause is active. # Two parts are created. - q(''' + q( + """ INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('one_min.x1', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('one_min.x1', 200, {timestamp}, '{date}', 2); -'''.format(timestamp=timestamp, date=date)) +""".format( + timestamp=timestamp, date=date + ) + ) - expected1 = '''\ + expected1 = """\ one_min.x1 100 {timestamp} {date} 1 one_min.x1 200 {timestamp} {date} 2 -'''.format(timestamp=timestamp, date=date) +""".format( + timestamp=timestamp, date=date + ) - assert TSV( - q('SELECT * FROM test.graphite ORDER BY updated') - ) == TSV(expected1) + assert TSV(q("SELECT * FROM test.graphite ORDER BY updated")) == TSV(expected1) - q('OPTIMIZE TABLE test.graphite') + q("OPTIMIZE TABLE test.graphite") # After rollup only the row with max version is retained. - expected2 = '''\ + expected2 = """\ one_min.x1 200 {timestamp} {date} 2 -'''.format(timestamp=rounded_timestamp, date=date) +""".format( + timestamp=rounded_timestamp, date=date + ) - assert TSV(q('SELECT * FROM test.graphite')) == TSV(expected2) + assert TSV(q("SELECT * FROM test.graphite")) == TSV(expected2) def test_rollup_versions_tagged(graphite_table): @@ -87,30 +97,38 @@ def test_rollup_versions_tagged(graphite_table): # Insert rows with timestamps relative to the current time so that the # first retention clause is active. # Two parts are created. - q(''' + q( + """ INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('x1?retention=one_min', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('x1?retention=one_min', 200, {timestamp}, '{date}', 2); -'''.format(timestamp=timestamp, date=date)) +""".format( + timestamp=timestamp, date=date + ) + ) - expected1 = '''\ + expected1 = """\ x1?retention=one_min 100 {timestamp} {date} 1 x1?retention=one_min 200 {timestamp} {date} 2 -'''.format(timestamp=timestamp, date=date) +""".format( + timestamp=timestamp, date=date + ) - result = q('SELECT * FROM test.graphite ORDER BY metric, updated') + result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected1) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n" - q('OPTIMIZE TABLE test.graphite') + q("OPTIMIZE TABLE test.graphite") # After rollup only the row with max version is retained. - expected2 = '''\ + expected2 = """\ x1?retention=one_min 200 {timestamp} {date} 2 -'''.format(timestamp=rounded_timestamp, date=date) +""".format( + timestamp=rounded_timestamp, date=date + ) - result = q('SELECT * FROM test.graphite ORDER BY metric, updated') + result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected2) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n" @@ -123,7 +141,8 @@ def test_rollup_versions_all(graphite_table): # Insert rows with timestamps relative to the current time so that the # first retention clause is active. # Two parts are created. - q(''' + q( + """ INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) @@ -132,35 +151,43 @@ INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1?env=staging', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1?env=staging', 200, {timestamp}, '{date}', 2); -'''.format(timestamp=timestamp, date=date)) +""".format( + timestamp=timestamp, date=date + ) + ) - expected1 = '''\ + expected1 = """\ ten_min.x1 100 {timestamp} {date} 1 ten_min.x1 200 {timestamp} {date} 2 ten_min.x1?env=staging 100 {timestamp} {date} 1 ten_min.x1?env=staging 200 {timestamp} {date} 2 -'''.format(timestamp=timestamp, date=date) +""".format( + timestamp=timestamp, date=date + ) - result = q('SELECT * FROM test.graphite ORDER BY metric, updated') + result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected1) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n" - q('OPTIMIZE TABLE test.graphite') + q("OPTIMIZE TABLE test.graphite") # After rollup only the row with max version is retained. - expected2 = '''\ + expected2 = """\ ten_min.x1 200 {timestamp} {date} 2 ten_min.x1?env=staging 200 {timestamp} {date} 2 -'''.format(timestamp=rounded_timestamp, date=date) +""".format( + timestamp=rounded_timestamp, date=date + ) - result = q('SELECT * FROM test.graphite ORDER BY metric, updated') + result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected2) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n" def test_rollup_aggregation_plain(graphite_table): # This query essentially emulates what rollup does. - result1 = q(''' + result1 = q( + """ SELECT avg(v), max(upd) FROM (SELECT timestamp, argMax(value, (updated, number)) AS v, @@ -174,16 +201,18 @@ FROM (SELECT timestamp, FROM system.numbers LIMIT 1000000) WHERE intDiv(timestamp, 600) * 600 = 1111444200 GROUP BY timestamp) -''') +""" + ) - expected1 = '''\ + expected1 = """\ 999634.9918367347 499999 -''' +""" assert TSV(result1) == TSV(expected1) # Timestamp 1111111111 is in sufficiently distant past # so that the last retention clause is active. - result2 = q(''' + result2 = q( + """ INSERT INTO test.graphite SELECT 'one_min.x' AS metric, toFloat64(number) AS value, @@ -195,18 +224,20 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - expected2 = '''\ + expected2 = """\ one_min.x 999634.9918367347 1111444200 2017-02-02 499999 -''' +""" assert TSV(result2) == TSV(expected2) def test_rollup_aggregation_tagged(graphite_table): # This query essentially emulates what rollup does. - result1 = q(''' + result1 = q( + """ SELECT avg(v), max(upd) FROM (SELECT timestamp, argMax(value, (updated, number)) AS v, @@ -220,16 +251,18 @@ FROM (SELECT timestamp, FROM system.numbers LIMIT 1000000) WHERE intDiv(timestamp, 600) * 600 = 1111444200 GROUP BY timestamp) -''') +""" + ) - expected1 = '''\ + expected1 = """\ 999634.9918367347 499999 -''' +""" assert TSV(result1) == TSV(expected1) # Timestamp 1111111111 is in sufficiently distant past # so that the last retention clause is active. - result2 = q(''' + result2 = q( + """ INSERT INTO test.graphite SELECT 'x?retention=one_min' AS metric, toFloat64(number) AS value, @@ -241,17 +274,19 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - expected2 = '''\ + expected2 = """\ x?retention=one_min 999634.9918367347 1111444200 2017-02-02 499999 -''' +""" assert TSV(result2) == TSV(expected2) def test_rollup_aggregation_2_plain(graphite_table): - result = q(''' + result = q( + """ INSERT INTO test.graphite SELECT 'one_min.x' AS metric, toFloat64(number) AS value, @@ -263,17 +298,19 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - expected = '''\ + expected = """\ one_min.x 24 1111110600 2017-02-02 100 -''' +""" assert TSV(result) == TSV(expected) def test_rollup_aggregation_2_tagged(graphite_table): - result = q(''' + result = q( + """ INSERT INTO test.graphite SELECT 'x?retention=one_min' AS metric, toFloat64(number) AS value, @@ -285,17 +322,19 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - expected = '''\ + expected = """\ x?retention=one_min 24 1111110600 2017-02-02 100 -''' +""" assert TSV(result) == TSV(expected) def test_multiple_paths_and_versions_plain(graphite_table): - result = q(''' + result = q( + """ INSERT INTO test.graphite SELECT 'one_min.x' AS metric, toFloat64(number) AS value, @@ -320,16 +359,18 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - with open(p.join(p.dirname(__file__), - 'test_multiple_paths_and_versions.reference.plain') - ) as reference: + with open( + p.join(p.dirname(__file__), "test_multiple_paths_and_versions.reference.plain") + ) as reference: assert TSV(result) == TSV(reference) def test_multiple_paths_and_versions_tagged(graphite_table): - result = q(''' + result = q( + """ INSERT INTO test.graphite SELECT 'x?retention=one_min' AS metric, toFloat64(number) AS value, @@ -354,97 +395,102 @@ INSERT INTO test.graphite OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - with open(p.join(p.dirname(__file__), - 'test_multiple_paths_and_versions.reference.tagged') - ) as reference: + with open( + p.join(p.dirname(__file__), "test_multiple_paths_and_versions.reference.tagged") + ) as reference: assert TSV(result) == TSV(reference) def test_multiple_output_blocks(graphite_table): MERGED_BLOCK_SIZE = 8192 - to_insert = '' - expected = '' + to_insert = "" + expected = "" for i in range(2 * MERGED_BLOCK_SIZE + 1): rolled_up_time = 1000000200 + 600 * i for j in range(3): cur_time = rolled_up_time + 100 * j - to_insert += 'one_min.x1 {} {} 2001-09-09 1\n'.format( - 10 * j, cur_time - ) - to_insert += 'one_min.x1 {} {} 2001-09-09 2\n'.format( + to_insert += "one_min.x1 {} {} 2001-09-09 1\n".format(10 * j, cur_time) + to_insert += "one_min.x1 {} {} 2001-09-09 2\n".format( 10 * (j + 1), cur_time ) - expected += 'one_min.x1 20 {} 2001-09-09 2\n'.format(rolled_up_time) + expected += "one_min.x1 20 {} 2001-09-09 2\n".format(rolled_up_time) - q('INSERT INTO test.graphite FORMAT TSV', to_insert) + q("INSERT INTO test.graphite FORMAT TSV", to_insert) - result = q(''' + result = q( + """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; -''') +""" + ) assert TSV(result) == TSV(expected) def test_paths_not_matching_any_pattern(graphite_table): - to_insert = '''\ + to_insert = """\ one_min.x1 100 1000000000 2001-09-09 1 zzzzzzzz 100 1000000001 2001-09-09 1 zzzzzzzz 200 1000000001 2001-09-09 2 -''' +""" - q('INSERT INTO test.graphite FORMAT TSV', to_insert) + q("INSERT INTO test.graphite FORMAT TSV", to_insert) - expected = '''\ + expected = """\ one_min.x1 100 999999600 2001-09-09 1 zzzzzzzz 200 1000000001 2001-09-09 2 -''' +""" - result = q(''' + result = q( + """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; -''') +""" + ) assert TSV(result) == TSV(expected) def test_rules_isolation(graphite_table): - to_insert = '''\ + to_insert = """\ one_min.x1 100 1000000000 2001-09-09 1 for_taggged 100 1000000001 2001-09-09 1 for_taggged 200 1000000001 2001-09-09 2 one_min?env=staging 100 1000000001 2001-09-09 1 one_min?env=staging 200 1000000001 2001-09-09 2 -''' +""" - q('INSERT INTO test.graphite FORMAT TSV', to_insert) + q("INSERT INTO test.graphite FORMAT TSV", to_insert) - expected = '''\ + expected = """\ for_taggged 200 1000000001 2001-09-09 2 one_min.x1 100 999999600 2001-09-09 1 one_min?env=staging 200 1000000001 2001-09-09 2 -''' +""" - result = q(''' + result = q( + """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; -''') +""" + ) - result = q('SELECT * FROM test.graphite ORDER BY metric, updated') + result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n" def test_system_graphite_retentions(graphite_table): - expected = ''' + expected = """ graphite_rollup plain \\\\.count$ sum 0 0 1 0 ['test'] ['graphite'] graphite_rollup plain \\\\.max$ max 0 0 2 0 ['test'] ['graphite'] graphite_rollup plain ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite'] @@ -465,13 +511,14 @@ graphite_rollup tagged ^for_taggged avg 0 60 7 0 ['test'] ['graphite'] graphite_rollup all ^ten_min\\\\. sum 31536000 28800 8 0 ['test'] ['graphite'] graphite_rollup all ^ten_min\\\\. sum 5184000 7200 8 0 ['test'] ['graphite'] graphite_rollup all ^ten_min\\\\. sum 0 600 8 0 ['test'] ['graphite'] - ''' - result = q('SELECT * from system.graphite_retentions') + """ + result = q("SELECT * from system.graphite_retentions") mismatch = csv_compare(result, expected) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n" - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite2; CREATE TABLE test.graphite2 (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -479,8 +526,9 @@ CREATE TABLE test.graphite2 PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=8192; - ''') - expected = ''' + """ + ) + expected = """ graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] @@ -489,19 +537,22 @@ graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] - ''' - result = q(''' + """ + result = q( + """ SELECT config_name, Tables.database, Tables.table FROM system.graphite_retentions - ''') + """ + ) assert csv_compare(result, expected), f"got\n{result}\nwant\n{expected}" def test_path_dangling_pointer(graphite_table): - q(''' + q( + """ DROP TABLE IF EXISTS test.graphite2; CREATE TABLE test.graphite2 (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) @@ -509,37 +560,48 @@ CREATE TABLE test.graphite2 PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=1; - ''') + """ + ) - path = 'abcd' * 4000000 # 16MB - q('INSERT INTO test.graphite2 FORMAT TSV', - "{}\t0.0\t0\t2018-01-01\t100\n".format(path)) - q('INSERT INTO test.graphite2 FORMAT TSV', - "{}\t0.0\t0\t2018-01-01\t101\n".format(path)) + path = "abcd" * 4000000 # 16MB + q( + "INSERT INTO test.graphite2 FORMAT TSV", + "{}\t0.0\t0\t2018-01-01\t100\n".format(path), + ) + q( + "INSERT INTO test.graphite2 FORMAT TSV", + "{}\t0.0\t0\t2018-01-01\t101\n".format(path), + ) for version in range(10): - q('INSERT INTO test.graphite2 FORMAT TSV', - "{}\t0.0\t0\t2018-01-01\t{}\n".format(path, version)) + q( + "INSERT INTO test.graphite2 FORMAT TSV", + "{}\t0.0\t0\t2018-01-01\t{}\n".format(path, version), + ) while True: - q('OPTIMIZE TABLE test.graphite2 PARTITION 201801 FINAL') - parts = int(q("SELECT count() FROM system.parts " - "WHERE active AND database='test' " - "AND table='graphite2'")) + q("OPTIMIZE TABLE test.graphite2 PARTITION 201801 FINAL") + parts = int( + q( + "SELECT count() FROM system.parts " + "WHERE active AND database='test' " + "AND table='graphite2'" + ) + ) if parts == 1: break - print(('Parts', parts)) + print(("Parts", parts)) - assert TSV( - q("SELECT value, timestamp, date, updated FROM test.graphite2") - ) == TSV("0\t0\t2018-01-01\t101\n") + assert TSV(q("SELECT value, timestamp, date, updated FROM test.graphite2")) == TSV( + "0\t0\t2018-01-01\t101\n" + ) - q('DROP TABLE test.graphite2') + q("DROP TABLE test.graphite2") def test_combined_rules(graphite_table): # 1487970000 ~ Sat 25 Feb 00:00:00 MSK 2017 - to_insert = 'INSERT INTO test.graphite VALUES ' - expected_unmerged = '' + to_insert = "INSERT INTO test.graphite VALUES " + expected_unmerged = "" for i in range(384): to_insert += "('five_min.count', {v}, {t}, toDate({t}), 1), ".format( v=1, t=1487970000 + (i * 300) @@ -547,18 +609,20 @@ def test_combined_rules(graphite_table): to_insert += "('five_min.max', {v}, {t}, toDate({t}), 1), ".format( v=i, t=1487970000 + (i * 300) ) - expected_unmerged += ("five_min.count\t{v1}\t{t}\n" - "five_min.max\t{v2}\t{t}\n").format( - v1=1, v2=i, - t=1487970000 + (i * 300) - ) + expected_unmerged += ( + "five_min.count\t{v1}\t{t}\n" "five_min.max\t{v2}\t{t}\n" + ).format(v1=1, v2=i, t=1487970000 + (i * 300)) q(to_insert) - assert TSV(q('SELECT metric, value, timestamp FROM test.graphite' - ' ORDER BY (timestamp, metric)')) == TSV(expected_unmerged) + assert TSV( + q( + "SELECT metric, value, timestamp FROM test.graphite" + " ORDER BY (timestamp, metric)" + ) + ) == TSV(expected_unmerged) - q('OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL') - expected_merged = ''' + q("OPTIMIZE TABLE test.graphite PARTITION 201702 FINAL") + expected_merged = """ five_min.count 48 1487970000 2017-02-25 1 five_min.count 48 1487984400 2017-02-25 1 five_min.count 48 1487998800 2017-02-25 1 @@ -575,6 +639,7 @@ def test_combined_rules(graphite_table): five_min.max 287 1488042000 2017-02-25 1 five_min.max 335 1488056400 2017-02-26 1 five_min.max 383 1488070800 2017-02-26 1 - ''' - assert TSV(q('SELECT * FROM test.graphite' - ' ORDER BY (metric, timestamp)')) == TSV(expected_merged) + """ + assert TSV(q("SELECT * FROM test.graphite" " ORDER BY (metric, timestamp)")) == TSV( + expected_merged + ) diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test.py b/tests/integration/test_groupBitmapAnd_on_distributed/test.py index b0fb55b13ff..4dbc81236e7 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test.py @@ -1,16 +1,35 @@ import pytest from helpers.cluster import ClickHouseCluster + cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/clusters.xml"], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=["configs/clusters.xml"], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=["configs/clusters.xml"], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=["configs/clusters.xml"], image='yandex/clickhouse-server', tag='21.5', with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/clusters.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/clusters.xml"], with_zookeeper=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/clusters.xml"], with_zookeeper=True +) +node4 = cluster.add_instance( + "node4", + main_configs=["configs/clusters.xml"], + image="yandex/clickhouse-server", + tag="21.5", + with_zookeeper=True, +) + def insert_data(node, table_name): - node.query("""INSERT INTO {} - VALUES (bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32))));""".format(table_name)) + node.query( + """INSERT INTO {} + VALUES (bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32))));""".format( + table_name + ) + ) + @pytest.fixture(scope="module") def start_cluster(): @@ -27,25 +46,36 @@ def test_groupBitmapAnd_on_distributed_table(start_cluster): cluster_name = "awesome_cluster" for node in (node1, node2): - node.query("""CREATE TABLE {} + node.query( + """CREATE TABLE {} ( z AggregateFunction(groupBitmap, UInt32) ) ENGINE = MergeTree() - ORDER BY tuple()""".format(local_table_name)) + ORDER BY tuple()""".format( + local_table_name + ) + ) - node.query("""CREATE TABLE {} + node.query( + """CREATE TABLE {} ( z AggregateFunction(groupBitmap, UInt32) ) - ENGINE = Distributed('{}', 'default', '{}')""".format(distributed_table_name, cluster_name, local_table_name)) + ENGINE = Distributed('{}', 'default', '{}')""".format( + distributed_table_name, cluster_name, local_table_name + ) + ) insert_data(node1, local_table_name) expected = "10" for node in (node1, node2): - result = node.query("select groupBitmapAnd(z) FROM {};".format(distributed_table_name)).strip() - assert(result == expected) + result = node.query( + "select groupBitmapAnd(z) FROM {};".format(distributed_table_name) + ).strip() + assert result == expected + def test_groupBitmapAnd_function_versioning(start_cluster): local_table_name = "bitmap_column_expr_versioning_test" @@ -53,30 +83,54 @@ def test_groupBitmapAnd_function_versioning(start_cluster): cluster_name = "test_version_cluster" for node in (node3, node4): - node.query("""CREATE TABLE {} + node.query( + """CREATE TABLE {} ( z AggregateFunction(groupBitmap, UInt32) ) ENGINE = MergeTree() - ORDER BY tuple()""".format(local_table_name)) + ORDER BY tuple()""".format( + local_table_name + ) + ) - node.query("""CREATE TABLE {} + node.query( + """CREATE TABLE {} ( z AggregateFunction(groupBitmap, UInt32) ) - ENGINE = Distributed('{}', 'default', '{}')""".format(distributed_table_name, cluster_name, local_table_name)) + ENGINE = Distributed('{}', 'default', '{}')""".format( + distributed_table_name, cluster_name, local_table_name + ) + ) - node.query("""INSERT INTO {} VALUES - (bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32))));""".format(local_table_name)) + node.query( + """INSERT INTO {} VALUES + (bitmapBuild(cast([1,2,3,4,5,6,7,8,9,10] as Array(UInt32))));""".format( + local_table_name + ) + ) expected = "10" - new_version_distributed_result = node3.query("select groupBitmapAnd(z) FROM {};".format(distributed_table_name)).strip() - old_version_distributed_result = node4.query("select groupBitmapAnd(z) FROM {};".format(distributed_table_name)).strip() - assert(new_version_distributed_result == expected) - assert(old_version_distributed_result == expected) + new_version_distributed_result = node3.query( + "select groupBitmapAnd(z) FROM {};".format(distributed_table_name) + ).strip() + old_version_distributed_result = node4.query( + "select groupBitmapAnd(z) FROM {};".format(distributed_table_name) + ).strip() + assert new_version_distributed_result == expected + assert old_version_distributed_result == expected - result_from_old_to_new_version = node3.query("select groupBitmapAnd(z) FROM remote('node4', default.{})".format(local_table_name)).strip() - assert(result_from_old_to_new_version == expected) + result_from_old_to_new_version = node3.query( + "select groupBitmapAnd(z) FROM remote('node4', default.{})".format( + local_table_name + ) + ).strip() + assert result_from_old_to_new_version == expected - result_from_new_to_old_version = node4.query("select groupBitmapAnd(z) FROM remote('node3', default.{})".format(local_table_name)).strip() - assert(result_from_new_to_old_version == expected) + result_from_new_to_old_version = node4.query( + "select groupBitmapAnd(z) FROM remote('node3', default.{})".format( + local_table_name + ) + ).strip() + assert result_from_new_to_old_version == expected diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 65ee3cb4261..109561dce1f 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -2,6 +2,8 @@ import os import pytest import sys import time +import pytz +import uuid import grpc from helpers.cluster import ClickHouseCluster, run_and_check from threading import Thread @@ -10,17 +12,21 @@ import lz4.frame GRPC_PORT = 9100 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -DEFAULT_ENCODING = 'utf-8' +DEFAULT_ENCODING = "utf-8" # Use grpcio-tools to generate *pb2.py files from *.proto. -proto_dir = os.path.join(SCRIPT_DIR, './protos') -gen_dir = os.path.join(SCRIPT_DIR, './_gen') +proto_dir = os.path.join(SCRIPT_DIR, "./protos") +gen_dir = os.path.join(SCRIPT_DIR, "./_gen") os.makedirs(gen_dir, exist_ok=True) run_and_check( - 'python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \ - {proto_dir}/clickhouse_grpc.proto'.format(proto_dir=proto_dir, gen_dir=gen_dir), shell=True) + "python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \ + {proto_dir}/clickhouse_grpc.proto".format( + proto_dir=proto_dir, gen_dir=gen_dir + ), + shell=True, +) sys.path.append(gen_dir) import clickhouse_grpc_pb2 @@ -29,13 +35,14 @@ import clickhouse_grpc_pb2_grpc # Utilities -config_dir = os.path.join(SCRIPT_DIR, './configs') +config_dir = os.path.join(SCRIPT_DIR, "./configs") cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml']) +node = cluster.add_instance("node", main_configs=["configs/grpc_config.xml"]) main_channel = None + def create_channel(): - node_ip_with_grpc_port = cluster.get_instance_ip('node') + ':' + str(GRPC_PORT) + node_ip_with_grpc_port = cluster.get_instance_ip("node") + ":" + str(GRPC_PORT) channel = grpc.insecure_channel(node_ip_with_grpc_port) grpc.channel_ready_future(channel).result(timeout=10) global main_channel @@ -43,30 +50,59 @@ def create_channel(): main_channel = channel return channel -def query_common(query_text, settings={}, input_data=[], input_data_delimiter='', output_format='TabSeparated', external_tables=[], - user_name='', password='', query_id='123', session_id='', stream_output=False, channel=None): + +def query_common( + query_text, + settings={}, + input_data=[], + input_data_delimiter="", + output_format="TabSeparated", + send_output_columns=False, + external_tables=[], + user_name="", + password="", + query_id="123", + session_id="", + stream_output=False, + channel=None, +): if type(input_data) is not list: input_data = [input_data] if type(input_data_delimiter) is str: - input_data_delimiter=input_data_delimiter.encode(DEFAULT_ENCODING) + input_data_delimiter = input_data_delimiter.encode(DEFAULT_ENCODING) if not channel: channel = main_channel stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(channel) + def query_info(): - input_data_part = input_data.pop(0) if input_data else b'' + input_data_part = input_data.pop(0) if input_data else b"" if type(input_data_part) is str: input_data_part = input_data_part.encode(DEFAULT_ENCODING) - return clickhouse_grpc_pb2.QueryInfo(query=query_text, settings=settings, input_data=input_data_part, - input_data_delimiter=input_data_delimiter, output_format=output_format, - external_tables=external_tables, user_name=user_name, password=password, query_id=query_id, - session_id=session_id, next_query_info=bool(input_data)) + return clickhouse_grpc_pb2.QueryInfo( + query=query_text, + settings=settings, + input_data=input_data_part, + input_data_delimiter=input_data_delimiter, + output_format=output_format, + send_output_columns=send_output_columns, + external_tables=external_tables, + user_name=user_name, + password=password, + query_id=query_id, + session_id=session_id, + next_query_info=bool(input_data), + ) + def send_query_info(): yield query_info() while input_data: input_data_part = input_data.pop(0) if type(input_data_part) is str: input_data_part = input_data_part.encode(DEFAULT_ENCODING) - yield clickhouse_grpc_pb2.QueryInfo(input_data=input_data_part, next_query_info=bool(input_data)) + yield clickhouse_grpc_pb2.QueryInfo( + input_data=input_data_part, next_query_info=bool(input_data) + ) + stream_input = len(input_data) > 1 if stream_input and stream_output: return list(stub.ExecuteQueryWithStreamIO(send_query_info())) @@ -77,58 +113,73 @@ def query_common(query_text, settings={}, input_data=[], input_data_delimiter='' else: return [stub.ExecuteQuery(query_info())] + def query_no_errors(*args, **kwargs): results = query_common(*args, **kwargs) - if results and results[-1].HasField('exception'): + if results and results[-1].HasField("exception"): raise Exception(results[-1].exception.display_text) return results + def query(*args, **kwargs): - output = b'' + output = b"" for result in query_no_errors(*args, **kwargs): output += result.output return output.decode(DEFAULT_ENCODING) + def query_and_get_error(*args, **kwargs): results = query_common(*args, **kwargs) - if not results or not results[-1].HasField('exception'): + if not results or not results[-1].HasField("exception"): raise Exception("Expected to be failed but succeeded!") return results[-1].exception + def query_and_get_totals(*args, **kwargs): - totals = b'' + totals = b"" for result in query_no_errors(*args, **kwargs): totals += result.totals return totals.decode(DEFAULT_ENCODING) + def query_and_get_extremes(*args, **kwargs): - extremes = b'' + extremes = b"" for result in query_no_errors(*args, **kwargs): extremes += result.extremes return extremes.decode(DEFAULT_ENCODING) + def query_and_get_logs(*args, **kwargs): logs = "" for result in query_no_errors(*args, **kwargs): for log_entry in result.logs: - #print(log_entry) + # print(log_entry) logs += log_entry.text + "\n" return logs + class QueryThread(Thread): - def __init__(self, query_text, expected_output, query_id, use_separate_channel=False): + def __init__( + self, query_text, expected_output, query_id, use_separate_channel=False + ): Thread.__init__(self) self.query_text = query_text self.expected_output = expected_output self.use_separate_channel = use_separate_channel self.query_id = query_id - + def run(self): if self.use_separate_channel: with create_channel() as channel: - assert query(self.query_text, query_id=self.query_id, channel=channel) == self.expected_output + assert ( + query(self.query_text, query_id=self.query_id, channel=channel) + == self.expected_output + ) else: - assert query(self.query_text, query_id=self.query_id) == self.expected_output + assert ( + query(self.query_text, query_id=self.query_id) == self.expected_output + ) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -136,10 +187,11 @@ def start_cluster(): try: with create_channel() as channel: yield cluster - + finally: cluster.shutdown() + @pytest.fixture(autouse=True) def reset_after_test(): yield @@ -148,12 +200,15 @@ def reset_after_test(): # Actual tests + def test_select_one(): assert query("SELECT 1") == "1\n" + def test_ordinary_query(): assert query("SELECT count() FROM numbers(100)") == "100\n" + def test_insert_query(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") query("INSERT INTO t VALUES (1),(2),(3)") @@ -162,76 +217,157 @@ def test_insert_query(): query("INSERT INTO t FORMAT TabSeparated", input_data="9\n10\n") assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" + def test_insert_query_streaming(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") - query("INSERT INTO t VALUES", input_data=["(1),(2),(3),", "(5),(4),(6),", "(7),(8),(9)"]) + query( + "INSERT INTO t VALUES", + input_data=["(1),(2),(3),", "(5),(4),(6),", "(7),(8),(9)"], + ) assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n7\n8\n9\n" + def test_insert_query_delimiter(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") - query("INSERT INTO t FORMAT CSV 1\n2", input_data=["3", "4\n5"], input_data_delimiter='\n') + query( + "INSERT INTO t FORMAT CSV 1\n2", + input_data=["3", "4\n5"], + input_data_delimiter="\n", + ) assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n" query("DROP TABLE t") query("CREATE TABLE t (a UInt8) ENGINE = Memory") query("INSERT INTO t FORMAT CSV 1\n2", input_data=["3", "4\n5"]) assert query("SELECT a FROM t ORDER BY a") == "1\n5\n234\n" + def test_insert_default_column(): - query("CREATE TABLE t (a UInt8, b Int32 DEFAULT 100 - a, c String DEFAULT 'c') ENGINE = Memory") + query( + "CREATE TABLE t (a UInt8, b Int32 DEFAULT 100 - a, c String DEFAULT 'c') ENGINE = Memory" + ) query("INSERT INTO t (c, a) VALUES ('x',1),('y',2)") query("INSERT INTO t (a) FORMAT TabSeparated", input_data="3\n4\n") - assert query("SELECT * FROM t ORDER BY a") == "1\t99\tx\n" \ - "2\t98\ty\n" \ - "3\t97\tc\n" \ - "4\t96\tc\n" + assert ( + query("SELECT * FROM t ORDER BY a") == "1\t99\tx\n" + "2\t98\ty\n" + "3\t97\tc\n" + "4\t96\tc\n" + ) + def test_insert_splitted_row(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") query("INSERT INTO t VALUES", input_data=["(1),(2),(", "3),(5),(4),(6)"]) assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n" + def test_output_format(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") query("INSERT INTO t VALUES (1),(2),(3)") - assert query("SELECT a FROM t ORDER BY a FORMAT JSONEachRow") == '{"a":1}\n{"a":2}\n{"a":3}\n' - assert query("SELECT a FROM t ORDER BY a", output_format="JSONEachRow") == '{"a":1}\n{"a":2}\n{"a":3}\n' + assert ( + query("SELECT a FROM t ORDER BY a FORMAT JSONEachRow") + == '{"a":1}\n{"a":2}\n{"a":3}\n' + ) + assert ( + query("SELECT a FROM t ORDER BY a", output_format="JSONEachRow") + == '{"a":1}\n{"a":2}\n{"a":3}\n' + ) + def test_totals_and_extremes(): query("CREATE TABLE t (x UInt8, y UInt8) ENGINE = Memory") query("INSERT INTO t VALUES (1, 2), (2, 4), (3, 2), (3, 3), (3, 4)") - assert query("SELECT sum(x), y FROM t GROUP BY y WITH TOTALS") == "4\t2\n3\t3\n5\t4\n" - assert query_and_get_totals("SELECT sum(x), y FROM t GROUP BY y WITH TOTALS") == "12\t0\n" + assert ( + query("SELECT sum(x), y FROM t GROUP BY y WITH TOTALS") == "4\t2\n3\t3\n5\t4\n" + ) + assert ( + query_and_get_totals("SELECT sum(x), y FROM t GROUP BY y WITH TOTALS") + == "12\t0\n" + ) assert query("SELECT x, y FROM t") == "1\t2\n2\t4\n3\t2\n3\t3\n3\t4\n" - assert query_and_get_extremes("SELECT x, y FROM t", settings={"extremes": "1"}) == "1\t2\n3\t4\n" + assert ( + query_and_get_extremes("SELECT x, y FROM t", settings={"extremes": "1"}) + == "1\t2\n3\t4\n" + ) + + +def test_get_query_details(): + result = list( + query_no_errors("CREATE TABLE t (a UInt8) ENGINE = Memory", query_id="123") + )[0] + assert result.query_id == "123" + pytz.timezone(result.time_zone) + assert result.output_format == "" + assert len(result.output_columns) == 0 + assert result.output == b"" + # + result = list( + query_no_errors("SELECT 'a', 1", query_id="", output_format="TabSeparated") + )[0] + uuid.UUID(result.query_id) + pytz.timezone(result.time_zone) + assert result.output_format == "TabSeparated" + assert len(result.output_columns) == 0 + assert result.output == b"a\t1\n" + # + result = list( + query_no_errors( + "SELECT 'a' AS x, 1 FORMAT JSONEachRow", + query_id="", + send_output_columns=True, + ) + )[0] + uuid.UUID(result.query_id) + pytz.timezone(result.time_zone) + assert result.output_format == "JSONEachRow" + assert ([(col.name, col.type) for col in result.output_columns]) == [ + ("x", "String"), + ("1", "UInt8"), + ] + assert result.output == b'{"x":"a","1":1}\n' + def test_errors_handling(): e = query_and_get_error("") - #print(e) + # print(e) assert "Empty query" in e.display_text query("CREATE TABLE t (a UInt8) ENGINE = Memory") e = query_and_get_error("CREATE TABLE t (a UInt8) ENGINE = Memory") assert "Table default.t already exists" in e.display_text + def test_authentication(): query("CREATE USER OR REPLACE john IDENTIFIED BY 'qwe123'") - assert query("SELECT currentUser()", user_name="john", password="qwe123") == "john\n" + assert ( + query("SELECT currentUser()", user_name="john", password="qwe123") == "john\n" + ) query("DROP USER john") + def test_logs(): - logs = query_and_get_logs("SELECT 1", settings={'send_logs_level':'debug'}) + logs = query_and_get_logs("SELECT 1", settings={"send_logs_level": "debug"}) assert "SELECT 1" in logs assert "Read 1 rows" in logs assert "Peak memory usage" in logs + def test_progress(): - results = query_no_errors("SELECT number, sleep(0.31) FROM numbers(8) SETTINGS max_block_size=2, interactive_delay=100000", stream_output=True) - #print(results) - assert str(results) ==\ -"""[progress { + results = query_no_errors( + "SELECT number, sleep(0.31) FROM numbers(8) SETTINGS max_block_size=2, interactive_delay=100000", + stream_output=True, + ) + for result in results: + result.time_zone = "" + result.query_id = "" + # print(results) + assert ( + str(results) + == """[progress { read_rows: 2 read_bytes: 16 total_rows_to_read: 8 } +output_format: "TabSeparated" , output: "0\\t0\\n1\\t0\\n" , progress { read_rows: 2 @@ -256,6 +392,8 @@ def test_progress(): rows_before_limit: 8 } ]""" + ) + def test_session_settings(): session_a = "session A" @@ -264,8 +402,21 @@ def test_session_settings(): query("SET custom_y=2", session_id=session_a) query("SET custom_x=3", session_id=session_b) query("SET custom_y=4", session_id=session_b) - assert query("SELECT getSetting('custom_x'), getSetting('custom_y')", session_id=session_a) == "1\t2\n" - assert query("SELECT getSetting('custom_x'), getSetting('custom_y')", session_id=session_b) == "3\t4\n" + assert ( + query( + "SELECT getSetting('custom_x'), getSetting('custom_y')", + session_id=session_a, + ) + == "1\t2\n" + ) + assert ( + query( + "SELECT getSetting('custom_x'), getSetting('custom_y')", + session_id=session_b, + ) + == "3\t4\n" + ) + def test_session_temp_tables(): session_a = "session A" @@ -278,182 +429,322 @@ def test_session_temp_tables(): assert query("SELECT * FROM my_temp_table", session_id=session_b) == "20\n" assert query("SELECT * FROM my_temp_table", session_id=session_a) == "10\n" + def test_no_session(): e = query_and_get_error("SET custom_x=1") assert "There is no session" in e.display_text e = query_and_get_error("CREATE TEMPORARY TABLE my_temp_table(a Int8)") assert "There is no session" in e.display_text + def test_input_function(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") - query("INSERT INTO t SELECT col1 * col2 FROM input('col1 UInt8, col2 UInt8') FORMAT CSV", input_data=["5,4\n", "8,11\n", "10,12\n"]) + query( + "INSERT INTO t SELECT col1 * col2 FROM input('col1 UInt8, col2 UInt8') FORMAT CSV", + input_data=["5,4\n", "8,11\n", "10,12\n"], + ) assert query("SELECT a FROM t ORDER BY a") == "20\n88\n120\n" - query("INSERT INTO t SELECT col1 * col2 FROM input('col1 UInt8, col2 UInt8') FORMAT CSV 11,13") + query( + "INSERT INTO t SELECT col1 * col2 FROM input('col1 UInt8, col2 UInt8') FORMAT CSV 11,13" + ) assert query("SELECT a FROM t ORDER BY a") == "20\n88\n120\n143\n" - query("INSERT INTO t SELECT col1 * col2 FROM input('col1 UInt8, col2 UInt8') FORMAT CSV 20,10\n", input_data="15,15\n") + query( + "INSERT INTO t SELECT col1 * col2 FROM input('col1 UInt8, col2 UInt8') FORMAT CSV 20,10\n", + input_data="15,15\n", + ) assert query("SELECT a FROM t ORDER BY a") == "20\n88\n120\n143\n200\n225\n" + def test_external_table(): - columns = [clickhouse_grpc_pb2.NameAndType(name='UserID', type='UInt64'), clickhouse_grpc_pb2.NameAndType(name='UserName', type='String')] - ext1 = clickhouse_grpc_pb2.ExternalTable(name='ext1', columns=columns, data=b'1\tAlex\n2\tBen\n3\tCarl\n', format='TabSeparated') - assert query("SELECT * FROM ext1 ORDER BY UserID", external_tables=[ext1]) == "1\tAlex\n"\ - "2\tBen\n"\ - "3\tCarl\n" - ext2 = clickhouse_grpc_pb2.ExternalTable(name='ext2', columns=columns, data=b'4,Daniel\n5,Ethan\n', format='CSV') - assert query("SELECT * FROM (SELECT * FROM ext1 UNION ALL SELECT * FROM ext2) ORDER BY UserID", external_tables=[ext1, ext2]) == "1\tAlex\n"\ - "2\tBen\n"\ - "3\tCarl\n"\ - "4\tDaniel\n"\ - "5\tEthan\n" - unnamed_columns = [clickhouse_grpc_pb2.NameAndType(type='UInt64'), clickhouse_grpc_pb2.NameAndType(type='String')] - unnamed_table = clickhouse_grpc_pb2.ExternalTable(columns=unnamed_columns, data=b'6\tGeorge\n7\tFred\n') - assert query("SELECT * FROM _data ORDER BY _2", external_tables=[unnamed_table]) == "7\tFred\n"\ - "6\tGeorge\n" + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + ext1 = clickhouse_grpc_pb2.ExternalTable( + name="ext1", + columns=columns, + data=b"1\tAlex\n2\tBen\n3\tCarl\n", + format="TabSeparated", + ) + assert ( + query("SELECT * FROM ext1 ORDER BY UserID", external_tables=[ext1]) + == "1\tAlex\n" + "2\tBen\n" + "3\tCarl\n" + ) + ext2 = clickhouse_grpc_pb2.ExternalTable( + name="ext2", columns=columns, data=b"4,Daniel\n5,Ethan\n", format="CSV" + ) + assert ( + query( + "SELECT * FROM (SELECT * FROM ext1 UNION ALL SELECT * FROM ext2) ORDER BY UserID", + external_tables=[ext1, ext2], + ) + == "1\tAlex\n" + "2\tBen\n" + "3\tCarl\n" + "4\tDaniel\n" + "5\tEthan\n" + ) + unnamed_columns = [ + clickhouse_grpc_pb2.NameAndType(type="UInt64"), + clickhouse_grpc_pb2.NameAndType(type="String"), + ] + unnamed_table = clickhouse_grpc_pb2.ExternalTable( + columns=unnamed_columns, data=b"6\tGeorge\n7\tFred\n" + ) + assert ( + query("SELECT * FROM _data ORDER BY _2", external_tables=[unnamed_table]) + == "7\tFred\n" + "6\tGeorge\n" + ) + def test_external_table_streaming(): - columns = [clickhouse_grpc_pb2.NameAndType(name='UserID', type='UInt64'), clickhouse_grpc_pb2.NameAndType(name='UserName', type='String')] + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + def send_query_info(): - yield clickhouse_grpc_pb2.QueryInfo(query="SELECT * FROM exts ORDER BY UserID", - external_tables=[clickhouse_grpc_pb2.ExternalTable(name='exts', columns=columns, data=b'1\tAlex\n2\tBen\n3\tCarl\n')], - next_query_info=True) - yield clickhouse_grpc_pb2.QueryInfo(external_tables=[clickhouse_grpc_pb2.ExternalTable(name='exts', data=b'4\tDaniel\n5\tEthan\n')]) + yield clickhouse_grpc_pb2.QueryInfo( + query="SELECT * FROM exts ORDER BY UserID", + external_tables=[ + clickhouse_grpc_pb2.ExternalTable( + name="exts", columns=columns, data=b"1\tAlex\n2\tBen\n3\tCarl\n" + ) + ], + next_query_info=True, + ) + yield clickhouse_grpc_pb2.QueryInfo( + external_tables=[ + clickhouse_grpc_pb2.ExternalTable( + name="exts", data=b"4\tDaniel\n5\tEthan\n" + ) + ] + ) + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQueryWithStreamInput(send_query_info()) - assert result.output == b'1\tAlex\n'\ - b'2\tBen\n'\ - b'3\tCarl\n'\ - b'4\tDaniel\n'\ - b'5\tEthan\n' + assert ( + result.output == b"1\tAlex\n" + b"2\tBen\n" + b"3\tCarl\n" + b"4\tDaniel\n" + b"5\tEthan\n" + ) + def test_simultaneous_queries_same_channel(): - threads=[] + threads = [] try: for i in range(0, 100): - thread = QueryThread("SELECT sum(number) FROM numbers(10)", expected_output="45\n", query_id='sqA'+str(i)) + thread = QueryThread( + "SELECT sum(number) FROM numbers(10)", + expected_output="45\n", + query_id="sqA" + str(i), + ) threads.append(thread) thread.start() finally: for thread in threads: thread.join() + def test_simultaneous_queries_multiple_channels(): - threads=[] + threads = [] try: for i in range(0, 100): - thread = QueryThread("SELECT sum(number) FROM numbers(10)", expected_output="45\n", query_id='sqB'+str(i), use_separate_channel=True) + thread = QueryThread( + "SELECT sum(number) FROM numbers(10)", + expected_output="45\n", + query_id="sqB" + str(i), + use_separate_channel=True, + ) threads.append(thread) thread.start() finally: for thread in threads: thread.join() + def test_cancel_while_processing_input(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") + def send_query_info(): - yield clickhouse_grpc_pb2.QueryInfo(query="INSERT INTO t FORMAT TabSeparated", input_data=b'1\n2\n3\n', next_query_info=True) - yield clickhouse_grpc_pb2.QueryInfo(input_data=b'4\n5\n6\n', next_query_info=True) + yield clickhouse_grpc_pb2.QueryInfo( + query="INSERT INTO t FORMAT TabSeparated", + input_data=b"1\n2\n3\n", + next_query_info=True, + ) + yield clickhouse_grpc_pb2.QueryInfo( + input_data=b"4\n5\n6\n", next_query_info=True + ) yield clickhouse_grpc_pb2.QueryInfo(cancel=True) + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQueryWithStreamInput(send_query_info()) assert result.cancelled == True assert result.progress.written_rows == 6 assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n" + def test_cancel_while_generating_output(): def send_query_info(): - yield clickhouse_grpc_pb2.QueryInfo(query="SELECT number, sleep(0.2) FROM numbers(10) SETTINGS max_block_size=2") + yield clickhouse_grpc_pb2.QueryInfo( + query="SELECT number, sleep(0.2) FROM numbers(10) SETTINGS max_block_size=2" + ) time.sleep(0.5) yield clickhouse_grpc_pb2.QueryInfo(cancel=True) + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) results = list(stub.ExecuteQueryWithStreamIO(send_query_info())) assert len(results) >= 1 assert results[-1].cancelled == True - output = b'' + output = b"" for result in results: output += result.output - assert output == b'0\t0\n1\t0\n2\t0\n3\t0\n' + assert output == b"0\t0\n1\t0\n2\t0\n3\t0\n" + def test_compressed_output(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000)", output_compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT 0 FROM numbers(1000)", output_compression_type="lz4" + ) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQuery(query_info) - assert lz4.frame.decompress(result.output) == (b'0\n')*1000 + assert lz4.frame.decompress(result.output) == (b"0\n") * 1000 + def test_compressed_output_streaming(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(100000)", output_compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT 0 FROM numbers(100000)", output_compression_type="lz4" + ) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) d_context = lz4.frame.create_decompression_context() - data = b'' + data = b"" for result in stub.ExecuteQueryWithStreamOutput(query_info): d1, _, _ = lz4.frame.decompress_chunk(d_context, result.output) data += d1 - assert data == (b'0\n')*100000 + assert data == (b"0\n") * 100000 + def test_compressed_output_gzip(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000)", output_compression_type="gzip", output_compression_level=6) + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT 0 FROM numbers(1000)", + output_compression_type="gzip", + output_compression_level=6, + ) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQuery(query_info) - assert gzip.decompress(result.output) == (b'0\n')*1000 + assert gzip.decompress(result.output) == (b"0\n") * 1000 + def test_compressed_totals_and_extremes(): query("CREATE TABLE t (x UInt8, y UInt8) ENGINE = Memory") query("INSERT INTO t VALUES (1, 2), (2, 4), (3, 2), (3, 3), (3, 4)") stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT sum(x), y FROM t GROUP BY y WITH TOTALS", output_compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT sum(x), y FROM t GROUP BY y WITH TOTALS", + output_compression_type="lz4", + ) result = stub.ExecuteQuery(query_info) - assert lz4.frame.decompress(result.totals) == b'12\t0\n' - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT x, y FROM t", settings={"extremes": "1"}, output_compression_type="lz4") + assert lz4.frame.decompress(result.totals) == b"12\t0\n" + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT x, y FROM t", + settings={"extremes": "1"}, + output_compression_type="lz4", + ) result = stub.ExecuteQuery(query_info) - assert lz4.frame.decompress(result.extremes) == b'1\t2\n3\t4\n' + assert lz4.frame.decompress(result.extremes) == b"1\t2\n3\t4\n" + def test_compressed_insert_query_streaming(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") - data = lz4.frame.compress(b'(1),(2),(3),(5),(4),(6),(7),(8),(9)') + data = lz4.frame.compress(b"(1),(2),(3),(5),(4),(6),(7),(8),(9)") sz1 = len(data) // 3 sz2 = len(data) // 3 d1 = data[:sz1] - d2 = data[sz1:sz1+sz2] - d3 = data[sz1+sz2:] + d2 = data[sz1 : sz1 + sz2] + d3 = data[sz1 + sz2 :] + def send_query_info(): - yield clickhouse_grpc_pb2.QueryInfo(query="INSERT INTO t VALUES", input_data=d1, input_compression_type="lz4", next_query_info=True) + yield clickhouse_grpc_pb2.QueryInfo( + query="INSERT INTO t VALUES", + input_data=d1, + input_compression_type="lz4", + next_query_info=True, + ) yield clickhouse_grpc_pb2.QueryInfo(input_data=d2, next_query_info=True) yield clickhouse_grpc_pb2.QueryInfo(input_data=d3) + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) stub.ExecuteQueryWithStreamInput(send_query_info()) assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n7\n8\n9\n" + def test_compressed_external_table(): - columns = [clickhouse_grpc_pb2.NameAndType(name='UserID', type='UInt64'), clickhouse_grpc_pb2.NameAndType(name='UserName', type='String')] - d1 = lz4.frame.compress(b'1\tAlex\n2\tBen\n3\tCarl\n') - d2 = gzip.compress(b'4,Daniel\n5,Ethan\n') - ext1 = clickhouse_grpc_pb2.ExternalTable(name='ext1', columns=columns, data=d1, format='TabSeparated', compression_type="lz4") - ext2 = clickhouse_grpc_pb2.ExternalTable(name='ext2', columns=columns, data=d2, format='CSV', compression_type="gzip") + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + d1 = lz4.frame.compress(b"1\tAlex\n2\tBen\n3\tCarl\n") + d2 = gzip.compress(b"4,Daniel\n5,Ethan\n") + ext1 = clickhouse_grpc_pb2.ExternalTable( + name="ext1", + columns=columns, + data=d1, + format="TabSeparated", + compression_type="lz4", + ) + ext2 = clickhouse_grpc_pb2.ExternalTable( + name="ext2", columns=columns, data=d2, format="CSV", compression_type="gzip" + ) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT * FROM (SELECT * FROM ext1 UNION ALL SELECT * FROM ext2) ORDER BY UserID", external_tables=[ext1, ext2]) + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT * FROM (SELECT * FROM ext1 UNION ALL SELECT * FROM ext2) ORDER BY UserID", + external_tables=[ext1, ext2], + ) result = stub.ExecuteQuery(query_info) - assert result.output == b"1\tAlex\n"\ - b"2\tBen\n"\ - b"3\tCarl\n"\ - b"4\tDaniel\n"\ - b"5\tEthan\n" + assert ( + result.output == b"1\tAlex\n" + b"2\tBen\n" + b"3\tCarl\n" + b"4\tDaniel\n" + b"5\tEthan\n" + ) + def test_transport_compression(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000000)", transport_compression_type='gzip', transport_compression_level=3) + query_info = clickhouse_grpc_pb2.QueryInfo( + query="SELECT 0 FROM numbers(1000000)", + transport_compression_type="gzip", + transport_compression_level=3, + ) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQuery(query_info) - assert result.output == (b'0\n')*1000000 + assert result.output == (b"0\n") * 1000000 + def test_opentelemetry_context_propagation(): trace_id = "80c190b5-9dc1-4eae-82b9-6c261438c817" parent_span_id = 123 trace_state = "some custom state" trace_id_hex = trace_id.replace("-", "") - parent_span_id_hex = f'{parent_span_id:0>16X}' - metadata = [("traceparent", f"00-{trace_id_hex}-{parent_span_id_hex}-01"), ("tracestate", trace_state)] + parent_span_id_hex = f"{parent_span_id:0>16X}" + metadata = [ + ("traceparent", f"00-{trace_id_hex}-{parent_span_id_hex}-01"), + ("tracestate", trace_state), + ] stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 1") result = stub.ExecuteQuery(query_info, metadata=metadata) assert result.output == b"1\n" node.query("SYSTEM FLUSH LOGS") - assert node.query(f"SELECT attribute['db.statement'], attribute['clickhouse.tracestate'] FROM system.opentelemetry_span_log " - f"WHERE trace_id='{trace_id}' AND parent_span_id={parent_span_id}") == "SELECT 1\tsome custom state\n" + assert ( + node.query( + f"SELECT attribute['db.statement'], attribute['clickhouse.tracestate'] FROM system.opentelemetry_span_log " + f"WHERE trace_id='{trace_id}' AND parent_span_id={parent_span_id}" + ) + == "SELECT 1\tsome custom state\n" + ) diff --git a/tests/integration/test_grpc_protocol_ssl/test.py b/tests/integration/test_grpc_protocol_ssl/test.py index 1f21fbe5f8a..60c3ccd7a9d 100644 --- a/tests/integration/test_grpc_protocol_ssl/test.py +++ b/tests/integration/test_grpc_protocol_ssl/test.py @@ -5,19 +5,23 @@ import grpc from helpers.cluster import ClickHouseCluster, run_and_check GRPC_PORT = 9100 -NODE_IP = '10.5.172.77' # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -DEFAULT_ENCODING = 'utf-8' +DEFAULT_ENCODING = "utf-8" # Use grpcio-tools to generate *pb2.py files from *.proto. -proto_dir = os.path.join(SCRIPT_DIR, './protos') -gen_dir = os.path.join(SCRIPT_DIR, './_gen') +proto_dir = os.path.join(SCRIPT_DIR, "./protos") +gen_dir = os.path.join(SCRIPT_DIR, "./_gen") os.makedirs(gen_dir, exist_ok=True) run_and_check( - 'python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \ - {proto_dir}/clickhouse_grpc.proto'.format(proto_dir=proto_dir, gen_dir=gen_dir), shell=True) + "python3 -m grpc_tools.protoc -I{proto_dir} --python_out={gen_dir} --grpc_python_out={gen_dir} \ + {proto_dir}/clickhouse_grpc.proto".format( + proto_dir=proto_dir, gen_dir=gen_dir + ), + shell=True, +) sys.path.append(gen_dir) import clickhouse_grpc_pb2 @@ -26,62 +30,80 @@ import clickhouse_grpc_pb2_grpc # Utilities -node_ip_with_grpc_port = NODE_IP + ':' + str(GRPC_PORT) -config_dir = os.path.join(SCRIPT_DIR, './configs') +node_ip_with_grpc_port = NODE_IP + ":" + str(GRPC_PORT) +config_dir = os.path.join(SCRIPT_DIR, "./configs") cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', ipv4_address=NODE_IP, main_configs=['configs/grpc_config.xml', 'configs/server-key.pem', 'configs/server-cert.pem', 'configs/ca-cert.pem']) +node = cluster.add_instance( + "node", + ipv4_address=NODE_IP, + main_configs=[ + "configs/grpc_config.xml", + "configs/server-key.pem", + "configs/server-cert.pem", + "configs/ca-cert.pem", + ], +) + def create_secure_channel(): - ca_cert = open(os.path.join(config_dir, 'ca-cert.pem'), 'rb').read() - client_key = open(os.path.join(config_dir, 'client-key.pem'), 'rb').read() - client_cert = open(os.path.join(config_dir, 'client-cert.pem'), 'rb').read() + ca_cert = open(os.path.join(config_dir, "ca-cert.pem"), "rb").read() + client_key = open(os.path.join(config_dir, "client-key.pem"), "rb").read() + client_cert = open(os.path.join(config_dir, "client-cert.pem"), "rb").read() credentials = grpc.ssl_channel_credentials(ca_cert, client_key, client_cert) channel = grpc.secure_channel(node_ip_with_grpc_port, credentials) grpc.channel_ready_future(channel).result(timeout=10) return channel + def create_insecure_channel(): channel = grpc.insecure_channel(node_ip_with_grpc_port) grpc.channel_ready_future(channel).result(timeout=2) return channel + def create_secure_channel_with_wrong_client_certificate(): - ca_cert = open(os.path.join(config_dir, 'ca-cert.pem'), 'rb').read() - client_key = open(os.path.join(config_dir, 'wrong-client-key.pem'), 'rb').read() - client_cert = open(os.path.join(config_dir, 'wrong-client-cert.pem'), 'rb').read() + ca_cert = open(os.path.join(config_dir, "ca-cert.pem"), "rb").read() + client_key = open(os.path.join(config_dir, "wrong-client-key.pem"), "rb").read() + client_cert = open(os.path.join(config_dir, "wrong-client-cert.pem"), "rb").read() credentials = grpc.ssl_channel_credentials(ca_cert, client_key, client_cert) channel = grpc.secure_channel(node_ip_with_grpc_port, credentials) grpc.channel_ready_future(channel).result(timeout=2) return channel + def query(query_text, channel): query_info = clickhouse_grpc_pb2.QueryInfo(query=query_text) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(channel) result = stub.ExecuteQuery(query_info) - if result and result.HasField('exception'): + if result and result.HasField("exception"): raise Exception(result.exception.display_text) return result.output.decode(DEFAULT_ENCODING) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): cluster.start() try: yield cluster - + finally: cluster.shutdown() + # Actual tests + def test_secure_channel(): with create_secure_channel() as channel: assert query("SELECT 'ok'", channel) == "ok\n" + def test_insecure_channel(): with pytest.raises(grpc.FutureTimeoutError): with create_insecure_channel() as channel: query("SELECT 'ok'", channel) + def test_wrong_client_certificate(): with pytest.raises(grpc.FutureTimeoutError): with create_insecure_channel() as channel: diff --git a/tests/integration/test_hedged_requests/test.py b/tests/integration/test_hedged_requests/test.py index b137dadfca9..5d24b66cd02 100644 --- a/tests/integration/test_hedged_requests/test.py +++ b/tests/integration/test_hedged_requests/test.py @@ -11,34 +11,46 @@ from helpers.network import PartitionManager from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -NODES = {'node_' + str(i): None for i in (1, 2, 3)} +NODES = {"node_" + str(i): None for i in (1, 2, 3)} -NODES['node'] = None +NODES["node"] = None # Sleep time in milliseconds. sleep_time = 30000 + @pytest.fixture(scope="module") def started_cluster(): - NODES['node'] = cluster.add_instance( - 'node', stay_alive=True, main_configs=['configs/remote_servers.xml'], user_configs=['configs/users.xml']) + NODES["node"] = cluster.add_instance( + "node", + stay_alive=True, + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.xml"], + ) for name in NODES: - if name != 'node': - NODES[name] = cluster.add_instance(name, user_configs=['configs/users1.xml']) + if name != "node": + NODES[name] = cluster.add_instance( + name, user_configs=["configs/users1.xml"] + ) try: cluster.start() for node_id, node in list(NODES.items()): - node.query('''CREATE TABLE test_hedged (id UInt32, date Date) ENGINE = - MergeTree() ORDER BY id PARTITION BY toYYYYMM(date)''') + node.query( + """CREATE TABLE test_hedged (id UInt32, date Date) ENGINE = + MergeTree() ORDER BY id PARTITION BY toYYYYMM(date)""" + ) - node.query("INSERT INTO test_hedged select number, toDate(number) from numbers(100);") + node.query( + "INSERT INTO test_hedged select number, toDate(number) from numbers(100);" + ) - - NODES['node'].query('''CREATE TABLE distributed (id UInt32, date Date) ENGINE = - Distributed('test_cluster', 'default', 'test_hedged')''') + NODES["node"].query( + """CREATE TABLE distributed (id UInt32, date Date) ENGINE = + Distributed('test_cluster', 'default', 'test_hedged')""" + ) yield cluster @@ -46,24 +58,28 @@ def started_cluster(): cluster.shutdown() -config = ''' +config = """ {sleep_in_send_tables_status_ms} {sleep_in_send_data_ms} -''' +""" def check_query(expected_replica, receive_timeout=300): - NODES['node'].restart_clickhouse() - + NODES["node"].restart_clickhouse() + # Without hedged requests select query will last more than 30 seconds, # with hedged requests it will last just around 1-2 second start = time.time() - result = NODES['node'].query("SELECT hostName(), id FROM distributed ORDER BY id LIMIT 1 SETTINGS receive_timeout={}".format(receive_timeout)); + result = NODES["node"].query( + "SELECT hostName(), id FROM distributed ORDER BY id LIMIT 1 SETTINGS receive_timeout={}".format( + receive_timeout + ) + ) query_time = time.time() - start assert TSV(result) == TSV(expected_replica + "\t0") @@ -75,9 +91,16 @@ def check_query(expected_replica, receive_timeout=300): def check_settings(node_name, sleep_in_send_tables_status_ms, sleep_in_send_data_ms): attempts = 0 while attempts < 1000: - setting1 = NODES[node_name].http_query("SELECT value FROM system.settings WHERE name='sleep_in_send_tables_status_ms'") - setting2 = NODES[node_name].http_query("SELECT value FROM system.settings WHERE name='sleep_in_send_data_ms'") - if int(setting1) == sleep_in_send_tables_status_ms and int(setting2) == sleep_in_send_data_ms: + setting1 = NODES[node_name].http_query( + "SELECT value FROM system.settings WHERE name='sleep_in_send_tables_status_ms'" + ) + setting2 = NODES[node_name].http_query( + "SELECT value FROM system.settings WHERE name='sleep_in_send_data_ms'" + ) + if ( + int(setting1) == sleep_in_send_tables_status_ms + and int(setting2) == sleep_in_send_data_ms + ): return time.sleep(0.1) attempts += 1 @@ -86,31 +109,56 @@ def check_settings(node_name, sleep_in_send_tables_status_ms, sleep_in_send_data def check_changing_replica_events(expected_count): - result = NODES['node'].query("SELECT value FROM system.events WHERE event='HedgedRequestsChangeReplica'") + result = NODES["node"].query( + "SELECT value FROM system.events WHERE event='HedgedRequestsChangeReplica'" + ) # If server load is high we can see more than expected # replica change events, but never less than expected assert int(result) >= expected_count -def update_configs(node_1_sleep_in_send_tables_status=0, node_1_sleep_in_send_data=0, - node_2_sleep_in_send_tables_status=0, node_2_sleep_in_send_data=0, - node_3_sleep_in_send_tables_status=0, node_3_sleep_in_send_data=0): - NODES['node_1'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_1_sleep_in_send_tables_status, sleep_in_send_data_ms=node_1_sleep_in_send_data)) +def update_configs( + node_1_sleep_in_send_tables_status=0, + node_1_sleep_in_send_data=0, + node_2_sleep_in_send_tables_status=0, + node_2_sleep_in_send_data=0, + node_3_sleep_in_send_tables_status=0, + node_3_sleep_in_send_data=0, +): + NODES["node_1"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_1_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_1_sleep_in_send_data, + ), + ) - NODES['node_2'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_2_sleep_in_send_tables_status, sleep_in_send_data_ms=node_2_sleep_in_send_data)) + NODES["node_2"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_2_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_2_sleep_in_send_data, + ), + ) - NODES['node_3'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_3_sleep_in_send_tables_status, sleep_in_send_data_ms=node_3_sleep_in_send_data)) + NODES["node_3"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_3_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_3_sleep_in_send_data, + ), + ) - check_settings('node_1', node_1_sleep_in_send_tables_status, node_1_sleep_in_send_data) - check_settings('node_2', node_2_sleep_in_send_tables_status, node_2_sleep_in_send_data) - check_settings('node_3', node_3_sleep_in_send_tables_status, node_3_sleep_in_send_data) + check_settings( + "node_1", node_1_sleep_in_send_tables_status, node_1_sleep_in_send_data + ) + check_settings( + "node_2", node_2_sleep_in_send_tables_status, node_2_sleep_in_send_data + ) + check_settings( + "node_3", node_3_sleep_in_send_tables_status, node_3_sleep_in_send_data + ) def test_stuck_replica(started_cluster): @@ -121,16 +169,22 @@ def test_stuck_replica(started_cluster): check_query(expected_replica="node_2") check_changing_replica_events(1) - result = NODES['node'].query("SELECT slowdowns_count FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + result = NODES["node"].query( + "SELECT slowdowns_count FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'" + ) assert TSV(result) == TSV("1") - result = NODES['node'].query("SELECT hostName(), id FROM distributed ORDER BY id LIMIT 1"); + result = NODES["node"].query( + "SELECT hostName(), id FROM distributed ORDER BY id LIMIT 1" + ) assert TSV(result) == TSV("node_2\t0") # Check that we didn't choose node_1 first again and slowdowns_count didn't increase. - result = NODES['node'].query("SELECT slowdowns_count FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + result = NODES["node"].query( + "SELECT slowdowns_count FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'" + ) assert TSV(result) == TSV("1") @@ -141,12 +195,16 @@ def test_long_query(started_cluster): update_configs() # Restart to reset pool states. - NODES['node'].restart_clickhouse() + NODES["node"].restart_clickhouse() - result = NODES['node'].query("select hostName(), max(id + sleep(1.5)) from distributed settings max_block_size = 1, max_threads = 1;") + result = NODES["node"].query( + "select hostName(), max(id + sleep(1.5)) from distributed settings max_block_size = 1, max_threads = 1;" + ) assert TSV(result) == TSV("node_1\t99") - NODES['node'].query("INSERT INTO distributed select number, toDate(number) from numbers(100);") + NODES["node"].query( + "INSERT INTO distributed select number, toDate(number) from numbers(100);" + ) def test_send_table_status_sleep(started_cluster): @@ -156,7 +214,10 @@ def test_send_table_status_sleep(started_cluster): def test_send_table_status_sleep2(started_cluster): - update_configs(node_1_sleep_in_send_tables_status=sleep_time, node_2_sleep_in_send_tables_status=sleep_time) + update_configs( + node_1_sleep_in_send_tables_status=sleep_time, + node_2_sleep_in_send_tables_status=sleep_time, + ) check_query(expected_replica="node_3") check_changing_replica_events(2) @@ -168,36 +229,48 @@ def test_send_data(started_cluster): def test_send_data2(started_cluster): - update_configs(node_1_sleep_in_send_data=sleep_time, node_2_sleep_in_send_data=sleep_time) + update_configs( + node_1_sleep_in_send_data=sleep_time, node_2_sleep_in_send_data=sleep_time + ) check_query(expected_replica="node_3") check_changing_replica_events(2) def test_combination1(started_cluster): - update_configs(node_1_sleep_in_send_tables_status=sleep_time, node_2_sleep_in_send_data=sleep_time) + update_configs( + node_1_sleep_in_send_tables_status=sleep_time, + node_2_sleep_in_send_data=sleep_time, + ) check_query(expected_replica="node_3") check_changing_replica_events(2) def test_combination2(started_cluster): - update_configs(node_1_sleep_in_send_data=sleep_time, node_2_sleep_in_send_tables_status=sleep_time) + update_configs( + node_1_sleep_in_send_data=sleep_time, + node_2_sleep_in_send_tables_status=sleep_time, + ) check_query(expected_replica="node_3") check_changing_replica_events(2) def test_combination3(started_cluster): - update_configs(node_1_sleep_in_send_data=sleep_time, - node_2_sleep_in_send_tables_status=1000, - node_3_sleep_in_send_data=sleep_time) + update_configs( + node_1_sleep_in_send_data=sleep_time, + node_2_sleep_in_send_tables_status=1000, + node_3_sleep_in_send_data=sleep_time, + ) check_query(expected_replica="node_2") check_changing_replica_events(3) def test_combination4(started_cluster): - update_configs(node_1_sleep_in_send_tables_status=1000, - node_1_sleep_in_send_data=sleep_time, - node_2_sleep_in_send_tables_status=1000, - node_3_sleep_in_send_tables_status=1000) + update_configs( + node_1_sleep_in_send_tables_status=1000, + node_1_sleep_in_send_data=sleep_time, + node_2_sleep_in_send_tables_status=1000, + node_3_sleep_in_send_tables_status=1000, + ) check_query(expected_replica="node_2") check_changing_replica_events(4) @@ -205,9 +278,11 @@ def test_combination4(started_cluster): def test_receive_timeout1(started_cluster): # Check the situation when first two replicas get receive timeout # in establishing connection, but the third replica is ok. - update_configs(node_1_sleep_in_send_tables_status=3000, - node_2_sleep_in_send_tables_status=3000, - node_3_sleep_in_send_data=1000) + update_configs( + node_1_sleep_in_send_tables_status=3000, + node_2_sleep_in_send_tables_status=3000, + node_3_sleep_in_send_data=1000, + ) check_query(expected_replica="node_3", receive_timeout=2) check_changing_replica_events(2) @@ -216,9 +291,10 @@ def test_receive_timeout2(started_cluster): # Check the situation when first replica get receive timeout # in packet receiving but there are replicas in process of # connection establishing. - update_configs(node_1_sleep_in_send_data=4000, - node_2_sleep_in_send_tables_status=2000, - node_3_sleep_in_send_tables_status=2000) + update_configs( + node_1_sleep_in_send_data=4000, + node_2_sleep_in_send_tables_status=2000, + node_3_sleep_in_send_tables_status=2000, + ) check_query(expected_replica="node_2", receive_timeout=3) check_changing_replica_events(3) - diff --git a/tests/integration/test_hedged_requests_parallel/test.py b/tests/integration/test_hedged_requests_parallel/test.py index 3ea6cf80622..ff83e99e6dd 100644 --- a/tests/integration/test_hedged_requests_parallel/test.py +++ b/tests/integration/test_hedged_requests_parallel/test.py @@ -11,33 +11,46 @@ from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -NODES = {'node_' + str(i): None for i in (1, 2, 3, 4)} -NODES['node'] = None +NODES = {"node_" + str(i): None for i in (1, 2, 3, 4)} +NODES["node"] = None # Cleep time in milliseconds. sleep_time = 30000 + @pytest.fixture(scope="module") def started_cluster(): cluster = ClickHouseCluster(__file__) - NODES['node'] = cluster.add_instance( - 'node', stay_alive=True, main_configs=['configs/remote_servers.xml'], user_configs=['configs/users.xml']) + NODES["node"] = cluster.add_instance( + "node", + stay_alive=True, + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.xml"], + ) for name in NODES: - if name != 'node': - NODES[name] = cluster.add_instance(name, user_configs=['configs/users1.xml']) - + if name != "node": + NODES[name] = cluster.add_instance( + name, user_configs=["configs/users1.xml"] + ) + try: cluster.start() for node_id, node in list(NODES.items()): - node.query('''CREATE TABLE test_hedged (id UInt32, date Date) ENGINE = - MergeTree() ORDER BY id PARTITION BY toYYYYMM(date)''') + node.query( + """CREATE TABLE test_hedged (id UInt32, date Date) ENGINE = + MergeTree() ORDER BY id PARTITION BY toYYYYMM(date)""" + ) - node.query("INSERT INTO test_hedged SELECT number, toDateTime(number) FROM numbers(100)") + node.query( + "INSERT INTO test_hedged SELECT number, toDateTime(number) FROM numbers(100)" + ) - NODES['node'].query('''CREATE TABLE distributed (id UInt32, date Date) ENGINE = - Distributed('test_cluster', 'default', 'test_hedged')''') + NODES["node"].query( + """CREATE TABLE distributed (id UInt32, date Date) ENGINE = + Distributed('test_cluster', 'default', 'test_hedged')""" + ) yield cluster @@ -45,14 +58,14 @@ def started_cluster(): cluster.shutdown() -config = ''' +config = """ {sleep_in_send_tables_status_ms} {sleep_in_send_data_ms} -''' +""" QUERY_1 = "SELECT count() FROM distributed" @@ -60,13 +73,13 @@ QUERY_2 = "SELECT * FROM distributed" def check_query(query=QUERY_1): - NODES['node'].restart_clickhouse() + NODES["node"].restart_clickhouse() # Without hedged requests select query will last more than 30 seconds, # with hedged requests it will last just around 1-2 second start = time.time() - NODES['node'].query(query); + NODES["node"].query(query) query_time = time.time() - start print("Query time:", query_time) @@ -76,9 +89,16 @@ def check_query(query=QUERY_1): def check_settings(node_name, sleep_in_send_tables_status_ms, sleep_in_send_data_ms): attempts = 0 while attempts < 1000: - setting1 = NODES[node_name].http_query("SELECT value FROM system.settings WHERE name='sleep_in_send_tables_status_ms'") - setting2 = NODES[node_name].http_query("SELECT value FROM system.settings WHERE name='sleep_in_send_data_ms'") - if int(setting1) == sleep_in_send_tables_status_ms and int(setting2) == sleep_in_send_data_ms: + setting1 = NODES[node_name].http_query( + "SELECT value FROM system.settings WHERE name='sleep_in_send_tables_status_ms'" + ) + setting2 = NODES[node_name].http_query( + "SELECT value FROM system.settings WHERE name='sleep_in_send_data_ms'" + ) + if ( + int(setting1) == sleep_in_send_tables_status_ms + and int(setting2) == sleep_in_send_data_ms + ): return time.sleep(0.1) attempts += 1 @@ -87,75 +107,116 @@ def check_settings(node_name, sleep_in_send_tables_status_ms, sleep_in_send_data def check_changing_replica_events(expected_count): - result = NODES['node'].query("SELECT value FROM system.events WHERE event='HedgedRequestsChangeReplica'") + result = NODES["node"].query( + "SELECT value FROM system.events WHERE event='HedgedRequestsChangeReplica'" + ) # If server load is high we can see more than expected # replica change events, but never less than expected assert int(result) >= expected_count -def update_configs(node_1_sleep_in_send_tables_status=0, node_1_sleep_in_send_data=0, - node_2_sleep_in_send_tables_status=0, node_2_sleep_in_send_data=0, - node_3_sleep_in_send_tables_status=0, node_3_sleep_in_send_data=0, - node_4_sleep_in_send_tables_status=0, node_4_sleep_in_send_data=0): - NODES['node_1'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_1_sleep_in_send_tables_status, sleep_in_send_data_ms=node_1_sleep_in_send_data)) +def update_configs( + node_1_sleep_in_send_tables_status=0, + node_1_sleep_in_send_data=0, + node_2_sleep_in_send_tables_status=0, + node_2_sleep_in_send_data=0, + node_3_sleep_in_send_tables_status=0, + node_3_sleep_in_send_data=0, + node_4_sleep_in_send_tables_status=0, + node_4_sleep_in_send_data=0, +): + NODES["node_1"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_1_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_1_sleep_in_send_data, + ), + ) - NODES['node_2'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_2_sleep_in_send_tables_status, sleep_in_send_data_ms=node_2_sleep_in_send_data)) + NODES["node_2"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_2_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_2_sleep_in_send_data, + ), + ) - NODES['node_3'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_3_sleep_in_send_tables_status, sleep_in_send_data_ms=node_3_sleep_in_send_data)) + NODES["node_3"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_3_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_3_sleep_in_send_data, + ), + ) - NODES['node_4'].replace_config( - '/etc/clickhouse-server/users.d/users1.xml', - config.format(sleep_in_send_tables_status_ms=node_4_sleep_in_send_tables_status, sleep_in_send_data_ms=node_4_sleep_in_send_data)) + NODES["node_4"].replace_config( + "/etc/clickhouse-server/users.d/users1.xml", + config.format( + sleep_in_send_tables_status_ms=node_4_sleep_in_send_tables_status, + sleep_in_send_data_ms=node_4_sleep_in_send_data, + ), + ) - check_settings('node_1', node_1_sleep_in_send_tables_status, node_1_sleep_in_send_data) - check_settings('node_2', node_2_sleep_in_send_tables_status, node_2_sleep_in_send_data) - check_settings('node_3', node_3_sleep_in_send_tables_status, node_3_sleep_in_send_data) - check_settings('node_4', node_4_sleep_in_send_tables_status, node_4_sleep_in_send_data) + check_settings( + "node_1", node_1_sleep_in_send_tables_status, node_1_sleep_in_send_data + ) + check_settings( + "node_2", node_2_sleep_in_send_tables_status, node_2_sleep_in_send_data + ) + check_settings( + "node_3", node_3_sleep_in_send_tables_status, node_3_sleep_in_send_data + ) + check_settings( + "node_4", node_4_sleep_in_send_tables_status, node_4_sleep_in_send_data + ) def test_send_table_status_sleep(started_cluster): - update_configs(node_1_sleep_in_send_tables_status=sleep_time, node_2_sleep_in_send_tables_status=sleep_time) + update_configs( + node_1_sleep_in_send_tables_status=sleep_time, + node_2_sleep_in_send_tables_status=sleep_time, + ) check_query() check_changing_replica_events(2) def test_send_data(started_cluster): - update_configs(node_1_sleep_in_send_data=sleep_time, node_2_sleep_in_send_data=sleep_time) + update_configs( + node_1_sleep_in_send_data=sleep_time, node_2_sleep_in_send_data=sleep_time + ) check_query() check_changing_replica_events(2) def test_combination1(started_cluster): - update_configs(node_1_sleep_in_send_tables_status=1000, - node_2_sleep_in_send_tables_status=1000, - node_3_sleep_in_send_data=sleep_time) + update_configs( + node_1_sleep_in_send_tables_status=1000, + node_2_sleep_in_send_tables_status=1000, + node_3_sleep_in_send_data=sleep_time, + ) check_query() check_changing_replica_events(3) def test_combination2(started_cluster): - update_configs(node_1_sleep_in_send_data=sleep_time, - node_2_sleep_in_send_tables_status=1000, - node_3_sleep_in_send_data=sleep_time, - node_4_sleep_in_send_tables_status=1000) + update_configs( + node_1_sleep_in_send_data=sleep_time, + node_2_sleep_in_send_tables_status=1000, + node_3_sleep_in_send_data=sleep_time, + node_4_sleep_in_send_tables_status=1000, + ) check_query() check_changing_replica_events(4) def test_query_with_no_data_to_sample(started_cluster): - update_configs(node_1_sleep_in_send_data=sleep_time, - node_2_sleep_in_send_data=sleep_time) + update_configs( + node_1_sleep_in_send_data=sleep_time, node_2_sleep_in_send_data=sleep_time + ) # When there is no way to sample data, the whole query will be performed by # the first replica and the second replica will just send EndOfStream, # so we will change only the first replica here. check_query(query=QUERY_2) check_changing_replica_events(1) - diff --git a/tests/integration/test_hive_query/test.py b/tests/integration/test_hive_query/test.py index a68ae0b066d..9e9a20fa6d1 100644 --- a/tests/integration/test_hive_query/test.py +++ b/tests/integration/test_hive_query/test.py @@ -16,66 +16,131 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) def started_cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance('h0_0_0', main_configs=['configs/config.xml'], extra_configs=[ 'configs/hdfs-site.xml'], with_hive=True) - + cluster.add_instance( + "h0_0_0", + main_configs=["configs/config.xml"], + extra_configs=["configs/hdfs-site.xml"], + with_hive=True, + ) + logging.info("Starting cluster ...") cluster.start() yield cluster finally: cluster.shutdown() + def test_create_parquet_table(started_cluster): - logging.info('Start testing creating hive table ...') - node = started_cluster.instances['h0_0_0'] - node.query("set input_format_parquet_allow_missing_columns = true") - result = node.query(""" - DROP TABLE IF EXISTS default.demo_parquet; - CREATE TABLE default.demo_parquet (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day) - """) - logging.info("create result {}".format(result)) - time.sleep(120) - assert result.strip() == '' + logging.info("Start testing creating hive table ...") + node = started_cluster.instances["h0_0_0"] + test_passed = False + for i in range(10): + node.query("set input_format_parquet_allow_missing_columns = true") + result = node.query( + """ +DROP TABLE IF EXISTS default.demo_parquet; +CREATE TABLE default.demo_parquet (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day) + """ + ) + logging.info("create result {}".format(result)) + if result.strip() == "": + test_passed = True + break + time.sleep(60) + assert test_passed + + +def test_create_parquet_table_1(started_cluster): + logging.info("Start testing creating hive table ...") + node = started_cluster.instances["h0_0_0"] + for i in range(10): + node.query("set input_format_parquet_allow_missing_columns = true") + result = node.query( + """ +DROP TABLE IF EXISTS default.demo_parquet_parts; +CREATE TABLE default.demo_parquet_parts (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String), `hour` String) ENGINE = Hive('thrift://hivetest:9083', 'test', 'parquet_demo') PARTITION BY(day, hour); + """ + ) + logging.info("create result {}".format(result)) + if result.strip() == "": + test_passed = True + break + time.sleep(60) + assert test_passed + def test_create_orc_table(started_cluster): - logging.info('Start testing creating hive table ...') - node = started_cluster.instances['h0_0_0'] - result = node.query(""" + logging.info("Start testing creating hive table ...") + node = started_cluster.instances["h0_0_0"] + test_passed = False + for i in range(10): + result = node.query( + """ DROP TABLE IF EXISTS default.demo_orc; CREATE TABLE default.demo_orc (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_orc') PARTITION BY(day) - """) - logging.info("create result {}".format(result)) - - assert result.strip() == '' + """ + ) + logging.info("create result {}".format(result)) + if result.strip() == "": + test_passed = True + break + time.sleep(60) + + assert test_passed + def test_create_text_table(started_cluster): - logging.info('Start testing creating hive table ...') - node = started_cluster.instances['h0_0_0'] - result = node.query(""" + logging.info("Start testing creating hive table ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ DROP TABLE IF EXISTS default.demo_text; CREATE TABLE default.demo_text (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo_text') PARTITION BY (tuple()) - """) + """ + ) logging.info("create result {}".format(result)) - - assert result.strip() == '' + + assert result.strip() == "" + def test_parquet_groupby(started_cluster): - logging.info('Start testing groupby ...') - node = started_cluster.instances['h0_0_0'] - result = node.query(""" + logging.info("Start testing groupby ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ SELECT day, count(*) FROM default.demo_parquet group by day order by day - """) + """ + ) expected_result = """2021-11-01 1 2021-11-05 2 2021-11-11 1 2021-11-16 2 """ assert result == expected_result + + +def test_parquet_in_filter(started_cluster): + logging.info("Start testing groupby ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ + SELECT count(*) FROM default.demo_parquet_parts where day = '2021-11-05' and hour in ('00') + """ + ) + expected_result = """2 +""" + logging.info("query result:{}".format(result)) + assert result == expected_result + + def test_orc_groupby(started_cluster): - logging.info('Start testing groupby ...') - node = started_cluster.instances['h0_0_0'] - result = node.query(""" + logging.info("Start testing groupby ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ SELECT day, count(*) FROM default.demo_orc group by day order by day - """) + """ + ) expected_result = """2021-11-01 1 2021-11-05 2 2021-11-11 1 @@ -83,11 +148,28 @@ def test_orc_groupby(started_cluster): """ assert result == expected_result + +def test_hive_columns_prunning(started_cluster): + logging.info("Start testing groupby ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ + SELECT count(*) FROM default.demo_parquet_parts where day = '2021-11-05' + """ + ) + expected_result = """4 +""" + logging.info("query result:{}".format(result)) + assert result == expected_result + + def test_text_count(started_cluster): - node = started_cluster.instances['h0_0_0'] - result = node.query(""" + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ SELECT day, count(*) FROM default.demo_orc group by day order by day SETTINGS format_csv_delimiter = '\x01' - """) + """ + ) expected_result = """2021-11-01 1 2021-11-05 2 2021-11-11 1 @@ -95,38 +177,61 @@ def test_text_count(started_cluster): """ assert result == expected_result + def test_parquet_groupby_with_cache(started_cluster): - logging.info('Start testing groupby ...') - node = started_cluster.instances['h0_0_0'] - result = node.query(""" + logging.info("Start testing groupby ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ SELECT day, count(*) FROM default.demo_parquet group by day order by day - """) + """ + ) expected_result = """2021-11-01 1 2021-11-05 2 2021-11-11 1 2021-11-16 2 """ assert result == expected_result + + +def test_parquet_groupby_by_hive_function(started_cluster): + logging.info("Start testing groupby ...") + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ + SELECT day, count(*) FROM hive('thrift://hivetest:9083', 'test', 'demo', '`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)', 'day') group by day order by day + """ + ) + expected_result = """2021-11-01 1 +2021-11-05 2 +2021-11-11 1 +2021-11-16 2 +""" + assert result == expected_result + + def test_cache_read_bytes(started_cluster): - node = started_cluster.instances['h0_0_0'] - node.query("set input_format_parquet_allow_missing_columns = true") - result = node.query(""" - DROP TABLE IF EXISTS default.demo_parquet; - CREATE TABLE default.demo_parquet (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day) - """) - result = node.query(""" - SELECT day, count(*) FROM default.demo_parquet group by day order by day - """) - result = node.query(""" - SELECT day, count(*) FROM default.demo_parquet group by day order by day - """) - expected_result = """2021-11-01 1 -2021-11-05 2 -2021-11-11 1 -2021-11-16 2 -""" - time.sleep(120) - assert result == expected_result - result = node.query("select sum(ProfileEvent_ExternalDataSourceLocalCacheReadBytes) from system.metric_log where ProfileEvent_ExternalDataSourceLocalCacheReadBytes > 0") - logging.info("Read bytes from cache:{}".format(result)) - assert result.strip() != '0' + node = started_cluster.instances["h0_0_0"] + result = node.query( + """ + CREATE TABLE IF NOT EXISTS default.demo_parquet_1 (`id` Nullable(String), `score` Nullable(Int32), `day` Nullable(String)) ENGINE = Hive('thrift://hivetest:9083', 'test', 'demo') PARTITION BY(day) + """ + ) + test_passed = False + for i in range(10): + result = node.query( + """ + SELECT day, count(*) FROM default.demo_parquet_1 group by day order by day settings input_format_parquet_allow_missing_columns = true + """ + ) + node.query("system flush logs") + result = node.query( + "select sum(ProfileEvent_ExternalDataSourceLocalCacheReadBytes) from system.metric_log where ProfileEvent_ExternalDataSourceLocalCacheReadBytes > 0" + ) + if result.strip() == "0": + logging.info("ProfileEvent_ExternalDataSourceLocalCacheReadBytes == 0") + time.sleep(10) + continue + test_passed = True + break + assert test_passed diff --git a/tests/integration/test_host_ip_change/test.py b/tests/integration/test_host_ip_change/test.py index 7525914e803..604f2e5dc76 100644 --- a/tests/integration/test_host_ip_change/test.py +++ b/tests/integration/test_host_ip_change/test.py @@ -10,19 +10,29 @@ cluster = ClickHouseCluster(__file__) def _fill_nodes(nodes, table_name): for node in nodes: node.query( - ''' + """ CREATE DATABASE IF NOT EXISTS test; CREATE TABLE IF NOT EXISTS {0}(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{0}', '{1}') ORDER BY id PARTITION BY toYYYYMM(date); - '''.format(table_name, node.name) + """.format( + table_name, node.name + ) ) -node1 = cluster.add_instance('node1', main_configs=['configs/listen_host.xml'], with_zookeeper=True, - ipv6_address='2001:3984:3989::1:1111') -node2 = cluster.add_instance('node2', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'], - with_zookeeper=True, ipv6_address='2001:3984:3989::1:1112') +node1 = cluster.add_instance( + "node1", + main_configs=["configs/listen_host.xml"], + with_zookeeper=True, + ipv6_address="2001:3984:3989::1:1111", +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/listen_host.xml", "configs/dns_update_long.xml"], + with_zookeeper=True, + ipv6_address="2001:3984:3989::1:1112", +) @pytest.fixture(scope="module") @@ -30,7 +40,7 @@ def cluster_without_dns_cache_update(): try: cluster.start() - _fill_nodes([node1, node2], 'test_table_drop') + _fill_nodes([node1, node2], "test_table_drop") yield cluster @@ -46,7 +56,9 @@ def cluster_without_dns_cache_update(): # node2 has long dns_cache_update_period, so dns cache update wouldn't work def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update): # First we check, that normal replication works - node1.query("INSERT INTO test_table_drop VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") + node1.query( + "INSERT INTO test_table_drop VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)" + ) assert node1.query("SELECT count(*) from test_table_drop") == "3\n" assert_eq_with_retry(node2, "SELECT count(*) from test_table_drop", "3") @@ -54,7 +66,9 @@ def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update): cluster.restart_instance_with_ip_change(node1, "2001:3984:3989::1:7777") # Put some data to source node1 - node1.query("INSERT INTO test_table_drop VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)") + node1.query( + "INSERT INTO test_table_drop VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)" + ) # Check that data is placed on node1 assert node1.query("SELECT count(*) from test_table_drop") == "6\n" @@ -73,11 +87,22 @@ def test_ip_change_drop_dns_cache(cluster_without_dns_cache_update): assert_eq_with_retry(node2, "SELECT count(*) from test_table_drop", "7") -node3 = cluster.add_instance('node3', main_configs=['configs/listen_host.xml'], - with_zookeeper=True, ipv6_address='2001:3984:3989::1:1113') -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/listen_host.xml', - 'configs/dns_update_short.xml'], - with_zookeeper=True, ipv6_address='2001:3984:3989::1:1114') +node3 = cluster.add_instance( + "node3", + main_configs=["configs/listen_host.xml"], + with_zookeeper=True, + ipv6_address="2001:3984:3989::1:1113", +) +node4 = cluster.add_instance( + "node4", + main_configs=[ + "configs/remote_servers.xml", + "configs/listen_host.xml", + "configs/dns_update_short.xml", + ], + with_zookeeper=True, + ipv6_address="2001:3984:3989::1:1114", +) @pytest.fixture(scope="module") @@ -85,7 +110,7 @@ def cluster_with_dns_cache_update(): try: cluster.start() - _fill_nodes([node3, node4], 'test_table_update') + _fill_nodes([node3, node4], "test_table_update") yield cluster @@ -101,7 +126,9 @@ def cluster_with_dns_cache_update(): # node4 has short dns_cache_update_period, so testing update of dns cache def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): # First we check, that normal replication works - node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") + node3.query( + "INSERT INTO test_table_update VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)" + ) assert node3.query("SELECT count(*) from test_table_update") == "3\n" assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "3") @@ -109,20 +136,26 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): cluster.restart_instance_with_ip_change(node3, "2001:3984:3989::1:8888") # Put some data to source node3 - node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)") + node3.query( + "INSERT INTO test_table_update VALUES ('2018-10-01', 5), ('2018-10-02', 6), ('2018-10-03', 7)" + ) # Check that data is placed on node3 assert node3.query("SELECT count(*) from test_table_update") == "6\n" curl_result = node4.exec_in_container(["bash", "-c", "curl -s 'node3:8123'"]) - assert curl_result == 'Ok.\n' + assert curl_result == "Ok.\n" cat_resolv = node4.exec_in_container(["bash", "-c", "cat /etc/resolv.conf"]) print(("RESOLV {}".format(cat_resolv))) - assert_eq_with_retry(node4, "SELECT * FROM remote('node3', 'system', 'one')", "0", sleep_time=0.5) + assert_eq_with_retry( + node4, "SELECT * FROM remote('node3', 'system', 'one')", "0", sleep_time=0.5 + ) # Because of DNS cache update, ip of node3 would be updated - assert_eq_with_retry(node4, "SELECT count(*) from test_table_update", "6", sleep_time=3) + assert_eq_with_retry( + node4, "SELECT count(*) from test_table_update", "6", sleep_time=3 + ) # Just to be sure check one more time node3.query("INSERT INTO test_table_update VALUES ('2018-10-01', 8)") @@ -131,39 +164,55 @@ def test_ip_change_update_dns_cache(cluster_with_dns_cache_update): def set_hosts(node, hosts): - new_content = '\\n'.join(['127.0.0.1 localhost', '::1 localhost'] + hosts) - node.exec_in_container(['bash', '-c', 'echo -e "{}" > /etc/hosts'.format(new_content)], privileged=True, - user='root') + new_content = "\\n".join(["127.0.0.1 localhost", "::1 localhost"] + hosts) + node.exec_in_container( + ["bash", "-c", 'echo -e "{}" > /etc/hosts'.format(new_content)], + privileged=True, + user="root", + ) def test_dns_cache_update(cluster_with_dns_cache_update): - set_hosts(node4, ['127.255.255.255 lost_host']) + set_hosts(node4, ["127.255.255.255 lost_host"]) with pytest.raises(QueryRuntimeException): node4.query("SELECT * FROM remote('lost_host', 'system', 'one')") node4.query( - "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") + "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')" + ) with pytest.raises(QueryRuntimeException): node4.query("SELECT * FROM distributed_lost_host") - set_hosts(node4, ['127.0.0.1 lost_host']) + set_hosts(node4, ["127.0.0.1 lost_host"]) # Wait a bit until dns cache will be updated - assert_eq_with_retry(node4, "SELECT * FROM remote('lost_host', 'system', 'one')", "0") + assert_eq_with_retry( + node4, "SELECT * FROM remote('lost_host', 'system', 'one')", "0" + ) assert_eq_with_retry(node4, "SELECT * FROM distributed_lost_host", "0") - assert TSV(node4.query( - "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV( - "lost_host\t127.0.0.1\n") + assert TSV( + node4.query( + "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'" + ) + ) == TSV("lost_host\t127.0.0.1\n") assert TSV(node4.query("SELECT hostName()")) == TSV("node4") # Check SYSTEM DROP DNS CACHE on node5 and background cache update on node6 -node5 = cluster.add_instance('node5', main_configs=['configs/listen_host.xml', 'configs/dns_update_long.xml'], - user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1115') -node6 = cluster.add_instance('node6', main_configs=['configs/listen_host.xml', 'configs/dns_update_short.xml'], - user_configs=['configs/users_with_hostname.xml'], ipv6_address='2001:3984:3989::1:1116') +node5 = cluster.add_instance( + "node5", + main_configs=["configs/listen_host.xml", "configs/dns_update_long.xml"], + user_configs=["configs/users_with_hostname.xml"], + ipv6_address="2001:3984:3989::1:1115", +) +node6 = cluster.add_instance( + "node6", + main_configs=["configs/listen_host.xml", "configs/dns_update_short.xml"], + user_configs=["configs/users_with_hostname.xml"], + ipv6_address="2001:3984:3989::1:1116", +) @pytest.mark.parametrize("node", [node5, node6]) @@ -171,16 +220,39 @@ def test_user_access_ip_change(cluster_with_dns_cache_update, node): node_name = node.name node_num = node.name[-1] # getaddrinfo(...) may hang for a log time without this options - node.exec_in_container(['bash', '-c', 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf'], - privileged=True, user='root') + node.exec_in_container( + [ + "bash", + "-c", + 'echo -e "options timeout:1\noptions attempts:2" >> /etc/resolv.conf', + ], + privileged=True, + user="root", + ) - assert node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n" - assert node4.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) == "0\n" + assert ( + node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) + == "0\n" + ) + assert ( + node4.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) + == "0\n" + ) - set_hosts(node, ['127.255.255.255 node3', '2001:3984:3989::1:88{}4 unknown_host'.format(node_num)]) + set_hosts( + node, + [ + "127.255.255.255 node3", + "2001:3984:3989::1:88{}4 unknown_host".format(node_num), + ], + ) - cluster.restart_instance_with_ip_change(node3, "2001:3984:3989::1:88{}3".format(node_num)) - cluster.restart_instance_with_ip_change(node4, "2001:3984:3989::1:88{}4".format(node_num)) + cluster.restart_instance_with_ip_change( + node3, "2001:3984:3989::1:88{}3".format(node_num) + ) + cluster.restart_instance_with_ip_change( + node4, "2001:3984:3989::1:88{}4".format(node_num) + ) with pytest.raises(QueryRuntimeException): node3.query("SELECT * FROM remote('{}', 'system', 'one')".format(node_name)) @@ -190,13 +262,26 @@ def test_user_access_ip_change(cluster_with_dns_cache_update, node): set_hosts(node, []) retry_count = 60 - if node_name == 'node5': + if node_name == "node5": # client is not allowed to connect, so execute it directly in container to send query from localhost - node.exec_in_container(['bash', '-c', 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], privileged=True, - user='root') + node.exec_in_container( + ["bash", "-c", 'clickhouse client -q "SYSTEM DROP DNS CACHE"'], + privileged=True, + user="root", + ) retry_count = 1 - assert_eq_with_retry(node3, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", - retry_count=retry_count, sleep_time=1) - assert_eq_with_retry(node4, "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), "0", - retry_count=retry_count, sleep_time=1) + assert_eq_with_retry( + node3, + "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), + "0", + retry_count=retry_count, + sleep_time=1, + ) + assert_eq_with_retry( + node4, + "SELECT * FROM remote('{}', 'system', 'one')".format(node_name), + "0", + retry_count=retry_count, + sleep_time=1, + ) diff --git a/tests/integration/test_http_and_readonly/test.py b/tests/integration/test_http_and_readonly/test.py index 9929e34c9d2..1ce3345bf80 100644 --- a/tests/integration/test_http_and_readonly/test.py +++ b/tests/integration/test_http_and_readonly/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -17,6 +17,12 @@ def setup_nodes(): def test_http_get_is_readonly(): assert "Cannot execute query in readonly mode" in instance.http_query_and_get_error( - "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)") - assert "Cannot modify 'readonly' setting in readonly mode" in instance.http_query_and_get_error( - "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", params={"readonly": 0}) + "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)" + ) + assert ( + "Cannot modify 'readonly' setting in readonly mode" + in instance.http_query_and_get_error( + "CREATE TABLE xxx (a Date) ENGINE = MergeTree(a, a, 256)", + params={"readonly": 0}, + ) + ) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index 01872a1d0c3..e73324ada8f 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -16,192 +16,485 @@ class SimpleCluster: def add_instance(self, name, config_dir): script_path = os.path.dirname(os.path.realpath(__file__)) - return self.cluster.add_instance(name, main_configs=[os.path.join(script_path, config_dir, 'config.xml')]) + return self.cluster.add_instance( + name, main_configs=[os.path.join(script_path, config_dir, "config.xml")] + ) def test_dynamic_query_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler")) as cluster: - test_query = urllib.parse.quote_plus('SELECT * FROM system.settings WHERE name = \'max_threads\'') + SimpleCluster( + ClickHouseCluster(__file__), "dynamic_handler", "test_dynamic_handler" + ) + ) as cluster: + test_query = urllib.parse.quote_plus( + "SELECT * FROM system.settings WHERE name = 'max_threads'" + ) - assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "?max_threads=1", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_dynamic_handler_get?max_threads=1", + method="POST", + headers={"XXX": "xxx"}, + ).status_code + ) - assert 404 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_dynamic_handler_get?max_threads=1", + method="GET", + headers={"XXX": "bad"}, + ).status_code + ) - assert 400 == cluster.instance.http_request('test_dynamic_handler_get?max_threads=1', method='GET', - headers={'XXX': 'xxx'}).status_code + assert ( + 400 + == cluster.instance.http_request( + "test_dynamic_handler_get?max_threads=1", + method="GET", + headers={"XXX": "xxx"}, + ).status_code + ) - assert 200 == cluster.instance.http_request( - 'test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=' + test_query, - method='GET', headers={'XXX': 'xxx'}).status_code + assert ( + 200 + == cluster.instance.http_request( + "test_dynamic_handler_get?max_threads=1&get_dynamic_handler_query=" + + test_query, + method="GET", + headers={"XXX": "xxx"}, + ).status_code + ) def test_predefined_query_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler")) as cluster: - assert 404 == cluster.instance.http_request('?max_threads=1', method='GET', headers={'XXX': 'xxx'}).status_code + SimpleCluster( + ClickHouseCluster(__file__), "predefined_handler", "test_predefined_handler" + ) + ) as cluster: + assert ( + 404 + == cluster.instance.http_request( + "?max_threads=1", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_predefined_handler_get?max_threads=1", + method="GET", + headers={"XXX": "bad"}, + ).status_code + ) - assert 404 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_predefined_handler_get?max_threads=1", + method="POST", + headers={"XXX": "xxx"}, + ).status_code + ) - assert 500 == cluster.instance.http_request('test_predefined_handler_get?max_threads=1', method='GET', - headers={'XXX': 'xxx'}).status_code + assert ( + 500 + == cluster.instance.http_request( + "test_predefined_handler_get?max_threads=1", + method="GET", + headers={"XXX": "xxx"}, + ).status_code + ) - assert b'max_threads\t1\n' == cluster.instance.http_request( - 'test_predefined_handler_get?max_threads=1&setting_name=max_threads', method='GET', - headers={'XXX': 'xxx'}).content + assert ( + b"max_threads\t1\n" + == cluster.instance.http_request( + "test_predefined_handler_get?max_threads=1&setting_name=max_threads", + method="GET", + headers={"XXX": "xxx"}, + ).content + ) - assert b'max_final_threads\t1\nmax_threads\t1\n' == cluster.instance.http_request( - 'query_param_with_url/max_threads?max_threads=1&max_final_threads=1', - headers={'XXX': 'max_final_threads'}).content + assert ( + b"max_final_threads\t1\nmax_threads\t1\n" + == cluster.instance.http_request( + "query_param_with_url/max_threads?max_threads=1&max_final_threads=1", + headers={"XXX": "max_final_threads"}, + ).content + ) def test_fixed_static_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: - assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + SimpleCluster( + ClickHouseCluster(__file__), "static_handler", "test_static_handler" + ) + ) as cluster: + assert ( + 404 + == cluster.instance.http_request( + "", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_fixed_static_handler", method="GET", headers={"XXX": "bad"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_fixed_static_handler', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_fixed_static_handler", method="POST", headers={"XXX": "xxx"} + ).status_code + ) - assert 402 == cluster.instance.http_request('test_get_fixed_static_handler', method='GET', - headers={'XXX': 'xxx'}).status_code - assert 'text/html; charset=UTF-8' == \ - cluster.instance.http_request('test_get_fixed_static_handler', method='GET', - headers={'XXX': 'xxx'}).headers['Content-Type'] - assert b'Test get static handler and fix content' == cluster.instance.http_request( - 'test_get_fixed_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert ( + 402 + == cluster.instance.http_request( + "test_get_fixed_static_handler", method="GET", headers={"XXX": "xxx"} + ).status_code + ) + assert ( + "text/html; charset=UTF-8" + == cluster.instance.http_request( + "test_get_fixed_static_handler", method="GET", headers={"XXX": "xxx"} + ).headers["Content-Type"] + ) + assert ( + b"Test get static handler and fix content" + == cluster.instance.http_request( + "test_get_fixed_static_handler", method="GET", headers={"XXX": "xxx"} + ).content + ) def test_config_static_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: - assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + SimpleCluster( + ClickHouseCluster(__file__), "static_handler", "test_static_handler" + ) + ) as cluster: + assert ( + 404 + == cluster.instance.http_request( + "", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_config_static_handler", method="GET", headers={"XXX": "bad"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_config_static_handler', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_config_static_handler", method="POST", headers={"XXX": "xxx"} + ).status_code + ) # check default status code - assert 200 == cluster.instance.http_request('test_get_config_static_handler', method='GET', - headers={'XXX': 'xxx'}).status_code - assert 'text/plain; charset=UTF-8' == \ - cluster.instance.http_request('test_get_config_static_handler', method='GET', - headers={'XXX': 'xxx'}).headers['Content-Type'] - assert b'Test get static handler and config content' == cluster.instance.http_request( - 'test_get_config_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert ( + 200 + == cluster.instance.http_request( + "test_get_config_static_handler", method="GET", headers={"XXX": "xxx"} + ).status_code + ) + assert ( + "text/plain; charset=UTF-8" + == cluster.instance.http_request( + "test_get_config_static_handler", method="GET", headers={"XXX": "xxx"} + ).headers["Content-Type"] + ) + assert ( + b"Test get static handler and config content" + == cluster.instance.http_request( + "test_get_config_static_handler", method="GET", headers={"XXX": "xxx"} + ).content + ) def test_absolute_path_static_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: + SimpleCluster( + ClickHouseCluster(__file__), "static_handler", "test_static_handler" + ) + ) as cluster: cluster.instance.exec_in_container( - ['bash', '-c', - 'echo "Absolute Path File" > /var/lib/clickhouse/user_files/absolute_path_file.html'], - privileged=True, user='root') + [ + "bash", + "-c", + 'echo "Absolute Path File" > /var/lib/clickhouse/user_files/absolute_path_file.html', + ], + privileged=True, + user="root", + ) - assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_absolute_path_static_handler", + method="GET", + headers={"XXX": "bad"}, + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_absolute_path_static_handler", + method="POST", + headers={"XXX": "xxx"}, + ).status_code + ) # check default status code - assert 200 == cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', - headers={'XXX': 'xxx'}).status_code - assert 'text/html; charset=UTF-8' == \ - cluster.instance.http_request('test_get_absolute_path_static_handler', method='GET', - headers={'XXX': 'xxx'}).headers['Content-Type'] - assert b'Absolute Path File\n' == cluster.instance.http_request( - 'test_get_absolute_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert ( + 200 + == cluster.instance.http_request( + "test_get_absolute_path_static_handler", + method="GET", + headers={"XXX": "xxx"}, + ).status_code + ) + assert ( + "text/html; charset=UTF-8" + == cluster.instance.http_request( + "test_get_absolute_path_static_handler", + method="GET", + headers={"XXX": "xxx"}, + ).headers["Content-Type"] + ) + assert ( + b"Absolute Path File\n" + == cluster.instance.http_request( + "test_get_absolute_path_static_handler", + method="GET", + headers={"XXX": "xxx"}, + ).content + ) def test_relative_path_static_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "static_handler", "test_static_handler")) as cluster: + SimpleCluster( + ClickHouseCluster(__file__), "static_handler", "test_static_handler" + ) + ) as cluster: cluster.instance.exec_in_container( - ['bash', '-c', - 'echo "Relative Path File" > /var/lib/clickhouse/user_files/relative_path_file.html'], - privileged=True, user='root') + [ + "bash", + "-c", + 'echo "Relative Path File" > /var/lib/clickhouse/user_files/relative_path_file.html', + ], + privileged=True, + user="root", + ) - assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_relative_path_static_handler", + method="GET", + headers={"XXX": "bad"}, + ).status_code + ) - assert 404 == cluster.instance.http_request('test_get_relative_path_static_handler', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_get_relative_path_static_handler", + method="POST", + headers={"XXX": "xxx"}, + ).status_code + ) # check default status code - assert 200 == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', - headers={'XXX': 'xxx'}).status_code - assert 'text/html; charset=UTF-8' == \ - cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', - headers={'XXX': 'xxx'}).headers['Content-Type'] - assert b'Relative Path File\n' == cluster.instance.http_request( - 'test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content + assert ( + 200 + == cluster.instance.http_request( + "test_get_relative_path_static_handler", + method="GET", + headers={"XXX": "xxx"}, + ).status_code + ) + assert ( + "text/html; charset=UTF-8" + == cluster.instance.http_request( + "test_get_relative_path_static_handler", + method="GET", + headers={"XXX": "xxx"}, + ).headers["Content-Type"] + ) + assert ( + b"Relative Path File\n" + == cluster.instance.http_request( + "test_get_relative_path_static_handler", + method="GET", + headers={"XXX": "xxx"}, + ).content + ) def test_defaults_http_handlers(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster: - assert 200 == cluster.instance.http_request('', method='GET').status_code - assert b'Default server response' == cluster.instance.http_request('', method='GET').content + SimpleCluster( + ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers" + ) + ) as cluster: + assert 200 == cluster.instance.http_request("", method="GET").status_code + assert ( + b"Default server response" + == cluster.instance.http_request("", method="GET").content + ) - assert 200 == cluster.instance.http_request('ping', method='GET').status_code - assert b'Ok.\n' == cluster.instance.http_request('ping', method='GET').content + assert 200 == cluster.instance.http_request("ping", method="GET").status_code + assert b"Ok.\n" == cluster.instance.http_request("ping", method="GET").content - assert 200 == cluster.instance.http_request('replicas_status', method='get').status_code - assert b'Ok.\n' == cluster.instance.http_request('replicas_status', method='get').content + assert ( + 200 + == cluster.instance.http_request( + "replicas_status", method="get" + ).status_code + ) + assert ( + b"Ok.\n" + == cluster.instance.http_request("replicas_status", method="get").content + ) - assert 200 == cluster.instance.http_request('replicas_status?verbose=1', method='get').status_code - assert b'' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content + assert ( + 200 + == cluster.instance.http_request( + "replicas_status?verbose=1", method="get" + ).status_code + ) + assert ( + b"" + == cluster.instance.http_request( + "replicas_status?verbose=1", method="get" + ).content + ) - assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code - assert b'1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content + assert ( + 200 + == cluster.instance.http_request( + "?query=SELECT+1", method="GET" + ).status_code + ) + assert ( + b"1\n" + == cluster.instance.http_request("?query=SELECT+1", method="GET").content + ) def test_prometheus_handler(): with contextlib.closing( - SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster: - assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + SimpleCluster( + ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler" + ) + ) as cluster: + assert ( + 404 + == cluster.instance.http_request( + "", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_prometheus", method="GET", headers={"XXX": "bad"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_prometheus', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_prometheus", method="POST", headers={"XXX": "xxx"} + ).status_code + ) - assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code - assert b'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', - headers={'XXX': 'xxx'}).content + assert ( + 200 + == cluster.instance.http_request( + "test_prometheus", method="GET", headers={"XXX": "xxx"} + ).status_code + ) + assert ( + b"ClickHouseProfileEvents_Query" + in cluster.instance.http_request( + "test_prometheus", method="GET", headers={"XXX": "xxx"} + ).content + ) def test_replicas_status_handler(): - with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", - "test_replicas_status_handler")) as cluster: - assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code + with contextlib.closing( + SimpleCluster( + ClickHouseCluster(__file__), + "replicas_status_handler", + "test_replicas_status_handler", + ) + ) as cluster: + assert ( + 404 + == cluster.instance.http_request( + "", method="GET", headers={"XXX": "xxx"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', - headers={'XXX': 'bad'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_replicas_status", method="GET", headers={"XXX": "bad"} + ).status_code + ) - assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', - headers={'XXX': 'xxx'}).status_code + assert ( + 404 + == cluster.instance.http_request( + "test_replicas_status", method="POST", headers={"XXX": "xxx"} + ).status_code + ) - assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', - headers={'XXX': 'xxx'}).status_code - assert b'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', - headers={'XXX': 'xxx'}).content + assert ( + 200 + == cluster.instance.http_request( + "test_replicas_status", method="GET", headers={"XXX": "xxx"} + ).status_code + ) + assert ( + b"Ok.\n" + == cluster.instance.http_request( + "test_replicas_status", method="GET", headers={"XXX": "xxx"} + ).content + ) diff --git a/tests/integration/test_https_replication/configs/config.xml b/tests/integration/test_https_replication/configs/config.xml index 4b8a61bc20b..4b3088d21e1 100644 --- a/tests/integration/test_https_replication/configs/config.xml +++ b/tests/integration/test_https_replication/configs/config.xml @@ -122,20 +122,6 @@ default - - - diff --git a/tests/integration/test_https_replication/test.py b/tests/integration/test_https_replication/test.py index 1008ce07ad3..4cf9f19b870 100644 --- a/tests/integration/test_https_replication/test.py +++ b/tests/integration/test_https_replication/test.py @@ -15,21 +15,40 @@ Both ssl_conf.xml and no_ssl_conf.xml have the same port def _fill_nodes(nodes, shard): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", - "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) -node2 = cluster.add_instance('node2', - main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", - "configs/server.key", "configs/dhparam.pem"], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/remote_servers.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + "configs/dhparam.pem", + ], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/remote_servers.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + "configs/dhparam.pem", + ], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -48,13 +67,13 @@ def both_https_cluster(): def test_both_https(both_https_cluster): node1.query("insert into test_table values ('2017-06-16', 111, 0)") - assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", "111") node2.query("insert into test_table values ('2017-06-17', 222, 1)") - assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", '111\n222') - assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", '111\n222') + assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", "111\n222") + assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", "111\n222") def test_replication_after_partition(both_https_cluster): @@ -80,12 +99,20 @@ def test_replication_after_partition(both_https_cluster): cres.wait() ires.wait() - assert_eq_with_retry(node1, "SELECT count() FROM test_table", '100') - assert_eq_with_retry(node2, "SELECT count() FROM test_table", '100') + assert_eq_with_retry(node1, "SELECT count() FROM test_table", "100") + assert_eq_with_retry(node2, "SELECT count() FROM test_table", "100") -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/remote_servers.xml", "configs/no_ssl_conf.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + main_configs=["configs/remote_servers.xml", "configs/no_ssl_conf.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -104,22 +131,31 @@ def both_http_cluster(): def test_both_http(both_http_cluster): node3.query("insert into test_table values ('2017-06-16', 111, 0)") - assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", "111") node4.query("insert into test_table values ('2017-06-17', 222, 1)") - assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111\n222') - assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111\n222') + assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", "111\n222") + assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", "111\n222") -node5 = cluster.add_instance('node5', - main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml', "configs/server.crt", - "configs/server.key", "configs/dhparam.pem"], - with_zookeeper=True) -node6 = cluster.add_instance('node6', - main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], - with_zookeeper=True) +node5 = cluster.add_instance( + "node5", + main_configs=[ + "configs/remote_servers.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + "configs/dhparam.pem", + ], + with_zookeeper=True, +) +node6 = cluster.add_instance( + "node6", + main_configs=["configs/remote_servers.xml", "configs/no_ssl_conf.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -138,10 +174,10 @@ def mixed_protocol_cluster(): def test_mixed_protocol(mixed_protocol_cluster): node5.query("insert into test_table values ('2017-06-16', 111, 0)") - assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '') + assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", "") node6.query("insert into test_table values ('2017-06-17', 222, 1)") - assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '222') + assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", "222") diff --git a/tests/integration/test_inherit_multiple_profiles/test.py b/tests/integration/test_inherit_multiple_profiles/test.py index 658ccc3f51b..46f2868fe36 100644 --- a/tests/integration/test_inherit_multiple_profiles/test.py +++ b/tests/integration/test_inherit_multiple_profiles/test.py @@ -5,8 +5,9 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - user_configs=['configs/combined_profile.xml']) +instance = cluster.add_instance( + "instance", user_configs=["configs/combined_profile.xml"] +) q = instance.query @@ -22,52 +23,63 @@ def started_cluster(): def test_combined_profile(started_cluster): - settings = q(''' + settings = q( + """ SELECT name, value FROM system.settings WHERE name IN ('max_insert_block_size', 'max_network_bytes', 'max_query_size', 'max_parallel_replicas', 'readonly') AND changed ORDER BY name -''', user='test_combined_profile') +""", + user="test_combined_profile", + ) - expected1 = '''\ + expected1 = """\ max_insert_block_size 654321 max_network_bytes 1234567890 max_parallel_replicas 2 max_query_size 400000000 -readonly 2''' +readonly 2""" assert TSV(settings) == TSV(expected1) with pytest.raises(QueryRuntimeException) as exc: - q(''' + q( + """ SET max_insert_block_size = 1000; - ''', user='test_combined_profile') + """, + user="test_combined_profile", + ) - assert ("max_insert_block_size shouldn't be less than 654320." in - str(exc.value)) + assert "max_insert_block_size shouldn't be less than 654320." in str(exc.value) with pytest.raises(QueryRuntimeException) as exc: - q(''' + q( + """ SET max_network_bytes = 2000000000; - ''', user='test_combined_profile') + """, + user="test_combined_profile", + ) - assert ("max_network_bytes shouldn't be greater than 1234567891." in - str(exc.value)) + assert "max_network_bytes shouldn't be greater than 1234567891." in str(exc.value) with pytest.raises(QueryRuntimeException) as exc: - q(''' + q( + """ SET max_parallel_replicas = 1000; - ''', user='test_combined_profile') + """, + user="test_combined_profile", + ) - assert ('max_parallel_replicas should not be changed.' in - str(exc.value)) + assert "max_parallel_replicas should not be changed." in str(exc.value) with pytest.raises(QueryRuntimeException) as exc: - q(''' + q( + """ SET max_memory_usage = 1000; - ''', user='test_combined_profile') + """, + user="test_combined_profile", + ) - assert ("max_memory_usage shouldn't be less than 300000000." in - str(exc.value)) + assert "max_memory_usage shouldn't be less than 300000000." in str(exc.value) diff --git a/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py b/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py index 1c686c7982e..2e4824a5a4f 100644 --- a/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py +++ b/tests/integration/test_input_format_parallel_parsing_memory_tracking/test.py @@ -8,13 +8,16 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=[ - 'configs/conf.xml', - 'configs/asynchronous_metrics_update_period_s.xml', -]) +instance = cluster.add_instance( + "instance", + main_configs=[ + "configs/conf.xml", + "configs/asynchronous_metrics_update_period_s.xml", + ], +) -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -26,11 +29,27 @@ def start_cluster(): # max_memory_usage_for_user cannot be used, since the memory for user accounted # correctly, only total is not (it is set via conf.xml) def test_memory_tracking_total(): - instance.query('CREATE TABLE null (row String) ENGINE=Null') - instance.exec_in_container(['bash', '-c', - 'clickhouse local -q "SELECT arrayStringConcat(arrayMap(x->toString(cityHash64(x)), range(1000)), \' \') from numbers(10000)" > data.json']) + instance.query("CREATE TABLE null (row String) ENGINE=Null") + instance.exec_in_container( + [ + "bash", + "-c", + "clickhouse local -q \"SELECT arrayStringConcat(arrayMap(x->toString(cityHash64(x)), range(1000)), ' ') from numbers(10000)\" > data.json", + ] + ) for it in range(0, 20): # the problem can be triggered only via HTTP, # since clickhouse-client parses the data by itself. - assert instance.exec_in_container(['curl', '--silent', '--show-error', '--data-binary', '@data.json', - 'http://127.1:8123/?query=INSERT%20INTO%20null%20FORMAT%20TSV']) == '', f'Failed on {it} iteration' + assert ( + instance.exec_in_container( + [ + "curl", + "--silent", + "--show-error", + "--data-binary", + "@data.json", + "http://127.1:8123/?query=INSERT%20INTO%20null%20FORMAT%20TSV", + ] + ) + == "" + ), f"Failed on {it} iteration" diff --git a/tests/integration/test_insert_distributed_async_extra_dirs/test.py b/tests/integration/test_insert_distributed_async_extra_dirs/test.py index 8365fce298d..b4421ba9590 100644 --- a/tests/integration/test_insert_distributed_async_extra_dirs/test.py +++ b/tests/integration/test_insert_distributed_async_extra_dirs/test.py @@ -8,9 +8,12 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/remote_servers.xml'], stay_alive=True) +node = cluster.add_instance( + "node", main_configs=["configs/remote_servers.xml"], stay_alive=True +) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -18,9 +21,11 @@ def start_cluster(): finally: cluster.shutdown() + def test_insert_distributed_async_send_success(): - node.query('CREATE TABLE data (key Int, value String) Engine=Null()') - node.query(""" + node.query("CREATE TABLE data (key Int, value String) Engine=Null()") + node.query( + """ CREATE TABLE dist AS data Engine=Distributed( test_cluster, @@ -28,16 +33,53 @@ def test_insert_distributed_async_send_success(): data, key ) - """) + """ + ) - node.exec_in_container(['bash', '-c', 'mkdir /var/lib/clickhouse/data/default/dist/shard10000_replica10000']) - node.exec_in_container(['bash', '-c', 'touch /var/lib/clickhouse/data/default/dist/shard10000_replica10000/1.bin']) + node.exec_in_container( + [ + "bash", + "-c", + "mkdir /var/lib/clickhouse/data/default/dist/shard10000_replica10000", + ] + ) + node.exec_in_container( + [ + "bash", + "-c", + "touch /var/lib/clickhouse/data/default/dist/shard10000_replica10000/1.bin", + ] + ) - node.exec_in_container(['bash', '-c', 'mkdir /var/lib/clickhouse/data/default/dist/shard1_replica10000']) - node.exec_in_container(['bash', '-c', 'touch /var/lib/clickhouse/data/default/dist/shard1_replica10000/1.bin']) + node.exec_in_container( + [ + "bash", + "-c", + "mkdir /var/lib/clickhouse/data/default/dist/shard1_replica10000", + ] + ) + node.exec_in_container( + [ + "bash", + "-c", + "touch /var/lib/clickhouse/data/default/dist/shard1_replica10000/1.bin", + ] + ) - node.exec_in_container(['bash', '-c', 'mkdir /var/lib/clickhouse/data/default/dist/shard10000_replica1']) - node.exec_in_container(['bash', '-c', 'touch /var/lib/clickhouse/data/default/dist/shard10000_replica1/1.bin']) + node.exec_in_container( + [ + "bash", + "-c", + "mkdir /var/lib/clickhouse/data/default/dist/shard10000_replica1", + ] + ) + node.exec_in_container( + [ + "bash", + "-c", + "touch /var/lib/clickhouse/data/default/dist/shard10000_replica1/1.bin", + ] + ) # will check that clickhouse-server is alive node.restart_clickhouse() diff --git a/tests/integration/test_insert_distributed_async_send/test.py b/tests/integration/test_insert_distributed_async_send/test.py index a9bf9801f4c..80b9081d3f2 100644 --- a/tests/integration/test_insert_distributed_async_send/test.py +++ b/tests/integration/test_insert_distributed_async_send/test.py @@ -13,34 +13,57 @@ from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) # n1 -- distributed_directory_monitor_batch_inserts=1 -n1 = cluster.add_instance('n1', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users.d/batch.xml']) +n1 = cluster.add_instance( + "n1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.d/batch.xml"], +) # n2 -- distributed_directory_monitor_batch_inserts=0 -n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users.d/no_batch.xml']) +n2 = cluster.add_instance( + "n2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.d/no_batch.xml"], +) # n3 -- distributed_directory_monitor_batch_inserts=1/distributed_directory_monitor_split_batch_on_failure=1 -n3 = cluster.add_instance('n3', main_configs=['configs/remote_servers_split.xml'], user_configs=[ - 'configs/users.d/batch.xml', - 'configs/users.d/split.xml', -]) +n3 = cluster.add_instance( + "n3", + main_configs=["configs/remote_servers_split.xml"], + user_configs=[ + "configs/users.d/batch.xml", + "configs/users.d/split.xml", + ], +) # n4 -- distributed_directory_monitor_batch_inserts=0/distributed_directory_monitor_split_batch_on_failure=1 -n4 = cluster.add_instance('n4', main_configs=['configs/remote_servers_split.xml'], user_configs=[ - 'configs/users.d/no_batch.xml', - 'configs/users.d/split.xml', -]) +n4 = cluster.add_instance( + "n4", + main_configs=["configs/remote_servers_split.xml"], + user_configs=[ + "configs/users.d/no_batch.xml", + "configs/users.d/split.xml", + ], +) -batch_params = pytest.mark.parametrize('batch', [ - (1), - (0), -]) +batch_params = pytest.mark.parametrize( + "batch", + [ + (1), + (0), + ], +) -batch_and_split_params = pytest.mark.parametrize('batch,split', [ - (1, 0), - (0, 0), - (1, 1), - (0, 1), -]) +batch_and_split_params = pytest.mark.parametrize( + "batch,split", + [ + (1, 0), + (0, 0), + (1, 1), + (0, 1), + ], +) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -51,8 +74,11 @@ def start_cluster(): def create_tables(remote_cluster_name): for _, instance in list(cluster.instances.items()): - instance.query('CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key') - instance.query(f""" + instance.query( + "CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key" + ) + instance.query( + f""" CREATE TABLE dist AS data Engine=Distributed( {remote_cluster_name}, @@ -60,26 +86,33 @@ def create_tables(remote_cluster_name): data, key ) - """) + """ + ) # only via SYSTEM FLUSH DISTRIBUTED - instance.query('SYSTEM STOP DISTRIBUTED SENDS dist') + instance.query("SYSTEM STOP DISTRIBUTED SENDS dist") + def drop_tables(): for _, instance in list(cluster.instances.items()): - instance.query('DROP TABLE IF EXISTS data') - instance.query('DROP TABLE IF EXISTS dist') + instance.query("DROP TABLE IF EXISTS data") + instance.query("DROP TABLE IF EXISTS dist") + # return amount of bytes of the 2.bin for n2 shard def insert_data(node): - node.query('INSERT INTO dist SELECT number, randomPrintableASCII(100) FROM numbers(10000)', settings={ - # do not do direct INSERT, always via SYSTEM FLUSH DISTRIBUTED - 'prefer_localhost_replica': 0, - }) + node.query( + "INSERT INTO dist SELECT number, randomPrintableASCII(100) FROM numbers(10000)", + settings={ + # do not do direct INSERT, always via SYSTEM FLUSH DISTRIBUTED + "prefer_localhost_replica": 0, + }, + ) path = get_path_to_dist_batch() - size = int(node.exec_in_container(['bash', '-c', f'wc -c < {path}'])) - assert size > 1<<16 + size = int(node.exec_in_container(["bash", "-c", f"wc -c < {path}"])) + assert size > 1 << 16 return size + def get_node(batch, split=None): if split: if batch: @@ -89,56 +122,65 @@ def get_node(batch, split=None): return n1 return n2 + def bootstrap(batch, split=None): drop_tables() - create_tables('insert_distributed_async_send_cluster_two_replicas') + create_tables("insert_distributed_async_send_cluster_two_replicas") return insert_data(get_node(batch, split)) -def get_path_to_dist_batch(file='2.bin'): + +def get_path_to_dist_batch(file="2.bin"): # There are: # - /var/lib/clickhouse/data/default/dist/shard1_replica1/1.bin # - /var/lib/clickhouse/data/default/dist/shard1_replica2/2.bin # # @return the file for the n2 shard - return f'/var/lib/clickhouse/data/default/dist/shard1_replica2/{file}' + return f"/var/lib/clickhouse/data/default/dist/shard1_replica2/{file}" + def check_dist_after_corruption(truncate, batch, split=None): node = get_node(batch, split) if batch: # In batch mode errors are ignored - node.query('SYSTEM FLUSH DISTRIBUTED dist') + node.query("SYSTEM FLUSH DISTRIBUTED dist") else: if truncate: - with pytest.raises(QueryRuntimeException, match="Cannot read all data. Bytes read:"): - node.query('SYSTEM FLUSH DISTRIBUTED dist') + with pytest.raises( + QueryRuntimeException, match="Cannot read all data. Bytes read:" + ): + node.query("SYSTEM FLUSH DISTRIBUTED dist") else: - with pytest.raises(QueryRuntimeException, match="Checksum doesn't match: corrupted data. Reference:"): - node.query('SYSTEM FLUSH DISTRIBUTED dist') + with pytest.raises( + QueryRuntimeException, + match="Checksum doesn't match: corrupted data. Reference:", + ): + node.query("SYSTEM FLUSH DISTRIBUTED dist") # send pending files # (since we have two nodes and corrupt file for only one of them) - node.query('SYSTEM FLUSH DISTRIBUTED dist') + node.query("SYSTEM FLUSH DISTRIBUTED dist") # but there is broken file - broken = get_path_to_dist_batch('broken') - node.exec_in_container(['bash', '-c', f'ls {broken}/2.bin']) + broken = get_path_to_dist_batch("broken") + node.exec_in_container(["bash", "-c", f"ls {broken}/2.bin"]) if split: - assert int(n3.query('SELECT count() FROM data')) == 10000 - assert int(n4.query('SELECT count() FROM data')) == 0 + assert int(n3.query("SELECT count() FROM data")) == 10000 + assert int(n4.query("SELECT count() FROM data")) == 0 else: - assert int(n1.query('SELECT count() FROM data')) == 10000 - assert int(n2.query('SELECT count() FROM data')) == 0 + assert int(n1.query("SELECT count() FROM data")) == 10000 + assert int(n2.query("SELECT count() FROM data")) == 0 @batch_params def test_insert_distributed_async_send_success(batch): bootstrap(batch) node = get_node(batch) - node.query('SYSTEM FLUSH DISTRIBUTED dist') - assert int(n1.query('SELECT count() FROM data')) == 10000 - assert int(n2.query('SELECT count() FROM data')) == 10000 + node.query("SYSTEM FLUSH DISTRIBUTED dist") + assert int(n1.query("SELECT count() FROM data")) == 10000 + assert int(n2.query("SELECT count() FROM data")) == 10000 + @batch_and_split_params def test_insert_distributed_async_send_truncated_1(batch, split): @@ -148,10 +190,13 @@ def test_insert_distributed_async_send_truncated_1(batch, split): new_size = size - 10 # we cannot use truncate, due to hardlinks - node.exec_in_container(['bash', '-c', f'mv {path} /tmp/bin && head -c {new_size} /tmp/bin > {path}']) + node.exec_in_container( + ["bash", "-c", f"mv {path} /tmp/bin && head -c {new_size} /tmp/bin > {path}"] + ) check_dist_after_corruption(True, batch, split) + @batch_params def test_insert_distributed_async_send_truncated_2(batch): bootstrap(batch) @@ -159,10 +204,13 @@ def test_insert_distributed_async_send_truncated_2(batch): node = get_node(batch) # we cannot use truncate, due to hardlinks - node.exec_in_container(['bash', '-c', f'mv {path} /tmp/bin && head -c 10000 /tmp/bin > {path}']) + node.exec_in_container( + ["bash", "-c", f"mv {path} /tmp/bin && head -c 10000 /tmp/bin > {path}"] + ) check_dist_after_corruption(True, batch) + # The difference from the test_insert_distributed_async_send_corrupted_small # is that small corruption will be seen only on local node @batch_params @@ -174,10 +222,17 @@ def test_insert_distributed_async_send_corrupted_big(batch): from_original_size = size - 8192 zeros_size = 8192 - node.exec_in_container(['bash', '-c', f'mv {path} /tmp/bin && head -c {from_original_size} /tmp/bin > {path} && head -c {zeros_size} /dev/zero >> {path}']) + node.exec_in_container( + [ + "bash", + "-c", + f"mv {path} /tmp/bin && head -c {from_original_size} /tmp/bin > {path} && head -c {zeros_size} /dev/zero >> {path}", + ] + ) check_dist_after_corruption(False, batch) + @batch_params def test_insert_distributed_async_send_corrupted_small(batch): size = bootstrap(batch) @@ -186,10 +241,17 @@ def test_insert_distributed_async_send_corrupted_small(batch): from_original_size = size - 60 zeros_size = 60 - node.exec_in_container(['bash', '-c', f'mv {path} /tmp/bin && head -c {from_original_size} /tmp/bin > {path} && head -c {zeros_size} /dev/zero >> {path}']) + node.exec_in_container( + [ + "bash", + "-c", + f"mv {path} /tmp/bin && head -c {from_original_size} /tmp/bin > {path} && head -c {zeros_size} /dev/zero >> {path}", + ] + ) check_dist_after_corruption(False, batch) + @batch_params def test_insert_distributed_async_send_different_header(batch): """ @@ -198,47 +260,66 @@ def test_insert_distributed_async_send_different_header(batch): """ drop_tables() - create_tables('insert_distributed_async_send_cluster_two_shards') + create_tables("insert_distributed_async_send_cluster_two_shards") node = get_node(batch) - node.query("INSERT INTO dist VALUES (0, 'f')", settings={ - 'prefer_localhost_replica': 0, - }) - node.query('ALTER TABLE dist MODIFY COLUMN value UInt64') - node.query("INSERT INTO dist VALUES (2, 1)", settings={ - 'prefer_localhost_replica': 0, - }) + node.query( + "INSERT INTO dist VALUES (0, 'f')", + settings={ + "prefer_localhost_replica": 0, + }, + ) + node.query("ALTER TABLE dist MODIFY COLUMN value UInt64") + node.query( + "INSERT INTO dist VALUES (2, 1)", + settings={ + "prefer_localhost_replica": 0, + }, + ) - n1.query('ALTER TABLE data MODIFY COLUMN value UInt64', settings={ - 'mutations_sync': 1, - }) + n1.query( + "ALTER TABLE data MODIFY COLUMN value UInt64", + settings={ + "mutations_sync": 1, + }, + ) if batch: # but only one batch will be sent, and first is with UInt64 column, so # one rows inserted, and for string ('f') exception will be throw. - with pytest.raises(QueryRuntimeException, match=r"DB::Exception: Cannot parse string 'f' as UInt64: syntax error at begin of string"): - node.query('SYSTEM FLUSH DISTRIBUTED dist') - assert int(n1.query('SELECT count() FROM data')) == 1 + with pytest.raises( + QueryRuntimeException, + match=r"DB::Exception: Cannot parse string 'f' as UInt64: syntax error at begin of string", + ): + node.query("SYSTEM FLUSH DISTRIBUTED dist") + assert int(n1.query("SELECT count() FROM data")) == 1 # but once underlying column String, implicit conversion will do the # thing, and insert left batch. - n1.query(""" + n1.query( + """ DROP TABLE data SYNC; CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key; - """) - node.query('SYSTEM FLUSH DISTRIBUTED dist') - assert int(n1.query('SELECT count() FROM data')) == 1 + """ + ) + node.query("SYSTEM FLUSH DISTRIBUTED dist") + assert int(n1.query("SELECT count() FROM data")) == 1 else: # first send with String ('f'), so zero rows will be inserted - with pytest.raises(QueryRuntimeException, match=r"DB::Exception: Cannot parse string 'f' as UInt64: syntax error at begin of string"): - node.query('SYSTEM FLUSH DISTRIBUTED dist') - assert int(n1.query('SELECT count() FROM data')) == 0 + with pytest.raises( + QueryRuntimeException, + match=r"DB::Exception: Cannot parse string 'f' as UInt64: syntax error at begin of string", + ): + node.query("SYSTEM FLUSH DISTRIBUTED dist") + assert int(n1.query("SELECT count() FROM data")) == 0 # but once underlying column String, implicit conversion will do the # thing, and insert 2 rows (mixed UInt64 and String). - n1.query(""" + n1.query( + """ DROP TABLE data SYNC; CREATE TABLE data (key Int, value String) Engine=MergeTree() ORDER BY key; - """) - node.query('SYSTEM FLUSH DISTRIBUTED dist') - assert int(n1.query('SELECT count() FROM data')) == 2 + """ + ) + node.query("SYSTEM FLUSH DISTRIBUTED dist") + assert int(n1.query("SELECT count() FROM data")) == 2 - assert int(n2.query('SELECT count() FROM data')) == 0 + assert int(n2.query("SELECT count() FROM data")) == 0 diff --git a/tests/integration/test_insert_distributed_load_balancing/test.py b/tests/integration/test_insert_distributed_load_balancing/test.py index 29cc953280f..5a17a6d5770 100644 --- a/tests/integration/test_insert_distributed_load_balancing/test.py +++ b/tests/integration/test_insert_distributed_load_balancing/test.py @@ -8,16 +8,19 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -n1 = cluster.add_instance('n1', main_configs=['configs/remote_servers.xml']) -n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml']) +n1 = cluster.add_instance("n1", main_configs=["configs/remote_servers.xml"]) +n2 = cluster.add_instance("n2", main_configs=["configs/remote_servers.xml"]) -params = pytest.mark.parametrize('cluster,q', [ - ('internal_replication', 0), - ('no_internal_replication', 1), -]) +params = pytest.mark.parametrize( + "cluster,q", + [ + ("internal_replication", 0), + ("no_internal_replication", 1), + ], +) -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -27,13 +30,14 @@ def start_cluster(): def create_tables(cluster): - n1.query('DROP TABLE IF EXISTS data') - n2.query('DROP TABLE IF EXISTS data') - n1.query('DROP TABLE IF EXISTS dist') + n1.query("DROP TABLE IF EXISTS data") + n2.query("DROP TABLE IF EXISTS data") + n1.query("DROP TABLE IF EXISTS dist") - n1.query('CREATE TABLE data (key Int) Engine=Memory()') - n2.query('CREATE TABLE data (key Int) Engine=Memory()') - n1.query(""" + n1.query("CREATE TABLE data (key Int) Engine=Memory()") + n2.query("CREATE TABLE data (key Int) Engine=Memory()") + n1.query( + """ CREATE TABLE dist AS data Engine=Distributed( {cluster}, @@ -41,45 +45,53 @@ def create_tables(cluster): data, rand() ) - """.format(cluster=cluster)) + """.format( + cluster=cluster + ) + ) def insert_data(cluster, **settings): create_tables(cluster) - n1.query('INSERT INTO dist SELECT * FROM numbers(10)', settings=settings) - n1.query('SYSTEM FLUSH DISTRIBUTED dist') + n1.query("INSERT INTO dist SELECT * FROM numbers(10)", settings=settings) + n1.query("SYSTEM FLUSH DISTRIBUTED dist") @params def test_prefer_localhost_replica_1(cluster, q): insert_data(cluster) - assert int(n1.query('SELECT count() FROM data')) == 10 - assert int(n2.query('SELECT count() FROM data')) == 10 * q + assert int(n1.query("SELECT count() FROM data")) == 10 + assert int(n2.query("SELECT count() FROM data")) == 10 * q @params def test_prefer_localhost_replica_1_load_balancing_in_order(cluster, q): - insert_data(cluster, load_balancing='in_order') - assert int(n1.query('SELECT count() FROM data')) == 10 - assert int(n2.query('SELECT count() FROM data')) == 10 * q + insert_data(cluster, load_balancing="in_order") + assert int(n1.query("SELECT count() FROM data")) == 10 + assert int(n2.query("SELECT count() FROM data")) == 10 * q @params def test_prefer_localhost_replica_0_load_balancing_nearest_hostname(cluster, q): - insert_data(cluster, load_balancing='nearest_hostname', prefer_localhost_replica=0) - assert int(n1.query('SELECT count() FROM data')) == 10 - assert int(n2.query('SELECT count() FROM data')) == 10 * q + insert_data(cluster, load_balancing="nearest_hostname", prefer_localhost_replica=0) + assert int(n1.query("SELECT count() FROM data")) == 10 + assert int(n2.query("SELECT count() FROM data")) == 10 * q @params def test_prefer_localhost_replica_0_load_balancing_in_order(cluster, q): - insert_data(cluster, load_balancing='in_order', prefer_localhost_replica=0) - assert int(n1.query('SELECT count() FROM data')) == 10 * q - assert int(n2.query('SELECT count() FROM data')) == 10 + insert_data(cluster, load_balancing="in_order", prefer_localhost_replica=0) + assert int(n1.query("SELECT count() FROM data")) == 10 * q + assert int(n2.query("SELECT count() FROM data")) == 10 @params def test_prefer_localhost_replica_0_load_balancing_in_order_sync(cluster, q): - insert_data(cluster, load_balancing='in_order', prefer_localhost_replica=0, insert_distributed_sync=1) - assert int(n1.query('SELECT count() FROM data')) == 10 * q - assert int(n2.query('SELECT count() FROM data')) == 10 + insert_data( + cluster, + load_balancing="in_order", + prefer_localhost_replica=0, + insert_distributed_sync=1, + ) + assert int(n1.query("SELECT count() FROM data")) == 10 * q + assert int(n2.query("SELECT count() FROM data")) == 10 diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index e2af59903bd..b8d94d2a043 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -8,21 +8,35 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_configs=['configs/remote_servers.xml']) +instance_test_reconnect = cluster.add_instance( + "instance_test_reconnect", main_configs=["configs/remote_servers.xml"] +) instance_test_inserts_batching = cluster.add_instance( - 'instance_test_inserts_batching', - main_configs=['configs/remote_servers.xml'], user_configs=['configs/enable_distributed_inserts_batching.xml']) -remote = cluster.add_instance('remote', main_configs=['configs/forbid_background_merges.xml']) + "instance_test_inserts_batching", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/enable_distributed_inserts_batching.xml"], +) +remote = cluster.add_instance( + "remote", main_configs=["configs/forbid_background_merges.xml"] +) instance_test_inserts_local_cluster = cluster.add_instance( - 'instance_test_inserts_local_cluster', - main_configs=['configs/remote_servers.xml']) + "instance_test_inserts_local_cluster", main_configs=["configs/remote_servers.xml"] +) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) -shard1 = cluster.add_instance('shard1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -shard2 = cluster.add_instance('shard2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +shard1 = cluster.add_instance( + "shard1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +shard2 = cluster.add_instance( + "shard2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -31,70 +45,107 @@ def started_cluster(): cluster.start() remote.query("CREATE TABLE local1 (x UInt32) ENGINE = Log") - instance_test_reconnect.query(''' + instance_test_reconnect.query( + """ CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local1') -''') +""" + ) - remote.query("CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)") - instance_test_inserts_batching.query(''' + remote.query( + "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)" + ) + instance_test_inserts_batching.query( + """ CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') SETTINGS fsync_after_insert=1, fsync_directories=1 -''') +""" + ) instance_test_inserts_local_cluster.query( - "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)") - instance_test_inserts_local_cluster.query(''' + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)" + ) + instance_test_inserts_local_cluster.query( + """ CREATE TABLE distributed_on_local (d Date, x UInt32) ENGINE = Distributed('test_local_cluster', 'default', 'local') -''') +""" + ) - node1.query(''' + node1.query( + """ CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node1', date, id, 8192) -''') - node2.query(''' +""" + ) + node2.query( + """ CREATE TABLE replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/replicated', 'node2', date, id, 8192) -''') +""" + ) - node1.query(''' + node1.query( + """ CREATE TABLE distributed (date Date, id UInt32) ENGINE = Distributed('shard_with_local_replica', 'default', 'replicated') -''') +""" + ) - node2.query(''' + node2.query( + """ CREATE TABLE distributed (date Date, id UInt32) ENGINE = Distributed('shard_with_local_replica', 'default', 'replicated') -''') +""" + ) - shard1.query(''' -CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)''') + shard1.query( + """ +CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)""" + ) - shard2.query(''' -CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)''') + shard2.query( + """ +CREATE TABLE low_cardinality (d Date, x UInt32, s LowCardinality(String)) ENGINE = MergeTree(d, x, 8192)""" + ) - shard1.query(''' -CREATE TABLE low_cardinality_all (d Date, x UInt32, s LowCardinality(String)) ENGINE = Distributed('shard_with_low_cardinality', 'default', 'low_cardinality', sipHash64(s))''') + shard1.query( + """ +CREATE TABLE low_cardinality_all (d Date, x UInt32, s LowCardinality(String)) ENGINE = Distributed('shard_with_low_cardinality', 'default', 'low_cardinality', sipHash64(s))""" + ) - node1.query(''' -CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n''') + node1.query( + """ +CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n""" + ) - node2.query(''' -CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n''') + node2.query( + """ +CREATE TABLE table_function (n UInt8, s String) ENGINE = MergeTree() ORDER BY n""" + ) - node1.query(''' + node1.query( + """ CREATE TABLE distributed_one_replica_internal_replication (date Date, id UInt32) ENGINE = Distributed('shard_with_local_replica_internal_replication', 'default', 'single_replicated') -''') +""" + ) - node2.query(''' + node2.query( + """ CREATE TABLE distributed_one_replica_internal_replication (date Date, id UInt32) ENGINE = Distributed('shard_with_local_replica_internal_replication', 'default', 'single_replicated') -''') +""" + ) - node1.query(''' + node1.query( + """ CREATE TABLE distributed_one_replica_no_internal_replication (date Date, id UInt32) ENGINE = Distributed('shard_with_local_replica', 'default', 'single_replicated') -''') +""" + ) - node2.query(''' + node2.query( + """ CREATE TABLE distributed_one_replica_no_internal_replication (date Date, id UInt32) ENGINE = Distributed('shard_with_local_replica', 'default', 'single_replicated') -''') +""" + ) - node2.query(''' + node2.query( + """ CREATE TABLE single_replicated(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/single_replicated', 'node2', date, id, 8192) -''') +""" + ) yield cluster @@ -109,10 +160,12 @@ def test_reconnect(started_cluster): # Open a connection for insertion. instance.query("INSERT INTO distributed VALUES (1)") time.sleep(1) - assert remote.query("SELECT count(*) FROM local1").strip() == '1' + assert remote.query("SELECT count(*) FROM local1").strip() == "1" # Now break the connection. - pm.partition_instances(instance, remote, action='REJECT --reject-with tcp-reset') + pm.partition_instances( + instance, remote, action="REJECT --reject-with tcp-reset" + ) instance.query("INSERT INTO distributed VALUES (2)") time.sleep(1) @@ -123,7 +176,7 @@ def test_reconnect(started_cluster): instance.query("INSERT INTO distributed VALUES (3)") time.sleep(5) - assert remote.query("SELECT count(*) FROM local1").strip() == '3' + assert remote.query("SELECT count(*) FROM local1").strip() == "3" def test_inserts_batching(started_cluster): @@ -139,10 +192,14 @@ def test_inserts_batching(started_cluster): instance.query("INSERT INTO distributed(x, d) VALUES (2, '2000-01-01')") for i in range(3, 7): - instance.query("INSERT INTO distributed(d, x) VALUES ('2000-01-01', {})".format(i)) + instance.query( + "INSERT INTO distributed(d, x) VALUES ('2000-01-01', {})".format(i) + ) for i in range(7, 9): - instance.query("INSERT INTO distributed(x, d) VALUES ({}, '2000-01-01')".format(i)) + instance.query( + "INSERT INTO distributed(x, d) VALUES ({}, '2000-01-01')".format(i) + ) instance.query("INSERT INTO distributed(d, x) VALUES ('2000-01-01', 9)") @@ -150,12 +207,16 @@ def test_inserts_batching(started_cluster): instance.query("ALTER TABLE distributed ADD COLUMN s String") for i in range(10, 13): - instance.query("INSERT INTO distributed(d, x) VALUES ('2000-01-01', {})".format(i)) + instance.query( + "INSERT INTO distributed(d, x) VALUES ('2000-01-01', {})".format(i) + ) instance.query("SYSTEM FLUSH DISTRIBUTED distributed") time.sleep(1.0) - result = remote.query("SELECT _part, groupArray(x) FROM local2 GROUP BY _part ORDER BY _part") + result = remote.query( + "SELECT _part, groupArray(x) FROM local2 GROUP BY _part ORDER BY _part" + ) # Explanation: as merges are turned off on remote instance, active parts in local2 table correspond 1-to-1 # to inserted blocks. @@ -166,13 +227,13 @@ def test_inserts_batching(started_cluster): # 3. Full batch of inserts before ALTER. # 4. Full batch of inserts after ALTER (that have different block structure). # 5. What was left to insert with the column structure before ALTER. - expected = '''\ + expected = """\ 20000101_20000101_1_1_0\t[1] 20000101_20000101_2_2_0\t[2,3,4] 20000101_20000101_3_3_0\t[5,6,7] 20000101_20000101_4_4_0\t[10,11,12] 20000101_20000101_5_5_0\t[8,9] -''' +""" assert TSV(result) == TSV(expected) @@ -180,11 +241,13 @@ def test_inserts_local(started_cluster): instance = instance_test_inserts_local_cluster instance.query("INSERT INTO distributed_on_local VALUES ('2000-01-01', 1)") time.sleep(0.5) - assert instance.query("SELECT count(*) FROM local").strip() == '1' + assert instance.query("SELECT count(*) FROM local").strip() == "1" def test_inserts_single_replica_local_internal_replication(started_cluster): - with pytest.raises(QueryRuntimeException, match="Table default.single_replicated doesn't exist"): + with pytest.raises( + QueryRuntimeException, match="Table default.single_replicated doesn't exist" + ): node1.query( "INSERT INTO distributed_one_replica_internal_replication VALUES ('2000-01-01', 1)", settings={ @@ -194,7 +257,7 @@ def test_inserts_single_replica_local_internal_replication(started_cluster): "load_balancing": "first_or_random", }, ) - assert node2.query("SELECT count(*) FROM single_replicated").strip() == '0' + assert node2.query("SELECT count(*) FROM single_replicated").strip() == "0" def test_inserts_single_replica_internal_replication(started_cluster): @@ -208,14 +271,16 @@ def test_inserts_single_replica_internal_replication(started_cluster): "load_balancing": "first_or_random", }, ) - assert node2.query("SELECT count(*) FROM single_replicated").strip() == '1' + assert node2.query("SELECT count(*) FROM single_replicated").strip() == "1" finally: node2.query("TRUNCATE TABLE single_replicated") def test_inserts_single_replica_no_internal_replication(started_cluster): try: - with pytest.raises(QueryRuntimeException, match="Table default.single_replicated doesn't exist"): + with pytest.raises( + QueryRuntimeException, match="Table default.single_replicated doesn't exist" + ): node1.query( "INSERT INTO distributed_one_replica_no_internal_replication VALUES ('2000-01-01', 1)", settings={ @@ -223,7 +288,7 @@ def test_inserts_single_replica_no_internal_replication(started_cluster): "prefer_localhost_replica": "0", }, ) - assert node2.query("SELECT count(*) FROM single_replicated").strip() == '1' + assert node2.query("SELECT count(*) FROM single_replicated").strip() == "1" finally: node2.query("TRUNCATE TABLE single_replicated") @@ -235,22 +300,22 @@ def test_prefer_localhost_replica(started_cluster): node2.query("INSERT INTO distributed VALUES (toDate('2017-06-17'), 22)") time.sleep(1.0) - expected_distributed = '''\ + expected_distributed = """\ 2017-06-17\t11 2017-06-17\t22 -''' +""" - expected_from_node2 = '''\ + expected_from_node2 = """\ 2017-06-17\t11 2017-06-17\t22 2017-06-17\t44 -''' +""" - expected_from_node1 = '''\ + expected_from_node1 = """\ 2017-06-17\t11 2017-06-17\t22 2017-06-17\t33 -''' +""" assert TSV(node1.query(test_query)) == TSV(expected_distributed) assert TSV(node2.query(test_query)) == TSV(expected_distributed) @@ -270,19 +335,30 @@ def test_prefer_localhost_replica(started_cluster): assert TSV(node2.query(test_query)) == TSV(expected_from_node2) # Now query is sent to node1, as it higher in order - assert TSV(node2.query(test_query + " SETTINGS load_balancing='in_order', prefer_localhost_replica=0")) == TSV( - expected_from_node1) + assert TSV( + node2.query( + test_query + + " SETTINGS load_balancing='in_order', prefer_localhost_replica=0" + ) + ) == TSV(expected_from_node1) def test_inserts_low_cardinality(started_cluster): instance = shard1 - instance.query("INSERT INTO low_cardinality_all (d,x,s) VALUES ('2018-11-12',1,'123')") + instance.query( + "INSERT INTO low_cardinality_all (d,x,s) VALUES ('2018-11-12',1,'123')" + ) time.sleep(0.5) - assert instance.query("SELECT count(*) FROM low_cardinality_all").strip() == '1' + assert instance.query("SELECT count(*) FROM low_cardinality_all").strip() == "1" def test_table_function(started_cluster): node1.query( - "insert into table function cluster('shard_with_local_replica', 'default', 'table_function') select number, concat('str_', toString(number)) from numbers(100000)") - assert node1.query( - "select count() from cluster('shard_with_local_replica', 'default', 'table_function')").rstrip() == '100000' + "insert into table function cluster('shard_with_local_replica', 'default', 'table_function') select number, concat('str_', toString(number)) from numbers(100000)" + ) + assert ( + node1.query( + "select count() from cluster('shard_with_local_replica', 'default', 'table_function')" + ).rstrip() + == "100000" + ) diff --git a/tests/integration/test_insert_into_distributed_sync_async/test.py b/tests/integration/test_insert_into_distributed_sync_async/test.py index 1f479003b99..e0c454feee6 100755 --- a/tests/integration/test_insert_into_distributed_sync_async/test.py +++ b/tests/integration/test_insert_into_distributed_sync_async/test.py @@ -11,8 +11,8 @@ from helpers.client import QueryRuntimeException, QueryTimeoutExceedException cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) +node1 = cluster.add_instance("node1", main_configs=["configs/remote_servers.xml"]) +node2 = cluster.add_instance("node2", main_configs=["configs/remote_servers.xml"]) @pytest.fixture(scope="module") @@ -21,13 +21,17 @@ def started_cluster(): cluster.start() for node in (node1, node2): - node.query(''' + node.query( + """ CREATE TABLE local_table(date Date, val UInt64) ENGINE = MergeTree(date, (date, val), 8192); -''') +""" + ) - node1.query(''' + node1.query( + """ CREATE TABLE distributed_table(date Date, val UInt64) ENGINE = Distributed(test_cluster, default, local_table) -''') +""" + ) yield cluster @@ -36,41 +40,61 @@ CREATE TABLE distributed_table(date Date, val UInt64) ENGINE = Distributed(test_ def test_insertion_sync(started_cluster): - node1.query('''SET insert_distributed_sync = 1, insert_distributed_timeout = 0; - INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers LIMIT 10000''') + node1.query( + """SET insert_distributed_sync = 1, insert_distributed_timeout = 0; + INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers LIMIT 10000""" + ) - assert node2.query("SELECT count() FROM local_table").rstrip() == '10000' + assert node2.query("SELECT count() FROM local_table").rstrip() == "10000" - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 1, insert_distributed_timeout = 1; - INSERT INTO distributed_table SELECT today() - 1 as date, number as val FROM system.numbers LIMIT 10000''') + INSERT INTO distributed_table SELECT today() - 1 as date, number as val FROM system.numbers LIMIT 10000""" + ) - assert node2.query("SELECT count() FROM local_table").rstrip() == '20000' + assert node2.query("SELECT count() FROM local_table").rstrip() == "20000" # Insert with explicitly specified columns. - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 1, insert_distributed_timeout = 1; - INSERT INTO distributed_table(date, val) VALUES ('2000-01-01', 100500)''') + INSERT INTO distributed_table(date, val) VALUES ('2000-01-01', 100500)""" + ) # Insert with columns specified in different order. - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 1, insert_distributed_timeout = 1; - INSERT INTO distributed_table(val, date) VALUES (100500, '2000-01-01')''') + INSERT INTO distributed_table(val, date) VALUES (100500, '2000-01-01')""" + ) # Insert with an incomplete list of columns. - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 1, insert_distributed_timeout = 1; - INSERT INTO distributed_table(val) VALUES (100500)''') + INSERT INTO distributed_table(val) VALUES (100500)""" + ) - expected = TSV(''' + expected = TSV( + """ 1970-01-01 100500 2000-01-01 100500 -2000-01-01 100500''') - assert TSV(node2.query('SELECT date, val FROM local_table WHERE val = 100500 ORDER BY date')) == expected +2000-01-01 100500""" + ) + assert ( + TSV( + node2.query( + "SELECT date, val FROM local_table WHERE val = 100500 ORDER BY date" + ) + ) + == expected + ) node1.query("TRUNCATE TABLE local_table SYNC") node2.query("TRUNCATE TABLE local_table SYNC") + """ def test_insertion_sync_fails_on_error(started_cluster): with PartitionManager() as pm: @@ -84,41 +108,53 @@ def test_insertion_sync_fails_on_error(started_cluster): def test_insertion_sync_fails_with_timeout(started_cluster): with pytest.raises(QueryRuntimeException): - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 1, insert_distributed_timeout = 1; - INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers''') + INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers""" + ) def test_insertion_without_sync_ignores_timeout(started_cluster): with pytest.raises(QueryTimeoutExceedException): - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 0, insert_distributed_timeout = 1; - INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers''', timeout=1.5) + INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers""", + timeout=1.5, + ) def test_insertion_sync_with_disabled_timeout(started_cluster): with pytest.raises(QueryTimeoutExceedException): - node1.query(''' + node1.query( + """ SET insert_distributed_sync = 1, insert_distributed_timeout = 0; - INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers''', timeout=1) + INSERT INTO distributed_table SELECT today() as date, number as val FROM system.numbers""", + timeout=1, + ) def test_async_inserts_into_local_shard(started_cluster): - node1.query('''CREATE TABLE shard_local (i Int64) ENGINE = Memory''') + node1.query("""CREATE TABLE shard_local (i Int64) ENGINE = Memory""") node1.query( - '''CREATE TABLE shard_distributed (i Int64) ENGINE = Distributed(local_shard_with_internal_replication, default, shard_local)''') - node1.query('''INSERT INTO shard_distributed VALUES (1)''', settings={"insert_distributed_sync": 0}) + """CREATE TABLE shard_distributed (i Int64) ENGINE = Distributed(local_shard_with_internal_replication, default, shard_local)""" + ) + node1.query( + """INSERT INTO shard_distributed VALUES (1)""", + settings={"insert_distributed_sync": 0}, + ) - assert TSV(node1.query('''SELECT count() FROM shard_distributed''')) == TSV("1\n") - node1.query('''DETACH TABLE shard_distributed''') - node1.query('''ATTACH TABLE shard_distributed''') - assert TSV(node1.query('''SELECT count() FROM shard_distributed''')) == TSV("1\n") + assert TSV(node1.query("""SELECT count() FROM shard_distributed""")) == TSV("1\n") + node1.query("""DETACH TABLE shard_distributed""") + node1.query("""ATTACH TABLE shard_distributed""") + assert TSV(node1.query("""SELECT count() FROM shard_distributed""")) == TSV("1\n") - node1.query('''DROP TABLE shard_distributed''') - node1.query('''DROP TABLE shard_local''') + node1.query("""DROP TABLE shard_distributed""") + node1.query("""DROP TABLE shard_local""") -if __name__ == '__main__': +if __name__ == "__main__": with contextmanager(started_cluster)() as cluster: for name, instance in list(cluster.instances.items()): print(name, instance.ip_address) diff --git a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py index 32edb6829c8..7c2ce9f05f2 100644 --- a/tests/integration/test_insert_into_distributed_through_materialized_view/test.py +++ b/tests/integration/test_insert_into_distributed_through_materialized_view/test.py @@ -7,15 +7,21 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_configs=['configs/remote_servers.xml']) +instance_test_reconnect = cluster.add_instance( + "instance_test_reconnect", main_configs=["configs/remote_servers.xml"] +) instance_test_inserts_batching = cluster.add_instance( - 'instance_test_inserts_batching', - main_configs=['configs/remote_servers.xml'], user_configs=['configs/enable_distributed_inserts_batching.xml']) -remote = cluster.add_instance('remote', main_configs=['configs/forbid_background_merges.xml']) + "instance_test_inserts_batching", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/enable_distributed_inserts_batching.xml"], +) +remote = cluster.add_instance( + "remote", main_configs=["configs/forbid_background_merges.xml"] +) instance_test_inserts_local_cluster = cluster.add_instance( - 'instance_test_inserts_local_cluster', - main_configs=['configs/remote_servers.xml']) + "instance_test_inserts_local_cluster", main_configs=["configs/remote_servers.xml"] +) @pytest.fixture(scope="module") @@ -25,29 +31,47 @@ def started_cluster(): remote.query("CREATE TABLE local1 (x UInt32) ENGINE = Log") - instance_test_reconnect.query(''' -CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local1') -''') - instance_test_reconnect.query("CREATE TABLE local1_source (x UInt32) ENGINE = Memory") instance_test_reconnect.query( - "CREATE MATERIALIZED VIEW local1_view to distributed AS SELECT x FROM local1_source") + """ +CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local1') +""" + ) + instance_test_reconnect.query( + "CREATE TABLE local1_source (x UInt32) ENGINE = Memory" + ) + instance_test_reconnect.query( + "CREATE MATERIALIZED VIEW local1_view to distributed AS SELECT x FROM local1_source" + ) - remote.query("CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)") - instance_test_inserts_batching.query(''' -CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') -''') - instance_test_inserts_batching.query("CREATE TABLE local2_source (d Date, x UInt32) ENGINE = Log") + remote.query( + "CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)" + ) instance_test_inserts_batching.query( - "CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x FROM local2_source") + """ +CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') +""" + ) + instance_test_inserts_batching.query( + "CREATE TABLE local2_source (d Date, x UInt32) ENGINE = Log" + ) + instance_test_inserts_batching.query( + "CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x FROM local2_source" + ) - instance_test_inserts_local_cluster.query("CREATE TABLE local_source (d Date, x UInt32) ENGINE = Memory") instance_test_inserts_local_cluster.query( - "CREATE MATERIALIZED VIEW local_view to distributed_on_local AS SELECT d,x FROM local_source") + "CREATE TABLE local_source (d Date, x UInt32) ENGINE = Memory" + ) instance_test_inserts_local_cluster.query( - "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)") - instance_test_inserts_local_cluster.query(''' + "CREATE MATERIALIZED VIEW local_view to distributed_on_local AS SELECT d,x FROM local_source" + ) + instance_test_inserts_local_cluster.query( + "CREATE TABLE local (d Date, x UInt32) ENGINE = MergeTree(d, x, 8192)" + ) + instance_test_inserts_local_cluster.query( + """ CREATE TABLE distributed_on_local (d Date, x UInt32) ENGINE = Distributed('test_local_cluster', 'default', 'local') -''') +""" + ) yield cluster @@ -62,10 +86,12 @@ def test_reconnect(started_cluster): # Open a connection for insertion. instance.query("INSERT INTO local1_source VALUES (1)") time.sleep(1) - assert remote.query("SELECT count(*) FROM local1").strip() == '1' + assert remote.query("SELECT count(*) FROM local1").strip() == "1" # Now break the connection. - pm.partition_instances(instance, remote, action='REJECT --reject-with tcp-reset') + pm.partition_instances( + instance, remote, action="REJECT --reject-with tcp-reset" + ) instance.query("INSERT INTO local1_source VALUES (2)") time.sleep(1) @@ -77,7 +103,7 @@ def test_reconnect(started_cluster): instance.query("INSERT INTO local1_source VALUES (3)") time.sleep(1) - assert remote.query("SELECT count(*) FROM local1").strip() == '3' + assert remote.query("SELECT count(*) FROM local1").strip() == "3" @pytest.mark.skip(reason="Flapping test") @@ -94,10 +120,14 @@ def test_inserts_batching(started_cluster): instance.query("INSERT INTO local2_source(x, d) VALUES (2, '2000-01-01')") for i in range(3, 7): - instance.query("INSERT INTO local2_source(d, x) VALUES ('2000-01-01', {})".format(i)) + instance.query( + "INSERT INTO local2_source(d, x) VALUES ('2000-01-01', {})".format(i) + ) for i in range(7, 9): - instance.query("INSERT INTO local2_source(x, d) VALUES ({}, '2000-01-01')".format(i)) + instance.query( + "INSERT INTO local2_source(x, d) VALUES ({}, '2000-01-01')".format(i) + ) instance.query("INSERT INTO local2_source(d, x) VALUES ('2000-01-01', 9)") @@ -107,15 +137,23 @@ def test_inserts_batching(started_cluster): # Memory Engine doesn't support ALTER so we just DROP/CREATE everything instance.query("DROP TABLE local2_source") - instance.query("CREATE TABLE local2_source (d Date, x UInt32, s String) ENGINE = Memory") - instance.query("CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x,s FROM local2_source") + instance.query( + "CREATE TABLE local2_source (d Date, x UInt32, s String) ENGINE = Memory" + ) + instance.query( + "CREATE MATERIALIZED VIEW local2_view to distributed AS SELECT d,x,s FROM local2_source" + ) for i in range(10, 13): - instance.query("INSERT INTO local2_source(d, x) VALUES ('2000-01-01', {})".format(i)) + instance.query( + "INSERT INTO local2_source(d, x) VALUES ('2000-01-01', {})".format(i) + ) time.sleep(1.0) - result = remote.query("SELECT _part, groupArray(x) FROM local2 GROUP BY _part ORDER BY _part") + result = remote.query( + "SELECT _part, groupArray(x) FROM local2 GROUP BY _part ORDER BY _part" + ) # Explanation: as merges are turned off on remote instance, active parts in local2 table correspond 1-to-1 # to inserted blocks. @@ -126,13 +164,13 @@ def test_inserts_batching(started_cluster): # 3. Full batch of inserts regardless order of columns thanks to the view. # 4. Full batch of inserts after ALTER (that have different block structure). # 5. What was left to insert before ALTER. - expected = '''\ + expected = """\ 20000101_20000101_1_1_0 [1] 20000101_20000101_2_2_0 [2,3,4] 20000101_20000101_3_3_0 [5,6,7] 20000101_20000101_4_4_0 [10,11,12] 20000101_20000101_5_5_0 [8,9] -''' +""" assert TSV(result) == TSV(expected) @@ -140,4 +178,4 @@ def test_inserts_local(started_cluster): instance = instance_test_inserts_local_cluster instance.query("INSERT INTO local_source VALUES ('2000-01-01', 1)") time.sleep(0.5) - assert instance.query("SELECT count(*) FROM local").strip() == '1' + assert instance.query("SELECT count(*) FROM local").strip() == "1" diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py index ef0308cc658..3807d6e1cea 100644 --- a/tests/integration/test_jbod_balancer/test.py +++ b/tests/integration/test_jbod_balancer/test.py @@ -14,7 +14,9 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", - main_configs=["configs/config.d/storage_configuration.xml",], + main_configs=[ + "configs/config.d/storage_configuration.xml", + ], with_zookeeper=True, stay_alive=True, tmpfs=["/jbod1:size=100M", "/jbod2:size=100M", "/jbod3:size=100M"], diff --git a/tests/integration/test_jbod_ha/test.py b/tests/integration/test_jbod_ha/test.py index 0a8631ff207..3dec61985b1 100644 --- a/tests/integration/test_jbod_ha/test.py +++ b/tests/integration/test_jbod_ha/test.py @@ -14,7 +14,9 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", - main_configs=["configs/config.d/storage_configuration.xml",], + main_configs=[ + "configs/config.d/storage_configuration.xml", + ], with_zookeeper=True, stay_alive=True, tmpfs=["/jbod1:size=100M", "/jbod2:size=100M", "/jbod3:size=100M"], diff --git a/tests/integration/test_jdbc_bridge/test.py b/tests/integration/test_jdbc_bridge/test.py index b5304c4cb10..0e41cc8c8b7 100644 --- a/tests/integration/test_jdbc_bridge/test.py +++ b/tests/integration/test_jdbc_bridge/test.py @@ -8,75 +8,108 @@ from helpers.test_tools import TSV from string import Template cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance("instance", main_configs=["configs/jdbc_bridge.xml"], with_jdbc_bridge=True) +instance = cluster.add_instance( + "instance", main_configs=["configs/jdbc_bridge.xml"], with_jdbc_bridge=True +) datasource = "self" records = 1000 + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - instance.query(''' + instance.query( + """ CREATE DATABASE test; CREATE TABLE test.ClickHouseTable(Num UInt32, Str String, Desc Nullable(String)) engine = Memory; INSERT INTO test.ClickHouseTable(Num, Str) SELECT number, toString(number) FROM system.numbers LIMIT {}; - '''.format(records)) + """.format( + records + ) + ) while True: datasources = instance.query("select * from jdbc('', 'show datasources')") - if 'self' in datasources: - logging.debug(f"JDBC Driver self datasource initialized.\n{datasources}") + if "self" in datasources: + logging.debug( + f"JDBC Driver self datasource initialized.\n{datasources}" + ) break else: - logging.debug(f"Waiting JDBC Driver to initialize 'self' datasource.\n{datasources}") + logging.debug( + f"Waiting JDBC Driver to initialize 'self' datasource.\n{datasources}" + ) yield cluster finally: cluster.shutdown() + def test_jdbc_query(started_cluster): """Test simple query with inline schema and query parameters""" expected = "{}\t{}".format(datasource, records) - actual = instance.query(''' + actual = instance.query( + """ SELECT * FROM jdbc( '{}?datasource_column&fetch_size=1', 'rows UInt32', 'SELECT count(1) AS rows FROM test.ClickHouseTable' ) - '''.format(datasource)) - assert TSV(actual) == TSV(expected), "expecting {} but got {}".format(expected, actual) + """.format( + datasource + ) + ) + assert TSV(actual) == TSV(expected), "expecting {} but got {}".format( + expected, actual + ) + def test_jdbc_distributed_query(started_cluster): """Test distributed query involving both JDBC table function and ClickHouse table""" - actual = instance.query(''' + actual = instance.query( + """ SELECT a.Num + 1 FROM jdbc('{0}', 'SELECT * FROM test.ClickHouseTable') a INNER JOIN jdbc('{0}', 'num UInt32', 'SELECT {1} - 1 AS num') b on a.Num = b.num INNER JOIN test.ClickHouseTable c on b.num = c.Num - '''.format(datasource, records)) + """.format( + datasource, records + ) + ) assert int(actual) == records, "expecting {} but got {}".format(records, actual) + def test_jdbc_insert(started_cluster): """Test insert query using JDBC table function""" - instance.query('DROP TABLE IF EXISTS test.test_insert') - instance.query(''' + instance.query("DROP TABLE IF EXISTS test.test_insert") + instance.query( + """ CREATE TABLE test.test_insert ENGINE = Memory AS SELECT * FROM test.ClickHouseTable; SELECT * FROM jdbc('{0}?mutation', 'INSERT INTO test.test_insert VALUES({1}, ''{1}'', ''{1}'')'); - '''.format(datasource, records)) + """.format( + datasource, records + ) + ) expected = records actual = instance.query( - "SELECT Desc FROM jdbc('{}', 'SELECT * FROM test.test_insert WHERE Num = {}')".format(datasource, records)) + "SELECT Desc FROM jdbc('{}', 'SELECT * FROM test.test_insert WHERE Num = {}')".format( + datasource, records + ) + ) assert int(actual) == expected, "expecting {} but got {}".format(records, actual) + def test_jdbc_update(started_cluster): """Test update query using JDBC table function""" secrets = str(uuid.uuid1()) - instance.query('DROP TABLE IF EXISTS test.test_update') - instance.query(''' + instance.query("DROP TABLE IF EXISTS test.test_update") + instance.query( + """ CREATE TABLE test.test_update ENGINE = Memory AS SELECT * FROM test.ClickHouseTable; SELECT * @@ -84,18 +117,29 @@ def test_jdbc_update(started_cluster): '{}?mutation', 'SET mutations_sync = 1; ALTER TABLE test.test_update UPDATE Str=''{}'' WHERE Num = {} - 1;' ) - '''.format(datasource, secrets, records)) + """.format( + datasource, secrets, records + ) + ) - actual = instance.query(''' + actual = instance.query( + """ SELECT Str FROM jdbc('{}', 'SELECT * FROM test.test_update WHERE Num = {} - 1') - '''.format(datasource, records)) - assert TSV(actual) == TSV(secrets), "expecting {} but got {}".format(secrets, actual) + """.format( + datasource, records + ) + ) + assert TSV(actual) == TSV(secrets), "expecting {} but got {}".format( + secrets, actual + ) + def test_jdbc_delete(started_cluster): """Test delete query using JDBC table function""" - instance.query('DROP TABLE IF EXISTS test.test_delete') - instance.query(''' + instance.query("DROP TABLE IF EXISTS test.test_delete") + instance.query( + """ CREATE TABLE test.test_delete ENGINE = Memory AS SELECT * FROM test.ClickHouseTable; SELECT * @@ -103,20 +147,31 @@ def test_jdbc_delete(started_cluster): '{}?mutation', 'SET mutations_sync = 1; ALTER TABLE test.test_delete DELETE WHERE Num < {} - 1;' ) - '''.format(datasource, records)) + """.format( + datasource, records + ) + ) expected = records - 1 actual = instance.query( - "SELECT Str FROM jdbc('{}', 'SELECT * FROM test.test_delete')".format(datasource, records)) + "SELECT Str FROM jdbc('{}', 'SELECT * FROM test.test_delete')".format( + datasource, records + ) + ) assert int(actual) == expected, "expecting {} but got {}".format(expected, actual) + def test_jdbc_table_engine(started_cluster): """Test query against a JDBC table""" - instance.query('DROP TABLE IF EXISTS test.jdbc_table') - actual = instance.query(''' + instance.query("DROP TABLE IF EXISTS test.jdbc_table") + actual = instance.query( + """ CREATE TABLE test.jdbc_table(Str String) ENGINE = JDBC('{}', 'test', 'ClickHouseTable'); SELECT count(1) FROM test.jdbc_table; - '''.format(datasource)) + """.format( + datasource + ) + ) assert int(actual) == records, "expecting {} but got {}".format(records, actual) diff --git a/tests/integration/test_jemalloc_percpu_arena/test.py b/tests/integration/test_jemalloc_percpu_arena/test.py index 6a4522c1b76..80d8e2ae36a 100755 --- a/tests/integration/test_jemalloc_percpu_arena/test.py +++ b/tests/integration/test_jemalloc_percpu_arena/test.py @@ -13,42 +13,54 @@ CPU_ID = 4 def run_command_in_container(cmd, *args): # /clickhouse is mounted by integration tests runner - alternative_binary = os.getenv('CLICKHOUSE_BINARY', '/clickhouse') + alternative_binary = os.getenv("CLICKHOUSE_BINARY", "/clickhouse") if alternative_binary: - args+=( - '--volume', f'{alternative_binary}:/usr/bin/clickhouse', + args += ( + "--volume", + f"{alternative_binary}:/usr/bin/clickhouse", ) - return subprocess.check_output(['docker', 'run', '--rm', - *args, - 'ubuntu:20.04', - 'sh', '-c', cmd, - ]) + return subprocess.check_output( + [ + "docker", + "run", + "--rm", + *args, + "ubuntu:20.04", + "sh", + "-c", + cmd, + ] + ) def run_with_cpu_limit(cmd, *args): with NamedTemporaryFile() as online_cpu: # NOTE: this is not the number of CPUs, but specific CPU ID - online_cpu.write(f'{CPU_ID}'.encode()) + online_cpu.write(f"{CPU_ID}".encode()) online_cpu.flush() # replace /sys/devices/system/cpu/online to full _SC_NPROCESSORS_ONLN # like LXD/LXC from [1] does. # # [1]: https://github.com/ClickHouse/ClickHouse/issues/32806 - args+=( - '--volume', f'{online_cpu.name}:/sys/devices/system/cpu/online', + args += ( + "--volume", + f"{online_cpu.name}:/sys/devices/system/cpu/online", ) return run_command_in_container(cmd, *args) def skip_if_jemalloc_disabled(): - output = run_command_in_container("""clickhouse local -q " + output = run_command_in_container( + """clickhouse local -q " SELECT value FROM system.build_options WHERE name = 'USE_JEMALLOC'" - """).strip() - if output != b'ON' and output != b'1': - pytest.skip(f'Compiled w/o jemalloc (USE_JEMALLOC={output})') + """ + ).strip() + if output != b"ON" and output != b"1": + pytest.skip(f"Compiled w/o jemalloc (USE_JEMALLOC={output})") + # Ensure that clickhouse works even when number of online CPUs # (_SC_NPROCESSORS_ONLN) is smaller then available (_SC_NPROCESSORS_CONF). @@ -59,29 +71,37 @@ def test_jemalloc_percpu_arena(): assert multiprocessing.cpu_count() > CPU_ID - online_cpus = int(run_with_cpu_limit('getconf _NPROCESSORS_ONLN')) + online_cpus = int(run_with_cpu_limit("getconf _NPROCESSORS_ONLN")) assert online_cpus == 1, online_cpus - all_cpus = int(run_with_cpu_limit('getconf _NPROCESSORS_CONF')) + all_cpus = int(run_with_cpu_limit("getconf _NPROCESSORS_CONF")) assert all_cpus == multiprocessing.cpu_count(), all_cpus # implicitly disable percpu arena - result = run_with_cpu_limit('clickhouse local -q "select 1"', + result = run_with_cpu_limit( + 'clickhouse local -q "select 1"', # NOTE: explicitly disable, since it is enabled by default in debug build # (and even though debug builds are not in CI let's state this). - '--env', 'MALLOC_CONF=abort_conf:false') + "--env", + "MALLOC_CONF=abort_conf:false", + ) assert int(result) == int(1), result # should fail because of abort_conf:true with pytest.raises(subprocess.CalledProcessError): - run_with_cpu_limit('clickhouse local -q "select 1"', - '--env', 'MALLOC_CONF=abort_conf:true') + run_with_cpu_limit( + 'clickhouse local -q "select 1"', "--env", "MALLOC_CONF=abort_conf:true" + ) # should not fail even with abort_conf:true, due to explicit narenas # NOTE: abort:false to make it compatible with debug build - run_with_cpu_limit('clickhouse local -q "select 1"', - '--env', f'MALLOC_CONF=abort_conf:true,abort:false,narenas:{all_cpus}') + run_with_cpu_limit( + 'clickhouse local -q "select 1"', + "--env", + f"MALLOC_CONF=abort_conf:true,abort:false,narenas:{all_cpus}", + ) + # For manual run. -if __name__ == '__main__': +if __name__ == "__main__": test_jemalloc_percpu_arena() diff --git a/tests/integration/test_join_set_family_s3/test.py b/tests/integration/test_join_set_family_s3/test.py index 9454acf1541..b09d5735628 100644 --- a/tests/integration/test_join_set_family_s3/test.py +++ b/tests/integration/test_join_set_family_s3/test.py @@ -9,9 +9,12 @@ from helpers.cluster import ClickHouseCluster def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=["configs/minio.xml", "configs/ssl.xml"], - with_minio=True, stay_alive=True) + cluster.add_instance( + "node", + main_configs=["configs/minio.xml", "configs/ssl.xml"], + with_minio=True, + stay_alive=True, + ) logging.info("Starting cluster...") cluster.start() @@ -22,7 +25,7 @@ def cluster(): cluster.shutdown() -def assert_objects_count(cluster, objects_count, path='data/'): +def assert_objects_count(cluster, objects_count, path="data/"): minio = cluster.minio_client s3_objects = list(minio.list_objects(cluster.minio_bucket, path)) if objects_count != len(s3_objects): @@ -41,22 +44,42 @@ def test_set_s3(cluster): node.query("INSERT INTO TABLE testLocalSet VALUES (1)") node.query("INSERT INTO TABLE testS3Set VALUES (1)") - assert node.query("SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3") == "0\t0\n1\t1\n0\t0\n" + assert ( + node.query( + "SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3" + ) + == "0\t0\n1\t1\n0\t0\n" + ) assert_objects_count(cluster, 1) node.query("INSERT INTO TABLE testLocalSet VALUES (2)") node.query("INSERT INTO TABLE testS3Set VALUES (2)") - assert node.query("SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3") == "0\t0\n1\t1\n1\t1\n" + assert ( + node.query( + "SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3" + ) + == "0\t0\n1\t1\n1\t1\n" + ) assert_objects_count(cluster, 2) node.restart_clickhouse() - assert node.query("SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3") == "0\t0\n1\t1\n1\t1\n" + assert ( + node.query( + "SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3" + ) + == "0\t0\n1\t1\n1\t1\n" + ) node.query("TRUNCATE TABLE testLocalSet") node.query("TRUNCATE TABLE testS3Set") - assert node.query("SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3") == "0\t0\n0\t0\n0\t0\n" + assert ( + node.query( + "SELECT number in testLocalSet, number in testS3Set FROM system.numbers LIMIT 3" + ) + == "0\t0\n0\t0\n0\t0\n" + ) assert_objects_count(cluster, 0) node.query("DROP TABLE testLocalSet") @@ -66,28 +89,52 @@ def test_set_s3(cluster): def test_join_s3(cluster): node = cluster.instances["node"] - node.query("CREATE TABLE testLocalJoin(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id)") - node.query("CREATE TABLE testS3Join(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id) SETTINGS disk='s3'") + node.query( + "CREATE TABLE testLocalJoin(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id)" + ) + node.query( + "CREATE TABLE testS3Join(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id) SETTINGS disk='s3'" + ) node.query("INSERT INTO testLocalJoin VALUES (1, 'a')") node.query("INSERT INTO testS3Join VALUES (1, 'a')") - assert node.query("SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3") == "\t\na\ta\n\t\n" + assert ( + node.query( + "SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3" + ) + == "\t\na\ta\n\t\n" + ) assert_objects_count(cluster, 1) node.query("INSERT INTO testLocalJoin VALUES (2, 'b')") node.query("INSERT INTO testS3Join VALUES (2, 'b')") - assert node.query("SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3") == "\t\na\ta\nb\tb\n" + assert ( + node.query( + "SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3" + ) + == "\t\na\ta\nb\tb\n" + ) assert_objects_count(cluster, 2) node.restart_clickhouse() - assert node.query("SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3") == "\t\na\ta\nb\tb\n" + assert ( + node.query( + "SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3" + ) + == "\t\na\ta\nb\tb\n" + ) node.query("TRUNCATE TABLE testLocalJoin") node.query("TRUNCATE TABLE testS3Join") - assert node.query("SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3") == "\t\n\t\n\t\n" + assert ( + node.query( + "SELECT joinGet('testLocalJoin', 'val', number) as local, joinGet('testS3Join', 'val', number) as s3 FROM system.numbers LIMIT 3" + ) + == "\t\n\t\n\t\n" + ) assert_objects_count(cluster, 0) node.query("DROP TABLE testLocalJoin") diff --git a/tests/integration/test_keeper_and_access_storage/test.py b/tests/integration/test_keeper_and_access_storage/test.py index 3a3c7535a85..ae6b0085094 100644 --- a/tests/integration/test_keeper_and_access_storage/test.py +++ b/tests/integration/test_keeper_and_access_storage/test.py @@ -6,7 +6,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/keeper.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/keeper.xml"], stay_alive=True +) # test that server is able to start @pytest.fixture(scope="module") @@ -17,5 +19,6 @@ def started_cluster(): finally: cluster.shutdown() + def test_create_replicated(started_cluster): assert node1.query("SELECT 1") == "1\n" diff --git a/tests/integration/test_keeper_auth/test.py b/tests/integration/test_keeper_auth/test.py index 6be78f95483..364d93dfc53 100644 --- a/tests/integration/test_keeper_auth/test.py +++ b/tests/integration/test_keeper_auth/test.py @@ -1,15 +1,26 @@ - import pytest from helpers.cluster import ClickHouseCluster from kazoo.client import KazooClient, KazooState from kazoo.security import ACL, make_digest_acl, make_acl -from kazoo.exceptions import AuthFailedError, InvalidACLError, NoAuthError, KazooException +from kazoo.exceptions import ( + AuthFailedError, + InvalidACLError, + NoAuthError, + KazooException, +) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/keeper_config.xml'], with_zookeeper=True, use_keeper=False, stay_alive=True) +node = cluster.add_instance( + "node", + main_configs=["configs/keeper_config.xml"], + with_zookeeper=True, + use_keeper=False, + stay_alive=True, +) SUPERAUTH = "super:admin" + @pytest.fixture(scope="module") def started_cluster(): try: @@ -20,63 +31,106 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip('node') + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip("node") + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def get_genuine_zk(): print("Zoo1", cluster.get_instance_ip("zoo1")) - return cluster.get_kazoo_client('zoo1') + return cluster.get_kazoo_client("zoo1") -@pytest.mark.parametrize( - ('get_zk'), - [ - get_genuine_zk, - get_fake_zk - ] -) +@pytest.mark.parametrize(("get_zk"), [get_genuine_zk, get_fake_zk]) def test_remove_acl(started_cluster, get_zk): auth_connection = get_zk() - auth_connection.add_auth('digest', 'user1:password1') + auth_connection.add_auth("digest", "user1:password1") # Consistent with zookeeper, accept generated digest - auth_connection.create("/test_remove_acl1", b"dataX", acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", read=True, write=False, create=False, delete=False, admin=False)]) - auth_connection.create("/test_remove_acl2", b"dataX", acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", read=True, write=True, create=False, delete=False, admin=False)]) - auth_connection.create("/test_remove_acl3", b"dataX", acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", all=True)]) + auth_connection.create( + "/test_remove_acl1", + b"dataX", + acl=[ + make_acl( + "digest", + "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", + read=True, + write=False, + create=False, + delete=False, + admin=False, + ) + ], + ) + auth_connection.create( + "/test_remove_acl2", + b"dataX", + acl=[ + make_acl( + "digest", + "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", + read=True, + write=True, + create=False, + delete=False, + admin=False, + ) + ], + ) + auth_connection.create( + "/test_remove_acl3", + b"dataX", + acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", all=True)], + ) auth_connection.delete("/test_remove_acl2") - auth_connection.create("/test_remove_acl4", b"dataX", acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", read=True, write=True, create=True, delete=False, admin=False)]) + auth_connection.create( + "/test_remove_acl4", + b"dataX", + acl=[ + make_acl( + "digest", + "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", + read=True, + write=True, + create=True, + delete=False, + admin=False, + ) + ], + ) acls, stat = auth_connection.get_acls("/test_remove_acl3") assert stat.aversion == 0 assert len(acls) == 1 for acl in acls: - assert acl.acl_list == ['ALL'] + assert acl.acl_list == ["ALL"] assert acl.perms == 31 -@pytest.mark.parametrize( - ('get_zk'), - [ - get_genuine_zk, - get_fake_zk - ] -) - +@pytest.mark.parametrize(("get_zk"), [get_genuine_zk, get_fake_zk]) def test_digest_auth_basic(started_cluster, get_zk): auth_connection = get_zk() - auth_connection.add_auth('digest', 'user1:password1') + auth_connection.add_auth("digest", "user1:password1") auth_connection.create("/test_no_acl", b"") - auth_connection.create("/test_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + auth_connection.create( + "/test_all_acl", b"data", acl=[make_acl("auth", "", all=True)] + ) # Consistent with zookeeper, accept generated digest - auth_connection.create("/test_all_digest_acl", b"dataX", acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", all=True)]) + auth_connection.create( + "/test_all_digest_acl", + b"dataX", + acl=[make_acl("digest", "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", all=True)], + ) assert auth_connection.get("/test_all_acl")[0] == b"data" assert auth_connection.get("/test_all_digest_acl")[0] == b"dataX" @@ -104,7 +158,7 @@ def test_digest_auth_basic(started_cluster, get_zk): no_auth_connection = get_zk() # wrong auth - no_auth_connection.add_auth('digest', 'user2:password2') + no_auth_connection.add_auth("digest", "user2:password2") with pytest.raises(NoAuthError): no_auth_connection.set("/test_all_acl", b"hello") @@ -122,7 +176,7 @@ def test_digest_auth_basic(started_cluster, get_zk): no_auth_connection.create("/some_allowed_node", b"data") # auth added, go on - no_auth_connection.add_auth('digest', 'user1:password1') + no_auth_connection.add_auth("digest", "user1:password1") for path in ["/test_no_acl", "/test_all_acl"]: no_auth_connection.set(path, b"auth_added") assert no_auth_connection.get(path)[0] == b"auth_added" @@ -131,62 +185,71 @@ def test_digest_auth_basic(started_cluster, get_zk): def test_super_auth(started_cluster): auth_connection = get_fake_zk() - auth_connection.add_auth('digest', 'user1:password1') + auth_connection.add_auth("digest", "user1:password1") auth_connection.create("/test_super_no_acl", b"") - auth_connection.create("/test_super_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + auth_connection.create( + "/test_super_all_acl", b"data", acl=[make_acl("auth", "", all=True)] + ) super_connection = get_fake_zk() - super_connection.add_auth('digest', 'super:admin') + super_connection.add_auth("digest", "super:admin") for path in ["/test_super_no_acl", "/test_super_all_acl"]: super_connection.set(path, b"value") assert super_connection.get(path)[0] == b"value" -@pytest.mark.parametrize( - ('get_zk'), - [ - get_genuine_zk, - get_fake_zk - ] -) +@pytest.mark.parametrize(("get_zk"), [get_genuine_zk, get_fake_zk]) def test_digest_auth_multiple(started_cluster, get_zk): auth_connection = get_zk() - auth_connection.add_auth('digest', 'user1:password1') - auth_connection.add_auth('digest', 'user2:password2') - auth_connection.add_auth('digest', 'user3:password3') + auth_connection.add_auth("digest", "user1:password1") + auth_connection.add_auth("digest", "user2:password2") + auth_connection.add_auth("digest", "user3:password3") - auth_connection.create("/test_multi_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + auth_connection.create( + "/test_multi_all_acl", b"data", acl=[make_acl("auth", "", all=True)] + ) one_auth_connection = get_zk() - one_auth_connection.add_auth('digest', 'user1:password1') + one_auth_connection.add_auth("digest", "user1:password1") one_auth_connection.set("/test_multi_all_acl", b"X") assert one_auth_connection.get("/test_multi_all_acl")[0] == b"X" other_auth_connection = get_zk() - other_auth_connection.add_auth('digest', 'user2:password2') + other_auth_connection.add_auth("digest", "user2:password2") other_auth_connection.set("/test_multi_all_acl", b"Y") assert other_auth_connection.get("/test_multi_all_acl")[0] == b"Y" -@pytest.mark.parametrize( - ('get_zk'), - [ - get_genuine_zk, - get_fake_zk - ] -) + +@pytest.mark.parametrize(("get_zk"), [get_genuine_zk, get_fake_zk]) def test_partial_auth(started_cluster, get_zk): auth_connection = get_zk() - auth_connection.add_auth('digest', 'user1:password1') + auth_connection.add_auth("digest", "user1:password1") - auth_connection.create("/test_partial_acl", b"data", acl=[make_acl("auth", "", read=False, write=True, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_partial_acl", + b"data", + acl=[ + make_acl( + "auth", "", read=False, write=True, create=True, delete=True, admin=True + ) + ], + ) auth_connection.set("/test_partial_acl", b"X") - auth_connection.create("/test_partial_acl/subnode", b"X", acl=[make_acl("auth", "", read=False, write=True, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_partial_acl/subnode", + b"X", + acl=[ + make_acl( + "auth", "", read=False, write=True, create=True, delete=True, admin=True + ) + ], + ) with pytest.raises(NoAuthError): auth_connection.get("/test_partial_acl") @@ -197,16 +260,40 @@ def test_partial_auth(started_cluster, get_zk): # exists works without read perm assert auth_connection.exists("/test_partial_acl") is not None - auth_connection.create("/test_partial_acl_create", b"data", acl=[make_acl("auth", "", read=True, write=True, create=False, delete=True, admin=True)]) + auth_connection.create( + "/test_partial_acl_create", + b"data", + acl=[ + make_acl( + "auth", "", read=True, write=True, create=False, delete=True, admin=True + ) + ], + ) with pytest.raises(NoAuthError): auth_connection.create("/test_partial_acl_create/subnode") - auth_connection.create("/test_partial_acl_set", b"data", acl=[make_acl("auth", "", read=True, write=False, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_partial_acl_set", + b"data", + acl=[ + make_acl( + "auth", "", read=True, write=False, create=True, delete=True, admin=True + ) + ], + ) with pytest.raises(NoAuthError): auth_connection.set("/test_partial_acl_set", b"X") # not allowed to delete child node - auth_connection.create("/test_partial_acl_delete", b"data", acl=[make_acl("auth", "", read=True, write=True, create=True, delete=False, admin=True)]) + auth_connection.create( + "/test_partial_acl_delete", + b"data", + acl=[ + make_acl( + "auth", "", read=True, write=True, create=True, delete=False, admin=True + ) + ], + ) auth_connection.create("/test_partial_acl_delete/subnode") with pytest.raises(NoAuthError): auth_connection.delete("/test_partial_acl_delete/subnode") @@ -216,85 +303,156 @@ def test_bad_auth(started_cluster): auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): - auth_connection.add_auth('world', 'anyone') + auth_connection.add_auth("world", "anyone") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 1") - auth_connection.add_auth('adssagf', 'user1:password1') + auth_connection.add_auth("adssagf", "user1:password1") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 2") - auth_connection.add_auth('digest', '') + auth_connection.add_auth("digest", "") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 3") - auth_connection.add_auth('', 'user1:password1') + auth_connection.add_auth("", "user1:password1") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 4") - auth_connection.add_auth('digest', 'user1') + auth_connection.add_auth("digest", "user1") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 5") - auth_connection.add_auth('digest', 'user1:password:otherpassword') + auth_connection.add_auth("digest", "user1:password:otherpassword") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 6") - auth_connection.add_auth('auth', 'user1:password') + auth_connection.add_auth("auth", "user1:password") auth_connection = get_fake_zk() with pytest.raises(AuthFailedError): print("Sending 7") - auth_connection.add_auth('world', 'somebody') + auth_connection.add_auth("world", "somebody") auth_connection = get_fake_zk() with pytest.raises(InvalidACLError): print("Sending 8") - auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("dasd", "", read=True, write=False, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_bad_acl", + b"data", + acl=[ + make_acl( + "dasd", + "", + read=True, + write=False, + create=True, + delete=True, + admin=True, + ) + ], + ) auth_connection = get_fake_zk() with pytest.raises(InvalidACLError): print("Sending 9") - auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("digest", "", read=True, write=False, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_bad_acl", + b"data", + acl=[ + make_acl( + "digest", + "", + read=True, + write=False, + create=True, + delete=True, + admin=True, + ) + ], + ) auth_connection = get_fake_zk() with pytest.raises(InvalidACLError): print("Sending 10") - auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("", "", read=True, write=False, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_bad_acl", + b"data", + acl=[ + make_acl( + "", "", read=True, write=False, create=True, delete=True, admin=True + ) + ], + ) auth_connection = get_fake_zk() with pytest.raises(InvalidACLError): print("Sending 11") - auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("digest", "dsdasda", read=True, write=False, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_bad_acl", + b"data", + acl=[ + make_acl( + "digest", + "dsdasda", + read=True, + write=False, + create=True, + delete=True, + admin=True, + ) + ], + ) auth_connection = get_fake_zk() with pytest.raises(InvalidACLError): print("Sending 12") - auth_connection.create("/test_bad_acl", b"data", acl=[make_acl("digest", "dsad:DSAa:d", read=True, write=False, create=True, delete=True, admin=True)]) + auth_connection.create( + "/test_bad_acl", + b"data", + acl=[ + make_acl( + "digest", + "dsad:DSAa:d", + read=True, + write=False, + create=True, + delete=True, + admin=True, + ) + ], + ) + def test_auth_snapshot(started_cluster): connection = get_fake_zk() - connection.add_auth('digest', 'user1:password1') + connection.add_auth("digest", "user1:password1") - connection.create("/test_snapshot_acl", b"data", acl=[make_acl("auth", "", all=True)]) + connection.create( + "/test_snapshot_acl", b"data", acl=[make_acl("auth", "", all=True)] + ) connection1 = get_fake_zk() - connection1.add_auth('digest', 'user2:password2') + connection1.add_auth("digest", "user2:password2") - connection1.create("/test_snapshot_acl1", b"data", acl=[make_acl("auth", "", all=True)]) + connection1.create( + "/test_snapshot_acl1", b"data", acl=[make_acl("auth", "", all=True)] + ) connection2 = get_fake_zk() connection2.create("/test_snapshot_acl2", b"data") for i in range(100): - connection.create(f"/test_snapshot_acl/path{i}", b"data", acl=[make_acl("auth", "", all=True)]) + connection.create( + f"/test_snapshot_acl/path{i}", b"data", acl=[make_acl("auth", "", all=True)] + ) node.restart_clickhouse() @@ -303,7 +461,7 @@ def test_auth_snapshot(started_cluster): with pytest.raises(NoAuthError): connection.get("/test_snapshot_acl") - connection.add_auth('digest', 'user1:password1') + connection.add_auth("digest", "user1:password1") assert connection.get("/test_snapshot_acl")[0] == b"data" @@ -316,14 +474,13 @@ def test_auth_snapshot(started_cluster): assert connection.get(f"/test_snapshot_acl/path{i}")[0] == b"data" connection1 = get_fake_zk() - connection1.add_auth('digest', 'user2:password2') + connection1.add_auth("digest", "user2:password2") assert connection1.get("/test_snapshot_acl1")[0] == b"data" with pytest.raises(NoAuthError): connection1.get("/test_snapshot_acl") - connection2 = get_fake_zk() assert connection2.get("/test_snapshot_acl2")[0] == b"data" with pytest.raises(NoAuthError): @@ -333,45 +490,55 @@ def test_auth_snapshot(started_cluster): connection2.get("/test_snapshot_acl1") -@pytest.mark.parametrize( - ('get_zk'), - [ - get_genuine_zk, - get_fake_zk - ] -) +@pytest.mark.parametrize(("get_zk"), [get_genuine_zk, get_fake_zk]) def test_get_set_acl(started_cluster, get_zk): auth_connection = get_zk() - auth_connection.add_auth('digest', 'username1:secret1') - auth_connection.add_auth('digest', 'username2:secret2') + auth_connection.add_auth("digest", "username1:secret1") + auth_connection.add_auth("digest", "username2:secret2") - auth_connection.create("/test_set_get_acl", b"data", acl=[make_acl("auth", "", all=True)]) + auth_connection.create( + "/test_set_get_acl", b"data", acl=[make_acl("auth", "", all=True)] + ) acls, stat = auth_connection.get_acls("/test_set_get_acl") assert stat.aversion == 0 assert len(acls) == 2 for acl in acls: - assert acl.acl_list == ['ALL'] - assert acl.id.scheme == 'digest' + assert acl.acl_list == ["ALL"] + assert acl.id.scheme == "digest" assert acl.perms == 31 - assert acl.id.id in ('username1:eGncMdBgOfGS/TCojt51xWsWv/Y=', 'username2:qgSSumukVlhftkVycylbHNvxhFU=') - + assert acl.id.id in ( + "username1:eGncMdBgOfGS/TCojt51xWsWv/Y=", + "username2:qgSSumukVlhftkVycylbHNvxhFU=", + ) other_auth_connection = get_zk() - other_auth_connection.add_auth('digest', 'username1:secret1') - other_auth_connection.add_auth('digest', 'username3:secret3') - other_auth_connection.set_acls("/test_set_get_acl", acls=[make_acl("auth", "", read=True, write=False, create=True, delete=True, admin=True)]) + other_auth_connection.add_auth("digest", "username1:secret1") + other_auth_connection.add_auth("digest", "username3:secret3") + other_auth_connection.set_acls( + "/test_set_get_acl", + acls=[ + make_acl( + "auth", "", read=True, write=False, create=True, delete=True, admin=True + ) + ], + ) acls, stat = other_auth_connection.get_acls("/test_set_get_acl") assert stat.aversion == 1 assert len(acls) == 2 for acl in acls: - assert acl.acl_list == ['READ', 'CREATE', 'DELETE', 'ADMIN'] - assert acl.id.scheme == 'digest' + assert acl.acl_list == ["READ", "CREATE", "DELETE", "ADMIN"] + assert acl.id.scheme == "digest" assert acl.perms == 29 - assert acl.id.id in ('username1:eGncMdBgOfGS/TCojt51xWsWv/Y=', 'username3:CvWITOxxTwk+u6S5PoGlQ4hNoWI=') + assert acl.id.id in ( + "username1:eGncMdBgOfGS/TCojt51xWsWv/Y=", + "username3:CvWITOxxTwk+u6S5PoGlQ4hNoWI=", + ) with pytest.raises(KazooException): - other_auth_connection.set_acls("/test_set_get_acl", acls=[make_acl("auth", "", all=True)], version=0) + other_auth_connection.set_acls( + "/test_set_get_acl", acls=[make_acl("auth", "", all=True)], version=0 + ) diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index f73b4671798..73fface02b4 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -7,16 +7,26 @@ import time from multiprocessing.dummy import Pool cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/enable_keeper.xml'], with_zookeeper=True, use_keeper=False) +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper.xml"], + with_zookeeper=True, + use_keeper=False, +) from kazoo.client import KazooClient, KazooState, KeeperState + def get_genuine_zk(): print("Zoo1", cluster.get_instance_ip("zoo1")) - return cluster.get_kazoo_client('zoo1') + return cluster.get_kazoo_client("zoo1") + def get_fake_zk(): print("node", cluster.get_instance_ip("node")) - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip("node") + ":9181", timeout=30.0) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip("node") + ":9181", timeout=30.0 + ) + def reset_last_zxid_listener(state): print("Fake zk callback called for state", state) nonlocal _fake_zk_instance @@ -27,14 +37,17 @@ def get_fake_zk(): _fake_zk_instance.start() return _fake_zk_instance + def random_string(length): - return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) + def create_random_path(prefix="", depth=1): if depth == 0: return prefix return create_random_path(os.path.join(prefix, random_string(3)), depth - 1) + def stop_zk(zk): try: if zk: @@ -82,7 +95,9 @@ def test_sequential_nodes(started_cluster): genuine_zk.create("/test_sequential_nodes") fake_zk.create("/test_sequential_nodes") for i in range(1, 11): - genuine_zk.create("/test_sequential_nodes/" + ("a" * i) + "-", sequence=True) + genuine_zk.create( + "/test_sequential_nodes/" + ("a" * i) + "-", sequence=True + ) genuine_zk.create("/test_sequential_nodes/" + ("b" * i)) fake_zk.create("/test_sequential_nodes/" + ("a" * i) + "-", sequence=True) fake_zk.create("/test_sequential_nodes/" + ("b" * i)) @@ -115,7 +130,9 @@ def test_sequential_nodes(started_cluster): assert genuine_throw == True assert fake_throw == True - genuine_childs_1 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_1"))) + genuine_childs_1 = list( + sorted(genuine_zk.get_children("/test_sequential_nodes_1")) + ) fake_childs_1 = list(sorted(fake_zk.get_children("/test_sequential_nodes_1"))) assert genuine_childs_1 == fake_childs_1 @@ -127,7 +144,9 @@ def test_sequential_nodes(started_cluster): genuine_zk.create("/test_sequential_nodes_2/node", sequence=True) fake_zk.create("/test_sequential_nodes_2/node", sequence=True) - genuine_childs_2 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_2"))) + genuine_childs_2 = list( + sorted(genuine_zk.get_children("/test_sequential_nodes_2")) + ) fake_childs_2 = list(sorted(fake_zk.get_children("/test_sequential_nodes_2"))) assert genuine_childs_2 == fake_childs_2 finally: @@ -139,10 +158,11 @@ def assert_eq_stats(stat1, stat2): assert stat1.version == stat2.version assert stat1.cversion == stat2.cversion assert stat1.aversion == stat2.aversion - assert stat1.aversion == stat2.aversion + assert stat1.ephemeralOwner == stat2.ephemeralOwner assert stat1.dataLength == stat2.dataLength assert stat1.numChildren == stat2.numChildren + def test_stats(started_cluster): try: genuine_zk = get_genuine_zk() @@ -162,10 +182,16 @@ def test_stats(started_cluster): fake_stats = fake_zk.exists("/test_stats_nodes") assert_eq_stats(genuine_stats, fake_stats) for i in range(1, 11): - print("/test_stats_nodes/" + ("a" * i) + "-" + "{:010d}".format((i - 1) * 2)) - genuine_zk.delete("/test_stats_nodes/" + ("a" * i) + "-" + "{:010d}".format((i - 1) * 2)) + print( + "/test_stats_nodes/" + ("a" * i) + "-" + "{:010d}".format((i - 1) * 2) + ) + genuine_zk.delete( + "/test_stats_nodes/" + ("a" * i) + "-" + "{:010d}".format((i - 1) * 2) + ) genuine_zk.delete("/test_stats_nodes/" + ("b" * i)) - fake_zk.delete("/test_stats_nodes/" + ("a" * i) + "-" + "{:010d}".format((i - 1) * 2)) + fake_zk.delete( + "/test_stats_nodes/" + ("a" * i) + "-" + "{:010d}".format((i - 1) * 2) + ) fake_zk.delete("/test_stats_nodes/" + ("b" * i)) genuine_stats = genuine_zk.exists("/test_stats_nodes") @@ -186,6 +212,7 @@ def test_stats(started_cluster): for zk in [genuine_zk, fake_zk]: stop_zk(zk) + def test_watchers(started_cluster): try: genuine_zk = get_genuine_zk() @@ -200,6 +227,7 @@ def test_watchers(started_cluster): genuine_data_watch_data = event fake_data_watch_data = None + def fake_callback(event): print("Fake data watch called") nonlocal fake_data_watch_data @@ -218,17 +246,18 @@ def test_watchers(started_cluster): print("Fake data", fake_data_watch_data) assert genuine_data_watch_data == fake_data_watch_data - genuine_zk.create("/test_data_watches/child", b"a") fake_zk.create("/test_data_watches/child", b"a") genuine_children = None + def genuine_child_callback(event): print("Genuine child watch called") nonlocal genuine_children genuine_children = event fake_children = None + def fake_child_callback(event): print("Fake child watch called") nonlocal fake_children @@ -262,32 +291,40 @@ def test_watchers(started_cluster): assert genuine_children == fake_children genuine_children_delete = None + def genuine_child_delete_callback(event): print("Genuine child watch called") nonlocal genuine_children_delete genuine_children_delete = event fake_children_delete = None + def fake_child_delete_callback(event): print("Fake child watch called") nonlocal fake_children_delete fake_children_delete = event genuine_child_delete = None + def genuine_own_delete_callback(event): print("Genuine child watch called") nonlocal genuine_child_delete genuine_child_delete = event fake_child_delete = None + def fake_own_delete_callback(event): print("Fake child watch called") nonlocal fake_child_delete fake_child_delete = event - genuine_zk.get_children("/test_data_watches", watch=genuine_child_delete_callback) + genuine_zk.get_children( + "/test_data_watches", watch=genuine_child_delete_callback + ) fake_zk.get_children("/test_data_watches", watch=fake_child_delete_callback) - genuine_zk.get_children("/test_data_watches/child", watch=genuine_own_delete_callback) + genuine_zk.get_children( + "/test_data_watches/child", watch=genuine_own_delete_callback + ) fake_zk.get_children("/test_data_watches/child", watch=fake_own_delete_callback) print("Calling genuine child delete") @@ -309,49 +346,59 @@ def test_watchers(started_cluster): for zk in [genuine_zk, fake_zk]: stop_zk(zk) + def test_multitransactions(started_cluster): try: genuine_zk = get_genuine_zk() fake_zk = get_fake_zk() for zk in [genuine_zk, fake_zk]: - zk.create('/test_multitransactions') + zk.create("/test_multitransactions") t = zk.transaction() - t.create('/test_multitransactions/freddy') - t.create('/test_multitransactions/fred', ephemeral=True) - t.create('/test_multitransactions/smith', sequence=True) + t.create("/test_multitransactions/freddy") + t.create("/test_multitransactions/fred", ephemeral=True) + t.create("/test_multitransactions/smith", sequence=True) results = t.commit() assert len(results) == 3 - assert results[0] == '/test_multitransactions/freddy' - assert results[2].startswith('/test_multitransactions/smith0') is True + assert results[0] == "/test_multitransactions/freddy" + assert results[2].startswith("/test_multitransactions/smith0") is True from kazoo.exceptions import RolledBackError, NoNodeError + for i, zk in enumerate([genuine_zk, fake_zk]): print("Processing ZK", i) t = zk.transaction() - t.create('/test_multitransactions/q') - t.delete('/test_multitransactions/a') - t.create('/test_multitransactions/x') + t.create("/test_multitransactions/q") + t.delete("/test_multitransactions/a") + t.create("/test_multitransactions/x") results = t.commit() print("Results", results) assert results[0].__class__ == RolledBackError assert results[1].__class__ == NoNodeError - assert zk.exists('/test_multitransactions/q') is None - assert zk.exists('/test_multitransactions/a') is None - assert zk.exists('/test_multitransactions/x') is None + assert zk.exists("/test_multitransactions/q") is None + assert zk.exists("/test_multitransactions/a") is None + assert zk.exists("/test_multitransactions/x") is None finally: for zk in [genuine_zk, fake_zk]: stop_zk(zk) + def exists(zk, path): result = zk.exists(path) return result is not None + def get(zk, path): result = zk.get(path) return result[0] + def get_children(zk, path): - return [elem for elem in list(sorted(zk.get_children(path))) if elem not in ('clickhouse', 'zookeeper')] + return [ + elem + for elem in list(sorted(zk.get_children(path))) + if elem not in ("clickhouse", "zookeeper") + ] + READ_REQUESTS = [ ("exists", exists), @@ -377,9 +424,8 @@ WRITE_REQUESTS = [ def delete(zk, path): zk.delete(path) -DELETE_REQUESTS = [ - ("delete", delete) -] + +DELETE_REQUESTS = [("delete", delete)] class Request(object): @@ -390,9 +436,10 @@ class Request(object): self.is_return = is_return def __str__(self): - arg_str = ', '.join([str(k) + "=" + str(v) for k, v in self.arguments.items()]) + arg_str = ", ".join([str(k) + "=" + str(v) for k, v in self.arguments.items()]) return "ZKRequest name {} with arguments {}".format(self.name, arg_str) + def generate_requests(prefix="/", iters=1): requests = [] existing_paths = [] @@ -404,18 +451,29 @@ def generate_requests(prefix="/", iters=1): path = create_random_path(path, 1) existing_paths.append(path) value = random_string(1000) - request = Request("create", {"path" : path, "value": value[0:10]}, lambda zk, path=path, value=value: create(zk, path, value), False) + request = Request( + "create", + {"path": path, "value": value[0:10]}, + lambda zk, path=path, value=value: create(zk, path, value), + False, + ) requests.append(request) for _ in range(100): path = random.choice(existing_paths) value = random_string(100) - request = Request("set", {"path": path, "value": value[0:10]}, lambda zk, path=path, value=value: set_data(zk, path, value), False) + request = Request( + "set", + {"path": path, "value": value[0:10]}, + lambda zk, path=path, value=value: set_data(zk, path, value), + False, + ) requests.append(request) for _ in range(100): path = random.choice(existing_paths) callback = random.choice(READ_REQUESTS) + def read_func1(zk, path=path, callback=callback): return callback[1](zk, path) @@ -424,13 +482,17 @@ def generate_requests(prefix="/", iters=1): for _ in range(30): path = random.choice(existing_paths) - request = Request("delete", {"path": path}, lambda zk, path=path: delete(zk, path), False) + request = Request( + "delete", {"path": path}, lambda zk, path=path: delete(zk, path), False + ) for _ in range(100): path = random.choice(existing_paths) callback = random.choice(READ_REQUESTS) + def read_func2(zk, path=path, callback=callback): return callback[1](zk, path) + request = Request(callback[0], {"path": path}, read_func2, True) requests.append(request) return requests @@ -463,15 +525,26 @@ def test_random_requests(started_cluster): print("Fake exception", str(ex)) fake_throw = True - assert fake_throw == genuine_throw, "Fake throw genuine not or vise versa request {}" + assert ( + fake_throw == genuine_throw + ), "Fake throw genuine not or vise versa request {}" assert fake_result == genuine_result, "Zookeeper results differ" - root_children_genuine = [elem for elem in list(sorted(genuine_zk.get_children("/test_random_requests"))) if elem not in ('clickhouse', 'zookeeper')] - root_children_fake = [elem for elem in list(sorted(fake_zk.get_children("/test_random_requests"))) if elem not in ('clickhouse', 'zookeeper')] + root_children_genuine = [ + elem + for elem in list(sorted(genuine_zk.get_children("/test_random_requests"))) + if elem not in ("clickhouse", "zookeeper") + ] + root_children_fake = [ + elem + for elem in list(sorted(fake_zk.get_children("/test_random_requests"))) + if elem not in ("clickhouse", "zookeeper") + ] assert root_children_fake == root_children_genuine finally: for zk in [genuine_zk, fake_zk]: stop_zk(zk) + def test_end_of_session(started_cluster): fake_zk1 = None @@ -484,20 +557,22 @@ def test_end_of_session(started_cluster): fake_zk1.start() fake_zk2 = KazooClient(hosts=cluster.get_instance_ip("node") + ":9181") fake_zk2.start() - genuine_zk1 = cluster.get_kazoo_client('zoo1') + genuine_zk1 = cluster.get_kazoo_client("zoo1") genuine_zk1.start() - genuine_zk2 = cluster.get_kazoo_client('zoo1') + genuine_zk2 = cluster.get_kazoo_client("zoo1") genuine_zk2.start() fake_zk1.create("/test_end_of_session") genuine_zk1.create("/test_end_of_session") fake_ephemeral_event = None + def fake_ephemeral_callback(event): print("Fake watch triggered") nonlocal fake_ephemeral_event fake_ephemeral_event = event genuine_ephemeral_event = None + def genuine_ephemeral_callback(event): print("Genuine watch triggered") nonlocal genuine_ephemeral_event @@ -509,8 +584,18 @@ def test_end_of_session(started_cluster): fake_zk1.create("/test_end_of_session/ephemeral_node", ephemeral=True) genuine_zk1.create("/test_end_of_session/ephemeral_node", ephemeral=True) - assert fake_zk2.exists("/test_end_of_session/ephemeral_node", watch=fake_ephemeral_callback) is not None - assert genuine_zk2.exists("/test_end_of_session/ephemeral_node", watch=genuine_ephemeral_callback) is not None + assert ( + fake_zk2.exists( + "/test_end_of_session/ephemeral_node", watch=fake_ephemeral_callback + ) + is not None + ) + assert ( + genuine_zk2.exists( + "/test_end_of_session/ephemeral_node", watch=genuine_ephemeral_callback + ) + is not None + ) print("Stopping genuine zk") genuine_zk1.stop() @@ -531,6 +616,7 @@ def test_end_of_session(started_cluster): for zk in [fake_zk1, fake_zk2, genuine_zk1, genuine_zk2]: stop_zk(zk) + def test_end_of_watches_session(started_cluster): fake_zk1 = None fake_zk2 = None @@ -544,6 +630,7 @@ def test_end_of_watches_session(started_cluster): fake_zk1.create("/test_end_of_watches_session") dummy_set = 0 + def dummy_callback(event): nonlocal dummy_set dummy_set += 1 @@ -551,22 +638,35 @@ def test_end_of_watches_session(started_cluster): for child_node in range(100): fake_zk1.create("/test_end_of_watches_session/" + str(child_node)) - fake_zk1.get_children("/test_end_of_watches_session/" + str(child_node), watch=dummy_callback) + fake_zk1.get_children( + "/test_end_of_watches_session/" + str(child_node), watch=dummy_callback + ) - fake_zk2.get_children("/test_end_of_watches_session/" + str(0), watch=dummy_callback) - fake_zk2.get_children("/test_end_of_watches_session/" + str(1), watch=dummy_callback) + fake_zk2.get_children( + "/test_end_of_watches_session/" + str(0), watch=dummy_callback + ) + fake_zk2.get_children( + "/test_end_of_watches_session/" + str(1), watch=dummy_callback + ) fake_zk1.stop() fake_zk1.close() for child_node in range(100): - fake_zk2.create("/test_end_of_watches_session/" + str(child_node) + "/" + str(child_node), b"somebytes") + fake_zk2.create( + "/test_end_of_watches_session/" + + str(child_node) + + "/" + + str(child_node), + b"somebytes", + ) assert dummy_set == 2 finally: for zk in [fake_zk1, fake_zk2]: stop_zk(zk) + def test_concurrent_watches(started_cluster): try: fake_zk = get_fake_zk() @@ -580,6 +680,7 @@ def test_concurrent_watches(started_cluster): existing_path = [] all_paths_created = [] watches_created = 0 + def create_path_and_watch(i): nonlocal watches_created nonlocal all_paths_created @@ -597,6 +698,7 @@ def test_concurrent_watches(started_cluster): existing_path.append(i) trigger_called = 0 + def trigger_watch(i): nonlocal trigger_called trigger_called += 1 diff --git a/tests/integration/test_keeper_four_word_command/configs/enable_keeper1.xml b/tests/integration/test_keeper_four_word_command/configs/enable_keeper1.xml index fcb3553bb98..095bb8a9530 100644 --- a/tests/integration/test_keeper_four_word_command/configs/enable_keeper1.xml +++ b/tests/integration/test_keeper_four_word_command/configs/enable_keeper1.xml @@ -4,7 +4,7 @@ 1 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots - * + * 5000 diff --git a/tests/integration/test_keeper_four_word_command/configs/enable_keeper2.xml b/tests/integration/test_keeper_four_word_command/configs/enable_keeper2.xml index 65011cd2637..33ca15c227d 100644 --- a/tests/integration/test_keeper_four_word_command/configs/enable_keeper2.xml +++ b/tests/integration/test_keeper_four_word_command/configs/enable_keeper2.xml @@ -4,7 +4,7 @@ 2 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots - * + * 5000 diff --git a/tests/integration/test_keeper_four_word_command/configs/enable_keeper3.xml b/tests/integration/test_keeper_four_word_command/configs/enable_keeper3.xml index a0d8c99f3d8..2a3f0b3c279 100644 --- a/tests/integration/test_keeper_four_word_command/configs/enable_keeper3.xml +++ b/tests/integration/test_keeper_four_word_command/configs/enable_keeper3.xml @@ -4,7 +4,7 @@ 3 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots - * + * 5000 diff --git a/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list.xml b/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_allow_list.xml similarity index 88% rename from tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list.xml rename to tests/integration/test_keeper_four_word_command/configs/keeper_config_with_allow_list.xml index c8dbf4a0ff2..feafd3f6b44 100644 --- a/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list.xml +++ b/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_allow_list.xml @@ -2,7 +2,7 @@ 9181 1 - ruok, conf + ruok, conf 1 diff --git a/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list_all.xml b/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_allow_list_all.xml similarity index 90% rename from tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list_all.xml rename to tests/integration/test_keeper_four_word_command/configs/keeper_config_with_allow_list_all.xml index e85e302f922..523e6b2fa27 100644 --- a/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list_all.xml +++ b/tests/integration/test_keeper_four_word_command/configs/keeper_config_with_allow_list_all.xml @@ -2,7 +2,7 @@ 9181 3 - * + * 1 diff --git a/tests/integration/test_keeper_four_word_command/configs/keeper_config_without_white_list.xml b/tests/integration/test_keeper_four_word_command/configs/keeper_config_without_allow_list.xml similarity index 100% rename from tests/integration/test_keeper_four_word_command/configs/keeper_config_without_white_list.xml rename to tests/integration/test_keeper_four_word_command/configs/keeper_config_without_allow_list.xml diff --git a/tests/integration/test_keeper_four_word_command/test.py b/tests/integration/test_keeper_four_word_command/test.py index 1887b18655a..caaf1d0c87a 100644 --- a/tests/integration/test_keeper_four_word_command/test.py +++ b/tests/integration/test_keeper_four_word_command/test.py @@ -13,12 +13,15 @@ import csv import re cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], - stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'], - stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml'], - stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/enable_keeper2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/enable_keeper3.xml"], stay_alive=True +) from kazoo.client import KazooClient, KazooState @@ -47,9 +50,9 @@ def clear_znodes(): zk = None try: zk = get_fake_zk(node3.name, timeout=30.0) - nodes = zk.get_children('/') - for node in [n for n in nodes if 'test_4lw_' in n]: - zk.delete('/' + node) + nodes = zk.get_children("/") + for node in [n for n in nodes if "test_4lw_" in n]: + zk.delete("/" + node) finally: destroy_zk_client(zk) @@ -77,7 +80,9 @@ def wait_nodes(): def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -99,14 +104,14 @@ def reset_node_stats(node_name=node1.name): client = None try: client = get_keeper_socket(node_name) - client.send(b'srst') + client.send(b"srst") client.recv(10) finally: if client is not None: client.close() -def send_4lw_cmd(node_name=node1.name, cmd='ruok'): +def send_4lw_cmd(node_name=node1.name, cmd="ruok"): client = None try: client = get_keeper_socket(node_name) @@ -123,7 +128,7 @@ def reset_conn_stats(node_name=node1.name): client = None try: client = get_keeper_socket(node_name) - client.send(b'crst') + client.send(b"crst") client.recv(10_000) finally: if client is not None: @@ -134,13 +139,15 @@ def test_cmd_ruok(started_cluster): client = None try: wait_nodes() - data = send_4lw_cmd(cmd='ruok') - assert data == 'imok' + data = send_4lw_cmd(cmd="ruok") + assert data == "imok" finally: close_keeper_socket(client) -def do_some_action(zk, create_cnt=0, get_cnt=0, set_cnt=0, ephemeral_cnt=0, watch_cnt=0, delete_cnt=0): +def do_some_action( + zk, create_cnt=0, get_cnt=0, set_cnt=0, ephemeral_cnt=0, watch_cnt=0, delete_cnt=0 +): assert create_cnt >= get_cnt assert create_cnt >= set_cnt assert create_cnt >= watch_cnt @@ -184,12 +191,20 @@ def test_cmd_mntr(started_cluster): reset_node_stats(node1.name) zk = get_fake_zk(node1.name, timeout=30.0) - do_some_action(zk, create_cnt=10, get_cnt=10, set_cnt=5, ephemeral_cnt=2, watch_cnt=2, delete_cnt=2) + do_some_action( + zk, + create_cnt=10, + get_cnt=10, + set_cnt=5, + ephemeral_cnt=2, + watch_cnt=2, + delete_cnt=2, + ) - data = send_4lw_cmd(cmd='mntr') + data = send_4lw_cmd(cmd="mntr") # print(data.decode()) - reader = csv.reader(data.split('\n'), delimiter='\t') + reader = csv.reader(data.split("\n"), delimiter="\t") result = {} for row in reader: @@ -205,7 +220,6 @@ def test_cmd_mntr(started_cluster): assert int(result["zk_min_latency"]) <= int(result["zk_avg_latency"]) assert int(result["zk_max_latency"]) >= int(result["zk_avg_latency"]) - assert int(result["zk_num_alive_connections"]) == 1 assert int(result["zk_outstanding_requests"]) == 0 @@ -239,14 +253,14 @@ def test_cmd_srst(started_cluster): wait_nodes() clear_znodes() - data = send_4lw_cmd(cmd='srst') + data = send_4lw_cmd(cmd="srst") assert data.strip() == "Server stats reset." - data = send_4lw_cmd(cmd='mntr') + data = send_4lw_cmd(cmd="mntr") assert len(data) != 0 # print(data) - reader = csv.reader(data.split('\n'), delimiter='\t') + reader = csv.reader(data.split("\n"), delimiter="\t") result = {} for row in reader: @@ -266,9 +280,9 @@ def test_cmd_conf(started_cluster): wait_nodes() clear_znodes() - data = send_4lw_cmd(cmd='conf') + data = send_4lw_cmd(cmd="conf") - reader = csv.reader(data.split('\n'), delimiter='=') + reader = csv.reader(data.split("\n"), delimiter="=") result = {} for row in reader: @@ -281,9 +295,12 @@ def test_cmd_conf(started_cluster): assert "tcp_port_secure" not in result assert "superdigest" not in result - assert result["four_letter_word_white_list"] == "*" + assert result["four_letter_word_allow_list"] == "*" assert result["log_storage_path"] == "/var/lib/clickhouse/coordination/log" - assert result["snapshot_storage_path"] == "/var/lib/clickhouse/coordination/snapshots" + assert ( + result["snapshot_storage_path"] + == "/var/lib/clickhouse/coordination/snapshots" + ) assert result["session_timeout_ms"] == "30000" assert result["min_session_timeout_ms"] == "10000" @@ -319,8 +336,8 @@ def test_cmd_conf(started_cluster): def test_cmd_isro(started_cluster): wait_nodes() - assert send_4lw_cmd(node1.name, 'isro') == 'rw' - assert send_4lw_cmd(node2.name, 'isro') == 'ro' + assert send_4lw_cmd(node1.name, "isro") == "rw" + assert send_4lw_cmd(node2.name, "isro") == "ro" def test_cmd_srvr(started_cluster): @@ -334,26 +351,26 @@ def test_cmd_srvr(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=10) - data = send_4lw_cmd(cmd='srvr') + data = send_4lw_cmd(cmd="srvr") print("srvr output -------------------------------------") print(data) - reader = csv.reader(data.split('\n'), delimiter=':') + reader = csv.reader(data.split("\n"), delimiter=":") result = {} for row in reader: if len(row) != 0: result[row[0].strip()] = row[1].strip() - assert 'ClickHouse Keeper version' in result - assert 'Latency min/avg/max' in result - assert result['Received'] == '10' - assert result['Sent'] == '10' - assert int(result['Connections']) == 1 - assert int(result['Zxid']) > 14 - assert result['Mode'] == 'leader' - assert result['Node count'] == '11' + assert "ClickHouse Keeper version" in result + assert "Latency min/avg/max" in result + assert result["Received"] == "10" + assert result["Sent"] == "10" + assert int(result["Connections"]) == 1 + assert int(result["Zxid"]) > 14 + assert result["Mode"] == "leader" + assert result["Node count"] == "11" finally: destroy_zk_client(zk) @@ -370,45 +387,45 @@ def test_cmd_stat(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=10) - data = send_4lw_cmd(cmd='stat') + data = send_4lw_cmd(cmd="stat") print("stat output -------------------------------------") print(data) # keeper statistics - stats = [n for n in data.split('\n') if '=' not in n] - reader = csv.reader(stats, delimiter=':') + stats = [n for n in data.split("\n") if "=" not in n] + reader = csv.reader(stats, delimiter=":") result = {} for row in reader: if len(row) != 0: result[row[0].strip()] = row[1].strip() - assert 'ClickHouse Keeper version' in result - assert 'Latency min/avg/max' in result - assert result['Received'] == '10' - assert result['Sent'] == '10' - assert int(result['Connections']) == 1 - assert int(result['Zxid']) > 14 - assert result['Mode'] == 'leader' - assert result['Node count'] == '11' + assert "ClickHouse Keeper version" in result + assert "Latency min/avg/max" in result + assert result["Received"] == "10" + assert result["Sent"] == "10" + assert int(result["Connections"]) == 1 + assert int(result["Zxid"]) > 14 + assert result["Mode"] == "leader" + assert result["Node count"] == "11" # filter connection statistics - cons = [n for n in data.split('\n') if '=' in n] + cons = [n for n in data.split("\n") if "=" in n] # filter connection created by 'cons' - cons = [n for n in cons if 'recved=0' not in n and len(n) > 0] + cons = [n for n in cons if "recved=0" not in n and len(n) > 0] assert len(cons) == 1 - conn_stat = re.match(r'(.*?)[:].*[(](.*?)[)].*', cons[0].strip(), re.S).group(2) + conn_stat = re.match(r"(.*?)[:].*[(](.*?)[)].*", cons[0].strip(), re.S).group(2) assert conn_stat is not None result = {} - for col in conn_stat.split(','): - col = col.strip().split('=') + for col in conn_stat.split(","): + col = col.strip().split("=") result[col[0]] = col[1] - assert result['recved'] == '10' - assert result['sent'] == '10' + assert result["recved"] == "10" + assert result["sent"] == "10" finally: destroy_zk_client(zk) @@ -424,36 +441,36 @@ def test_cmd_cons(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=10) - data = send_4lw_cmd(cmd='cons') + data = send_4lw_cmd(cmd="cons") print("cons output -------------------------------------") print(data) # filter connection created by 'cons' - cons = [n for n in data.split('\n') if 'recved=0' not in n and len(n) > 0] + cons = [n for n in data.split("\n") if "recved=0" not in n and len(n) > 0] assert len(cons) == 1 - conn_stat = re.match(r'(.*?)[:].*[(](.*?)[)].*', cons[0].strip(), re.S).group(2) + conn_stat = re.match(r"(.*?)[:].*[(](.*?)[)].*", cons[0].strip(), re.S).group(2) assert conn_stat is not None result = {} - for col in conn_stat.split(','): - col = col.strip().split('=') + for col in conn_stat.split(","): + col = col.strip().split("=") result[col[0]] = col[1] - assert result['recved'] == '10' - assert result['sent'] == '10' - assert 'sid' in result - assert result['lop'] == 'Create' - assert 'est' in result - assert result['to'] == '30000' - assert result['lcxid'] == '0x000000000000000a' - assert 'lzxid' in result - assert 'lresp' in result - assert int(result['llat']) >= 0 - assert int(result['minlat']) >= 0 - assert int(result['avglat']) >= 0 - assert int(result['maxlat']) >= 0 + assert result["recved"] == "10" + assert result["sent"] == "10" + assert "sid" in result + assert result["lop"] == "Create" + assert "est" in result + assert result["to"] == "30000" + assert result["lcxid"] == "0x000000000000000a" + assert "lzxid" in result + assert "lresp" in result + assert int(result["llat"]) >= 0 + assert int(result["minlat"]) >= 0 + assert int(result["avglat"]) >= 0 + assert int(result["maxlat"]) >= 0 finally: destroy_zk_client(zk) @@ -469,43 +486,43 @@ def test_cmd_crst(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=10) - data = send_4lw_cmd(cmd='crst') + data = send_4lw_cmd(cmd="crst") print("crst output -------------------------------------") print(data) - data = send_4lw_cmd(cmd='cons') + data = send_4lw_cmd(cmd="cons") print("cons output(after crst) -------------------------------------") print(data) # 2 connections, 1 for 'cons' command, 1 for zk - cons = [n for n in data.split('\n') if len(n) > 0] + cons = [n for n in data.split("\n") if len(n) > 0] assert len(cons) == 2 # connection for zk - zk_conn = [n for n in cons if not n.__contains__('sid=0xffffffffffffffff')][0] + zk_conn = [n for n in cons if not n.__contains__("sid=0xffffffffffffffff")][0] - conn_stat = re.match(r'(.*?)[:].*[(](.*?)[)].*', zk_conn.strip(), re.S).group(2) + conn_stat = re.match(r"(.*?)[:].*[(](.*?)[)].*", zk_conn.strip(), re.S).group(2) assert conn_stat is not None result = {} - for col in conn_stat.split(','): - col = col.strip().split('=') + for col in conn_stat.split(","): + col = col.strip().split("=") result[col[0]] = col[1] - assert result['recved'] == '0' - assert result['sent'] == '0' - assert 'sid' in result - assert result['lop'] == 'NA' - assert 'est' in result - assert result['to'] == '30000' - assert 'lcxid' not in result - assert result['lzxid'] == '0xffffffffffffffff' - assert result['lresp'] == '0' - assert int(result['llat']) == 0 - assert int(result['minlat']) == 0 - assert int(result['avglat']) == 0 - assert int(result['maxlat']) == 0 + assert result["recved"] == "0" + assert result["sent"] == "0" + assert "sid" in result + assert result["lop"] == "NA" + assert "est" in result + assert result["to"] == "30000" + assert "lcxid" not in result + assert result["lzxid"] == "0xffffffffffffffff" + assert result["lresp"] == "0" + assert int(result["llat"]) == 0 + assert int(result["minlat"]) == 0 + assert int(result["avglat"]) == 0 + assert int(result["maxlat"]) == 0 finally: destroy_zk_client(zk) @@ -521,18 +538,18 @@ def test_cmd_dump(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, ephemeral_cnt=2) - data = send_4lw_cmd(cmd='dump') + data = send_4lw_cmd(cmd="dump") print("dump output -------------------------------------") print(data) - list_data = data.split('\n') + list_data = data.split("\n") - session_count = int(re.match(r'.*[(](.*?)[)].*', list_data[0], re.S).group(1)) + session_count = int(re.match(r".*[(](.*?)[)].*", list_data[0], re.S).group(1)) assert session_count == 1 - assert '\t' + '/test_4lw_ephemeral_node_0' in list_data - assert '\t' + '/test_4lw_ephemeral_node_1' in list_data + assert "\t" + "/test_4lw_ephemeral_node_0" in list_data + assert "\t" + "/test_4lw_ephemeral_node_1" in list_data finally: destroy_zk_client(zk) @@ -547,19 +564,23 @@ def test_cmd_wchs(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=2, watch_cnt=2) - data = send_4lw_cmd(cmd='wchs') + data = send_4lw_cmd(cmd="wchs") print("wchs output -------------------------------------") print(data) - list_data = [n for n in data.split('\n') if len(n.strip()) > 0] + list_data = [n for n in data.split("\n") if len(n.strip()) > 0] # 37 connections watching 632141 paths # Total watches:632141 - matcher = re.match(r'([0-9].*) connections watching ([0-9].*) paths', list_data[0], re.S) + matcher = re.match( + r"([0-9].*) connections watching ([0-9].*) paths", list_data[0], re.S + ) conn_count = int(matcher.group(1)) watch_path_count = int(matcher.group(2)) - watch_count = int(re.match(r'Total watches:([0-9].*)', list_data[1], re.S).group(1)) + watch_count = int( + re.match(r"Total watches:([0-9].*)", list_data[1], re.S).group(1) + ) assert conn_count == 1 assert watch_path_count == 2 @@ -578,16 +599,16 @@ def test_cmd_wchc(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=2, watch_cnt=2) - data = send_4lw_cmd(cmd='wchc') + data = send_4lw_cmd(cmd="wchc") print("wchc output -------------------------------------") print(data) - list_data = [n for n in data.split('\n') if len(n.strip()) > 0] + list_data = [n for n in data.split("\n") if len(n.strip()) > 0] assert len(list_data) == 3 - assert '\t' + '/test_4lw_normal_node_0' in list_data - assert '\t' + '/test_4lw_normal_node_1' in list_data + assert "\t" + "/test_4lw_normal_node_0" in list_data + assert "\t" + "/test_4lw_normal_node_1" in list_data finally: destroy_zk_client(zk) @@ -602,16 +623,15 @@ def test_cmd_wchp(started_cluster): zk = get_fake_zk(node1.name, timeout=30.0) do_some_action(zk, create_cnt=2, watch_cnt=2) - data = send_4lw_cmd(cmd='wchp') + data = send_4lw_cmd(cmd="wchp") print("wchp output -------------------------------------") print(data) - list_data = [n for n in data.split('\n') if len(n.strip()) > 0] + list_data = [n for n in data.split("\n") if len(n.strip()) > 0] assert len(list_data) == 4 - assert '/test_4lw_normal_node_0' in list_data - assert '/test_4lw_normal_node_1' in list_data + assert "/test_4lw_normal_node_0" in list_data + assert "/test_4lw_normal_node_1" in list_data finally: destroy_zk_client(zk) - diff --git a/tests/integration/test_keeper_four_word_command/test_white_list.py b/tests/integration/test_keeper_four_word_command/test_allow_list.py similarity index 67% rename from tests/integration/test_keeper_four_word_command/test_white_list.py rename to tests/integration/test_keeper_four_word_command/test_allow_list.py index f0428147257..026bd1d59af 100644 --- a/tests/integration/test_keeper_four_word_command/test_white_list.py +++ b/tests/integration/test_keeper_four_word_command/test_allow_list.py @@ -3,10 +3,20 @@ import pytest from helpers.cluster import ClickHouseCluster import time -cluster = ClickHouseCluster(__file__, name="test_keeper_4lw_white_list") -node1 = cluster.add_instance('node1', main_configs=['configs/keeper_config_with_white_list.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/keeper_config_without_white_list.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/keeper_config_with_white_list_all.xml'], stay_alive=True) +cluster = ClickHouseCluster(__file__, name="test_keeper_4lw_allow_list") +node1 = cluster.add_instance( + "node1", main_configs=["configs/keeper_config_with_allow_list.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/keeper_config_without_allow_list.xml"], + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/keeper_config_with_allow_list_all.xml"], + stay_alive=True, +) from kazoo.client import KazooClient, KazooState @@ -62,7 +72,9 @@ def get_keeper_socket(nodename): def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -73,7 +85,7 @@ def close_keeper_socket(cli): cli.close() -def send_cmd(node_name, command = "ruok"): +def send_cmd(node_name, command="ruok"): client = None try: wait_nodes() @@ -85,13 +97,13 @@ def send_cmd(node_name, command = "ruok"): close_keeper_socket(client) -def test_white_list(started_cluster): +def test_allow_list(started_cluster): client = None try: wait_nodes() - assert send_cmd(node1.name) == 'imok' - assert send_cmd(node1.name, command = 'mntr') == '' - assert send_cmd(node2.name) == 'imok' - assert send_cmd(node3.name) == 'imok' + assert send_cmd(node1.name) == "imok" + assert send_cmd(node1.name, command="mntr") == "" + assert send_cmd(node2.name) == "imok" + assert send_cmd(node3.name) == "imok" finally: close_keeper_socket(client) diff --git a/tests/integration/test_keeper_incorrect_config/test.py b/tests/integration/test_keeper_incorrect_config/test.py index 4ab6b87d853..52c76c84e23 100644 --- a/tests/integration/test_keeper_incorrect_config/test.py +++ b/tests/integration/test_keeper_incorrect_config/test.py @@ -4,7 +4,10 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) + @pytest.fixture(scope="module") def started_cluster(): @@ -102,18 +105,25 @@ NORMAL_CONFIG = """ """ + def test_duplicate_endpoint(started_cluster): node1.stop_clickhouse() - node1.replace_config("/etc/clickhouse-server/config.d/enable_keeper1.xml", DUPLICATE_ENDPOINT_CONFIG) + node1.replace_config( + "/etc/clickhouse-server/config.d/enable_keeper1.xml", DUPLICATE_ENDPOINT_CONFIG + ) with pytest.raises(Exception): node1.start_clickhouse(start_wait_sec=10) - node1.replace_config("/etc/clickhouse-server/config.d/enable_keeper1.xml", DUPLICATE_ID_CONFIG) + node1.replace_config( + "/etc/clickhouse-server/config.d/enable_keeper1.xml", DUPLICATE_ID_CONFIG + ) with pytest.raises(Exception): node1.start_clickhouse(start_wait_sec=10) - node1.replace_config("/etc/clickhouse-server/config.d/enable_keeper1.xml", NORMAL_CONFIG) + node1.replace_config( + "/etc/clickhouse-server/config.d/enable_keeper1.xml", NORMAL_CONFIG + ) node1.start_clickhouse() assert node1.query("SELECT 1") == "1\n" diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index b4bf62f9a37..2d45e95e4ff 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -8,12 +8,40 @@ import os import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_secure_keeper1.xml', 'configs/ssl_conf.xml', 'configs/server.crt', 'configs/server.key', 'configs/rootCA.pem']) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_secure_keeper2.xml', 'configs/ssl_conf.xml', 'configs/server.crt', 'configs/server.key', 'configs/rootCA.pem']) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_secure_keeper3.xml', 'configs/ssl_conf.xml', 'configs/server.crt', 'configs/server.key', 'configs/rootCA.pem']) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/enable_secure_keeper1.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + "configs/rootCA.pem", + ], +) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/enable_secure_keeper2.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + "configs/rootCA.pem", + ], +) +node3 = cluster.add_instance( + "node3", + main_configs=[ + "configs/enable_secure_keeper3.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + "configs/rootCA.pem", + ], +) from kazoo.client import KazooClient, KazooState + @pytest.fixture(scope="module") def started_cluster(): try: @@ -24,11 +52,15 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def test_secure_raft_works(started_cluster): try: node1_zk = get_fake_zk("node1") diff --git a/tests/integration/test_keeper_multinode_blocade_leader/test.py b/tests/integration/test_keeper_multinode_blocade_leader/test.py index 2101c2a973f..c2d4039e122 100644 --- a/tests/integration/test_keeper_multinode_blocade_leader/test.py +++ b/tests/integration/test_keeper_multinode_blocade_leader/test.py @@ -9,9 +9,21 @@ from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml', 'configs/use_keeper.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml', 'configs/use_keeper.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml', 'configs/use_keeper.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/enable_keeper1.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/enable_keeper2.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/enable_keeper3.xml", "configs/use_keeper.xml"], + stay_alive=True, +) from kazoo.client import KazooClient, KazooState @@ -27,6 +39,7 @@ TODO find (or write) not so smart python client. TODO remove this when jepsen tests will be written. """ + @pytest.fixture(scope="module") def started_cluster(): try: @@ -37,8 +50,10 @@ def started_cluster(): finally: cluster.shutdown() + def smaller_exception(ex): - return '\n'.join(str(ex).split('\n')[0:2]) + return "\n".join(str(ex).split("\n")[0:2]) + def wait_node(node): for _ in range(100): @@ -59,13 +74,16 @@ def wait_node(node): else: raise Exception("Can't wait node", node.name, "to become ready") + def wait_nodes(): for node in [node1, node2, node3]: wait_node(node) def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -78,7 +96,11 @@ def test_blocade_leader(started_cluster): try: for i, node in enumerate([node1, node2, node3]): node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary") - node.query("CREATE TABLE IF NOT EXISTS ordinary.t1 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t1', '{}') ORDER BY tuple()".format(i + 1)) + node.query( + "CREATE TABLE IF NOT EXISTS ordinary.t1 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t1', '{}') ORDER BY tuple()".format( + i + 1 + ) + ) break except Exception as ex: print("Got exception from node", smaller_exception(ex)) @@ -99,7 +121,9 @@ def test_blocade_leader(started_cluster): for i in range(100): try: - restart_replica_for_sure(node2, "ordinary.t1", "/clickhouse/t1/replicas/2") + restart_replica_for_sure( + node2, "ordinary.t1", "/clickhouse/t1/replicas/2" + ) node2.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100)") break except Exception as ex: @@ -111,12 +135,16 @@ def test_blocade_leader(started_cluster): time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t1', '/clickhouse/t1/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t1", "/clickhouse/t1/replicas/{}".format(num + 1) + ) assert False, "Cannot insert anything node2" for i in range(100): try: - restart_replica_for_sure(node3, "ordinary.t1", "/clickhouse/t1/replicas/3") + restart_replica_for_sure( + node3, "ordinary.t1", "/clickhouse/t1/replicas/3" + ) node3.query("INSERT INTO ordinary.t1 SELECT rand() FROM numbers(100)") break except Exception as ex: @@ -128,19 +156,26 @@ def test_blocade_leader(started_cluster): time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t1', '/clickhouse/t1/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t1", "/clickhouse/t1/replicas/{}".format(num + 1) + ) assert False, "Cannot insert anything node3" for n, node in enumerate([node1, node2, node3]): for i in range(100): try: - restart_replica_for_sure(node, "ordinary.t1", "/clickhouse/t1/replicas/{}".format(n + 1)) + restart_replica_for_sure( + node, "ordinary.t1", "/clickhouse/t1/replicas/{}".format(n + 1) + ) break except Exception as ex: try: node.query("ATTACH TABLE ordinary.t1") except Exception as attach_ex: - print("Got exception node{}".format(n + 1), smaller_exception(attach_ex)) + print( + "Got exception node{}".format(n + 1), + smaller_exception(attach_ex), + ) print("Got exception node{}".format(n + 1), smaller_exception(ex)) time.sleep(0.5) @@ -156,31 +191,42 @@ def test_blocade_leader(started_cluster): time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t1', '/clickhouse/t1/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t1", "/clickhouse/t1/replicas/{}".format(num + 1) + ) assert False, "Cannot insert anything node1" for n, node in enumerate([node1, node2, node3]): for i in range(100): try: - restart_replica_for_sure(node, "ordinary.t1", "/clickhouse/t1/replicas/{}".format(n + 1)) + restart_replica_for_sure( + node, "ordinary.t1", "/clickhouse/t1/replicas/{}".format(n + 1) + ) node.query("SYSTEM SYNC REPLICA ordinary.t1", timeout=10) break except Exception as ex: try: node.query("ATTACH TABLE ordinary.t1") except Exception as attach_ex: - print("Got exception node{}".format(n + 1), smaller_exception(attach_ex)) + print( + "Got exception node{}".format(n + 1), + smaller_exception(attach_ex), + ) print("Got exception node{}".format(n + 1), smaller_exception(ex)) time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t1', '/clickhouse/t1/replicas/{}'.format(num + 1)) - assert False, "Cannot sync replica node{}".format(n+1) + dump_zk( + node, "/clickhouse/t1", "/clickhouse/t1/replicas/{}".format(num + 1) + ) + assert False, "Cannot sync replica node{}".format(n + 1) if node1.query("SELECT COUNT() FROM ordinary.t1") != "310\n": for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t1', '/clickhouse/t1/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t1", "/clickhouse/t1/replicas/{}".format(num + 1) + ) assert_eq_with_retry(node1, "SELECT COUNT() FROM ordinary.t1", "310") assert_eq_with_retry(node2, "SELECT COUNT() FROM ordinary.t1", "310") @@ -192,13 +238,38 @@ def dump_zk(node, zk_path, replica_path): print("Replicas") print(node.query("SELECT * FROM system.replicas FORMAT Vertical")) print("Replica 2 info") - print(node.query("SELECT * FROM system.zookeeper WHERE path = '{}' FORMAT Vertical".format(zk_path))) + print( + node.query( + "SELECT * FROM system.zookeeper WHERE path = '{}' FORMAT Vertical".format( + zk_path + ) + ) + ) print("Queue") - print(node.query("SELECT * FROM system.zookeeper WHERE path = '{}/queue' FORMAT Vertical".format(replica_path))) + print( + node.query( + "SELECT * FROM system.zookeeper WHERE path = '{}/queue' FORMAT Vertical".format( + replica_path + ) + ) + ) print("Log") - print(node.query("SELECT * FROM system.zookeeper WHERE path = '{}/log' FORMAT Vertical".format(zk_path))) + print( + node.query( + "SELECT * FROM system.zookeeper WHERE path = '{}/log' FORMAT Vertical".format( + zk_path + ) + ) + ) print("Parts") - print(node.query("SELECT name FROM system.zookeeper WHERE path = '{}/parts' FORMAT Vertical".format(replica_path))) + print( + node.query( + "SELECT name FROM system.zookeeper WHERE path = '{}/parts' FORMAT Vertical".format( + replica_path + ) + ) + ) + def restart_replica_for_sure(node, table_name, zk_replica_path): fake_zk = None @@ -218,7 +289,6 @@ def restart_replica_for_sure(node, table_name, zk_replica_path): fake_zk.close() - # in extremely rare case it can take more than 5 minutes in debug build with sanitizer @pytest.mark.timeout(600) def test_blocade_leader_twice(started_cluster): @@ -227,7 +297,11 @@ def test_blocade_leader_twice(started_cluster): try: for i, node in enumerate([node1, node2, node3]): node.query("CREATE DATABASE IF NOT EXISTS ordinary ENGINE=Ordinary") - node.query("CREATE TABLE IF NOT EXISTS ordinary.t2 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t2', '{}') ORDER BY tuple()".format(i + 1)) + node.query( + "CREATE TABLE IF NOT EXISTS ordinary.t2 (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t2', '{}') ORDER BY tuple()".format( + i + 1 + ) + ) break except Exception as ex: print("Got exception from node", smaller_exception(ex)) @@ -248,7 +322,9 @@ def test_blocade_leader_twice(started_cluster): for i in range(100): try: - restart_replica_for_sure(node2, "ordinary.t2", "/clickhouse/t2/replicas/2") + restart_replica_for_sure( + node2, "ordinary.t2", "/clickhouse/t2/replicas/2" + ) node2.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") break except Exception as ex: @@ -260,12 +336,16 @@ def test_blocade_leader_twice(started_cluster): time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t2', '/clickhouse/t2/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t2", "/clickhouse/t2/replicas/{}".format(num + 1) + ) assert False, "Cannot reconnect for node2" for i in range(100): try: - restart_replica_for_sure(node3, "ordinary.t2", "/clickhouse/t2/replicas/3") + restart_replica_for_sure( + node3, "ordinary.t2", "/clickhouse/t2/replicas/3" + ) node3.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) node3.query("INSERT INTO ordinary.t2 SELECT rand() FROM numbers(100)") break @@ -278,7 +358,9 @@ def test_blocade_leader_twice(started_cluster): time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t2', '/clickhouse/t2/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t2", "/clickhouse/t2/replicas/{}".format(num + 1) + ) assert False, "Cannot reconnect for node3" node2.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) @@ -306,19 +388,26 @@ def test_blocade_leader_twice(started_cluster): for n, node in enumerate([node1, node2, node3]): for i in range(100): try: - restart_replica_for_sure(node, "ordinary.t2", "/clickhouse/t2/replicas/{}".format(n + 1)) + restart_replica_for_sure( + node, "ordinary.t2", "/clickhouse/t2/replicas/{}".format(n + 1) + ) break except Exception as ex: try: node.query("ATTACH TABLE ordinary.t2") except Exception as attach_ex: - print("Got exception node{}".format(n + 1), smaller_exception(attach_ex)) + print( + "Got exception node{}".format(n + 1), + smaller_exception(attach_ex), + ) print("Got exception node{}".format(n + 1), smaller_exception(ex)) time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t2', '/clickhouse/t2/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t2", "/clickhouse/t2/replicas/{}".format(num + 1) + ) assert False, "Cannot reconnect for node{}".format(n + 1) for n, node in enumerate([node1, node2, node3]): @@ -331,14 +420,18 @@ def test_blocade_leader_twice(started_cluster): time.sleep(0.5) else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t2', '/clickhouse/t2/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t2", "/clickhouse/t2/replicas/{}".format(num + 1) + ) assert False, "Cannot reconnect for node{}".format(n + 1) for i in range(100): all_done = True for n, node in enumerate([node1, node2, node3]): try: - restart_replica_for_sure(node, "ordinary.t2", "/clickhouse/t2/replicas/{}".format(n + 1)) + restart_replica_for_sure( + node, "ordinary.t2", "/clickhouse/t2/replicas/{}".format(n + 1) + ) node.query("SYSTEM SYNC REPLICA ordinary.t2", timeout=10) break except Exception as ex: @@ -346,7 +439,10 @@ def test_blocade_leader_twice(started_cluster): try: node.query("ATTACH TABLE ordinary.t2") except Exception as attach_ex: - print("Got exception node{}".format(n + 1), smaller_exception(attach_ex)) + print( + "Got exception node{}".format(n + 1), + smaller_exception(attach_ex), + ) print("Got exception node{}".format(n + 1), smaller_exception(ex)) time.sleep(0.5) @@ -355,13 +451,17 @@ def test_blocade_leader_twice(started_cluster): break else: for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t2', '/clickhouse/t2/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t2", "/clickhouse/t2/replicas/{}".format(num + 1) + ) assert False, "Cannot reconnect in i {} retries".format(i) assert_eq_with_retry(node1, "SELECT COUNT() FROM ordinary.t2", "510") if node2.query("SELECT COUNT() FROM ordinary.t2") != "510\n": for num, node in enumerate([node1, node2, node3]): - dump_zk(node, '/clickhouse/t2', '/clickhouse/t2/replicas/{}'.format(num + 1)) + dump_zk( + node, "/clickhouse/t2", "/clickhouse/t2/replicas/{}".format(num + 1) + ) assert_eq_with_retry(node2, "SELECT COUNT() FROM ordinary.t2", "510") assert_eq_with_retry(node3, "SELECT COUNT() FROM ordinary.t2", "510") diff --git a/tests/integration/test_keeper_multinode_simple/test.py b/tests/integration/test_keeper_multinode_simple/test.py index d7cd4dd927e..694600acc67 100644 --- a/tests/integration/test_keeper_multinode_simple/test.py +++ b/tests/integration/test_keeper_multinode_simple/test.py @@ -9,12 +9,25 @@ from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml', 'configs/use_keeper.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml', 'configs/use_keeper.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml', 'configs/use_keeper.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/enable_keeper1.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/enable_keeper2.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/enable_keeper3.xml", "configs/use_keeper.xml"], + stay_alive=True, +) from kazoo.client import KazooClient, KazooState + @pytest.fixture(scope="module") def started_cluster(): try: @@ -25,8 +38,10 @@ def started_cluster(): finally: cluster.shutdown() + def smaller_exception(ex): - return '\n'.join(str(ex).split('\n')[0:2]) + return "\n".join(str(ex).split("\n")[0:2]) + def wait_node(node): for _ in range(100): @@ -47,16 +62,20 @@ def wait_node(node): else: raise Exception("Can't wait node", node.name, "to become ready") + def wait_nodes(): for node in [node1, node2, node3]: wait_node(node) def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def test_read_write_multinode(started_cluster): try: wait_nodes() @@ -111,6 +130,7 @@ def test_watch_on_follower(started_cluster): node3_zk.set("/test_data_watches", b"world") node1_data = None + def node1_callback(event): print("node1 data watch called") nonlocal node1_data @@ -119,6 +139,7 @@ def test_watch_on_follower(started_cluster): node1_zk.get("/test_data_watches", watch=node1_callback) node2_data = None + def node2_callback(event): print("node2 data watch called") nonlocal node2_data @@ -127,6 +148,7 @@ def test_watch_on_follower(started_cluster): node2_zk.get("/test_data_watches", watch=node2_callback) node3_data = None + def node3_callback(event): print("node3 data watch called") nonlocal node3_data @@ -169,7 +191,10 @@ def test_session_expiration(started_cluster): node3_zk.stop() node3_zk.close() for _ in range(100): - if node1_zk.exists("/test_ephemeral_node") is None and node2_zk.exists("/test_ephemeral_node") is None: + if ( + node1_zk.exists("/test_ephemeral_node") is None + and node2_zk.exists("/test_ephemeral_node") is None + ): break print("Node1 exists", node1_zk.exists("/test_ephemeral_node")) print("Node2 exists", node2_zk.exists("/test_ephemeral_node")) @@ -221,7 +246,11 @@ def test_follower_restart(started_cluster): def test_simple_replicated_table(started_cluster): wait_nodes() for i, node in enumerate([node1, node2, node3]): - node.query("CREATE TABLE t (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t', '{}') ORDER BY tuple()".format(i + 1)) + node.query( + "CREATE TABLE t (value UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/t', '{}') ORDER BY tuple()".format( + i + 1 + ) + ) node2.query("INSERT INTO t SELECT number FROM numbers(10)") diff --git a/tests/integration/test_keeper_nodes_add/test.py b/tests/integration/test_keeper_nodes_add/test.py index ae4a996f6b3..ad7d7c21182 100644 --- a/tests/integration/test_keeper_nodes_add/test.py +++ b/tests/integration/test_keeper_nodes_add/test.py @@ -12,15 +12,19 @@ from helpers.test_tools import assert_eq_with_retry from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) -CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'configs') +CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=[], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=[], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance("node2", main_configs=[], stay_alive=True) +node3 = cluster.add_instance("node3", main_configs=[], stay_alive=True) def get_fake_zk(node, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -35,8 +39,9 @@ def started_cluster(): finally: cluster.shutdown() + def start(node): - node.start_clickhouse() + node.start_clickhouse() def test_nodes_add(started_cluster): @@ -47,9 +52,15 @@ def test_nodes_add(started_cluster): p = Pool(3) node2.stop_clickhouse() - node2.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_2.xml"), "/etc/clickhouse-server/config.d/enable_keeper2.xml") + node2.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_2.xml"), + "/etc/clickhouse-server/config.d/enable_keeper2.xml", + ) waiter = p.apply_async(start, (node2,)) - node1.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_1.xml"), "/etc/clickhouse-server/config.d/enable_keeper1.xml") + node1.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_1.xml"), + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + ) node1.query("SYSTEM RELOAD CONFIG") waiter.wait() @@ -65,10 +76,19 @@ def test_nodes_add(started_cluster): node3.stop_clickhouse() - node3.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_three_nodes_3.xml"), "/etc/clickhouse-server/config.d/enable_keeper3.xml") + node3.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_three_nodes_3.xml"), + "/etc/clickhouse-server/config.d/enable_keeper3.xml", + ) waiter = p.apply_async(start, (node3,)) - node2.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_three_nodes_2.xml"), "/etc/clickhouse-server/config.d/enable_keeper2.xml") - node1.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_three_nodes_1.xml"), "/etc/clickhouse-server/config.d/enable_keeper1.xml") + node2.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_three_nodes_2.xml"), + "/etc/clickhouse-server/config.d/enable_keeper2.xml", + ) + node1.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_three_nodes_1.xml"), + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + ) node1.query("SYSTEM RELOAD CONFIG") node2.query("SYSTEM RELOAD CONFIG") diff --git a/tests/integration/test_keeper_nodes_move/test.py b/tests/integration/test_keeper_nodes_move/test.py index e3f1a161b07..9a571cd8ed6 100644 --- a/tests/integration/test_keeper_nodes_move/test.py +++ b/tests/integration/test_keeper_nodes_move/test.py @@ -15,12 +15,18 @@ from helpers.test_tools import assert_eq_with_retry from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) -CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'configs') +CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml'], stay_alive=True) -node4 = cluster.add_instance('node4', stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/enable_keeper2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/enable_keeper3.xml"], stay_alive=True +) +node4 = cluster.add_instance("node4", stay_alive=True) @pytest.fixture(scope="module") @@ -37,8 +43,11 @@ def started_cluster(): def start(node): node.start_clickhouse() + def get_fake_zk(node, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -60,11 +69,20 @@ def test_node_move(started_cluster): assert zk_conn3.exists("test_four_" + str(i)) is not None node4.stop_clickhouse() - node4.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_node4_4.xml"), "/etc/clickhouse-server/config.d/enable_keeper4.xml") + node4.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_node4_4.xml"), + "/etc/clickhouse-server/config.d/enable_keeper4.xml", + ) p = Pool(3) waiter = p.apply_async(start, (node4,)) - node1.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_node4_1.xml"), "/etc/clickhouse-server/config.d/enable_keeper1.xml") - node2.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_node4_2.xml"), "/etc/clickhouse-server/config.d/enable_keeper2.xml") + node1.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_node4_1.xml"), + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + ) + node2.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_node4_2.xml"), + "/etc/clickhouse-server/config.d/enable_keeper2.xml", + ) node1.query("SYSTEM RELOAD CONFIG") node2.query("SYSTEM RELOAD CONFIG") diff --git a/tests/integration/test_keeper_nodes_remove/test.py b/tests/integration/test_keeper_nodes_remove/test.py index 6df4ee1c497..13303d320eb 100644 --- a/tests/integration/test_keeper_nodes_remove/test.py +++ b/tests/integration/test_keeper_nodes_remove/test.py @@ -6,11 +6,18 @@ import os from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) -CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'configs') +CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs") + +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/enable_keeper2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/enable_keeper3.xml"], stay_alive=True +) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml'], stay_alive=True) @pytest.fixture(scope="module") def started_cluster(): @@ -24,7 +31,9 @@ def started_cluster(): def get_fake_zk(node, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -45,8 +54,14 @@ def test_nodes_remove(started_cluster): assert zk_conn2.exists("test_two_" + str(i)) is not None assert zk_conn3.exists("test_two_" + str(i)) is not None - node2.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_2.xml"), "/etc/clickhouse-server/config.d/enable_keeper2.xml") - node1.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_1.xml"), "/etc/clickhouse-server/config.d/enable_keeper1.xml") + node2.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_2.xml"), + "/etc/clickhouse-server/config.d/enable_keeper2.xml", + ) + node1.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_keeper_two_nodes_1.xml"), + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + ) node1.query("SYSTEM RELOAD CONFIG") node2.query("SYSTEM RELOAD CONFIG") @@ -70,7 +85,10 @@ def test_nodes_remove(started_cluster): node3.stop_clickhouse() - node1.copy_file_to_container(os.path.join(CONFIG_DIR, "enable_single_keeper1.xml"), "/etc/clickhouse-server/config.d/enable_keeper1.xml") + node1.copy_file_to_container( + os.path.join(CONFIG_DIR, "enable_single_keeper1.xml"), + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + ) node1.query("SYSTEM RELOAD CONFIG") zk_conn = get_fake_zk(node1) diff --git a/tests/integration/test_keeper_persistent_log/test.py b/tests/integration/test_keeper_persistent_log/test.py index eec2d4cbbdc..377fa436a87 100644 --- a/tests/integration/test_keeper_persistent_log/test.py +++ b/tests/integration/test_keeper_persistent_log/test.py @@ -10,17 +10,23 @@ from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/enable_keeper.xml', 'configs/use_keeper.xml'], stay_alive=True) +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper.xml", "configs/use_keeper.xml"], + stay_alive=True, +) def random_string(length): - return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) + def create_random_path(prefix="", depth=1): if depth == 0: return prefix return create_random_path(os.path.join(prefix, random_string(3)), depth - 1) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -31,11 +37,15 @@ def started_cluster(): finally: cluster.shutdown() + def get_connection_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def test_state_after_restart(started_cluster): try: node_zk = None @@ -59,10 +69,18 @@ def test_state_after_restart(started_cluster): assert node_zk2.get("/test_state_after_restart")[0] == b"somevalue" for i in range(100): if i % 7 == 0: - assert node_zk2.exists("/test_state_after_restart/node" + str(i)) is None + assert ( + node_zk2.exists("/test_state_after_restart/node" + str(i)) is None + ) else: - assert len(node_zk2.get("/test_state_after_restart/node" + str(i))[0]) == 123 - assert node_zk2.get("/test_state_after_restart/node" + str(i))[0] == strs[i] + assert ( + len(node_zk2.get("/test_state_after_restart/node" + str(i))[0]) + == 123 + ) + assert ( + node_zk2.get("/test_state_after_restart/node" + str(i))[0] + == strs[i] + ) finally: try: if node_zk is not None: @@ -75,6 +93,7 @@ def test_state_after_restart(started_cluster): except: pass + def test_state_duplicate_restart(started_cluster): try: node_zk = None @@ -107,10 +126,19 @@ def test_state_duplicate_restart(started_cluster): assert node_zk3.get("/test_state_duplicated_restart")[0] == b"somevalue" for i in range(100): if i % 7 == 0: - assert node_zk3.exists("/test_state_duplicated_restart/node" + str(i)) is None + assert ( + node_zk3.exists("/test_state_duplicated_restart/node" + str(i)) + is None + ) else: - assert len(node_zk3.get("/test_state_duplicated_restart/node" + str(i))[0]) == 123 - assert node_zk3.get("/test_state_duplicated_restart/node" + str(i))[0] == strs[i] + assert ( + len(node_zk3.get("/test_state_duplicated_restart/node" + str(i))[0]) + == 123 + ) + assert ( + node_zk3.get("/test_state_duplicated_restart/node" + str(i))[0] + == strs[i] + ) finally: try: if node_zk is not None: @@ -129,7 +157,6 @@ def test_state_duplicate_restart(started_cluster): pass - # http://zookeeper-user.578899.n2.nabble.com/Why-are-ephemeral-nodes-written-to-disk-tp7583403p7583418.html def test_ephemeral_after_restart(started_cluster): try: @@ -141,7 +168,9 @@ def test_ephemeral_after_restart(started_cluster): strs = [] for i in range(100): strs.append(random_string(123).encode()) - node_zk.create("/test_ephemeral_after_restart/node" + str(i), strs[i], ephemeral=True) + node_zk.create( + "/test_ephemeral_after_restart/node" + str(i), strs[i], ephemeral=True + ) for i in range(100): if i % 7 == 0: @@ -154,10 +183,19 @@ def test_ephemeral_after_restart(started_cluster): assert node_zk2.get("/test_ephemeral_after_restart")[0] == b"somevalue" for i in range(100): if i % 7 == 0: - assert node_zk2.exists("/test_ephemeral_after_restart/node" + str(i)) is None + assert ( + node_zk2.exists("/test_ephemeral_after_restart/node" + str(i)) + is None + ) else: - assert len(node_zk2.get("/test_ephemeral_after_restart/node" + str(i))[0]) == 123 - assert node_zk2.get("/test_ephemeral_after_restart/node" + str(i))[0] == strs[i] + assert ( + len(node_zk2.get("/test_ephemeral_after_restart/node" + str(i))[0]) + == 123 + ) + assert ( + node_zk2.get("/test_ephemeral_after_restart/node" + str(i))[0] + == strs[i] + ) finally: try: if node_zk is not None: diff --git a/tests/integration/test_keeper_persistent_log_multinode/test.py b/tests/integration/test_keeper_persistent_log_multinode/test.py index 8c02f269a60..f15e772fd5f 100644 --- a/tests/integration/test_keeper_persistent_log_multinode/test.py +++ b/tests/integration/test_keeper_persistent_log_multinode/test.py @@ -7,12 +7,25 @@ import os import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml', 'configs/use_keeper.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml', 'configs/use_keeper.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml', 'configs/use_keeper.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/enable_keeper1.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/enable_keeper2.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/enable_keeper3.xml", "configs/use_keeper.xml"], + stay_alive=True, +) from kazoo.client import KazooClient, KazooState + @pytest.fixture(scope="module") def started_cluster(): try: @@ -23,11 +36,15 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def stop_zk(zk): try: if zk: @@ -36,6 +53,7 @@ def stop_zk(zk): except: pass + def test_restart_multinode(started_cluster): try: node1_zk = node2_zk = node3_zk = None @@ -45,7 +63,10 @@ def test_restart_multinode(started_cluster): node3_zk = get_fake_zk("node3") for i in range(100): - node1_zk.create("/test_read_write_multinode_node" + str(i), ("somedata" + str(i)).encode()) + node1_zk.create( + "/test_read_write_multinode_node" + str(i), + ("somedata" + str(i)).encode(), + ) for i in range(100): if i % 10 == 0: @@ -56,11 +77,21 @@ def test_restart_multinode(started_cluster): for i in range(100): if i % 10 != 0: - assert node2_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() - assert node3_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() + assert ( + node2_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node3_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) else: - assert node2_zk.exists("/test_read_write_multinode_node" + str(i)) is None - assert node3_zk.exists("/test_read_write_multinode_node" + str(i)) is None + assert ( + node2_zk.exists("/test_read_write_multinode_node" + str(i)) is None + ) + assert ( + node3_zk.exists("/test_read_write_multinode_node" + str(i)) is None + ) finally: for zk in [node1_zk, node2_zk, node3_zk]: @@ -76,13 +107,31 @@ def test_restart_multinode(started_cluster): node3_zk = get_fake_zk("node3") for i in range(100): if i % 10 != 0: - assert node1_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() - assert node2_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() - assert node3_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() + assert ( + node1_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node2_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node3_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) else: - assert node1_zk.exists("/test_read_write_multinode_node" + str(i)) is None - assert node2_zk.exists("/test_read_write_multinode_node" + str(i)) is None - assert node3_zk.exists("/test_read_write_multinode_node" + str(i)) is None + assert ( + node1_zk.exists("/test_read_write_multinode_node" + str(i)) + is None + ) + assert ( + node2_zk.exists("/test_read_write_multinode_node" + str(i)) + is None + ) + assert ( + node3_zk.exists("/test_read_write_multinode_node" + str(i)) + is None + ) break except Exception as ex: print("Got exception as ex", ex) diff --git a/tests/integration/test_keeper_restore_from_snapshot/test.py b/tests/integration/test_keeper_restore_from_snapshot/test.py index 7a0323d95b4..7270c84bdda 100644 --- a/tests/integration/test_keeper_restore_from_snapshot/test.py +++ b/tests/integration/test_keeper_restore_from_snapshot/test.py @@ -7,12 +7,19 @@ import os import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/enable_keeper2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/enable_keeper3.xml"], stay_alive=True +) from kazoo.client import KazooClient, KazooState + @pytest.fixture(scope="module") def started_cluster(): try: @@ -23,11 +30,15 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def stop_zk(zk): try: if zk: @@ -57,7 +68,10 @@ def test_recover_from_snapshot(started_cluster): # at least we will have 2 snapshots for i in range(435): - node1_zk.create("/test_snapshot_multinode_recover" + str(i), ("somedata" + str(i)).encode()) + node1_zk.create( + "/test_snapshot_multinode_recover" + str(i), + ("somedata" + str(i)).encode(), + ) for i in range(435): if i % 10 == 0: @@ -88,13 +102,28 @@ def test_recover_from_snapshot(started_cluster): for i in range(435): if i % 10 != 0: - assert node1_zk.get("/test_snapshot_multinode_recover" + str(i))[0] == ("somedata" + str(i)).encode() - assert node2_zk.get("/test_snapshot_multinode_recover" + str(i))[0] == ("somedata" + str(i)).encode() - assert node3_zk.get("/test_snapshot_multinode_recover" + str(i))[0] == ("somedata" + str(i)).encode() + assert ( + node1_zk.get("/test_snapshot_multinode_recover" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node2_zk.get("/test_snapshot_multinode_recover" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node3_zk.get("/test_snapshot_multinode_recover" + str(i))[0] + == ("somedata" + str(i)).encode() + ) else: - assert node1_zk.exists("/test_snapshot_multinode_recover" + str(i)) is None - assert node2_zk.exists("/test_snapshot_multinode_recover" + str(i)) is None - assert node3_zk.exists("/test_snapshot_multinode_recover" + str(i)) is None + assert ( + node1_zk.exists("/test_snapshot_multinode_recover" + str(i)) is None + ) + assert ( + node2_zk.exists("/test_snapshot_multinode_recover" + str(i)) is None + ) + assert ( + node3_zk.exists("/test_snapshot_multinode_recover" + str(i)) is None + ) finally: for zk in [node1_zk, node2_zk, node3_zk]: stop_zk(zk) diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index fe03ed8dcf8..55e00880da0 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -6,8 +6,25 @@ import os import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_secure_keeper.xml', 'configs/ssl_conf.xml', "configs/dhparam.pem", "configs/server.crt", "configs/server.key"]) -node2 = cluster.add_instance('node2', main_configs=['configs/use_secure_keeper.xml', 'configs/ssl_conf.xml', "configs/server.crt", "configs/server.key"]) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/enable_secure_keeper.xml", + "configs/ssl_conf.xml", + "configs/dhparam.pem", + "configs/server.crt", + "configs/server.key", + ], +) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/use_secure_keeper.xml", + "configs/ssl_conf.xml", + "configs/server.crt", + "configs/server.key", + ], +) @pytest.fixture(scope="module") diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index 6867f2bf5cd..30db4d9548c 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -5,21 +5,24 @@ import socket import struct from kazoo.client import KazooClient + # from kazoo.protocol.serialization import Connect, read_buffer, write_buffer cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/keeper_config.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/keeper_config.xml"], stay_alive=True +) -bool_struct = struct.Struct('B') -int_struct = struct.Struct('!i') -int_int_struct = struct.Struct('!ii') -int_int_long_struct = struct.Struct('!iiq') +bool_struct = struct.Struct("B") +int_struct = struct.Struct("!i") +int_int_struct = struct.Struct("!ii") +int_int_long_struct = struct.Struct("!iiq") -int_long_int_long_struct = struct.Struct('!iqiq') -long_struct = struct.Struct('!q') -multiheader_struct = struct.Struct('!iBi') -reply_header_struct = struct.Struct('!iqi') -stat_struct = struct.Struct('!qqqqiiiqiiq') +int_long_int_long_struct = struct.Struct("!iqiq") +long_struct = struct.Struct("!q") +multiheader_struct = struct.Struct("!iBi") +reply_header_struct = struct.Struct("!iqi") +stat_struct = struct.Struct("!qqqqiiiqiiq") @pytest.fixture(scope="module") @@ -63,7 +66,9 @@ def wait_nodes(): def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -96,7 +101,7 @@ def read_buffer(bytes, offset): else: index = offset offset += length - return bytes[index:index + length], offset + return bytes[index : index + length], offset def handshake(node_name=node1.name, session_timeout=1000, session_id=0): @@ -105,14 +110,18 @@ def handshake(node_name=node1.name, session_timeout=1000, session_id=0): client = get_keeper_socket(node_name) protocol_version = 0 last_zxid_seen = 0 - session_passwd = b'\x00' * 16 + session_passwd = b"\x00" * 16 read_only = 0 # Handshake serialize and deserialize code is from 'kazoo.protocol.serialization'. # serialize handshake req = bytearray() - req.extend(int_long_int_long_struct.pack(protocol_version, last_zxid_seen, session_timeout, session_id)) + req.extend( + int_long_int_long_struct.pack( + protocol_version, last_zxid_seen, session_timeout, session_id + ) + ) req.extend(write_buffer(session_passwd)) req.extend([1 if read_only else 0]) # add header @@ -127,7 +136,9 @@ def handshake(node_name=node1.name, session_timeout=1000, session_id=0): print("handshake response - len:", data.hex(), len(data)) # ignore header offset = 4 - proto_version, negotiated_timeout, session_id = int_int_long_struct.unpack_from(data, offset) + proto_version, negotiated_timeout, session_id = int_int_long_struct.unpack_from( + data, offset + ) offset += int_int_long_struct.size password, offset = read_buffer(data, offset) try: @@ -153,4 +164,4 @@ def test_session_timeout(started_cluster): assert negotiated_timeout == 8000 negotiated_timeout, _ = handshake(node1.name, session_timeout=20000, session_id=0) - assert negotiated_timeout == 10000 + assert negotiated_timeout == 10000 diff --git a/tests/integration/test_keeper_snapshot_small_distance/test.py b/tests/integration/test_keeper_snapshot_small_distance/test.py index 4acd76806b4..4351c5ac96f 100644 --- a/tests/integration/test_keeper_snapshot_small_distance/test.py +++ b/tests/integration/test_keeper_snapshot_small_distance/test.py @@ -10,42 +10,68 @@ import os import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/keeper_config1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/keeper_config2.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/keeper_config3.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/keeper_config1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/keeper_config2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/keeper_config3.xml"], stay_alive=True +) + def start_zookeeper(node): - node1.exec_in_container(['bash', '-c', '/opt/zookeeper/bin/zkServer.sh start']) + node1.exec_in_container(["bash", "-c", "/opt/zookeeper/bin/zkServer.sh start"]) + def stop_zookeeper(node): - node.exec_in_container(['bash', '-c', '/opt/zookeeper/bin/zkServer.sh stop']) + node.exec_in_container(["bash", "-c", "/opt/zookeeper/bin/zkServer.sh stop"]) + def clear_zookeeper(node): - node.exec_in_container(['bash', '-c', 'rm -fr /zookeeper/*']) + node.exec_in_container(["bash", "-c", "rm -fr /zookeeper/*"]) + def restart_and_clear_zookeeper(node): stop_zookeeper(node) clear_zookeeper(node) start_zookeeper(node) + def clear_clickhouse_data(node): - node.exec_in_container(['bash', '-c', 'rm -fr /var/lib/clickhouse/coordination/logs/* /var/lib/clickhouse/coordination/snapshots/*']) + node.exec_in_container( + [ + "bash", + "-c", + "rm -fr /var/lib/clickhouse/coordination/logs/* /var/lib/clickhouse/coordination/snapshots/*", + ] + ) + def convert_zookeeper_data(node): - cmd = '/usr/bin/clickhouse keeper-converter --zookeeper-logs-dir /zookeeper/version-2/ --zookeeper-snapshots-dir /zookeeper/version-2/ --output-dir /var/lib/clickhouse/coordination/snapshots' - node.exec_in_container(['bash', '-c', cmd]) - return os.path.join('/var/lib/clickhouse/coordination/snapshots', node.exec_in_container(['bash', '-c', 'ls /var/lib/clickhouse/coordination/snapshots']).strip()) + cmd = "/usr/bin/clickhouse keeper-converter --zookeeper-logs-dir /zookeeper/version-2/ --zookeeper-snapshots-dir /zookeeper/version-2/ --output-dir /var/lib/clickhouse/coordination/snapshots" + node.exec_in_container(["bash", "-c", cmd]) + return os.path.join( + "/var/lib/clickhouse/coordination/snapshots", + node.exec_in_container( + ["bash", "-c", "ls /var/lib/clickhouse/coordination/snapshots"] + ).strip(), + ) + def stop_clickhouse(node): node.stop_clickhouse() + def start_clickhouse(node): node.start_clickhouse() + def copy_zookeeper_data(make_zk_snapshots, node): stop_zookeeper(node) - if make_zk_snapshots: # force zookeeper to create snapshot + if make_zk_snapshots: # force zookeeper to create snapshot start_zookeeper(node) stop_zookeeper(node) @@ -66,13 +92,19 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(node, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def get_genuine_zk(node, timeout=30.0): - _genuine_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":2181", timeout=timeout) + _genuine_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(node.name) + ":2181", timeout=timeout + ) _genuine_zk_instance.start() return _genuine_zk_instance @@ -99,7 +131,9 @@ def test_snapshot_and_load(started_cluster): print("Resulted path", resulted_path) for node in [node2, node3]: print("Copy snapshot from", node1.name, "to", node.name) - cluster.copy_file_from_container_to_container(node1, resulted_path, node, '/var/lib/clickhouse/coordination/snapshots') + cluster.copy_file_from_container_to_container( + node1, resulted_path, node, "/var/lib/clickhouse/coordination/snapshots" + ) print("Starting clickhouses") diff --git a/tests/integration/test_keeper_snapshots/test.py b/tests/integration/test_keeper_snapshots/test.py index 607e461d835..08f60e538a4 100644 --- a/tests/integration/test_keeper_snapshots/test.py +++ b/tests/integration/test_keeper_snapshots/test.py @@ -13,16 +13,24 @@ from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) # clickhouse itself will use external zookeeper -node = cluster.add_instance('node', main_configs=['configs/enable_keeper.xml'], stay_alive=True, with_zookeeper=True) +node = cluster.add_instance( + "node", + main_configs=["configs/enable_keeper.xml"], + stay_alive=True, + with_zookeeper=True, +) + def random_string(length): - return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length)) + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) + def create_random_path(prefix="", depth=1): if depth == 0: return prefix return create_random_path(os.path.join(prefix, random_string(3)), depth - 1) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -33,11 +41,15 @@ def started_cluster(): finally: cluster.shutdown() + def get_connection_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def test_state_after_restart(started_cluster): try: node_zk = None @@ -57,7 +69,6 @@ def test_state_after_restart(started_cluster): else: existing_children.append("node" + str(i)) - node.restart_clickhouse(kill=True) node_zk2 = get_connection_zk("node") @@ -65,14 +76,18 @@ def test_state_after_restart(started_cluster): assert node_zk2.get("/test_state_after_restart")[0] == b"somevalue" for i in range(100): if i % 7 == 0: - assert node_zk2.exists("/test_state_after_restart/node" + str(i)) is None + assert ( + node_zk2.exists("/test_state_after_restart/node" + str(i)) is None + ) else: data, stat = node_zk2.get("/test_state_after_restart/node" + str(i)) assert len(data) == 123 assert data == strs[i] assert stat.ephemeralOwner == 0 - assert list(sorted(existing_children)) == list(sorted(node_zk2.get_children("/test_state_after_restart"))) + assert list(sorted(existing_children)) == list( + sorted(node_zk2.get_children("/test_state_after_restart")) + ) finally: try: if node_zk is not None: @@ -97,7 +112,9 @@ def test_ephemeral_after_restart(started_cluster): strs = [] for i in range(100): strs.append(random_string(123).encode()) - node_zk.create("/test_ephemeral_after_restart/node" + str(i), strs[i], ephemeral=True) + node_zk.create( + "/test_ephemeral_after_restart/node" + str(i), strs[i], ephemeral=True + ) existing_children = [] for i in range(100): @@ -113,13 +130,18 @@ def test_ephemeral_after_restart(started_cluster): assert node_zk2.get("/test_ephemeral_after_restart")[0] == b"somevalue" for i in range(100): if i % 7 == 0: - assert node_zk2.exists("/test_ephemeral_after_restart/node" + str(i)) is None + assert ( + node_zk2.exists("/test_ephemeral_after_restart/node" + str(i)) + is None + ) else: data, stat = node_zk2.get("/test_ephemeral_after_restart/node" + str(i)) assert len(data) == 123 assert data == strs[i] assert stat.ephemeralOwner == session_id - assert list(sorted(existing_children)) == list(sorted(node_zk2.get_children("/test_ephemeral_after_restart"))) + assert list(sorted(existing_children)) == list( + sorted(node_zk2.get_children("/test_ephemeral_after_restart")) + ) finally: try: if node_zk is not None: diff --git a/tests/integration/test_keeper_snapshots_multinode/test.py b/tests/integration/test_keeper_snapshots_multinode/test.py index de4ed3a1a8f..1461f35e6a4 100644 --- a/tests/integration/test_keeper_snapshots_multinode/test.py +++ b/tests/integration/test_keeper_snapshots_multinode/test.py @@ -7,12 +7,19 @@ import os import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/enable_keeper2.xml"], stay_alive=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/enable_keeper3.xml"], stay_alive=True +) from kazoo.client import KazooClient, KazooState + @pytest.fixture(scope="module") def started_cluster(): try: @@ -23,11 +30,15 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def stop_zk(zk): try: if zk: @@ -36,6 +47,7 @@ def stop_zk(zk): except: pass + def test_restart_multinode(started_cluster): try: node1_zk = node2_zk = node3_zk = None @@ -45,7 +57,10 @@ def test_restart_multinode(started_cluster): node3_zk = get_fake_zk("node3") for i in range(100): - node1_zk.create("/test_read_write_multinode_node" + str(i), ("somedata" + str(i)).encode()) + node1_zk.create( + "/test_read_write_multinode_node" + str(i), + ("somedata" + str(i)).encode(), + ) for i in range(100): if i % 10 == 0: @@ -56,11 +71,21 @@ def test_restart_multinode(started_cluster): for i in range(100): if i % 10 != 0: - assert node2_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() - assert node3_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() + assert ( + node2_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node3_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) else: - assert node2_zk.exists("/test_read_write_multinode_node" + str(i)) is None - assert node3_zk.exists("/test_read_write_multinode_node" + str(i)) is None + assert ( + node2_zk.exists("/test_read_write_multinode_node" + str(i)) is None + ) + assert ( + node3_zk.exists("/test_read_write_multinode_node" + str(i)) is None + ) finally: for zk in [node1_zk, node2_zk, node3_zk]: @@ -76,13 +101,31 @@ def test_restart_multinode(started_cluster): node3_zk = get_fake_zk("node3") for i in range(100): if i % 10 != 0: - assert node1_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() - assert node2_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() - assert node3_zk.get("/test_read_write_multinode_node" + str(i))[0] == ("somedata" + str(i)).encode() + assert ( + node1_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node2_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) + assert ( + node3_zk.get("/test_read_write_multinode_node" + str(i))[0] + == ("somedata" + str(i)).encode() + ) else: - assert node1_zk.exists("/test_read_write_multinode_node" + str(i)) is None - assert node2_zk.exists("/test_read_write_multinode_node" + str(i)) is None - assert node3_zk.exists("/test_read_write_multinode_node" + str(i)) is None + assert ( + node1_zk.exists("/test_read_write_multinode_node" + str(i)) + is None + ) + assert ( + node2_zk.exists("/test_read_write_multinode_node" + str(i)) + is None + ) + assert ( + node3_zk.exists("/test_read_write_multinode_node" + str(i)) + is None + ) break except Exception as ex: print("Got exception as ex", ex) diff --git a/tests/integration/test_keeper_three_nodes_start/test.py b/tests/integration/test_keeper_three_nodes_start/test.py index 7828f21d0d7..f23ef5440c1 100644 --- a/tests/integration/test_keeper_three_nodes_start/test.py +++ b/tests/integration/test_keeper_three_nodes_start/test.py @@ -13,14 +13,22 @@ from helpers.test_tools import assert_eq_with_retry from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/enable_keeper1.xml"], stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/enable_keeper2.xml"], stay_alive=True +) + def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def test_smoke(): try: cluster.start() diff --git a/tests/integration/test_keeper_three_nodes_two_alive/test.py b/tests/integration/test_keeper_three_nodes_two_alive/test.py index d79a185b367..11ff1d8cc08 100644 --- a/tests/integration/test_keeper_three_nodes_two_alive/test.py +++ b/tests/integration/test_keeper_three_nodes_two_alive/test.py @@ -11,13 +11,27 @@ from helpers.test_tools import assert_eq_with_retry from kazoo.client import KazooClient, KazooState cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml', 'configs/keeper_conf.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml', 'configs/keeper_conf.xml'], stay_alive=True) -node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml', 'configs/keeper_conf.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/enable_keeper1.xml", "configs/keeper_conf.xml"], + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/enable_keeper2.xml", "configs/keeper_conf.xml"], + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/enable_keeper3.xml", "configs/keeper_conf.xml"], + stay_alive=True, +) def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance @@ -32,8 +46,9 @@ def started_cluster(): finally: cluster.shutdown() + def start(node): - node.start_clickhouse() + node.start_clickhouse() def delete_with_retry(node_name, path): @@ -59,8 +74,12 @@ def test_start_offline(started_cluster): time.sleep(3) p.map(start, [node2, node3]) - assert node2.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start") - assert node3.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start") + assert node2.contains_in_log( + "Cannot connect to ZooKeeper (or Keeper) before internal Keeper start" + ) + assert node3.contains_in_log( + "Cannot connect to ZooKeeper (or Keeper) before internal Keeper start" + ) node2_zk = get_fake_zk("node2") node2_zk.create("/c", b"data") @@ -77,20 +96,40 @@ def test_start_non_existing(started_cluster): node2.stop_clickhouse() node3.stop_clickhouse() - node1.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper1.xml', 'node3', 'non_existing_node') - node2.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper2.xml', 'node3', 'non_existing_node') + node1.replace_in_config( + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + "node3", + "non_existing_node", + ) + node2.replace_in_config( + "/etc/clickhouse-server/config.d/enable_keeper2.xml", + "node3", + "non_existing_node", + ) time.sleep(3) p.map(start, [node2, node1]) - assert node1.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start") - assert node2.contains_in_log("Cannot connect to ZooKeeper (or Keeper) before internal Keeper start") + assert node1.contains_in_log( + "Cannot connect to ZooKeeper (or Keeper) before internal Keeper start" + ) + assert node2.contains_in_log( + "Cannot connect to ZooKeeper (or Keeper) before internal Keeper start" + ) node2_zk = get_fake_zk("node2") node2_zk.create("/test_non_exising", b"data") finally: - node1.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper1.xml', 'non_existing_node', 'node3') - node2.replace_in_config('/etc/clickhouse-server/config.d/enable_keeper2.xml', 'non_existing_node', 'node3') + node1.replace_in_config( + "/etc/clickhouse-server/config.d/enable_keeper1.xml", + "non_existing_node", + "node3", + ) + node2.replace_in_config( + "/etc/clickhouse-server/config.d/enable_keeper2.xml", + "non_existing_node", + "node3", + ) p.map(start, [node1, node2, node3]) delete_with_retry("node2", "/test_non_exising") @@ -101,5 +140,7 @@ def test_restart_third_node(started_cluster): node3.restart_clickhouse() - assert node3.contains_in_log("Connected to ZooKeeper (or Keeper) before internal Keeper start") + assert node3.contains_in_log( + "Connected to ZooKeeper (or Keeper) before internal Keeper start" + ) node1_zk.delete("/test_restart") diff --git a/tests/integration/test_keeper_two_nodes_cluster/test.py b/tests/integration/test_keeper_two_nodes_cluster/test.py index 4cafa1d17f4..8c0276f7d77 100644 --- a/tests/integration/test_keeper_two_nodes_cluster/test.py +++ b/tests/integration/test_keeper_two_nodes_cluster/test.py @@ -11,11 +11,20 @@ from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml', 'configs/use_keeper.xml'], stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml', 'configs/use_keeper.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/enable_keeper1.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/enable_keeper2.xml", "configs/use_keeper.xml"], + stay_alive=True, +) from kazoo.client import KazooClient, KazooState + @pytest.fixture(scope="module") def started_cluster(): try: @@ -26,8 +35,10 @@ def started_cluster(): finally: cluster.shutdown() + def smaller_exception(ex): - return '\n'.join(str(ex).split('\n')[0:2]) + return "\n".join(str(ex).split("\n")[0:2]) + def wait_node(node): for _ in range(100): @@ -48,16 +59,20 @@ def wait_node(node): else: raise Exception("Can't wait node", node.name, "to become ready") + def wait_nodes(): for node in [node1, node2]: wait_node(node) def get_fake_zk(nodename, timeout=30.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def test_read_write_two_nodes(started_cluster): try: wait_nodes() @@ -89,6 +104,7 @@ def test_read_write_two_nodes(started_cluster): except: pass + def test_read_write_two_nodes_with_blocade(started_cluster): try: wait_nodes() @@ -108,7 +124,6 @@ def test_read_write_two_nodes_with_blocade(started_cluster): with pytest.raises(Exception): node2_zk.create("/test_read_write_blocked_node2", b"somedata2") - print("Nodes unblocked") for i in range(10): try: @@ -118,7 +133,6 @@ def test_read_write_two_nodes_with_blocade(started_cluster): except: time.sleep(0.5) - for i in range(100): try: node1_zk.create("/test_after_block1", b"somedata12") diff --git a/tests/integration/test_keeper_znode_time/__init__.py b/tests/integration/test_keeper_znode_time/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_keeper_znode_time/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_keeper_znode_time/configs/enable_keeper1.xml b/tests/integration/test_keeper_znode_time/configs/enable_keeper1.xml new file mode 100644 index 00000000000..17455ed12f5 --- /dev/null +++ b/tests/integration/test_keeper_znode_time/configs/enable_keeper1.xml @@ -0,0 +1,41 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + true + 3 + + + 2 + node2 + 9234 + true + true + 2 + + + 3 + node3 + 9234 + true + true + 1 + + + + diff --git a/tests/integration/test_keeper_znode_time/configs/enable_keeper2.xml b/tests/integration/test_keeper_znode_time/configs/enable_keeper2.xml new file mode 100644 index 00000000000..03a23984cc2 --- /dev/null +++ b/tests/integration/test_keeper_znode_time/configs/enable_keeper2.xml @@ -0,0 +1,41 @@ + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + true + 3 + + + 2 + node2 + 9234 + true + true + 2 + + + 3 + node3 + 9234 + true + true + 1 + + + + diff --git a/tests/integration/test_keeper_znode_time/configs/enable_keeper3.xml b/tests/integration/test_keeper_znode_time/configs/enable_keeper3.xml new file mode 100644 index 00000000000..a3196ac3061 --- /dev/null +++ b/tests/integration/test_keeper_znode_time/configs/enable_keeper3.xml @@ -0,0 +1,41 @@ + + + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 75 + trace + + + + + 1 + node1 + 9234 + true + 3 + + + 2 + node2 + 9234 + true + true + 2 + + + 3 + node3 + 9234 + true + true + 1 + + + + diff --git a/tests/integration/test_keeper_znode_time/configs/use_keeper.xml b/tests/integration/test_keeper_znode_time/configs/use_keeper.xml new file mode 100644 index 00000000000..384e984f210 --- /dev/null +++ b/tests/integration/test_keeper_znode_time/configs/use_keeper.xml @@ -0,0 +1,16 @@ + + + + node1 + 9181 + + + node2 + 9181 + + + node3 + 9181 + + + diff --git a/tests/integration/test_keeper_znode_time/test.py b/tests/integration/test_keeper_znode_time/test.py new file mode 100644 index 00000000000..f50f03ac168 --- /dev/null +++ b/tests/integration/test_keeper_znode_time/test.py @@ -0,0 +1,149 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import random +import string +import os +import time +from multiprocessing.dummy import Pool +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/enable_keeper1.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/enable_keeper2.xml", "configs/use_keeper.xml"], + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/enable_keeper3.xml", "configs/use_keeper.xml"], + stay_alive=True, +) + +from kazoo.client import KazooClient, KazooState + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def smaller_exception(ex): + return "\n".join(str(ex).split("\n")[0:2]) + + +def wait_node(node): + for _ in range(100): + zk = None + try: + node.query("SELECT * FROM system.zookeeper WHERE path = '/'") + zk = get_fake_zk(node.name, timeout=30.0) + zk.create("/test", sequence=True) + print("node", node.name, "ready") + break + except Exception as ex: + time.sleep(0.2) + print("Waiting until", node.name, "will be ready, exception", ex) + finally: + if zk: + zk.stop() + zk.close() + else: + raise Exception("Can't wait node", node.name, "to become ready") + + +def wait_nodes(): + for node in [node1, node2, node3]: + wait_node(node) + + +def get_fake_zk(nodename, timeout=30.0): + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout + ) + _fake_zk_instance.start() + return _fake_zk_instance + + +def assert_eq_stats(stat1, stat2): + assert stat1.version == stat2.version + assert stat1.cversion == stat2.cversion + assert stat1.aversion == stat2.aversion + assert stat1.aversion == stat2.aversion + assert stat1.dataLength == stat2.dataLength + assert stat1.numChildren == stat2.numChildren + assert stat1.ctime == stat2.ctime + assert stat1.mtime == stat2.mtime + + +def test_between_servers(started_cluster): + try: + wait_nodes() + node1_zk = get_fake_zk("node1") + node2_zk = get_fake_zk("node2") + node3_zk = get_fake_zk("node3") + + node1_zk.create("/test_between_servers") + for child_node in range(1000): + node1_zk.create("/test_between_servers/" + str(child_node)) + + for child_node in range(1000): + node1_zk.set("/test_between_servers/" + str(child_node), b"somevalue") + + for child_node in range(1000): + stats1 = node1_zk.exists("/test_between_servers/" + str(child_node)) + stats2 = node2_zk.exists("/test_between_servers/" + str(child_node)) + stats3 = node3_zk.exists("/test_between_servers/" + str(child_node)) + assert_eq_stats(stats1, stats2) + assert_eq_stats(stats2, stats3) + + finally: + try: + for zk_conn in [node1_zk, node2_zk, node3_zk]: + zk_conn.stop() + zk_conn.close() + except: + pass + + +def test_server_restart(started_cluster): + try: + wait_nodes() + node1_zk = get_fake_zk("node1") + + node1_zk.create("/test_server_restart") + for child_node in range(1000): + node1_zk.create("/test_server_restart/" + str(child_node)) + + for child_node in range(1000): + node1_zk.set("/test_server_restart/" + str(child_node), b"somevalue") + + node3.restart_clickhouse(kill=True) + + node2_zk = get_fake_zk("node2") + node3_zk = get_fake_zk("node3") + for child_node in range(1000): + stats1 = node1_zk.exists("/test_between_servers/" + str(child_node)) + stats2 = node2_zk.exists("/test_between_servers/" + str(child_node)) + stats3 = node3_zk.exists("/test_between_servers/" + str(child_node)) + assert_eq_stats(stats1, stats2) + assert_eq_stats(stats2, stats3) + + finally: + try: + for zk_conn in [node1_zk, node2_zk, node3_zk]: + zk_conn.stop() + zk_conn.close() + except: + pass diff --git a/tests/integration/test_keeper_zookeeper_converter/test.py b/tests/integration/test_keeper_zookeeper_converter/test.py index 6829b4a9000..87975a57019 100644 --- a/tests/integration/test_keeper_zookeeper_converter/test.py +++ b/tests/integration/test_keeper_zookeeper_converter/test.py @@ -3,44 +3,68 @@ import pytest from helpers.cluster import ClickHouseCluster from kazoo.client import KazooClient, KazooState from kazoo.security import ACL, make_digest_acl, make_acl -from kazoo.exceptions import AuthFailedError, InvalidACLError, NoAuthError, KazooException +from kazoo.exceptions import ( + AuthFailedError, + InvalidACLError, + NoAuthError, + KazooException, +) import os cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/keeper_config.xml', 'configs/logs_conf.xml'], stay_alive=True) +node = cluster.add_instance( + "node", + main_configs=["configs/keeper_config.xml", "configs/logs_conf.xml"], + stay_alive=True, +) + def start_zookeeper(): - node.exec_in_container(['bash', '-c', '/opt/zookeeper/bin/zkServer.sh start']) + node.exec_in_container(["bash", "-c", "/opt/zookeeper/bin/zkServer.sh start"]) + def stop_zookeeper(): - node.exec_in_container(['bash', '-c', '/opt/zookeeper/bin/zkServer.sh stop']) + node.exec_in_container(["bash", "-c", "/opt/zookeeper/bin/zkServer.sh stop"]) + def clear_zookeeper(): - node.exec_in_container(['bash', '-c', 'rm -fr /zookeeper/*']) + node.exec_in_container(["bash", "-c", "rm -fr /zookeeper/*"]) + def restart_and_clear_zookeeper(): stop_zookeeper() clear_zookeeper() start_zookeeper() + def clear_clickhouse_data(): - node.exec_in_container(['bash', '-c', 'rm -fr /var/lib/clickhouse/coordination/logs/* /var/lib/clickhouse/coordination/snapshots/*']) + node.exec_in_container( + [ + "bash", + "-c", + "rm -fr /var/lib/clickhouse/coordination/logs/* /var/lib/clickhouse/coordination/snapshots/*", + ] + ) + def convert_zookeeper_data(): - cmd = '/usr/bin/clickhouse keeper-converter --zookeeper-logs-dir /zookeeper/version-2/ --zookeeper-snapshots-dir /zookeeper/version-2/ --output-dir /var/lib/clickhouse/coordination/snapshots' - node.exec_in_container(['bash', '-c', cmd]) + cmd = "/usr/bin/clickhouse keeper-converter --zookeeper-logs-dir /zookeeper/version-2/ --zookeeper-snapshots-dir /zookeeper/version-2/ --output-dir /var/lib/clickhouse/coordination/snapshots" + node.exec_in_container(["bash", "-c", cmd]) + def stop_clickhouse(): node.stop_clickhouse() + def start_clickhouse(): node.start_clickhouse() + def copy_zookeeper_data(make_zk_snapshots): stop_zookeeper() - if make_zk_snapshots: # force zookeeper to create snapshot + if make_zk_snapshots: # force zookeeper to create snapshot start_zookeeper() stop_zookeeper() @@ -50,6 +74,7 @@ def copy_zookeeper_data(make_zk_snapshots): start_zookeeper() start_clickhouse() + @pytest.fixture(scope="module") def started_cluster(): try: @@ -60,26 +85,97 @@ def started_cluster(): finally: cluster.shutdown() + def get_fake_zk(timeout=60.0): - _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip('node') + ":9181", timeout=timeout) + _fake_zk_instance = KazooClient( + hosts=cluster.get_instance_ip("node") + ":9181", timeout=timeout + ) _fake_zk_instance.start() return _fake_zk_instance + def get_genuine_zk(timeout=60.0): - _genuine_zk_instance = KazooClient(hosts=cluster.get_instance_ip('node') + ":2181", timeout=timeout) + _genuine_zk_instance = KazooClient( + hosts=cluster.get_instance_ip("node") + ":2181", timeout=timeout + ) _genuine_zk_instance.start() return _genuine_zk_instance + def compare_stats(stat1, stat2, path): - assert stat1.czxid == stat2.czxid, "path " + path + " cxzids not equal for stats: " + str(stat1.czxid) + " != " + str(stat2.zxid) - assert stat1.mzxid == stat2.mzxid, "path " + path + " mxzids not equal for stats: " + str(stat1.mzxid) + " != " + str(stat2.mzxid) - assert stat1.version == stat2.version, "path " + path + " versions not equal for stats: " + str(stat1.version) + " != " + str(stat2.version) - assert stat1.cversion == stat2.cversion, "path " + path + " cversions not equal for stats: " + str(stat1.cversion) + " != " + str(stat2.cversion) - assert stat1.aversion == stat2.aversion, "path " + path + " aversions not equal for stats: " + str(stat1.aversion) + " != " + str(stat2.aversion) - assert stat1.ephemeralOwner == stat2.ephemeralOwner,"path " + path + " ephemeralOwners not equal for stats: " + str(stat1.ephemeralOwner) + " != " + str(stat2.ephemeralOwner) - assert stat1.dataLength == stat2.dataLength , "path " + path + " ephemeralOwners not equal for stats: " + str(stat1.dataLength) + " != " + str(stat2.dataLength) - assert stat1.numChildren == stat2.numChildren, "path " + path + " numChildren not equal for stats: " + str(stat1.numChildren) + " != " + str(stat2.numChildren) - assert stat1.pzxid == stat2.pzxid, "path " + path + " pzxid not equal for stats: " + str(stat1.pzxid) + " != " + str(stat2.pzxid) + assert stat1.czxid == stat2.czxid, ( + "path " + + path + + " cxzids not equal for stats: " + + str(stat1.czxid) + + " != " + + str(stat2.zxid) + ) + assert stat1.mzxid == stat2.mzxid, ( + "path " + + path + + " mxzids not equal for stats: " + + str(stat1.mzxid) + + " != " + + str(stat2.mzxid) + ) + assert stat1.version == stat2.version, ( + "path " + + path + + " versions not equal for stats: " + + str(stat1.version) + + " != " + + str(stat2.version) + ) + assert stat1.cversion == stat2.cversion, ( + "path " + + path + + " cversions not equal for stats: " + + str(stat1.cversion) + + " != " + + str(stat2.cversion) + ) + assert stat1.aversion == stat2.aversion, ( + "path " + + path + + " aversions not equal for stats: " + + str(stat1.aversion) + + " != " + + str(stat2.aversion) + ) + assert stat1.ephemeralOwner == stat2.ephemeralOwner, ( + "path " + + path + + " ephemeralOwners not equal for stats: " + + str(stat1.ephemeralOwner) + + " != " + + str(stat2.ephemeralOwner) + ) + assert stat1.dataLength == stat2.dataLength, ( + "path " + + path + + " ephemeralOwners not equal for stats: " + + str(stat1.dataLength) + + " != " + + str(stat2.dataLength) + ) + assert stat1.numChildren == stat2.numChildren, ( + "path " + + path + + " numChildren not equal for stats: " + + str(stat1.numChildren) + + " != " + + str(stat2.numChildren) + ) + assert stat1.pzxid == stat2.pzxid, ( + "path " + + path + + " pzxid not equal for stats: " + + str(stat1.pzxid) + + " != " + + str(stat2.pzxid) + ) + def compare_states(zk1, zk2, path="/"): data1, stat1 = zk1.get(path) @@ -101,12 +197,8 @@ def compare_states(zk1, zk2, path="/"): print("Checking child", os.path.join(path, children)) compare_states(zk1, zk2, os.path.join(path, children)) -@pytest.mark.parametrize( - ('create_snapshots'), - [ - True, False - ] -) + +@pytest.mark.parametrize(("create_snapshots"), [True, False]) def test_smoke(started_cluster, create_snapshots): restart_and_clear_zookeeper() @@ -122,15 +214,12 @@ def test_smoke(started_cluster, create_snapshots): compare_states(genuine_connection, fake_connection) + def get_bytes(s): return s.encode() -@pytest.mark.parametrize( - ('create_snapshots'), - [ - True, False - ] -) + +@pytest.mark.parametrize(("create_snapshots"), [True, False]) def test_simple_crud_requests(started_cluster, create_snapshots): restart_and_clear_zookeeper() @@ -151,14 +240,19 @@ def test_simple_crud_requests(started_cluster, create_snapshots): genuine_connection.create(path, get_bytes("data" + str(i))) path = os.path.join(path, str(i)) - genuine_connection.create("/test_sequential", b"") for i in range(10): - genuine_connection.create("/test_sequential/" + "a" * i + "-", get_bytes("dataX" + str(i)), sequence=True) + genuine_connection.create( + "/test_sequential/" + "a" * i + "-", + get_bytes("dataX" + str(i)), + sequence=True, + ) genuine_connection.create("/test_ephemeral", b"") for i in range(10): - genuine_connection.create("/test_ephemeral/" + str(i), get_bytes("dataX" + str(i)), ephemeral=True) + genuine_connection.create( + "/test_ephemeral/" + str(i), get_bytes("dataX" + str(i)), ephemeral=True + ) copy_zookeeper_data(create_snapshots) @@ -168,54 +262,64 @@ def test_simple_crud_requests(started_cluster, create_snapshots): compare_states(genuine_connection, fake_connection) # especially ensure that counters are the same - genuine_connection.create("/test_sequential/" + "a" * 10 + "-", get_bytes("dataX" + str(i)), sequence=True) - fake_connection.create("/test_sequential/" + "a" * 10 + "-", get_bytes("dataX" + str(i)), sequence=True) + genuine_connection.create( + "/test_sequential/" + "a" * 10 + "-", get_bytes("dataX" + str(i)), sequence=True + ) + fake_connection.create( + "/test_sequential/" + "a" * 10 + "-", get_bytes("dataX" + str(i)), sequence=True + ) first_children = list(sorted(genuine_connection.get_children("/test_sequential"))) second_children = list(sorted(fake_connection.get_children("/test_sequential"))) assert first_children == second_children, "Childrens are not equal on path " + path -@pytest.mark.parametrize( - ('create_snapshots'), - [ - True, False - ] -) + +@pytest.mark.parametrize(("create_snapshots"), [True, False]) def test_multi_and_failed_requests(started_cluster, create_snapshots): restart_and_clear_zookeeper() genuine_connection = get_genuine_zk() - genuine_connection.create('/test_multitransactions') + genuine_connection.create("/test_multitransactions") for i in range(10): t = genuine_connection.transaction() - t.create('/test_multitransactions/freddy' + str(i), get_bytes('data' + str(i))) - t.create('/test_multitransactions/fred' + str(i), get_bytes('value' + str(i)), ephemeral=True) - t.create('/test_multitransactions/smith' + str(i), get_bytes('entity' + str(i)), sequence=True) - t.set_data('/test_multitransactions', get_bytes("somedata" + str(i))) + t.create("/test_multitransactions/freddy" + str(i), get_bytes("data" + str(i))) + t.create( + "/test_multitransactions/fred" + str(i), + get_bytes("value" + str(i)), + ephemeral=True, + ) + t.create( + "/test_multitransactions/smith" + str(i), + get_bytes("entity" + str(i)), + sequence=True, + ) + t.set_data("/test_multitransactions", get_bytes("somedata" + str(i))) t.commit() with pytest.raises(Exception): - genuine_connection.set('/test_multitransactions/freddy0', get_bytes('mustfail' + str(i)), version=1) + genuine_connection.set( + "/test_multitransactions/freddy0", get_bytes("mustfail" + str(i)), version=1 + ) t = genuine_connection.transaction() - t.create('/test_bad_transaction', get_bytes('data' + str(1))) - t.check('/test_multitransactions', version=32) - t.create('/test_bad_transaction1', get_bytes('data' + str(2))) + t.create("/test_bad_transaction", get_bytes("data" + str(1))) + t.check("/test_multitransactions", version=32) + t.create("/test_bad_transaction1", get_bytes("data" + str(2))) # should fail t.commit() - assert genuine_connection.exists('/test_bad_transaction') is None - assert genuine_connection.exists('/test_bad_transaction1') is None + assert genuine_connection.exists("/test_bad_transaction") is None + assert genuine_connection.exists("/test_bad_transaction1") is None t = genuine_connection.transaction() - t.create('/test_bad_transaction2', get_bytes('data' + str(1))) - t.delete('/test_multitransactions/freddy0', version=5) + t.create("/test_bad_transaction2", get_bytes("data" + str(1))) + t.delete("/test_multitransactions/freddy0", version=5) # should fail t.commit() - assert genuine_connection.exists('/test_bad_transaction2') is None - assert genuine_connection.exists('/test_multitransactions/freddy0') is not None + assert genuine_connection.exists("/test_bad_transaction2") is None + assert genuine_connection.exists("/test_multitransactions/freddy0") is not None copy_zookeeper_data(create_snapshots) @@ -224,35 +328,40 @@ def test_multi_and_failed_requests(started_cluster, create_snapshots): compare_states(genuine_connection, fake_connection) -@pytest.mark.parametrize( - ('create_snapshots'), - [ - True, False - ] -) + +@pytest.mark.parametrize(("create_snapshots"), [True, False]) def test_acls(started_cluster, create_snapshots): restart_and_clear_zookeeper() genuine_connection = get_genuine_zk() - genuine_connection.add_auth('digest', 'user1:password1') - genuine_connection.add_auth('digest', 'user2:password2') - genuine_connection.add_auth('digest', 'user3:password3') + genuine_connection.add_auth("digest", "user1:password1") + genuine_connection.add_auth("digest", "user2:password2") + genuine_connection.add_auth("digest", "user3:password3") - genuine_connection.create("/test_multi_all_acl", b"data", acl=[make_acl("auth", "", all=True)]) + genuine_connection.create( + "/test_multi_all_acl", b"data", acl=[make_acl("auth", "", all=True)] + ) other_connection = get_genuine_zk() - other_connection.add_auth('digest', 'user1:password1') + other_connection.add_auth("digest", "user1:password1") other_connection.set("/test_multi_all_acl", b"X") assert other_connection.get("/test_multi_all_acl")[0] == b"X" yet_other_auth_connection = get_genuine_zk() - yet_other_auth_connection.add_auth('digest', 'user2:password2') + yet_other_auth_connection.add_auth("digest", "user2:password2") yet_other_auth_connection.set("/test_multi_all_acl", b"Y") - genuine_connection.add_auth('digest', 'user3:password3') + genuine_connection.add_auth("digest", "user3:password3") # just to check that we are able to deserialize it - genuine_connection.set_acls("/test_multi_all_acl", acls=[make_acl("auth", "", read=True, write=False, create=True, delete=True, admin=True)]) + genuine_connection.set_acls( + "/test_multi_all_acl", + acls=[ + make_acl( + "auth", "", read=True, write=False, create=True, delete=True, admin=True + ) + ], + ) no_auth_connection = get_genuine_zk() @@ -262,14 +371,14 @@ def test_acls(started_cluster, create_snapshots): copy_zookeeper_data(create_snapshots) genuine_connection = get_genuine_zk() - genuine_connection.add_auth('digest', 'user1:password1') - genuine_connection.add_auth('digest', 'user2:password2') - genuine_connection.add_auth('digest', 'user3:password3') + genuine_connection.add_auth("digest", "user1:password1") + genuine_connection.add_auth("digest", "user2:password2") + genuine_connection.add_auth("digest", "user3:password3") fake_connection = get_fake_zk() - fake_connection.add_auth('digest', 'user1:password1') - fake_connection.add_auth('digest', 'user2:password2') - fake_connection.add_auth('digest', 'user3:password3') + fake_connection.add_auth("digest", "user1:password1") + fake_connection.add_auth("digest", "user2:password2") + fake_connection.add_auth("digest", "user3:password3") compare_states(genuine_connection, fake_connection) @@ -278,7 +387,11 @@ def test_acls(started_cluster, create_snapshots): assert stat.aversion == 1 assert len(acls) == 3 for acl in acls: - assert acl.acl_list == ['READ', 'CREATE', 'DELETE', 'ADMIN'] - assert acl.id.scheme == 'digest' + assert acl.acl_list == ["READ", "CREATE", "DELETE", "ADMIN"] + assert acl.id.scheme == "digest" assert acl.perms == 29 - assert acl.id.id in ('user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=', 'user2:lo/iTtNMP+gEZlpUNaCqLYO3i5U=', 'user3:wr5Y0kEs9nFX3bKrTMKxrlcFeWo=') + assert acl.id.id in ( + "user1:XDkd2dsEuhc9ImU3q8pa8UOdtpI=", + "user2:lo/iTtNMP+gEZlpUNaCqLYO3i5U=", + "user3:wr5Y0kEs9nFX3bKrTMKxrlcFeWo=", + ) diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index 12a967ebaa4..6e2c2ec0597 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -8,13 +8,18 @@ from helpers.cluster import ClickHouseCluster, run_and_check cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - dictionaries=['configs/dictionaries/dict1.xml'], main_configs=['configs/config.d/config.xml'], stay_alive=True) +instance = cluster.add_instance( + "instance", + dictionaries=["configs/dictionaries/dict1.xml"], + main_configs=["configs/config.d/config.xml"], + stay_alive=True, +) def create_dict_simple(): - instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') - instance.query(''' + instance.query("DROP DICTIONARY IF EXISTS lib_dict_c") + instance.query( + """ CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so')) LAYOUT(CACHE( @@ -24,30 +29,52 @@ def create_dict_simple(): READ_BUFFER_SIZE 1048576 MAX_STORED_KEYS 1048576)) LIFETIME(2) ; - ''') + """ + ) @pytest.fixture(scope="module") def ch_cluster(): try: cluster.start() - instance.query('CREATE DATABASE test') - container_lib_path = '/etc/clickhouse-server/config.d/dictionarites_lib/dict_lib.cpp' + instance.query("CREATE DATABASE test") + container_lib_path = ( + "/etc/clickhouse-server/config.d/dictionarites_lib/dict_lib.cpp" + ) - instance.copy_file_to_container(os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs/dict_lib.cpp"), - "/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp") + instance.copy_file_to_container( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs/dict_lib.cpp" + ), + "/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp", + ) instance.query("SYSTEM RELOAD CONFIG") instance.exec_in_container( - ['bash', '-c', - '/usr/bin/g++ -shared -o /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp'], - user='root') + [ + "bash", + "-c", + "/usr/bin/g++ -shared -o /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp", + ], + user="root", + ) instance.exec_in_container( - ['bash', '-c', - '/usr/bin/g++ -shared -o /dict_lib_copy.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp'], user='root') - instance.exec_in_container(['bash', '-c', 'ln -s /dict_lib_copy.so /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so']) + [ + "bash", + "-c", + "/usr/bin/g++ -shared -o /dict_lib_copy.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp", + ], + user="root", + ) + instance.exec_in_container( + [ + "bash", + "-c", + "ln -s /dict_lib_copy.so /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so", + ] + ) yield cluster @@ -64,8 +91,9 @@ def test_load_all(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - instance.query('DROP DICTIONARY IF EXISTS lib_dict') - instance.query(''' + instance.query("DROP DICTIONARY IF EXISTS lib_dict") + instance.query( + """ CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library( @@ -73,41 +101,45 @@ def test_load_all(ch_cluster): SETTINGS (test_type test_simple))) LAYOUT(HASHED()) LIFETIME (MIN 0 MAX 10) - ''') + """ + ) - result = instance.query('SELECT * FROM lib_dict ORDER BY key') + result = instance.query("SELECT * FROM lib_dict ORDER BY key") expected = ( -"0\t10\t20\t30\n" + -"1\t11\t21\t31\n" + -"2\t12\t22\t32\n" + -"3\t13\t23\t33\n" + -"4\t14\t24\t34\n" + -"5\t15\t25\t35\n" + -"6\t16\t26\t36\n" + -"7\t17\t27\t37\n" + -"8\t18\t28\t38\n" + -"9\t19\t29\t39\n" -) - instance.query('SYSTEM RELOAD DICTIONARY dict1') - instance.query('DROP DICTIONARY lib_dict') - assert(result == expected) + "0\t10\t20\t30\n" + + "1\t11\t21\t31\n" + + "2\t12\t22\t32\n" + + "3\t13\t23\t33\n" + + "4\t14\t24\t34\n" + + "5\t15\t25\t35\n" + + "6\t16\t26\t36\n" + + "7\t17\t27\t37\n" + + "8\t18\t28\t38\n" + + "9\t19\t29\t39\n" + ) + instance.query("SYSTEM RELOAD DICTIONARY dict1") + instance.query("DROP DICTIONARY lib_dict") + assert result == expected - instance.query(""" + instance.query( + """ CREATE TABLE IF NOT EXISTS `dict1_table` ( key UInt64, value1 UInt64, value2 UInt64, value3 UInt64 ) ENGINE = Dictionary(dict1) - """) + """ + ) - result = instance.query('SELECT * FROM dict1_table ORDER BY key') - assert(result == expected) + result = instance.query("SELECT * FROM dict1_table ORDER BY key") + assert result == expected def test_load_ids(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') - instance.query(''' + instance.query("DROP DICTIONARY IF EXISTS lib_dict_c") + instance.query( + """ CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so')) LAYOUT(CACHE( @@ -117,37 +149,46 @@ def test_load_ids(ch_cluster): READ_BUFFER_SIZE 1048576 MAX_STORED_KEYS 1048576)) LIFETIME(2) ; - ''') + """ + ) - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(0));''') - assert(result.strip() == '100') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(0));""") + assert result.strip() == "100" # Just check bridge is ok with a large vector of random ids - instance.query('''select number, dictGet(lib_dict_c, 'value1', toUInt64(rand())) from numbers(1000);''') + instance.query( + """select number, dictGet(lib_dict_c, 'value1', toUInt64(rand())) from numbers(1000);""" + ) - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') - instance.query('DROP DICTIONARY lib_dict_c') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" + instance.query("DROP DICTIONARY lib_dict_c") def test_load_keys(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - instance.query('DROP DICTIONARY IF EXISTS lib_dict_ckc') - instance.query(''' + instance.query("DROP DICTIONARY IF EXISTS lib_dict_ckc") + instance.query( + """ CREATE DICTIONARY lib_dict_ckc (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so')) LAYOUT(COMPLEX_KEY_CACHE( SIZE_IN_CELLS 10000000)) LIFETIME(2); - ''') + """ + ) - result = instance.query('''select dictGet(lib_dict_ckc, 'value1', tuple(toUInt64(0)));''') - assert(result.strip() == '100') - result = instance.query('''select dictGet(lib_dict_ckc, 'value2', tuple(toUInt64(0)));''') - assert(result.strip() == '200') - instance.query('DROP DICTIONARY lib_dict_ckc') + result = instance.query( + """select dictGet(lib_dict_ckc, 'value1', tuple(toUInt64(0)));""" + ) + assert result.strip() == "100" + result = instance.query( + """select dictGet(lib_dict_ckc, 'value2', tuple(toUInt64(0)));""" + ) + assert result.strip() == "200" + instance.query("DROP DICTIONARY lib_dict_ckc") def test_load_all_many_rows(ch_cluster): @@ -155,9 +196,10 @@ def test_load_all_many_rows(ch_cluster): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") num_rows = [1000, 10000, 100000, 1000000] - instance.query('DROP DICTIONARY IF EXISTS lib_dict') + instance.query("DROP DICTIONARY IF EXISTS lib_dict") for num in num_rows: - instance.query(''' + instance.query( + """ CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library( @@ -165,28 +207,35 @@ def test_load_all_many_rows(ch_cluster): SETTINGS (num_rows {} test_type test_many_rows))) LAYOUT(HASHED()) LIFETIME (MIN 0 MAX 10) - '''.format(num)) + """.format( + num + ) + ) - result = instance.query('SELECT * FROM lib_dict ORDER BY key') - expected = instance.query('SELECT number, number, number, number FROM numbers({})'.format(num)) - instance.query('DROP DICTIONARY lib_dict') - assert(result == expected) + result = instance.query("SELECT * FROM lib_dict ORDER BY key") + expected = instance.query( + "SELECT number, number, number, number FROM numbers({})".format(num) + ) + instance.query("DROP DICTIONARY lib_dict") + assert result == expected def test_null_values(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - instance.query('SYSTEM RELOAD DICTIONARY dict2') - instance.query(""" + instance.query("SYSTEM RELOAD DICTIONARY dict2") + instance.query( + """ CREATE TABLE IF NOT EXISTS `dict2_table` ( key UInt64, value1 UInt64, value2 UInt64, value3 UInt64 ) ENGINE = Dictionary(dict2) - """) + """ + ) - result = instance.query('SELECT * FROM dict2_table ORDER BY key') + result = instance.query("SELECT * FROM dict2_table ORDER BY key") expected = "0\t12\t12\t12\n" - assert(result == expected) + assert result == expected def test_recover_after_bridge_crash(ch_cluster): @@ -195,21 +244,25 @@ def test_recover_after_bridge_crash(ch_cluster): create_dict_simple() - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(0));''') - assert(result.strip() == '100') - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(0));""") + assert result.strip() == "100" + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" - instance.exec_in_container(['bash', '-c', 'kill -9 `pidof clickhouse-library-bridge`'], user='root') - instance.query('SYSTEM RELOAD DICTIONARY lib_dict_c') + instance.exec_in_container( + ["bash", "-c", "kill -9 `pidof clickhouse-library-bridge`"], user="root" + ) + instance.query("SYSTEM RELOAD DICTIONARY lib_dict_c") - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(0));''') - assert(result.strip() == '100') - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(0));""") + assert result.strip() == "100" + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" - instance.exec_in_container(['bash', '-c', 'kill -9 `pidof clickhouse-library-bridge`'], user='root') - instance.query('DROP DICTIONARY lib_dict_c') + instance.exec_in_container( + ["bash", "-c", "kill -9 `pidof clickhouse-library-bridge`"], user="root" + ) + instance.query("DROP DICTIONARY lib_dict_c") def test_server_restart_bridge_might_be_stil_alive(ch_cluster): @@ -218,32 +271,36 @@ def test_server_restart_bridge_might_be_stil_alive(ch_cluster): create_dict_simple() - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" instance.restart_clickhouse() - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" - instance.exec_in_container(['bash', '-c', 'kill -9 `pidof clickhouse-library-bridge`'], user='root') + instance.exec_in_container( + ["bash", "-c", "kill -9 `pidof clickhouse-library-bridge`"], user="root" + ) instance.restart_clickhouse() - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" - instance.query('DROP DICTIONARY lib_dict_c') + instance.query("DROP DICTIONARY lib_dict_c") def test_bridge_dies_with_parent(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") if instance.is_built_with_address_sanitizer(): - pytest.skip("Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge") + pytest.skip( + "Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge" + ) create_dict_simple() - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" clickhouse_pid = instance.get_process_pid("clickhouse server") bridge_pid = instance.get_process_pid("library-bridge") @@ -252,7 +309,9 @@ def test_bridge_dies_with_parent(ch_cluster): while clickhouse_pid is not None: try: - instance.exec_in_container(["kill", str(clickhouse_pid)], privileged=True, user='root') + instance.exec_in_container( + ["kill", str(clickhouse_pid)], privileged=True, user="root" + ) except: pass clickhouse_pid = instance.get_process_pid("clickhouse server") @@ -265,22 +324,26 @@ def test_bridge_dies_with_parent(ch_cluster): break if bridge_pid: - out = instance.exec_in_container(["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], - privileged=True, user='root') + out = instance.exec_in_container( + ["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], + privileged=True, + user="root", + ) logging.debug(f"Bridge is running, gdb output:\n{out}") assert clickhouse_pid is None assert bridge_pid is None instance.start_clickhouse(20) - instance.query('DROP DICTIONARY lib_dict_c') + instance.query("DROP DICTIONARY lib_dict_c") def test_path_validation(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") - instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') - instance.query(''' + instance.query("DROP DICTIONARY IF EXISTS lib_dict_c") + instance.query( + """ CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so')) LAYOUT(CACHE( @@ -290,13 +353,15 @@ def test_path_validation(ch_cluster): READ_BUFFER_SIZE 1048576 MAX_STORED_KEYS 1048576)) LIFETIME(2) ; - ''') + """ + ) - result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert(result.strip() == '101') + result = instance.query("""select dictGet(lib_dict_c, 'value1', toUInt64(1));""") + assert result.strip() == "101" - instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') - instance.query(''' + instance.query("DROP DICTIONARY IF EXISTS lib_dict_c") + instance.query( + """ CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so')) LAYOUT(CACHE( @@ -306,12 +371,18 @@ def test_path_validation(ch_cluster): READ_BUFFER_SIZE 1048576 MAX_STORED_KEYS 1048576)) LIFETIME(2) ; - ''') - result = instance.query_and_get_error('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') - assert('DB::Exception: File path /etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so is not inside /etc/clickhouse-server/config.d/dictionaries_lib' in result) + """ + ) + result = instance.query_and_get_error( + """select dictGet(lib_dict_c, 'value1', toUInt64(1));""" + ) + assert ( + "DB::Exception: File path /etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so is not inside /etc/clickhouse-server/config.d/dictionaries_lib" + in result + ) -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_limited_replicated_fetches/test.py b/tests/integration/test_limited_replicated_fetches/test.py index 7b0c7aed15d..e3271100b74 100644 --- a/tests/integration/test_limited_replicated_fetches/test.py +++ b/tests/integration/test_limited_replicated_fetches/test.py @@ -10,11 +10,16 @@ import os cluster = ClickHouseCluster(__file__) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -node1 = cluster.add_instance('node1', user_configs=['configs/custom_settings.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', user_configs=['configs/custom_settings.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", user_configs=["configs/custom_settings.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", user_configs=["configs/custom_settings.xml"], with_zookeeper=True +) MAX_THREADS_FOR_FETCH = 3 + @pytest.fixture(scope="module") def started_cluster(): try: @@ -27,32 +32,72 @@ def started_cluster(): def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) def test_limited_fetches(started_cluster): """ - Test checks that that we utilize all available threads for fetches + Test checks that that we utilize all available threads for fetches """ - node1.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple() PARTITION BY key") - node2.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple() PARTITION BY key") + node1.query( + "CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple() PARTITION BY key" + ) + node2.query( + "CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple() PARTITION BY key" + ) with PartitionManager() as pm: node2.query("SYSTEM STOP FETCHES t") - node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 3, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 4, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 5, '{}' FROM numbers(5000)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 6, '{}' FROM numbers(5000)".format(get_random_string(104857))) + node1.query( + "INSERT INTO t SELECT 1, '{}' FROM numbers(5000)".format( + get_random_string(104857) + ) + ) + node1.query( + "INSERT INTO t SELECT 2, '{}' FROM numbers(5000)".format( + get_random_string(104857) + ) + ) + node1.query( + "INSERT INTO t SELECT 3, '{}' FROM numbers(5000)".format( + get_random_string(104857) + ) + ) + node1.query( + "INSERT INTO t SELECT 4, '{}' FROM numbers(5000)".format( + get_random_string(104857) + ) + ) + node1.query( + "INSERT INTO t SELECT 5, '{}' FROM numbers(5000)".format( + get_random_string(104857) + ) + ) + node1.query( + "INSERT INTO t SELECT 6, '{}' FROM numbers(5000)".format( + get_random_string(104857) + ) + ) pm.add_network_delay(node1, 80) node2.query("SYSTEM START FETCHES t") fetches_result = [] background_fetches_metric = [] fetched_parts = set([]) for _ in range(1000): - result = node2.query("SELECT result_part_name FROM system.replicated_fetches").strip().split() - background_fetches_metric.append(int(node2.query("select value from system.metrics where metric = 'BackgroundFetchesPoolTask'").strip())) + result = ( + node2.query("SELECT result_part_name FROM system.replicated_fetches") + .strip() + .split() + ) + background_fetches_metric.append( + int( + node2.query( + "select value from system.metrics where metric = 'BackgroundFetchesPoolTask'" + ).strip() + ) + ) if not result: if len(fetched_parts) == 6: break @@ -67,10 +112,16 @@ def test_limited_fetches(started_cluster): for concurrently_fetching_parts in fetches_result: if len(concurrently_fetching_parts) > MAX_THREADS_FOR_FETCH: - assert False, "Found more than {} concurrently fetching parts: {}".format(MAX_THREADS_FOR_FETCH, ', '.join(concurrently_fetching_parts)) + assert False, "Found more than {} concurrently fetching parts: {}".format( + MAX_THREADS_FOR_FETCH, ", ".join(concurrently_fetching_parts) + ) - assert max([len(parts) for parts in fetches_result]) == 3, "Strange, but we don't utilize max concurrent threads for fetches" - assert(max(background_fetches_metric)) == 3, "Just checking metric consistent with table" + assert ( + max([len(parts) for parts in fetches_result]) == 3 + ), "Strange, but we don't utilize max concurrent threads for fetches" + assert ( + max(background_fetches_metric) + ) == 3, "Just checking metric consistent with table" node1.query("DROP TABLE IF EXISTS t SYNC") node2.query("DROP TABLE IF EXISTS t SYNC") diff --git a/tests/integration/test_log_family_hdfs/test.py b/tests/integration/test_log_family_hdfs/test.py index 7bb9cdfeaf5..e8afe364ec4 100644 --- a/tests/integration/test_log_family_hdfs/test.py +++ b/tests/integration/test_log_family_hdfs/test.py @@ -11,26 +11,27 @@ from pyhdfs import HdfsClient def started_cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=["configs/storage_conf.xml"], - with_hdfs=True) + cluster.add_instance( + "node", main_configs=["configs/storage_conf.xml"], with_hdfs=True + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") fs = HdfsClient(hosts=cluster.hdfs_ip) - fs.mkdirs('/clickhouse') + fs.mkdirs("/clickhouse") yield cluster finally: cluster.shutdown() -def assert_objects_count(started_cluster, objects_count, path='data/'): +def assert_objects_count(started_cluster, objects_count, path="data/"): fs = HdfsClient(hosts=started_cluster.hdfs_ip) - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert objects_count == len(hdfs_objects) + # TinyLog: files: id.bin, sizes.json # INSERT overwrites 1 file (`sizes.json`) and appends 1 file (`id.bin`), so # files_overhead=1, files_overhead_per_insert=1 @@ -44,26 +45,41 @@ def assert_objects_count(started_cluster, objects_count, path='data/'): # files_overhead=1, files_overhead_per_insert=2 @pytest.mark.parametrize( "log_engine,files_overhead,files_overhead_per_insert", - [("TinyLog", 1, 1), ("Log", 1, 2), ("StripeLog", 1, 2)]) -def test_log_family_hdfs(started_cluster, log_engine, files_overhead, files_overhead_per_insert): + [("TinyLog", 1, 1), ("Log", 1, 2), ("StripeLog", 1, 2)], +) +def test_log_family_hdfs( + started_cluster, log_engine, files_overhead, files_overhead_per_insert +): node = started_cluster.instances["node"] - node.query("CREATE TABLE hdfs_test (id UInt64) ENGINE={} SETTINGS disk = 'hdfs'".format(log_engine)) + node.query( + "CREATE TABLE hdfs_test (id UInt64) ENGINE={} SETTINGS disk = 'hdfs'".format( + log_engine + ) + ) node.query("INSERT INTO hdfs_test SELECT number FROM numbers(5)") assert node.query("SELECT * FROM hdfs_test") == "0\n1\n2\n3\n4\n" assert_objects_count(started_cluster, files_overhead_per_insert + files_overhead) node.query("INSERT INTO hdfs_test SELECT number + 5 FROM numbers(3)") - assert node.query("SELECT * FROM hdfs_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n" - assert_objects_count(started_cluster, files_overhead_per_insert * 2 + files_overhead) + assert ( + node.query("SELECT * FROM hdfs_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n" + ) + assert_objects_count( + started_cluster, files_overhead_per_insert * 2 + files_overhead + ) node.query("INSERT INTO hdfs_test SELECT number + 8 FROM numbers(1)") - assert node.query("SELECT * FROM hdfs_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n8\n" - assert_objects_count(started_cluster, files_overhead_per_insert * 3 + files_overhead) + assert ( + node.query("SELECT * FROM hdfs_test order by id") + == "0\n1\n2\n3\n4\n5\n6\n7\n8\n" + ) + assert_objects_count( + started_cluster, files_overhead_per_insert * 3 + files_overhead + ) node.query("TRUNCATE TABLE hdfs_test") assert_objects_count(started_cluster, 0) node.query("DROP TABLE hdfs_test") - diff --git a/tests/integration/test_log_family_s3/test.py b/tests/integration/test_log_family_s3/test.py index 8531edd635f..234b079ba00 100644 --- a/tests/integration/test_log_family_s3/test.py +++ b/tests/integration/test_log_family_s3/test.py @@ -9,9 +9,11 @@ from helpers.cluster import ClickHouseCluster def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=["configs/minio.xml", "configs/ssl.xml"], - with_minio=True) + cluster.add_instance( + "node", + main_configs=["configs/minio.xml", "configs/ssl.xml"], + with_minio=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -21,7 +23,7 @@ def cluster(): cluster.shutdown() -def assert_objects_count(cluster, objects_count, path='data/'): +def assert_objects_count(cluster, objects_count, path="data/"): minio = cluster.minio_client s3_objects = list(minio.list_objects(cluster.minio_bucket, path)) if objects_count != len(s3_objects): @@ -30,6 +32,7 @@ def assert_objects_count(cluster, objects_count, path='data/'): logging.info("Existing S3 object: %s", str(object_meta)) assert objects_count == len(s3_objects) + # TinyLog: files: id.bin, sizes.json # INSERT overwrites 1 file (`sizes.json`) and appends 1 file (`id.bin`), so # files_overhead=1, files_overhead_per_insert=1 @@ -43,11 +46,16 @@ def assert_objects_count(cluster, objects_count, path='data/'): # files_overhead=1, files_overhead_per_insert=2 @pytest.mark.parametrize( "log_engine,files_overhead,files_overhead_per_insert", - [("TinyLog", 1, 1), ("Log", 1, 2), ("StripeLog", 1, 2)]) + [("TinyLog", 1, 1), ("Log", 1, 2), ("StripeLog", 1, 2)], +) def test_log_family_s3(cluster, log_engine, files_overhead, files_overhead_per_insert): node = cluster.instances["node"] - node.query("CREATE TABLE s3_test (id UInt64) ENGINE={} SETTINGS disk = 's3'".format(log_engine)) + node.query( + "CREATE TABLE s3_test (id UInt64) ENGINE={} SETTINGS disk = 's3'".format( + log_engine + ) + ) node.query("INSERT INTO s3_test SELECT number FROM numbers(5)") assert node.query("SELECT * FROM s3_test") == "0\n1\n2\n3\n4\n" @@ -58,7 +66,9 @@ def test_log_family_s3(cluster, log_engine, files_overhead, files_overhead_per_i assert_objects_count(cluster, files_overhead_per_insert * 2 + files_overhead) node.query("INSERT INTO s3_test SELECT number + 8 FROM numbers(1)") - assert node.query("SELECT * FROM s3_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n8\n" + assert ( + node.query("SELECT * FROM s3_test order by id") == "0\n1\n2\n3\n4\n5\n6\n7\n8\n" + ) assert_objects_count(cluster, files_overhead_per_insert * 3 + files_overhead) node.query("TRUNCATE TABLE s3_test") diff --git a/tests/integration/test_log_levels_update/test.py b/tests/integration/test_log_levels_update/test.py index f631677a400..86719390f33 100644 --- a/tests/integration/test_log_levels_update/test.py +++ b/tests/integration/test_log_levels_update/test.py @@ -4,14 +4,14 @@ import re from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="log_quries_probability") -node = cluster.add_instance('node', with_zookeeper=False) +node = cluster.add_instance("node", with_zookeeper=False) -config = ''' +config = """ information /var/log/clickhouse-server/clickhouse-server.log -''' +""" @pytest.fixture(scope="module") @@ -25,7 +25,10 @@ def start_cluster(): def get_log(node): - return node.exec_in_container(["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"]) + return node.exec_in_container( + ["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"] + ) + def test_log_levels_update(start_cluster): # Make sure that there are enough log messages for the test @@ -37,14 +40,13 @@ def test_log_levels_update(start_cluster): node.replace_config("/etc/clickhouse-server/config.d/log.xml", config) node.query("SYSTEM RELOAD CONFIG;") - node.exec_in_container(["bash", "-c", "> /var/log/clickhouse-server/clickhouse-server.log"]) - + node.exec_in_container( + ["bash", "-c", "> /var/log/clickhouse-server/clickhouse-server.log"] + ) + for i in range(5): node.query("SELECT 1") log = get_log(node) assert len(log) > 0 assert not re.search("(|)", log) - - - diff --git a/tests/integration/test_log_lz4_streaming/test.py b/tests/integration/test_log_lz4_streaming/test.py index 75b46a378c5..05c0c809b5a 100644 --- a/tests/integration/test_log_lz4_streaming/test.py +++ b/tests/integration/test_log_lz4_streaming/test.py @@ -5,7 +5,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/logs.xml'], stay_alive=True) +node = cluster.add_instance("node", main_configs=["configs/logs.xml"], stay_alive=True) + @pytest.fixture(scope="module") def started_cluster(): @@ -20,10 +21,26 @@ def started_cluster(): def check_log_file(): assert node.path_exists("/var/log/clickhouse-server/clickhouse-server.log.lz4") - lz4_output = node.exec_in_container(["bash", "-c", "lz4 -t /var/log/clickhouse-server/clickhouse-server.log.lz4 2>&1"], user='root') - assert lz4_output.count('Error') == 0, lz4_output + lz4_output = node.exec_in_container( + [ + "bash", + "-c", + "lz4 -t /var/log/clickhouse-server/clickhouse-server.log.lz4 2>&1", + ], + user="root", + ) + assert lz4_output.count("Error") == 0, lz4_output - compressed_size = int(node.exec_in_container(["bash", "-c", "du -b /var/log/clickhouse-server/clickhouse-server.log.lz4 | awk {' print $1 '}"], user='root')) + compressed_size = int( + node.exec_in_container( + [ + "bash", + "-c", + "du -b /var/log/clickhouse-server/clickhouse-server.log.lz4 | awk {' print $1 '}", + ], + user="root", + ) + ) uncompressed_size = int(lz4_output.split()[3]) assert 0 < compressed_size < uncompressed_size, lz4_output diff --git a/tests/integration/test_log_query_probability/test.py b/tests/integration/test_log_query_probability/test.py index d1e19974e75..d13ecc276cb 100644 --- a/tests/integration/test_log_query_probability/test.py +++ b/tests/integration/test_log_query_probability/test.py @@ -3,8 +3,8 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="log_quries_probability") -node1 = cluster.add_instance('node1', with_zookeeper=False) -node2 = cluster.add_instance('node2', with_zookeeper=False) +node1 = cluster.add_instance("node1", with_zookeeper=False) +node2 = cluster.add_instance("node2", with_zookeeper=False) @pytest.fixture(scope="module") @@ -19,26 +19,48 @@ def start_cluster(): def test_log_quries_probability_one(start_cluster): for i in range(100): - node1.query("SELECT 12345", settings={"log_queries_probability":0.5}) + node1.query("SELECT 12345", settings={"log_queries_probability": 0.5}) node1.query("SYSTEM FLUSH LOGS") - assert node1.query("SELECT count() < (2 * 100) FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'") == "1\n" - assert node1.query("SELECT count() > 0 FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'") == "1\n" - assert node1.query("SELECT count() % 2 FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'") == "0\n" + assert ( + node1.query( + "SELECT count() < (2 * 100) FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'" + ) + == "1\n" + ) + assert ( + node1.query( + "SELECT count() > 0 FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'" + ) + == "1\n" + ) + assert ( + node1.query( + "SELECT count() % 2 FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'" + ) + == "0\n" + ) node1.query("TRUNCATE TABLE system.query_log") def test_log_quries_probability_two(start_cluster): for i in range(100): - node1.query("SELECT 12345 FROM remote('node2', system, one)", settings={"log_queries_probability":0.5}) + node1.query( + "SELECT 12345 FROM remote('node2', system, one)", + settings={"log_queries_probability": 0.5}, + ) node1.query("SYSTEM FLUSH LOGS") node2.query("SYSTEM FLUSH LOGS") - ans1 = node1.query("SELECT count() FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'") - ans2 = node2.query("SELECT count() FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'") + ans1 = node1.query( + "SELECT count() FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'" + ) + ans2 = node2.query( + "SELECT count() FROM system.query_log WHERE query LIKE '%12345%' AND query NOT LIKE '%system.query_log%'" + ) assert ans1 == ans2 diff --git a/tests/integration/test_logs_level/test.py b/tests/integration/test_logs_level/test.py index 9aa3f7ffd9a..7262861944f 100644 --- a/tests/integration/test_logs_level/test.py +++ b/tests/integration/test_logs_level/test.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/config_information.xml']) +node = cluster.add_instance("node", main_configs=["configs/config_information.xml"]) @pytest.fixture(scope="module") @@ -16,5 +16,7 @@ def start_cluster(): def test_check_client_logs_level(start_cluster): - logs = node.query_and_get_answer_with_error("SELECT 1", settings={"send_logs_level": 'trace'})[1] - assert logs.count('Trace') != 0 + logs = node.query_and_get_answer_with_error( + "SELECT 1", settings={"send_logs_level": "trace"} + )[1] + assert logs.count("Trace") != 0 diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 7b2d54a5ea4..405888c552b 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -10,8 +10,9 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -25,16 +26,24 @@ def start_cluster(): def remove_part_from_disk(node, table, part_name): part_path = node.query( - "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() if not part_path: raise Exception("Part " + part_name + "doesn't exist") - node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True) + node.exec_in_container( + ["bash", "-c", "rm -r {p}/*".format(p=part_path)], privileged=True + ) def test_lost_part_same_replica(start_cluster): for node in [node1, node2]: node.query( - "CREATE TABLE mt0 (id UInt64, date Date) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple() PARTITION BY date".format(node.name)) + "CREATE TABLE mt0 (id UInt64, date Date) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple() PARTITION BY date".format( + node.name + ) + ) node1.query("SYSTEM STOP MERGES mt0") node2.query("SYSTEM STOP REPLICATION QUEUES") @@ -43,7 +52,9 @@ def test_lost_part_same_replica(start_cluster): node1.query("INSERT INTO mt0 VALUES ({}, toDate('2020-10-01'))".format(i)) for i in range(20): - parts_to_merge = node1.query("SELECT parts_to_merge FROM system.replication_queue") + parts_to_merge = node1.query( + "SELECT parts_to_merge FROM system.replication_queue" + ) if parts_to_merge: parts_list = list(sorted(ast.literal_eval(parts_to_merge))) print("Got parts list", parts_list) @@ -55,7 +66,7 @@ def test_lost_part_same_replica(start_cluster): victim_part_from_the_middle = random.choice(parts_list[1:-1]) print("Will corrupt part", victim_part_from_the_middle) - remove_part_from_disk(node1, 'mt0', victim_part_from_the_middle) + remove_part_from_disk(node1, "mt0", victim_part_from_the_middle) node1.query("DETACH TABLE mt0") @@ -69,9 +80,15 @@ def test_lost_part_same_replica(start_cluster): break time.sleep(1) else: - assert False, "Still have something in replication queue:\n" + node1.query("SELECT count() FROM system.replication_queue FORMAT Vertical") + assert False, "Still have something in replication queue:\n" + node1.query( + "SELECT count() FROM system.replication_queue FORMAT Vertical" + ) - assert node1.contains_in_log("Created empty part"), "Seems like empty part {} is not created or log message changed".format(victim_part_from_the_middle) + assert node1.contains_in_log( + "Created empty part" + ), "Seems like empty part {} is not created or log message changed".format( + victim_part_from_the_middle + ) assert node1.query("SELECT COUNT() FROM mt0") == "4\n" @@ -80,10 +97,14 @@ def test_lost_part_same_replica(start_cluster): assert_eq_with_retry(node2, "SELECT COUNT() FROM mt0", "4") assert_eq_with_retry(node2, "SELECT COUNT() FROM system.replication_queue", "0") + def test_lost_part_other_replica(start_cluster): for node in [node1, node2]: node.query( - "CREATE TABLE mt1 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', '{}') ORDER BY tuple()".format(node.name)) + "CREATE TABLE mt1 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', '{}') ORDER BY tuple()".format( + node.name + ) + ) node1.query("SYSTEM STOP MERGES mt1") node2.query("SYSTEM STOP REPLICATION QUEUES") @@ -92,7 +113,9 @@ def test_lost_part_other_replica(start_cluster): node1.query("INSERT INTO mt1 VALUES ({})".format(i)) for i in range(20): - parts_to_merge = node1.query("SELECT parts_to_merge FROM system.replication_queue") + parts_to_merge = node1.query( + "SELECT parts_to_merge FROM system.replication_queue" + ) if parts_to_merge: parts_list = list(sorted(ast.literal_eval(parts_to_merge))) print("Got parts list", parts_list) @@ -104,7 +127,7 @@ def test_lost_part_other_replica(start_cluster): victim_part_from_the_middle = random.choice(parts_list[1:-1]) print("Will corrupt part", victim_part_from_the_middle) - remove_part_from_disk(node1, 'mt1', victim_part_from_the_middle) + remove_part_from_disk(node1, "mt1", victim_part_from_the_middle) # other way to detect broken parts node1.query("CHECK TABLE mt1") @@ -117,9 +140,15 @@ def test_lost_part_other_replica(start_cluster): break time.sleep(1) else: - assert False, "Still have something in replication queue:\n" + node2.query("SELECT * FROM system.replication_queue FORMAT Vertical") + assert False, "Still have something in replication queue:\n" + node2.query( + "SELECT * FROM system.replication_queue FORMAT Vertical" + ) - assert node1.contains_in_log("Created empty part"), "Seems like empty part {} is not created or log message changed".format(victim_part_from_the_middle) + assert node1.contains_in_log( + "Created empty part" + ), "Seems like empty part {} is not created or log message changed".format( + victim_part_from_the_middle + ) assert_eq_with_retry(node2, "SELECT COUNT() FROM mt1", "4") assert_eq_with_retry(node2, "SELECT COUNT() FROM system.replication_queue", "0") @@ -129,10 +158,14 @@ def test_lost_part_other_replica(start_cluster): assert_eq_with_retry(node1, "SELECT COUNT() FROM mt1", "4") assert_eq_with_retry(node1, "SELECT COUNT() FROM system.replication_queue", "0") + def test_lost_part_mutation(start_cluster): for node in [node1, node2]: node.query( - "CREATE TABLE mt2 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t2', '{}') ORDER BY tuple()".format(node.name)) + "CREATE TABLE mt2 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t2', '{}') ORDER BY tuple()".format( + node.name + ) + ) node1.query("SYSTEM STOP MERGES mt2") node2.query("SYSTEM STOP REPLICATION QUEUES") @@ -140,7 +173,9 @@ def test_lost_part_mutation(start_cluster): for i in range(2): node1.query("INSERT INTO mt2 VALUES ({})".format(i)) - node1.query("ALTER TABLE mt2 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"}) + node1.query( + "ALTER TABLE mt2 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} + ) for i in range(20): parts_to_mutate = node1.query("SELECT count() FROM system.replication_queue") @@ -149,7 +184,7 @@ def test_lost_part_mutation(start_cluster): break time.sleep(1) - remove_part_from_disk(node1, 'mt2', 'all_1_1_0') + remove_part_from_disk(node1, "mt2", "all_1_1_0") # other way to detect broken parts node1.query("CHECK TABLE mt2") @@ -162,7 +197,9 @@ def test_lost_part_mutation(start_cluster): break time.sleep(1) else: - assert False, "Still have something in replication queue:\n" + node1.query("SELECT * FROM system.replication_queue FORMAT Vertical") + assert False, "Still have something in replication queue:\n" + node1.query( + "SELECT * FROM system.replication_queue FORMAT Vertical" + ) assert_eq_with_retry(node1, "SELECT COUNT() FROM mt2", "1") assert_eq_with_retry(node1, "SELECT SUM(id) FROM mt2", "777") @@ -179,7 +216,8 @@ def test_lost_last_part(start_cluster): for node in [node1, node2]: node.query( "CREATE TABLE mt3 (id UInt64, p String) ENGINE ReplicatedMergeTree('/clickhouse/tables/t3', '{}') " - "ORDER BY tuple() PARTITION BY p".format(node.name)) + "ORDER BY tuple() PARTITION BY p".format(node.name) + ) node1.query("SYSTEM STOP MERGES mt3") node2.query("SYSTEM STOP REPLICATION QUEUES") @@ -188,10 +226,12 @@ def test_lost_last_part(start_cluster): node1.query("INSERT INTO mt3 VALUES ({}, 'x')".format(i)) # actually not important - node1.query("ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"}) + node1.query( + "ALTER TABLE mt3 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} + ) partition_id = node1.query("select partitionId('x')").strip() - remove_part_from_disk(node1, 'mt3', '{}_0_0_0'.format(partition_id)) + remove_part_from_disk(node1, "mt3", "{}_0_0_0".format(partition_id)) # other way to detect broken parts node1.query("CHECK TABLE mt3") @@ -200,8 +240,12 @@ def test_lost_last_part(start_cluster): for i in range(10): result = node1.query("SELECT count() FROM system.replication_queue") - assert int(result) <= 1, "Have a lot of entries in queue {}".format(node1.query("SELECT * FROM system.replication_queue FORMAT Vertical")) - if node1.contains_in_log("Cannot create empty part") and node1.contains_in_log("DROP/DETACH PARTITION"): + assert int(result) <= 1, "Have a lot of entries in queue {}".format( + node1.query("SELECT * FROM system.replication_queue FORMAT Vertical") + ) + if node1.contains_in_log("Cannot create empty part") and node1.contains_in_log( + "DROP/DETACH PARTITION" + ): break time.sleep(1) else: diff --git a/tests/integration/test_lost_part_during_startup/test.py b/tests/integration/test_lost_part_during_startup/test.py index f9d24682354..b110a17704b 100644 --- a/tests/integration/test_lost_part_during_startup/test.py +++ b/tests/integration/test_lost_part_during_startup/test.py @@ -6,8 +6,8 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, stay_alive=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, stay_alive=True) +node1 = cluster.add_instance("node1", with_zookeeper=True, stay_alive=True) +node2 = cluster.add_instance("node2", with_zookeeper=True, stay_alive=True) @pytest.fixture(scope="module") @@ -22,17 +22,25 @@ def start_cluster(): finally: cluster.shutdown() + def remove_part_from_disk(node, table, part_name): part_path = node.query( - "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format(table, part_name)).strip() + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() if not part_path: raise Exception("Part " + part_name + "doesn't exist") - node.exec_in_container(['bash', '-c', 'rm -r {p}/*'.format(p=part_path)], privileged=True) + node.exec_in_container( + ["bash", "-c", "rm -r {p}/*".format(p=part_path)], privileged=True + ) def test_lost_part_during_startup(start_cluster): for i, node in enumerate([node1, node2]): - node.query(f"CREATE TABLE test_lost (value UInt64) Engine = ReplicatedMergeTree('/clickhouse/test_lost', '{i + 1}') ORDER BY tuple()") + node.query( + f"CREATE TABLE test_lost (value UInt64) Engine = ReplicatedMergeTree('/clickhouse/test_lost', '{i + 1}') ORDER BY tuple()" + ) for i in range(4): node2.query(f"INSERT INTO test_lost VALUES({i})") @@ -40,9 +48,14 @@ def test_lost_part_during_startup(start_cluster): node2.query("OPTIMIZE TABLE test_lost FINAL") node1.query("SYSTEM SYNC REPLICA test_lost") - assert node2.query("SELECT sum(value) FROM test_lost") == str(sum(i for i in range(4))) + '\n' - assert node1.query("SELECT sum(value) FROM test_lost") == str(sum(i for i in range(4))) + '\n' - + assert ( + node2.query("SELECT sum(value) FROM test_lost") + == str(sum(i for i in range(4))) + "\n" + ) + assert ( + node1.query("SELECT sum(value) FROM test_lost") + == str(sum(i for i in range(4))) + "\n" + ) remove_part_from_disk(node2, "test_lost", "all_0_3_1") remove_part_from_disk(node2, "test_lost", "all_1_1_0") @@ -71,5 +84,11 @@ def test_lost_part_during_startup(start_cluster): node2.query("SYSTEM SYNC REPLICA test_lost") node1.query("SYSTEM SYNC REPLICA test_lost") - assert node2.query("SELECT sum(value) FROM test_lost") == str(sum(i for i in range(4)) + sum(i for i in range(7, 13))) + '\n' - assert node1.query("SELECT sum(value) FROM test_lost") == str(sum(i for i in range(4)) + sum(i for i in range(7, 13))) + '\n' + assert ( + node2.query("SELECT sum(value) FROM test_lost") + == str(sum(i for i in range(4)) + sum(i for i in range(7, 13))) + "\n" + ) + assert ( + node1.query("SELECT sum(value) FROM test_lost") + == str(sum(i for i in range(4)) + sum(i for i in range(7, 13))) + "\n" + ) diff --git a/tests/integration/test_match_process_uid_against_data_owner/test.py b/tests/integration/test_match_process_uid_against_data_owner/test.py index cf8a4bc711b..bbcee941833 100644 --- a/tests/integration/test_match_process_uid_against_data_owner/test.py +++ b/tests/integration/test_match_process_uid_against_data_owner/test.py @@ -5,10 +5,11 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True) -other_user_id = pwd.getpwnam('nobody').pw_uid +node = cluster.add_instance("node", stay_alive=True) +other_user_id = pwd.getpwnam("nobody").pw_uid current_user_id = os.getuid() + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -25,14 +26,22 @@ def started_cluster(): def test_different_user(started_cluster): with pytest.raises(Exception): node.stop_clickhouse() - node.exec_in_container(["bash", "-c", f"chown {other_user_id} /var/lib/clickhouse"], privileged=True) + node.exec_in_container( + ["bash", "-c", f"chown {other_user_id} /var/lib/clickhouse"], + privileged=True, + ) node.start_clickhouse(start_wait_sec=3) log = node.grep_in_log("Effective") expected_message = "Effective user of the process \(.*\) does not match the owner of the data \(.*\)\. Run under 'sudo -u .*'\." if re.search(expected_message, log) is None: pytest.fail( - 'Expected the server to fail with a message "{}", but the last message is "{}"'.format(expected_message, log)) - node.exec_in_container(["bash", "-c", f"chown {current_user_id} /var/lib/clickhouse"], privileged=True) + 'Expected the server to fail with a message "{}", but the last message is "{}"'.format( + expected_message, log + ) + ) + node.exec_in_container( + ["bash", "-c", f"chown {current_user_id} /var/lib/clickhouse"], privileged=True + ) node.start_clickhouse() node.rotate_logs() diff --git a/tests/integration/test_materialized_mysql_database/configs/timezone_config.xml b/tests/integration/test_materialized_mysql_database/configs/timezone_config.xml new file mode 100644 index 00000000000..42369fdf488 --- /dev/null +++ b/tests/integration/test_materialized_mysql_database/configs/timezone_config.xml @@ -0,0 +1,3 @@ + + Asia/Istanbul + diff --git a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py index 377a48be7ed..227c872c111 100644 --- a/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialized_mysql_database/materialize_with_ddl.py @@ -12,8 +12,9 @@ import threading from multiprocessing.dummy import Pool from helpers.test_tools import assert_eq_with_retry + def check_query(clickhouse_node, query, result_set, retry_count=10, interval_seconds=3): - lastest_result = '' + lastest_result = "" for i in range(retry_count): try: @@ -27,7 +28,10 @@ def check_query(clickhouse_node, query, result_set, retry_count=10, interval_sec logging.debug(f"check_query retry {i+1} exception {e}") time.sleep(interval_seconds) else: - assert clickhouse_node.query(query) == result_set + result_got = clickhouse_node.query(query) + assert ( + result_got == result_set + ), f"Got result {result_got}, while expected result {result_set}" def dml_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): @@ -36,49 +40,67 @@ def dml_with_materialized_mysql_database(clickhouse_node, mysql_node, service_na mysql_node.query("CREATE DATABASE test_database_dml DEFAULT CHARACTER SET 'utf8'") # existed before the mapping was created - mysql_node.query("CREATE TABLE test_database_dml.test_table_1 (" - "`key` INT NOT NULL PRIMARY KEY, " - "unsigned_tiny_int TINYINT UNSIGNED, tiny_int TINYINT, " - "unsigned_small_int SMALLINT UNSIGNED, small_int SMALLINT, " - "unsigned_medium_int MEDIUMINT UNSIGNED, medium_int MEDIUMINT, " - "unsigned_int INT UNSIGNED, _int INT, " - "unsigned_integer INTEGER UNSIGNED, _integer INTEGER, " - "unsigned_bigint BIGINT UNSIGNED, _bigint BIGINT, " - "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" - "unsigned_float FLOAT UNSIGNED, _float FLOAT, " - "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " - "_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), " - "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" - "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_dml.test_table_1 (" + "`key` INT NOT NULL PRIMARY KEY, " + "unsigned_tiny_int TINYINT UNSIGNED, tiny_int TINYINT, " + "unsigned_small_int SMALLINT UNSIGNED, small_int SMALLINT, " + "unsigned_medium_int MEDIUMINT UNSIGNED, medium_int MEDIUMINT, " + "unsigned_int INT UNSIGNED, _int INT, " + "unsigned_integer INTEGER UNSIGNED, _integer INTEGER, " + "unsigned_bigint BIGINT UNSIGNED, _bigint BIGINT, " + "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" + "unsigned_float FLOAT UNSIGNED, _float FLOAT, " + "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " + "_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), " + "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" + "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;" + ) # it already has some data - mysql_node.query(""" + mysql_node.query( + """ INSERT INTO test_database_dml.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary', '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', true); - """) + """ + ) clickhouse_node.query( "CREATE DATABASE test_database_dml ENGINE = MaterializeMySQL('{}:3306', 'test_database_dml', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_dml" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SELECT * FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", - "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" - "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n") + check_query( + clickhouse_node, + "SELECT * FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", + "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" + "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n", + ) - mysql_node.query(""" + mysql_node.query( + """ INSERT INTO test_database_dml.test_table_1 VALUES(2, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary', '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', false); - """) + """ + ) - check_query(clickhouse_node, "SELECT * FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", - "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" - "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n") + check_query( + clickhouse_node, + "SELECT * FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", + "1\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" + "2020-01-01 00:00:00\t2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" + "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t2020-01-01 00:00:00\t0\n", + ) - mysql_node.query("UPDATE test_database_dml.test_table_1 SET unsigned_tiny_int = 2 WHERE `key` = 1") + mysql_node.query( + "UPDATE test_database_dml.test_table_1 SET unsigned_tiny_int = 2 WHERE `key` = 1" + ) - check_query(clickhouse_node, """ + check_query( + clickhouse_node, + """ SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int, small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, @@ -87,31 +109,46 @@ def dml_with_materialized_mysql_database(clickhouse_node, mysql_node, service_na """, "1\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" "2020-01-01 00:00:00\t1\n2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n") + "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n", + ) # update primary key - mysql_node.query("UPDATE test_database_dml.test_table_1 SET `key` = 3 WHERE `unsigned_tiny_int` = 2") + mysql_node.query( + "UPDATE test_database_dml.test_table_1 SET `key` = 3 WHERE `unsigned_tiny_int` = 2" + ) - check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," - " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, " - " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " - " _bool FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", - "2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" - "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t" - "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t1\n") + check_query( + clickhouse_node, + "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," + " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, " + " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " + " _bool FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", + "2\t1\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\t" + "varchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t0\n3\t2\t-1\t2\t-2\t3\t-3\t" + "4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t2020-01-01 00:00:00\t1\n", + ) - mysql_node.query('DELETE FROM test_database_dml.test_table_1 WHERE `key` = 2') - check_query(clickhouse_node, "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," - " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " - " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, " - " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " - " _bool FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", - "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" - "2020-01-01 00:00:00\t1\n") + mysql_node.query("DELETE FROM test_database_dml.test_table_1 WHERE `key` = 2") + check_query( + clickhouse_node, + "SELECT key, unsigned_tiny_int, tiny_int, unsigned_small_int," + " small_int, unsigned_medium_int, medium_int, unsigned_int, _int, unsigned_integer, _integer, " + " unsigned_bigint, _bigint, unsigned_float, _float, unsigned_double, _double, _varchar, _char, binary_col, " + " _date, _datetime, /* exclude it, because ON UPDATE CURRENT_TIMESTAMP _timestamp, */ " + " _bool FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", + "3\t2\t-1\t2\t-2\t3\t-3\t4\t-4\t5\t-5\t6\t-6\t3.2\t-3.2\t3.4\t-3.4\tvarchar\tchar\tbinary\\0\\0\t2020-01-01\t" + "2020-01-01 00:00:00\t1\n", + ) - mysql_node.query('DELETE FROM test_database_dml.test_table_1 WHERE `unsigned_tiny_int` = 2') - check_query(clickhouse_node, "SELECT * FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", "") + mysql_node.query( + "DELETE FROM test_database_dml.test_table_1 WHERE `unsigned_tiny_int` = 2" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_dml.test_table_1 ORDER BY key FORMAT TSV", + "", + ) clickhouse_node.query("DROP DATABASE test_database_dml") mysql_node.query("DROP DATABASE test_database_dml") @@ -123,109 +160,232 @@ def materialized_mysql_database_with_views(clickhouse_node, mysql_node, service_ mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") # existed before the mapping was created - mysql_node.query("CREATE TABLE test_database.test_table_1 (" - "`key` INT NOT NULL PRIMARY KEY, " - "unsigned_tiny_int TINYINT UNSIGNED, tiny_int TINYINT, " - "unsigned_small_int SMALLINT UNSIGNED, small_int SMALLINT, " - "unsigned_medium_int MEDIUMINT UNSIGNED, medium_int MEDIUMINT, " - "unsigned_int INT UNSIGNED, _int INT, " - "unsigned_integer INTEGER UNSIGNED, _integer INTEGER, " - "unsigned_bigint BIGINT UNSIGNED, _bigint BIGINT, " - "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" - "unsigned_float FLOAT UNSIGNED, _float FLOAT, " - "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " - "_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), " - "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" - "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database.test_table_1 (" + "`key` INT NOT NULL PRIMARY KEY, " + "unsigned_tiny_int TINYINT UNSIGNED, tiny_int TINYINT, " + "unsigned_small_int SMALLINT UNSIGNED, small_int SMALLINT, " + "unsigned_medium_int MEDIUMINT UNSIGNED, medium_int MEDIUMINT, " + "unsigned_int INT UNSIGNED, _int INT, " + "unsigned_integer INTEGER UNSIGNED, _integer INTEGER, " + "unsigned_bigint BIGINT UNSIGNED, _bigint BIGINT, " + "/* Need ClickHouse support read mysql decimal unsigned_decimal DECIMAL(19, 10) UNSIGNED, _decimal DECIMAL(19, 10), */" + "unsigned_float FLOAT UNSIGNED, _float FLOAT, " + "unsigned_double DOUBLE UNSIGNED, _double DOUBLE, " + "_varchar VARCHAR(10), _char CHAR(10), binary_col BINARY(8), " + "/* Need ClickHouse support Enum('a', 'b', 'v') _enum ENUM('a', 'b', 'c'), */" + "_date Date, _datetime DateTime, _timestamp TIMESTAMP, _bool BOOLEAN) ENGINE = InnoDB;" + ) - mysql_node.query("CREATE VIEW test_database.test_table_1_view AS SELECT SUM(tiny_int) FROM test_database.test_table_1 GROUP BY _date;") + mysql_node.query( + "CREATE VIEW test_database.test_table_1_view AS SELECT SUM(tiny_int) FROM test_database.test_table_1 GROUP BY _date;" + ) # it already has some data - mysql_node.query(""" + mysql_node.query( + """ INSERT INTO test_database.test_table_1 VALUES(1, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6, 3.2, -3.2, 3.4, -3.4, 'varchar', 'char', 'binary', '2020-01-01', '2020-01-01 00:00:00', '2020-01-01 00:00:00', true); - """) + """ + ) clickhouse_node.query( "CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n") + check_query( + clickhouse_node, "SHOW TABLES FROM test_database FORMAT TSV", "test_table_1\n" + ) clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") -def materialized_mysql_database_with_datetime_and_decimal(clickhouse_node, mysql_node, service_name): +def materialized_mysql_database_with_datetime_and_decimal( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_dt") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_dt") mysql_node.query("CREATE DATABASE test_database_dt DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test_database_dt.test_table_1 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), _decimal DECIMAL(65, 30)) ENGINE = InnoDB;") - mysql_node.query("INSERT INTO test_database_dt.test_table_1 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + ('9' * 35) + "." + ('9' * 30) + ")") - mysql_node.query("INSERT INTO test_database_dt.test_table_1 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + ('0' * 29) + "1)") - mysql_node.query("INSERT INTO test_database_dt.test_table_1 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + ('9' * 35) + "." + ('9' * 30) + ")") - mysql_node.query("INSERT INTO test_database_dt.test_table_1 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + ('0' * 29) + "1)") + mysql_node.query( + "CREATE TABLE test_database_dt.test_table_1 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), _decimal DECIMAL(65, 30)) ENGINE = InnoDB;" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_1 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + + ("9" * 35) + + "." + + ("9" * 30) + + ")" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_1 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + + ("0" * 29) + + "1)" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_1 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + + ("9" * 35) + + "." + + ("9" * 30) + + ")" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_1 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + + ("0" * 29) + + "1)" + ) - clickhouse_node.query("CREATE DATABASE test_database_dt ENGINE = MaterializedMySQL('{}:3306', 'test_database_dt', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE test_database_dt ENGINE = MaterializedMySQL('{}:3306', 'test_database_dt', 'root', 'clickhouse')".format( + service_name + ) + ) assert "test_database_dt" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SELECT * FROM test_database_dt.test_table_1 ORDER BY key FORMAT TSV", - "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" - "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" - "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + ('9' * 35) + "." + ('9' * 30) + "\n" - "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + ('0' * 29) + "1\n") + check_query( + clickhouse_node, + "SELECT * FROM test_database_dt.test_table_1 ORDER BY key FORMAT TSV", + "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + + ("9" * 35) + + "." + + ("9" * 30) + + "\n" + "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + + ("0" * 29) + + "1\n" + "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + + ("9" * 35) + + "." + + ("9" * 30) + + "\n" + "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + + ("0" * 29) + + "1\n", + ) - mysql_node.query("CREATE TABLE test_database_dt.test_table_2 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), _decimal DECIMAL(65, 30)) ENGINE = InnoDB;") - mysql_node.query("INSERT INTO test_database_dt.test_table_2 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + ('9' * 35) + "." + ('9' * 30) + ")") - mysql_node.query("INSERT INTO test_database_dt.test_table_2 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + ('0' * 29) + "1)") - mysql_node.query("INSERT INTO test_database_dt.test_table_2 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + ('9' * 35) + "." + ('9' * 30) + ")") - mysql_node.query("INSERT INTO test_database_dt.test_table_2 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + ('0' * 29) + "1)") + mysql_node.query( + "CREATE TABLE test_database_dt.test_table_2 (`key` INT NOT NULL PRIMARY KEY, _datetime DateTime(6), _timestamp TIMESTAMP(3), _decimal DECIMAL(65, 30)) ENGINE = InnoDB;" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_2 VALUES(1, '2020-01-01 01:02:03.999999', '2020-01-01 01:02:03.999', " + + ("9" * 35) + + "." + + ("9" * 30) + + ")" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_2 VALUES(2, '2020-01-01 01:02:03.000000', '2020-01-01 01:02:03.000', ." + + ("0" * 29) + + "1)" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_2 VALUES(3, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.99', -" + + ("9" * 35) + + "." + + ("9" * 30) + + ")" + ) + mysql_node.query( + "INSERT INTO test_database_dt.test_table_2 VALUES(4, '2020-01-01 01:02:03.9999', '2020-01-01 01:02:03.9999', -." + + ("0" * 29) + + "1)" + ) - check_query(clickhouse_node, "SELECT * FROM test_database_dt.test_table_2 ORDER BY key FORMAT TSV", - "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + ('9' * 35) + "." + ('9' * 30) + "\n" - "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + ('0' * 29) + "1\n" - "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + ('9' * 35) + "." + ('9' * 30) + "\n" - "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + ('0' * 29) + "1\n") + check_query( + clickhouse_node, + "SELECT * FROM test_database_dt.test_table_2 ORDER BY key FORMAT TSV", + "1\t2020-01-01 01:02:03.999999\t2020-01-01 01:02:03.999\t" + + ("9" * 35) + + "." + + ("9" * 30) + + "\n" + "2\t2020-01-01 01:02:03.000000\t2020-01-01 01:02:03.000\t0." + + ("0" * 29) + + "1\n" + "3\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:03.990\t-" + + ("9" * 35) + + "." + + ("9" * 30) + + "\n" + "4\t2020-01-01 01:02:03.999900\t2020-01-01 01:02:04.000\t-0." + + ("0" * 29) + + "1\n", + ) clickhouse_node.query("DROP DATABASE test_database_dt") mysql_node.query("DROP DATABASE test_database_dt") -def drop_table_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def drop_table_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_drop") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_drop") mysql_node.query("CREATE DATABASE test_database_drop DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test_database_drop.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_drop.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) mysql_node.query("DROP TABLE test_database_drop.test_table_1;") - mysql_node.query("CREATE TABLE test_database_drop.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_drop.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) mysql_node.query("TRUNCATE TABLE test_database_drop.test_table_2;") # create mapping clickhouse_node.query( "CREATE DATABASE test_database_drop ENGINE = MaterializedMySQL('{}:3306', 'test_database_drop', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_drop" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SELECT * FROM test_database_drop.test_table_2 ORDER BY id FORMAT TSV", "") + check_query( + clickhouse_node, + "SELECT * FROM test_database_drop.test_table_2 ORDER BY id FORMAT TSV", + "", + ) - mysql_node.query("INSERT INTO test_database_drop.test_table_2 VALUES(1), (2), (3), (4), (5), (6)") - mysql_node.query("CREATE TABLE test_database_drop.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_drop FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "SELECT * FROM test_database_drop.test_table_2 ORDER BY id FORMAT TSV", - "1\n2\n3\n4\n5\n6\n") + mysql_node.query( + "INSERT INTO test_database_drop.test_table_2 VALUES(1), (2), (3), (4), (5), (6)" + ) + mysql_node.query( + "CREATE TABLE test_database_drop.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_drop FORMAT TSV", + "test_table_1\ntest_table_2\n", + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_drop.test_table_2 ORDER BY id FORMAT TSV", + "1\n2\n3\n4\n5\n6\n", + ) mysql_node.query("DROP TABLE test_database_drop.test_table_1;") mysql_node.query("TRUNCATE TABLE test_database_drop.test_table_2;") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_drop FORMAT TSV", "test_table_2\n") - check_query(clickhouse_node, "SELECT * FROM test_database_drop.test_table_2 ORDER BY id FORMAT TSV", "") + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_drop FORMAT TSV", + "test_table_2\n", + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_drop.test_table_2 ORDER BY id FORMAT TSV", + "", + ) clickhouse_node.query("DROP DATABASE test_database_drop") mysql_node.query("DROP DATABASE test_database_drop") -def create_table_like_with_materialize_mysql_database(clickhouse_node, mysql_node, service_name): +def create_table_like_with_materialize_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS create_like") mysql_node.query("DROP DATABASE IF EXISTS create_like2") clickhouse_node.query("DROP DATABASE IF EXISTS create_like") @@ -236,7 +396,8 @@ def create_table_like_with_materialize_mysql_database(clickhouse_node, mysql_nod mysql_node.query("CREATE TABLE create_like2.t1 LIKE create_like.t1") clickhouse_node.query( - f"CREATE DATABASE create_like ENGINE = MaterializeMySQL('{service_name}:3306', 'create_like', 'root', 'clickhouse')") + f"CREATE DATABASE create_like ENGINE = MaterializeMySQL('{service_name}:3306', 'create_like', 'root', 'clickhouse')" + ) mysql_node.query("CREATE TABLE create_like.t2 LIKE create_like.t1") check_query(clickhouse_node, "SHOW TABLES FROM create_like", "t1\nt2\n") @@ -252,204 +413,388 @@ def create_table_like_with_materialize_mysql_database(clickhouse_node, mysql_nod mysql_node.query("DROP DATABASE create_like2") -def create_table_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def create_table_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_create") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_create") - mysql_node.query("CREATE DATABASE test_database_create DEFAULT CHARACTER SET 'utf8'") + mysql_node.query( + "CREATE DATABASE test_database_create DEFAULT CHARACTER SET 'utf8'" + ) # existed before the mapping was created - mysql_node.query("CREATE TABLE test_database_create.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_create.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) # it already has some data - mysql_node.query("INSERT INTO test_database_create.test_table_1 VALUES(1), (2), (3), (5), (6), (7);") + mysql_node.query( + "INSERT INTO test_database_create.test_table_1 VALUES(1), (2), (3), (5), (6), (7);" + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_create ENGINE = MaterializedMySQL('{}:3306', 'test_database_create', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) # Check for pre-existing status assert "test_database_create" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SELECT * FROM test_database_create.test_table_1 ORDER BY id FORMAT TSV", - "1\n2\n3\n5\n6\n7\n") + check_query( + clickhouse_node, + "SELECT * FROM test_database_create.test_table_1 ORDER BY id FORMAT TSV", + "1\n2\n3\n5\n6\n7\n", + ) - mysql_node.query("CREATE TABLE test_database_create.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") - mysql_node.query("INSERT INTO test_database_create.test_table_2 VALUES(1), (2), (3), (4), (5), (6);") - check_query(clickhouse_node, "SELECT * FROM test_database_create.test_table_2 ORDER BY id FORMAT TSV", - "1\n2\n3\n4\n5\n6\n") + mysql_node.query( + "CREATE TABLE test_database_create.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) + mysql_node.query( + "INSERT INTO test_database_create.test_table_2 VALUES(1), (2), (3), (4), (5), (6);" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_create.test_table_2 ORDER BY id FORMAT TSV", + "1\n2\n3\n4\n5\n6\n", + ) clickhouse_node.query("DROP DATABASE test_database_create") mysql_node.query("DROP DATABASE test_database_create") -def rename_table_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def rename_table_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_rename") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_rename") - mysql_node.query("CREATE DATABASE test_database_rename DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test_database_rename.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") + mysql_node.query( + "CREATE DATABASE test_database_rename DEFAULT CHARACTER SET 'utf8'" + ) + mysql_node.query( + "CREATE TABLE test_database_rename.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) - mysql_node.query("RENAME TABLE test_database_rename.test_table_1 TO test_database_rename.test_table_2") + mysql_node.query( + "RENAME TABLE test_database_rename.test_table_1 TO test_database_rename.test_table_2" + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_rename ENGINE = MaterializedMySQL('{}:3306', 'test_database_rename', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_rename" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_rename FORMAT TSV", "test_table_2\n") - mysql_node.query("RENAME TABLE test_database_rename.test_table_2 TO test_database_rename.test_table_1") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_rename FORMAT TSV", "test_table_1\n") + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_rename FORMAT TSV", + "test_table_2\n", + ) + mysql_node.query( + "RENAME TABLE test_database_rename.test_table_2 TO test_database_rename.test_table_1" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_rename FORMAT TSV", + "test_table_1\n", + ) clickhouse_node.query("DROP DATABASE test_database_rename") mysql_node.query("DROP DATABASE test_database_rename") -def alter_add_column_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def alter_add_column_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_add") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_add") mysql_node.query("CREATE DATABASE test_database_add DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test_database_add.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_add.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) - mysql_node.query("ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_1 INT NOT NULL") - mysql_node.query("ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_2 INT NOT NULL FIRST") - mysql_node.query("ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_3 INT NOT NULL AFTER add_column_1") - mysql_node.query("ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + ( - "0" if service_name == "mysql57" else "(id)")) + mysql_node.query( + "ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_1 INT NOT NULL" + ) + mysql_node.query( + "ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_2 INT NOT NULL FIRST" + ) + mysql_node.query( + "ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_3 INT NOT NULL AFTER add_column_1" + ) + mysql_node.query( + "ALTER TABLE test_database_add.test_table_1 ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + + ("0" if service_name == "mysql57" else "(id)") + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_add ENGINE = MaterializedMySQL('{}:3306', 'test_database_add', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_add" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "DESC test_database_add.test_table_1 FORMAT TSV", - "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("CREATE TABLE test_database_add.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_add FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database_add.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query( + clickhouse_node, + "DESC test_database_add.test_table_1 FORMAT TSV", + "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "ALTER TABLE test_database_add.test_table_2 ADD COLUMN add_column_1 INT NOT NULL, ADD COLUMN add_column_2 INT NOT NULL FIRST") + "CREATE TABLE test_database_add.test_table_2 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_add FORMAT TSV", + "test_table_1\ntest_table_2\n", + ) + check_query( + clickhouse_node, + "DESC test_database_add.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "ALTER TABLE test_database_add.test_table_2 ADD COLUMN add_column_3 INT NOT NULL AFTER add_column_1, ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + ( - "0" if service_name == "mysql57" else "(id)")) + "ALTER TABLE test_database_add.test_table_2 ADD COLUMN add_column_1 INT NOT NULL, ADD COLUMN add_column_2 INT NOT NULL FIRST" + ) + mysql_node.query( + "ALTER TABLE test_database_add.test_table_2 ADD COLUMN add_column_3 INT NOT NULL AFTER add_column_1, ADD COLUMN add_column_4 INT NOT NULL DEFAULT " + + ("0" if service_name == "mysql57" else "(id)") + ) default_expression = "DEFAULT\t0" if service_name == "mysql57" else "DEFAULT\tid" - check_query(clickhouse_node, "DESC test_database_add.test_table_2 FORMAT TSV", - "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t" + default_expression + "\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query( + clickhouse_node, + "DESC test_database_add.test_table_2 FORMAT TSV", + "add_column_2\tInt32\t\t\t\t\t\nid\tInt32\t\t\t\t\t\nadd_column_1\tInt32\t\t\t\t\t\nadd_column_3\tInt32\t\t\t\t\t\nadd_column_4\tInt32\t" + + default_expression + + "\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) - mysql_node.query("INSERT INTO test_database_add.test_table_2 VALUES(1, 2, 3, 4, 5), (6, 7, 8, 9, 10)") - check_query(clickhouse_node, "SELECT * FROM test_database_add.test_table_2 ORDER BY id FORMAT TSV", - "1\t2\t3\t4\t5\n6\t7\t8\t9\t10\n") + mysql_node.query( + "INSERT INTO test_database_add.test_table_2 VALUES(1, 2, 3, 4, 5), (6, 7, 8, 9, 10)" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_add.test_table_2 ORDER BY id FORMAT TSV", + "1\t2\t3\t4\t5\n6\t7\t8\t9\t10\n", + ) clickhouse_node.query("DROP DATABASE test_database_add") mysql_node.query("DROP DATABASE test_database_add") -def alter_drop_column_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def alter_drop_column_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_alter_drop") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_alter_drop") - mysql_node.query("CREATE DATABASE test_database_alter_drop DEFAULT CHARACTER SET 'utf8'") mysql_node.query( - "CREATE TABLE test_database_alter_drop.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;") + "CREATE DATABASE test_database_alter_drop DEFAULT CHARACTER SET 'utf8'" + ) + mysql_node.query( + "CREATE TABLE test_database_alter_drop.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;" + ) - mysql_node.query("ALTER TABLE test_database_alter_drop.test_table_1 DROP COLUMN drop_column") + mysql_node.query( + "ALTER TABLE test_database_alter_drop.test_table_1 DROP COLUMN drop_column" + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_alter_drop ENGINE = MaterializedMySQL('{}:3306', 'test_database_alter_drop', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_alter_drop" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_alter_drop FORMAT TSV", "test_table_1\n") - check_query(clickhouse_node, "DESC test_database_alter_drop.test_table_1 FORMAT TSV", - "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_alter_drop FORMAT TSV", + "test_table_1\n", + ) + check_query( + clickhouse_node, + "DESC test_database_alter_drop.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "CREATE TABLE test_database_alter_drop.test_table_2 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_alter_drop FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database_alter_drop.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database_alter_drop.test_table_2 DROP COLUMN drop_column") - check_query(clickhouse_node, "DESC test_database_alter_drop.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "CREATE TABLE test_database_alter_drop.test_table_2 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_alter_drop FORMAT TSV", + "test_table_1\ntest_table_2\n", + ) + check_query( + clickhouse_node, + "DESC test_database_alter_drop.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE test_database_alter_drop.test_table_2 DROP COLUMN drop_column" + ) + check_query( + clickhouse_node, + "DESC test_database_alter_drop.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) - mysql_node.query("INSERT INTO test_database_alter_drop.test_table_2 VALUES(1), (2), (3), (4), (5)") - check_query(clickhouse_node, "SELECT * FROM test_database_alter_drop.test_table_2 ORDER BY id FORMAT TSV", "1\n2\n3\n4\n5\n") + mysql_node.query( + "INSERT INTO test_database_alter_drop.test_table_2 VALUES(1), (2), (3), (4), (5)" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_alter_drop.test_table_2 ORDER BY id FORMAT TSV", + "1\n2\n3\n4\n5\n", + ) clickhouse_node.query("DROP DATABASE test_database_alter_drop") mysql_node.query("DROP DATABASE test_database_alter_drop") -def alter_rename_column_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def alter_rename_column_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_alter_rename") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_alter_rename") - mysql_node.query("CREATE DATABASE test_database_alter_rename DEFAULT CHARACTER SET 'utf8'") + mysql_node.query( + "CREATE DATABASE test_database_alter_rename DEFAULT CHARACTER SET 'utf8'" + ) # maybe should test rename primary key? mysql_node.query( - "CREATE TABLE test_database_alter_rename.test_table_1 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;") + "CREATE TABLE test_database_alter_rename.test_table_1 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;" + ) - mysql_node.query("ALTER TABLE test_database_alter_rename.test_table_1 RENAME COLUMN rename_column TO new_column_name") + mysql_node.query( + "ALTER TABLE test_database_alter_rename.test_table_1 RENAME COLUMN rename_column TO new_column_name" + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_alter_rename ENGINE = MaterializedMySQL('{}:3306', 'test_database_alter_rename', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_alter_rename" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "DESC test_database_alter_rename.test_table_1 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query( + clickhouse_node, + "DESC test_database_alter_rename.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "CREATE TABLE test_database_alter_rename.test_table_2 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;") - check_query(clickhouse_node, "DESC test_database_alter_rename.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nrename_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database_alter_rename.test_table_2 RENAME COLUMN rename_column TO new_column_name") - check_query(clickhouse_node, "DESC test_database_alter_rename.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "CREATE TABLE test_database_alter_rename.test_table_2 (id INT NOT NULL PRIMARY KEY, rename_column INT NOT NULL) ENGINE = InnoDB;" + ) + check_query( + clickhouse_node, + "DESC test_database_alter_rename.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nrename_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE test_database_alter_rename.test_table_2 RENAME COLUMN rename_column TO new_column_name" + ) + check_query( + clickhouse_node, + "DESC test_database_alter_rename.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nnew_column_name\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) - mysql_node.query("INSERT INTO test_database_alter_rename.test_table_2 VALUES(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)") - check_query(clickhouse_node, "SELECT * FROM test_database_alter_rename.test_table_2 ORDER BY id FORMAT TSV", - "1\t2\n3\t4\n5\t6\n7\t8\n9\t10\n") + mysql_node.query( + "INSERT INTO test_database_alter_rename.test_table_2 VALUES(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_alter_rename.test_table_2 ORDER BY id FORMAT TSV", + "1\t2\n3\t4\n5\t6\n7\t8\n9\t10\n", + ) clickhouse_node.query("DROP DATABASE test_database_alter_rename") mysql_node.query("DROP DATABASE test_database_alter_rename") -def alter_modify_column_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def alter_modify_column_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_alter_modify") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_alter_modify") - mysql_node.query("CREATE DATABASE test_database_alter_modify DEFAULT CHARACTER SET 'utf8'") + mysql_node.query( + "CREATE DATABASE test_database_alter_modify DEFAULT CHARACTER SET 'utf8'" + ) # maybe should test rename primary key? mysql_node.query( - "CREATE TABLE test_database_alter_modify.test_table_1 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;") + "CREATE TABLE test_database_alter_modify.test_table_1 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;" + ) - mysql_node.query("ALTER TABLE test_database_alter_modify.test_table_1 MODIFY COLUMN modify_column INT") + mysql_node.query( + "ALTER TABLE test_database_alter_modify.test_table_1 MODIFY COLUMN modify_column INT" + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_alter_modify ENGINE = MaterializedMySQL('{}:3306', 'test_database_alter_modify', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_alter_modify" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_alter_modify FORMAT TSV", "test_table_1\n") - check_query(clickhouse_node, "DESC test_database_alter_modify.test_table_1 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_alter_modify FORMAT TSV", + "test_table_1\n", + ) + check_query( + clickhouse_node, + "DESC test_database_alter_modify.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "CREATE TABLE test_database_alter_modify.test_table_2 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_alter_modify FORMAT TSV", "test_table_1\ntest_table_2\n") - check_query(clickhouse_node, "DESC test_database_alter_modify.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nmodify_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database_alter_modify.test_table_2 MODIFY COLUMN modify_column INT") - check_query(clickhouse_node, "DESC test_database_alter_modify.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database_alter_modify.test_table_2 MODIFY COLUMN modify_column INT FIRST") - check_query(clickhouse_node, "DESC test_database_alter_modify.test_table_2 FORMAT TSV", - "modify_column\tNullable(Int32)\t\t\t\t\t\nid\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE test_database_alter_modify.test_table_2 MODIFY COLUMN modify_column INT AFTER id") - check_query(clickhouse_node, "DESC test_database_alter_modify.test_table_2 FORMAT TSV", - "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "CREATE TABLE test_database_alter_modify.test_table_2 (id INT NOT NULL PRIMARY KEY, modify_column INT NOT NULL) ENGINE = InnoDB;" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_alter_modify FORMAT TSV", + "test_table_1\ntest_table_2\n", + ) + check_query( + clickhouse_node, + "DESC test_database_alter_modify.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE test_database_alter_modify.test_table_2 MODIFY COLUMN modify_column INT" + ) + check_query( + clickhouse_node, + "DESC test_database_alter_modify.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE test_database_alter_modify.test_table_2 MODIFY COLUMN modify_column INT FIRST" + ) + check_query( + clickhouse_node, + "DESC test_database_alter_modify.test_table_2 FORMAT TSV", + "modify_column\tNullable(Int32)\t\t\t\t\t\nid\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE test_database_alter_modify.test_table_2 MODIFY COLUMN modify_column INT AFTER id" + ) + check_query( + clickhouse_node, + "DESC test_database_alter_modify.test_table_2 FORMAT TSV", + "id\tInt32\t\t\t\t\t\nmodify_column\tNullable(Int32)\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) - mysql_node.query("INSERT INTO test_database_alter_modify.test_table_2 VALUES(1, 2), (3, NULL)") - check_query(clickhouse_node, "SELECT * FROM test_database_alter_modify.test_table_2 ORDER BY id FORMAT TSV", "1\t2\n3\t\\N\n") + mysql_node.query( + "INSERT INTO test_database_alter_modify.test_table_2 VALUES(1, 2), (3, NULL)" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_alter_modify.test_table_2 ORDER BY id FORMAT TSV", + "1\t2\n3\t\\N\n", + ) clickhouse_node.query("DROP DATABASE test_database_alter_modify") mysql_node.query("DROP DATABASE test_database_alter_modify") @@ -459,38 +804,76 @@ def alter_modify_column_with_materialized_mysql_database(clickhouse_node, mysql_ # def test_mysql_alter_change_column_for_materialized_mysql_database(started_cluster): # pass -def alter_rename_table_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): + +def alter_rename_table_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_rename_table") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_rename_table") - mysql_node.query("CREATE DATABASE test_database_rename_table DEFAULT CHARACTER SET 'utf8'") mysql_node.query( - "CREATE TABLE test_database_rename_table.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;") + "CREATE DATABASE test_database_rename_table DEFAULT CHARACTER SET 'utf8'" + ) + mysql_node.query( + "CREATE TABLE test_database_rename_table.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT) ENGINE = InnoDB;" + ) mysql_node.query( - "ALTER TABLE test_database_rename_table.test_table_1 DROP COLUMN drop_column, RENAME TO test_database_rename_table.test_table_2, RENAME TO test_database_rename_table.test_table_3") + "ALTER TABLE test_database_rename_table.test_table_1 DROP COLUMN drop_column, RENAME TO test_database_rename_table.test_table_2, RENAME TO test_database_rename_table.test_table_3" + ) # create mapping clickhouse_node.query( "CREATE DATABASE test_database_rename_table ENGINE = MaterializedMySQL('{}:3306', 'test_database_rename_table', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) assert "test_database_rename_table" in clickhouse_node.query("SHOW DATABASES") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_rename_table FORMAT TSV", "test_table_3\n") - check_query(clickhouse_node, "DESC test_database_rename_table.test_table_3 FORMAT TSV", - "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_rename_table FORMAT TSV", + "test_table_3\n", + ) + check_query( + clickhouse_node, + "DESC test_database_rename_table.test_table_3 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "CREATE TABLE test_database_rename_table.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_rename_table FORMAT TSV", "test_table_1\ntest_table_3\n") - check_query(clickhouse_node, "DESC test_database_rename_table.test_table_1 FORMAT TSV", - "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "CREATE TABLE test_database_rename_table.test_table_1 (id INT NOT NULL PRIMARY KEY, drop_column INT NOT NULL) ENGINE = InnoDB;" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_rename_table FORMAT TSV", + "test_table_1\ntest_table_3\n", + ) + check_query( + clickhouse_node, + "DESC test_database_rename_table.test_table_1 FORMAT TSV", + "id\tInt32\t\t\t\t\t\ndrop_column\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) mysql_node.query( - "ALTER TABLE test_database_rename_table.test_table_1 DROP COLUMN drop_column, RENAME TO test_database_rename_table.test_table_2, RENAME TO test_database_rename_table.test_table_4") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_rename_table FORMAT TSV", "test_table_3\ntest_table_4\n") - check_query(clickhouse_node, "DESC test_database_rename_table.test_table_4 FORMAT TSV", - "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + "ALTER TABLE test_database_rename_table.test_table_1 DROP COLUMN drop_column, RENAME TO test_database_rename_table.test_table_2, RENAME TO test_database_rename_table.test_table_4" + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_rename_table FORMAT TSV", + "test_table_3\ntest_table_4\n", + ) + check_query( + clickhouse_node, + "DESC test_database_rename_table.test_table_4 FORMAT TSV", + "id\tInt32\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) - mysql_node.query("INSERT INTO test_database_rename_table.test_table_4 VALUES(1), (2), (3), (4), (5)") - check_query(clickhouse_node, "SELECT * FROM test_database_rename_table.test_table_4 ORDER BY id FORMAT TSV", "1\n2\n3\n4\n5\n") + mysql_node.query( + "INSERT INTO test_database_rename_table.test_table_4 VALUES(1), (2), (3), (4), (5)" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_rename_table.test_table_4 ORDER BY id FORMAT TSV", + "1\n2\n3\n4\n5\n", + ) clickhouse_node.query("DROP DATABASE test_database_rename_table") mysql_node.query("DROP DATABASE test_database_rename_table") @@ -502,12 +885,16 @@ def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name mysql_node.query("CREATE DATABASE test_database_event") mysql_node.query("RESET MASTER") - mysql_node.query("CREATE TABLE test_database_event.t1(a INT NOT NULL PRIMARY KEY, b VARCHAR(255) DEFAULT 'BEGIN')") + mysql_node.query( + "CREATE TABLE test_database_event.t1(a INT NOT NULL PRIMARY KEY, b VARCHAR(255) DEFAULT 'BEGIN')" + ) mysql_node.query("INSERT INTO test_database_event.t1(a) VALUES(1)") clickhouse_node.query( "CREATE DATABASE test_database_event ENGINE = MaterializedMySQL('{}:3306', 'test_database_event', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) # Reject one empty GTID QUERY event with 'BEGIN' and 'COMMIT' mysql_cursor = mysql_node.alloc_connection().cursor(pymysql.cursors.DictCursor) @@ -525,8 +912,14 @@ def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name mysql_node.query("INSERT INTO test_database_event.t1(a) VALUES(2)") mysql_node.query("/* start */ commit /* end */") - check_query(clickhouse_node, "SHOW TABLES FROM test_database_event FORMAT TSV", "t1\n") - check_query(clickhouse_node, "SELECT * FROM test_database_event.t1 ORDER BY a FORMAT TSV", "1\tBEGIN\n2\tBEGIN\n") + check_query( + clickhouse_node, "SHOW TABLES FROM test_database_event FORMAT TSV", "t1\n" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_event.t1 ORDER BY a FORMAT TSV", + "1\tBEGIN\n2\tBEGIN\n", + ) clickhouse_node.query("DROP DATABASE test_database_event") mysql_node.query("DROP DATABASE test_database_event") @@ -537,7 +930,10 @@ def select_without_columns(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE db") mysql_node.query("CREATE TABLE db.t (a INT PRIMARY KEY, b INT)") clickhouse_node.query( - "CREATE DATABASE db ENGINE = MaterializedMySQL('{}:3306', 'db', 'root', 'clickhouse') SETTINGS max_flush_data_time = 100000".format(service_name)) + "CREATE DATABASE db ENGINE = MaterializedMySQL('{}:3306', 'db', 'root', 'clickhouse') SETTINGS max_flush_data_time = 100000".format( + service_name + ) + ) check_query(clickhouse_node, "SHOW TABLES FROM db FORMAT TSV", "t\n") clickhouse_node.query("SYSTEM STOP MERGES db.t") clickhouse_node.query("CREATE VIEW v AS SELECT * FROM db.t") @@ -546,26 +942,51 @@ def select_without_columns(clickhouse_node, mysql_node, service_name): # We need to execute a DDL for flush data buffer mysql_node.query("CREATE TABLE db.temporary(a INT PRIMARY KEY, b INT)") - optimize_on_insert = clickhouse_node.query("SELECT value FROM system.settings WHERE name='optimize_on_insert'").strip() + optimize_on_insert = clickhouse_node.query( + "SELECT value FROM system.settings WHERE name='optimize_on_insert'" + ).strip() if optimize_on_insert == "0": res = ["3\n", "2\n", "2\n"] else: res = ["2\n", "2\n", "1\n"] - check_query(clickhouse_node, "SELECT count((_sign, _version)) FROM db.t FORMAT TSV", res[0]) + check_query( + clickhouse_node, "SELECT count((_sign, _version)) FROM db.t FORMAT TSV", res[0] + ) assert clickhouse_node.query("SELECT count(_sign) FROM db.t FORMAT TSV") == res[1] - assert_eq_with_retry(clickhouse_node, "SELECT count(_version) FROM db.t", res[2].strip(), sleep_time=2, retry_count=3) + assert_eq_with_retry( + clickhouse_node, + "SELECT count(_version) FROM db.t", + res[2].strip(), + sleep_time=2, + retry_count=3, + ) assert clickhouse_node.query("SELECT count() FROM db.t FORMAT TSV") == "1\n" assert clickhouse_node.query("SELECT count(*) FROM db.t FORMAT TSV") == "1\n" - assert clickhouse_node.query("SELECT count() FROM (SELECT * FROM db.t) FORMAT TSV") == "1\n" + assert ( + clickhouse_node.query("SELECT count() FROM (SELECT * FROM db.t) FORMAT TSV") + == "1\n" + ) assert clickhouse_node.query("SELECT count() FROM v FORMAT TSV") == "1\n" - assert clickhouse_node.query("SELECT count() FROM merge('db', 't') FORMAT TSV") == "1\n" - assert clickhouse_node.query("SELECT count() FROM remote('localhost', 'db', 't') FORMAT TSV") == "1\n" + assert ( + clickhouse_node.query("SELECT count() FROM merge('db', 't') FORMAT TSV") + == "1\n" + ) + assert ( + clickhouse_node.query( + "SELECT count() FROM remote('localhost', 'db', 't') FORMAT TSV" + ) + == "1\n" + ) assert clickhouse_node.query("SELECT _part FROM db.t FORMAT TSV") == "0_1_1_0\n" - assert clickhouse_node.query("SELECT _part FROM remote('localhost', 'db', 't') FORMAT TSV") == "0_1_1_0\n" - + assert ( + clickhouse_node.query( + "SELECT _part FROM remote('localhost', 'db', 't') FORMAT TSV" + ) + == "0_1_1_0\n" + ) clickhouse_node.query("DROP VIEW v") clickhouse_node.query("DROP DATABASE db") @@ -575,45 +996,79 @@ def select_without_columns(clickhouse_node, mysql_node, service_name): def insert_with_modify_binlog_checksum(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE DATABASE test_checksum") mysql_node.query("CREATE TABLE test_checksum.t (a INT PRIMARY KEY, b varchar(200))") - clickhouse_node.query("CREATE DATABASE test_checksum ENGINE = MaterializedMySQL('{}:3306', 'test_checksum', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE test_checksum ENGINE = MaterializedMySQL('{}:3306', 'test_checksum', 'root', 'clickhouse')".format( + service_name + ) + ) check_query(clickhouse_node, "SHOW TABLES FROM test_checksum FORMAT TSV", "t\n") mysql_node.query("INSERT INTO test_checksum.t VALUES(1, '1111')") - check_query(clickhouse_node, "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", "1\t1111\n") + check_query( + clickhouse_node, + "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", + "1\t1111\n", + ) mysql_node.query("SET GLOBAL binlog_checksum=NONE") mysql_node.query("INSERT INTO test_checksum.t VALUES(2, '2222')") - check_query(clickhouse_node, "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n") + check_query( + clickhouse_node, + "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", + "1\t1111\n2\t2222\n", + ) mysql_node.query("SET GLOBAL binlog_checksum=CRC32") mysql_node.query("INSERT INTO test_checksum.t VALUES(3, '3333')") - check_query(clickhouse_node, "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", "1\t1111\n2\t2222\n3\t3333\n") + check_query( + clickhouse_node, + "SELECT * FROM test_checksum.t ORDER BY a FORMAT TSV", + "1\t1111\n2\t2222\n3\t3333\n", + ) clickhouse_node.query("DROP DATABASE test_checksum") mysql_node.query("DROP DATABASE test_checksum") -def err_sync_user_privs_with_materialized_mysql_database(clickhouse_node, mysql_node, service_name): +def err_sync_user_privs_with_materialized_mysql_database( + clickhouse_node, mysql_node, service_name +): clickhouse_node.query("DROP DATABASE IF EXISTS priv_err_db") mysql_node.query("DROP DATABASE IF EXISTS priv_err_db") mysql_node.query("CREATE DATABASE priv_err_db DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE priv_err_db.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE priv_err_db.test_table_1 (id INT NOT NULL PRIMARY KEY) ENGINE = InnoDB;" + ) mysql_node.query("INSERT INTO priv_err_db.test_table_1 VALUES(1);") mysql_node.create_min_priv_user("test", "123") mysql_node.result("SHOW GRANTS FOR 'test'@'%';") clickhouse_node.query( "CREATE DATABASE priv_err_db ENGINE = MaterializedMySQL('{}:3306', 'priv_err_db', 'test', '123')".format( - service_name)) + service_name + ) + ) - check_query(clickhouse_node, "SELECT count() FROM priv_err_db.test_table_1 FORMAT TSV", "1\n", 30, 5) + check_query( + clickhouse_node, + "SELECT count() FROM priv_err_db.test_table_1 FORMAT TSV", + "1\n", + 30, + 5, + ) mysql_node.query("INSERT INTO priv_err_db.test_table_1 VALUES(2);") - check_query(clickhouse_node, "SELECT count() FROM priv_err_db.test_table_1 FORMAT TSV", "2\n") + check_query( + clickhouse_node, + "SELECT count() FROM priv_err_db.test_table_1 FORMAT TSV", + "2\n", + ) clickhouse_node.query("DROP DATABASE priv_err_db;") mysql_node.query("REVOKE REPLICATION SLAVE ON *.* FROM 'test'@'%'") clickhouse_node.query( "CREATE DATABASE priv_err_db ENGINE = MaterializedMySQL('{}:3306', 'priv_err_db', 'test', '123')".format( - service_name)) + service_name + ) + ) assert "priv_err_db" in clickhouse_node.query("SHOW DATABASES") assert "test_table_1" not in clickhouse_node.query("SHOW TABLES FROM priv_err_db") clickhouse_node.query("DROP DATABASE priv_err_db") @@ -621,7 +1076,9 @@ def err_sync_user_privs_with_materialized_mysql_database(clickhouse_node, mysql_ mysql_node.query("REVOKE REPLICATION CLIENT, RELOAD ON *.* FROM 'test'@'%'") clickhouse_node.query( "CREATE DATABASE priv_err_db ENGINE = MaterializedMySQL('{}:3306', 'priv_err_db', 'test', '123')".format( - service_name)) + service_name + ) + ) assert "priv_err_db" in clickhouse_node.query("SHOW DATABASES") assert "test_table_1" not in clickhouse_node.query("SHOW TABLES FROM priv_err_db") clickhouse_node.query_with_retry("DETACH DATABASE priv_err_db") @@ -632,7 +1089,7 @@ def err_sync_user_privs_with_materialized_mysql_database(clickhouse_node, mysql_ with pytest.raises(QueryRuntimeException) as exception: clickhouse_node.query("ATTACH DATABASE priv_err_db") - assert 'MySQL SYNC USER ACCESS ERR:' in str(exception.value) + assert "MySQL SYNC USER ACCESS ERR:" in str(exception.value) assert "priv_err_db" not in clickhouse_node.query("SHOW DATABASES") mysql_node.query("GRANT SELECT ON priv_err_db.* TO 'test'@'%'") @@ -645,52 +1102,101 @@ def err_sync_user_privs_with_materialized_mysql_database(clickhouse_node, mysql_ mysql_node.query("DROP USER 'test'@'%'") -def restore_instance_mysql_connections(clickhouse_node, pm, action='REJECT'): +def restore_instance_mysql_connections(clickhouse_node, pm, action="REJECT"): pm._check_instance(clickhouse_node) - pm._delete_rule({'source': clickhouse_node.ip_address, 'destination_port': 3306, 'action': action}) - pm._delete_rule({'destination': clickhouse_node.ip_address, 'source_port': 3306, 'action': action}) + pm._delete_rule( + { + "source": clickhouse_node.ip_address, + "destination_port": 3306, + "action": action, + } + ) + pm._delete_rule( + { + "destination": clickhouse_node.ip_address, + "source_port": 3306, + "action": action, + } + ) time.sleep(5) -def drop_instance_mysql_connections(clickhouse_node, pm, action='REJECT'): + +def drop_instance_mysql_connections(clickhouse_node, pm, action="REJECT"): pm._check_instance(clickhouse_node) - pm._add_rule({'source': clickhouse_node.ip_address, 'destination_port': 3306, 'action': action}) - pm._add_rule({'destination': clickhouse_node.ip_address, 'source_port': 3306, 'action': action}) + pm._add_rule( + { + "source": clickhouse_node.ip_address, + "destination_port": 3306, + "action": action, + } + ) + pm._add_rule( + { + "destination": clickhouse_node.ip_address, + "source_port": 3306, + "action": action, + } + ) time.sleep(5) + def network_partition_test(clickhouse_node, mysql_node, service_name): clickhouse_node.query("DROP DATABASE IF EXISTS test_database_network") clickhouse_node.query("DROP DATABASE IF EXISTS test") mysql_node.query("DROP DATABASE IF EXISTS test_database_network") mysql_node.query("DROP DATABASE IF EXISTS test") mysql_node.query("CREATE DATABASE test_database_network;") - mysql_node.query("CREATE TABLE test_database_network.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_network.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) mysql_node.query("CREATE DATABASE test;") clickhouse_node.query( - "CREATE DATABASE test_database_network ENGINE = MaterializedMySQL('{}:3306', 'test_database_network', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SELECT * FROM test_database_network.test_table", '') + "CREATE DATABASE test_database_network ENGINE = MaterializedMySQL('{}:3306', 'test_database_network', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query(clickhouse_node, "SELECT * FROM test_database_network.test_table", "") with PartitionManager() as pm: drop_instance_mysql_connections(clickhouse_node, pm) - mysql_node.query('INSERT INTO test_database_network.test_table VALUES(1)') - check_query(clickhouse_node, "SELECT * FROM test_database_network.test_table", '') + mysql_node.query("INSERT INTO test_database_network.test_table VALUES(1)") + check_query( + clickhouse_node, "SELECT * FROM test_database_network.test_table", "" + ) with pytest.raises(QueryRuntimeException) as exception: clickhouse_node.query( - "CREATE DATABASE test ENGINE = MaterializedMySQL('{}:3306', 'test', 'root', 'clickhouse')".format(service_name)) + "CREATE DATABASE test ENGINE = MaterializedMySQL('{}:3306', 'test', 'root', 'clickhouse')".format( + service_name + ) + ) assert "Can't connect to MySQL server" in str(exception.value) restore_instance_mysql_connections(clickhouse_node, pm) - check_query(clickhouse_node, "SELECT * FROM test_database_network.test_table FORMAT TSV", '1\n') + check_query( + clickhouse_node, + "SELECT * FROM test_database_network.test_table FORMAT TSV", + "1\n", + ) clickhouse_node.query( - "CREATE DATABASE test ENGINE = MaterializedMySQL('{}:3306', 'test', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SHOW TABLES FROM test_database_network FORMAT TSV", "test_table\n") + "CREATE DATABASE test ENGINE = MaterializedMySQL('{}:3306', 'test', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM test_database_network FORMAT TSV", + "test_table\n", + ) - mysql_node.query("CREATE TABLE test.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + mysql_node.query( + "CREATE TABLE test.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) check_query(clickhouse_node, "SHOW TABLES FROM test FORMAT TSV", "test\n") clickhouse_node.query("DROP DATABASE test_database_network") @@ -705,28 +1211,53 @@ def mysql_kill_sync_thread_restore_test(clickhouse_node, mysql_node, service_nam mysql_node.query("DROP DATABASE IF EXISTS test_database;") mysql_node.query("CREATE DATABASE test_database;") - mysql_node.query("CREATE TABLE test_database.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + mysql_node.query( + "CREATE TABLE test_database.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) mysql_node.query("INSERT INTO test_database.test_table VALUES (1)") mysql_node.query("DROP DATABASE IF EXISTS test_database_auto;") mysql_node.query("CREATE DATABASE test_database_auto;") - mysql_node.query("CREATE TABLE test_database_auto.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + mysql_node.query( + "CREATE TABLE test_database_auto.test_table ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) mysql_node.query("INSERT INTO test_database_auto.test_table VALUES (11)") - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse') SETTINGS max_wait_time_when_mysql_unavailable=-1".format(service_name)) - clickhouse_node.query("CREATE DATABASE test_database_auto ENGINE = MaterializedMySQL('{}:3306', 'test_database_auto', 'root', 'clickhouse')".format(service_name)) - - check_query(clickhouse_node, "SELECT * FROM test_database.test_table FORMAT TSV", '1\n') - check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table FORMAT TSV", '11\n') + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse') SETTINGS max_wait_time_when_mysql_unavailable=-1".format( + service_name + ) + ) + clickhouse_node.query( + "CREATE DATABASE test_database_auto ENGINE = MaterializedMySQL('{}:3306', 'test_database_auto', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, "SELECT * FROM test_database.test_table FORMAT TSV", "1\n" + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_auto.test_table FORMAT TSV", + "11\n", + ) # When ClickHouse dump all history data we can query it on ClickHouse # but it don't mean that the sync thread is already to connect to MySQL. # So After ClickHouse can query data, insert some rows to MySQL. Use this to re-check sync successed. mysql_node.query("INSERT INTO test_database_auto.test_table VALUES (22)") mysql_node.query("INSERT INTO test_database.test_table VALUES (2)") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", '1\n2\n') - check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", '11\n22\n') + check_query( + clickhouse_node, + "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", + "1\n2\n", + ) + check_query( + clickhouse_node, + "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", + "11\n22\n", + ) get_sync_id_query = "SELECT id FROM information_schema.processlist WHERE state LIKE '% has sent all binlog to % waiting for more updates%';" result = mysql_node.query_and_get_data(get_sync_id_query) @@ -745,13 +1276,25 @@ def mysql_kill_sync_thread_restore_test(clickhouse_node, mysql_node, service_nam clickhouse_node.query_with_retry("DETACH DATABASE test_database") clickhouse_node.query("ATTACH DATABASE test_database") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", '1\n2\n') + check_query( + clickhouse_node, + "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", + "1\n2\n", + ) mysql_node.query("INSERT INTO test_database.test_table VALUES (3)") - check_query(clickhouse_node, "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", '1\n2\n3\n') + check_query( + clickhouse_node, + "SELECT * FROM test_database.test_table ORDER BY id FORMAT TSV", + "1\n2\n3\n", + ) mysql_node.query("INSERT INTO test_database_auto.test_table VALUES (33)") - check_query(clickhouse_node, "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", '11\n22\n33\n') + check_query( + clickhouse_node, + "SELECT * FROM test_database_auto.test_table ORDER BY id FORMAT TSV", + "11\n22\n33\n", + ) clickhouse_node.query("DROP DATABASE test_database") clickhouse_node.query("DROP DATABASE test_database_auto") @@ -763,14 +1306,25 @@ def mysql_killed_while_insert(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS kill_mysql_while_insert") clickhouse_node.query("DROP DATABASE IF EXISTS kill_mysql_while_insert") mysql_node.query("CREATE DATABASE kill_mysql_while_insert") - mysql_node.query("CREATE TABLE kill_mysql_while_insert.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") - clickhouse_node.query("CREATE DATABASE kill_mysql_while_insert ENGINE = MaterializedMySQL('{}:3306', 'kill_mysql_while_insert', 'root', 'clickhouse') SETTINGS max_wait_time_when_mysql_unavailable=-1".format(service_name)) - check_query(clickhouse_node, "SHOW TABLES FROM kill_mysql_while_insert FORMAT TSV", 'test\n') + mysql_node.query( + "CREATE TABLE kill_mysql_while_insert.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) + clickhouse_node.query( + "CREATE DATABASE kill_mysql_while_insert ENGINE = MaterializedMySQL('{}:3306', 'kill_mysql_while_insert', 'root', 'clickhouse') SETTINGS max_wait_time_when_mysql_unavailable=-1".format( + service_name + ) + ) + check_query( + clickhouse_node, "SHOW TABLES FROM kill_mysql_while_insert FORMAT TSV", "test\n" + ) try: + def insert(num): for i in range(num): - query = "INSERT INTO kill_mysql_while_insert.test VALUES({v});".format( v = i + 1 ) + query = "INSERT INTO kill_mysql_while_insert.test VALUES({v});".format( + v=i + 1 + ) mysql_node.query(query) t = threading.Thread(target=insert, args=(10000,)) @@ -787,10 +1341,14 @@ def mysql_killed_while_insert(clickhouse_node, mysql_node, service_name): clickhouse_node.query_with_retry("DETACH DATABASE kill_mysql_while_insert") clickhouse_node.query("ATTACH DATABASE kill_mysql_while_insert") - result = mysql_node.query_and_get_data("SELECT COUNT(1) FROM kill_mysql_while_insert.test") + result = mysql_node.query_and_get_data( + "SELECT COUNT(1) FROM kill_mysql_while_insert.test" + ) for row in result: - res = str(row[0]) + '\n' - check_query(clickhouse_node, "SELECT count() FROM kill_mysql_while_insert.test", res) + res = str(row[0]) + "\n" + check_query( + clickhouse_node, "SELECT count() FROM kill_mysql_while_insert.test", res + ) mysql_node.query("DROP DATABASE kill_mysql_while_insert") clickhouse_node.query("DROP DATABASE kill_mysql_while_insert") @@ -799,13 +1357,25 @@ def mysql_killed_while_insert(clickhouse_node, mysql_node, service_name): def clickhouse_killed_while_insert(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS kill_clickhouse_while_insert") mysql_node.query("CREATE DATABASE kill_clickhouse_while_insert") - mysql_node.query("CREATE TABLE kill_clickhouse_while_insert.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") - clickhouse_node.query("CREATE DATABASE kill_clickhouse_while_insert ENGINE = MaterializedMySQL('{}:3306', 'kill_clickhouse_while_insert', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SHOW TABLES FROM kill_clickhouse_while_insert FORMAT TSV", 'test\n') + mysql_node.query( + "CREATE TABLE kill_clickhouse_while_insert.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) + clickhouse_node.query( + "CREATE DATABASE kill_clickhouse_while_insert ENGINE = MaterializedMySQL('{}:3306', 'kill_clickhouse_while_insert', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SHOW TABLES FROM kill_clickhouse_while_insert FORMAT TSV", + "test\n", + ) def insert(num): for i in range(num): - query = "INSERT INTO kill_clickhouse_while_insert.test VALUES({v});".format( v = i + 1 ) + query = "INSERT INTO kill_clickhouse_while_insert.test VALUES({v});".format( + v=i + 1 + ) mysql_node.query(query) t = threading.Thread(target=insert, args=(1000,)) @@ -815,76 +1385,154 @@ def clickhouse_killed_while_insert(clickhouse_node, mysql_node, service_name): clickhouse_node.restart_clickhouse(20, kill=True) t.join() - result = mysql_node.query_and_get_data("SELECT COUNT(1) FROM kill_clickhouse_while_insert.test") + result = mysql_node.query_and_get_data( + "SELECT COUNT(1) FROM kill_clickhouse_while_insert.test" + ) for row in result: - res = str(row[0]) + '\n' - check_query(clickhouse_node, "SELECT count() FROM kill_clickhouse_while_insert.test FORMAT TSV", res) + res = str(row[0]) + "\n" + check_query( + clickhouse_node, + "SELECT count() FROM kill_clickhouse_while_insert.test FORMAT TSV", + res, + ) mysql_node.query("DROP DATABASE kill_clickhouse_while_insert") clickhouse_node.query("DROP DATABASE kill_clickhouse_while_insert") + def utf8mb4_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS utf8mb4_test") clickhouse_node.query("DROP DATABASE IF EXISTS utf8mb4_test") mysql_node.query("CREATE DATABASE utf8mb4_test") - mysql_node.query("CREATE TABLE utf8mb4_test.test (id INT(11) NOT NULL PRIMARY KEY, name VARCHAR(255)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4") + mysql_node.query( + "CREATE TABLE utf8mb4_test.test (id INT(11) NOT NULL PRIMARY KEY, name VARCHAR(255)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4" + ) mysql_node.query("INSERT INTO utf8mb4_test.test VALUES(1, '🦄'),(2, '\u2601')") - clickhouse_node.query("CREATE DATABASE utf8mb4_test ENGINE = MaterializedMySQL('{}:3306', 'utf8mb4_test', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE utf8mb4_test ENGINE = MaterializedMySQL('{}:3306', 'utf8mb4_test', 'root', 'clickhouse')".format( + service_name + ) + ) check_query(clickhouse_node, "SHOW TABLES FROM utf8mb4_test FORMAT TSV", "test\n") - check_query(clickhouse_node, "SELECT id, name FROM utf8mb4_test.test ORDER BY id", "1\t\U0001F984\n2\t\u2601\n") + check_query( + clickhouse_node, + "SELECT id, name FROM utf8mb4_test.test ORDER BY id", + "1\t\U0001F984\n2\t\u2601\n", + ) + def system_parts_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS system_parts_test") clickhouse_node.query("DROP DATABASE IF EXISTS system_parts_test") mysql_node.query("CREATE DATABASE system_parts_test") - mysql_node.query("CREATE TABLE system_parts_test.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") + mysql_node.query( + "CREATE TABLE system_parts_test.test ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) mysql_node.query("INSERT INTO system_parts_test.test VALUES(1),(2),(3)") + def check_active_parts(num): - check_query(clickhouse_node, "SELECT count() FROM system.parts WHERE database = 'system_parts_test' AND table = 'test' AND active = 1", "{}\n".format(num)) - clickhouse_node.query("CREATE DATABASE system_parts_test ENGINE = MaterializedMySQL('{}:3306', 'system_parts_test', 'root', 'clickhouse')".format(service_name)) + check_query( + clickhouse_node, + "SELECT count() FROM system.parts WHERE database = 'system_parts_test' AND table = 'test' AND active = 1", + "{}\n".format(num), + ) + + clickhouse_node.query( + "CREATE DATABASE system_parts_test ENGINE = MaterializedMySQL('{}:3306', 'system_parts_test', 'root', 'clickhouse')".format( + service_name + ) + ) check_active_parts(1) mysql_node.query("INSERT INTO system_parts_test.test VALUES(4),(5),(6)") check_active_parts(2) clickhouse_node.query("OPTIMIZE TABLE system_parts_test.test") check_active_parts(1) + def multi_table_update_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS multi_table_update") clickhouse_node.query("DROP DATABASE IF EXISTS multi_table_update") mysql_node.query("CREATE DATABASE multi_table_update") - mysql_node.query("CREATE TABLE multi_table_update.a (id INT(11) NOT NULL PRIMARY KEY, value VARCHAR(255))") - mysql_node.query("CREATE TABLE multi_table_update.b (id INT(11) NOT NULL PRIMARY KEY, othervalue VARCHAR(255))") + mysql_node.query( + "CREATE TABLE multi_table_update.a (id INT(11) NOT NULL PRIMARY KEY, value VARCHAR(255))" + ) + mysql_node.query( + "CREATE TABLE multi_table_update.b (id INT(11) NOT NULL PRIMARY KEY, othervalue VARCHAR(255))" + ) mysql_node.query("INSERT INTO multi_table_update.a VALUES(1, 'foo')") mysql_node.query("INSERT INTO multi_table_update.b VALUES(1, 'bar')") - clickhouse_node.query("CREATE DATABASE multi_table_update ENGINE = MaterializedMySQL('{}:3306', 'multi_table_update', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE multi_table_update ENGINE = MaterializedMySQL('{}:3306', 'multi_table_update', 'root', 'clickhouse')".format( + service_name + ) + ) check_query(clickhouse_node, "SHOW TABLES FROM multi_table_update", "a\nb\n") - mysql_node.query("UPDATE multi_table_update.a, multi_table_update.b SET value='baz', othervalue='quux' where a.id=b.id") + mysql_node.query( + "UPDATE multi_table_update.a, multi_table_update.b SET value='baz', othervalue='quux' where a.id=b.id" + ) check_query(clickhouse_node, "SELECT * FROM multi_table_update.a", "1\tbaz\n") check_query(clickhouse_node, "SELECT * FROM multi_table_update.b", "1\tquux\n") + def system_tables_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS system_tables_test") clickhouse_node.query("DROP DATABASE IF EXISTS system_tables_test") mysql_node.query("CREATE DATABASE system_tables_test") - mysql_node.query("CREATE TABLE system_tables_test.test (id int NOT NULL PRIMARY KEY) ENGINE=InnoDB") - clickhouse_node.query("CREATE DATABASE system_tables_test ENGINE = MaterializedMySQL('{}:3306', 'system_tables_test', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SELECT partition_key, sorting_key, primary_key FROM system.tables WHERE database = 'system_tables_test' AND name = 'test'", "intDiv(id, 4294967)\tid\tid\n") + mysql_node.query( + "CREATE TABLE system_tables_test.test (id int NOT NULL PRIMARY KEY) ENGINE=InnoDB" + ) + clickhouse_node.query( + "CREATE DATABASE system_tables_test ENGINE = MaterializedMySQL('{}:3306', 'system_tables_test', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SELECT partition_key, sorting_key, primary_key FROM system.tables WHERE database = 'system_tables_test' AND name = 'test'", + "intDiv(id, 4294967)\tid\tid\n", + ) + def materialize_with_column_comments_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS materialize_with_column_comments_test") - clickhouse_node.query("DROP DATABASE IF EXISTS materialize_with_column_comments_test") + clickhouse_node.query( + "DROP DATABASE IF EXISTS materialize_with_column_comments_test" + ) mysql_node.query("CREATE DATABASE materialize_with_column_comments_test") - mysql_node.query("CREATE TABLE materialize_with_column_comments_test.test (id int NOT NULL PRIMARY KEY, value VARCHAR(255) COMMENT 'test comment') ENGINE=InnoDB") - clickhouse_node.query("CREATE DATABASE materialize_with_column_comments_test ENGINE = MaterializedMySQL('{}:3306', 'materialize_with_column_comments_test', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "DESCRIBE TABLE materialize_with_column_comments_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\ttest comment\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE materialize_with_column_comments_test.test MODIFY value VARCHAR(255) COMMENT 'comment test'") - check_query(clickhouse_node, "DESCRIBE TABLE materialize_with_column_comments_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\tcomment test\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("ALTER TABLE materialize_with_column_comments_test.test ADD value2 int COMMENT 'test comment 2'") - check_query(clickhouse_node, "DESCRIBE TABLE materialize_with_column_comments_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\tcomment test\t\t\nvalue2\tNullable(Int32)\t\t\ttest comment 2\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + mysql_node.query( + "CREATE TABLE materialize_with_column_comments_test.test (id int NOT NULL PRIMARY KEY, value VARCHAR(255) COMMENT 'test comment') ENGINE=InnoDB" + ) + clickhouse_node.query( + "CREATE DATABASE materialize_with_column_comments_test ENGINE = MaterializedMySQL('{}:3306', 'materialize_with_column_comments_test', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE materialize_with_column_comments_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\ttest comment\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE materialize_with_column_comments_test.test MODIFY value VARCHAR(255) COMMENT 'comment test'" + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE materialize_with_column_comments_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\tcomment test\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "ALTER TABLE materialize_with_column_comments_test.test ADD value2 int COMMENT 'test comment 2'" + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE materialize_with_column_comments_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(String)\t\t\tcomment test\t\t\nvalue2\tNullable(Int32)\t\t\ttest comment 2\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) clickhouse_node.query("DROP DATABASE materialize_with_column_comments_test") mysql_node.query("DROP DATABASE materialize_with_column_comments_test") + def materialize_with_enum8_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS materialize_with_enum8_test") clickhouse_node.query("DROP DATABASE IF EXISTS materialize_with_enum8_test") @@ -893,20 +1541,49 @@ def materialize_with_enum8_test(clickhouse_node, mysql_node, service_name): enum8_values = "" enum8_values_with_backslash = "" for i in range(1, enum8_values_count): - enum8_values += '\'' + str(i) + "\', " - enum8_values_with_backslash += "\\\'" + str(i) +"\\\' = " + str(i) + ", " - enum8_values += '\'' + str(enum8_values_count) + '\'' - enum8_values_with_backslash += "\\\'" + str(enum8_values_count) +"\\\' = " + str(enum8_values_count) - mysql_node.query("CREATE TABLE materialize_with_enum8_test.test (id int NOT NULL PRIMARY KEY, value ENUM(" + enum8_values + ")) ENGINE=InnoDB") - mysql_node.query("INSERT INTO materialize_with_enum8_test.test (id, value) VALUES (1, '1'),(2, '2')") - clickhouse_node.query("CREATE DATABASE materialize_with_enum8_test ENGINE = MaterializedMySQL('{}:3306', 'materialize_with_enum8_test', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SELECT value FROM materialize_with_enum8_test.test ORDER BY id", "1\n2\n") - mysql_node.query("INSERT INTO materialize_with_enum8_test.test (id, value) VALUES (3, '127')") - check_query(clickhouse_node, "SELECT value FROM materialize_with_enum8_test.test ORDER BY id", "1\n2\n127\n") - check_query(clickhouse_node, "DESCRIBE TABLE materialize_with_enum8_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum8(" + enum8_values_with_backslash + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + enum8_values += "'" + str(i) + "', " + enum8_values_with_backslash += "\\'" + str(i) + "\\' = " + str(i) + ", " + enum8_values += "'" + str(enum8_values_count) + "'" + enum8_values_with_backslash += ( + "\\'" + str(enum8_values_count) + "\\' = " + str(enum8_values_count) + ) + mysql_node.query( + "CREATE TABLE materialize_with_enum8_test.test (id int NOT NULL PRIMARY KEY, value ENUM(" + + enum8_values + + ")) ENGINE=InnoDB" + ) + mysql_node.query( + "INSERT INTO materialize_with_enum8_test.test (id, value) VALUES (1, '1'),(2, '2')" + ) + clickhouse_node.query( + "CREATE DATABASE materialize_with_enum8_test ENGINE = MaterializedMySQL('{}:3306', 'materialize_with_enum8_test', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SELECT value FROM materialize_with_enum8_test.test ORDER BY id", + "1\n2\n", + ) + mysql_node.query( + "INSERT INTO materialize_with_enum8_test.test (id, value) VALUES (3, '127')" + ) + check_query( + clickhouse_node, + "SELECT value FROM materialize_with_enum8_test.test ORDER BY id", + "1\n2\n127\n", + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE materialize_with_enum8_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum8(" + + enum8_values_with_backslash + + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) clickhouse_node.query("DROP DATABASE materialize_with_enum8_test") mysql_node.query("DROP DATABASE materialize_with_enum8_test") + def materialize_with_enum16_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS materialize_with_enum16_test") clickhouse_node.query("DROP DATABASE IF EXISTS materialize_with_enum16_test") @@ -915,20 +1592,49 @@ def materialize_with_enum16_test(clickhouse_node, mysql_node, service_name): enum16_values = "" enum16_values_with_backslash = "" for i in range(1, enum16_values_count): - enum16_values += '\'' + str(i) + "\', " - enum16_values_with_backslash += "\\\'" + str(i) +"\\\' = " + str(i) + ", " - enum16_values += '\'' + str(enum16_values_count) + '\'' - enum16_values_with_backslash += "\\\'" + str(enum16_values_count) +"\\\' = " + str(enum16_values_count) - mysql_node.query("CREATE TABLE materialize_with_enum16_test.test (id int NOT NULL PRIMARY KEY, value ENUM(" + enum16_values + ")) ENGINE=InnoDB") - mysql_node.query("INSERT INTO materialize_with_enum16_test.test (id, value) VALUES (1, '1'),(2, '2')") - clickhouse_node.query("CREATE DATABASE materialize_with_enum16_test ENGINE = MaterializedMySQL('{}:3306', 'materialize_with_enum16_test', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SELECT value FROM materialize_with_enum16_test.test ORDER BY id", "1\n2\n") - mysql_node.query("INSERT INTO materialize_with_enum16_test.test (id, value) VALUES (3, '500')") - check_query(clickhouse_node, "SELECT value FROM materialize_with_enum16_test.test ORDER BY id", "1\n2\n500\n") - check_query(clickhouse_node, "DESCRIBE TABLE materialize_with_enum16_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum16(" + enum16_values_with_backslash + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + enum16_values += "'" + str(i) + "', " + enum16_values_with_backslash += "\\'" + str(i) + "\\' = " + str(i) + ", " + enum16_values += "'" + str(enum16_values_count) + "'" + enum16_values_with_backslash += ( + "\\'" + str(enum16_values_count) + "\\' = " + str(enum16_values_count) + ) + mysql_node.query( + "CREATE TABLE materialize_with_enum16_test.test (id int NOT NULL PRIMARY KEY, value ENUM(" + + enum16_values + + ")) ENGINE=InnoDB" + ) + mysql_node.query( + "INSERT INTO materialize_with_enum16_test.test (id, value) VALUES (1, '1'),(2, '2')" + ) + clickhouse_node.query( + "CREATE DATABASE materialize_with_enum16_test ENGINE = MaterializedMySQL('{}:3306', 'materialize_with_enum16_test', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SELECT value FROM materialize_with_enum16_test.test ORDER BY id", + "1\n2\n", + ) + mysql_node.query( + "INSERT INTO materialize_with_enum16_test.test (id, value) VALUES (3, '500')" + ) + check_query( + clickhouse_node, + "SELECT value FROM materialize_with_enum16_test.test ORDER BY id", + "1\n2\n500\n", + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE materialize_with_enum16_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum16(" + + enum16_values_with_backslash + + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) clickhouse_node.query("DROP DATABASE materialize_with_enum16_test") mysql_node.query("DROP DATABASE materialize_with_enum16_test") + def alter_enum8_to_enum16_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS alter_enum8_to_enum16_test") clickhouse_node.query("DROP DATABASE IF EXISTS alter_enum8_to_enum16_test") @@ -938,106 +1644,219 @@ def alter_enum8_to_enum16_test(clickhouse_node, mysql_node, service_name): enum8_values = "" enum8_values_with_backslash = "" for i in range(1, enum8_values_count): - enum8_values += '\'' + str(i) + "\', " - enum8_values_with_backslash += "\\\'" + str(i) +"\\\' = " + str(i) + ", " - enum8_values += '\'' + str(enum8_values_count) + '\'' - enum8_values_with_backslash += "\\\'" + str(enum8_values_count) +"\\\' = " + str(enum8_values_count) - mysql_node.query("CREATE TABLE alter_enum8_to_enum16_test.test (id int NOT NULL PRIMARY KEY, value ENUM(" + enum8_values + ")) ENGINE=InnoDB") - mysql_node.query("INSERT INTO alter_enum8_to_enum16_test.test (id, value) VALUES (1, '1'),(2, '2')") - clickhouse_node.query("CREATE DATABASE alter_enum8_to_enum16_test ENGINE = MaterializedMySQL('{}:3306', 'alter_enum8_to_enum16_test', 'root', 'clickhouse')".format(service_name)) - mysql_node.query("INSERT INTO alter_enum8_to_enum16_test.test (id, value) VALUES (3, '75')") - check_query(clickhouse_node, "SELECT value FROM alter_enum8_to_enum16_test.test ORDER BY id", "1\n2\n75\n") - check_query(clickhouse_node, "DESCRIBE TABLE alter_enum8_to_enum16_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum8(" + enum8_values_with_backslash + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") + enum8_values += "'" + str(i) + "', " + enum8_values_with_backslash += "\\'" + str(i) + "\\' = " + str(i) + ", " + enum8_values += "'" + str(enum8_values_count) + "'" + enum8_values_with_backslash += ( + "\\'" + str(enum8_values_count) + "\\' = " + str(enum8_values_count) + ) + mysql_node.query( + "CREATE TABLE alter_enum8_to_enum16_test.test (id int NOT NULL PRIMARY KEY, value ENUM(" + + enum8_values + + ")) ENGINE=InnoDB" + ) + mysql_node.query( + "INSERT INTO alter_enum8_to_enum16_test.test (id, value) VALUES (1, '1'),(2, '2')" + ) + clickhouse_node.query( + "CREATE DATABASE alter_enum8_to_enum16_test ENGINE = MaterializedMySQL('{}:3306', 'alter_enum8_to_enum16_test', 'root', 'clickhouse')".format( + service_name + ) + ) + mysql_node.query( + "INSERT INTO alter_enum8_to_enum16_test.test (id, value) VALUES (3, '75')" + ) + check_query( + clickhouse_node, + "SELECT value FROM alter_enum8_to_enum16_test.test ORDER BY id", + "1\n2\n75\n", + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE alter_enum8_to_enum16_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum8(" + + enum8_values_with_backslash + + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) enum16_values_count = 600 enum16_values = "" enum16_values_with_backslash = "" for i in range(1, enum16_values_count): - enum16_values += '\'' + str(i) + "\', " - enum16_values_with_backslash += "\\\'" + str(i) +"\\\' = " + str(i) + ", " - enum16_values += '\'' + str(enum16_values_count) + '\'' - enum16_values_with_backslash += "\\\'" + str(enum16_values_count) +"\\\' = " + str(enum16_values_count) - mysql_node.query("ALTER TABLE alter_enum8_to_enum16_test.test MODIFY COLUMN value ENUM(" + enum16_values + ")") - check_query(clickhouse_node, "DESCRIBE TABLE alter_enum8_to_enum16_test.test", "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum16(" + enum16_values_with_backslash + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n") - mysql_node.query("INSERT INTO alter_enum8_to_enum16_test.test (id, value) VALUES (4, '500')") - check_query(clickhouse_node, "SELECT value FROM alter_enum8_to_enum16_test.test ORDER BY id", "1\n2\n75\n500\n") + enum16_values += "'" + str(i) + "', " + enum16_values_with_backslash += "\\'" + str(i) + "\\' = " + str(i) + ", " + enum16_values += "'" + str(enum16_values_count) + "'" + enum16_values_with_backslash += ( + "\\'" + str(enum16_values_count) + "\\' = " + str(enum16_values_count) + ) + mysql_node.query( + "ALTER TABLE alter_enum8_to_enum16_test.test MODIFY COLUMN value ENUM(" + + enum16_values + + ")" + ) + check_query( + clickhouse_node, + "DESCRIBE TABLE alter_enum8_to_enum16_test.test", + "id\tInt32\t\t\t\t\t\nvalue\tNullable(Enum16(" + + enum16_values_with_backslash + + "))\t\t\t\t\t\n_sign\tInt8\tMATERIALIZED\t1\t\t\t\n_version\tUInt64\tMATERIALIZED\t1\t\t\t\n", + ) + mysql_node.query( + "INSERT INTO alter_enum8_to_enum16_test.test (id, value) VALUES (4, '500')" + ) + check_query( + clickhouse_node, + "SELECT value FROM alter_enum8_to_enum16_test.test ORDER BY id", + "1\n2\n75\n500\n", + ) clickhouse_node.query("DROP DATABASE alter_enum8_to_enum16_test") mysql_node.query("DROP DATABASE alter_enum8_to_enum16_test") + def move_to_prewhere_and_column_filtering(clickhouse_node, mysql_node, service_name): clickhouse_node.query("DROP DATABASE IF EXISTS cond_on_key_col") mysql_node.query("DROP DATABASE IF EXISTS cond_on_key_col") mysql_node.query("CREATE DATABASE cond_on_key_col") - clickhouse_node.query("CREATE DATABASE cond_on_key_col ENGINE = MaterializedMySQL('{}:3306', 'cond_on_key_col', 'root', 'clickhouse')".format(service_name)) - mysql_node.query("create table cond_on_key_col.products (id int primary key, product_id int not null, catalog_id int not null, brand_id int not null, name text)") - mysql_node.query("insert into cond_on_key_col.products (id, name, catalog_id, brand_id, product_id) values (915, 'ertyui', 5287, 15837, 0), (990, 'wer', 1053, 24390, 1), (781, 'qwerty', 1041, 1176, 2);") - mysql_node.query("create table cond_on_key_col.test (id int(11) NOT NULL AUTO_INCREMENT, a int(11) DEFAULT NULL, b int(11) DEFAULT NULL, PRIMARY KEY (id)) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4;") + clickhouse_node.query( + "CREATE DATABASE cond_on_key_col ENGINE = MaterializedMySQL('{}:3306', 'cond_on_key_col', 'root', 'clickhouse')".format( + service_name + ) + ) + mysql_node.query( + "create table cond_on_key_col.products (id int primary key, product_id int not null, catalog_id int not null, brand_id int not null, name text)" + ) + mysql_node.query( + "insert into cond_on_key_col.products (id, name, catalog_id, brand_id, product_id) values (915, 'ertyui', 5287, 15837, 0), (990, 'wer', 1053, 24390, 1), (781, 'qwerty', 1041, 1176, 2);" + ) + mysql_node.query( + "create table cond_on_key_col.test (id int(11) NOT NULL AUTO_INCREMENT, a int(11) DEFAULT NULL, b int(11) DEFAULT NULL, PRIMARY KEY (id)) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4;" + ) mysql_node.query("insert into cond_on_key_col.test values (42, 123, 1);") - mysql_node.query("CREATE TABLE cond_on_key_col.balance_change_record (id bigint(20) NOT NULL AUTO_INCREMENT, type tinyint(4) DEFAULT NULL, value decimal(10,4) DEFAULT NULL, time timestamp NULL DEFAULT NULL, " - "initiative_id varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, passivity_id varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, " - "person_id varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, tenant_code varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, " - "created_time timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', updated_time timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, " - "value_snapshot decimal(10,4) DEFAULT NULL, PRIMARY KEY (id), KEY balance_change_record_initiative_id (person_id) USING BTREE, " - "KEY type (type) USING BTREE, KEY balance_change_record_type (time) USING BTREE, KEY initiative_id (initiative_id) USING BTREE, " - "KEY balance_change_record_tenant_code (passivity_id) USING BTREE, KEY tenant_code (tenant_code) USING BTREE) ENGINE=InnoDB AUTO_INCREMENT=1691049 DEFAULT CHARSET=utf8") - mysql_node.query("insert into cond_on_key_col.balance_change_record values (123, 1, 3.14, null, 'qwe', 'asd', 'zxc', 'rty', null, null, 2.7);") - mysql_node.query("CREATE TABLE cond_on_key_col.test1 (id int(11) NOT NULL AUTO_INCREMENT, c1 varchar(32) NOT NULL, c2 varchar(32), PRIMARY KEY (id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4") - mysql_node.query("insert into cond_on_key_col.test1(c1,c2) values ('a','b'), ('c', null);") - check_query(clickhouse_node, "SELECT DISTINCT P.id, P.name, P.catalog_id FROM cond_on_key_col.products P WHERE P.name ILIKE '%e%' and P.catalog_id=5287", '915\tertyui\t5287\n') - check_query(clickhouse_node, "select count(a) from cond_on_key_col.test where b = 1;", "1\n") - check_query(clickhouse_node, "select id from cond_on_key_col.balance_change_record where type=1;", "123\n") - check_query(clickhouse_node, "select count(c1) from cond_on_key_col.test1 where c2='b';", "1\n") + mysql_node.query( + "CREATE TABLE cond_on_key_col.balance_change_record (id bigint(20) NOT NULL AUTO_INCREMENT, type tinyint(4) DEFAULT NULL, value decimal(10,4) DEFAULT NULL, time timestamp NULL DEFAULT NULL, " + "initiative_id varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, passivity_id varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, " + "person_id varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, tenant_code varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, " + "created_time timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', updated_time timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, " + "value_snapshot decimal(10,4) DEFAULT NULL, PRIMARY KEY (id), KEY balance_change_record_initiative_id (person_id) USING BTREE, " + "KEY type (type) USING BTREE, KEY balance_change_record_type (time) USING BTREE, KEY initiative_id (initiative_id) USING BTREE, " + "KEY balance_change_record_tenant_code (passivity_id) USING BTREE, KEY tenant_code (tenant_code) USING BTREE) ENGINE=InnoDB AUTO_INCREMENT=1691049 DEFAULT CHARSET=utf8" + ) + mysql_node.query( + "insert into cond_on_key_col.balance_change_record values (123, 1, 3.14, null, 'qwe', 'asd', 'zxc', 'rty', null, null, 2.7);" + ) + mysql_node.query( + "CREATE TABLE cond_on_key_col.test1 (id int(11) NOT NULL AUTO_INCREMENT, c1 varchar(32) NOT NULL, c2 varchar(32), PRIMARY KEY (id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" + ) + mysql_node.query( + "insert into cond_on_key_col.test1(c1,c2) values ('a','b'), ('c', null);" + ) + check_query( + clickhouse_node, + "SELECT DISTINCT P.id, P.name, P.catalog_id FROM cond_on_key_col.products P WHERE P.name ILIKE '%e%' and P.catalog_id=5287", + "915\tertyui\t5287\n", + ) + check_query( + clickhouse_node, "select count(a) from cond_on_key_col.test where b = 1;", "1\n" + ) + check_query( + clickhouse_node, + "select id from cond_on_key_col.balance_change_record where type=1;", + "123\n", + ) + check_query( + clickhouse_node, + "select count(c1) from cond_on_key_col.test1 where c2='b';", + "1\n", + ) clickhouse_node.query("DROP DATABASE cond_on_key_col") mysql_node.query("DROP DATABASE cond_on_key_col") + def mysql_settings_test(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS test_database") clickhouse_node.query("DROP DATABASE IF EXISTS test_database") mysql_node.query("CREATE DATABASE test_database") - mysql_node.query("CREATE TABLE test_database.a (id INT(11) NOT NULL PRIMARY KEY, value VARCHAR(255))") + mysql_node.query( + "CREATE TABLE test_database.a (id INT(11) NOT NULL PRIMARY KEY, value VARCHAR(255))" + ) mysql_node.query("INSERT INTO test_database.a VALUES(1, 'foo')") mysql_node.query("INSERT INTO test_database.a VALUES(2, 'bar')") - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SELECT COUNT() FROM test_database.a FORMAT TSV", "2\n") + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, "SELECT COUNT() FROM test_database.a FORMAT TSV", "2\n" + ) - assert clickhouse_node.query("SELECT COUNT(DISTINCT blockNumber()) FROM test_database.a FORMAT TSV") == "2\n" + assert ( + clickhouse_node.query( + "SELECT COUNT(DISTINCT blockNumber()) FROM test_database.a FORMAT TSV" + ) + == "2\n" + ) clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") + def materialized_mysql_large_transaction(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS largetransaction") clickhouse_node.query("DROP DATABASE IF EXISTS largetransaction") mysql_node.query("CREATE DATABASE largetransaction") - mysql_node.query("CREATE TABLE largetransaction.test_table (" - "`key` INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " - "`value` INT NOT NULL) ENGINE = InnoDB;") + mysql_node.query( + "CREATE TABLE largetransaction.test_table (" + "`key` INT NOT NULL PRIMARY KEY AUTO_INCREMENT, " + "`value` INT NOT NULL) ENGINE = InnoDB;" + ) num_rows = 200000 rows_per_insert = 5000 values = ",".join(["(1)" for _ in range(rows_per_insert)]) - for i in range(num_rows//rows_per_insert): - mysql_node.query(f"INSERT INTO largetransaction.test_table (`value`) VALUES {values};") + for i in range(num_rows // rows_per_insert): + mysql_node.query( + f"INSERT INTO largetransaction.test_table (`value`) VALUES {values};" + ) - - clickhouse_node.query("CREATE DATABASE largetransaction ENGINE = MaterializedMySQL('{}:3306', 'largetransaction', 'root', 'clickhouse')".format(service_name)) - check_query(clickhouse_node, "SELECT COUNT() FROM largetransaction.test_table", f"{num_rows}\n") + clickhouse_node.query( + "CREATE DATABASE largetransaction ENGINE = MaterializedMySQL('{}:3306', 'largetransaction', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SELECT COUNT() FROM largetransaction.test_table", + f"{num_rows}\n", + ) mysql_node.query("UPDATE largetransaction.test_table SET value = 2;") # Attempt to restart clickhouse after it has started processing # the transaction, but before it has completed it. - while int(clickhouse_node.query("SELECT COUNT() FROM largetransaction.test_table WHERE value = 2")) == 0: + while ( + int( + clickhouse_node.query( + "SELECT COUNT() FROM largetransaction.test_table WHERE value = 2" + ) + ) + == 0 + ): time.sleep(0.2) clickhouse_node.restart_clickhouse() - check_query(clickhouse_node, "SELECT COUNT() FROM largetransaction.test_table WHERE value = 2", f"{num_rows}\n") + check_query( + clickhouse_node, + "SELECT COUNT() FROM largetransaction.test_table WHERE value = 2", + f"{num_rows}\n", + ) clickhouse_node.query("DROP DATABASE largetransaction") mysql_node.query("DROP DATABASE largetransaction") + def table_table(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS table_test") clickhouse_node.query("DROP DATABASE IF EXISTS table_test") @@ -1047,78 +1866,112 @@ def table_table(clickhouse_node, mysql_node, service_name): mysql_node.query("CREATE TABLE table_test.table (id INT UNSIGNED PRIMARY KEY)") mysql_node.query("INSERT INTO table_test.table VALUES (0),(1),(2),(3),(4)") - clickhouse_node.query("CREATE DATABASE table_test ENGINE=MaterializeMySQL('{}:3306', 'table_test', 'root', 'clickhouse')".format(service_name)) + clickhouse_node.query( + "CREATE DATABASE table_test ENGINE=MaterializeMySQL('{}:3306', 'table_test', 'root', 'clickhouse')".format( + service_name + ) + ) check_query(clickhouse_node, "SELECT COUNT(*) FROM table_test.table", "5\n") mysql_node.query("DROP DATABASE table_test") clickhouse_node.query("DROP DATABASE table_test") + def table_overrides(clickhouse_node, mysql_node, service_name): mysql_node.query("DROP DATABASE IF EXISTS table_overrides") clickhouse_node.query("DROP DATABASE IF EXISTS table_overrides") mysql_node.query("CREATE DATABASE table_overrides") - mysql_node.query("CREATE TABLE table_overrides.t1 (sensor_id INT UNSIGNED, timestamp DATETIME, temperature FLOAT, PRIMARY KEY(timestamp, sensor_id))") + mysql_node.query( + "CREATE TABLE table_overrides.t1 (sensor_id INT UNSIGNED, timestamp DATETIME, temperature FLOAT, PRIMARY KEY(timestamp, sensor_id))" + ) for id in range(10): mysql_node.query("BEGIN") for day in range(100): - mysql_node.query(f"INSERT INTO table_overrides.t1 VALUES({id}, TIMESTAMP('2021-01-01') + INTERVAL {day} DAY, (RAND()*20)+20)") + mysql_node.query( + f"INSERT INTO table_overrides.t1 VALUES({id}, TIMESTAMP('2021-01-01') + INTERVAL {day} DAY, (RAND()*20)+20)" + ) mysql_node.query("COMMIT") - clickhouse_node.query(f""" + clickhouse_node.query( + f""" CREATE DATABASE table_overrides ENGINE=MaterializeMySQL('{service_name}:3306', 'table_overrides', 'root', 'clickhouse') TABLE OVERRIDE t1 (COLUMNS (sensor_id UInt64, temp_f Nullable(Float32) ALIAS if(isNull(temperature), NULL, (temperature * 9 / 5) + 32))) - """) + """ + ) check_query( clickhouse_node, "SELECT type FROM system.columns WHERE database = 'table_overrides' AND table = 't1' AND name = 'sensor_id'", - "UInt64\n") + "UInt64\n", + ) check_query( clickhouse_node, "SELECT type, default_kind FROM system.columns WHERE database = 'table_overrides' AND table = 't1' AND name = 'temp_f'", - "Nullable(Float32)\tALIAS\n") + "Nullable(Float32)\tALIAS\n", + ) check_query(clickhouse_node, "SELECT count() FROM table_overrides.t1", "1000\n") - mysql_node.query("INSERT INTO table_overrides.t1 VALUES(1001, '2021-10-01 00:00:00', 42.0)") + mysql_node.query( + "INSERT INTO table_overrides.t1 VALUES(1001, '2021-10-01 00:00:00', 42.0)" + ) check_query(clickhouse_node, "SELECT count() FROM table_overrides.t1", "1001\n") explain_with_table_func = f"EXPLAIN TABLE OVERRIDE mysql('{service_name}:3306', 'table_overrides', 't1', 'root', 'clickhouse')" - for what in ['ORDER BY', 'PRIMARY KEY', 'SAMPLE BY', 'PARTITION BY', 'TTL']: + for what in ["ORDER BY", "PRIMARY KEY", "SAMPLE BY", "PARTITION BY", "TTL"]: with pytest.raises(QueryRuntimeException) as exc: clickhouse_node.query(f"{explain_with_table_func} {what} temperature") - assert f'{what} override refers to nullable column `temperature`' in \ - str(exc.value) - assert f"{what} uses columns: `temperature` Nullable(Float32)" in \ - clickhouse_node.query(f"{explain_with_table_func} {what} assumeNotNull(temperature)") + assert f"{what} override refers to nullable column `temperature`" in str( + exc.value + ) + assert ( + f"{what} uses columns: `temperature` Nullable(Float32)" + in clickhouse_node.query( + f"{explain_with_table_func} {what} assumeNotNull(temperature)" + ) + ) for testcase in [ - ('COLUMNS (temperature Nullable(Float32) MATERIALIZED 1.0)', - 'column `temperature`: modifying default specifier is not allowed'), - ('COLUMNS (sensor_id UInt64 ALIAS 42)', - 'column `sensor_id`: modifying default specifier is not allowed') + ( + "COLUMNS (temperature Nullable(Float32) MATERIALIZED 1.0)", + "column `temperature`: modifying default specifier is not allowed", + ), + ( + "COLUMNS (sensor_id UInt64 ALIAS 42)", + "column `sensor_id`: modifying default specifier is not allowed", + ), ]: with pytest.raises(QueryRuntimeException) as exc: clickhouse_node.query(f"{explain_with_table_func} {testcase[0]}") assert testcase[1] in str(exc.value) for testcase in [ - ('COLUMNS (temperature Nullable(Float64))', - 'Modified columns: `temperature` Nullable(Float32) -> Nullable(Float64)'), - ('COLUMNS (temp_f Nullable(Float32) ALIAS if(temperature IS NULL, NULL, (temperature * 9.0 / 5.0) + 32),\ - temp_k Nullable(Float32) ALIAS if(temperature IS NULL, NULL, temperature + 273.15))', - 'Added columns: `temp_f` Nullable(Float32), `temp_k` Nullable(Float32)') + ( + "COLUMNS (temperature Nullable(Float64))", + "Modified columns: `temperature` Nullable(Float32) -> Nullable(Float64)", + ), + ( + "COLUMNS (temp_f Nullable(Float32) ALIAS if(temperature IS NULL, NULL, (temperature * 9.0 / 5.0) + 32),\ + temp_k Nullable(Float32) ALIAS if(temperature IS NULL, NULL, temperature + 273.15))", + "Added columns: `temp_f` Nullable(Float32), `temp_k` Nullable(Float32)", + ), ]: assert testcase[1] in clickhouse_node.query( - f"{explain_with_table_func} {testcase[0]}") + f"{explain_with_table_func} {testcase[0]}" + ) clickhouse_node.query("DROP DATABASE IF EXISTS table_overrides") mysql_node.query("DROP DATABASE IF EXISTS table_overrides") -def materialized_database_support_all_kinds_of_mysql_datatype(clickhouse_node, mysql_node, service_name): +def materialized_database_support_all_kinds_of_mysql_datatype( + clickhouse_node, mysql_node, service_name +): mysql_node.query("DROP DATABASE IF EXISTS test_database_datatype") clickhouse_node.query("DROP DATABASE IF EXISTS test_database_datatype") - mysql_node.query("CREATE DATABASE test_database_datatype DEFAULT CHARACTER SET 'utf8'") - mysql_node.query(""" + mysql_node.query( + "CREATE DATABASE test_database_datatype DEFAULT CHARACTER SET 'utf8'" + ) + mysql_node.query( + """ CREATE TABLE test_database_datatype.t1 ( `v1` int(10) unsigned AUTO_INCREMENT, `v2` TINYINT, @@ -1155,31 +2008,146 @@ def materialized_database_support_all_kinds_of_mysql_datatype(clickhouse_node, m `v32` ENUM('RED','GREEN','BLUE'), PRIMARY KEY (`v1`) ) ENGINE=InnoDB; - """) + """ + ) - mysql_node.query(""" + mysql_node.query( + """ INSERT INTO test_database_datatype.t1 (v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v28, v29, v30, v31, v32) values (1, 11, 9223372036854775807, -1, 1, 11, 18446744073709551615, -1.1, 1.1, -1.111, 1.111, 1.1111, '2021-10-06', 'text', 'varchar', 'BLOB', '2021-10-06 18:32:57', '2021-10-06 18:32:57.482786', '2021-10-06 18:32:57', '2021-10-06 18:32:57.482786', '2021', '838:59:59', '838:59:59.000000', ST_GeometryFromText('point(0.0 0.0)'), b'1010', 'a', 11, 'varbinary', 'binary', 'RED'); - """) + """ + ) clickhouse_node.query( "CREATE DATABASE test_database_datatype ENGINE = MaterializeMySQL('{}:3306', 'test_database_datatype', 'root', 'clickhouse')".format( - service_name)) + service_name + ) + ) - check_query(clickhouse_node, "SELECT name FROM system.tables WHERE database = 'test_database_datatype'", "t1\n") + check_query( + clickhouse_node, + "SELECT name FROM system.tables WHERE database = 'test_database_datatype'", + "t1\n", + ) # full synchronization check - check_query(clickhouse_node, "SELECT v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, hex(v25), v26, v28, v29, v30, v32 FROM test_database_datatype.t1 FORMAT TSV", - "1\t1\t11\t9223372036854775807\t-1\t1\t11\t18446744073709551615\t-1.1\t1.1\t-1.111\t1.111\t1.1111\t2021-10-06\ttext\tvarchar\tBLOB\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786\t2021-10-06 18:32:57" + - "\t2021-10-06 18:32:57.482786\t2021\t3020399000000\t3020399000000\t00000000010100000000000000000000000000000000000000\t10\t1\t11\tvarbinary\tRED\n") + check_query( + clickhouse_node, + "SELECT v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, hex(v25), v26, v28, v29, v30, v32 FROM test_database_datatype.t1 FORMAT TSV", + "1\t1\t11\t9223372036854775807\t-1\t1\t11\t18446744073709551615\t-1.1\t1.1\t-1.111\t1.111\t1.1111\t2021-10-06\ttext\tvarchar\tBLOB\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786\t2021-10-06 18:32:57" + + "\t2021-10-06 18:32:57.482786\t2021\t3020399000000\t3020399000000\t00000000010100000000000000000000000000000000000000\t10\t1\t11\tvarbinary\tRED\n", + ) - mysql_node.query(""" + mysql_node.query( + """ INSERT INTO test_database_datatype.t1 (v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v28, v29, v30, v31, v32) values (2, 22, 9223372036854775807, -2, 2, 22, 18446744073709551615, -2.2, 2.2, -2.22, 2.222, 2.2222, '2021-10-07', 'text', 'varchar', 'BLOB', '2021-10-07 18:32:57', '2021-10-07 18:32:57.482786', '2021-10-07 18:32:57', '2021-10-07 18:32:57.482786', '2021', '-838:59:59', '-12:59:58.000001', ST_GeometryFromText('point(120.153576 30.287459)'), b'1011', 'a,c', 22, 'varbinary', 'binary', 'GREEN' ); - """) + """ + ) # increment synchronization check - check_query(clickhouse_node, "SELECT v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, hex(v25), v26, v28, v29, v30, v32 FROM test_database_datatype.t1 FORMAT TSV", - "1\t1\t11\t9223372036854775807\t-1\t1\t11\t18446744073709551615\t-1.1\t1.1\t-1.111\t1.111\t1.1111\t2021-10-06\ttext\tvarchar\tBLOB\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786" + - "\t2021\t3020399000000\t3020399000000\t00000000010100000000000000000000000000000000000000\t10\t1\t11\tvarbinary\tRED\n" + - "2\t2\t22\t9223372036854775807\t-2\t2\t22\t18446744073709551615\t-2.2\t2.2\t-2.22\t2.222\t2.2222\t2021-10-07\ttext\tvarchar\tBLOB\t2021-10-07 18:32:57\t2021-10-07 18:32:57.482786\t2021-10-07 18:32:57\t2021-10-07 18:32:57.482786" + - "\t2021\t-3020399000000\t-46798000001\t000000000101000000D55C6E30D4095E40DCF0BBE996493E40\t11\t3\t22\tvarbinary\tGREEN\n") + check_query( + clickhouse_node, + "SELECT v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, hex(v25), v26, v28, v29, v30, v32 FROM test_database_datatype.t1 FORMAT TSV", + "1\t1\t11\t9223372036854775807\t-1\t1\t11\t18446744073709551615\t-1.1\t1.1\t-1.111\t1.111\t1.1111\t2021-10-06\ttext\tvarchar\tBLOB\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786\t2021-10-06 18:32:57\t2021-10-06 18:32:57.482786" + + "\t2021\t3020399000000\t3020399000000\t00000000010100000000000000000000000000000000000000\t10\t1\t11\tvarbinary\tRED\n" + + "2\t2\t22\t9223372036854775807\t-2\t2\t22\t18446744073709551615\t-2.2\t2.2\t-2.22\t2.222\t2.2222\t2021-10-07\ttext\tvarchar\tBLOB\t2021-10-07 18:32:57\t2021-10-07 18:32:57.482786\t2021-10-07 18:32:57\t2021-10-07 18:32:57.482786" + + "\t2021\t-3020399000000\t-46798000001\t000000000101000000D55C6E30D4095E40DCF0BBE996493E40\t11\t3\t22\tvarbinary\tGREEN\n", + ) + + +def materialized_database_settings_materialized_mysql_tables_list( + clickhouse_node, mysql_node, service_name +): + mysql_node.query("DROP DATABASE IF EXISTS test_database") + clickhouse_node.query("DROP DATABASE IF EXISTS test_database") + mysql_node.query("CREATE DATABASE test_database") + mysql_node.query( + "CREATE TABLE test_database.a (id INT(11) NOT NULL PRIMARY KEY, value VARCHAR(255))" + ) + mysql_node.query("INSERT INTO test_database.a VALUES(1, 'foo')") + mysql_node.query("INSERT INTO test_database.a VALUES(2, 'bar')") + # table b(include json type, not in materialized_mysql_tables_list) can be skip + mysql_node.query( + "CREATE TABLE test_database.b (id INT(11) NOT NULL PRIMARY KEY, value JSON)" + ) + + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse') SETTINGS materialized_mysql_tables_list = ' a,c,d'".format( + service_name + ) + ) + + check_query( + clickhouse_node, + "SELECT name from system.tables where database = 'test_database' FORMAT TSV", + "a\n", + ) + check_query( + clickhouse_node, "SELECT COUNT() FROM test_database.a FORMAT TSV", "2\n" + ) + + # mysql data(binlog) can be skip + mysql_node.query('INSERT INTO test_database.b VALUES(1, \'{"name":"testjson"}\')') + mysql_node.query('INSERT INTO test_database.b VALUES(2, \'{"name":"testjson"}\')') + + # irrelevant database can be skip + mysql_node.query("DROP DATABASE IF EXISTS other_database") + mysql_node.query("CREATE DATABASE other_database") + mysql_node.query( + "CREATE TABLE other_database.d (id INT(11) NOT NULL PRIMARY KEY, value json)" + ) + mysql_node.query('INSERT INTO other_database.d VALUES(1, \'{"name":"testjson"}\')') + + mysql_node.query( + "CREATE TABLE test_database.c (id INT(11) NOT NULL PRIMARY KEY, value VARCHAR(255))" + ) + mysql_node.query("INSERT INTO test_database.c VALUES(1, 'foo')") + mysql_node.query("INSERT INTO test_database.c VALUES(2, 'bar')") + + check_query( + clickhouse_node, + "SELECT name from system.tables where database = 'test_database' FORMAT TSV", + "a\nc\n", + ) + check_query( + clickhouse_node, "SELECT COUNT() FROM test_database.c FORMAT TSV", "2\n" + ) + + clickhouse_node.query("DROP DATABASE test_database") + mysql_node.query("DROP DATABASE test_database") + + +def materialized_database_mysql_date_type_to_date32( + clickhouse_node, mysql_node, service_name +): + mysql_node.query("DROP DATABASE IF EXISTS test_database") + clickhouse_node.query("DROP DATABASE IF EXISTS test_database") + mysql_node.query("CREATE DATABASE test_database") + mysql_node.query( + "CREATE TABLE test_database.a (a INT(11) NOT NULL PRIMARY KEY, b date DEFAULT NULL)" + ) + # can't support date that less than 1925 year for now + mysql_node.query("INSERT INTO test_database.a VALUES(1, '1900-04-16')") + # test date that is older than 1925 + mysql_node.query("INSERT INTO test_database.a VALUES(3, '1971-02-16')") + mysql_node.query("INSERT INTO test_database.a VALUES(4, '2101-05-16')") + + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MaterializedMySQL('{}:3306', 'test_database', 'root', 'clickhouse')".format( + service_name + ) + ) + check_query( + clickhouse_node, + "SELECT b from test_database.a order by a FORMAT TSV", + "1970-01-01\n1971-02-16\n2101-05-16\n", + ) + + mysql_node.query("INSERT INTO test_database.a VALUES(6, '2022-02-16')") + mysql_node.query("INSERT INTO test_database.a VALUES(7, '2104-06-06')") + + check_query( + clickhouse_node, + "SELECT b from test_database.a order by a FORMAT TSV", + "1970-01-01\n1971-02-16\n2101-05-16\n2022-02-16\n" + "2104-06-06\n", + ) diff --git a/tests/integration/test_materialized_mysql_database/test.py b/tests/integration/test_materialized_mysql_database/test.py index 501c0cd78fa..a672ec72275 100644 --- a/tests/integration/test_materialized_mysql_database/test.py +++ b/tests/integration/test_materialized_mysql_database/test.py @@ -5,7 +5,12 @@ import pwd import re import pymysql.cursors import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance, get_docker_compose_path, run_and_check +from helpers.cluster import ( + ClickHouseCluster, + ClickHouseInstance, + get_docker_compose_path, + run_and_check, +) import docker import logging @@ -17,9 +22,28 @@ cluster = ClickHouseCluster(__file__) mysql_node = None mysql8_node = None -node_db = cluster.add_instance('node1', user_configs=["configs/users.xml"], with_mysql=True, with_mysql8=True, stay_alive=True) -node_disable_bytes_settings = cluster.add_instance('node2', user_configs=["configs/users_disable_bytes_settings.xml"], with_mysql=False, stay_alive=True) -node_disable_rows_settings = cluster.add_instance('node3', user_configs=["configs/users_disable_rows_settings.xml"], with_mysql=False, stay_alive=True) +node_db = cluster.add_instance( + "node1", + main_configs=["configs/timezone_config.xml"], + user_configs=["configs/users.xml"], + with_mysql=True, + with_mysql8=True, + stay_alive=True, +) +node_disable_bytes_settings = cluster.add_instance( + "node2", + main_configs=["configs/timezone_config.xml"], + user_configs=["configs/users_disable_bytes_settings.xml"], + with_mysql=False, + stay_alive=True, +) +node_disable_rows_settings = cluster.add_instance( + "node3", + main_configs=["configs/timezone_config.xml"], + user_configs=["configs/users_disable_rows_settings.xml"], + with_mysql=False, + stay_alive=True, +) @pytest.fixture(scope="module") @@ -32,7 +56,15 @@ def started_cluster(): class MySQLConnection: - def __init__(self, port, user='root', password='clickhouse', ip_address=None, docker_compose=None, project_name=cluster.project_name): + def __init__( + self, + port, + user="root", + password="clickhouse", + ip_address=None, + docker_compose=None, + project_name=cluster.project_name, + ): self.user = user self.port = port self.ip_address = ip_address @@ -44,11 +76,20 @@ class MySQLConnection: for _ in range(5): try: if self.mysql_connection is None: - self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.ip_address, - port=self.port, autocommit=True) + self.mysql_connection = pymysql.connect( + user=self.user, + password=self.password, + host=self.ip_address, + port=self.port, + autocommit=True, + ) else: self.mysql_connection.ping(reconnect=True) - logging.debug("MySQL Connection establised: {}:{}".format(self.ip_address, self.port)) + logging.debug( + "MySQL Connection establised: {}:{}".format( + self.ip_address, self.port + ) + ) return self.mysql_connection except Exception as e: errors += [str(e)] @@ -63,8 +104,12 @@ class MySQLConnection: self.query("CREATE USER '" + user + "'@'%' IDENTIFIED BY '" + password + "'") self.grant_min_priv_for_user(user) - def grant_min_priv_for_user(self, user, db='priv_err_db'): - self.query("GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO '" + user + "'@'%'") + def grant_min_priv_for_user(self, user, db="priv_err_db"): + self.query( + "GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO '" + + user + + "'@'%'" + ) self.query("GRANT SELECT ON " + db + ".* TO '" + user + "'@'%'") def result(self, execution_query): @@ -85,175 +130,382 @@ class MySQLConnection: @pytest.fixture(scope="module") def started_mysql_5_7(): - mysql_node = MySQLConnection(cluster.mysql_port, 'root', 'clickhouse', cluster.mysql_ip) + mysql_node = MySQLConnection( + cluster.mysql_port, "root", "clickhouse", cluster.mysql_ip + ) yield mysql_node @pytest.fixture(scope="module") def started_mysql_8_0(): - mysql8_node = MySQLConnection(cluster.mysql8_port, 'root', 'clickhouse', cluster.mysql8_ip) + mysql8_node = MySQLConnection( + cluster.mysql8_port, "root", "clickhouse", cluster.mysql8_ip + ) yield mysql8_node -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def clickhouse_node(): yield node_db -def test_materialized_database_dml_with_mysql_5_7(started_cluster, started_mysql_5_7, clickhouse_node: ClickHouseInstance): - materialize_with_ddl.dml_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.materialized_mysql_database_with_views(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.materialized_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.move_to_prewhere_and_column_filtering(clickhouse_node, started_mysql_5_7, "mysql57") +def test_materialized_database_dml_with_mysql_5_7( + started_cluster, started_mysql_5_7, clickhouse_node: ClickHouseInstance +): + materialize_with_ddl.dml_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.materialized_mysql_database_with_views( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.materialized_mysql_database_with_datetime_and_decimal( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.move_to_prewhere_and_column_filtering( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_materialized_database_dml_with_mysql_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.dml_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.materialized_mysql_database_with_views(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.materialized_mysql_database_with_datetime_and_decimal(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.move_to_prewhere_and_column_filtering(clickhouse_node, started_mysql_8_0, "mysql80") +def test_materialized_database_dml_with_mysql_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.dml_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialized_mysql_database_with_views( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialized_mysql_database_with_datetime_and_decimal( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.move_to_prewhere_and_column_filtering( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_materialized_database_ddl_with_mysql_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.drop_table_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.create_table_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.rename_table_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.alter_add_column_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.alter_drop_column_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") +def test_materialized_database_ddl_with_mysql_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.drop_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.create_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.rename_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.alter_add_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.alter_drop_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) # mysql 5.7 cannot support alter rename column # materialize_with_ddl.alter_rename_column_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.alter_rename_table_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.alter_modify_column_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.create_table_like_with_materialize_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") + materialize_with_ddl.alter_rename_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.alter_modify_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.create_table_like_with_materialize_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_materialized_database_ddl_with_mysql_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.drop_table_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.create_table_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.rename_table_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.alter_add_column_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.alter_drop_column_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.alter_rename_table_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.alter_rename_column_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.alter_modify_column_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.create_table_like_with_materialize_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") +def test_materialized_database_ddl_with_mysql_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.drop_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.create_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.rename_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.alter_add_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.alter_drop_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.alter_rename_table_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.alter_rename_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.alter_modify_column_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.create_table_like_with_materialize_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_materialized_database_ddl_with_empty_transaction_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_5_7, "mysql57") +def test_materialized_database_ddl_with_empty_transaction_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.query_event_with_empty_transaction( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_materialized_database_ddl_with_empty_transaction_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_8_0, "mysql80") +def test_materialized_database_ddl_with_empty_transaction_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.query_event_with_empty_transaction( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_select_without_columns_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_5_7, "mysql57") +def test_select_without_columns_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.select_without_columns( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_select_without_columns_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_8_0, "mysql80") +def test_select_without_columns_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.select_without_columns( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_insert_with_modify_binlog_checksum_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.insert_with_modify_binlog_checksum(clickhouse_node, started_mysql_5_7, "mysql57") +def test_insert_with_modify_binlog_checksum_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.insert_with_modify_binlog_checksum( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_insert_with_modify_binlog_checksum_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.insert_with_modify_binlog_checksum(clickhouse_node, started_mysql_8_0, "mysql80") +def test_insert_with_modify_binlog_checksum_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.insert_with_modify_binlog_checksum( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_materialized_database_err_sync_user_privs_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.err_sync_user_privs_with_materialized_mysql_database(clickhouse_node, started_mysql_5_7, "mysql57") +def test_materialized_database_err_sync_user_privs_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.err_sync_user_privs_with_materialized_mysql_database( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_materialized_database_err_sync_user_privs_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.err_sync_user_privs_with_materialized_mysql_database(clickhouse_node, started_mysql_8_0, "mysql80") +def test_materialized_database_err_sync_user_privs_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.err_sync_user_privs_with_materialized_mysql_database( + clickhouse_node, started_mysql_8_0, "mysql80" + ) def test_network_partition_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.network_partition_test(clickhouse_node, started_mysql_5_7, "mysql57") + materialize_with_ddl.network_partition_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) def test_network_partition_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.network_partition_test(clickhouse_node, started_mysql_8_0, "mysql80") + materialize_with_ddl.network_partition_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_mysql_kill_sync_thread_restore_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.mysql_kill_sync_thread_restore_test(clickhouse_node, started_mysql_5_7, "mysql57") +def test_mysql_kill_sync_thread_restore_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.mysql_kill_sync_thread_restore_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_mysql_kill_sync_thread_restore_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.mysql_kill_sync_thread_restore_test(clickhouse_node, started_mysql_8_0, "mysql80") +def test_mysql_kill_sync_thread_restore_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.mysql_kill_sync_thread_restore_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_mysql_killed_while_insert_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.mysql_killed_while_insert(clickhouse_node, started_mysql_5_7, "mysql57") +def test_mysql_killed_while_insert_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.mysql_killed_while_insert( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_mysql_killed_while_insert_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.mysql_killed_while_insert(clickhouse_node, started_mysql_8_0, "mysql80") +def test_mysql_killed_while_insert_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.mysql_killed_while_insert( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_clickhouse_killed_while_insert_5_7(started_cluster, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.clickhouse_killed_while_insert(clickhouse_node, started_mysql_5_7, "mysql57") +def test_clickhouse_killed_while_insert_5_7( + started_cluster, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.clickhouse_killed_while_insert( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_clickhouse_killed_while_insert_8_0(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.clickhouse_killed_while_insert(clickhouse_node, started_mysql_8_0, "mysql80") +def test_clickhouse_killed_while_insert_8_0( + started_cluster, started_mysql_8_0, clickhouse_node +): + materialize_with_ddl.clickhouse_killed_while_insert( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_utf8mb4(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): +def test_utf8mb4( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): materialize_with_ddl.utf8mb4_test(clickhouse_node, started_mysql_5_7, "mysql57") materialize_with_ddl.utf8mb4_test(clickhouse_node, started_mysql_8_0, "mysql80") def test_system_parts_table(started_cluster, started_mysql_8_0, clickhouse_node): - materialize_with_ddl.system_parts_test(clickhouse_node, started_mysql_8_0, "mysql80") + materialize_with_ddl.system_parts_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_multi_table_update(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.multi_table_update_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.multi_table_update_test(clickhouse_node, started_mysql_8_0, "mysql80") +def test_multi_table_update( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.multi_table_update_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.multi_table_update_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_system_tables_table(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.system_tables_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.system_tables_test(clickhouse_node, started_mysql_8_0, "mysql80") +def test_system_tables_table( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.system_tables_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.system_tables_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_materialized_with_column_comments(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.materialize_with_column_comments_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.materialize_with_column_comments_test(clickhouse_node, started_mysql_8_0, "mysql80") +def test_materialized_with_column_comments( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.materialize_with_column_comments_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.materialize_with_column_comments_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_materialized_with_enum(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.materialize_with_enum8_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.materialize_with_enum16_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.alter_enum8_to_enum16_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.materialize_with_enum8_test(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.materialize_with_enum16_test(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.alter_enum8_to_enum16_test(clickhouse_node, started_mysql_8_0, "mysql80") +def test_materialized_with_enum( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.materialize_with_enum8_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.materialize_with_enum16_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.alter_enum8_to_enum16_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.materialize_with_enum8_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialize_with_enum16_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.alter_enum8_to_enum16_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -@pytest.mark.parametrize(('clickhouse_node'), [node_disable_bytes_settings, node_disable_rows_settings]) -def test_mysql_settings(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.mysql_settings_test(clickhouse_node, started_mysql_5_7, "mysql57") - materialize_with_ddl.mysql_settings_test(clickhouse_node, started_mysql_8_0, "mysql80") +@pytest.mark.parametrize( + ("clickhouse_node"), [node_disable_bytes_settings, node_disable_rows_settings] +) +def test_mysql_settings( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.mysql_settings_test( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + materialize_with_ddl.mysql_settings_test( + clickhouse_node, started_mysql_8_0, "mysql80" + ) -def test_large_transaction(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.materialized_mysql_large_transaction(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.materialized_mysql_large_transaction(clickhouse_node, started_mysql_5_7, "mysql57") +def test_large_transaction( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.materialized_mysql_large_transaction( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialized_mysql_large_transaction( + clickhouse_node, started_mysql_5_7, "mysql57" + ) -def test_table_table(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): + +def test_table_table( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): materialize_with_ddl.table_table(clickhouse_node, started_mysql_8_0, "mysql80") materialize_with_ddl.table_table(clickhouse_node, started_mysql_5_7, "mysql57") -def test_table_overrides(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): + +def test_table_overrides( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): materialize_with_ddl.table_overrides(clickhouse_node, started_mysql_5_7, "mysql57") materialize_with_ddl.table_overrides(clickhouse_node, started_mysql_8_0, "mysql80") -def test_materialized_database_support_all_kinds_of_mysql_datatype(started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node): - materialize_with_ddl.materialized_database_support_all_kinds_of_mysql_datatype(clickhouse_node, started_mysql_8_0, "mysql80") - materialize_with_ddl.materialized_database_support_all_kinds_of_mysql_datatype(clickhouse_node, started_mysql_5_7, "mysql57") + +def test_materialized_database_support_all_kinds_of_mysql_datatype( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.materialized_database_support_all_kinds_of_mysql_datatype( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialized_database_support_all_kinds_of_mysql_datatype( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + + +def test_materialized_database_settings_materialized_mysql_tables_list( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.materialized_database_settings_materialized_mysql_tables_list( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialized_database_settings_materialized_mysql_tables_list( + clickhouse_node, started_mysql_5_7, "mysql57" + ) + + +def test_materialized_database_mysql_date_type_to_date32( + started_cluster, started_mysql_8_0, started_mysql_5_7, clickhouse_node +): + materialize_with_ddl.materialized_database_mysql_date_type_to_date32( + clickhouse_node, started_mysql_8_0, "mysql80" + ) + materialize_with_ddl.materialized_database_mysql_date_type_to_date32( + clickhouse_node, started_mysql_5_7, "mysql57" + ) diff --git a/tests/integration/test_max_http_connections_for_replication/test.py b/tests/integration/test_max_http_connections_for_replication/test.py index 67b3c5b53aa..bcb779ee913 100644 --- a/tests/integration/test_max_http_connections_for_replication/test.py +++ b/tests/integration/test_max_http_connections_for_replication/test.py @@ -9,7 +9,7 @@ from helpers.test_tools import assert_eq_with_retry def _fill_nodes(nodes, shard, connections_count): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) @@ -19,14 +19,25 @@ def _fill_nodes(nodes, shard, connections_count): SETTINGS replicated_max_parallel_fetches_for_host={connections}, index_granularity=8192; - '''.format(shard=shard, replica=node.name, connections=connections_count)) + """.format( + shard=shard, replica=node.name, connections=connections_count + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', user_configs=[], - main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', user_configs=[], - main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + user_configs=[], + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + user_configs=[], + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -45,6 +56,7 @@ def start_small_cluster(): def test_single_endpoint_connections_count(start_small_cluster): node1.query("TRUNCATE TABLE test_table") node2.query("SYSTEM SYNC REPLICA test_table") + def task(count): print(("Inserting ten times from {}".format(count))) for i in range(count, count + 10): @@ -56,7 +68,12 @@ def test_single_endpoint_connections_count(start_small_cluster): assert_eq_with_retry(node1, "select count() from test_table", "100") assert_eq_with_retry(node2, "select count() from test_table", "100") - assert node2.query("SELECT value FROM system.events where event='CreatedHTTPConnections'") == '1\n' + assert ( + node2.query( + "SELECT value FROM system.events where event='CreatedHTTPConnections'" + ) + == "1\n" + ) def test_keepalive_timeout(start_small_cluster): @@ -75,12 +92,29 @@ def test_keepalive_timeout(start_small_cluster): assert_eq_with_retry(node2, "select count() from test_table", str(2)) - assert not node2.contains_in_log("No message received"), "Found 'No message received' in clickhouse-server.log" + assert not node2.contains_in_log( + "No message received" + ), "Found 'No message received' in clickhouse-server.log" -node3 = cluster.add_instance('node3', user_configs=[], main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', user_configs=[], main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node5 = cluster.add_instance('node5', user_configs=[], main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node3 = cluster.add_instance( + "node3", + user_configs=[], + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + user_configs=[], + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) +node5 = cluster.add_instance( + "node5", + user_configs=[], + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -115,4 +149,9 @@ def test_multiple_endpoint_connections_count(start_big_cluster): assert_eq_with_retry(node5, "select count() from test_table", "100") # Two per each host or sometimes less, if fetches are not performed in parallel. But not more. - assert node5.query("SELECT value FROM system.events where event='CreatedHTTPConnections'") <= '4\n' + assert ( + node5.query( + "SELECT value FROM system.events where event='CreatedHTTPConnections'" + ) + <= "4\n" + ) diff --git a/tests/integration/test_max_suspicious_broken_parts/test.py b/tests/integration/test_max_suspicious_broken_parts/test.py index 31f53fdbc3c..c1f34adbb62 100644 --- a/tests/integration/test_max_suspicious_broken_parts/test.py +++ b/tests/integration/test_max_suspicious_broken_parts/test.py @@ -8,9 +8,10 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True) +node = cluster.add_instance("node", stay_alive=True) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -18,52 +19,69 @@ def start_cluster(): finally: cluster.shutdown() + def break_part(table, part_name): - node.exec_in_container(['bash', '-c', f'rm /var/lib/clickhouse/data/default/{table}/{part_name}/columns.txt']) + node.exec_in_container( + [ + "bash", + "-c", + f"rm /var/lib/clickhouse/data/default/{table}/{part_name}/columns.txt", + ] + ) + def remove_part(table, part_name): - node.exec_in_container(['bash', '-c', f'rm -r /var/lib/clickhouse/data/default/{table}/{part_name}']) + node.exec_in_container( + ["bash", "-c", f"rm -r /var/lib/clickhouse/data/default/{table}/{part_name}"] + ) + def get_count(table): - return int(node.query(f'SELECT count() FROM {table}').strip()) + return int(node.query(f"SELECT count() FROM {table}").strip()) + def detach_table(table): - node.query(f'DETACH TABLE {table}') + node.query(f"DETACH TABLE {table}") + + def attach_table(table): - node.query(f'ATTACH TABLE {table}') + node.query(f"ATTACH TABLE {table}") + def check_table(table): rows = 900 per_part_rows = 90 - node.query(f'INSERT INTO {table} SELECT * FROM numbers(900)') + node.query(f"INSERT INTO {table} SELECT * FROM numbers(900)") assert get_count(table) == rows # break one part, and check that clickhouse will be alive - break_part(table, '0_1_1_0') + break_part(table, "0_1_1_0") rows -= per_part_rows detach_table(table) attach_table(table) assert get_count(table) == rows # break two parts, and check that clickhouse will not start - break_part(table, '1_2_2_0') - break_part(table, '2_3_3_0') - rows -= per_part_rows*2 + break_part(table, "1_2_2_0") + break_part(table, "2_3_3_0") + rows -= per_part_rows * 2 detach_table(table) with pytest.raises(QueryRuntimeException): attach_table(table) # now remove one part, and check - remove_part(table, '1_2_2_0') + remove_part(table, "1_2_2_0") attach_table(table) assert get_count(table) == rows - node.query(f'DROP TABLE {table}') + node.query(f"DROP TABLE {table}") + def test_max_suspicious_broken_parts(): - node.query(""" + node.query( + """ CREATE TABLE test_max_suspicious_broken_parts ( key Int ) @@ -72,11 +90,14 @@ def test_max_suspicious_broken_parts(): PARTITION BY key%10 SETTINGS max_suspicious_broken_parts = 1; - """) - check_table('test_max_suspicious_broken_parts') + """ + ) + check_table("test_max_suspicious_broken_parts") + def test_max_suspicious_broken_parts_bytes(): - node.query(""" + node.query( + """ CREATE TABLE test_max_suspicious_broken_parts_bytes ( key Int ) @@ -87,11 +108,14 @@ def test_max_suspicious_broken_parts_bytes(): max_suspicious_broken_parts = 10, /* one part takes ~751 byte, so we allow failure of one part with these limit */ max_suspicious_broken_parts_bytes = 1000; - """) - check_table('test_max_suspicious_broken_parts_bytes') + """ + ) + check_table("test_max_suspicious_broken_parts_bytes") + def test_max_suspicious_broken_parts__wide(): - node.query(""" + node.query( + """ CREATE TABLE test_max_suspicious_broken_parts__wide ( key Int ) @@ -101,11 +125,14 @@ def test_max_suspicious_broken_parts__wide(): SETTINGS min_bytes_for_wide_part = 0, max_suspicious_broken_parts = 1; - """) - check_table('test_max_suspicious_broken_parts__wide') + """ + ) + check_table("test_max_suspicious_broken_parts__wide") + def test_max_suspicious_broken_parts_bytes__wide(): - node.query(""" + node.query( + """ CREATE TABLE test_max_suspicious_broken_parts_bytes__wide ( key Int ) @@ -117,5 +144,6 @@ def test_max_suspicious_broken_parts_bytes__wide(): max_suspicious_broken_parts = 10, /* one part takes ~750 byte, so we allow failure of one part with these limit */ max_suspicious_broken_parts_bytes = 1000; - """) - check_table('test_max_suspicious_broken_parts_bytes__wide') + """ + ) + check_table("test_max_suspicious_broken_parts_bytes__wide") diff --git a/tests/integration/test_merge_table_over_distributed/test.py b/tests/integration/test_merge_table_over_distributed/test.py index ab294867126..5ee542079a7 100644 --- a/tests/integration/test_merge_table_over_distributed/test.py +++ b/tests/integration/test_merge_table_over_distributed/test.py @@ -5,8 +5,8 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) +node1 = cluster.add_instance("node1", main_configs=["configs/remote_servers.xml"]) +node2 = cluster.add_instance("node2", main_configs=["configs/remote_servers.xml"]) @pytest.fixture(scope="module") @@ -15,19 +15,23 @@ def started_cluster(): cluster.start() for node in (node1, node2): - node.query(''' + node.query( + """ CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id; CREATE TABLE local_table_2(id UInt32, val String) ENGINE = MergeTree ORDER BY id; -''') +""" + ) node1.query("INSERT INTO local_table VALUES (1, 'node1')") node2.query("INSERT INTO local_table VALUES (2, 'node2')") - node1.query(''' + node1.query( + """ CREATE TABLE distributed_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table); CREATE TABLE distributed_table_2(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table_2); CREATE TABLE merge_table(id UInt32, val String) ENGINE = Merge(default, '^distributed_table') -''') +""" + ) yield cluster @@ -36,26 +40,47 @@ CREATE TABLE merge_table(id UInt32, val String) ENGINE = Merge(default, '^distri def test_global_in(started_cluster): - assert node1.query( - "SELECT val FROM distributed_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)").rstrip() \ - == 'node2' + assert ( + node1.query( + "SELECT val FROM distributed_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)" + ).rstrip() + == "node2" + ) - assert node1.query( - "SELECT val FROM merge_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)").rstrip() \ - == 'node2' + assert ( + node1.query( + "SELECT val FROM merge_table WHERE id GLOBAL IN (SELECT toUInt32(3 - id) FROM local_table)" + ).rstrip() + == "node2" + ) def test_filtering(started_cluster): - assert node1.query("SELECT id, val FROM merge_table WHERE id = 1").rstrip() == '1\tnode1' + assert ( + node1.query("SELECT id, val FROM merge_table WHERE id = 1").rstrip() + == "1\tnode1" + ) - assert node1.query("SELECT id + 1, val FROM merge_table WHERE id = 1").rstrip() == '2\tnode1' + assert ( + node1.query("SELECT id + 1, val FROM merge_table WHERE id = 1").rstrip() + == "2\tnode1" + ) - assert node1.query("SELECT id + 1 FROM merge_table WHERE val = 'node1'").rstrip() == '2' + assert ( + node1.query("SELECT id + 1 FROM merge_table WHERE val = 'node1'").rstrip() + == "2" + ) - assert node1.query( - "SELECT id + 1, val FROM merge_table PREWHERE id = 1 WHERE _table != '_dummy'").rstrip() == '2\tnode1' + assert ( + node1.query( + "SELECT id + 1, val FROM merge_table PREWHERE id = 1 WHERE _table != '_dummy'" + ).rstrip() + == "2\tnode1" + ) - assert node1.query("SELECT count() FROM merge_table PREWHERE id = 1").rstrip() == '1' + assert ( + node1.query("SELECT count() FROM merge_table PREWHERE id = 1").rstrip() == "1" + ) def test_select_table_name_from_merge_over_distributed(started_cluster): @@ -63,10 +88,12 @@ def test_select_table_name_from_merge_over_distributed(started_cluster): node2.query("INSERT INTO local_table_2 VALUES (2, 'node2')") node1.query("select _table == 'distributed_table' from merge_table") - node1.query("select * from (select _table == 'distributed_table' from merge_table limit 1)") + node1.query( + "select * from (select _table == 'distributed_table' from merge_table limit 1)" + ) -if __name__ == '__main__': +if __name__ == "__main__": with contextmanager(started_cluster)() as cluster: for name, instance in list(cluster.instances.items()): print(name, instance.ip_address) diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index 92b9d52cf86..bc549210b39 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -8,7 +8,10 @@ from helpers.utility import generate_values, replace_config, SafeThread SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/storage_conf.xml'.format(get_instances_dir())) +CONFIG_PATH = os.path.join( + SCRIPT_DIR, + "./{}/node/configs/config.d/storage_conf.xml".format(get_instances_dir()), +) NODE_NAME = "node" TABLE_NAME = "blob_storage_table" @@ -21,9 +24,14 @@ CONTAINER_NAME = "cont" def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance(NODE_NAME, - main_configs=["configs/config.d/storage_conf.xml", "configs/config.d/bg_processing_pool_conf.xml"], - with_azurite=True) + cluster.add_instance( + NODE_NAME, + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/config.d/bg_processing_pool_conf.xml", + ], + with_azurite=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -32,6 +40,7 @@ def cluster(): finally: cluster.shutdown() + # Note: use this for selects and inserts and create table queries. # For inserts there is no guarantee that retries will not result in duplicates. # But it is better to retry anyway because 'Connection was closed by the server' error @@ -42,7 +51,7 @@ def azure_query(node, query, try_num=3): return node.query(query) except Exception as ex: retriable_errors = [ - 'DB::Exception: Azure::Core::Http::TransportException: Connection was closed by the server while trying to read a response', + "DB::Exception: Azure::Core::Http::TransportException: Connection was closed by the server while trying to read a response", ] retry = False for error in retriable_errors: @@ -54,11 +63,12 @@ def azure_query(node, query, try_num=3): raise Exception(ex) continue + def create_table(node, table_name, **additional_settings): settings = { "storage_policy": "blob_storage_policy", "old_parts_lifetime": 1, - "index_granularity": 512 + "index_granularity": 512, } settings.update(additional_settings) @@ -75,7 +85,9 @@ def create_table(node, table_name, **additional_settings): node.query(f"DROP TABLE IF EXISTS {table_name}") azure_query(node, create_table_statement) - assert azure_query(node, f"SELECT COUNT(*) FROM {table_name} FORMAT Values") == "(0)" + assert ( + azure_query(node, f"SELECT COUNT(*) FROM {table_name} FORMAT Values") == "(0)" + ) def test_create_table(cluster): @@ -92,10 +104,16 @@ def test_read_after_cache_is_wiped(cluster): azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {values}") # Wipe cache - cluster.exec_in_container(cluster.get_container_id(NODE_NAME), ["rm", "-rf", "/var/lib/clickhouse/disks/blob_storage_disk/cache/"]) + cluster.exec_in_container( + cluster.get_container_id(NODE_NAME), + ["rm", "-rf", "/var/lib/clickhouse/disks/blob_storage_disk/cache/"], + ) # After cache is populated again, only .bin files should be accessed from Blob Storage. - assert azure_query(node, f"SELECT * FROM {TABLE_NAME} order by dt, id FORMAT Values") == values + assert ( + azure_query(node, f"SELECT * FROM {TABLE_NAME} order by dt, id FORMAT Values") + == values + ) def test_simple_insert_select(cluster): @@ -104,55 +122,96 @@ def test_simple_insert_select(cluster): values = "('2021-11-13',3,'hello')" azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {values}") - assert azure_query(node, f"SELECT dt, id, data FROM {TABLE_NAME} FORMAT Values") == values - blob_container_client = cluster.blob_service_client.get_container_client(CONTAINER_NAME) - assert len(list(blob_container_client.list_blobs())) >= 12 # 1 format file + 2 skip index files + 9 regular MergeTree files + leftovers from other tests + assert ( + azure_query(node, f"SELECT dt, id, data FROM {TABLE_NAME} FORMAT Values") + == values + ) + blob_container_client = cluster.blob_service_client.get_container_client( + CONTAINER_NAME + ) + assert ( + len(list(blob_container_client.list_blobs())) >= 12 + ) # 1 format file + 2 skip index files + 9 regular MergeTree files + leftovers from other tests def test_inserts_selects(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - values1 = generate_values('2020-01-03', 4096) + values1 = generate_values("2020-01-03", 4096) azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {values1}") - assert azure_query(node, f"SELECT * FROM {TABLE_NAME} order by dt, id FORMAT Values") == values1 + assert ( + azure_query(node, f"SELECT * FROM {TABLE_NAME} order by dt, id FORMAT Values") + == values1 + ) - values2 = generate_values('2020-01-04', 4096) + values2 = generate_values("2020-01-04", 4096) azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {values2}") - assert azure_query(node, f"SELECT * FROM {TABLE_NAME} ORDER BY dt, id FORMAT Values") == values1 + "," + values2 + assert ( + azure_query(node, f"SELECT * FROM {TABLE_NAME} ORDER BY dt, id FORMAT Values") + == values1 + "," + values2 + ) - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} where id = 1 FORMAT Values") == "(2)" + assert ( + azure_query( + node, f"SELECT count(*) FROM {TABLE_NAME} where id = 1 FORMAT Values" + ) + == "(2)" + ) @pytest.mark.parametrize( - "merge_vertical", [ + "merge_vertical", + [ (True), (False), -]) + ], +) def test_insert_same_partition_and_merge(cluster, merge_vertical): settings = {} if merge_vertical: - settings['vertical_merge_algorithm_min_rows_to_activate'] = 0 - settings['vertical_merge_algorithm_min_columns_to_activate'] = 0 + settings["vertical_merge_algorithm_min_rows_to_activate"] = 0 + settings["vertical_merge_algorithm_min_columns_to_activate"] = 0 node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME, **settings) node.query(f"SYSTEM STOP MERGES {TABLE_NAME}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 1024)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 2048)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 1024, -1)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 2048, -1)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096, -1)}") + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 1024)}" + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 2048)}" + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 1024, -1)}", + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 2048, -1)}", + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096, -1)}", + ) assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(distinct(id)) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + assert ( + azure_query(node, f"SELECT count(distinct(id)) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) node.query(f"SYSTEM START MERGES {TABLE_NAME}") # Wait for merges and old parts deletion for attempt in range(0, 10): - parts_count = azure_query(node, f"SELECT COUNT(*) FROM system.parts WHERE table = '{TABLE_NAME}' FORMAT Values") + parts_count = azure_query( + node, + f"SELECT COUNT(*) FROM system.parts WHERE table = '{TABLE_NAME}' FORMAT Values", + ) if parts_count == "(1)": break @@ -162,63 +221,123 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical): time.sleep(1) assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(distinct(id)) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + assert ( + azure_query(node, f"SELECT count(distinct(id)) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) def test_alter_table_columns(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096, -1)}") + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096, -1)}", + ) node.query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN col1 UInt64 DEFAULT 1") # To ensure parts have been merged node.query(f"OPTIMIZE TABLE {TABLE_NAME}") - assert azure_query(node, f"SELECT sum(col1) FROM {TABLE_NAME} FORMAT Values") == "(8192)" - assert azure_query(node, f"SELECT sum(col1) FROM {TABLE_NAME} WHERE id > 0 FORMAT Values") == "(4096)" + assert ( + azure_query(node, f"SELECT sum(col1) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) + assert ( + azure_query( + node, f"SELECT sum(col1) FROM {TABLE_NAME} WHERE id > 0 FORMAT Values" + ) + == "(4096)" + ) - node.query(f"ALTER TABLE {TABLE_NAME} MODIFY COLUMN col1 String", settings={"mutations_sync": 2}) + node.query( + f"ALTER TABLE {TABLE_NAME} MODIFY COLUMN col1 String", + settings={"mutations_sync": 2}, + ) - assert azure_query(node, f"SELECT distinct(col1) FROM {TABLE_NAME} FORMAT Values") == "('1')" + assert ( + azure_query(node, f"SELECT distinct(col1) FROM {TABLE_NAME} FORMAT Values") + == "('1')" + ) def test_attach_detach_partition(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}" + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) node.query(f"ALTER TABLE {TABLE_NAME} DETACH PARTITION '2020-01-03'") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(4096)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(4096)" + ) node.query(f"ALTER TABLE {TABLE_NAME} ATTACH PARTITION '2020-01-03'") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) node.query(f"ALTER TABLE {TABLE_NAME} DROP PARTITION '2020-01-03'") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(4096)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(4096)" + ) node.query(f"ALTER TABLE {TABLE_NAME} DETACH PARTITION '2020-01-04'") - node.query(f"ALTER TABLE {TABLE_NAME} DROP DETACHED PARTITION '2020-01-04'", settings={"allow_drop_detached": 1}) - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(0)" + node.query( + f"ALTER TABLE {TABLE_NAME} DROP DETACHED PARTITION '2020-01-04'", + settings={"allow_drop_detached": 1}, + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(0)" + ) def test_move_partition_to_another_disk(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}" + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) - node.query(f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{LOCAL_DISK}'") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + node.query( + f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{LOCAL_DISK}'" + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) - node.query(f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{AZURE_BLOB_STORAGE_DISK}'") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + node.query( + f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{AZURE_BLOB_STORAGE_DISK}'" + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) def test_table_manipulations(cluster): @@ -227,21 +346,33 @@ def test_table_manipulations(cluster): renamed_table = TABLE_NAME + "_renamed" - node.query_with_retry(f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") - node.query_with_retry(f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}") + node.query_with_retry( + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) + node.query_with_retry( + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}" + ) node.query(f"RENAME TABLE {TABLE_NAME} TO {renamed_table}") - assert azure_query(node, f"SELECT count(*) FROM {renamed_table} FORMAT Values") == "(8192)" + assert ( + azure_query(node, f"SELECT count(*) FROM {renamed_table} FORMAT Values") + == "(8192)" + ) node.query(f"RENAME TABLE {renamed_table} TO {TABLE_NAME}") assert node.query(f"CHECK TABLE {TABLE_NAME} FORMAT Values") == "(1)" node.query(f"DETACH TABLE {TABLE_NAME}") node.query(f"ATTACH TABLE {TABLE_NAME}") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(8192)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(8192)" + ) node.query(f"TRUNCATE TABLE {TABLE_NAME}") - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(0)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(0)" + ) @pytest.mark.long_run @@ -251,38 +382,87 @@ def test_move_replace_partition_to_another_table(cluster): table_clone_name = TABLE_NAME + "_clone" - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 256)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 256)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-05', 256, -1)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-06', 256, -1)}") + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 256)}" + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 256)}" + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-05', 256, -1)}", + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-06', 256, -1)}", + ) assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(1024)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(1024)" + ) create_table(node, table_clone_name) - node.query(f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-03' TO TABLE {table_clone_name}") - node.query(f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-05' TO TABLE {table_clone_name}") + node.query( + f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-03' TO TABLE {table_clone_name}" + ) + node.query( + f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-05' TO TABLE {table_clone_name}" + ) assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(512)" - assert azure_query(node, f"SELECT sum(id) FROM {table_clone_name} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {table_clone_name} FORMAT Values") == "(512)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(512)" + ) + assert ( + azure_query(node, f"SELECT sum(id) FROM {table_clone_name} FORMAT Values") + == "(0)" + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {table_clone_name} FORMAT Values") + == "(512)" + ) # Add new partitions to source table, but with different values and replace them from copied table. - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 256, -1)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-05', 256)}") + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 256, -1)}", + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-05', 256)}" + ) assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(1024)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(1024)" + ) - node.query(f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-03' FROM {table_clone_name}") - node.query(f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-05' FROM {table_clone_name}") + node.query( + f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-03' FROM {table_clone_name}" + ) + node.query( + f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-05' FROM {table_clone_name}" + ) assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(1024)" - assert azure_query(node, f"SELECT sum(id) FROM {table_clone_name} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {table_clone_name} FORMAT Values") == "(512)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(1024)" + ) + assert ( + azure_query(node, f"SELECT sum(id) FROM {table_clone_name} FORMAT Values") + == "(0)" + ) + assert ( + azure_query(node, f"SELECT count(*) FROM {table_clone_name} FORMAT Values") + == "(512)" + ) node.query(f"DROP TABLE {table_clone_name} NO DELAY") assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" - assert azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(1024)" + assert ( + azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") + == "(1024)" + ) node.query(f"ALTER TABLE {TABLE_NAME} FREEZE") @@ -293,18 +473,24 @@ def test_freeze_unfreeze(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - backup1 = 'backup1' - backup2 = 'backup2' + backup1 = "backup1" + backup2 = "backup2" - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) node.query(f"ALTER TABLE {TABLE_NAME} FREEZE WITH NAME '{backup1}'") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}") + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}" + ) node.query(f"ALTER TABLE {TABLE_NAME} FREEZE WITH NAME '{backup2}'") azure_query(node, f"TRUNCATE TABLE {TABLE_NAME}") # Unfreeze single partition from backup1. - node.query(f"ALTER TABLE {TABLE_NAME} UNFREEZE PARTITION '2020-01-03' WITH NAME '{backup1}'") + node.query( + f"ALTER TABLE {TABLE_NAME} UNFREEZE PARTITION '2020-01-03' WITH NAME '{backup1}'" + ) # Unfreeze all partitions from backup2. node.query(f"ALTER TABLE {TABLE_NAME} UNFREEZE WITH NAME '{backup2}'") @@ -313,16 +499,22 @@ def test_apply_new_settings(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}") + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}" + ) # Force multi-part upload mode. replace_config( CONFIG_PATH, "33554432", - "4096") + "4096", + ) node.query("SYSTEM RELOAD CONFIG") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096, -1)}") + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096, -1)}", + ) # NOTE: this test takes a couple of minutes when run together with other tests @@ -332,16 +524,25 @@ def test_restart_during_load(cluster): create_table(node, TABLE_NAME) # Force multi-part upload mode. - replace_config(CONFIG_PATH, "false", "") - - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}") - azure_query(node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-05', 4096, -1)}") + replace_config( + CONFIG_PATH, "false", "" + ) + azure_query( + node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}" + ) + azure_query( + node, + f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-05', 4096, -1)}", + ) def read(): for ii in range(0, 5): logging.info(f"Executing {ii} query") - assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" + assert ( + azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") + == "(0)" + ) logging.info(f"Query {ii} executed") time.sleep(0.2) @@ -368,5 +569,8 @@ def test_restart_during_load(cluster): def test_big_insert(cluster): node = cluster.instances[NODE_NAME] create_table(node, TABLE_NAME) - azure_query(node, f"INSERT INTO {TABLE_NAME} select '2020-01-03', number, toString(number) from numbers(5000000)") + azure_query( + node, + f"INSERT INTO {TABLE_NAME} select '2020-01-03', number, toString(number) from numbers(5000000)", + ) assert int(azure_query(node, f"SELECT count() FROM {TABLE_NAME}")) == 5000000 diff --git a/tests/integration/test_merge_tree_empty_parts/test.py b/tests/integration/test_merge_tree_empty_parts/test.py index bc2679d4c92..7ca275e96de 100644 --- a/tests/integration/test_merge_tree_empty_parts/test.py +++ b/tests/integration/test_merge_tree_empty_parts/test.py @@ -5,7 +5,11 @@ from helpers.test_tools import assert_eq_with_retry cluster = helpers.cluster.ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/cleanup_thread.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/cleanup_thread.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -17,22 +21,38 @@ def started_cluster(): finally: cluster.shutdown() + def test_empty_parts_alter_delete(started_cluster): - node1.query("CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1', d, key, 8192)") + node1.query( + "CREATE TABLE empty_parts_delete (d Date, key UInt64, value String) \ + ENGINE = ReplicatedMergeTree('/clickhouse/tables/empty_parts_delete', 'r1', d, key, 8192)" + ) node1.query("INSERT INTO empty_parts_delete VALUES (toDate('2020-10-10'), 1, 'a')") - node1.query("ALTER TABLE empty_parts_delete DELETE WHERE 1 SETTINGS mutations_sync = 2") + node1.query( + "ALTER TABLE empty_parts_delete DELETE WHERE 1 SETTINGS mutations_sync = 2" + ) print(node1.query("SELECT count() FROM empty_parts_delete")) - assert_eq_with_retry(node1, "SELECT count() FROM system.parts WHERE table = 'empty_parts_delete' AND active", "0") + assert_eq_with_retry( + node1, + "SELECT count() FROM system.parts WHERE table = 'empty_parts_delete' AND active", + "0", + ) + def test_empty_parts_summing(started_cluster): - node1.query("CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1', d, key, 8192)") + node1.query( + "CREATE TABLE empty_parts_summing (d Date, key UInt64, value Int64) \ + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/empty_parts_summing', 'r1', d, key, 8192)" + ) node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, 1)") node1.query("INSERT INTO empty_parts_summing VALUES (toDate('2020-10-10'), 1, -1)") node1.query("OPTIMIZE TABLE empty_parts_summing FINAL") - assert_eq_with_retry(node1, "SELECT count() FROM system.parts WHERE table = 'empty_parts_summing' AND active", "0") + assert_eq_with_retry( + node1, + "SELECT count() FROM system.parts WHERE table = 'empty_parts_summing' AND active", + "0", + ) diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index d6e3315e45d..132e1027586 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -9,7 +9,9 @@ from helpers.utility import generate_values from pyhdfs import HdfsClient SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node/configs/config.d/storage_conf.xml') +CONFIG_PATH = os.path.join( + SCRIPT_DIR, "./_instances/node/configs/config.d/storage_conf.xml" +) def create_table(cluster, table_name, additional_settings=None): @@ -26,7 +28,9 @@ def create_table(cluster, table_name, additional_settings=None): storage_policy='hdfs', old_parts_lifetime=0, index_granularity=512 - """.format(table_name) + """.format( + table_name + ) if additional_settings: create_table_statement += "," @@ -45,13 +49,15 @@ FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml"], with_hdfs=True) + cluster.add_instance( + "node", main_configs=["configs/config.d/storage_conf.xml"], with_hdfs=True + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") fs = HdfsClient(hosts=cluster.hdfs_ip) - fs.mkdirs('/clickhouse') + fs.mkdirs("/clickhouse") logging.info("Created HDFS directory") @@ -63,12 +69,12 @@ def cluster(): def wait_for_delete_hdfs_objects(cluster, expected, num_tries=30): fs = HdfsClient(hosts=cluster.hdfs_ip) while num_tries > 0: - num_hdfs_objects = len(fs.listdir('/clickhouse')) + num_hdfs_objects = len(fs.listdir("/clickhouse")) if num_hdfs_objects == expected: break num_tries -= 1 time.sleep(1) - assert(len(fs.listdir('/clickhouse')) == expected) + assert len(fs.listdir("/clickhouse")) == expected @pytest.fixture(autouse=True) @@ -76,46 +82,63 @@ def drop_table(cluster): node = cluster.instances["node"] fs = HdfsClient(hosts=cluster.hdfs_ip) - hdfs_objects = fs.listdir('/clickhouse') - print('Number of hdfs objects to delete:', len(hdfs_objects), sep=' ') + hdfs_objects = fs.listdir("/clickhouse") + print("Number of hdfs objects to delete:", len(hdfs_objects), sep=" ") node.query("DROP TABLE IF EXISTS hdfs_test SYNC") try: wait_for_delete_hdfs_objects(cluster, 0) finally: - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") if len(hdfs_objects) == 0: return - print("Manually removing extra objects to prevent tests cascade failing: ", hdfs_objects) + print( + "Manually removing extra objects to prevent tests cascade failing: ", + hdfs_objects, + ) for path in hdfs_objects: fs.delete(path) -@pytest.mark.parametrize("min_rows_for_wide_part,files_per_part", [(0, FILES_OVERHEAD_PER_PART_WIDE), (8192, FILES_OVERHEAD_PER_PART_COMPACT)]) +@pytest.mark.parametrize( + "min_rows_for_wide_part,files_per_part", + [(0, FILES_OVERHEAD_PER_PART_WIDE), (8192, FILES_OVERHEAD_PER_PART_COMPACT)], +) def test_simple_insert_select(cluster, min_rows_for_wide_part, files_per_part): - create_table(cluster, "hdfs_test", additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part)) + create_table( + cluster, + "hdfs_test", + additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part), + ) node = cluster.instances["node"] - values1 = generate_values('2020-01-03', 4096) + values1 = generate_values("2020-01-03", 4096) node.query("INSERT INTO hdfs_test VALUES {}".format(values1)) - assert node.query("SELECT * FROM hdfs_test order by dt, id FORMAT Values") == values1 + assert ( + node.query("SELECT * FROM hdfs_test order by dt, id FORMAT Values") == values1 + ) fs = HdfsClient(hosts=cluster.hdfs_ip) - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") print(hdfs_objects) assert len(hdfs_objects) == FILES_OVERHEAD + files_per_part - values2 = generate_values('2020-01-04', 4096) + values2 = generate_values("2020-01-04", 4096) node.query("INSERT INTO hdfs_test VALUES {}".format(values2)) - assert node.query("SELECT * FROM hdfs_test ORDER BY dt, id FORMAT Values") == values1 + "," + values2 + assert ( + node.query("SELECT * FROM hdfs_test ORDER BY dt, id FORMAT Values") + == values1 + "," + values2 + ) - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + files_per_part * 2 - assert node.query("SELECT count(*) FROM hdfs_test where id = 1 FORMAT Values") == "(2)" + assert ( + node.query("SELECT count(*) FROM hdfs_test where id = 1 FORMAT Values") == "(2)" + ) def test_alter_table_columns(cluster): @@ -124,27 +147,47 @@ def test_alter_table_columns(cluster): node = cluster.instances["node"] fs = HdfsClient(hosts=cluster.hdfs_ip) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format( + generate_values("2020-01-03", 4096, -1) + ) + ) node.query("ALTER TABLE hdfs_test ADD COLUMN col1 UInt64 DEFAULT 1") # To ensure parts have merged node.query("OPTIMIZE TABLE hdfs_test") assert node.query("SELECT sum(col1) FROM hdfs_test FORMAT Values") == "(8192)" - assert node.query("SELECT sum(col1) FROM hdfs_test WHERE id > 0 FORMAT Values") == "(4096)" - wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN) + assert ( + node.query("SELECT sum(col1) FROM hdfs_test WHERE id > 0 FORMAT Values") + == "(4096)" + ) + wait_for_delete_hdfs_objects( + cluster, + FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN, + ) - node.query("ALTER TABLE hdfs_test MODIFY COLUMN col1 String", settings={"mutations_sync": 2}) + node.query( + "ALTER TABLE hdfs_test MODIFY COLUMN col1 String", + settings={"mutations_sync": 2}, + ) assert node.query("SELECT distinct(col1) FROM hdfs_test FORMAT Values") == "('1')" # and file with mutation - wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + 1) + wait_for_delete_hdfs_objects( + cluster, + FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + 1, + ) node.query("ALTER TABLE hdfs_test DROP COLUMN col1", settings={"mutations_sync": 2}) # and 2 files with mutations - wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2) + wait_for_delete_hdfs_objects( + cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2 + ) def test_attach_detach_partition(cluster): @@ -153,36 +196,43 @@ def test_attach_detach_partition(cluster): node = cluster.instances["node"] fs = HdfsClient(hosts=cluster.hdfs_ip) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE hdfs_test DETACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE hdfs_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE hdfs_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE node.query("ALTER TABLE hdfs_test DETACH PARTITION '2020-01-04'") - node.query("ALTER TABLE hdfs_test DROP DETACHED PARTITION '2020-01-04'", settings={"allow_drop_detached": 1}) + node.query( + "ALTER TABLE hdfs_test DROP DETACHED PARTITION '2020-01-04'", + settings={"allow_drop_detached": 1}, + ) assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(0)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD @@ -192,23 +242,27 @@ def test_move_partition_to_another_disk(cluster): node = cluster.instances["node"] fs = HdfsClient(hosts=cluster.hdfs_ip) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("ALTER TABLE hdfs_test MOVE PARTITION '2020-01-04' TO DISK 'hdd'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE node.query("ALTER TABLE hdfs_test MOVE PARTITION '2020-01-04' TO DISK 'hdfs'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 @@ -218,13 +272,17 @@ def test_table_manipulations(cluster): node = cluster.instances["node"] fs = HdfsClient(hosts=cluster.hdfs_ip) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) node.query("RENAME TABLE hdfs_test TO hdfs_renamed") assert node.query("SELECT count(*) FROM hdfs_renamed FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("RENAME TABLE hdfs_renamed TO hdfs_test") @@ -234,13 +292,13 @@ def test_table_manipulations(cluster): node.query("ATTACH TABLE hdfs_test") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 node.query("TRUNCATE TABLE hdfs_test") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(0)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD @@ -250,14 +308,26 @@ def test_move_replace_partition_to_another_table(cluster): node = cluster.instances["node"] fs = HdfsClient(hosts=cluster.hdfs_ip) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-04', 4096))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-05', 4096, -1))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-06', 4096, -1))) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format( + generate_values("2020-01-05", 4096, -1) + ) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format( + generate_values("2020-01-06", 4096, -1) + ) + ) assert node.query("SELECT sum(id) FROM hdfs_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(16384)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 create_table(cluster, "hdfs_clone") @@ -270,16 +340,22 @@ def test_move_replace_partition_to_another_table(cluster): assert node.query("SELECT count(*) FROM hdfs_clone FORMAT Values") == "(8192)" # Number of objects in HDFS should be unchanged. - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 # Add new partitions to source table, but with different values and replace them from copied table. - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) - node.query("INSERT INTO hdfs_test VALUES {}".format(generate_values('2020-01-05', 4096))) + node.query( + "INSERT INTO hdfs_test VALUES {}".format( + generate_values("2020-01-03", 4096, -1) + ) + ) + node.query( + "INSERT INTO hdfs_test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) assert node.query("SELECT sum(id) FROM hdfs_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(16384)" - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 node.query("ALTER TABLE hdfs_test REPLACE PARTITION '2020-01-03' FROM hdfs_clone") @@ -291,13 +367,14 @@ def test_move_replace_partition_to_another_table(cluster): # Wait for outdated partitions deletion. print(1) - wait_for_delete_hdfs_objects(cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4) + wait_for_delete_hdfs_objects( + cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + ) node.query("DROP TABLE hdfs_clone NO DELAY") assert node.query("SELECT sum(id) FROM hdfs_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(16384)" # Data should remain in hdfs - hdfs_objects = fs.listdir('/clickhouse') + hdfs_objects = fs.listdir("/clickhouse") assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 - diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml index a0fe0a6f609..2f1b8275a0b 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/storage_conf.xml @@ -19,6 +19,14 @@ local / + + s3 + http://minio1:9001/root/data/ + minio + minio123 + 33554432 + 1 + @@ -38,6 +46,13 @@ + + +
+ s3_with_cache +
+ + diff --git a/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py b/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py index 1f8fcc4bbfd..c4a1e5ea1f5 100644 --- a/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py +++ b/tests/integration/test_merge_tree_s3/s3_mocks/unstable_proxy.py @@ -11,13 +11,19 @@ random.seed("Unstable proxy/1.0") def request(command, url, headers={}, data=None): - """ Mini-requests. """ + """Mini-requests.""" + class Dummy: pass parts = urllib.parse.urlparse(url) c = http.client.HTTPConnection(parts.hostname, parts.port) - c.request(command, urllib.parse.urlunparse(parts._replace(scheme='', netloc='')), headers=headers, body=data) + c.request( + command, + urllib.parse.urlunparse(parts._replace(scheme="", netloc="")), + headers=headers, + body=data, + ) r = c.getresponse() result = Dummy() result.status_code = r.status @@ -45,13 +51,18 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): content_length = self.headers.get("Content-Length") data = self.rfile.read(int(content_length)) if content_length else None - r = request(self.command, f"http://{UPSTREAM_HOST}{self.path}", headers=self.headers, data=data) + r = request( + self.command, + f"http://{UPSTREAM_HOST}{self.path}", + headers=self.headers, + data=data, + ) self.send_response(r.status_code) for k, v in r.headers.items(): self.send_header(k, v) self.end_headers() - if random.random() < 0.25 and len(r.content) > 1024*1024: - r.content = r.content[:len(r.content)//2] + if random.random() < 0.25 and len(r.content) > 1024 * 1024: + r.content = r.content[: len(r.content) // 2] self.wfile.write(r.content) self.wfile.close() diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 04981523432..b7ef3ce3ef2 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -8,17 +8,24 @@ from helpers.utility import generate_values, replace_config, SafeThread SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/storage_conf.xml'.format(get_instances_dir())) +CONFIG_PATH = os.path.join( + SCRIPT_DIR, + "./{}/node/configs/config.d/storage_conf.xml".format(get_instances_dir()), +) @pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=["configs/config.d/storage_conf.xml", - "configs/config.d/bg_processing_pool_conf.xml"], - with_minio=True) + cluster.add_instance( + "node", + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/config.d/bg_processing_pool_conf.xml", + ], + with_minio=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -39,7 +46,7 @@ def create_table(node, table_name, **additional_settings): settings = { "storage_policy": "s3", "old_parts_lifetime": 0, - "index_granularity": 512 + "index_granularity": 512, } settings.update(additional_settings) @@ -60,28 +67,39 @@ def create_table(node, table_name, **additional_settings): def run_s3_mocks(cluster): logging.info("Starting s3 mocks") - mocks = ( - ("unstable_proxy.py", "resolver", "8081"), - ) + mocks = (("unstable_proxy.py", "resolver", "8081"),) for mock_filename, container, port in mocks: container_id = cluster.get_container_id(container) current_dir = os.path.dirname(__file__) - cluster.copy_file_to_container(container_id, os.path.join(current_dir, "s3_mocks", mock_filename), mock_filename) - cluster.exec_in_container(container_id, ["python", mock_filename, port], detach=True) + cluster.copy_file_to_container( + container_id, + os.path.join(current_dir, "s3_mocks", mock_filename), + mock_filename, + ) + cluster.exec_in_container( + container_id, ["python", mock_filename, port], detach=True + ) # Wait for S3 mocks to start for mock_filename, container, port in mocks: num_attempts = 100 for attempt in range(num_attempts): - ping_response = cluster.exec_in_container(cluster.get_container_id(container), - ["curl", "-s", f"http://localhost:{port}/"], nothrow=True) + ping_response = cluster.exec_in_container( + cluster.get_container_id(container), + ["curl", "-s", f"http://localhost:{port}/"], + nothrow=True, + ) if ping_response != "OK": if attempt == num_attempts - 1: - assert ping_response == "OK", f'Expected "OK", but got "{ping_response}"' + assert ( + ping_response == "OK" + ), f'Expected "OK", but got "{ping_response}"' else: time.sleep(1) else: - logging.debug(f"mock {mock_filename} ({port}) answered {ping_response} on attempt {attempt}") + logging.debug( + f"mock {mock_filename} ({port}) answered {ping_response} on attempt {attempt}" + ) break logging.info("S3 mocks started") @@ -90,11 +108,11 @@ def run_s3_mocks(cluster): def wait_for_delete_s3_objects(cluster, expected, timeout=30): minio = cluster.minio_client while timeout > 0: - if len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == expected: + if len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == expected: return timeout -= 1 time.sleep(1) - assert(len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == expected) + assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == expected @pytest.fixture(autouse=True) @@ -110,7 +128,7 @@ def drop_table(cluster, node_name): wait_for_delete_s3_objects(cluster, 0) finally: # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -118,59 +136,86 @@ def drop_table(cluster, node_name): "min_rows_for_wide_part,files_per_part,node_name", [ (0, FILES_OVERHEAD_PER_PART_WIDE, "node"), - (8192, FILES_OVERHEAD_PER_PART_COMPACT, "node") - ] + (8192, FILES_OVERHEAD_PER_PART_COMPACT, "node"), + ], ) -def test_simple_insert_select(cluster, min_rows_for_wide_part, files_per_part, node_name): +def test_simple_insert_select( + cluster, min_rows_for_wide_part, files_per_part, node_name +): node = cluster.instances[node_name] create_table(node, "s3_test", min_rows_for_wide_part=min_rows_for_wide_part) minio = cluster.minio_client - values1 = generate_values('2020-01-03', 4096) + values1 = generate_values("2020-01-03", 4096) node.query("INSERT INTO s3_test VALUES {}".format(values1)) assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values") == values1 - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + files_per_part + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + files_per_part + ) - values2 = generate_values('2020-01-04', 4096) + values2 = generate_values("2020-01-04", 4096) node.query("INSERT INTO s3_test VALUES {}".format(values2)) - assert node.query("SELECT * FROM s3_test ORDER BY dt, id FORMAT Values") == values1 + "," + values2 - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + files_per_part * 2 + assert ( + node.query("SELECT * FROM s3_test ORDER BY dt, id FORMAT Values") + == values1 + "," + values2 + ) + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + files_per_part * 2 + ) - assert node.query("SELECT count(*) FROM s3_test where id = 1 FORMAT Values") == "(2)" + assert ( + node.query("SELECT count(*) FROM s3_test where id = 1 FORMAT Values") == "(2)" + ) -@pytest.mark.parametrize( - "merge_vertical,node_name", [ - (True, "node"), - (False, "node") -]) +@pytest.mark.parametrize("merge_vertical,node_name", [(True, "node"), (False, "node")]) def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): settings = {} if merge_vertical: - settings['vertical_merge_algorithm_min_rows_to_activate'] = 0 - settings['vertical_merge_algorithm_min_columns_to_activate'] = 0 + settings["vertical_merge_algorithm_min_rows_to_activate"] = 0 + settings["vertical_merge_algorithm_min_columns_to_activate"] = 0 node = cluster.instances[node_name] create_table(node, "s3_test", **settings) minio = cluster.minio_client node.query("SYSTEM STOP MERGES s3_test") - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 1024))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 2048))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 1024, -1))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 2048, -1))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 1024)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 2048)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 1024, -1)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 2048, -1)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096, -1)) + ) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" - assert node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD + assert ( + node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" + ) + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD + ) node.query("SYSTEM START MERGES s3_test") # Wait for merges and old parts deletion for attempt in range(0, 10): - parts_count = node.query("SELECT COUNT(*) FROM system.parts WHERE table = 's3_test' FORMAT Values") + parts_count = node.query( + "SELECT COUNT(*) FROM system.parts WHERE table = 's3_test' FORMAT Values" + ) if parts_count == "(1)": break @@ -180,7 +225,9 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): time.sleep(1) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" - assert node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" + assert ( + node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" + ) wait_for_delete_s3_objects(cluster, FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD) @@ -190,27 +237,44 @@ def test_alter_table_columns(cluster, node_name): create_table(node, "s3_test") minio = cluster.minio_client - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096, -1)) + ) node.query("ALTER TABLE s3_test ADD COLUMN col1 UInt64 DEFAULT 1") # To ensure parts have merged node.query("OPTIMIZE TABLE s3_test") assert node.query("SELECT sum(col1) FROM s3_test FORMAT Values") == "(8192)" - assert node.query("SELECT sum(col1) FROM s3_test WHERE id > 0 FORMAT Values") == "(4096)" - wait_for_delete_s3_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN) + assert ( + node.query("SELECT sum(col1) FROM s3_test WHERE id > 0 FORMAT Values") + == "(4096)" + ) + wait_for_delete_s3_objects( + cluster, + FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN, + ) - node.query("ALTER TABLE s3_test MODIFY COLUMN col1 String", settings={"mutations_sync": 2}) + node.query( + "ALTER TABLE s3_test MODIFY COLUMN col1 String", settings={"mutations_sync": 2} + ) assert node.query("SELECT distinct(col1) FROM s3_test FORMAT Values") == "('1')" # and file with mutation - wait_for_delete_s3_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + 1) + wait_for_delete_s3_objects( + cluster, + FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD_PER_COLUMN + 1, + ) node.query("ALTER TABLE s3_test DROP COLUMN col1", settings={"mutations_sync": 2}) # and 2 files with mutations - wait_for_delete_s3_objects(cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2) + wait_for_delete_s3_objects( + cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2 + ) @pytest.mark.parametrize("node_name", ["node"]) @@ -219,30 +283,48 @@ def test_attach_detach_partition(cluster, node_name): create_table(node, "s3_test") minio = cluster.minio_client - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) node.query("ALTER TABLE s3_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) node.query("ALTER TABLE s3_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + ) node.query("ALTER TABLE s3_test DETACH PARTITION '2020-01-04'") - node.query("ALTER TABLE s3_test DROP DETACHED PARTITION '2020-01-04'", settings={"allow_drop_detached": 1}) + node.query( + "ALTER TABLE s3_test DROP DETACHED PARTITION '2020-01-04'", + settings={"allow_drop_detached": 1}, + ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + ) @pytest.mark.parametrize("node_name", ["node"]) @@ -251,20 +333,31 @@ def test_move_partition_to_another_disk(cluster, node_name): create_table(node, "s3_test") minio = cluster.minio_client - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 'hdd'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 's3'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) @pytest.mark.parametrize("node_name", ["node"]) @@ -273,13 +366,19 @@ def test_table_manipulations(cluster, node_name): create_table(node, "s3_test") minio = cluster.minio_client - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) node.query("RENAME TABLE s3_test TO s3_renamed") assert node.query("SELECT count(*) FROM s3_renamed FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) node.query("RENAME TABLE s3_renamed TO s3_test") assert node.query("CHECK TABLE s3_test FORMAT Values") == "(1)" @@ -287,12 +386,16 @@ def test_table_manipulations(cluster, node_name): node.query("DETACH TABLE s3_test") node.query("ATTACH TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) node.query("TRUNCATE TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + ) @pytest.mark.parametrize("node_name", ["node"]) @@ -301,14 +404,24 @@ def test_move_replace_partition_to_another_table(cluster, node_name): create_table(node, "s3_test") minio = cluster.minio_client - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-05', 4096, -1))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-06', 4096, -1))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-05", 4096, -1)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-06", 4096, -1)) + ) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + ) create_table(node, "s3_clone") @@ -319,16 +432,24 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT sum(id) FROM s3_clone FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_clone FORMAT Values") == "(8192)" # Number of objects in S3 should be unchanged. - assert len(list( - minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + ) # Add new partitions to source table, but with different values and replace them from copied table. - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-05', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096, -1)) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" - assert len(list( - minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 + ) node.query("ALTER TABLE s3_test REPLACE PARTITION '2020-01-03' FROM s3_clone") node.query("ALTER TABLE s3_test REPLACE PARTITION '2020-01-05' FROM s3_clone") @@ -338,26 +459,32 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT count(*) FROM s3_clone FORMAT Values") == "(8192)" # Wait for outdated partitions deletion. - wait_for_delete_s3_objects(cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4) + wait_for_delete_s3_objects( + cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + ) node.query("DROP TABLE s3_clone NO DELAY") assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" # Data should remain in S3 - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + ) node.query("ALTER TABLE s3_test FREEZE") # Number S3 objects should be unchanged. - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + ) node.query("DROP TABLE s3_test NO DELAY") # Backup data should remain in S3. wait_for_delete_s3_objects(cluster, FILES_OVERHEAD_PER_PART_WIDE * 4) - for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) @@ -367,23 +494,32 @@ def test_freeze_unfreeze(cluster, node_name): create_table(node, "s3_test") minio = cluster.minio_client - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) node.query("ALTER TABLE s3_test FREEZE WITH NAME 'backup1'") - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096)) + ) node.query("ALTER TABLE s3_test FREEZE WITH NAME 'backup2'") node.query("TRUNCATE TABLE s3_test") - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) + == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + ) # Unfreeze single partition from backup1. - node.query("ALTER TABLE s3_test UNFREEZE PARTITION '2020-01-03' WITH NAME 'backup1'") + node.query( + "ALTER TABLE s3_test UNFREEZE PARTITION '2020-01-03' WITH NAME 'backup1'" + ) # Unfreeze all partitions from backup2. node.query("ALTER TABLE s3_test UNFREEZE WITH NAME 'backup2'") # Data should be removed from S3. - assert len( - list(minio.list_objects(cluster.minio_bucket, 'data/'))) == FILES_OVERHEAD + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == FILES_OVERHEAD + ) @pytest.mark.parametrize("node_name", ["node"]) @@ -393,21 +529,31 @@ def test_s3_disk_apply_new_settings(cluster, node_name): def get_s3_requests(): node.query("SYSTEM FLUSH LOGS") - return int(node.query("SELECT value FROM system.events WHERE event='S3WriteRequestsCount'")) + return int( + node.query( + "SELECT value FROM system.events WHERE event='S3WriteRequestsCount'" + ) + ) s3_requests_before = get_s3_requests() - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-03', 4096))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) s3_requests_to_write_partition = get_s3_requests() - s3_requests_before # Force multi-part upload mode. - replace_config(CONFIG_PATH, + replace_config( + CONFIG_PATH, "33554432", - "0") + "0", + ) node.query("SYSTEM RELOAD CONFIG") s3_requests_before = get_s3_requests() - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) + node.query( + "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096, -1)) + ) # There should be 3 times more S3 requests because multi-part upload mode uses 3 requests to upload object. assert get_s3_requests() - s3_requests_before == s3_requests_to_write_partition * 3 @@ -418,8 +564,16 @@ def test_s3_disk_restart_during_load(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-04', 1024 * 1024))) - node.query("INSERT INTO s3_test VALUES {}".format(generate_values('2020-01-05', 1024 * 1024, -1))) + node.query( + "INSERT INTO s3_test VALUES {}".format( + generate_values("2020-01-04", 1024 * 1024) + ) + ) + node.query( + "INSERT INTO s3_test VALUES {}".format( + generate_values("2020-01-05", 1024 * 1024, -1) + ) + ) def read(): for ii in range(0, 20): @@ -451,21 +605,29 @@ def test_s3_disk_restart_during_load(cluster, node_name): @pytest.mark.parametrize("node_name", ["node"]) def test_s3_disk_reads_on_unstable_connection(cluster, node_name): node = cluster.instances[node_name] - create_table(node, "s3_test", storage_policy='unstable_s3') - node.query("INSERT INTO s3_test SELECT today(), *, toString(*) FROM system.numbers LIMIT 9000000") + create_table(node, "s3_test", storage_policy="unstable_s3") + node.query( + "INSERT INTO s3_test SELECT today(), *, toString(*) FROM system.numbers LIMIT 9000000" + ) for i in range(30): print(f"Read sequence {i}") - assert node.query("SELECT sum(id) FROM s3_test").splitlines() == ["40499995500000"] + assert node.query("SELECT sum(id) FROM s3_test").splitlines() == [ + "40499995500000" + ] @pytest.mark.parametrize("node_name", ["node"]) def test_lazy_seek_optimization_for_async_read(cluster, node_name): node = cluster.instances[node_name] node.query("DROP TABLE IF EXISTS s3_test NO DELAY") - node.query("CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3';") - node.query("INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000000") + node.query( + "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3';" + ) + node.query( + "INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000000" + ) node.query("SELECT * FROM s3_test WHERE value LIKE '%abc%' ORDER BY value LIMIT 10") node.query("DROP TABLE IF EXISTS s3_test NO DELAY") minio = cluster.minio_client - for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index 9f5c7b1c8ce..b6567dfebc5 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -10,73 +10,75 @@ cache = {} mutex = Lock() -@route('/fail_request/<_request_number>') +@route("/fail_request/<_request_number>") def fail_request(_request_number): request_number = int(_request_number) if request_number > 0: - cache['request_number'] = request_number + cache["request_number"] = request_number else: - cache.pop('request_number', None) - return 'OK' + cache.pop("request_number", None) + return "OK" -@route('/throttle_request/<_request_number>') +@route("/throttle_request/<_request_number>") def fail_request(_request_number): request_number = int(_request_number) if request_number > 0: - cache['throttle_request_number'] = request_number + cache["throttle_request_number"] = request_number else: - cache.pop('throttle_request_number', None) - return 'OK' + cache.pop("throttle_request_number", None) + return "OK" # Handle for MultipleObjectsDelete. -@route('/<_bucket>', ['POST']) +@route("/<_bucket>", ["POST"]) def delete(_bucket): - response.set_header("Location", "http://minio1:9001/" + _bucket + "?" + request.query_string) + response.set_header( + "Location", "http://minio1:9001/" + _bucket + "?" + request.query_string + ) response.status = 307 - return 'Redirected' + return "Redirected" -@route('/<_bucket>/<_path:path>', ['GET', 'POST', 'PUT', 'DELETE']) +@route("/<_bucket>/<_path:path>", ["GET", "POST", "PUT", "DELETE"]) def server(_bucket, _path): # It's delete query for failed part - if _path.endswith('delete'): - response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) + if _path.endswith("delete"): + response.set_header("Location", "http://minio1:9001/" + _bucket + "/" + _path) response.status = 307 - return 'Redirected' + return "Redirected" mutex.acquire() try: - if cache.get('request_number', None): - request_number = cache.pop('request_number') - 1 + if cache.get("request_number", None): + request_number = cache.pop("request_number") - 1 if request_number > 0: - cache['request_number'] = request_number + cache["request_number"] = request_number else: response.status = 500 - response.content_type = 'text/xml' + response.content_type = "text/xml" return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' - if cache.get('throttle_request_number', None): - request_number = cache.pop('throttle_request_number') - 1 + if cache.get("throttle_request_number", None): + request_number = cache.pop("throttle_request_number") - 1 if request_number > 0: - cache['throttle_request_number'] = request_number + cache["throttle_request_number"] = request_number else: response.status = 429 - response.content_type = 'text/xml' + response.content_type = "text/xml" return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' finally: mutex.release() - response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) + response.set_header("Location", "http://minio1:9001/" + _bucket + "/" + _path) response.status = 307 - return 'Redirected' + return "Redirected" -@route('/') +@route("/") def ping(): - return 'OK' + return "OK" -run(host='0.0.0.0', port=8080) +run(host="0.0.0.0", port=8080) diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index 44e7e0ae5ad..d4c691fdb55 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -11,19 +11,28 @@ from helpers.cluster import ClickHouseCluster # Runs custom python-based S3 endpoint. def run_endpoint(cluster): logging.info("Starting custom S3 endpoint") - container_id = cluster.get_container_id('resolver') + container_id = cluster.get_container_id("resolver") current_dir = os.path.dirname(__file__) - cluster.copy_file_to_container(container_id, os.path.join(current_dir, "s3_endpoint", "endpoint.py"), "endpoint.py") + cluster.copy_file_to_container( + container_id, + os.path.join(current_dir, "s3_endpoint", "endpoint.py"), + "endpoint.py", + ) cluster.exec_in_container(container_id, ["python", "endpoint.py"], detach=True) # Wait for S3 endpoint start num_attempts = 100 for attempt in range(num_attempts): - ping_response = cluster.exec_in_container(cluster.get_container_id('resolver'), - ["curl", "-s", "http://resolver:8080/"], nothrow=True) - if ping_response != 'OK': + ping_response = cluster.exec_in_container( + cluster.get_container_id("resolver"), + ["curl", "-s", "http://resolver:8080/"], + nothrow=True, + ) + if ping_response != "OK": if attempt == num_attempts - 1: - assert ping_response == 'OK', 'Expected "OK", but got "{}"'.format(ping_response) + assert ping_response == "OK", 'Expected "OK", but got "{}"'.format( + ping_response + ) else: time.sleep(1) else: @@ -33,25 +42,34 @@ def run_endpoint(cluster): def fail_request(cluster, request): - response = cluster.exec_in_container(cluster.get_container_id('resolver'), - ["curl", "-s", "http://resolver:8080/fail_request/{}".format(request)]) - assert response == 'OK', 'Expected "OK", but got "{}"'.format(response) + response = cluster.exec_in_container( + cluster.get_container_id("resolver"), + ["curl", "-s", "http://resolver:8080/fail_request/{}".format(request)], + ) + assert response == "OK", 'Expected "OK", but got "{}"'.format(response) + def throttle_request(cluster, request): - response = cluster.exec_in_container(cluster.get_container_id('resolver'), - ["curl", "-s", "http://resolver:8080/throttle_request/{}".format(request)]) - assert response == 'OK', 'Expected "OK", but got "{}"'.format(response) + response = cluster.exec_in_container( + cluster.get_container_id("resolver"), + ["curl", "-s", "http://resolver:8080/throttle_request/{}".format(request)], + ) + assert response == "OK", 'Expected "OK", but got "{}"'.format(response) @pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=["configs/config.d/storage_conf.xml", - "configs/config.d/instant_moves.xml", - "configs/config.d/part_log.xml"], - with_minio=True) + cluster.add_instance( + "node", + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/config.d/instant_moves.xml", + "configs/config.d/part_log.xml", + ], + with_minio=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -72,7 +90,9 @@ def drop_table(cluster): # S3 request will be failed for an appropriate part file write. FILES_PER_PART_BASE = 5 # partition.dat, default_compression_codec.txt, count.txt, columns.txt, checksums.txt -FILES_PER_PART_WIDE = FILES_PER_PART_BASE + 1 + 1 + 3 * 2 # Primary index, MinMax, Mark and data file for column(s) +FILES_PER_PART_WIDE = ( + FILES_PER_PART_BASE + 1 + 1 + 3 * 2 +) # Primary index, MinMax, Mark and data file for column(s) # In debug build there are additional requests (from MergeTreeDataPartWriterWide.cpp:554 due to additional validation). FILES_PER_PART_WIDE_DEBUG = 2 # Additional requests to S3 in debug build @@ -85,10 +105,12 @@ FILES_PER_PART_COMPACT_DEBUG = 0 "min_bytes_for_wide_part,request_count,debug_request_count", [ (0, FILES_PER_PART_WIDE, FILES_PER_PART_WIDE_DEBUG), - (1024 * 1024, FILES_PER_PART_COMPACT, FILES_PER_PART_COMPACT_DEBUG) - ] + (1024 * 1024, FILES_PER_PART_COMPACT, FILES_PER_PART_COMPACT_DEBUG), + ], ) -def test_write_failover(cluster, min_bytes_for_wide_part, request_count, debug_request_count): +def test_write_failover( + cluster, min_bytes_for_wide_part, request_count, debug_request_count +): node = cluster.instances["node"] node.query( @@ -101,8 +123,9 @@ def test_write_failover(cluster, min_bytes_for_wide_part, request_count, debug_r ORDER BY id PARTITION BY dt SETTINGS storage_policy='s3', min_bytes_for_wide_part={} - """ - .format(min_bytes_for_wide_part) + """.format( + min_bytes_for_wide_part + ) ) is_debug_mode = False @@ -113,7 +136,9 @@ def test_write_failover(cluster, min_bytes_for_wide_part, request_count, debug_r fail_request(cluster, request + 1) data = "('2020-03-01',0,'data'),('2020-03-01',1,'data')" - positive = request >= (request_count + debug_request_count if is_debug_mode else request_count) + positive = request >= ( + request_count + debug_request_count if is_debug_mode else request_count + ) try: node.query("INSERT INTO s3_failover_test VALUES {}".format(data)) assert positive, "Insert query should be failed, request {}".format(request) @@ -123,17 +148,26 @@ def test_write_failover(cluster, min_bytes_for_wide_part, request_count, debug_r is_debug_mode = True positive = False - assert not positive, "Insert query shouldn't be failed, request {}".format(request) - assert str(e).find("Expected Error") != -1, "Unexpected error {}".format(str(e)) + assert not positive, "Insert query shouldn't be failed, request {}".format( + request + ) + assert str(e).find("Expected Error") != -1, "Unexpected error {}".format( + str(e) + ) if positive: # Disable request failing. fail_request(cluster, 0) - assert node.query("CHECK TABLE s3_failover_test") == '1\n' - assert success_count > 1 or node.query("SELECT * FROM s3_failover_test FORMAT Values") == data + assert node.query("CHECK TABLE s3_failover_test") == "1\n" + assert ( + success_count > 1 + or node.query("SELECT * FROM s3_failover_test FORMAT Values") == data + ) - assert success_count == (1 if is_debug_mode else debug_request_count + 1), "Insert query should be successful at least once" + assert success_count == ( + 1 if is_debug_mode else debug_request_count + 1 + ), "Insert query should be successful at least once" # Check that second data part move is ended successfully if first attempt was failed. @@ -156,15 +190,21 @@ def test_move_failover(cluster): # Fail a request to S3 to break first TTL move. fail_request(cluster, 1) - node.query("INSERT INTO s3_failover_test VALUES (now() - 2, 0, 'data'), (now() - 2, 1, 'data')") + node.query( + "INSERT INTO s3_failover_test VALUES (now() - 2, 0, 'data'), (now() - 2, 1, 'data')" + ) # Wait for part move to S3. max_attempts = 10 for attempt in range(max_attempts + 1): - disk = node.query("SELECT disk_name FROM system.parts WHERE table='s3_failover_test' LIMIT 1") + disk = node.query( + "SELECT disk_name FROM system.parts WHERE table='s3_failover_test' LIMIT 1" + ) if disk != "s3\n": if attempt == max_attempts: - assert disk == "s3\n", "Expected move to S3 while part still on disk " + disk + assert disk == "s3\n", ( + "Expected move to S3 while part still on disk " + disk + ) else: time.sleep(1) else: @@ -174,23 +214,33 @@ def test_move_failover(cluster): node.query("SYSTEM FLUSH LOGS") # There should be 2 attempts to move part. - assert node.query(""" + assert ( + node.query( + """ SELECT count(*) FROM system.part_log WHERE event_type='MovePart' AND table='s3_failover_test' - """) == '2\n' + """ + ) + == "2\n" + ) # First attempt should be failed with expected error. - exception = node.query(""" + exception = node.query( + """ SELECT exception FROM system.part_log WHERE event_type='MovePart' AND table='s3_failover_test' AND notEmpty(exception) ORDER BY event_time LIMIT 1 - """) + """ + ) assert exception.find("Expected Error") != -1, exception # Ensure data is not corrupted. - assert node.query("CHECK TABLE s3_failover_test") == '1\n' - assert node.query("SELECT id,data FROM s3_failover_test FORMAT Values") == "(0,'data'),(1,'data')" + assert node.query("CHECK TABLE s3_failover_test") == "1\n" + assert ( + node.query("SELECT id,data FROM s3_failover_test FORMAT Values") + == "(0,'data'),(1,'data')" + ) # Check that throttled request retries and does not cause an error on disk with default `retry_attempts` (>0) @@ -212,6 +262,11 @@ def test_throttle_retry(cluster): throttle_request(cluster, 1) - assert node.query(""" + assert ( + node.query( + """ SELECT * FROM s3_throttle_retry_test - """) == '42\n' + """ + ) + == "42\n" + ) diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py index acbcd8c04cf..6ae63db52ef 100644 --- a/tests/integration/test_merge_tree_s3_restore/test.py +++ b/tests/integration/test_merge_tree_s3_restore/test.py @@ -9,16 +9,24 @@ from helpers.cluster import ClickHouseCluster, get_instances_dir SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -NOT_RESTORABLE_CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node_not_restorable/configs/config.d/storage_conf_not_restorable.xml'.format(get_instances_dir())) -COMMON_CONFIGS = ["configs/config.d/bg_processing_pool_conf.xml", "configs/config.d/clusters.xml"] +NOT_RESTORABLE_CONFIG_PATH = os.path.join( + SCRIPT_DIR, + "./{}/node_not_restorable/configs/config.d/storage_conf_not_restorable.xml".format( + get_instances_dir() + ), +) +COMMON_CONFIGS = [ + "configs/config.d/bg_processing_pool_conf.xml", + "configs/config.d/clusters.xml", +] def replace_config(old, new): - config = open(NOT_RESTORABLE_CONFIG_PATH, 'r') + config = open(NOT_RESTORABLE_CONFIG_PATH, "r") config_lines = config.readlines() config.close() config_lines = [line.replace(old, new) for line in config_lines] - config = open(NOT_RESTORABLE_CONFIG_PATH, 'w') + config = open(NOT_RESTORABLE_CONFIG_PATH, "w") config.writelines(config_lines) config.close() @@ -28,20 +36,34 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"], - macros={"cluster": "node", "replica": "0"}, - with_minio=True, with_zookeeper=True, stay_alive=True) - cluster.add_instance("node_another_bucket", - main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf_another_bucket.xml"], - macros={"cluster": "node_another_bucket", "replica": "0"}, - with_zookeeper=True, stay_alive=True) - cluster.add_instance("node_another_bucket_path", - main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf_another_bucket_path.xml"], - stay_alive=True) - cluster.add_instance("node_not_restorable", - main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf_not_restorable.xml"], - stay_alive=True) + cluster.add_instance( + "node", + main_configs=COMMON_CONFIGS + ["configs/config.d/storage_conf.xml"], + macros={"cluster": "node", "replica": "0"}, + with_minio=True, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node_another_bucket", + main_configs=COMMON_CONFIGS + + ["configs/config.d/storage_conf_another_bucket.xml"], + macros={"cluster": "node_another_bucket", "replica": "0"}, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node_another_bucket_path", + main_configs=COMMON_CONFIGS + + ["configs/config.d/storage_conf_another_bucket_path.xml"], + stay_alive=True, + ) + cluster.add_instance( + "node_not_restorable", + main_configs=COMMON_CONFIGS + + ["configs/config.d/storage_conf_not_restorable.xml"], + stay_alive=True, + ) logging.info("Starting cluster...") cluster.start() @@ -54,7 +76,7 @@ def cluster(): def random_string(length): letters = string.ascii_letters - return ''.join(random.choice(letters) for i in range(length)) + return "".join(random.choice(letters) for i in range(length)) def generate_values(date_str, count, sign=1): @@ -63,8 +85,14 @@ def generate_values(date_str, count, sign=1): return ",".join(["('{}',{},'{}',{})".format(x, y, z, 0) for x, y, z in data]) -def create_table(node, table_name, attach=False, replicated=False, db_atomic=False, uuid=""): - node.query("CREATE DATABASE IF NOT EXISTS s3 ENGINE = {engine}".format(engine="Atomic" if db_atomic else "Ordinary")) +def create_table( + node, table_name, attach=False, replicated=False, db_atomic=False, uuid="" +): + node.query( + "CREATE DATABASE IF NOT EXISTS s3 ENGINE = {engine}".format( + engine="Atomic" if db_atomic else "Ordinary" + ) + ) create_table_statement = """ {create} TABLE s3.{table_name} {uuid} {on_cluster} ( @@ -80,11 +108,15 @@ def create_table(node, table_name, attach=False, replicated=False, db_atomic=Fal storage_policy='s3', old_parts_lifetime=600, index_granularity=512 - """.format(create="ATTACH" if attach else "CREATE", - table_name=table_name, - uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "", - on_cluster="ON CLUSTER '{}'".format(node.name) if replicated else "", - engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')" if replicated else "MergeTree()") + """.format( + create="ATTACH" if attach else "CREATE", + table_name=table_name, + uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "", + on_cluster="ON CLUSTER '{}'".format(node.name) if replicated else "", + engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')" + if replicated + else "MergeTree()", + ) node.query(create_table_statement) @@ -98,37 +130,68 @@ def purge_s3(cluster, bucket): def drop_s3_metadata(node): - node.exec_in_container(['bash', '-c', 'rm -rf /var/lib/clickhouse/disks/s3/*'], user='root') + node.exec_in_container( + ["bash", "-c", "rm -rf /var/lib/clickhouse/disks/s3/*"], user="root" + ) def drop_shadow_information(node): - node.exec_in_container(['bash', '-c', 'rm -rf /var/lib/clickhouse/shadow/*'], user='root') + node.exec_in_container( + ["bash", "-c", "rm -rf /var/lib/clickhouse/shadow/*"], user="root" + ) def create_restore_file(node, revision=None, bucket=None, path=None, detached=None): - node.exec_in_container(['bash', '-c', 'mkdir -p /var/lib/clickhouse/disks/s3/'], user='root') - node.exec_in_container(['bash', '-c', 'touch /var/lib/clickhouse/disks/s3/restore'], user='root') + node.exec_in_container( + ["bash", "-c", "mkdir -p /var/lib/clickhouse/disks/s3/"], user="root" + ) + node.exec_in_container( + ["bash", "-c", "touch /var/lib/clickhouse/disks/s3/restore"], user="root" + ) add_restore_option = 'echo -en "{}={}\n" >> /var/lib/clickhouse/disks/s3/restore' if revision: - node.exec_in_container(['bash', '-c', add_restore_option.format('revision', revision)], user='root') + node.exec_in_container( + ["bash", "-c", add_restore_option.format("revision", revision)], user="root" + ) if bucket: - node.exec_in_container(['bash', '-c', add_restore_option.format('source_bucket', bucket)], user='root') + node.exec_in_container( + ["bash", "-c", add_restore_option.format("source_bucket", bucket)], + user="root", + ) if path: - node.exec_in_container(['bash', '-c', add_restore_option.format('source_path', path)], user='root') + node.exec_in_container( + ["bash", "-c", add_restore_option.format("source_path", path)], user="root" + ) if detached: - node.exec_in_container(['bash', '-c', add_restore_option.format('detached', 'true')], user='root') + node.exec_in_container( + ["bash", "-c", add_restore_option.format("detached", "true")], user="root" + ) def get_revision_counter(node, backup_number): - return int(node.exec_in_container( - ['bash', '-c', 'cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt'.format(backup_number)], user='root')) + return int( + node.exec_in_container( + [ + "bash", + "-c", + "cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt".format( + backup_number + ), + ], + user="root", + ) + ) def get_table_uuid(node, db_atomic, table): uuid = "" if db_atomic: - uuid = node.query("SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format(table)).strip() + uuid = node.query( + "SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format( + table + ) + ).strip() return uuid @@ -136,7 +199,12 @@ def get_table_uuid(node, db_atomic, table): def drop_table(cluster): yield - node_names = ["node", "node_another_bucket", "node_another_bucket_path", "node_not_restorable"] + node_names = [ + "node", + "node_another_bucket", + "node_another_bucket_path", + "node_not_restorable", + ] for node_name in node_names: node = cluster.instances[node_name] @@ -151,21 +219,25 @@ def drop_table(cluster): purge_s3(cluster, bucket) -@pytest.mark.parametrize( - "replicated", [False, True] -) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("replicated", [False, True]) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_full_restore(cluster, replicated, db_atomic): node = cluster.instances["node"] create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1)) + ) node.query("DETACH TABLE s3.test") drop_s3_metadata(node) @@ -173,66 +245,94 @@ def test_full_restore(cluster, replicated, db_atomic): node.query("SYSTEM RESTART DISK s3") node.query("ATTACH TABLE s3.test") - assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) + assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format( + 4096 * 4 + ) assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_restore_another_bucket_path(cluster, db_atomic): node = cluster.instances["node"] create_table(node, "test", db_atomic=db_atomic) uuid = get_table_uuid(node, db_atomic, "test") - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1)) + ) # To ensure parts have merged node.query("OPTIMIZE TABLE s3.test") - assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) + assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format( + 4096 * 4 + ) assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) node_another_bucket = cluster.instances["node_another_bucket"] create_restore_file(node_another_bucket, bucket="root") node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid + ) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 4) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) node_another_bucket_path = cluster.instances["node_another_bucket_path"] create_restore_file(node_another_bucket_path, bucket="root2", path="data") node_another_bucket_path.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket_path, "test", attach=True, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket_path, "test", attach=True, db_atomic=db_atomic, uuid=uuid + ) - assert node_another_bucket_path.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) - assert node_another_bucket_path.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) + assert node_another_bucket_path.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 4) + assert node_another_bucket_path.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_restore_different_revisions(cluster, db_atomic): node = cluster.instances["node"] create_table(node, "test", db_atomic=db_atomic) uuid = get_table_uuid(node, db_atomic, "test") - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1)) + ) node.query("ALTER TABLE s3.test FREEZE") revision1 = get_revision_counter(node, 1) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1)) + ) node.query("ALTER TABLE s3.test FREEZE") revision2 = get_revision_counter(node, 2) @@ -243,20 +343,33 @@ def test_restore_different_revisions(cluster, db_atomic): node.query("ALTER TABLE s3.test FREEZE") revision3 = get_revision_counter(node, 3) - assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) + assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format( + 4096 * 4 + ) assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node.query("SELECT count(*) from system.parts where table = 'test'") == '5\n' + assert node.query("SELECT count(*) from system.parts where table = 'test'") == "5\n" node_another_bucket = cluster.instances["node_another_bucket"] # Restore to revision 1 (2 parts). create_restore_file(node_another_bucket, revision=revision1, bucket="root") node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid + ) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT count(*) from system.parts where table = 'test'") == '2\n' + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 2) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert ( + node_another_bucket.query( + "SELECT count(*) from system.parts where table = 'test'" + ) + == "2\n" + ) # Restore to revision 2 (4 parts). node_another_bucket.query("DETACH TABLE s3.test") @@ -264,9 +377,18 @@ def test_restore_different_revisions(cluster, db_atomic): node_another_bucket.query("SYSTEM RESTART DISK s3") node_another_bucket.query("ATTACH TABLE s3.test") - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT count(*) from system.parts where table = 'test'") == '4\n' + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 4) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert ( + node_another_bucket.query( + "SELECT count(*) from system.parts where table = 'test'" + ) + == "4\n" + ) # Restore to revision 3 (4 parts + 1 merged). node_another_bucket.query("DETACH TABLE s3.test") @@ -274,27 +396,40 @@ def test_restore_different_revisions(cluster, db_atomic): node_another_bucket.query("SYSTEM RESTART DISK s3") node_another_bucket.query("ATTACH TABLE s3.test") - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT count(*) from system.parts where table = 'test'") == '5\n' + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 4) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert ( + node_another_bucket.query( + "SELECT count(*) from system.parts where table = 'test'" + ) + == "5\n" + ) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_restore_mutations(cluster, db_atomic): node = cluster.instances["node"] create_table(node, "test", db_atomic=db_atomic) uuid = get_table_uuid(node, db_atomic, "test") - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096, -1)) + ) node.query("ALTER TABLE s3.test FREEZE") revision_before_mutation = get_revision_counter(node, 1) - node.query("ALTER TABLE s3.test UPDATE counter = 1 WHERE 1", settings={"mutations_sync": 2}) + node.query( + "ALTER TABLE s3.test UPDATE counter = 1 WHERE 1", settings={"mutations_sync": 2} + ) node.query("ALTER TABLE s3.test FREEZE") revision_after_mutation = get_revision_counter(node, 2) @@ -302,24 +437,44 @@ def test_restore_mutations(cluster, db_atomic): node_another_bucket = cluster.instances["node_another_bucket"] # Restore to revision before mutation. - create_restore_file(node_another_bucket, revision=revision_before_mutation, bucket="root") + create_restore_file( + node_another_bucket, revision=revision_before_mutation, bucket="root" + ) node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid + ) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test FORMAT Values") == "({})".format(0) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 2) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test FORMAT Values" + ) == "({})".format(0) # Restore to revision after mutation. node_another_bucket.query("DETACH TABLE s3.test") - create_restore_file(node_another_bucket, revision=revision_after_mutation, bucket="root") + create_restore_file( + node_another_bucket, revision=revision_after_mutation, bucket="root" + ) node_another_bucket.query("SYSTEM RESTART DISK s3") node_another_bucket.query("ATTACH TABLE s3.test") - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values") == "({})".format(4096) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 2) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 2) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values" + ) == "({})".format(4096) # Restore to revision in the middle of mutation. # Unfinished mutation should be completed after table startup. @@ -332,31 +487,51 @@ def test_restore_mutations(cluster, db_atomic): # Wait for unfinished mutation completion. time.sleep(3) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values") == "({})".format(4096) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 2) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 2) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values" + ) == "({})".format(4096) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_migrate_to_restorable_schema(cluster, db_atomic): node = cluster.instances["node_not_restorable"] create_table(node, "test", db_atomic=db_atomic) uuid = get_table_uuid(node, db_atomic, "test") - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096, -1)) + ) - replace_config("false", "true") + replace_config( + "false", "true" + ) node.restart_clickhouse() - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-06', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-06', 4096, -1))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-06", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-06", 4096, -1)) + ) node.query("ALTER TABLE s3.test FREEZE") revision = get_revision_counter(node, 1) @@ -366,34 +541,50 @@ def test_migrate_to_restorable_schema(cluster, db_atomic): node_another_bucket = cluster.instances["node_another_bucket"] # Restore to revision before mutation. - create_restore_file(node_another_bucket, revision=revision, bucket="root", path="another_data") + create_restore_file( + node_another_bucket, revision=revision, bucket="root", path="another_data" + ) node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid + ) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 6) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 6) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) -@pytest.mark.parametrize( - "replicated", [False, True] -) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("replicated", [False, True]) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_restore_to_detached(cluster, replicated, db_atomic): node = cluster.instances["node"] create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic) uuid = get_table_uuid(node, db_atomic, "test") - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-05', 4096))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-06', 4096, -1))) - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-07', 4096, 0))) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-04", 4096, -1)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-05", 4096)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-06", 4096, -1)) + ) + node.query( + "INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-07", 4096, 0)) + ) # Add some mutation. - node.query("ALTER TABLE s3.test UPDATE counter = 1 WHERE 1", settings={"mutations_sync": 2}) + node.query( + "ALTER TABLE s3.test UPDATE counter = 1 WHERE 1", settings={"mutations_sync": 2} + ) # Detach some partition. node.query("ALTER TABLE s3.test DETACH PARTITION '2020-01-07'") @@ -403,42 +594,64 @@ def test_restore_to_detached(cluster, replicated, db_atomic): node_another_bucket = cluster.instances["node_another_bucket"] - create_restore_file(node_another_bucket, revision=revision, bucket="root", path="data", detached=True) + create_restore_file( + node_another_bucket, + revision=revision, + bucket="root", + path="data", + detached=True, + ) node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", replicated=replicated, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket, + "test", + replicated=replicated, + db_atomic=db_atomic, + uuid=uuid, + ) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(0) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(0) node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-03'") node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-04'") node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-05'") node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-06'") - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 4) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 4) # Attach partition that was already detached before backup-restore. node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-07'") - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 5) - assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) - assert node_another_bucket.query("SELECT sum(counter) FROM s3.test FORMAT Values") == "({})".format(4096 * 5) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 5) + assert node_another_bucket.query( + "SELECT sum(id) FROM s3.test FORMAT Values" + ) == "({})".format(0) + assert node_another_bucket.query( + "SELECT sum(counter) FROM s3.test FORMAT Values" + ) == "({})".format(4096 * 5) -@pytest.mark.parametrize( - "replicated", [False, True] -) -@pytest.mark.parametrize( - "db_atomic", [False, True] -) +@pytest.mark.parametrize("replicated", [False, True]) +@pytest.mark.parametrize("db_atomic", [False, True]) def test_restore_without_detached(cluster, replicated, db_atomic): node = cluster.instances["node"] create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic) uuid = get_table_uuid(node, db_atomic, "test") - node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 1))) + node.query("INSERT INTO s3.test VALUES {}".format(generate_values("2020-01-03", 1))) assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(1) @@ -447,12 +660,28 @@ def test_restore_without_detached(cluster, replicated, db_atomic): node_another_bucket = cluster.instances["node_another_bucket"] - create_restore_file(node_another_bucket, revision=revision, bucket="root", path="data", detached=True) + create_restore_file( + node_another_bucket, + revision=revision, + bucket="root", + path="data", + detached=True, + ) node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", replicated=replicated, db_atomic=db_atomic, uuid=uuid) + create_table( + node_another_bucket, + "test", + replicated=replicated, + db_atomic=db_atomic, + uuid=uuid, + ) - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(0) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(0) node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-03'") - assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(1) + assert node_another_bucket.query( + "SELECT count(*) FROM s3.test FORMAT Values" + ) == "({})".format(1) diff --git a/tests/integration/test_merge_tree_s3_with_cache/test.py b/tests/integration/test_merge_tree_s3_with_cache/test.py index be3d2709873..89b5a400b1b 100644 --- a/tests/integration/test_merge_tree_s3_with_cache/test.py +++ b/tests/integration/test_merge_tree_s3_with_cache/test.py @@ -8,9 +8,16 @@ from helpers.cluster import ClickHouseCluster def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/config.d/ssl_conf.xml", - "configs/config.d/query_log.xml"], - user_configs=["configs/config.d/users.xml"], with_minio=True) + cluster.add_instance( + "node", + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/config.d/ssl_conf.xml", + "configs/config.d/query_log.xml", + ], + user_configs=["configs/config.d/users.xml"], + with_minio=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -23,12 +30,16 @@ def cluster(): def get_query_stat(instance, hint): result = {} instance.query("SYSTEM FLUSH LOGS") - events = instance.query(''' + events = instance.query( + """ SELECT ProfileEvents.keys, ProfileEvents.values FROM system.query_log ARRAY JOIN ProfileEvents WHERE type != 1 AND query LIKE '%{}%' - '''.format(hint.replace("'", "\\'"))).split("\n") + """.format( + hint.replace("'", "\\'") + ) + ).split("\n") for event in events: ev = event.split("\t") if len(ev) == 2: @@ -36,6 +47,7 @@ def get_query_stat(instance, hint): result[ev[0]] = int(ev[1]) return result + @pytest.mark.parametrize("min_rows_for_wide_part,read_requests", [(0, 2), (8192, 1)]) def test_write_is_cached(cluster, min_rows_for_wide_part, read_requests): node = cluster.instances["node"] @@ -48,7 +60,9 @@ def test_write_is_cached(cluster, min_rows_for_wide_part, read_requests): ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='s3', min_rows_for_wide_part={} - """.format(min_rows_for_wide_part) + """.format( + min_rows_for_wide_part + ) ) node.query("SYSTEM FLUSH LOGS") @@ -65,8 +79,13 @@ def test_write_is_cached(cluster, min_rows_for_wide_part, read_requests): node.query("DROP TABLE IF EXISTS s3_test NO DELAY") -@pytest.mark.parametrize("min_rows_for_wide_part,all_files,bin_files", [(0, 4, 2), (8192, 2, 1)]) -def test_read_after_cache_is_wiped(cluster, min_rows_for_wide_part, all_files, bin_files): + +@pytest.mark.parametrize( + "min_rows_for_wide_part,all_files,bin_files", [(0, 4, 2), (8192, 2, 1)] +) +def test_read_after_cache_is_wiped( + cluster, min_rows_for_wide_part, all_files, bin_files +): node = cluster.instances["node"] node.query( @@ -77,7 +96,9 @@ def test_read_after_cache_is_wiped(cluster, min_rows_for_wide_part, all_files, b ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='s3', min_rows_for_wide_part={} - """.format(min_rows_for_wide_part) + """.format( + min_rows_for_wide_part + ) ) node.query("SYSTEM FLUSH LOGS") @@ -86,7 +107,10 @@ def test_read_after_cache_is_wiped(cluster, min_rows_for_wide_part, all_files, b node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") # Wipe cache - cluster.exec_in_container(cluster.get_container_id("node"), ["rm", "-rf", "/var/lib/clickhouse/disks/s3/cache/"]) + cluster.exec_in_container( + cluster.get_container_id("node"), + ["rm", "-rf", "/var/lib/clickhouse/disks/s3/cache/"], + ) select_query = "SELECT * FROM s3_test" node.query(select_query) @@ -99,7 +123,7 @@ def test_read_after_cache_is_wiped(cluster, min_rows_for_wide_part, all_files, b assert node.query(select_query) == "(0,'data'),(1,'data')" # With async reads profile events are not updated because reads are done in a separate thread. - #stat = get_query_stat(node, select_query) - #assert stat["S3ReadRequestsCount"] == bin_files + # stat = get_query_stat(node, select_query) + # assert stat["S3ReadRequestsCount"] == bin_files node.query("DROP TABLE IF EXISTS s3_test NO DELAY") diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index e2b30b8f90e..be07e4a1b8c 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -12,21 +12,31 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml', 'configs/config.d/storage_configuration.xml', - 'configs/config.d/cluster.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 1}) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/logs_config.xml", + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", + ], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=40M", "/jbod2:size=40M", "/external:size=200M"], + macros={"shard": 0, "replica": 1}, +) -node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml', 'configs/config.d/storage_configuration.xml', - 'configs/config.d/cluster.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 2}) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/logs_config.xml", + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", + ], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=40M", "/jbod2:size=40M", "/external:size=200M"], + macros={"shard": 0, "replica": 2}, +) @pytest.fixture(scope="module") @@ -44,28 +54,31 @@ def test_system_tables(start_cluster): { "name": "default", "path": "/var/lib/clickhouse/", - "keep_free_space": '1024', + "keep_free_space": "1024", }, { "name": "jbod1", "path": "/jbod1/", - "keep_free_space": '0', + "keep_free_space": "0", }, { "name": "jbod2", "path": "/jbod2/", - "keep_free_space": '10485760', + "keep_free_space": "10485760", }, { "name": "external", "path": "/external/", - "keep_free_space": '0', - } + "keep_free_space": "0", + }, ] - click_disk_data = json.loads(node1.query("SELECT name, path, keep_free_space FROM system.disks FORMAT JSON"))[ - "data"] - assert sorted(click_disk_data, key=lambda x: x["name"]) == sorted(expected_disks_data, key=lambda x: x["name"]) + click_disk_data = json.loads( + node1.query("SELECT name, path, keep_free_space FROM system.disks FORMAT JSON") + )["data"] + assert sorted(click_disk_data, key=lambda x: x["name"]) == sorted( + expected_disks_data, key=lambda x: x["name"] + ) expected_policies_data = [ { @@ -230,131 +243,211 @@ def test_system_tables(start_cluster): }, ] - clickhouse_policies_data = \ - json.loads(node1.query("SELECT * FROM system.storage_policies WHERE policy_name != 'default' FORMAT JSON"))[ - "data"] + clickhouse_policies_data = json.loads( + node1.query( + "SELECT * FROM system.storage_policies WHERE policy_name != 'default' FORMAT JSON" + ) + )["data"] def key(x): return (x["policy_name"], x["volume_name"], x["volume_priority"]) - assert sorted(clickhouse_policies_data, key=key) == sorted(expected_policies_data, key=key) + assert sorted(clickhouse_policies_data, key=key) == sorted( + expected_policies_data, key=key + ) def test_query_parser(start_cluster): try: with pytest.raises(QueryRuntimeException): - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS table_with_absent_policy ( d UInt64 ) ENGINE = MergeTree() ORDER BY d SETTINGS storage_policy='very_exciting_policy' - """) + """ + ) with pytest.raises(QueryRuntimeException): - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS table_with_absent_policy ( d UInt64 ) ENGINE = MergeTree() ORDER BY d SETTINGS storage_policy='jbod1' - """) + """ + ) - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS table_with_normal_policy ( d UInt64 ) ENGINE = MergeTree() ORDER BY d SETTINGS storage_policy='default' - """) + """ + ) node1.query("INSERT INTO table_with_normal_policy VALUES (5)") with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE table_with_normal_policy MOVE PARTITION tuple() TO VOLUME 'some_volume'") - - with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE table_with_normal_policy MOVE PARTITION tuple() TO DISK 'some_volume'") - - with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE table_with_normal_policy MOVE PART 'xxxxx' TO DISK 'jbod1'") - - with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE table_with_normal_policy MOVE PARTITION 'yyyy' TO DISK 'jbod1'") + node1.query( + "ALTER TABLE table_with_normal_policy MOVE PARTITION tuple() TO VOLUME 'some_volume'" + ) with pytest.raises(QueryRuntimeException): node1.query( - "ALTER TABLE table_with_normal_policy MODIFY SETTING storage_policy='moving_jbod_with_external'") + "ALTER TABLE table_with_normal_policy MOVE PARTITION tuple() TO DISK 'some_volume'" + ) + + with pytest.raises(QueryRuntimeException): + node1.query( + "ALTER TABLE table_with_normal_policy MOVE PART 'xxxxx' TO DISK 'jbod1'" + ) + + with pytest.raises(QueryRuntimeException): + node1.query( + "ALTER TABLE table_with_normal_policy MOVE PARTITION 'yyyy' TO DISK 'jbod1'" + ) + + with pytest.raises(QueryRuntimeException): + node1.query( + "ALTER TABLE table_with_normal_policy MODIFY SETTING storage_policy='moving_jbod_with_external'" + ) finally: node1.query("DROP TABLE IF EXISTS table_with_normal_policy SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("test_alter_policy", "MergeTree()", id="mt"), - pytest.param("replicated_test_alter_policy", "ReplicatedMergeTree('/clickhouse/test_alter_policy', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("test_alter_policy", "MergeTree()", id="mt"), + pytest.param( + "replicated_test_alter_policy", + "ReplicatedMergeTree('/clickhouse/test_alter_policy', '1')", + id="replicated", + ), + ], +) def test_alter_policy(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( - name=name)) == "small_jbod_with_external\n" + assert ( + node1.query( + """SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name + ) + ) + == "small_jbod_with_external\n" + ) with pytest.raises(QueryRuntimeException): node1.query( """ALTER TABLE {name} MODIFY SETTING storage_policy='one_more_small_jbod_with_external'""".format( - name=name)) + name=name + ) + ) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( - name=name)) == "small_jbod_with_external\n" + assert ( + node1.query( + """SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name + ) + ) + == "small_jbod_with_external\n" + ) - node1.query_with_retry("""ALTER TABLE {name} MODIFY SETTING storage_policy='jbods_with_external'""".format(name=name)) + node1.query_with_retry( + """ALTER TABLE {name} MODIFY SETTING storage_policy='jbods_with_external'""".format( + name=name + ) + ) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( - name=name)) == "jbods_with_external\n" + assert ( + node1.query( + """SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name + ) + ) + == "jbods_with_external\n" + ) with pytest.raises(QueryRuntimeException): node1.query( - """ALTER TABLE {name} MODIFY SETTING storage_policy='small_jbod_with_external'""".format(name=name)) + """ALTER TABLE {name} MODIFY SETTING storage_policy='small_jbod_with_external'""".format( + name=name + ) + ) - assert node1.query("""SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( - name=name)) == "jbods_with_external\n" + assert ( + node1.query( + """SELECT storage_policy FROM system.tables WHERE name = '{name}'""".format( + name=name + ) + ) + == "jbods_with_external\n" + ) finally: node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) def get_used_disks_for_table(node, table_name): - return tuple(node.query( - "select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format( - table_name)).strip().split('\n')) + return tuple( + node.query( + "select disk_name from system.parts where table == '{}' and active=1 order by modification_time".format( + table_name + ) + ) + .strip() + .split("\n") + ) def get_used_parts_for_table(node, table_name): - return node.query("SELECT name FROM system.parts WHERE table = '{}' AND active = 1 ORDER BY modification_time".format(table_name)).splitlines() + return node.query( + "SELECT name FROM system.parts WHERE table = '{}' AND active = 1 ORDER BY modification_time".format( + table_name + ) + ).splitlines() + def test_no_warning_about_zero_max_data_part_size(start_cluster): def get_log(node): - return node.exec_in_container(["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"]) + return node.exec_in_container( + ["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"] + ) for node in (node1, node2): - node.query(""" + node.query( + """ CREATE TABLE IF NOT EXISTS default.test_warning_table ( s String ) ENGINE = MergeTree ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' - """) + """ + ) node.query("DROP TABLE IF EXISTS default.test_warning_table SYNC") log = get_log(node) assert not re.search("Warning.*Volume.*special_warning_zero_volume", log) @@ -363,38 +456,55 @@ def test_no_warning_about_zero_max_data_part_size(start_cluster): assert not re.search("Warning.*Volume.*special_warning_big_volume", log) -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_on_jbod", "MergeTree()", id="mt"), - pytest.param("replicated_mt_on_jbod", "ReplicatedMergeTree('/clickhouse/replicated_mt_on_jbod', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("mt_on_jbod", "MergeTree()", id="mt"), + pytest.param( + "replicated_mt_on_jbod", + "ReplicatedMergeTree('/clickhouse/replicated_mt_on_jbod', '1')", + id="replicated", + ), + ], +) def test_round_robin(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) # first should go to the jbod1 - node1.query_with_retry("insert into {} select * from numbers(10000)".format(name)) + node1.query_with_retry( + "insert into {} select * from numbers(10000)".format(name) + ) used_disk = get_used_disks_for_table(node1, name) - assert len(used_disk) == 1, 'More than one disk used for single insert' + assert len(used_disk) == 1, "More than one disk used for single insert" # sleep is required because we order disks by their modification time, and if insert will be fast # modification time of two disks will be equal, then sort will not provide deterministic results time.sleep(5) - node1.query_with_retry("insert into {} select * from numbers(10000, 10000)".format(name)) + node1.query_with_retry( + "insert into {} select * from numbers(10000, 10000)".format(name) + ) used_disks = get_used_disks_for_table(node1, name) - assert len(used_disks) == 2, 'Two disks should be used for two parts' + assert len(used_disks) == 2, "Two disks should be used for two parts" assert used_disks[0] != used_disks[1], "Should write to different disks" time.sleep(5) - node1.query_with_retry("insert into {} select * from numbers(20000, 10000)".format(name)) + node1.query_with_retry( + "insert into {} select * from numbers(20000, 10000)".format(name) + ) used_disks = get_used_disks_for_table(node1, name) # jbod1 -> jbod2 -> jbod1 -> jbod2 ... etc @@ -405,46 +515,79 @@ def test_round_robin(start_cluster, name, engine): node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_with_huge_part", "MergeTree()", id="mt"), - pytest.param("replicated_mt_with_huge_part", "ReplicatedMergeTree('/clickhouse/replicated_mt_with_huge_part', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("mt_with_huge_part", "MergeTree()", id="mt"), + pytest.param( + "replicated_mt_with_huge_part", + "ReplicatedMergeTree('/clickhouse/replicated_mt_with_huge_part', '1')", + id="replicated", + ), + ], +) def test_max_data_part_size(start_cluster, name, engine): try: - assert int(*node1.query("""SELECT max_data_part_size FROM system.storage_policies WHERE policy_name = 'jbods_with_external' AND volume_name = 'main'""").splitlines()) == 10*1024*1024 + assert ( + int( + *node1.query( + """SELECT max_data_part_size FROM system.storage_policies WHERE policy_name = 'jbods_with_external' AND volume_name = 'main'""" + ).splitlines() + ) + == 10 * 1024 * 1024 + ) - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( s1 String ) ENGINE = {engine} ORDER BY tuple() SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) data = [] # 10MB in total for i in range(10): data.append(get_random_string(1024 * 1024)) # 1MB row - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert len(used_disks) == 1 - assert used_disks[0] == 'external' + assert used_disks[0] == "external" finally: node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_with_overflow", "MergeTree()", id="mt"), - pytest.param("replicated_mt_with_overflow", "ReplicatedMergeTree('/clickhouse/replicated_mt_with_overflow', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("mt_with_overflow", "MergeTree()", id="mt"), + pytest.param( + "replicated_mt_with_overflow", + "ReplicatedMergeTree('/clickhouse/replicated_mt_with_overflow', '1')", + id="replicated", + ), + ], +) def test_jbod_overflow(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( s1 String ) ENGINE = {engine} ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query(f"SYSTEM STOP MERGES {name}") @@ -453,21 +596,29 @@ def test_jbod_overflow(start_cluster, name, engine): data = [] # 5MB in total for i in range(5): data.append(get_random_string(1024 * 1024)) # 1MB row - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) - assert used_disks == tuple('jbod1' for _ in used_disks) + assert used_disks == tuple("jbod1" for _ in used_disks) # should go to the external disk (jbod is overflown) data = [] # 10MB in total for i in range(10): data.append(get_random_string(1024 * 1024)) # 1MB row - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) - assert used_disks[-1] == 'external' + assert used_disks[-1] == "external" node1.query(f"SYSTEM START MERGES {name}") time.sleep(1) @@ -475,29 +626,46 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query_with_retry("OPTIMIZE TABLE {} FINAL".format(name)) time.sleep(2) - disks_for_merges = tuple(node1.query( - "SELECT disk_name FROM system.parts WHERE table == '{}' AND level >= 1 and active = 1 ORDER BY modification_time".format( - name)).strip().split('\n')) + disks_for_merges = tuple( + node1.query( + "SELECT disk_name FROM system.parts WHERE table == '{}' AND level >= 1 and active = 1 ORDER BY modification_time".format( + name + ) + ) + .strip() + .split("\n") + ) - assert disks_for_merges == tuple('external' for _ in disks_for_merges) + assert disks_for_merges == tuple("external" for _ in disks_for_merges) finally: node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("moving_mt", "MergeTree()", id="mt"), - pytest.param("moving_replicated_mt", "ReplicatedMergeTree('/clickhouse/moving_replicated_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("moving_mt", "MergeTree()", id="mt"), + pytest.param( + "moving_replicated_mt", + "ReplicatedMergeTree('/clickhouse/moving_replicated_mt', '1')", + id="replicated", + ), + ], +) def test_background_move(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( s1 String ) ENGINE = {engine} ORDER BY tuple() SETTINGS storage_policy='moving_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query(f"SYSTEM STOP MERGES {name}") @@ -506,25 +674,31 @@ def test_background_move(start_cluster, name, engine): for i in range(5): data.append(get_random_string(1024 * 1024)) # 1MB row # small jbod size is 40MB, so lets insert 5MB batch 5 times - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) retry = 20 i = 0 - while not sum(1 for x in used_disks if x == 'jbod1') <= 2 and i < retry: + while not sum(1 for x in used_disks if x == "jbod1") <= 2 and i < retry: time.sleep(0.5) used_disks = get_used_disks_for_table(node1, name) i += 1 - assert sum(1 for x in used_disks if x == 'jbod1') <= 2 + assert sum(1 for x in used_disks if x == "jbod1") <= 2 # first (oldest) part was moved to external - assert used_disks[0] == 'external' + assert used_disks[0] == "external" path = node1.query( "SELECT path_on_disk FROM system.part_log WHERE table = '{}' AND event_type='MovePart' ORDER BY event_time LIMIT 1".format( - name)) + name + ) + ) # first (oldest) part was moved to external assert path.startswith("/external") @@ -535,19 +709,30 @@ def test_background_move(start_cluster, name, engine): node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("stopped_moving_mt", "MergeTree()", id="mt"), - pytest.param("stopped_moving_replicated_mt", "ReplicatedMergeTree('/clickhouse/stopped_moving_replicated_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("stopped_moving_mt", "MergeTree()", id="mt"), + pytest.param( + "stopped_moving_replicated_mt", + "ReplicatedMergeTree('/clickhouse/stopped_moving_replicated_mt', '1')", + id="replicated", + ), + ], +) def test_start_stop_moves(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( s1 String ) ENGINE = {engine} ORDER BY tuple() SETTINGS storage_policy='moving_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query_with_retry("INSERT INTO {} VALUES ('HELLO')".format(name)) node1.query_with_retry("INSERT INTO {} VALUES ('WORLD')".format(name)) @@ -557,23 +742,37 @@ def test_start_stop_moves(start_cluster, name, engine): first_part = node1.query( "SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format( - name)).strip() + name + ) + ).strip() node1.query("SYSTEM STOP MOVES") with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, first_part)) + node1.query( + "ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format( + name, first_part + ) + ) used_disks = get_used_disks_for_table(node1, name) - assert all(d == "jbod1" for d in used_disks), "Blocked moves doesn't actually move something" + assert all( + d == "jbod1" for d in used_disks + ), "Blocked moves doesn't actually move something" node1.query("SYSTEM START MOVES") - node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, first_part)) + node1.query( + "ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format( + name, first_part + ) + ) disk = node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, - first_part)).strip() + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format( + name, first_part + ) + ).strip() assert disk == "external" @@ -587,36 +786,40 @@ def test_start_stop_moves(start_cluster, name, engine): for i in range(5): data.append(get_random_string(1024 * 1024)) # 1MB row # jbod size is 40MB, so lets insert 5MB batch 7 times - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) retry = 5 i = 0 - while not sum(1 for x in used_disks if x == 'jbod1') <= 2 and i < retry: + while not sum(1 for x in used_disks if x == "jbod1") <= 2 and i < retry: time.sleep(0.1) used_disks = get_used_disks_for_table(node1, name) i += 1 # first (oldest) part doesn't move anywhere - assert used_disks[0] == 'jbod1' + assert used_disks[0] == "jbod1" node1.query("SYSTEM START MOVES {}".format(name)) # wait sometime until background backoff finishes retry = 30 i = 0 - while not sum(1 for x in used_disks if x == 'jbod1') <= 2 and i < retry: + while not sum(1 for x in used_disks if x == "jbod1") <= 2 and i < retry: time.sleep(1) used_disks = get_used_disks_for_table(node1, name) i += 1 node1.query("SYSTEM START MERGES {}".format(name)) - assert sum(1 for x in used_disks if x == 'jbod1') <= 2 + assert sum(1 for x in used_disks if x == "jbod1") <= 2 # first (oldest) part moved to external - assert used_disks[0] == 'external' + assert used_disks[0] == "external" finally: node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") @@ -626,7 +829,9 @@ def get_path_for_part_from_part_log(node, table, part_name): node.query("SYSTEM FLUSH LOGS") path = node.query( "SELECT path_on_disk FROM system.part_log WHERE table = '{}' and part_name = '{}' ORDER BY event_time DESC LIMIT 1".format( - table, part_name)) + table, part_name + ) + ) return path.strip() @@ -634,18 +839,24 @@ def get_paths_for_partition_from_part_log(node, table, partition_id): node.query("SYSTEM FLUSH LOGS") paths = node.query( "SELECT path_on_disk FROM system.part_log WHERE table = '{}' and partition_id = '{}' ORDER BY event_time DESC".format( - table, partition_id)) - return paths.strip().split('\n') + table, partition_id + ) + ) + return paths.strip().split("\n") -@pytest.mark.parametrize("name,engine", [ - pytest.param("altering_mt", "MergeTree()", id="mt"), - # ("altering_replicated_mt","ReplicatedMergeTree('/clickhouse/altering_replicated_mt', '1')",), - # SYSTEM STOP MERGES doesn't disable merges assignments -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("altering_mt", "MergeTree()", id="mt"), + # ("altering_replicated_mt","ReplicatedMergeTree('/clickhouse/altering_replicated_mt', '1')",), + # SYSTEM STOP MERGES doesn't disable merges assignments + ], +) def test_alter_move(start_cluster, name, engine): try: - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( EventDate Date, number UInt64 @@ -653,7 +864,10 @@ def test_alter_move(start_cluster, name, engine): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query("SYSTEM STOP MERGES {}".format(name)) # to avoid conflicts @@ -662,47 +876,83 @@ def test_alter_move(start_cluster, name, engine): node1.query("INSERT INTO {} VALUES(toDate('2019-04-10'), 42)".format(name)) node1.query("INSERT INTO {} VALUES(toDate('2019-04-11'), 43)".format(name)) used_disks = get_used_disks_for_table(node1, name) - assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods" + assert all( + d.startswith("jbod") for d in used_disks + ), "All writes should go to jbods" first_part = node1.query( "SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format( - name)).strip() + name + ) + ).strip() time.sleep(1) - node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, first_part)) + node1.query( + "ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format( + name, first_part + ) + ) disk = node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, - first_part)).strip() - assert disk == 'external' - assert get_path_for_part_from_part_log(node1, name, first_part).startswith("/external") + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format( + name, first_part + ) + ).strip() + assert disk == "external" + assert get_path_for_part_from_part_log(node1, name, first_part).startswith( + "/external" + ) time.sleep(1) - node1.query("ALTER TABLE {} MOVE PART '{}' TO DISK 'jbod1'".format(name, first_part)) + node1.query( + "ALTER TABLE {} MOVE PART '{}' TO DISK 'jbod1'".format(name, first_part) + ) disk = node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, - first_part)).strip() - assert disk == 'jbod1' - assert get_path_for_part_from_part_log(node1, name, first_part).startswith("/jbod1") + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format( + name, first_part + ) + ).strip() + assert disk == "jbod1" + assert get_path_for_part_from_part_log(node1, name, first_part).startswith( + "/jbod1" + ) time.sleep(1) - node1.query("ALTER TABLE {} MOVE PARTITION 201904 TO VOLUME 'external'".format(name)) - disks = node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format( - name)).strip().split('\n') + node1.query( + "ALTER TABLE {} MOVE PARTITION 201904 TO VOLUME 'external'".format(name) + ) + disks = ( + node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format( + name + ) + ) + .strip() + .split("\n") + ) assert len(disks) == 2 assert all(d == "external" for d in disks) assert all( - path.startswith("/external") for path in get_paths_for_partition_from_part_log(node1, name, '201904')[:2]) + path.startswith("/external") + for path in get_paths_for_partition_from_part_log(node1, name, "201904")[:2] + ) time.sleep(1) node1.query("ALTER TABLE {} MOVE PARTITION 201904 TO DISK 'jbod2'".format(name)) - disks = node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format( - name)).strip().split('\n') + disks = ( + node1.query( + "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201904' and active = 1".format( + name + ) + ) + .strip() + .split("\n") + ) assert len(disks) == 2 assert all(d == "jbod2" for d in disks) assert all( - path.startswith("/jbod2") for path in get_paths_for_partition_from_part_log(node1, name, '201904')[:2]) + path.startswith("/jbod2") + for path in get_paths_for_partition_from_part_log(node1, name, "201904")[:2] + ) assert node1.query("SELECT COUNT() FROM {}".format(name)) == "4\n" @@ -710,15 +960,13 @@ def test_alter_move(start_cluster, name, engine): node1.query(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("volume_or_disk", [ - "DISK", - "VOLUME" -]) +@pytest.mark.parametrize("volume_or_disk", ["DISK", "VOLUME"]) def test_alter_move_half_of_partition(start_cluster, volume_or_disk): name = "alter_move_half_of_partition" engine = "MergeTree()" try: - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( EventDate Date, number UInt64 @@ -726,31 +974,49 @@ def test_alter_move_half_of_partition(start_cluster, volume_or_disk): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query("SYSTEM STOP MERGES {}".format(name)) node1.query("INSERT INTO {} VALUES(toDate('2019-03-15'), 65)".format(name)) node1.query("INSERT INTO {} VALUES(toDate('2019-03-16'), 42)".format(name)) used_disks = get_used_disks_for_table(node1, name) - assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods" + assert all( + d.startswith("jbod") for d in used_disks + ), "All writes should go to jbods" time.sleep(1) - parts = node1.query("SELECT name FROM system.parts WHERE table = '{}' and active = 1".format(name)).splitlines() + parts = node1.query( + "SELECT name FROM system.parts WHERE table = '{}' and active = 1".format( + name + ) + ).splitlines() assert len(parts) == 2 - node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, parts[0])) + node1.query( + "ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, parts[0]) + ) disks = node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, parts[ - 0])).splitlines() + "SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format( + name, parts[0] + ) + ).splitlines() assert disks == ["external"] time.sleep(1) - node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, - volume_or_disk=volume_or_disk)) + node1.query( + "ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format( + name, volume_or_disk=volume_or_disk + ) + ) disks = node1.query( "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format( - name)).splitlines() + name + ) + ).splitlines() assert disks == ["external"] * 2 assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n" @@ -759,15 +1025,13 @@ def test_alter_move_half_of_partition(start_cluster, volume_or_disk): node1.query(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("volume_or_disk", [ - "DISK", - "VOLUME" -]) +@pytest.mark.parametrize("volume_or_disk", ["DISK", "VOLUME"]) def test_alter_double_move_partition(start_cluster, volume_or_disk): name = "alter_double_move_partition" engine = "MergeTree()" try: - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( EventDate Date, number UInt64 @@ -775,29 +1039,42 @@ def test_alter_double_move_partition(start_cluster, volume_or_disk): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query("SYSTEM STOP MERGES {}".format(name)) node1.query("INSERT INTO {} VALUES(toDate('2019-03-15'), 65)".format(name)) node1.query("INSERT INTO {} VALUES(toDate('2019-03-16'), 42)".format(name)) used_disks = get_used_disks_for_table(node1, name) - assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods" + assert all( + d.startswith("jbod") for d in used_disks + ), "All writes should go to jbods" time.sleep(1) - node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, - volume_or_disk=volume_or_disk)) + node1.query( + "ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format( + name, volume_or_disk=volume_or_disk + ) + ) disks = node1.query( "SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format( - name)).splitlines() + name + ) + ).splitlines() assert disks == ["external"] * 2 assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n" time.sleep(1) with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, - volume_or_disk=volume_or_disk)) + node1.query( + "ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format( + name, volume_or_disk=volume_or_disk + ) + ) finally: node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @@ -808,8 +1085,15 @@ def produce_alter_move(node, name): if move_type == "PART": for _ in range(10): try: - parts = node1.query( - "SELECT name from system.parts where table = '{}' and active = 1".format(name)).strip().split('\n') + parts = ( + node1.query( + "SELECT name from system.parts where table = '{}' and active = 1".format( + name + ) + ) + .strip() + .split("\n") + ) break except QueryRuntimeException: pass @@ -826,20 +1110,30 @@ def produce_alter_move(node, name): else: move_volume = random.choice(["'main'", "'external'"]) try: - node1.query("ALTER TABLE {} MOVE {mt} {mp} TO {md} {mv}".format( - name, mt=move_type, mp=move_part, md=move_disk, mv=move_volume)) + node1.query( + "ALTER TABLE {} MOVE {mt} {mp} TO {md} {mv}".format( + name, mt=move_type, mp=move_part, md=move_disk, mv=move_volume + ) + ) except QueryRuntimeException as ex: pass -@pytest.mark.parametrize("name,engine", [ - pytest.param("concurrently_altering_mt", "MergeTree()", id="mt"), - pytest.param("concurrently_altering_replicated_mt", - "ReplicatedMergeTree('/clickhouse/concurrently_altering_replicated_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("concurrently_altering_mt", "MergeTree()", id="mt"), + pytest.param( + "concurrently_altering_replicated_mt", + "ReplicatedMergeTree('/clickhouse/concurrently_altering_replicated_mt', '1')", + id="replicated", + ), + ], +) def test_concurrent_alter_move(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( EventDate Date, number UInt64 @@ -847,7 +1141,10 @@ def test_concurrent_alter_move(start_cluster, name, engine): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) @@ -855,8 +1152,12 @@ def test_concurrent_alter_move(start_cluster, name, engine): for i in range(num): day = random.randint(11, 30) value = values.pop() - month = '0' + str(random.choice([3, 4])) - node1.query_with_retry("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) + month = "0" + str(random.choice([3, 4])) + node1.query_with_retry( + "INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format( + name, m=month, d=day, v=value + ) + ) def alter_move(num): for i in range(num): @@ -864,7 +1165,9 @@ def test_concurrent_alter_move(start_cluster, name, engine): def alter_update(num): for i in range(num): - node1.query("ALTER TABLE {} UPDATE number = number + 1 WHERE 1".format(name)) + node1.query( + "ALTER TABLE {} UPDATE number = number + 1 WHERE 1".format(name) + ) def optimize_table(num): for i in range(num): @@ -887,14 +1190,21 @@ def test_concurrent_alter_move(start_cluster, name, engine): node1.query(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("concurrently_dropping_mt", "MergeTree()", id="mt"), - pytest.param("concurrently_dropping_replicated_mt", - "ReplicatedMergeTree('/clickhouse/concurrently_dropping_replicated_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("concurrently_dropping_mt", "MergeTree()", id="mt"), + pytest.param( + "concurrently_dropping_replicated_mt", + "ReplicatedMergeTree('/clickhouse/concurrently_dropping_replicated_mt', '1')", + id="replicated", + ), + ], +) def test_concurrent_alter_move_and_drop(start_cluster, name, engine): try: - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( EventDate Date, number UInt64 @@ -902,7 +1212,10 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) @@ -910,8 +1223,12 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): for i in range(num): day = random.randint(11, 30) value = values.pop() - month = '0' + str(random.choice([3, 4])) - node1.query_with_retry("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) + month = "0" + str(random.choice([3, 4])) + node1.query_with_retry( + "INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format( + name, m=month, d=day, v=value + ) + ) def alter_move(num): for i in range(num): @@ -921,7 +1238,9 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): for i in range(num): partition = random.choice([201903, 201904]) drach = random.choice(["drop", "detach"]) - node1.query("ALTER TABLE {} {} PARTITION {}".format(name, drach, partition)) + node1.query( + "ALTER TABLE {} {} PARTITION {}".format(name, drach, partition) + ) insert(100) p = Pool(15) @@ -940,29 +1259,49 @@ def test_concurrent_alter_move_and_drop(start_cluster, name, engine): node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("detach_attach_mt", "MergeTree()", id="mt"), - pytest.param("replicated_detach_attach_mt", "ReplicatedMergeTree('/clickhouse/replicated_detach_attach_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("detach_attach_mt", "MergeTree()", id="mt"), + pytest.param( + "replicated_detach_attach_mt", + "ReplicatedMergeTree('/clickhouse/replicated_detach_attach_mt', '1')", + id="replicated", + ), + ], +) def test_detach_attach(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( s1 String ) ENGINE = {engine} ORDER BY tuple() SETTINGS storage_policy='moving_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) data = [] # 5MB in total for i in range(5): data.append(get_random_string(1024 * 1024)) # 1MB row - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) node1.query("ALTER TABLE {} DETACH PARTITION tuple()".format(name)) assert node1.query("SELECT count() FROM {}".format(name)).strip() == "0" - assert node1.query("SELECT disk FROM system.detached_parts WHERE table = '{}'".format(name)).strip() == "jbod1" + assert ( + node1.query( + "SELECT disk FROM system.detached_parts WHERE table = '{}'".format(name) + ).strip() + == "jbod1" + ) node1.query("ALTER TABLE {} ATTACH PARTITION tuple()".format(name)) assert node1.query("SELECT count() FROM {}".format(name)).strip() == "5" @@ -971,59 +1310,101 @@ def test_detach_attach(start_cluster, name, engine): node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("mutating_mt", "MergeTree()", id="mt"), - pytest.param("replicated_mutating_mt", "ReplicatedMergeTree('/clickhouse/replicated_mutating_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("mutating_mt", "MergeTree()", id="mt"), + pytest.param( + "replicated_mutating_mt", + "ReplicatedMergeTree('/clickhouse/replicated_mutating_mt', '1')", + id="replicated", + ), + ], +) def test_mutate_to_another_disk(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( s1 String ) ENGINE = {engine} ORDER BY tuple() SETTINGS storage_policy='moving_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) for i in range(5): data = [] # 5MB in total for i in range(5): data.append(get_random_string(1024 * 1024)) # 1MB row - node1.query_with_retry("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + node1.query_with_retry( + "INSERT INTO {} VALUES {}".format( + name, ",".join(["('" + x + "')" for x in data]) + ) + ) node1.query("ALTER TABLE {} UPDATE s1 = concat(s1, 'x') WHERE 1".format(name)) retry = 20 - while node1.query("SELECT * FROM system.mutations WHERE is_done = 0") != "" and retry > 0: + while ( + node1.query("SELECT * FROM system.mutations WHERE is_done = 0") != "" + and retry > 0 + ): retry -= 1 time.sleep(0.5) - if node1.query("SELECT latest_fail_reason FROM system.mutations WHERE table = '{}'".format(name)) == "": - assert node1.query("SELECT sum(endsWith(s1, 'x')) FROM {}".format(name)) == "25\n" + if ( + node1.query( + "SELECT latest_fail_reason FROM system.mutations WHERE table = '{}'".format( + name + ) + ) + == "" + ): + assert ( + node1.query("SELECT sum(endsWith(s1, 'x')) FROM {}".format(name)) + == "25\n" + ) else: # mutation failed, let's try on another disk print("Mutation failed") node1.query_with_retry("OPTIMIZE TABLE {} FINAL".format(name)) - node1.query("ALTER TABLE {} UPDATE s1 = concat(s1, 'x') WHERE 1".format(name)) + node1.query( + "ALTER TABLE {} UPDATE s1 = concat(s1, 'x') WHERE 1".format(name) + ) retry = 20 - while node1.query("SELECT * FROM system.mutations WHERE is_done = 0") != "" and retry > 0: + while ( + node1.query("SELECT * FROM system.mutations WHERE is_done = 0") != "" + and retry > 0 + ): retry -= 1 time.sleep(0.5) - assert node1.query("SELECT sum(endsWith(s1, 'x')) FROM {}".format(name)) == "25\n" - - + assert ( + node1.query("SELECT sum(endsWith(s1, 'x')) FROM {}".format(name)) + == "25\n" + ) finally: node1.query_with_retry(f"DROP TABLE IF EXISTS {name} SYNC") -@pytest.mark.parametrize("name,engine", [ - pytest.param("alter_modifying_mt", "MergeTree()", id="mt"), - pytest.param("replicated_alter_modifying_mt", "ReplicatedMergeTree('/clickhouse/replicated_alter_modifying_mt', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("alter_modifying_mt", "MergeTree()", id="mt"), + pytest.param( + "replicated_alter_modifying_mt", + "ReplicatedMergeTree('/clickhouse/replicated_alter_modifying_mt', '1')", + id="replicated", + ), + ], +) def test_concurrent_alter_modify(start_cluster, name, engine): try: - node1.query_with_retry(""" + node1.query_with_retry( + """ CREATE TABLE IF NOT EXISTS {name} ( EventDate Date, number UInt64 @@ -1031,7 +1412,10 @@ def test_concurrent_alter_modify(start_cluster, name, engine): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) @@ -1039,8 +1423,12 @@ def test_concurrent_alter_modify(start_cluster, name, engine): for i in range(num): day = random.randint(11, 30) value = values.pop() - month = '0' + str(random.choice([3, 4])) - node1.query_with_retry("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) + month = "0" + str(random.choice([3, 4])) + node1.query_with_retry( + "INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format( + name, m=month, d=day, v=value + ) + ) def alter_move(num): for i in range(num): @@ -1050,7 +1438,11 @@ def test_concurrent_alter_modify(start_cluster, name, engine): for i in range(num): column_type = random.choice(["UInt64", "String"]) try: - node1.query("ALTER TABLE {} MODIFY COLUMN number {}".format(name, column_type)) + node1.query( + "ALTER TABLE {} MODIFY COLUMN number {}".format( + name, column_type + ) + ) except: if "Replicated" not in engine: raise @@ -1078,13 +1470,17 @@ def test_concurrent_alter_modify(start_cluster, name, engine): def test_simple_replication_and_moves(start_cluster): try: for i, node in enumerate([node1, node2]): - node.query_with_retry(""" + node.query_with_retry( + """ CREATE TABLE IF NOT EXISTS replicated_table_for_moves ( s1 String ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_table_for_moves', '{}') ORDER BY tuple() SETTINGS storage_policy='moving_jbod_with_external', old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=2 - """.format(i + 1)) + """.format( + i + 1 + ) + ) def insert(num): for i in range(num): @@ -1092,8 +1488,11 @@ def test_simple_replication_and_moves(start_cluster): data = [] # 1MB in total for i in range(2): data.append(get_random_string(512 * 1024)) # 500KB value - node.query_with_retry("INSERT INTO replicated_table_for_moves VALUES {}".format( - ','.join(["('" + x + "')" for x in data]))) + node.query_with_retry( + "INSERT INTO replicated_table_for_moves VALUES {}".format( + ",".join(["('" + x + "')" for x in data]) + ) + ) def optimize(num): for i in range(num): @@ -1108,7 +1507,10 @@ def test_simple_replication_and_moves(start_cluster): for task in tasks: task.get(timeout=60) - node1.query_with_retry("SYSTEM SYNC REPLICA ON CLUSTER test_cluster replicated_table_for_moves", timeout=5) + node1.query_with_retry( + "SYSTEM SYNC REPLICA ON CLUSTER test_cluster replicated_table_for_moves", + timeout=5, + ) node1.query("SELECT COUNT() FROM replicated_table_for_moves") == "40\n" node2.query("SELECT COUNT() FROM replicated_table_for_moves") == "40\n" @@ -1122,9 +1524,15 @@ def test_simple_replication_and_moves(start_cluster): node2.query("SYSTEM STOP MERGES") node1.query_with_retry( - "INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + "INSERT INTO replicated_table_for_moves VALUES {}".format( + ",".join(["('" + x + "')" for x in data]) + ) + ) node2.query_with_retry( - "INSERT INTO replicated_table_for_moves VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + "INSERT INTO replicated_table_for_moves VALUES {}".format( + ",".join(["('" + x + "')" for x in data]) + ) + ) time.sleep(3) # nothing was moved @@ -1143,24 +1551,33 @@ def test_simple_replication_and_moves(start_cluster): def test_download_appropriate_disk(start_cluster): try: for i, node in enumerate([node1, node2]): - node.query_with_retry(""" + node.query_with_retry( + """ CREATE TABLE IF NOT EXISTS replicated_table_for_download ( s1 String ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_table_for_download', '{}') ORDER BY tuple() SETTINGS storage_policy='moving_jbod_with_external', old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=2 - """.format(i + 1)) + """.format( + i + 1 + ) + ) data = [] for i in range(50): data.append(get_random_string(1024 * 1024)) # 1MB value node1.query_with_retry( - "INSERT INTO replicated_table_for_download VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + "INSERT INTO replicated_table_for_download VALUES {}".format( + ",".join(["('" + x + "')" for x in data]) + ) + ) for _ in range(10): try: print("Syncing replica") - node2.query_with_retry("SYSTEM SYNC REPLICA replicated_table_for_download") + node2.query_with_retry( + "SYSTEM SYNC REPLICA replicated_table_for_download" + ) break except: time.sleep(0.5) @@ -1171,24 +1588,32 @@ def test_download_appropriate_disk(start_cluster): finally: for node in [node1, node2]: - node.query_with_retry("DROP TABLE IF EXISTS replicated_table_for_download SYNC") + node.query_with_retry( + "DROP TABLE IF EXISTS replicated_table_for_download SYNC" + ) def test_rename(start_cluster): try: - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS default.renaming_table ( s String ) ENGINE = MergeTree ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' - """) + """ + ) for _ in range(5): data = [] for i in range(10): data.append(get_random_string(1024 * 1024)) # 1MB value - node1.query("INSERT INTO renaming_table VALUES {}".format(','.join(["('" + x + "')" for x in data]))) + node1.query( + "INSERT INTO renaming_table VALUES {}".format( + ",".join(["('" + x + "')" for x in data]) + ) + ) disks = get_used_disks_for_table(node1, "renaming_table") assert len(disks) > 1 @@ -1215,7 +1640,8 @@ def test_rename(start_cluster): def test_freeze(start_cluster): try: - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS default.freezing_table ( d Date, s String @@ -1223,7 +1649,8 @@ def test_freeze(start_cluster): ORDER BY tuple() PARTITION BY toYYYYMM(d) SETTINGS storage_policy='small_jbod_with_external' - """) + """ + ) for _ in range(5): data = [] @@ -1231,8 +1658,11 @@ def test_freeze(start_cluster): for i in range(10): data.append(get_random_string(1024 * 1024)) # 1MB value dates.append("toDate('2019-03-05')") - node1.query("INSERT INTO freezing_table VALUES {}".format( - ','.join(["(" + d + ", '" + s + "')" for d, s in zip(dates, data)]))) + node1.query( + "INSERT INTO freezing_table VALUES {}".format( + ",".join(["(" + d + ", '" + s + "')" for d, s in zip(dates, data)]) + ) + ) disks = get_used_disks_for_table(node1, "freezing_table") assert len(disks) > 1 @@ -1240,8 +1670,12 @@ def test_freeze(start_cluster): node1.query("ALTER TABLE freezing_table FREEZE PARTITION 201903") # check shadow files (backups) exists - node1.exec_in_container(["bash", "-c", "find /jbod1/shadow -name '*.mrk2' | grep '.*'"]) - node1.exec_in_container(["bash", "-c", "find /external/shadow -name '*.mrk2' | grep '.*'"]) + node1.exec_in_container( + ["bash", "-c", "find /jbod1/shadow -name '*.mrk2' | grep '.*'"] + ) + node1.exec_in_container( + ["bash", "-c", "find /external/shadow -name '*.mrk2' | grep '.*'"] + ) finally: node1.query("DROP TABLE IF EXISTS default.freezing_table SYNC") @@ -1252,19 +1686,27 @@ def test_kill_while_insert(start_cluster): try: name = "test_kill_while_insert" - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( s String ) ENGINE = MergeTree ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name)) + """.format( + name=name + ) + ) data = [] dates = [] for i in range(10): data.append(get_random_string(1024 * 1024)) # 1MB value - node1.query("INSERT INTO {name} VALUES {}".format(','.join(["('" + s + "')" for s in data]), name=name)) + node1.query( + "INSERT INTO {name} VALUES {}".format( + ",".join(["('" + s + "')" for s in data]), name=name + ) + ) disks = get_used_disks_for_table(node1, name) assert set(disks) == {"jbod1"} @@ -1276,12 +1718,19 @@ def test_kill_while_insert(start_cluster): """(っಠ‿ಠ)っ""" start_time = time.time() - long_select = threading.Thread(target=ignore_exceptions, args=(node1.query, "SELECT sleep(3) FROM {name}".format(name=name))) + long_select = threading.Thread( + target=ignore_exceptions, + args=(node1.query, "SELECT sleep(3) FROM {name}".format(name=name)), + ) long_select.start() time.sleep(0.5) - node1.query("ALTER TABLE {name} MOVE PARTITION tuple() TO DISK 'external'".format(name=name)) + node1.query( + "ALTER TABLE {name} MOVE PARTITION tuple() TO DISK 'external'".format( + name=name + ) + ) assert time.time() - start_time < 2 node1.restart_clickhouse(kill=True) @@ -1290,7 +1739,9 @@ def test_kill_while_insert(start_cluster): except: """""" - assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["10"] + assert node1.query( + "SELECT count() FROM {name}".format(name=name) + ).splitlines() == ["10"] finally: try: @@ -1303,13 +1754,17 @@ def test_move_while_merge(start_cluster): try: name = "test_move_while_merge" - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( n Int64 ) ENGINE = MergeTree ORDER BY sleep(2) SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name)) + """.format( + name=name + ) + ) node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) node1.query("INSERT INTO {name} VALUES (2)".format(name=name)) @@ -1326,7 +1781,11 @@ def test_move_while_merge(start_cluster): time.sleep(0.5) with pytest.raises(QueryRuntimeException): - node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) + node1.query( + "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format( + name=name, part=parts[0] + ) + ) exiting = False no_exception = {} @@ -1335,8 +1794,11 @@ def test_move_while_merge(start_cluster): while not exiting: try: node1.query( - "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) - no_exception['missing'] = 'exception' + "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format( + name=name, part=parts[0] + ) + ) + no_exception["missing"] = "exception" break except QueryRuntimeException: """""" @@ -1352,7 +1814,9 @@ def test_move_while_merge(start_cluster): alter_thread.join() assert len(no_exception) == 0 - assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["2"] + assert node1.query( + "SELECT count() FROM {name}".format(name=name) + ).splitlines() == ["2"] finally: node1.query(f"DROP TABLE IF EXISTS {name} SYNC") @@ -1362,47 +1826,85 @@ def test_move_across_policies_does_not_work(start_cluster): try: name = "test_move_across_policies_does_not_work" - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name} ( n Int64 ) ENGINE = MergeTree ORDER BY tuple() SETTINGS storage_policy='jbods_with_external' - """.format(name=name)) + """.format( + name=name + ) + ) - node1.query(""" + node1.query( + """ CREATE TABLE IF NOT EXISTS {name}2 ( n Int64 ) ENGINE = MergeTree ORDER BY tuple() SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name)) + """.format( + name=name + ) + ) node1.query("""INSERT INTO {name} VALUES (1)""".format(name=name)) try: - node1.query("""ALTER TABLE {name} MOVE PARTITION tuple() TO DISK 'jbod2'""".format(name=name)) + node1.query( + """ALTER TABLE {name} MOVE PARTITION tuple() TO DISK 'jbod2'""".format( + name=name + ) + ) except QueryRuntimeException: """All parts of partition 'all' are already on disk 'jbod2'.""" - with pytest.raises(QueryRuntimeException, match='.*because disk does not belong to storage policy.*'): - node1.query("""ALTER TABLE {name}2 ATTACH PARTITION tuple() FROM {name}""".format(name=name)) + with pytest.raises( + QueryRuntimeException, + match=".*because disk does not belong to storage policy.*", + ): + node1.query( + """ALTER TABLE {name}2 ATTACH PARTITION tuple() FROM {name}""".format( + name=name + ) + ) - with pytest.raises(QueryRuntimeException, match='.*because disk does not belong to storage policy.*'): - node1.query("""ALTER TABLE {name}2 REPLACE PARTITION tuple() FROM {name}""".format(name=name)) + with pytest.raises( + QueryRuntimeException, + match=".*because disk does not belong to storage policy.*", + ): + node1.query( + """ALTER TABLE {name}2 REPLACE PARTITION tuple() FROM {name}""".format( + name=name + ) + ) - with pytest.raises(QueryRuntimeException, match='.*should have the same storage policy of source table.*'): - node1.query("""ALTER TABLE {name} MOVE PARTITION tuple() TO TABLE {name}2""".format(name=name)) + with pytest.raises( + QueryRuntimeException, + match=".*should have the same storage policy of source table.*", + ): + node1.query( + """ALTER TABLE {name} MOVE PARTITION tuple() TO TABLE {name}2""".format( + name=name + ) + ) - assert node1.query("""SELECT * FROM {name}""".format(name=name)).splitlines() == ["1"] + assert node1.query( + """SELECT * FROM {name}""".format(name=name) + ).splitlines() == ["1"] finally: node1.query(f"DROP TABLE IF EXISTS {name} SYNC") node1.query(f"DROP TABLE IF EXISTS {name}2 SYNC") -def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, parts_after_cmds): +def _insert_merge_execute( + node, name, policy, parts, cmds, parts_before_cmds, parts_after_cmds +): try: - node.query(""" + node.query( + """ CREATE TABLE IF NOT EXISTS {name} ( n Int64 ) ENGINE = MergeTree @@ -1410,7 +1912,10 @@ def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, pa PARTITION BY tuple() TTL now()-1 TO VOLUME 'external' SETTINGS storage_policy='{policy}' - """.format(name=name, policy=policy)) + """.format( + name=name, policy=policy + ) + ) for i in range(parts): node.query("""INSERT INTO {name} VALUES ({n})""".format(name=name, n=i)) @@ -1437,29 +1942,45 @@ def _insert_merge_execute(node, name, policy, parts, cmds, parts_before_cmds, pa def _check_merges_are_working(node, storage_policy, volume, shall_work): try: - name = "_check_merges_are_working_{storage_policy}_{volume}".format(storage_policy=storage_policy, volume=volume) + name = "_check_merges_are_working_{storage_policy}_{volume}".format( + storage_policy=storage_policy, volume=volume + ) - node.query(""" + node.query( + """ CREATE TABLE IF NOT EXISTS {name} ( n Int64 ) ENGINE = MergeTree ORDER BY tuple() PARTITION BY tuple() SETTINGS storage_policy='{storage_policy}' - """.format(name=name, storage_policy=storage_policy)) + """.format( + name=name, storage_policy=storage_policy + ) + ) created_parts = 24 for i in range(created_parts): node.query("""INSERT INTO {name} VALUES ({n})""".format(name=name, n=i)) try: - node.query("""ALTER TABLE {name} MOVE PARTITION tuple() TO VOLUME '{volume}' """.format(name=name, volume=volume)) + node.query( + """ALTER TABLE {name} MOVE PARTITION tuple() TO VOLUME '{volume}' """.format( + name=name, volume=volume + ) + ) except: """Ignore 'nothing to move'.""" - expected_disks = set(node.query(""" + expected_disks = set( + node.query( + """ SELECT disks FROM system.storage_policies ARRAY JOIN disks WHERE volume_name = '{volume_name}' - """.format(volume_name=volume)).splitlines()) + """.format( + volume_name=volume + ) + ).splitlines() + ) disks = get_used_disks_for_table(node, name) assert set(disks) <= expected_disks @@ -1474,11 +1995,22 @@ def _check_merges_are_working(node, storage_policy, volume, shall_work): def _get_prefer_not_to_merge_for_storage_policy(node, storage_policy): - return list(map(int, node.query("SELECT prefer_not_to_merge FROM system.storage_policies WHERE policy_name = '{}' ORDER BY volume_priority".format(storage_policy)).splitlines())) + return list( + map( + int, + node.query( + "SELECT prefer_not_to_merge FROM system.storage_policies WHERE policy_name = '{}' ORDER BY volume_priority".format( + storage_policy + ) + ).splitlines(), + ) + ) def test_simple_merge_tree_merges_are_disabled(start_cluster): - _check_merges_are_working(node1, "small_jbod_with_external_no_merges", "external", False) + _check_merges_are_working( + node1, "small_jbod_with_external_no_merges", "external", False + ) def test_no_merges_in_configuration_allow_from_query_without_reload(start_cluster): @@ -1489,9 +2021,15 @@ def test_no_merges_in_configuration_allow_from_query_without_reload(start_cluste assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 1] _check_merges_are_working(node1, policy, "external", False) - _insert_merge_execute(node1, name, policy, 2, [ - "SYSTEM START MERGES ON VOLUME {}.external".format(policy) - ], 2, 1) + _insert_merge_execute( + node1, + name, + policy, + 2, + ["SYSTEM START MERGES ON VOLUME {}.external".format(policy)], + 2, + 1, + ) assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 0] _check_merges_are_working(node1, policy, "external", True) @@ -1507,17 +2045,28 @@ def test_no_merges_in_configuration_allow_from_query_with_reload(start_cluster): assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 1] _check_merges_are_working(node1, policy, "external", False) - _insert_merge_execute(node1, name, policy, 2, [ + _insert_merge_execute( + node1, + name, + policy, + 2, + [ "SYSTEM START MERGES ON VOLUME {}.external".format(policy), - "SYSTEM RELOAD CONFIG" - ], 2, 1) + "SYSTEM RELOAD CONFIG", + ], + 2, + 1, + ) assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 0] _check_merges_are_working(node1, policy, "external", True) finally: node1.query("SYSTEM STOP MERGES ON VOLUME {}.external".format(policy)) -def test_no_merges_in_configuration_allow_from_query_with_reload_on_cluster(start_cluster): + +def test_no_merges_in_configuration_allow_from_query_with_reload_on_cluster( + start_cluster, +): try: name = "test_no_merges_in_configuration_allow_from_query_with_reload" policy = "small_jbod_with_external_no_merges" @@ -1525,15 +2074,29 @@ def test_no_merges_in_configuration_allow_from_query_with_reload_on_cluster(star assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 1] _check_merges_are_working(node1, policy, "external", False) - _insert_merge_execute(node1, name, policy, 2, [ - "SYSTEM START MERGES ON CLUSTER test_cluster ON VOLUME {}.external".format(policy), - "SYSTEM RELOAD CONFIG ON CLUSTER test_cluster" - ], 2, 1) + _insert_merge_execute( + node1, + name, + policy, + 2, + [ + "SYSTEM START MERGES ON CLUSTER test_cluster ON VOLUME {}.external".format( + policy + ), + "SYSTEM RELOAD CONFIG ON CLUSTER test_cluster", + ], + 2, + 1, + ) assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 0] _check_merges_are_working(node1, policy, "external", True) finally: - node1.query("SYSTEM STOP MERGES ON CLUSTER test_cluster ON VOLUME {}.external".format(policy)) + node1.query( + "SYSTEM STOP MERGES ON CLUSTER test_cluster ON VOLUME {}.external".format( + policy + ) + ) def test_yes_merges_in_configuration_disallow_from_query_without_reload(start_cluster): @@ -1544,10 +2107,18 @@ def test_yes_merges_in_configuration_disallow_from_query_without_reload(start_cl assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 0] _check_merges_are_working(node1, policy, "external", True) - _insert_merge_execute(node1, name, policy, 2, [ + _insert_merge_execute( + node1, + name, + policy, + 2, + [ "SYSTEM STOP MERGES ON VOLUME {}.external".format(policy), - "INSERT INTO {name} VALUES (2)".format(name=name) - ], 1, 2) + "INSERT INTO {name} VALUES (2)".format(name=name), + ], + 1, + 2, + ) assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 1] _check_merges_are_working(node1, policy, "external", False) @@ -1563,11 +2134,19 @@ def test_yes_merges_in_configuration_disallow_from_query_with_reload(start_clust assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 0] _check_merges_are_working(node1, policy, "external", True) - _insert_merge_execute(node1, name, policy, 2, [ + _insert_merge_execute( + node1, + name, + policy, + 2, + [ "SYSTEM STOP MERGES ON VOLUME {}.external".format(policy), "INSERT INTO {name} VALUES (2)".format(name=name), - "SYSTEM RELOAD CONFIG" - ], 1, 2) + "SYSTEM RELOAD CONFIG", + ], + 1, + 2, + ) assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 1] _check_merges_are_working(node1, policy, "external", False) diff --git a/tests/integration/test_mutations_hardlinks/test.py b/tests/integration/test_mutations_hardlinks/test.py index 7ac7fe12108..f70cbccefa5 100644 --- a/tests/integration/test_mutations_hardlinks/test.py +++ b/tests/integration/test_mutations_hardlinks/test.py @@ -8,7 +8,7 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/wide_parts_only.xml']) +node1 = cluster.add_instance("node1", main_configs=["configs/wide_parts_only.xml"]) @pytest.fixture(scope="module") @@ -21,42 +21,61 @@ def started_cluster(): def check_hardlinks(table, part_path, column_file, count): - column_path = os.path.join("/var/lib/clickhouse/data/default", table, part_path, column_file) + column_path = os.path.join( + "/var/lib/clickhouse/data/default", table, part_path, column_file + ) script = """ export INODE=`ls -i {column_path} | awk '{{print $1}}'` export COUNT=`find /var/lib/clickhouse -inum $INODE | wc -l` test $COUNT = {count} - """.format(column_path=column_path, count=count) + """.format( + column_path=column_path, count=count + ) node1.exec_in_container(["bash", "-c", script]) def check_exists(table, part_path, column_file): - column_path = os.path.join("/var/lib/clickhouse/data/default", table, part_path, column_file) + column_path = os.path.join( + "/var/lib/clickhouse/data/default", table, part_path, column_file + ) node1.exec_in_container(["bash", "-c", "test -f {}".format(column_path)]) def test_update_mutation(started_cluster): node1.query( - "CREATE TABLE table_for_update(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + "CREATE TABLE table_for_update(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()" + ) - node1.query("INSERT INTO table_for_update SELECT number, number, toString(number) from numbers(100)") + node1.query( + "INSERT INTO table_for_update SELECT number, number, toString(number) from numbers(100)" + ) - assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(range(100)) + assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum( + range(100) + ) - node1.query("ALTER TABLE table_for_update UPDATE value1 = value1 * value1 WHERE 1", - settings={"mutations_sync": "2"}) - assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(i * i for i in range(100)) + node1.query( + "ALTER TABLE table_for_update UPDATE value1 = value1 * value1 WHERE 1", + settings={"mutations_sync": "2"}, + ) + assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum( + i * i for i in range(100) + ) check_hardlinks("table_for_update", "all_1_1_0_2", "key.bin", 2) check_hardlinks("table_for_update", "all_1_1_0_2", "value2.bin", 2) check_hardlinks("table_for_update", "all_1_1_0_2", "value1.bin", 1) - node1.query("ALTER TABLE table_for_update UPDATE key=key, value1=value1, value2=value2 WHERE 1", - settings={"mutations_sync": "2"}) + node1.query( + "ALTER TABLE table_for_update UPDATE key=key, value1=value1, value2=value2 WHERE 1", + settings={"mutations_sync": "2"}, + ) - assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum(i * i for i in range(100)) + assert int(node1.query("SELECT sum(value1) FROM table_for_update").strip()) == sum( + i * i for i in range(100) + ) check_hardlinks("table_for_update", "all_1_1_0_3", "key.bin", 1) check_hardlinks("table_for_update", "all_1_1_0_3", "value1.bin", 1) @@ -65,15 +84,25 @@ def test_update_mutation(started_cluster): def test_modify_mutation(started_cluster): node1.query( - "CREATE TABLE table_for_modify(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + "CREATE TABLE table_for_modify(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()" + ) - node1.query("INSERT INTO table_for_modify SELECT number, number, toString(number) from numbers(100)") + node1.query( + "INSERT INTO table_for_modify SELECT number, number, toString(number) from numbers(100)" + ) - assert int(node1.query("SELECT sum(value1) FROM table_for_modify").strip()) == sum(range(100)) + assert int(node1.query("SELECT sum(value1) FROM table_for_modify").strip()) == sum( + range(100) + ) - node1.query("ALTER TABLE table_for_modify MODIFY COLUMN value2 UInt64", settings={"mutations_sync": "2"}) + node1.query( + "ALTER TABLE table_for_modify MODIFY COLUMN value2 UInt64", + settings={"mutations_sync": "2"}, + ) - assert int(node1.query("SELECT sum(value2) FROM table_for_modify").strip()) == sum(range(100)) + assert int(node1.query("SELECT sum(value2) FROM table_for_modify").strip()) == sum( + range(100) + ) check_hardlinks("table_for_modify", "all_1_1_0_2", "key.bin", 2) check_hardlinks("table_for_modify", "all_1_1_0_2", "value1.bin", 2) @@ -82,13 +111,21 @@ def test_modify_mutation(started_cluster): def test_drop_mutation(started_cluster): node1.query( - "CREATE TABLE table_for_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + "CREATE TABLE table_for_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()" + ) - node1.query("INSERT INTO table_for_drop SELECT number, number, toString(number) from numbers(100)") + node1.query( + "INSERT INTO table_for_drop SELECT number, number, toString(number) from numbers(100)" + ) - assert int(node1.query("SELECT sum(value1) FROM table_for_drop").strip()) == sum(range(100)) + assert int(node1.query("SELECT sum(value1) FROM table_for_drop").strip()) == sum( + range(100) + ) - node1.query("ALTER TABLE table_for_drop DROP COLUMN value2", settings={"mutations_sync": "2"}) + node1.query( + "ALTER TABLE table_for_drop DROP COLUMN value2", + settings={"mutations_sync": "2"}, + ) check_hardlinks("table_for_drop", "all_1_1_0_2", "key.bin", 2) check_hardlinks("table_for_drop", "all_1_1_0_2", "value1.bin", 2) @@ -101,23 +138,31 @@ def test_drop_mutation(started_cluster): def test_delete_and_drop_mutation(started_cluster): node1.query( - "CREATE TABLE table_for_delete_and_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()") + "CREATE TABLE table_for_delete_and_drop(key UInt64, value1 UInt64, value2 String) ENGINE MergeTree() ORDER BY tuple()" + ) - node1.query("INSERT INTO table_for_delete_and_drop SELECT number, number, toString(number) from numbers(100)") + node1.query( + "INSERT INTO table_for_delete_and_drop SELECT number, number, toString(number) from numbers(100)" + ) - assert int(node1.query("SELECT sum(value1) FROM table_for_delete_and_drop").strip()) == sum(range(100)) + assert int( + node1.query("SELECT sum(value1) FROM table_for_delete_and_drop").strip() + ) == sum(range(100)) node1.query("SYSTEM STOP MERGES") def mutate(): - node1.query("ALTER TABLE table_for_delete_and_drop DELETE WHERE key % 2 == 0, DROP COLUMN value2") + node1.query( + "ALTER TABLE table_for_delete_and_drop DELETE WHERE key % 2 == 0, DROP COLUMN value2" + ) p = Pool(2) p.apply_async(mutate) for _ in range(1, 100): result = node1.query( - "SELECT COUNT() FROM system.mutations WHERE table = 'table_for_delete_and_drop' and is_done=0") + "SELECT COUNT() FROM system.mutations WHERE table = 'table_for_delete_and_drop' and is_done=0" + ) try: if int(result.strip()) == 2: break @@ -129,8 +174,11 @@ def test_delete_and_drop_mutation(started_cluster): node1.query("SYSTEM START MERGES") - assert_eq_with_retry(node1, "SELECT COUNT() FROM table_for_delete_and_drop", - str(sum(1 for i in range(100) if i % 2 != 0))) + assert_eq_with_retry( + node1, + "SELECT COUNT() FROM table_for_delete_and_drop", + str(sum(1 for i in range(100) if i % 2 != 0)), + ) check_hardlinks("table_for_delete_and_drop", "all_1_1_0_3", "key.bin", 1) check_hardlinks("table_for_delete_and_drop", "all_1_1_0_3", "value1.bin", 1) diff --git a/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py index 2abeaf50cbd..2ab5816e5b1 100644 --- a/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py +++ b/tests/integration/test_mutations_in_partitions_of_merge_tree/test.py @@ -5,11 +5,19 @@ import helpers.cluster cluster = helpers.cluster.ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'], - with_zookeeper=True, stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/logs_config.xml", "configs/cluster.xml"], + with_zookeeper=True, + stay_alive=True, +) -node2 = cluster.add_instance('node2', main_configs=['configs/logs_config.xml', 'configs/cluster.xml'], - with_zookeeper=True, stay_alive=True) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/logs_config.xml", "configs/cluster.xml"], + with_zookeeper=True, + stay_alive=True, +) @pytest.fixture(scope="module") @@ -26,19 +34,39 @@ def test_trivial_alter_in_partition_merge_tree_without_where(started_cluster): try: name = "test_trivial_alter_in_partition_merge_tree_without_where" node1.query("DROP TABLE IF EXISTS {}".format(name)) - node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name)) + node1.query( + "CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format( + name + ) + ) node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name)) with pytest.raises(helpers.client.QueryRuntimeException): - node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] with pytest.raises(helpers.client.QueryRuntimeException): - node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] with pytest.raises(helpers.client.QueryRuntimeException): - node1.query("ALTER TABLE {} DELETE IN PARTITION 1 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 1 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] with pytest.raises(helpers.client.QueryRuntimeException): - node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["5"] finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) @@ -48,16 +76,39 @@ def test_trivial_alter_in_partition_merge_tree_with_where(started_cluster): try: name = "test_trivial_alter_in_partition_merge_tree_with_where" node1.query("DROP TABLE IF EXISTS {}".format(name)) - node1.query("CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format(name)) + node1.query( + "CREATE TABLE {} (p Int64, x Int64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY p".format( + name + ) + ) node1.query("INSERT INTO {} VALUES (1, 2), (2, 3)".format(name)) - node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) - assert node1.query("SELECT x FROM {} ORDER BY p".format(name)).splitlines() == ["2", "4"] + node1.query( + "ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) + assert node1.query("SELECT x FROM {} ORDER BY p".format(name)).splitlines() == [ + "2", + "4", + ] assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] - node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] - node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] - node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) assert node1.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) @@ -72,28 +123,60 @@ def test_trivial_alter_in_partition_replicated_merge_tree(started_cluster): for node in (node1, node2): node.query( - "CREATE TABLE {name} (p Int64, x Int64) ENGINE=ReplicatedMergeTree('/clickhouse/{name}', '{{instance}}') ORDER BY tuple() PARTITION BY p" - .format(name=name)) + "CREATE TABLE {name} (p Int64, x Int64) ENGINE=ReplicatedMergeTree('/clickhouse/{name}', '{{instance}}') ORDER BY tuple() PARTITION BY p".format( + name=name + ) + ) node1.query("INSERT INTO {} VALUES (1, 2)".format(name)) node2.query("INSERT INTO {} VALUES (2, 3)".format(name)) - node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE 1 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 2 WHERE 1 SETTINGS mutations_sync = 2".format( + name + ) + ) for node in (node1, node2): - assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] - node1.query("ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == [ + "6" + ] + node1.query( + "ALTER TABLE {} UPDATE x = x + 1 IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) for node in (node1, node2): - assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == [ + "6" + ] with pytest.raises(helpers.client.QueryRuntimeException): - node1.query("ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format(name)) + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 2 SETTINGS mutations_sync = 2".format( + name + ) + ) for node in (node1, node2): - assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["6"] - node1.query("ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == [ + "6" + ] + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 2 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) for node in (node1, node2): - assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] - node1.query("ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format(name)) + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == [ + "2" + ] + node1.query( + "ALTER TABLE {} DELETE IN PARTITION 1 WHERE p = 2 SETTINGS mutations_sync = 2".format( + name + ) + ) for node in (node1, node2): - assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == ["2"] + assert node.query("SELECT sum(x) FROM {}".format(name)).splitlines() == [ + "2" + ] finally: node1.query("DROP TABLE IF EXISTS {}".format(name)) node2.query("DROP TABLE IF EXISTS {}".format(name)) diff --git a/tests/integration/test_mutations_with_merge_tree/test.py b/tests/integration/test_mutations_with_merge_tree/test.py index 72ef8c9a373..d1843017b9f 100644 --- a/tests/integration/test_mutations_with_merge_tree/test.py +++ b/tests/integration/test_mutations_with_merge_tree/test.py @@ -5,8 +5,11 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance_test_mutations = cluster.add_instance('test_mutations_with_merge_tree', main_configs=['configs/config.xml'], - user_configs=['configs/users.xml']) +instance_test_mutations = cluster.add_instance( + "test_mutations_with_merge_tree", + main_configs=["configs/config.xml"], + user_configs=["configs/users.xml"], +) @pytest.fixture(scope="module") @@ -14,9 +17,11 @@ def started_cluster(): try: cluster.start() instance_test_mutations.query( - '''CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)''') + """CREATE TABLE test_mutations_with_ast_elements(date Date, a UInt64, b String) ENGINE = MergeTree(date, (a, date), 8192)""" + ) instance_test_mutations.query( - '''INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1) SETTINGS force_index_by_date = 0, force_primary_key = 0''') + """INSERT INTO test_mutations_with_ast_elements SELECT '2019-07-29' AS date, 1, toString(number) FROM numbers(1) SETTINGS force_index_by_date = 0, force_primary_key = 0""" + ) yield cluster finally: cluster.shutdown() @@ -28,110 +33,161 @@ def test_mutations_in_partition_background(started_cluster): name = "test_mutations_in_partition" instance_test_mutations.query( - f'''CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a''') + f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a""" + ) instance_test_mutations.query( - f'''INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})''') + f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})""" + ) for i in range(0, numbers, 3): - instance_test_mutations.query(f'''ALTER TABLE {name} DELETE IN PARTITION {i} WHERE a = {i}''') + instance_test_mutations.query( + f"""ALTER TABLE {name} DELETE IN PARTITION {i} WHERE a = {i}""" + ) for i in range(1, numbers, 3): - instance_test_mutations.query(f'''ALTER TABLE {name} UPDATE b = 'changed' IN PARTITION {i} WHERE a = {i} ''') + instance_test_mutations.query( + f"""ALTER TABLE {name} UPDATE b = 'changed' IN PARTITION {i} WHERE a = {i} """ + ) def count_and_changed(): - return instance_test_mutations.query(f"SELECT count(), countIf(b == 'changed') FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() + return instance_test_mutations.query( + f"SELECT count(), countIf(b == 'changed') FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() all_done = False - for wait_times_for_mutation in range(100): # wait for replication 80 seconds max + for wait_times_for_mutation in range( + 100 + ): # wait for replication 80 seconds max time.sleep(0.8) if count_and_changed() == ["66,33"]: all_done = True break - print(instance_test_mutations.query( - f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames")) + print( + instance_test_mutations.query( + f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames" + ) + ) assert (count_and_changed(), all_done) == (["66,33"], True) - assert instance_test_mutations.query(f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() == ["67,67"] + assert instance_test_mutations.query( + f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() == ["67,67"] finally: - instance_test_mutations.query(f'''DROP TABLE {name}''') + instance_test_mutations.query(f"""DROP TABLE {name}""") -@pytest.mark.parametrize("sync", [ - ("last",), - ("all",) -]) +@pytest.mark.parametrize("sync", [("last",), ("all",)]) def test_mutations_in_partition_sync(started_cluster, sync): try: numbers = 10 name = "test_mutations_in_partition_sync" instance_test_mutations.query( - f'''CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a''') + f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a""" + ) instance_test_mutations.query( - f'''INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})''') + f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})""" + ) for i in range(0, numbers, 3): - instance_test_mutations.query(f'''ALTER TABLE {name} DELETE IN PARTITION {i} WHERE a = {i}''' - + (' SETTINGS mutations_sync = 1' if sync == 'all' else '')) + instance_test_mutations.query( + f"""ALTER TABLE {name} DELETE IN PARTITION {i} WHERE a = {i}""" + + (" SETTINGS mutations_sync = 1" if sync == "all" else "") + ) - for reverse_index, i in reversed(list(enumerate(reversed(range(1, numbers, 3))))): - instance_test_mutations.query(f'''ALTER TABLE {name} UPDATE b = 'changed' IN PARTITION {i} WHERE a = {i}''' - + (' SETTINGS mutations_sync = 1' if not reverse_index or sync == 'all' else '')) + for reverse_index, i in reversed( + list(enumerate(reversed(range(1, numbers, 3)))) + ): + instance_test_mutations.query( + f"""ALTER TABLE {name} UPDATE b = 'changed' IN PARTITION {i} WHERE a = {i}""" + + ( + " SETTINGS mutations_sync = 1" + if not reverse_index or sync == "all" + else "" + ) + ) def count_and_changed(): - return instance_test_mutations.query(f"SELECT count(), countIf(b == 'changed') FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() + return instance_test_mutations.query( + f"SELECT count(), countIf(b == 'changed') FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() - print(instance_test_mutations.query( - f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames")) + print( + instance_test_mutations.query( + f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames" + ) + ) assert count_and_changed() == ["6,3"] - assert instance_test_mutations.query(f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() == ["7,7"] + assert instance_test_mutations.query( + f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() == ["7,7"] finally: - instance_test_mutations.query(f'''DROP TABLE {name}''') + instance_test_mutations.query(f"""DROP TABLE {name}""") def test_mutations_with_merge_background_task(started_cluster): - instance_test_mutations.query('''SYSTEM STOP MERGES test_mutations_with_ast_elements''') + instance_test_mutations.query( + """SYSTEM STOP MERGES test_mutations_with_ast_elements""" + ) ## The number of asts per query is 15 for execution_times_for_mutation in range(100): instance_test_mutations.query( - '''ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)''') + """ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)""" + ) all_done = False for wait_times_for_mutation in range(100): # wait for replication 80 seconds max time.sleep(0.8) def get_done_mutations(instance): - instance_test_mutations.query('''DETACH TABLE test_mutations_with_ast_elements''') - instance_test_mutations.query('''ATTACH TABLE test_mutations_with_ast_elements''') - return int(instance.query( - "SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' SETTINGS force_index_by_date = 0, force_primary_key = 0").rstrip()) + instance_test_mutations.query( + """DETACH TABLE test_mutations_with_ast_elements""" + ) + instance_test_mutations.query( + """ATTACH TABLE test_mutations_with_ast_elements""" + ) + return int( + instance.query( + "SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' SETTINGS force_index_by_date = 0, force_primary_key = 0" + ).rstrip() + ) if get_done_mutations(instance_test_mutations) == 100: all_done = True break - print(instance_test_mutations.query( - "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames")) + print( + instance_test_mutations.query( + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations_with_ast_elements' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames" + ) + ) assert all_done def test_mutations_with_truncate_table(started_cluster): - instance_test_mutations.query('''SYSTEM STOP MERGES test_mutations_with_ast_elements''') + instance_test_mutations.query( + """SYSTEM STOP MERGES test_mutations_with_ast_elements""" + ) ## The number of asts per query is 15 for execute_number in range(100): instance_test_mutations.query( - '''ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)''') + """ALTER TABLE test_mutations_with_ast_elements DELETE WHERE 1 = 1 AND toUInt32(b) IN (1)""" + ) instance_test_mutations.query("TRUNCATE TABLE test_mutations_with_ast_elements") - assert instance_test_mutations.query( - "SELECT COUNT() FROM system.mutations WHERE table = 'test_mutations_with_ast_elements SETTINGS force_index_by_date = 0, force_primary_key = 0'").rstrip() == '0' + assert ( + instance_test_mutations.query( + "SELECT COUNT() FROM system.mutations WHERE table = 'test_mutations_with_ast_elements SETTINGS force_index_by_date = 0, force_primary_key = 0'" + ).rstrip() + == "0" + ) def test_mutations_will_not_hang_for_non_existing_parts_sync(started_cluster): @@ -140,21 +196,32 @@ def test_mutations_will_not_hang_for_non_existing_parts_sync(started_cluster): name = "test_mutations_will_not_hang_for_non_existing_parts_sync" instance_test_mutations.query( - f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a""") + f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a""" + ) instance_test_mutations.query( - f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})""") + f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})""" + ) for i in range(0, numbers, 3): - instance_test_mutations.query(f"""ALTER TABLE {name} DELETE IN PARTITION {i+1000} WHERE a = {i} SETTINGS mutations_sync = 1""") + instance_test_mutations.query( + f"""ALTER TABLE {name} DELETE IN PARTITION {i+1000} WHERE a = {i} SETTINGS mutations_sync = 1""" + ) def count(): - return instance_test_mutations.query(f"SELECT count() FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() + return instance_test_mutations.query( + f"SELECT count() FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() - print(instance_test_mutations.query( - f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames")) + print( + instance_test_mutations.query( + f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames" + ) + ) assert count() == [f"{numbers}"] - assert instance_test_mutations.query(f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() == [f"34,34"] + assert instance_test_mutations.query( + f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() == [f"34,34"] finally: instance_test_mutations.query(f"""DROP TABLE {name}""") @@ -166,29 +233,42 @@ def test_mutations_will_not_hang_for_non_existing_parts_async(started_cluster): name = "test_mutations_will_not_hang_for_non_existing_parts_async" instance_test_mutations.query( - f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a""") + f"""CREATE TABLE {name} (date Date, a UInt64, b String) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY a""" + ) instance_test_mutations.query( - f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})""") + f"""INSERT INTO {name} SELECT '2019-07-29' AS date, number, toString(number) FROM numbers({numbers})""" + ) for i in range(0, numbers, 3): - instance_test_mutations.query(f"""ALTER TABLE {name} DELETE IN PARTITION {i+1000} WHERE a = {i}""") + instance_test_mutations.query( + f"""ALTER TABLE {name} DELETE IN PARTITION {i+1000} WHERE a = {i}""" + ) def count(): - return instance_test_mutations.query(f"SELECT count() FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() + return instance_test_mutations.query( + f"SELECT count() FROM {name} SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() def count_and_sum_is_done(): - return instance_test_mutations.query(f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV").splitlines() + return instance_test_mutations.query( + f"SELECT count(), sum(is_done) FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() all_done = False - for wait_times_for_mutation in range(100): # wait for replication 80 seconds max + for wait_times_for_mutation in range( + 100 + ): # wait for replication 80 seconds max time.sleep(0.8) if count_and_sum_is_done() == ["34,34"]: all_done = True break - print(instance_test_mutations.query( - f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames")) + print( + instance_test_mutations.query( + f"SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = '{name}' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames" + ) + ) assert count() == [f"{numbers}"] assert count_and_sum_is_done() == ["34,34"] diff --git a/tests/integration/test_mysql_database_engine/test.py b/tests/integration/test_mysql_database_engine/test.py index ff1c955d78b..96a4e2d692c 100644 --- a/tests/integration/test_mysql_database_engine/test.py +++ b/tests/integration/test_mysql_database_engine/test.py @@ -9,7 +9,12 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -clickhouse_node = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/named_collections.xml'], with_mysql=True, stay_alive=True) +clickhouse_node = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/named_collections.xml"], + with_mysql=True, + stay_alive=True, +) @pytest.fixture(scope="module") @@ -32,12 +37,17 @@ class MySQLNodeInstance: def query(self, execution_query): if self.mysql_connection is None: - self.mysql_connection = pymysql.connect(user=self.user, password=self.password, host=self.hostname, - port=self.port) + self.mysql_connection = pymysql.connect( + user=self.user, + password=self.password, + host=self.hostname, + port=self.port, + ) with self.mysql_connection.cursor() as cursor: + def execute(query): res = cursor.execute(query) - if query.lstrip().lower().startswith(('select', 'show')): + if query.lstrip().lower().startswith(("select", "show")): # Mimic output of the ClickHouseInstance, which is: # tab-sparated values and newline (\n)-separated rows. rows = [] @@ -57,168 +67,279 @@ class MySQLNodeInstance: def test_mysql_ddl_for_mysql_database(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: mysql_node.query("DROP DATABASE IF EXISTS test_database") mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')") - assert 'test_database' in clickhouse_node.query('SHOW DATABASES') + "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')" + ) + assert "test_database" in clickhouse_node.query("SHOW DATABASES") mysql_node.query( - 'CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') - assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') + "CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) + assert "test_table" in clickhouse_node.query("SHOW TABLES FROM test_database") time.sleep( - 3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained - mysql_node.query('ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11)') - assert 'add_column' in clickhouse_node.query( - "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + 3 + ) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + mysql_node.query( + "ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11)" + ) + assert "add_column" in clickhouse_node.query( + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + ) time.sleep( - 3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained - mysql_node.query('ALTER TABLE `test_database`.`test_table` DROP COLUMN `add_column`') - assert 'add_column' not in clickhouse_node.query( - "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + 3 + ) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + mysql_node.query( + "ALTER TABLE `test_database`.`test_table` DROP COLUMN `add_column`" + ) + assert "add_column" not in clickhouse_node.query( + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + ) - mysql_node.query('DROP TABLE `test_database`.`test_table`;') - assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') + mysql_node.query("DROP TABLE `test_database`.`test_table`;") + assert "test_table" not in clickhouse_node.query( + "SHOW TABLES FROM test_database" + ) clickhouse_node.query("DROP DATABASE test_database") - assert 'test_database' not in clickhouse_node.query('SHOW DATABASES') + assert "test_database" not in clickhouse_node.query("SHOW DATABASES") mysql_node.query("DROP DATABASE test_database") def test_clickhouse_ddl_for_mysql_database(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") mysql_node.query( - 'CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') + "CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')") + "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')" + ) - assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') + assert "test_table" in clickhouse_node.query("SHOW TABLES FROM test_database") clickhouse_node.query("DROP TABLE test_database.test_table") - assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') + assert "test_table" not in clickhouse_node.query( + "SHOW TABLES FROM test_database" + ) clickhouse_node.query("ATTACH TABLE test_database.test_table") - assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') + assert "test_table" in clickhouse_node.query("SHOW TABLES FROM test_database") clickhouse_node.query("DETACH TABLE test_database.test_table") - assert 'test_table' not in clickhouse_node.query('SHOW TABLES FROM test_database') + assert "test_table" not in clickhouse_node.query( + "SHOW TABLES FROM test_database" + ) clickhouse_node.query("ATTACH TABLE test_database.test_table") - assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_database') + assert "test_table" in clickhouse_node.query("SHOW TABLES FROM test_database") clickhouse_node.query("DROP DATABASE test_database") - assert 'test_database' not in clickhouse_node.query('SHOW DATABASES') + assert "test_database" not in clickhouse_node.query("SHOW DATABASES") mysql_node.query("DROP DATABASE test_database") def test_clickhouse_dml_for_mysql_database(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") mysql_node.query( - 'CREATE TABLE `test_database`.`test_table` ( `i``d` int(11) NOT NULL, PRIMARY KEY (`i``d`)) ENGINE=InnoDB;') + "CREATE TABLE `test_database`.`test_table` ( `i``d` int(11) NOT NULL, PRIMARY KEY (`i``d`)) ENGINE=InnoDB;" + ) clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', test_database, 'root', 'clickhouse')") + "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', test_database, 'root', 'clickhouse')" + ) - assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '0' - clickhouse_node.query("INSERT INTO `test_database`.`test_table`(`i``d`) select number from numbers(10000)") - assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '10000' + assert ( + clickhouse_node.query( + "SELECT count() FROM `test_database`.`test_table`" + ).rstrip() + == "0" + ) + clickhouse_node.query( + "INSERT INTO `test_database`.`test_table`(`i``d`) select number from numbers(10000)" + ) + assert ( + clickhouse_node.query( + "SELECT count() FROM `test_database`.`test_table`" + ).rstrip() + == "10000" + ) clickhouse_node.query("DROP DATABASE test_database") - assert 'test_database' not in clickhouse_node.query('SHOW DATABASES') + assert "test_database" not in clickhouse_node.query("SHOW DATABASES") mysql_node.query("DROP DATABASE test_database") def test_clickhouse_join_for_mysql_database(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: - mysql_node.query("CREATE DATABASE IF NOT EXISTS test DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE test.t1_mysql_local (" - "pays VARCHAR(55) DEFAULT 'FRA' NOT NULL," - "service VARCHAR(5) DEFAULT '' NOT NULL," - "opco CHAR(3) DEFAULT '' NOT NULL" - ")") - mysql_node.query("CREATE TABLE test.t2_mysql_local (" - "service VARCHAR(5) DEFAULT '' NOT NULL," - "opco VARCHAR(5) DEFAULT ''" - ")") + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: + mysql_node.query( + "CREATE DATABASE IF NOT EXISTS test DEFAULT CHARACTER SET 'utf8'" + ) + mysql_node.query( + "CREATE TABLE test.t1_mysql_local (" + "pays VARCHAR(55) DEFAULT 'FRA' NOT NULL," + "service VARCHAR(5) DEFAULT '' NOT NULL," + "opco CHAR(3) DEFAULT '' NOT NULL" + ")" + ) + mysql_node.query( + "CREATE TABLE test.t2_mysql_local (" + "service VARCHAR(5) DEFAULT '' NOT NULL," + "opco VARCHAR(5) DEFAULT ''" + ")" + ) clickhouse_node.query( - "CREATE TABLE default.t1_remote_mysql AS mysql('mysql57:3306','test','t1_mysql_local','root','clickhouse')") + "CREATE TABLE default.t1_remote_mysql AS mysql('mysql57:3306','test','t1_mysql_local','root','clickhouse')" + ) clickhouse_node.query( - "CREATE TABLE default.t2_remote_mysql AS mysql('mysql57:3306','test','t2_mysql_local','root','clickhouse')") - clickhouse_node.query("INSERT INTO `default`.`t1_remote_mysql` VALUES ('EN','A',''),('RU','B','AAA')") - clickhouse_node.query("INSERT INTO `default`.`t2_remote_mysql` VALUES ('A','AAA'),('Z','')") + "CREATE TABLE default.t2_remote_mysql AS mysql('mysql57:3306','test','t2_mysql_local','root','clickhouse')" + ) + clickhouse_node.query( + "INSERT INTO `default`.`t1_remote_mysql` VALUES ('EN','A',''),('RU','B','AAA')" + ) + clickhouse_node.query( + "INSERT INTO `default`.`t2_remote_mysql` VALUES ('A','AAA'),('Z','')" + ) - assert clickhouse_node.query("SELECT s.pays " - "FROM default.t1_remote_mysql AS s " - "LEFT JOIN default.t1_remote_mysql AS s_ref " - "ON (s_ref.opco = s.opco AND s_ref.service = s.service) " - "WHERE s_ref.opco != '' AND s.opco != '' ").rstrip() == 'RU' + assert ( + clickhouse_node.query( + "SELECT s.pays " + "FROM default.t1_remote_mysql AS s " + "LEFT JOIN default.t1_remote_mysql AS s_ref " + "ON (s_ref.opco = s.opco AND s_ref.service = s.service) " + "WHERE s_ref.opco != '' AND s.opco != '' " + ).rstrip() + == "RU" + ) mysql_node.query("DROP DATABASE test") def test_bad_arguments_for_mysql_database_engine(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, port=started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", + "clickhouse", + started_cluster.mysql_ip, + port=started_cluster.mysql_port, + ) + ) as mysql_node: with pytest.raises(QueryRuntimeException) as exception: - mysql_node.query("CREATE DATABASE IF NOT EXISTS test_bad_arguments DEFAULT CHARACTER SET 'utf8'") + mysql_node.query( + "CREATE DATABASE IF NOT EXISTS test_bad_arguments DEFAULT CHARACTER SET 'utf8'" + ) clickhouse_node.query( - "CREATE DATABASE test_database_bad_arguments ENGINE = MySQL('mysql57:3306', test_bad_arguments, root, 'clickhouse')") - assert 'Database engine MySQL requested literal argument.' in str(exception.value) + "CREATE DATABASE test_database_bad_arguments ENGINE = MySQL('mysql57:3306', test_bad_arguments, root, 'clickhouse')" + ) + assert "Database engine MySQL requested literal argument." in str( + exception.value + ) mysql_node.query("DROP DATABASE test_bad_arguments") + def test_column_comments_for_mysql_database_engine(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: mysql_node.query("DROP DATABASE IF EXISTS test_database") mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") clickhouse_node.query( - "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')") - assert 'test_database' in clickhouse_node.query('SHOW DATABASES') + "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', 'test_database', 'root', 'clickhouse')" + ) + assert "test_database" in clickhouse_node.query("SHOW DATABASES") mysql_node.query( - "CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`), `test` int COMMENT 'test comment') ENGINE=InnoDB;") - assert 'test comment' in clickhouse_node.query('DESCRIBE TABLE `test_database`.`test_table`') + "CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`), `test` int COMMENT 'test comment') ENGINE=InnoDB;" + ) + assert "test comment" in clickhouse_node.query( + "DESCRIBE TABLE `test_database`.`test_table`" + ) time.sleep( - 3) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained - mysql_node.query("ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11) COMMENT 'add_column comment'") - assert 'add_column comment' in clickhouse_node.query( - "SELECT comment FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + 3 + ) # Because the unit of MySQL modification time is seconds, modifications made in the same second cannot be obtained + mysql_node.query( + "ALTER TABLE `test_database`.`test_table` ADD COLUMN `add_column` int(11) COMMENT 'add_column comment'" + ) + assert "add_column comment" in clickhouse_node.query( + "SELECT comment FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + ) clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") def test_data_types_support_level_for_mysql_database_engine(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: - mysql_node.query("CREATE DATABASE IF NOT EXISTS test DEFAULT CHARACTER SET 'utf8'") - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', test, 'root', 'clickhouse')", - settings={"mysql_datatypes_support_level": "decimal,datetime64"}) + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: + mysql_node.query( + "CREATE DATABASE IF NOT EXISTS test DEFAULT CHARACTER SET 'utf8'" + ) + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MySQL('mysql57:3306', test, 'root', 'clickhouse')", + settings={"mysql_datatypes_support_level": "decimal,datetime64"}, + ) - assert "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" in clickhouse_node.query("SHOW CREATE DATABASE test_database FORMAT TSV") + assert ( + "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" + in clickhouse_node.query("SHOW CREATE DATABASE test_database FORMAT TSV") + ) clickhouse_node.query("DETACH DATABASE test_database") # without context settings clickhouse_node.query("ATTACH DATABASE test_database") - assert "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" in clickhouse_node.query("SHOW CREATE DATABASE test_database FORMAT TSV") + assert ( + "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" + in clickhouse_node.query("SHOW CREATE DATABASE test_database FORMAT TSV") + ) clickhouse_node.query( "CREATE DATABASE test_database_1 ENGINE = MySQL('mysql57:3306', test, 'root', 'clickhouse') SETTINGS mysql_datatypes_support_level = 'decimal,datetime64'", - settings={"mysql_datatypes_support_level": "decimal"}) + settings={"mysql_datatypes_support_level": "decimal"}, + ) - assert "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" in clickhouse_node.query("SHOW CREATE DATABASE test_database_1 FORMAT TSV") + assert ( + "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" + in clickhouse_node.query("SHOW CREATE DATABASE test_database_1 FORMAT TSV") + ) clickhouse_node.query("DETACH DATABASE test_database_1") # without context settings clickhouse_node.query("ATTACH DATABASE test_database_1") - assert "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" in clickhouse_node.query("SHOW CREATE DATABASE test_database_1 FORMAT TSV") + assert ( + "SETTINGS mysql_datatypes_support_level = \\'decimal,datetime64\\'" + in clickhouse_node.query("SHOW CREATE DATABASE test_database_1 FORMAT TSV") + ) clickhouse_node.query("DROP DATABASE test_database") clickhouse_node.query("DROP DATABASE test_database_1") - assert 'test_database' not in clickhouse_node.query('SHOW DATABASES') + assert "test_database" not in clickhouse_node.query("SHOW DATABASES") mysql_node.query("DROP DATABASE test") @@ -226,7 +347,10 @@ def test_data_types_support_level_for_mysql_database_engine(started_cluster): # float_values = ['NULL'] # float_values = [0] mysql returns 0 while clickhouse returns 0.0, so cannot compare using == directly int32_values = [0, 1, -1, 2147483647, -2147483648] -uint32_values = [0, 1] # [FIXME] seems client have issue with value 4294967295, it returns -1 for it +uint32_values = [ + 0, + 1, +] # [FIXME] seems client have issue with value 4294967295, it returns -1 for it mint_values = [0, 1, -1, 8388607, -8388608] umint_values = [0, 1, 16777215] int16_values = [0, 1, -1, 32767, -32768] @@ -235,215 +359,548 @@ int8_values = [0, 1, -1, 127, -128] uint8_values = [0, 1, 255] # string_values = ["'ClickHouse'", 'NULL'] string_values = ["'ClickHouse'"] +date_values = ["'1970-01-01'"] +date2Date32_values = ["'1925-01-01'", "'2283-11-11'"] +date2String_values = ["'1000-01-01'", "'9999-12-31'"] -decimal_values = [0, 0.123, 0.4, 5.67, 8.91011, 123456789.123, -0.123, -0.4, -5.67, -8.91011, -123456789.123] +decimal_values = [ + 0, + 0.123, + 0.4, + 5.67, + 8.91011, + 123456789.123, + -0.123, + -0.4, + -5.67, + -8.91011, + -123456789.123, +] timestamp_values = ["'2015-05-18 07:40:01.123'", "'2019-09-16 19:20:11.123'"] timestamp_values_no_subsecond = ["'2015-05-18 07:40:01'", "'2019-09-16 19:20:11'"] -@pytest.mark.parametrize("case_name, mysql_type, expected_ch_type, mysql_values, setting_mysql_datatypes_support_level", - [ - # test common type mapping - # ("common_types", "FLOAT", "Nullable(Float32)", float_values, ""), - # ("common_types", "FLOAT UNSIGNED", "Nullable(Float32)", float_values, ""), - - pytest.param("common_types", "INT", "Nullable(Int32)", int32_values, "", id="common_types_1"), - pytest.param("common_types", "INT NOT NULL", "Int32", int32_values, "", id="common_types_2"), - pytest.param("common_types", "INT UNSIGNED NOT NULL", "UInt32", uint32_values, "", id="common_types_3"), - pytest.param("common_types", "INT UNSIGNED", "Nullable(UInt32)", uint32_values, "", id="common_types_4"), - pytest.param("common_types", "INT UNSIGNED DEFAULT NULL", "Nullable(UInt32)", uint32_values, "", id="common_types_5"), - pytest.param("common_types", "INT UNSIGNED DEFAULT '1'", "Nullable(UInt32)", uint32_values, "", id="common_types_6"), - pytest.param("common_types", "INT(10)", "Nullable(Int32)", int32_values, "", id="common_types_7"), - pytest.param("common_types", "INT(10) NOT NULL", "Int32", int32_values, "", id="common_types_8"), - pytest.param("common_types", "INT(10) UNSIGNED NOT NULL", "UInt32", uint32_values, "", id="common_types_8"), - pytest.param("common_types", "INT(10) UNSIGNED", "Nullable(UInt32)", uint32_values, "", id="common_types_9"), - pytest.param("common_types", "INT(10) UNSIGNED DEFAULT NULL", "Nullable(UInt32)", uint32_values, "", id="common_types_10"), - pytest.param("common_types", "INT(10) UNSIGNED DEFAULT '1'", "Nullable(UInt32)", uint32_values, "", id="common_types_11"), - pytest.param("common_types", "INTEGER", "Nullable(Int32)", int32_values, "", id="common_types_12"), - pytest.param("common_types", "INTEGER UNSIGNED", "Nullable(UInt32)", uint32_values, "", id="common_types_13"), - - pytest.param("common_types", "MEDIUMINT", "Nullable(Int32)", mint_values, "", id="common_types_14"), - pytest.param("common_types", "MEDIUMINT UNSIGNED", "Nullable(UInt32)", umint_values, "", id="common_types_15"), - - pytest.param("common_types", "SMALLINT", "Nullable(Int16)", int16_values, "", id="common_types_16"), - pytest.param("common_types", "SMALLINT UNSIGNED", "Nullable(UInt16)", uint16_values, "", id="common_types_17"), - - pytest.param("common_types", "TINYINT", "Nullable(Int8)", int8_values, "", id="common_types_18"), - pytest.param("common_types", "TINYINT UNSIGNED", "Nullable(UInt8)", uint8_values, "", id="common_types_19"), - - pytest.param("common_types", "VARCHAR(10)", "Nullable(String)", string_values, "", id="common_types_20"), - - - pytest.param("decimal_default", "decimal NOT NULL", "Decimal(10, 0)", decimal_values, - "decimal,datetime64", id="decimal_1"), - pytest.param("decimal_default_nullable", "decimal", "Nullable(Decimal(10, 0))", decimal_values, - "decimal,datetime64", id="decimal_2"), - pytest.param("decimal_18_6", "decimal(18, 6) NOT NULL", "Decimal(18, 6)", decimal_values, - "decimal,datetime64", id="decimal_3"), - pytest.param("decimal_38_6", "decimal(38, 6) NOT NULL", "Decimal(38, 6)", decimal_values, - "decimal,datetime64", id="decimal_4"), - - # Due to python DB driver roundtrip MySQL timestamp and datetime values - # are printed with 6 digits after decimal point, so to simplify tests a bit, - # we only validate precision of 0 and 6. - pytest.param("timestamp_default", "timestamp", "DateTime", timestamp_values, "decimal,datetime64", id="timestamp_default"), - pytest.param("timestamp_6", "timestamp(6)", "DateTime64(6)", timestamp_values, "decimal,datetime64", id="timestamp_6"), - pytest.param("datetime_default", "DATETIME NOT NULL", "DateTime64(0)", timestamp_values, - "decimal,datetime64", id="datetime_default"), - pytest.param("datetime_6", "DATETIME(6) NOT NULL", "DateTime64(6)", timestamp_values, - "decimal,datetime64", id="datetime_6_1"), - - # right now precision bigger than 39 is not supported by ClickHouse's Decimal, hence fall back to String - pytest.param("decimal_40_6", "decimal(40, 6) NOT NULL", "String", decimal_values, - "decimal,datetime64", id="decimal_40_6"), - pytest.param("decimal_18_6", "decimal(18, 6) NOT NULL", "String", decimal_values, "datetime64", id="decimal_18_6_1"), - pytest.param("decimal_18_6", "decimal(18, 6) NOT NULL", "String", decimal_values, "", id="decimal_18_6_2"), - pytest.param("datetime_6", "DATETIME(6) NOT NULL", "DateTime", timestamp_values_no_subsecond, - "decimal", id="datetime_6_2"), - pytest.param("datetime_6", "DATETIME(6) NOT NULL", "DateTime", timestamp_values_no_subsecond, "", id="datetime_6_3"), - ]) -def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, mysql_values, - setting_mysql_datatypes_support_level): - """ Verify that values written to MySQL can be read on ClickHouse side via DB engine MySQL, +@pytest.mark.parametrize( + "case_name, mysql_type, expected_ch_type, mysql_values, setting_mysql_datatypes_support_level", + [ + # test common type mapping + # ("common_types", "FLOAT", "Nullable(Float32)", float_values, ""), + # ("common_types", "FLOAT UNSIGNED", "Nullable(Float32)", float_values, ""), + pytest.param( + "common_types", + "INT", + "Nullable(Int32)", + int32_values, + "", + id="common_types_1", + ), + pytest.param( + "common_types", + "INT NOT NULL", + "Int32", + int32_values, + "", + id="common_types_2", + ), + pytest.param( + "common_types", + "INT UNSIGNED NOT NULL", + "UInt32", + uint32_values, + "", + id="common_types_3", + ), + pytest.param( + "common_types", + "INT UNSIGNED", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_4", + ), + pytest.param( + "common_types", + "INT UNSIGNED DEFAULT NULL", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_5", + ), + pytest.param( + "common_types", + "INT UNSIGNED DEFAULT '1'", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_6", + ), + pytest.param( + "common_types", + "INT(10)", + "Nullable(Int32)", + int32_values, + "", + id="common_types_7", + ), + pytest.param( + "common_types", + "INT(10) NOT NULL", + "Int32", + int32_values, + "", + id="common_types_8", + ), + pytest.param( + "common_types", + "INT(10) UNSIGNED NOT NULL", + "UInt32", + uint32_values, + "", + id="common_types_8", + ), + pytest.param( + "common_types", + "INT(10) UNSIGNED", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_9", + ), + pytest.param( + "common_types", + "INT(10) UNSIGNED DEFAULT NULL", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_10", + ), + pytest.param( + "common_types", + "INT(10) UNSIGNED DEFAULT '1'", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_11", + ), + pytest.param( + "common_types", + "INTEGER", + "Nullable(Int32)", + int32_values, + "", + id="common_types_12", + ), + pytest.param( + "common_types", + "INTEGER UNSIGNED", + "Nullable(UInt32)", + uint32_values, + "", + id="common_types_13", + ), + pytest.param( + "common_types", + "MEDIUMINT", + "Nullable(Int32)", + mint_values, + "", + id="common_types_14", + ), + pytest.param( + "common_types", + "MEDIUMINT UNSIGNED", + "Nullable(UInt32)", + umint_values, + "", + id="common_types_15", + ), + pytest.param( + "common_types", + "SMALLINT", + "Nullable(Int16)", + int16_values, + "", + id="common_types_16", + ), + pytest.param( + "common_types", + "SMALLINT UNSIGNED", + "Nullable(UInt16)", + uint16_values, + "", + id="common_types_17", + ), + pytest.param( + "common_types", + "TINYINT", + "Nullable(Int8)", + int8_values, + "", + id="common_types_18", + ), + pytest.param( + "common_types", + "TINYINT UNSIGNED", + "Nullable(UInt8)", + uint8_values, + "", + id="common_types_19", + ), + pytest.param( + "common_types", + "VARCHAR(10)", + "Nullable(String)", + string_values, + "", + id="common_types_20", + ), + pytest.param( + "common_types", + "DATE", + "Nullable(Date)", + date_values, + "", + id="common_types_21", + ), + pytest.param( + "common_types", + "DATE", + "Nullable(Date32)", + date2Date32_values, + "date2Date32", + id="common_types_22", + ), + pytest.param( + "common_types", + "DATE", + "Nullable(String)", + date2String_values, + "date2String", + id="common_types_23", + ), + pytest.param( + "decimal_default", + "decimal NOT NULL", + "Decimal(10, 0)", + decimal_values, + "decimal,datetime64", + id="decimal_1", + ), + pytest.param( + "decimal_default_nullable", + "decimal", + "Nullable(Decimal(10, 0))", + decimal_values, + "decimal,datetime64", + id="decimal_2", + ), + pytest.param( + "decimal_18_6", + "decimal(18, 6) NOT NULL", + "Decimal(18, 6)", + decimal_values, + "decimal,datetime64", + id="decimal_3", + ), + pytest.param( + "decimal_38_6", + "decimal(38, 6) NOT NULL", + "Decimal(38, 6)", + decimal_values, + "decimal,datetime64", + id="decimal_4", + ), + # Due to python DB driver roundtrip MySQL timestamp and datetime values + # are printed with 6 digits after decimal point, so to simplify tests a bit, + # we only validate precision of 0 and 6. + pytest.param( + "timestamp_default", + "timestamp", + "DateTime", + timestamp_values, + "decimal,datetime64", + id="timestamp_default", + ), + pytest.param( + "timestamp_6", + "timestamp(6)", + "DateTime64(6)", + timestamp_values, + "decimal,datetime64", + id="timestamp_6", + ), + pytest.param( + "datetime_default", + "DATETIME NOT NULL", + "DateTime64(0)", + timestamp_values, + "decimal,datetime64", + id="datetime_default", + ), + pytest.param( + "datetime_6", + "DATETIME(6) NOT NULL", + "DateTime64(6)", + timestamp_values, + "decimal,datetime64", + id="datetime_6_1", + ), + # right now precision bigger than 39 is not supported by ClickHouse's Decimal, hence fall back to String + pytest.param( + "decimal_40_6", + "decimal(40, 6) NOT NULL", + "String", + decimal_values, + "decimal,datetime64", + id="decimal_40_6", + ), + pytest.param( + "decimal_18_6", + "decimal(18, 6) NOT NULL", + "String", + decimal_values, + "datetime64", + id="decimal_18_6_1", + ), + pytest.param( + "decimal_18_6", + "decimal(18, 6) NOT NULL", + "String", + decimal_values, + "", + id="decimal_18_6_2", + ), + pytest.param( + "datetime_6", + "DATETIME(6) NOT NULL", + "DateTime", + timestamp_values_no_subsecond, + "decimal", + id="datetime_6_2", + ), + pytest.param( + "datetime_6", + "DATETIME(6) NOT NULL", + "DateTime", + timestamp_values_no_subsecond, + "", + id="datetime_6_3", + ), + ], +) +def test_mysql_types( + started_cluster, + case_name, + mysql_type, + expected_ch_type, + mysql_values, + setting_mysql_datatypes_support_level, +): + """Verify that values written to MySQL can be read on ClickHouse side via DB engine MySQL, or Table engine MySQL, or mysql() table function. Make sure that type is converted properly and values match exactly. """ substitutes = dict( - mysql_db='decimal_support', + mysql_db="decimal_support", table_name=case_name, mysql_type=mysql_type, - mysql_values=', '.join('({})'.format(x) for x in mysql_values), - ch_mysql_db='mysql_db', - ch_mysql_table='mysql_table_engine_' + case_name, + mysql_values=", ".join("({})".format(x) for x in mysql_values), + ch_mysql_db="mysql_db", + ch_mysql_table="mysql_table_engine_" + case_name, expected_ch_type=expected_ch_type, ) clickhouse_query_settings = dict( mysql_datatypes_support_level=setting_mysql_datatypes_support_level, - output_format_decimal_trailing_zeros=1 + output_format_decimal_trailing_zeros=1, ) def execute_query(node, query, **kwargs): def do_execute(query): query = Template(query).safe_substitute(substitutes) res = node.query(query, **kwargs) - return res if isinstance(res, int) else res.rstrip('\n\r') + return res if isinstance(res, int) else res.rstrip("\n\r") if isinstance(query, (str, bytes)): return do_execute(query) else: return [do_execute(q) for q in query] - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, port=started_cluster.mysql_port)) as mysql_node: - execute_query(mysql_node, [ - "DROP DATABASE IF EXISTS ${mysql_db}", - "CREATE DATABASE ${mysql_db} DEFAULT CHARACTER SET 'utf8'", - "CREATE TABLE `${mysql_db}`.`${table_name}` (value ${mysql_type})", - "INSERT INTO `${mysql_db}`.`${table_name}` (value) VALUES ${mysql_values}", - "SELECT * FROM `${mysql_db}`.`${table_name}`", - "FLUSH TABLES" - ]) + with contextlib.closing( + MySQLNodeInstance( + "root", + "clickhouse", + started_cluster.mysql_ip, + port=started_cluster.mysql_port, + ) + ) as mysql_node: + execute_query( + mysql_node, + [ + "DROP DATABASE IF EXISTS ${mysql_db}", + "CREATE DATABASE ${mysql_db} DEFAULT CHARACTER SET 'utf8'", + "CREATE TABLE `${mysql_db}`.`${table_name}` (value ${mysql_type})", + "INSERT INTO `${mysql_db}`.`${table_name}` (value) VALUES ${mysql_values}", + "SELECT * FROM `${mysql_db}`.`${table_name}`", + "FLUSH TABLES", + ], + ) - assert execute_query(mysql_node, "SELECT COUNT(*) FROM ${mysql_db}.${table_name}") \ - == \ - "{}".format(len(mysql_values)) + assert execute_query( + mysql_node, "SELECT COUNT(*) FROM ${mysql_db}.${table_name}" + ) == "{}".format(len(mysql_values)) # MySQL TABLE ENGINE - execute_query(clickhouse_node, [ - "DROP TABLE IF EXISTS ${ch_mysql_table};", - "CREATE TABLE ${ch_mysql_table} (value ${expected_ch_type}) ENGINE = MySQL('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')", - ], settings=clickhouse_query_settings) + execute_query( + clickhouse_node, + [ + "DROP TABLE IF EXISTS ${ch_mysql_table};", + "CREATE TABLE ${ch_mysql_table} (value ${expected_ch_type}) ENGINE = MySQL('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')", + ], + settings=clickhouse_query_settings, + ) # Validate type - assert \ - execute_query(clickhouse_node, "SELECT toTypeName(value) FROM ${ch_mysql_table} LIMIT 1", - settings=clickhouse_query_settings) \ - == \ - expected_ch_type + assert ( + execute_query( + clickhouse_node, + "SELECT toTypeName(value) FROM ${ch_mysql_table} LIMIT 1", + settings=clickhouse_query_settings, + ) + == expected_ch_type + ) # Validate values - assert \ - execute_query(clickhouse_node, "SELECT value FROM ${ch_mysql_table}", - settings=clickhouse_query_settings) \ - == \ - execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") + assert execute_query( + clickhouse_node, + "SELECT value FROM ${ch_mysql_table}", + settings=clickhouse_query_settings, + ) == execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") # MySQL DATABASE ENGINE - execute_query(clickhouse_node, [ - "DROP DATABASE IF EXISTS ${ch_mysql_db}", - "CREATE DATABASE ${ch_mysql_db} ENGINE = MySQL('mysql57:3306', '${mysql_db}', 'root', 'clickhouse')" - ], settings=clickhouse_query_settings) + execute_query( + clickhouse_node, + [ + "DROP DATABASE IF EXISTS ${ch_mysql_db}", + "CREATE DATABASE ${ch_mysql_db} ENGINE = MySQL('mysql57:3306', '${mysql_db}', 'root', 'clickhouse')", + ], + settings=clickhouse_query_settings, + ) # Validate type - assert \ - execute_query(clickhouse_node, "SELECT toTypeName(value) FROM ${ch_mysql_db}.${table_name} LIMIT 1", - settings=clickhouse_query_settings) \ - == \ - expected_ch_type + assert ( + execute_query( + clickhouse_node, + "SELECT toTypeName(value) FROM ${ch_mysql_db}.${table_name} LIMIT 1", + settings=clickhouse_query_settings, + ) + == expected_ch_type + ) # Validate values - assert \ - execute_query(clickhouse_node, "SELECT value FROM ${ch_mysql_db}.${table_name}", - settings=clickhouse_query_settings) \ - == \ - execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") + assert execute_query( + clickhouse_node, + "SELECT value FROM ${ch_mysql_db}.${table_name}", + settings=clickhouse_query_settings, + ) == execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") # MySQL TABLE FUNCTION # Validate type - assert \ - execute_query(clickhouse_node, - "SELECT toTypeName(value) FROM mysql('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse') LIMIT 1", - settings=clickhouse_query_settings) \ - == \ - expected_ch_type + assert ( + execute_query( + clickhouse_node, + "SELECT toTypeName(value) FROM mysql('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse') LIMIT 1", + settings=clickhouse_query_settings, + ) + == expected_ch_type + ) # Validate values - assert \ - execute_query(mysql_node, "SELECT value FROM ${mysql_db}.${table_name}") \ - == \ - execute_query(clickhouse_node, - "SELECT value FROM mysql('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')", - settings=clickhouse_query_settings) + assert execute_query( + mysql_node, "SELECT value FROM ${mysql_db}.${table_name}" + ) == execute_query( + clickhouse_node, + "SELECT value FROM mysql('mysql57:3306', '${mysql_db}', '${table_name}', 'root', 'clickhouse')", + settings=clickhouse_query_settings, + ) def test_predefined_connection_configuration(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: mysql_node.query("DROP DATABASE IF EXISTS test_database") mysql_node.query("CREATE DATABASE test_database DEFAULT CHARACTER SET 'utf8'") - mysql_node.query('CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;') + mysql_node.query( + "CREATE TABLE `test_database`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) clickhouse_node.query("DROP DATABASE IF EXISTS test_database") clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL(mysql1)") - clickhouse_node.query("INSERT INTO `test_database`.`test_table` select number from numbers(100)") - assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '100' + clickhouse_node.query( + "INSERT INTO `test_database`.`test_table` select number from numbers(100)" + ) + assert ( + clickhouse_node.query( + "SELECT count() FROM `test_database`.`test_table`" + ).rstrip() + == "100" + ) clickhouse_node.query("DROP DATABASE test_database") - clickhouse_node.query_and_get_error("CREATE DATABASE test_database ENGINE = MySQL(mysql2)") - clickhouse_node.query_and_get_error("CREATE DATABASE test_database ENGINE = MySQL(unknown_collection)") - clickhouse_node.query_and_get_error("CREATE DATABASE test_database ENGINE = MySQL(mysql1, 1)") + clickhouse_node.query_and_get_error( + "CREATE DATABASE test_database ENGINE = MySQL(mysql2)" + ) + clickhouse_node.query_and_get_error( + "CREATE DATABASE test_database ENGINE = MySQL(unknown_collection)" + ) + clickhouse_node.query_and_get_error( + "CREATE DATABASE test_database ENGINE = MySQL(mysql1, 1)" + ) - clickhouse_node.query("CREATE DATABASE test_database ENGINE = MySQL(mysql1, port=3306)") - assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '100' + clickhouse_node.query( + "CREATE DATABASE test_database ENGINE = MySQL(mysql1, port=3306)" + ) + assert ( + clickhouse_node.query( + "SELECT count() FROM `test_database`.`test_table`" + ).rstrip() + == "100" + ) def test_restart_server(started_cluster): - with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', started_cluster.mysql_ip, started_cluster.mysql_port)) as mysql_node: + with contextlib.closing( + MySQLNodeInstance( + "root", "clickhouse", started_cluster.mysql_ip, started_cluster.mysql_port + ) + ) as mysql_node: mysql_node.query("DROP DATABASE IF EXISTS test_restart") clickhouse_node.query("DROP DATABASE IF EXISTS test_restart") - clickhouse_node.query_and_get_error("CREATE DATABASE test_restart ENGINE = MySQL('mysql57:3306', 'test_restart', 'root', 'clickhouse')") - assert 'test_restart' not in clickhouse_node.query('SHOW DATABASES') + clickhouse_node.query_and_get_error( + "CREATE DATABASE test_restart ENGINE = MySQL('mysql57:3306', 'test_restart', 'root', 'clickhouse')" + ) + assert "test_restart" not in clickhouse_node.query("SHOW DATABASES") mysql_node.query("CREATE DATABASE test_restart DEFAULT CHARACTER SET 'utf8'") - mysql_node.query("CREATE TABLE `test_restart`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;") - clickhouse_node.query("CREATE DATABASE test_restart ENGINE = MySQL('mysql57:3306', 'test_restart', 'root', 'clickhouse')") + mysql_node.query( + "CREATE TABLE `test_restart`.`test_table` ( `id` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB;" + ) + clickhouse_node.query( + "CREATE DATABASE test_restart ENGINE = MySQL('mysql57:3306', 'test_restart', 'root', 'clickhouse')" + ) - assert 'test_restart' in clickhouse_node.query('SHOW DATABASES') - assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_restart') + assert "test_restart" in clickhouse_node.query("SHOW DATABASES") + assert "test_table" in clickhouse_node.query("SHOW TABLES FROM test_restart") with PartitionManager() as pm: - pm.partition_instances(clickhouse_node, mysql_node, action='REJECT --reject-with tcp-reset') + pm.partition_instances( + clickhouse_node, mysql_node, action="REJECT --reject-with tcp-reset" + ) clickhouse_node.restart_clickhouse() - clickhouse_node.query_and_get_error('SHOW TABLES FROM test_restart') - assert 'test_table' in clickhouse_node.query('SHOW TABLES FROM test_restart') + clickhouse_node.query_and_get_error("SHOW TABLES FROM test_restart") + assert "test_table" in clickhouse_node.query("SHOW TABLES FROM test_restart") diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index 0b3f6ea95af..78049e0f123 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -16,12 +16,23 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/ssl_conf.xml", "configs/mysql.xml", "configs/dhparam.pem", - "configs/server.crt", "configs/server.key"], - user_configs=["configs/users.xml"], env_variables={'UBSAN_OPTIONS': 'print_stacktrace=1'}, with_mysql_client=True) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/ssl_conf.xml", + "configs/mysql.xml", + "configs/dhparam.pem", + "configs/server.crt", + "configs/server.key", + ], + user_configs=["configs/users.xml"], + env_variables={"UBSAN_OPTIONS": "print_stacktrace=1"}, + with_mysql_client=True, +) server_port = 9001 + @pytest.fixture(scope="module") def started_cluster(): cluster.start() @@ -31,77 +42,177 @@ def started_cluster(): cluster.shutdown() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def golang_container(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_golang_client.yml') + docker_compose = os.path.join( + DOCKER_COMPOSE_PATH, "docker_compose_mysql_golang_client.yml" + ) run_and_check( - ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) - yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=cluster.docker_api_version, timeout=600).containers.get(cluster.project_name + '_golang1_1') + [ + "docker-compose", + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--no-build", + ] + ) + yield docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=cluster.docker_api_version, + timeout=600, + ).containers.get(cluster.project_name + "_golang1_1") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def php_container(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_php_client.yml') + docker_compose = os.path.join( + DOCKER_COMPOSE_PATH, "docker_compose_mysql_php_client.yml" + ) run_and_check( - ['docker-compose', '--env-file', cluster.instances["node"].env_file, '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) - yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=cluster.docker_api_version, timeout=600).containers.get(cluster.project_name + '_php1_1') + [ + "docker-compose", + "--env-file", + cluster.instances["node"].env_file, + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--no-build", + ] + ) + yield docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=cluster.docker_api_version, + timeout=600, + ).containers.get(cluster.project_name + "_php1_1") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def nodejs_container(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_js_client.yml') + docker_compose = os.path.join( + DOCKER_COMPOSE_PATH, "docker_compose_mysql_js_client.yml" + ) run_and_check( - ['docker-compose', '--env-file', cluster.instances["node"].env_file, '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) - yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=cluster.docker_api_version, timeout=600).containers.get(cluster.project_name + '_mysqljs1_1') + [ + "docker-compose", + "--env-file", + cluster.instances["node"].env_file, + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--no-build", + ] + ) + yield docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=cluster.docker_api_version, + timeout=600, + ).containers.get(cluster.project_name + "_mysqljs1_1") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def java_container(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_mysql_java_client.yml') + docker_compose = os.path.join( + DOCKER_COMPOSE_PATH, "docker_compose_mysql_java_client.yml" + ) run_and_check( - ['docker-compose', '--env-file', cluster.instances["node"].env_file, '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--no-build']) - yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=cluster.docker_api_version, timeout=600).containers.get(cluster.project_name + '_java1_1') + [ + "docker-compose", + "--env-file", + cluster.instances["node"].env_file, + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--no-build", + ] + ) + yield docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=cluster.docker_api_version, + timeout=600, + ).containers.get(cluster.project_name + "_java1_1") def test_mysql_client(started_cluster): # type: (Container, str) -> None - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u user_with_double_sha1 --password=abacaba -e "SELECT 1;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) logging.debug(f"test_mysql_client code:{code} stdout:{stdout}, stderr:{stderr}") - assert stdout.decode() == '\n'.join(['1', '1', '']) + assert stdout.decode() == "\n".join(["1", "1", ""]) - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "SELECT 1 as a;" -e "SELECT 'тест' as b;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) - assert stdout.decode() == '\n'.join(['a', '1', 'b', 'тест', '']) + assert stdout.decode() == "\n".join(["a", "1", "b", "тест", ""]) - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=abc -e "select 1 as a;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) - assert stderr.decode() == 'mysql: [Warning] Using a password on the command line interface can be insecure.\n' \ - 'ERROR 516 (00000): default: Authentication failed: password is incorrect or there is no user with such name\n' + assert ( + stderr.decode() + == "mysql: [Warning] Using a password on the command line interface can be insecure.\n" + "ERROR 516 (00000): default: Authentication failed: password is incorrect or there is no user with such name\n" + ) - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "use system;" -e "select count(*) from (select name from tables limit 1);" -e "use system2;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) - assert stdout.decode() == 'count()\n1\n' - expected_msg = '\n'.join([ - "mysql: [Warning] Using a password on the command line interface can be insecure.", - "ERROR 81 (00000) at line 1: Code: 81. DB::Exception: Database system2 doesn't exist", - ]) - assert stderr[:len(expected_msg)].decode() == expected_msg + assert stdout.decode() == "count()\n1\n" + expected_msg = "\n".join( + [ + "mysql: [Warning] Using a password on the command line interface can be insecure.", + "ERROR 81 (00000) at line 1: Code: 81. DB::Exception: Database system2 doesn't exist", + ] + ) + assert stderr[: len(expected_msg)].decode() == expected_msg - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "CREATE DATABASE x;" -e "USE x;" @@ -113,134 +224,223 @@ def test_mysql_client(started_cluster): -e "CREATE TEMPORARY TABLE tmp (tmp_column UInt32);" -e "INSERT INTO tmp VALUES (0), (1);" -e "SELECT * FROM tmp ORDER BY tmp_column;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) - assert stdout.decode() == '\n'.join(['column', '0', '0', '1', '1', '5', '5', 'tmp_column', '0', '1', '']) + assert stdout.decode() == "\n".join( + ["column", "0", "0", "1", "1", "5", "5", "tmp_column", "0", "1", ""] + ) def test_mysql_client_exception(started_cluster): # Poco exception. - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "CREATE TABLE default.t1_remote_mysql AS mysql('127.0.0.1:10086','default','t1_local','default','');" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) - expected_msg = '\n'.join([ - "mysql: [Warning] Using a password on the command line interface can be insecure.", - "ERROR 1000 (00000) at line 1: Poco::Exception. Code: 1000, e.code() = 0, Exception: Connections to all replicas failed: default@127.0.0.1:10086 as user default", - ]) - assert stderr[:len(expected_msg)].decode() == expected_msg + expected_msg = "\n".join( + [ + "mysql: [Warning] Using a password on the command line interface can be insecure.", + "ERROR 1000 (00000) at line 1: Poco::Exception. Code: 1000, e.code() = 0, Exception: Connections to all replicas failed: default@127.0.0.1:10086 as user default", + ] + ) + assert stderr[: len(expected_msg)].decode() == expected_msg def test_mysql_affected_rows(started_cluster): - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "CREATE TABLE IF NOT EXISTS default.t1 (n UInt64) ENGINE MergeTree() ORDER BY tuple();" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql -vvv --protocol tcp -h {host} -P {port} default -u default --password=123 -e "INSERT INTO default.t1(n) VALUES(1);" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert "1 row affected" in stdout.decode() - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql -vvv --protocol tcp -h {host} -P {port} default -u default --password=123 -e "INSERT INTO default.t1(n) SELECT * FROM numbers(1000)" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert "1000 rows affected" in stdout.decode() - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "DROP TABLE default.t1;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 def test_mysql_replacement_query(started_cluster): # SHOW TABLE STATUS LIKE. - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "show table status like 'xx';" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 # SHOW VARIABLES. - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "show variables;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 # KILL QUERY. - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "kill query 0;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "kill query where query_id='mysql:0';" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 # SELECT DATABASE(). - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "select database();" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'DATABASE()\ndefault\n' + assert stdout.decode() == "DATABASE()\ndefault\n" - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "select DATABASE();" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'DATABASE()\ndefault\n' + assert stdout.decode() == "DATABASE()\ndefault\n" def test_mysql_select_user(started_cluster): - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "select user();" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'currentUser()\ndefault\n' + assert stdout.decode() == "currentUser()\ndefault\n" + def test_mysql_explain(started_cluster): # EXPLAIN SELECT 1 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "EXPLAIN SELECT 1;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 # EXPLAIN AST SELECT 1 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "EXPLAIN AST SELECT 1;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 # EXPLAIN PLAN SELECT 1 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "EXPLAIN PLAN SELECT 1;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 # EXPLAIN PIPELINE graph=1 SELECT 1 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e "EXPLAIN PIPELINE graph=1 SELECT 1;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 @@ -248,25 +448,40 @@ def test_mysql_federated(started_cluster): # For some reason it occasionally fails without retries. retries = 100 for try_num in range(retries): - node.query('''DROP DATABASE IF EXISTS mysql_federated''', settings={"password": "123"}) - node.query('''CREATE DATABASE mysql_federated''', settings={"password": "123"}) - node.query('''CREATE TABLE mysql_federated.test (col UInt32) ENGINE = Log''', settings={"password": "123"}) - node.query('''INSERT INTO mysql_federated.test VALUES (0), (1), (5)''', settings={"password": "123"}) + node.query( + """DROP DATABASE IF EXISTS mysql_federated""", settings={"password": "123"} + ) + node.query("""CREATE DATABASE mysql_federated""", settings={"password": "123"}) + node.query( + """CREATE TABLE mysql_federated.test (col UInt32) ENGINE = Log""", + settings={"password": "123"}, + ) + node.query( + """INSERT INTO mysql_federated.test VALUES (0), (1), (5)""", + settings={"password": "123"}, + ) def check_retryable_error_in_stderr(stderr): stderr = stderr.decode() - return ("Can't connect to local MySQL server through socket" in stderr - or "MySQL server has gone away" in stderr - or "Server shutdown in progress" in stderr) + return ( + "Can't connect to local MySQL server through socket" in stderr + or "MySQL server has gone away" in stderr + or "Server shutdown in progress" in stderr + ) - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql -e "DROP SERVER IF EXISTS clickhouse;" -e "CREATE SERVER clickhouse FOREIGN DATA WRAPPER mysql OPTIONS (USER 'default', PASSWORD '123', HOST '{host}', PORT {port}, DATABASE 'mysql_federated');" -e "DROP DATABASE IF EXISTS mysql_federated;" -e "CREATE DATABASE mysql_federated;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) if code != 0: print(("stdout", stdout)) @@ -276,11 +491,16 @@ def test_mysql_federated(started_cluster): continue assert code == 0 - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql -e "CREATE TABLE mysql_federated.test(`col` int UNSIGNED) ENGINE=FEDERATED CONNECTION='clickhouse';" -e "SELECT * FROM mysql_federated.test ORDER BY col;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) if code != 0: print(("stdout", stdout)) @@ -290,13 +510,18 @@ def test_mysql_federated(started_cluster): continue assert code == 0 - assert stdout.decode() == '\n'.join(['col', '0', '1', '5', '']) + assert stdout.decode() == "\n".join(["col", "0", "1", "5", ""]) - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql -e "INSERT INTO mysql_federated.test VALUES (0), (1), (5);" -e "SELECT * FROM mysql_federated.test ORDER BY col;" - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) if code != 0: print(("stdout", stdout)) @@ -306,11 +531,12 @@ def test_mysql_federated(started_cluster): continue assert code == 0 - assert stdout.decode() == '\n'.join(['col', '0', '0', '1', '1', '5', '5', '']) + assert stdout.decode() == "\n".join(["col", "0", "0", "1", "1", "5", "5", ""]) def test_mysql_set_variables(started_cluster): - code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run(''' + code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( + """ mysql --protocol tcp -h {host} -P {port} default -u default --password=123 -e " @@ -322,82 +548,118 @@ def test_mysql_set_variables(started_cluster): SET @@wait_timeout = 2147483; SET SESSION TRANSACTION ISOLATION LEVEL READ; " - '''.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + """.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - def test_python_client(started_cluster): - client = pymysql.connections.Connection(host=started_cluster.get_instance_ip('node'), user='user_with_double_sha1', password='abacaba', - database='default', port=server_port) + client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="user_with_double_sha1", + password="abacaba", + database="default", + port=server_port, + ) with pytest.raises(pymysql.InternalError) as exc_info: - client.query('select name from tables') + client.query("select name from tables") - assert exc_info.value.args[1].startswith("Code: 60. DB::Exception: Table default.tables doesn't exist"), exc_info.value.args[1] + assert exc_info.value.args[1].startswith( + "Code: 60. DB::Exception: Table default.tables doesn't exist" + ), exc_info.value.args[1] cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute("select 1 as a, 'тест' as b") - assert cursor.fetchall() == [{'a': 1, 'b': 'тест'}] + assert cursor.fetchall() == [{"a": 1, "b": "тест"}] with pytest.raises(pymysql.InternalError) as exc_info: - pymysql.connections.Connection(host=started_cluster.get_instance_ip('node'), user='default', password='abacab', database='default', - port=server_port) + pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="abacab", + database="default", + port=server_port, + ) assert exc_info.value.args == ( - 516, 'default: Authentication failed: password is incorrect or there is no user with such name') + 516, + "default: Authentication failed: password is incorrect or there is no user with such name", + ) - client = pymysql.connections.Connection(host=started_cluster.get_instance_ip('node'), user='default', password='123', database='default', - port=server_port) + client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="123", + database="default", + port=server_port, + ) with pytest.raises(pymysql.InternalError) as exc_info: - client.query('select name from tables') + client.query("select name from tables") - assert exc_info.value.args[1].startswith("Code: 60. DB::Exception: Table default.tables doesn't exist"), exc_info.value.args[1] + assert exc_info.value.args[1].startswith( + "Code: 60. DB::Exception: Table default.tables doesn't exist" + ), exc_info.value.args[1] cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute("select 1 as a, 'тест' as b") - assert cursor.fetchall() == [{'a': 1, 'b': 'тест'}] + assert cursor.fetchall() == [{"a": 1, "b": "тест"}] - client.select_db('system') + client.select_db("system") with pytest.raises(pymysql.InternalError) as exc_info: - client.select_db('system2') + client.select_db("system2") - assert exc_info.value.args[1].startswith("Code: 81. DB::Exception: Database system2 doesn't exist"), exc_info.value.args[1] + assert exc_info.value.args[1].startswith( + "Code: 81. DB::Exception: Database system2 doesn't exist" + ), exc_info.value.args[1] cursor = client.cursor(pymysql.cursors.DictCursor) - cursor.execute('CREATE DATABASE x') - client.select_db('x') + cursor.execute("CREATE DATABASE x") + client.select_db("x") cursor.execute("CREATE TABLE table1 (a UInt32) ENGINE = Memory") cursor.execute("INSERT INTO table1 VALUES (1), (3)") cursor.execute("INSERT INTO table1 VALUES (1), (4)") cursor.execute("SELECT * FROM table1 ORDER BY a") - assert cursor.fetchall() == [{'a': 1}, {'a': 1}, {'a': 3}, {'a': 4}] + assert cursor.fetchall() == [{"a": 1}, {"a": 1}, {"a": 3}, {"a": 4}] def test_golang_client(started_cluster, golang_container): # type: (str, Container) -> None - with open(os.path.join(SCRIPT_DIR, 'golang.reference'), 'rb') as fp: + with open(os.path.join(SCRIPT_DIR, "golang.reference"), "rb") as fp: reference = fp.read() code, (stdout, stderr) = golang_container.exec_run( - './main --host {host} --port {port} --user default --password 123 --database ' - 'abc'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "./main --host {host} --port {port} --user default --password 123 --database " + "abc".format(host=started_cluster.get_instance_ip("node"), port=server_port), + demux=True, + ) assert code == 1 assert stderr.decode() == "Error 81: Database abc doesn't exist\n" code, (stdout, stderr) = golang_container.exec_run( - './main --host {host} --port {port} --user default --password 123 --database ' - 'default'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "./main --host {host} --port {port} --user default --password 123 --database " + "default".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert stdout == reference code, (stdout, stderr) = golang_container.exec_run( - './main --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' - 'default'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "./main --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database " + "default".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert stdout == reference @@ -405,86 +667,135 @@ def test_golang_client(started_cluster, golang_container): def test_php_client(started_cluster, php_container): # type: (str, Container) -> None code, (stdout, stderr) = php_container.exec_run( - 'php -f test.php {host} {port} default 123'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "php -f test.php {host} {port} default 123".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'tables\ntables\n' + assert stdout.decode() == "tables\ntables\n" code, (stdout, stderr) = php_container.exec_run( - 'php -f test_ssl.php {host} {port} default 123'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "php -f test_ssl.php {host} {port} default 123".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'tables\ntables\n' + assert stdout.decode() == "tables\ntables\n" code, (stdout, stderr) = php_container.exec_run( - 'php -f test.php {host} {port} user_with_double_sha1 abacaba'.format(host=started_cluster.get_instance_ip('node'), port=server_port), - demux=True) + "php -f test.php {host} {port} user_with_double_sha1 abacaba".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'tables\ntables\n' + assert stdout.decode() == "tables\ntables\n" code, (stdout, stderr) = php_container.exec_run( - 'php -f test_ssl.php {host} {port} user_with_double_sha1 abacaba'.format(host=started_cluster.get_instance_ip('node'), port=server_port), - demux=True) + "php -f test_ssl.php {host} {port} user_with_double_sha1 abacaba".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 - assert stdout.decode() == 'tables\ntables\n' + assert stdout.decode() == "tables\ntables\n" def test_mysqljs_client(started_cluster, nodejs_container): code, (_, stderr) = nodejs_container.exec_run( - 'node test.js {host} {port} user_with_sha256 abacaba'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "node test.js {host} {port} user_with_sha256 abacaba".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 1 - assert 'MySQL is requesting the sha256_password authentication method, which is not supported.' in stderr.decode() + assert ( + "MySQL is requesting the sha256_password authentication method, which is not supported." + in stderr.decode() + ) code, (_, stderr) = nodejs_container.exec_run( - 'node test.js {host} {port} user_with_empty_password ""'.format(host=started_cluster.get_instance_ip('node'), port=server_port), - demux=True) + 'node test.js {host} {port} user_with_empty_password ""'.format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 code, (_, _) = nodejs_container.exec_run( - 'node test.js {host} {port} user_with_double_sha1 abacaba'.format(host=started_cluster.get_instance_ip('node'), port=server_port), - demux=True) + "node test.js {host} {port} user_with_double_sha1 abacaba".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 code, (_, _) = nodejs_container.exec_run( - 'node test.js {host} {port} user_with_empty_password 123'.format(host=started_cluster.get_instance_ip('node'), port=server_port), - demux=True) + "node test.js {host} {port} user_with_empty_password 123".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 1 def test_java_client(started_cluster, java_container): # type: (str, Container) -> None - with open(os.path.join(SCRIPT_DIR, 'java.reference')) as fp: + with open(os.path.join(SCRIPT_DIR, "java.reference")) as fp: reference = fp.read() # database not exists exception. code, (stdout, stderr) = java_container.exec_run( - 'java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' - 'abc'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database " + "abc".format(host=started_cluster.get_instance_ip("node"), port=server_port), + demux=True, + ) assert code == 1 # empty password passed. code, (stdout, stderr) = java_container.exec_run( - 'java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database ' - 'default'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "java JavaConnectorTest --host {host} --port {port} --user user_with_empty_password --database " + "default".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert stdout.decode() == reference # non-empty password passed. code, (stdout, stderr) = java_container.exec_run( - 'java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' - 'default'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database " + "default".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert stdout.decode() == reference # double-sha1 password passed. code, (stdout, stderr) = java_container.exec_run( - 'java JavaConnectorTest --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database ' - 'default'.format(host=started_cluster.get_instance_ip('node'), port=server_port), demux=True) + "java JavaConnectorTest --host {host} --port {port} --user user_with_double_sha1 --password abacaba --database " + "default".format( + host=started_cluster.get_instance_ip("node"), port=server_port + ), + demux=True, + ) assert code == 0 assert stdout.decode() == reference def test_types(started_cluster): - client = pymysql.connections.Connection(host=started_cluster.get_instance_ip('node'), user='default', password='123', database='default', - port=server_port) + client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="123", + database="default", + port=server_port, + ) cursor = client.cursor(pymysql.cursors.DictCursor) cursor.execute( @@ -511,24 +822,24 @@ def test_types(started_cluster): result = cursor.fetchall()[0] expected = [ - ('Int8_column', -2 ** 7), - ('UInt8_column', 2 ** 8 - 1), - ('Int16_column', -2 ** 15), - ('UInt16_column', 2 ** 16 - 1), - ('Int32_column', -2 ** 31), - ('UInt32_column', 2 ** 32 - 1), - ('Int64_column', -2 ** 63), - ('UInt64_column', 2 ** 64 - 1), - ('String_column', 'тест'), - ('FixedString_column', 'тест'), - ('Float32_column', 1.5), - ('Float64_column', 1.5), - ('Float32_NaN_column', float('nan')), - ('Float64_Inf_column', float('-inf')), - ('Date_column', datetime.date(2019, 12, 8)), - ('Date_min_column', datetime.date(1970, 1, 1)), - ('Date_after_min_column', datetime.date(1970, 1, 2)), - ('DateTime_column', datetime.datetime(2019, 12, 8, 8, 24, 3)), + ("Int8_column", -(2**7)), + ("UInt8_column", 2**8 - 1), + ("Int16_column", -(2**15)), + ("UInt16_column", 2**16 - 1), + ("Int32_column", -(2**31)), + ("UInt32_column", 2**32 - 1), + ("Int64_column", -(2**63)), + ("UInt64_column", 2**64 - 1), + ("String_column", "тест"), + ("FixedString_column", "тест"), + ("Float32_column", 1.5), + ("Float64_column", 1.5), + ("Float32_NaN_column", float("nan")), + ("Float64_Inf_column", float("-inf")), + ("Date_column", datetime.date(2019, 12, 8)), + ("Date_min_column", datetime.date(1970, 1, 1)), + ("Date_after_min_column", datetime.date(1970, 1, 2)), + ("DateTime_column", datetime.datetime(2019, 12, 8, 8, 24, 3)), ] for key, value in expected: diff --git a/tests/integration/test_nlp/test.py b/tests/integration/test_nlp/test.py index 24935153608..e15c9ecfaa6 100644 --- a/tests/integration/test_nlp/test.py +++ b/tests/integration/test_nlp/test.py @@ -10,38 +10,140 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=['configs/dicts_config.xml']) +instance = cluster.add_instance("instance", main_configs=["configs/dicts_config.xml"]) + def copy_file_to_container(local_path, dist_path, container_id): - os.system("docker cp {local} {cont_id}:{dist}".format(local=local_path, cont_id=container_id, dist=dist_path)) + os.system( + "docker cp {local} {cont_id}:{dist}".format( + local=local_path, cont_id=container_id, dist=dist_path + ) + ) + @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() - copy_file_to_container(os.path.join(SCRIPT_DIR, 'dictionaries/.'), '/etc/clickhouse-server/dictionaries', instance.docker_id) + copy_file_to_container( + os.path.join(SCRIPT_DIR, "dictionaries/."), + "/etc/clickhouse-server/dictionaries", + instance.docker_id, + ) yield cluster finally: cluster.shutdown() + def test_lemmatize(start_cluster): - assert instance.query("SELECT lemmatize('en', 'wolves')", settings={"allow_experimental_nlp_functions": 1}) == "wolf\n" - assert instance.query("SELECT lemmatize('en', 'dogs')", settings={"allow_experimental_nlp_functions": 1}) == "dog\n" - assert instance.query("SELECT lemmatize('en', 'looking')", settings={"allow_experimental_nlp_functions": 1}) == "look\n" - assert instance.query("SELECT lemmatize('en', 'took')", settings={"allow_experimental_nlp_functions": 1}) == "take\n" - assert instance.query("SELECT lemmatize('en', 'imported')", settings={"allow_experimental_nlp_functions": 1}) == "import\n" - assert instance.query("SELECT lemmatize('en', 'tokenized')", settings={"allow_experimental_nlp_functions": 1}) == "tokenize\n" - assert instance.query("SELECT lemmatize('en', 'flown')", settings={"allow_experimental_nlp_functions": 1}) == "fly\n" + assert ( + instance.query( + "SELECT lemmatize('en', 'wolves')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "wolf\n" + ) + assert ( + instance.query( + "SELECT lemmatize('en', 'dogs')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "dog\n" + ) + assert ( + instance.query( + "SELECT lemmatize('en', 'looking')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "look\n" + ) + assert ( + instance.query( + "SELECT lemmatize('en', 'took')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "take\n" + ) + assert ( + instance.query( + "SELECT lemmatize('en', 'imported')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "import\n" + ) + assert ( + instance.query( + "SELECT lemmatize('en', 'tokenized')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "tokenize\n" + ) + assert ( + instance.query( + "SELECT lemmatize('en', 'flown')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "fly\n" + ) + def test_synonyms_extensions(start_cluster): - assert instance.query("SELECT synonyms('en', 'crucial')", settings={"allow_experimental_nlp_functions": 1}) == "['important','big','critical','crucial','essential']\n" - assert instance.query("SELECT synonyms('en', 'cheerful')", settings={"allow_experimental_nlp_functions": 1}) == "['happy','cheerful','delighted','ecstatic']\n" - assert instance.query("SELECT synonyms('en', 'yet')", settings={"allow_experimental_nlp_functions": 1}) == "['however','nonetheless','but','yet']\n" - assert instance.query("SELECT synonyms('en', 'quiz')", settings={"allow_experimental_nlp_functions": 1}) == "['quiz','query','check','exam']\n" - - assert instance.query("SELECT synonyms('ru', 'главный')", settings={"allow_experimental_nlp_functions": 1}) == "['важный','большой','высокий','хороший','главный']\n" - assert instance.query("SELECT synonyms('ru', 'веселый')", settings={"allow_experimental_nlp_functions": 1}) == "['веселый','счастливый','живой','яркий','смешной']\n" - assert instance.query("SELECT synonyms('ru', 'правда')", settings={"allow_experimental_nlp_functions": 1}) == "['хотя','однако','но','правда']\n" - assert instance.query("SELECT synonyms('ru', 'экзамен')", settings={"allow_experimental_nlp_functions": 1}) == "['экзамен','испытание','проверка']\n" + assert ( + instance.query( + "SELECT synonyms('en', 'crucial')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['important','big','critical','crucial','essential']\n" + ) + assert ( + instance.query( + "SELECT synonyms('en', 'cheerful')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['happy','cheerful','delighted','ecstatic']\n" + ) + assert ( + instance.query( + "SELECT synonyms('en', 'yet')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['however','nonetheless','but','yet']\n" + ) + assert ( + instance.query( + "SELECT synonyms('en', 'quiz')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['quiz','query','check','exam']\n" + ) + + assert ( + instance.query( + "SELECT synonyms('ru', 'главный')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['важный','большой','высокий','хороший','главный']\n" + ) + assert ( + instance.query( + "SELECT synonyms('ru', 'веселый')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['веселый','счастливый','живой','яркий','смешной']\n" + ) + assert ( + instance.query( + "SELECT synonyms('ru', 'правда')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['хотя','однако','но','правда']\n" + ) + assert ( + instance.query( + "SELECT synonyms('ru', 'экзамен')", + settings={"allow_experimental_nlp_functions": 1}, + ) + == "['экзамен','испытание','проверка']\n" + ) diff --git a/tests/integration/test_no_local_metadata_node/test.py b/tests/integration/test_no_local_metadata_node/test.py index f976cc005bd..a4f04035a11 100644 --- a/tests/integration/test_no_local_metadata_node/test.py +++ b/tests/integration/test_no_local_metadata_node/test.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) @pytest.fixture(scope="module") @@ -17,11 +17,13 @@ def start_cluster(): def test_table_start_without_metadata(start_cluster): - node1.query(""" + node1.query( + """ CREATE TABLE test (date Date) ENGINE = ReplicatedMergeTree('/clickhouse/table/test_table', '1') ORDER BY tuple() - """) + """ + ) node1.query("INSERT INTO test VALUES(toDate('2019-12-01'))") @@ -33,7 +35,7 @@ def test_table_start_without_metadata(start_cluster): assert node1.query("SELECT date FROM test") == "2019-12-01\n" node1.query("DETACH TABLE test") - zk_cli = cluster.get_kazoo_client('zoo1') + zk_cli = cluster.get_kazoo_client("zoo1") # simulate update from old version zk_cli.delete("/clickhouse/table/test_table/replicas/1/metadata") diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 0cfffd28e12..e0a67a5db95 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -6,19 +6,42 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/zstd_compression_by_default.xml'], - user_configs=['configs/allow_suspicious_codecs.xml']) -node2 = cluster.add_instance('node2', main_configs=['configs/lz4hc_compression_by_default.xml'], - user_configs=['configs/allow_suspicious_codecs.xml']) -node3 = cluster.add_instance('node3', main_configs=['configs/custom_compression_by_default.xml'], - user_configs=['configs/allow_suspicious_codecs.xml']) -node4 = cluster.add_instance('node4', user_configs=['configs/enable_uncompressed_cache.xml', - 'configs/allow_suspicious_codecs.xml']) -node5 = cluster.add_instance('node5', main_configs=['configs/zstd_compression_by_default.xml'], - user_configs=['configs/enable_uncompressed_cache.xml', - 'configs/allow_suspicious_codecs.xml']) -node6 = cluster.add_instance('node6', main_configs=['configs/allow_experimental_codecs.xml'], - user_configs=['configs/allow_suspicious_codecs.xml']) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/zstd_compression_by_default.xml"], + user_configs=["configs/allow_suspicious_codecs.xml"], +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/lz4hc_compression_by_default.xml"], + user_configs=["configs/allow_suspicious_codecs.xml"], +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/custom_compression_by_default.xml"], + user_configs=["configs/allow_suspicious_codecs.xml"], +) +node4 = cluster.add_instance( + "node4", + user_configs=[ + "configs/enable_uncompressed_cache.xml", + "configs/allow_suspicious_codecs.xml", + ], +) +node5 = cluster.add_instance( + "node5", + main_configs=["configs/zstd_compression_by_default.xml"], + user_configs=[ + "configs/enable_uncompressed_cache.xml", + "configs/allow_suspicious_codecs.xml", + ], +) +node6 = cluster.add_instance( + "node6", + main_configs=["configs/allow_experimental_codecs.xml"], + user_configs=["configs/allow_suspicious_codecs.xml"], +) + @pytest.fixture(scope="module") def start_cluster(): @@ -32,109 +55,192 @@ def start_cluster(): def test_preconfigured_default_codec(start_cluster): for node in [node1, node2]: - node.query(""" + node.query( + """ CREATE TABLE compression_codec_multiple_with_key ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4), somecolumn Float64 ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; - """) + """ + ) node.query( - "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)") - assert node.query("SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0") == "2\n" - assert node.query( - "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id") == "777.777\n88.88\n99.99\n" - assert node.query( - "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100") == "hello\nworld\n" + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)" + ) + assert ( + node.query( + "SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0" + ) + == "2\n" + ) + assert ( + node.query( + "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id" + ) + == "777.777\n88.88\n99.99\n" + ) + assert ( + node.query( + "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100" + ) + == "hello\nworld\n" + ) node.query( - "INSERT INTO compression_codec_multiple_with_key SELECT toDate('2018-10-12'), number, toString(number), 1.0 FROM system.numbers LIMIT 10000") + "INSERT INTO compression_codec_multiple_with_key SELECT toDate('2018-10-12'), number, toString(number), 1.0 FROM system.numbers LIMIT 10000" + ) - assert node.query("SELECT COUNT(id) FROM compression_codec_multiple_with_key WHERE id % 10 == 0") == "1001\n" - assert node.query("SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key") == str( - 777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" - assert node.query("SELECT count(*) FROM compression_codec_multiple_with_key GROUP BY somedate") == "10003\n" + assert ( + node.query( + "SELECT COUNT(id) FROM compression_codec_multiple_with_key WHERE id % 10 == 0" + ) + == "1001\n" + ) + assert ( + node.query( + "SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key" + ) + == str(777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" + ) + assert ( + node.query( + "SELECT count(*) FROM compression_codec_multiple_with_key GROUP BY somedate" + ) + == "10003\n" + ) def test_preconfigured_custom_codec(start_cluster): - node3.query(""" + node3.query( + """ CREATE TABLE compression_codec_multiple_with_key ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String, somecolumn Float64 CODEC(ZSTD(2), LZ4HC, NONE, NONE, NONE, LZ4HC(5)) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; - """) + """ + ) node3.query( - "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)") - assert node3.query("SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0") == "2\n" - assert node3.query( - "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id") == "777.777\n88.88\n99.99\n" - assert node3.query( - "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100") == "hello\nworld\n" + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)" + ) + assert ( + node3.query( + "SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0" + ) + == "2\n" + ) + assert ( + node3.query( + "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id" + ) + == "777.777\n88.88\n99.99\n" + ) + assert ( + node3.query( + "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100" + ) + == "hello\nworld\n" + ) node3.query( "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format( - ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) + "".join( + random.choice(string.ascii_uppercase + string.digits) + for _ in range(10000) + ) + ) + ) node3.query("OPTIMIZE TABLE compression_codec_multiple_with_key FINAL") - assert node3.query( - "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert ( + node3.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1" + ) + == "10000\n" + ) for i in range(10): node3.query( - "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), {}, '{}', 88.88)".format(i, - ''.join( - random.choice( - string.ascii_uppercase + string.digits) - for - _ - in - range( - 10000)))) + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), {}, '{}', 88.88)".format( + i, + "".join( + random.choice(string.ascii_uppercase + string.digits) + for _ in range(10000) + ), + ) + ) node3.query("OPTIMIZE TABLE compression_codec_multiple_with_key FINAL") - assert node3.query("SELECT COUNT(*) from compression_codec_multiple_with_key WHERE length(data) = 10000") == "11\n" + assert ( + node3.query( + "SELECT COUNT(*) from compression_codec_multiple_with_key WHERE length(data) = 10000" + ) + == "11\n" + ) def test_uncompressed_cache_custom_codec(start_cluster): - node4.query(""" + node4.query( + """ CREATE TABLE compression_codec_multiple_with_key ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String, somecolumn Float64 CODEC(ZSTD(2), LZ4HC, NONE, NONE, NONE, LZ4HC(5)) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; - """) + """ + ) node4.query( "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format( - ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10000)))) + "".join( + random.choice(string.ascii_uppercase + string.digits) + for _ in range(10000) + ) + ) + ) # two equal requests one by one, to get into UncompressedCache for the first block - assert node4.query( - "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert ( + node4.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1" + ) + == "10000\n" + ) - assert node4.query( - "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert ( + node4.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1" + ) + == "10000\n" + ) def test_uncompressed_cache_plus_zstd_codec(start_cluster): - node5.query(""" + node5.query( + """ CREATE TABLE compression_codec_multiple_with_key ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String, somecolumn Float64 CODEC(ZSTD(2), LZ4HC, NONE, NONE, NONE, LZ4HC(5)) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; - """) + """ + ) node5.query( "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, '{}', 88.88)".format( - 'a' * 10000)) + "a" * 10000 + ) + ) - assert node5.query( - "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + assert ( + node5.query( + "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1" + ) + == "10000\n" + ) diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 8d3a8773bc4..06028af63c5 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -12,11 +12,19 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT from multiprocessing.dummy import Pool cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_odbc_drivers=True, with_mysql=True, with_postgres=True, - main_configs=['configs/openssl.xml', 'configs/odbc_logging.xml'], - dictionaries=['configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml', - 'configs/dictionaries/sqlite3_odbc_cached_dictionary.xml', - 'configs/dictionaries/postgres_odbc_hashed_dictionary.xml'], stay_alive=True) +node1 = cluster.add_instance( + "node1", + with_odbc_drivers=True, + with_mysql=True, + with_postgres=True, + main_configs=["configs/openssl.xml", "configs/odbc_logging.xml"], + dictionaries=[ + "configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml", + "configs/dictionaries/sqlite3_odbc_cached_dictionary.xml", + "configs/dictionaries/postgres_odbc_hashed_dictionary.xml", + ], + stay_alive=True, +) drop_table_sql_template = """ @@ -45,10 +53,17 @@ def get_mysql_conn(): for _ in range(15): try: if conn is None: - conn = pymysql.connect(user='root', password='clickhouse', host=cluster.mysql_ip, port=cluster.mysql_port) + conn = pymysql.connect( + user="root", + password="clickhouse", + host=cluster.mysql_ip, + port=cluster.mysql_port, + ) else: conn.ping(reconnect=True) - logging.debug(f"MySQL Connection establised: {cluster.mysql_ip}:{cluster.mysql_port}") + logging.debug( + f"MySQL Connection establised: {cluster.mysql_ip}:{cluster.mysql_port}" + ) return conn except Exception as e: errors += [str(e)] @@ -70,7 +85,9 @@ def create_mysql_table(conn, table_name): def get_postgres_conn(started_cluster): - conn_string = "host={} port={} user='postgres' password='mysecretpassword'".format(started_cluster.postgres_ip, started_cluster.postgres_port) + conn_string = "host={} port={} user='postgres' password='mysecretpassword'".format( + started_cluster.postgres_ip, started_cluster.postgres_port + ) errors = [] for _ in range(15): try: @@ -83,7 +100,9 @@ def get_postgres_conn(started_cluster): errors += [str(e)] time.sleep(1) - raise Exception("Postgre connection not establised DSN={}, {}".format(conn_string, errors)) + raise Exception( + "Postgre connection not establised DSN={}, {}".format(conn_string, errors) + ) def create_postgres_db(conn, name): @@ -99,36 +118,67 @@ def started_cluster(): logging.debug(f"sqlite data received: {sqlite_db}") node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);"], - privileged=True, user='root') + [ + "sqlite3", + sqlite_db, + "CREATE TABLE t1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);", + ], + privileged=True, + user="root", + ) node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t2(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], - privileged=True, user='root') + [ + "sqlite3", + sqlite_db, + "CREATE TABLE t2(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);", + ], + privileged=True, + user="root", + ) node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t3(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], - privileged=True, user='root') + [ + "sqlite3", + sqlite_db, + "CREATE TABLE t3(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);", + ], + privileged=True, + user="root", + ) node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t4(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], - privileged=True, user='root') + [ + "sqlite3", + sqlite_db, + "CREATE TABLE t4(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);", + ], + privileged=True, + user="root", + ) node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE tf1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);"], - privileged=True, user='root') + [ + "sqlite3", + sqlite_db, + "CREATE TABLE tf1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);", + ], + privileged=True, + user="root", + ) logging.debug("sqlite tables created") mysql_conn = get_mysql_conn() logging.debug("mysql connection received") ## create mysql db and table - create_mysql_db(mysql_conn, 'clickhouse') + create_mysql_db(mysql_conn, "clickhouse") logging.debug("mysql database created") postgres_conn = get_postgres_conn(cluster) logging.debug("postgres connection received") - create_postgres_db(postgres_conn, 'clickhouse') + create_postgres_db(postgres_conn, "clickhouse") logging.debug("postgres db created") cursor = postgres_conn.cursor() cursor.execute( - "create table if not exists clickhouse.test_table (id int primary key, column1 int not null, column2 varchar(40) not null)") + "create table if not exists clickhouse.test_table (id int primary key, column1 int not null, column2 varchar(40) not null)" + ) yield cluster @@ -144,7 +194,7 @@ def test_mysql_simple_select_works(started_cluster): mysql_setup = node1.odbc_drivers["MySQL"] - table_name = 'test_insert_select' + table_name = "test_insert_select" conn = get_mysql_conn() create_mysql_table(conn, table_name) @@ -152,27 +202,66 @@ def test_mysql_simple_select_works(started_cluster): with conn.cursor() as cursor: cursor.execute( "INSERT INTO clickhouse.{} VALUES(50, 'null-guy', 127, 255, NULL), (100, 'non-null-guy', 127, 255, 511);".format( - table_name)) + table_name + ) + ) conn.commit() - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), - settings={"external_table_functions_use_nulls": "1"}) == '\\N\n511\n' - assert node1.query("SELECT column_x FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name), - settings={"external_table_functions_use_nulls": "0"}) == '0\n511\n' + assert ( + node1.query( + "SELECT column_x FROM odbc('DSN={}', '{}')".format( + mysql_setup["DSN"], table_name + ), + settings={"external_table_functions_use_nulls": "1"}, + ) + == "\\N\n511\n" + ) + assert ( + node1.query( + "SELECT column_x FROM odbc('DSN={}', '{}')".format( + mysql_setup["DSN"], table_name + ), + settings={"external_table_functions_use_nulls": "0"}, + ) + == "0\n511\n" + ) - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nullable(UInt32)) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money, column_x) select number, concat('name_', toString(number)), 3, NULL from numbers(49) ".format( - table_name)) + table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money, column_x) select number, concat('name_', toString(number)), 3, 42 from numbers(51, 49) ".format( - table_name)) + table_name + ) + ) - assert node1.query("SELECT COUNT () FROM {} WHERE column_x IS NOT NULL".format(table_name)) == '50\n' - assert node1.query("SELECT COUNT () FROM {} WHERE column_x IS NULL".format(table_name)) == '50\n' - assert node1.query("SELECT count(*) FROM odbc('DSN={}', '{}')".format(mysql_setup["DSN"], table_name)) == '100\n' + assert ( + node1.query( + "SELECT COUNT () FROM {} WHERE column_x IS NOT NULL".format(table_name) + ) + == "50\n" + ) + assert ( + node1.query("SELECT COUNT () FROM {} WHERE column_x IS NULL".format(table_name)) + == "50\n" + ) + assert ( + node1.query( + "SELECT count(*) FROM odbc('DSN={}', '{}')".format( + mysql_setup["DSN"], table_name + ) + ) + == "100\n" + ) # previously this test fails with segfault # just to be sure :) @@ -185,23 +274,40 @@ def test_mysql_insert(started_cluster): skip_test_msan(node1) mysql_setup = node1.odbc_drivers["MySQL"] - table_name = 'test_insert' + table_name = "test_insert" conn = get_mysql_conn() create_mysql_table(conn, table_name) - odbc_args = "'DSN={}', '{}', '{}'".format(mysql_setup["DSN"], mysql_setup["Database"], table_name) + odbc_args = "'DSN={}', '{}', '{}'".format( + mysql_setup["DSN"], mysql_setup["Database"], table_name + ) node1.query( "create table mysql_insert (id Int64, name String, age UInt8, money Float, column_x Nullable(Int16)) Engine=ODBC({})".format( - odbc_args)) - node1.query("insert into mysql_insert values (1, 'test', 11, 111, 1111), (2, 'odbc', 22, 222, NULL)") - assert node1.query("select * from mysql_insert") == "1\ttest\t11\t111\t1111\n2\todbc\t22\t222\t\\N\n" + odbc_args + ) + ) + node1.query( + "insert into mysql_insert values (1, 'test', 11, 111, 1111), (2, 'odbc', 22, 222, NULL)" + ) + assert ( + node1.query("select * from mysql_insert") + == "1\ttest\t11\t111\t1111\n2\todbc\t22\t222\t\\N\n" + ) - node1.query("insert into table function odbc({}) values (3, 'insert', 33, 333, 3333)".format(odbc_args)) + node1.query( + "insert into table function odbc({}) values (3, 'insert', 33, 333, 3333)".format( + odbc_args + ) + ) node1.query( "insert into table function odbc({}) (id, name, age, money) select id*4, upper(name), age*4, money*4 from odbc({}) where id=1".format( - odbc_args, odbc_args)) - assert node1.query( - "select * from mysql_insert where id in (3, 4)") == "3\tinsert\t33\t333\t3333\n4\tTEST\t44\t444\t\\N\n" + odbc_args, odbc_args + ) + ) + assert ( + node1.query("select * from mysql_insert where id in (3, 4)") + == "3\tinsert\t33\t333\t3333\n4\tTEST\t44\t444\t\\N\n" + ) def test_sqlite_simple_select_function_works(started_cluster): @@ -210,17 +316,57 @@ def test_sqlite_simple_select_function_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t1 values(1, 1, 2, 3);"], - privileged=True, user='root') - assert node1.query("select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t1\t2\t3\n" + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO t1 values(1, 1, 2, 3);"], + privileged=True, + user="root", + ) + assert ( + node1.query( + "select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], "t1") + ) + == "1\t1\t2\t3\n" + ) + + assert ( + node1.query( + "select y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], "t1") + ) + == "2\n" + ) + assert ( + node1.query( + "select z from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], "t1") + ) + == "3\n" + ) + assert ( + node1.query( + "select x from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], "t1") + ) + == "1\n" + ) + assert ( + node1.query( + "select x, y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], "t1") + ) + == "1\t2\n" + ) + assert ( + node1.query( + "select z, x, y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], "t1") + ) + == "3\t1\t2\n" + ) + assert ( + node1.query( + "select count(), sum(x) from odbc('DSN={}', '{}') group by x".format( + sqlite_setup["DSN"], "t1" + ) + ) + == "1\t1\n" + ) - assert node1.query("select y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "2\n" - assert node1.query("select z from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "3\n" - assert node1.query("select x from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\n" - assert node1.query("select x, y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t2\n" - assert node1.query("select z, x, y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "3\t1\t2\n" - assert node1.query( - "select count(), sum(x) from odbc('DSN={}', '{}') group by x".format(sqlite_setup["DSN"], 't1')) == "1\t1\n" def test_sqlite_table_function(started_cluster): skip_test_msan(node1) @@ -228,9 +374,16 @@ def test_sqlite_table_function(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO tf1 values(1, 1, 2, 3);"], - privileged=True, user='root') - node1.query("create table odbc_tf as odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 'tf1')) + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO tf1 values(1, 1, 2, 3);"], + privileged=True, + user="root", + ) + node1.query( + "create table odbc_tf as odbc('DSN={}', '{}')".format( + sqlite_setup["DSN"], "tf1" + ) + ) assert node1.query("select * from odbc_tf") == "1\t1\t2\t3\n" assert node1.query("select y from odbc_tf") == "2\n" @@ -240,16 +393,23 @@ def test_sqlite_table_function(started_cluster): assert node1.query("select z, x, y from odbc_tf") == "3\t1\t2\n" assert node1.query("select count(), sum(x) from odbc_tf group by x") == "1\t1\n" + def test_sqlite_simple_select_storage_works(started_cluster): skip_test_msan(node1) sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t4 values(1, 1, 2, 3);"], - privileged=True, user='root') - node1.query("create table SqliteODBC (x Int32, y String, z String) engine = ODBC('DSN={}', '', 't4')".format( - sqlite_setup["DSN"])) + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO t4 values(1, 1, 2, 3);"], + privileged=True, + user="root", + ) + node1.query( + "create table SqliteODBC (x Int32, y String, z String) engine = ODBC('DSN={}', '', 't4')".format( + sqlite_setup["DSN"] + ) + ) assert node1.query("select * from SqliteODBC") == "1\t2\t3\n" assert node1.query("select y from SqliteODBC") == "2\n" @@ -264,70 +424,118 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): skip_test_msan(node1) sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 1, 2, 3);"], - privileged=True, user='root') + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 1, 2, 3);"], + privileged=True, + user="root", + ) node1.query("SYSTEM RELOAD DICTIONARY sqlite3_odbc_hashed") - first_update_time = node1.query("SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'") + first_update_time = node1.query( + "SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'" + ) logging.debug(f"First update time {first_update_time}") - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3") - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "1") # default + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3" + ) + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "1" + ) # default - second_update_time = node1.query("SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'") + second_update_time = node1.query( + "SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'" + ) # Reloaded with new data logging.debug(f"Second update time {second_update_time}") while first_update_time == second_update_time: - second_update_time = node1.query("SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'") + second_update_time = node1.query( + "SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'" + ) logging.debug("Waiting dictionary to update for the second time") time.sleep(0.1) - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(200, 200, 2, 7);"], - privileged=True, user='root') + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO t2 values(200, 200, 2, 7);"], + privileged=True, + user="root", + ) # No reload because of invalidate query - third_update_time = node1.query("SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'") + third_update_time = node1.query( + "SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'" + ) logging.debug(f"Third update time {second_update_time}") counter = 0 while third_update_time == second_update_time: - third_update_time = node1.query("SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'") + third_update_time = node1.query( + "SELECT last_successful_update_time FROM system.dictionaries WHERE name = 'sqlite3_odbc_hashed'" + ) time.sleep(0.1) if counter > 50: break counter += 1 - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3") - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "1") # still default + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3" + ) + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "1" + ) # still default - node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t2 values(1, 1, 2, 5);"], - privileged=True, user='root') + node1.exec_in_container( + ["sqlite3", sqlite_db, "REPLACE INTO t2 values(1, 1, 2, 5);"], + privileged=True, + user="root", + ) - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "5") - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "7") + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "5" + ) + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "7" + ) def test_sqlite_odbc_cached_dictionary(started_cluster): skip_test_msan(node1) sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t3 values(1, 1, 2, 3);"], - privileged=True, user='root') + node1.exec_in_container( + ["sqlite3", sqlite_db, "INSERT INTO t3 values(1, 1, 2, 3);"], + privileged=True, + user="root", + ) - assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "3\n" + assert ( + node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") + == "3\n" + ) # Allow insert - node1.exec_in_container(["chmod", "a+rw", "/tmp"], privileged=True, user='root') - node1.exec_in_container(["chmod", "a+rw", sqlite_db], privileged=True, user='root') + node1.exec_in_container(["chmod", "a+rw", "/tmp"], privileged=True, user="root") + node1.exec_in_container(["chmod", "a+rw", sqlite_db], privileged=True, user="root") - node1.query("insert into table function odbc('DSN={};ReadOnly=0', '', 't3') values (200, 200, 2, 7)".format( - node1.odbc_drivers["SQLite3"]["DSN"])) + node1.query( + "insert into table function odbc('DSN={};ReadOnly=0', '', 't3') values (200, 200, 2, 7)".format( + node1.odbc_drivers["SQLite3"]["DSN"] + ) + ) - assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value + assert ( + node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") + == "7\n" + ) # new value - node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t3 values(1, 1, 2, 12);"], - privileged=True, user='root') + node1.exec_in_container( + ["sqlite3", sqlite_db, "REPLACE INTO t3 values(1, 1, 2, 12);"], + privileged=True, + user="root", + ) - assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))", "12") + assert_eq_with_retry( + node1, "select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))", "12" + ) def test_postgres_odbc_hashed_dictionary_with_schema(started_cluster): @@ -336,12 +544,24 @@ def test_postgres_odbc_hashed_dictionary_with_schema(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() cursor.execute("truncate table clickhouse.test_table") - cursor.execute("insert into clickhouse.test_table values(1, 1, 'hello'),(2, 2, 'world')") + cursor.execute( + "insert into clickhouse.test_table values(1, 1, 'hello'),(2, 2, 'world')" + ) node1.query("SYSTEM RELOAD DICTIONARY postgres_odbc_hashed") - node1.exec_in_container(["ss", "-K", "dport", "postgresql"], privileged=True, user='root') + node1.exec_in_container( + ["ss", "-K", "dport", "postgresql"], privileged=True, user="root" + ) node1.query("SYSTEM RELOAD DICTIONARY postgres_odbc_hashed") - assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))", "hello") - assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(2))", "world") + assert_eq_with_retry( + node1, + "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))", + "hello", + ) + assert_eq_with_retry( + node1, + "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(2))", + "world", + ) def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster): @@ -357,7 +577,11 @@ def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster): except Exception as ex: assert False, "Exception occured -- odbc-bridge hangs: " + str(ex) - assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(3))", "xxx") + assert_eq_with_retry( + node1, + "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(3))", + "xxx", + ) def test_postgres_insert(started_cluster): @@ -371,16 +595,26 @@ def test_postgres_insert(started_cluster): # reconstruction of connection string. node1.query( - "create table pg_insert (id UInt64, column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") + "create table pg_insert (id UInt64, column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')" + ) node1.query("insert into pg_insert values (1, 1, 'hello'), (2, 2, 'world')") - assert node1.query("select * from pg_insert") == '1\t1\thello\n2\t2\tworld\n' - node1.query("insert into table function odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table') format CSV 3,3,test") + assert node1.query("select * from pg_insert") == "1\t1\thello\n2\t2\tworld\n" node1.query( - "insert into table function odbc('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')" \ - " select number, number, 's' || toString(number) from numbers (4, 7)") - assert node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" - assert node1.query( - "select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table')))") == "55\t10\n" + "insert into table function odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table') format CSV 3,3,test" + ) + node1.query( + "insert into table function odbc('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')" + " select number, number, 's' || toString(number) from numbers (4, 7)" + ) + assert ( + node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" + ) + assert ( + node1.query( + "select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table')))" + ) + == "55\t10\n" + ) def test_bridge_dies_with_parent(started_cluster): @@ -390,7 +624,9 @@ def test_bridge_dies_with_parent(started_cluster): # TODO: Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge in this test and # that's linked somehow with that we have replaced getauxval() in glibc-compatibility. # The leak sanitizer calls getauxval() for its own purposes, and our replaced version doesn't seem to be equivalent in that case. - pytest.skip("Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge") + pytest.skip( + "Leak sanitizer falsely reports about a leak of 16 bytes in clickhouse-odbc-bridge" + ) node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") @@ -401,7 +637,9 @@ def test_bridge_dies_with_parent(started_cluster): while clickhouse_pid is not None: try: - node1.exec_in_container(["kill", str(clickhouse_pid)], privileged=True, user='root') + node1.exec_in_container( + ["kill", str(clickhouse_pid)], privileged=True, user="root" + ) except: pass clickhouse_pid = node1.get_process_pid("clickhouse server") @@ -414,8 +652,11 @@ def test_bridge_dies_with_parent(started_cluster): break if bridge_pid: - out = node1.exec_in_container(["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], - privileged=True, user='root') + out = node1.exec_in_container( + ["gdb", "-p", str(bridge_pid), "--ex", "thread apply all bt", "--ex", "q"], + privileged=True, + user="root", + ) logging.debug(f"Bridge is running, gdb output:\n{out}") assert clickhouse_pid is None @@ -426,9 +667,11 @@ def test_bridge_dies_with_parent(started_cluster): def test_odbc_postgres_date_data_type(started_cluster): skip_test_msan(node1) - conn = get_postgres_conn(started_cluster); + conn = get_postgres_conn(started_cluster) cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS clickhouse.test_date (id integer, column1 integer, column2 date)") + cursor.execute( + "CREATE TABLE IF NOT EXISTS clickhouse.test_date (id integer, column1 integer, column2 date)" + ) cursor.execute("INSERT INTO clickhouse.test_date VALUES (1, 1, '2020-12-01')") cursor.execute("INSERT INTO clickhouse.test_date VALUES (2, 2, '2020-12-02')") @@ -436,13 +679,14 @@ def test_odbc_postgres_date_data_type(started_cluster): conn.commit() node1.query( - ''' + """ CREATE TABLE test_date (id UInt64, column1 UInt64, column2 Date) - ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_date')''') + ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_date')""" + ) - expected = '1\t1\t2020-12-01\n2\t2\t2020-12-02\n3\t3\t2020-12-03\n' - result = node1.query('SELECT * FROM test_date'); - assert(result == expected) + expected = "1\t1\t2020-12-01\n2\t2\t2020-12-02\n3\t3\t2020-12-03\n" + result = node1.query("SELECT * FROM test_date") + assert result == expected cursor.execute("DROP TABLE IF EXISTS clickhouse.test_date") node1.query("DROP TABLE IF EXISTS test_date") @@ -454,39 +698,53 @@ def test_odbc_postgres_conversions(started_cluster): cursor = conn.cursor() cursor.execute( - '''CREATE TABLE IF NOT EXISTS clickhouse.test_types ( + """CREATE TABLE IF NOT EXISTS clickhouse.test_types ( a smallint, b integer, c bigint, d real, e double precision, f serial, g bigserial, - h timestamp)''') - - node1.query(''' - INSERT INTO TABLE FUNCTION - odbc('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_types') - VALUES (-32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12')''') - - result = node1.query(''' - SELECT a, b, c, d, e, f, g, h - FROM odbc('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_types') - ''') - - assert(result == '-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12\n') - cursor.execute("DROP TABLE IF EXISTS clickhouse.test_types") - - cursor.execute("""CREATE TABLE IF NOT EXISTS clickhouse.test_types (column1 Timestamp, column2 Numeric)""") + h timestamp)""" + ) node1.query( - ''' + """ + INSERT INTO TABLE FUNCTION + odbc('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_types') + VALUES (-32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12')""" + ) + + result = node1.query( + """ + SELECT a, b, c, d, e, f, g, h + FROM odbc('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_types') + """ + ) + + assert ( + result + == "-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12\n" + ) + cursor.execute("DROP TABLE IF EXISTS clickhouse.test_types") + + cursor.execute( + """CREATE TABLE IF NOT EXISTS clickhouse.test_types (column1 Timestamp, column2 Numeric)""" + ) + + node1.query( + """ CREATE TABLE test_types (column1 DateTime64, column2 Decimal(5, 1)) - ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_types')''') + ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_types')""" + ) node1.query( """INSERT INTO test_types - SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow'), toDecimal32(1.1, 1)""") + SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Etc/UTC'), toDecimal32(1.1, 1)""" + ) - expected = node1.query("SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow'), toDecimal32(1.1, 1)") + expected = node1.query( + "SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Etc/UTC'), toDecimal32(1.1, 1)" + ) result = node1.query("SELECT * FROM test_types") logging.debug(result) cursor.execute("DROP TABLE IF EXISTS clickhouse.test_types") - assert(result == expected) + assert result == expected def test_odbc_cyrillic_with_varchar(started_cluster): @@ -498,17 +756,21 @@ def test_odbc_cyrillic_with_varchar(started_cluster): cursor.execute("DROP TABLE IF EXISTS clickhouse.test_cyrillic") cursor.execute("CREATE TABLE clickhouse.test_cyrillic (name varchar(11))") - node1.query(''' + node1.query( + """ CREATE TABLE test_cyrillic (name String) - ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_cyrillic')''') + ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_cyrillic')""" + ) cursor.execute("INSERT INTO clickhouse.test_cyrillic VALUES ('A-nice-word')") cursor.execute("INSERT INTO clickhouse.test_cyrillic VALUES ('Красивенько')") - result = node1.query(''' SELECT * FROM test_cyrillic ORDER BY name''') - assert(result == 'A-nice-word\nКрасивенько\n') - result = node1.query(''' SELECT name FROM odbc('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_cyrillic') ''') - assert(result == 'A-nice-word\nКрасивенько\n') + result = node1.query(""" SELECT * FROM test_cyrillic ORDER BY name""") + assert result == "A-nice-word\nКрасивенько\n" + result = node1.query( + """ SELECT name FROM odbc('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_cyrillic') """ + ) + assert result == "A-nice-word\nКрасивенько\n" def test_many_connections(started_cluster): @@ -517,22 +779,24 @@ def test_many_connections(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() - cursor.execute('DROP TABLE IF EXISTS clickhouse.test_pg_table') - cursor.execute('CREATE TABLE clickhouse.test_pg_table (key integer, value integer)') + cursor.execute("DROP TABLE IF EXISTS clickhouse.test_pg_table") + cursor.execute("CREATE TABLE clickhouse.test_pg_table (key integer, value integer)") - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test_pg_table; CREATE TABLE test_pg_table (key UInt32, value UInt32) - ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_pg_table')''') + ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_pg_table')""" + ) node1.query("INSERT INTO test_pg_table SELECT number, number FROM numbers(10)") query = "SELECT count() FROM (" - for i in range (24): + for i in range(24): query += "SELECT key FROM {t} UNION ALL " query += "SELECT key FROM {t})" - assert node1.query(query.format(t='test_pg_table')) == '250\n' + assert node1.query(query.format(t="test_pg_table")) == "250\n" def test_concurrent_queries(started_cluster): @@ -541,41 +805,58 @@ def test_concurrent_queries(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test_pg_table; CREATE TABLE test_pg_table (key UInt32, value UInt32) - ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_pg_table')''') + ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_pg_table')""" + ) - cursor.execute('DROP TABLE IF EXISTS clickhouse.test_pg_table') - cursor.execute('CREATE TABLE clickhouse.test_pg_table (key integer, value integer)') + cursor.execute("DROP TABLE IF EXISTS clickhouse.test_pg_table") + cursor.execute("CREATE TABLE clickhouse.test_pg_table (key integer, value integer)") def node_insert(_): for i in range(5): - node1.query("INSERT INTO test_pg_table SELECT number, number FROM numbers(1000)", user='default') + node1.query( + "INSERT INTO test_pg_table SELECT number, number FROM numbers(1000)", + user="default", + ) busy_pool = Pool(5) p = busy_pool.map_async(node_insert, range(5)) p.wait() - assert_eq_with_retry(node1, "SELECT count() FROM test_pg_table", str(5*5*1000), retry_count=100) + assert_eq_with_retry( + node1, "SELECT count() FROM test_pg_table", str(5 * 5 * 1000), retry_count=100 + ) def node_insert_select(_): for i in range(5): - result = node1.query("INSERT INTO test_pg_table SELECT number, number FROM numbers(1000)", user='default') - result = node1.query("SELECT * FROM test_pg_table LIMIT 100", user='default') + result = node1.query( + "INSERT INTO test_pg_table SELECT number, number FROM numbers(1000)", + user="default", + ) + result = node1.query( + "SELECT * FROM test_pg_table LIMIT 100", user="default" + ) busy_pool = Pool(5) p = busy_pool.map_async(node_insert_select, range(5)) p.wait() - assert_eq_with_retry(node1, "SELECT count() FROM test_pg_table", str(5*5*1000*2), retry_count=100) + assert_eq_with_retry( + node1, + "SELECT count() FROM test_pg_table", + str(5 * 5 * 1000 * 2), + retry_count=100, + ) - node1.query('DROP TABLE test_pg_table;') - cursor.execute('DROP TABLE clickhouse.test_pg_table;') + node1.query("DROP TABLE test_pg_table;") + cursor.execute("DROP TABLE clickhouse.test_pg_table;") def test_odbc_long_column_names(started_cluster): skip_test_msan(node1) - conn = get_postgres_conn(started_cluster); + conn = get_postgres_conn(started_cluster) cursor = conn.cursor() column_name = "column" * 8 @@ -586,7 +867,11 @@ def test_odbc_long_column_names(started_cluster): create_table += "{} integer".format(column_name + str(i)) create_table += ")" cursor.execute(create_table) - insert = "INSERT INTO clickhouse.test_long_column_names SELECT i" + ", i" * 999 + " FROM generate_series(0, 99) as t(i)" + insert = ( + "INSERT INTO clickhouse.test_long_column_names SELECT i" + + ", i" * 999 + + " FROM generate_series(0, 99) as t(i)" + ) cursor.execute(insert) conn.commit() @@ -596,11 +881,11 @@ def test_odbc_long_column_names(started_cluster): create_table += ", " create_table += "{} UInt32".format(column_name + str(i)) create_table += ") ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_long_column_names')" - result = node1.query(create_table); + result = node1.query(create_table) - result = node1.query('SELECT * FROM test_long_column_names'); + result = node1.query("SELECT * FROM test_long_column_names") expected = node1.query("SELECT number" + ", number" * 999 + " FROM numbers(100)") - assert(result == expected) + assert result == expected cursor.execute("DROP TABLE IF EXISTS clickhouse.test_long_column_names") node1.query("DROP TABLE IF EXISTS test_long_column_names") @@ -612,20 +897,30 @@ def test_odbc_long_text(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() cursor.execute("drop table if exists clickhouse.test_long_text") - cursor.execute("create table clickhouse.test_long_text(flen int, field1 text)"); + cursor.execute("create table clickhouse.test_long_text(flen int, field1 text)") # sample test from issue 9363 text_from_issue = """BEGIN These examples only show the order that data is arranged in. The values from different columns are stored separately, and data from the same column is stored together. Examples of a column-oriented DBMS: Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, and kdb+. Different orders for storing data are better suited to different scenarios. The data access scenario refers to what queries are made, how often, and in what proportion; how much data is read for each type of query – rows, columns, and bytes; the relationship between reading and updating data; the working size of the data and how locally it is used; whether transactions are used, and how isolated they are; requirements for data replication and logical integrity; requirements for latency and throughput for each type of query, and so on. The higher the load on the system, the more important it is to customize the system set up to match the requirements of the usage scenario, and the more fine grained this customization becomes. There is no system that is equally well-suited to significantly different scenarios. If a system is adaptable to a wide set of scenarios, under a high load, the system will handle all the scenarios equally poorly, or will work well for just one or few of possible scenarios. Key Properties of OLAP Scenario¶ The vast majority of requests are for read access. Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. Data is added to the DB but is not modified. For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. Tables are "wide," meaning they contain a large number of columns. Queries are relatively rare (usually hundreds of queries per server or less per second). For simple queries, latencies around 50 ms are allowed. Column values are fairly small: numbers and short strings (for example, 60 bytes per URL). Requires high throughput when processing a single query (up to billions of rows per second per server). Transactions are not necessary. Low requirements for data consistency. There is one large table per query. All tables are small, except for one. A query result is significantly smaller than the source data. In other words, data is filtered or aggregated, so the result fits in a single server"s RAM. It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it doesn"t make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Redis for analytics, you will get very poor performance compared to OLAP databases. Why Column-Oriented Databases Work Better in the OLAP Scenario¶ Column-oriented databases are better suited to OLAP scenarios: they are at least 100 times faster in processing most queries. The reasons are explained in detail below, but the fact is easier to demonstrate visually. END""" - cursor.execute("""insert into clickhouse.test_long_text (flen, field1) values (3248, '{}')""".format(text_from_issue)); + cursor.execute( + """insert into clickhouse.test_long_text (flen, field1) values (3248, '{}')""".format( + text_from_issue + ) + ) - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test_long_test; CREATE TABLE test_long_text (flen UInt32, field1 String) - ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_long_text')''') + ENGINE = ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_long_text')""" + ) result = node1.query("select field1 from test_long_text;") - assert(result.strip() == text_from_issue) + assert result.strip() == text_from_issue long_text = "text" * 1000000 - cursor.execute("""insert into clickhouse.test_long_text (flen, field1) values (400000, '{}')""".format(long_text)); + cursor.execute( + """insert into clickhouse.test_long_text (flen, field1) values (400000, '{}')""".format( + long_text + ) + ) result = node1.query("select field1 from test_long_text where flen=400000;") - assert(result.strip() == long_text) + assert result.strip() == long_text diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index 1870ecf4c9d..beef294b792 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -4,27 +4,69 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node18_14 = cluster.add_instance('node18_14', image='yandex/clickhouse-server', tag='18.14.19', - with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_1 = cluster.add_instance('node19_1', image='yandex/clickhouse-server', tag='19.1.16', with_installed_binary=True, - main_configs=["configs/config.d/test_cluster.xml"]) -node19_4 = cluster.add_instance('node19_4', image='yandex/clickhouse-server', tag='19.4.5.35', - with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_8 = cluster.add_instance('node19_8', image='yandex/clickhouse-server', tag='19.8.3.8', - with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_11 = cluster.add_instance('node19_11', image='yandex/clickhouse-server', tag='19.11.13.74', - with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_13 = cluster.add_instance('node19_13', image='yandex/clickhouse-server', tag='19.13.7.57', - with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) -node19_16 = cluster.add_instance('node19_16', image='yandex/clickhouse-server', tag='19.16.2.2', - with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"]) +node18_14 = cluster.add_instance( + "node18_14", + image="yandex/clickhouse-server", + tag="18.14.19", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) +node19_1 = cluster.add_instance( + "node19_1", + image="yandex/clickhouse-server", + tag="19.1.16", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) +node19_4 = cluster.add_instance( + "node19_4", + image="yandex/clickhouse-server", + tag="19.4.5.35", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) +node19_8 = cluster.add_instance( + "node19_8", + image="yandex/clickhouse-server", + tag="19.8.3.8", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) +node19_11 = cluster.add_instance( + "node19_11", + image="yandex/clickhouse-server", + tag="19.11.13.74", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) +node19_13 = cluster.add_instance( + "node19_13", + image="yandex/clickhouse-server", + tag="19.13.7.57", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) +node19_16 = cluster.add_instance( + "node19_16", + image="yandex/clickhouse-server", + tag="19.16.2.2", + with_installed_binary=True, + main_configs=["configs/config.d/test_cluster.xml"], +) old_nodes = [node18_14, node19_1, node19_4, node19_8, node19_11, node19_13, node19_16] -new_node = cluster.add_instance('node_new') +new_node = cluster.add_instance("node_new") def query_from_one_node_to_another(client_node, server_node, query): client_node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host {} --query {!r}".format(server_node.name, query)]) + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host {} --query {!r}".format( + server_node.name, query + ), + ] + ) @pytest.fixture(scope="module") @@ -33,11 +75,14 @@ def setup_nodes(): cluster.start() for n in old_nodes + [new_node]: - n.query('''CREATE TABLE test_table (id UInt32, value UInt64) ENGINE = MergeTree() ORDER BY tuple()''') + n.query( + """CREATE TABLE test_table (id UInt32, value UInt64) ENGINE = MergeTree() ORDER BY tuple()""" + ) for n in old_nodes: n.query( - '''CREATE TABLE dist_table AS test_table ENGINE = Distributed('test_cluster', 'default', 'test_table')''') + """CREATE TABLE dist_table AS test_table ENGINE = Distributed('test_cluster', 'default', 'test_table')""" + ) yield cluster finally: @@ -47,18 +92,25 @@ def setup_nodes(): def test_client_is_older_than_server(setup_nodes): server = new_node for i, client in enumerate(old_nodes): - query_from_one_node_to_another(client, server, "INSERT INTO test_table VALUES (1, {})".format(i)) + query_from_one_node_to_another( + client, server, "INSERT INTO test_table VALUES (1, {})".format(i) + ) for client in old_nodes: query_from_one_node_to_another(client, server, "SELECT COUNT() FROM test_table") - assert server.query("SELECT COUNT() FROM test_table WHERE id=1") == str(len(old_nodes)) + "\n" + assert ( + server.query("SELECT COUNT() FROM test_table WHERE id=1") + == str(len(old_nodes)) + "\n" + ) def test_server_is_older_than_client(setup_nodes): client = new_node for i, server in enumerate(old_nodes): - query_from_one_node_to_another(client, server, "INSERT INTO test_table VALUES (2, {})".format(i)) + query_from_one_node_to_another( + client, server, "INSERT INTO test_table VALUES (2, {})".format(i) + ) for server in old_nodes: query_from_one_node_to_another(client, server, "SELECT COUNT() FROM test_table") @@ -73,7 +125,13 @@ def test_distributed_query_initiator_is_older_than_shard(setup_nodes): for i, initiator in enumerate(distributed_query_initiator_old_nodes): initiator.query("INSERT INTO dist_table VALUES (3, {})".format(i)) - assert_eq_with_retry(shard, "SELECT COUNT() FROM test_table WHERE id=3", - str(len(distributed_query_initiator_old_nodes))) - assert_eq_with_retry(initiator, "SELECT COUNT() FROM dist_table WHERE id=3", - str(len(distributed_query_initiator_old_nodes))) + assert_eq_with_retry( + shard, + "SELECT COUNT() FROM test_table WHERE id=3", + str(len(distributed_query_initiator_old_nodes)), + ) + assert_eq_with_retry( + initiator, + "SELECT COUNT() FROM dist_table WHERE id=3", + str(len(distributed_query_initiator_old_nodes)), + ) diff --git a/tests/integration/test_on_cluster_timeouts/test.py b/tests/integration/test_on_cluster_timeouts/test.py index 544153d0d00..06f19fabd68 100644 --- a/tests/integration/test_on_cluster_timeouts/test.py +++ b/tests/integration/test_on_cluster_timeouts/test.py @@ -4,14 +4,30 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/users_config.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/users_config.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/users_config.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/users_config.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users_config.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users_config.xml"], + with_zookeeper=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users_config.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users_config.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -26,23 +42,33 @@ def started_cluster(): def test_long_query(started_cluster): node1.query( - "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '1') ORDER BY tuple()") + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '1') ORDER BY tuple()" + ) node2.query( - "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '2') ORDER BY tuple()") + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/1/cluster_table', '2') ORDER BY tuple()" + ) - node1.query("INSERT INTO cluster_table SELECT number, toString(number) FROM numbers(20)") + node1.query( + "INSERT INTO cluster_table SELECT number, toString(number) FROM numbers(20)" + ) node2.query("SYSTEM SYNC REPLICA cluster_table") node3.query( - "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '1') ORDER BY tuple()") + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '1') ORDER BY tuple()" + ) node4.query( - "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '2') ORDER BY tuple()") - node3.query("INSERT INTO cluster_table SELECT number, toString(number) FROM numbers(20)") + "CREATE TABLE cluster_table (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test/2/cluster_table', '2') ORDER BY tuple()" + ) + node3.query( + "INSERT INTO cluster_table SELECT number, toString(number) FROM numbers(20)" + ) node4.query("SYSTEM SYNC REPLICA cluster_table") - node1.query("ALTER TABLE cluster_table ON CLUSTER 'test_cluster' UPDATE key = 1 WHERE sleepEachRow(1) == 0", - settings={"mutations_sync": "2"}) + node1.query( + "ALTER TABLE cluster_table ON CLUSTER 'test_cluster' UPDATE key = 1 WHERE sleepEachRow(1) == 0", + settings={"mutations_sync": "2"}, + ) assert node1.query("SELECT SUM(key) FROM cluster_table") == "20\n" assert node2.query("SELECT SUM(key) FROM cluster_table") == "20\n" diff --git a/tests/integration/test_optimize_on_insert/test.py b/tests/integration/test_optimize_on_insert/test.py index da4e20edf0c..0dfec53cf9c 100644 --- a/tests/integration/test_optimize_on_insert/test.py +++ b/tests/integration/test_optimize_on_insert/test.py @@ -5,8 +5,9 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -20,18 +21,27 @@ def start_cluster(): def get_data_files_for_table(node, table_name): - raw_output = node.exec_in_container(["bash", "-c", "ls /var/lib/clickhouse/data/default/{}".format(table_name)]) + raw_output = node.exec_in_container( + ["bash", "-c", "ls /var/lib/clickhouse/data/default/{}".format(table_name)] + ) return raw_output.strip().split("\n") + def test_empty_parts_optimize(start_cluster): for n, node in enumerate([node1, node2]): - node.query(""" + node.query( + """ CREATE TABLE empty (key UInt32, val UInt32, date Datetime) ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '{}', val) PARTITION BY date ORDER BY key; - """.format(n+1)) + """.format( + n + 1 + ) + ) - node1.query("INSERT INTO empty VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01')") + node1.query( + "INSERT INTO empty VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01')" + ) node2.query("SYSTEM SYNC REPLICA empty", timeout=15) @@ -39,10 +49,19 @@ def test_empty_parts_optimize(start_cluster): assert node2.query("SELECT * FROM empty") == "" # No other tmp files exists - assert set(get_data_files_for_table(node1, "empty")) == {"detached", "format_version.txt"} - assert set(get_data_files_for_table(node2, "empty")) == {"detached", "format_version.txt"} + assert set(get_data_files_for_table(node1, "empty")) == { + "detached", + "format_version.txt", + } + assert set(get_data_files_for_table(node2, "empty")) == { + "detached", + "format_version.txt", + } - node1.query("INSERT INTO empty VALUES (1, 1, '2020-02-01'), (1, 1, '2020-02-01'), (1, -2, '2020-02-01')", settings={"insert_quorum": 2}) + node1.query( + "INSERT INTO empty VALUES (1, 1, '2020-02-01'), (1, 1, '2020-02-01'), (1, -2, '2020-02-01')", + settings={"insert_quorum": 2}, + ) assert node1.query("SELECT * FROM empty") == "" assert node2.query("SELECT * FROM empty") == "" diff --git a/tests/integration/test_part_log_table/configs/config_disk_name_test.xml b/tests/integration/test_part_log_table/configs/config_disk_name_test.xml new file mode 100644 index 00000000000..c8831031674 --- /dev/null +++ b/tests/integration/test_part_log_table/configs/config_disk_name_test.xml @@ -0,0 +1,30 @@ + + + + + local + /path1/ + + + local + /path2/ + + + + + +
+ test1 +
+ + + + +
+ test2 +
+ + + + + diff --git a/tests/integration/test_part_log_table/test.py b/tests/integration/test_part_log_table/test.py index 050e8c831c7..d81990a9d47 100644 --- a/tests/integration/test_part_log_table/test.py +++ b/tests/integration/test_part_log_table/test.py @@ -3,9 +3,18 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", main_configs=["configs/config_without_standard_part_log.xml"]) -node2 = cluster.add_instance("node2", main_configs=["configs/config_with_standard_part_log.xml"]) -node3 = cluster.add_instance("node3", main_configs=["configs/config_with_non_standard_part_log.xml"]) +node1 = cluster.add_instance( + "node1", main_configs=["configs/config_without_standard_part_log.xml"] +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/config_with_standard_part_log.xml"] +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/config_with_non_standard_part_log.xml"] +) +node4 = cluster.add_instance( + "node4", main_configs=["configs/config_disk_name_test.xml"] +) @pytest.fixture(scope="module") @@ -18,25 +27,54 @@ def start_cluster(): def test_config_without_part_log(start_cluster): - assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") - node1.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value") - assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") + assert "Table system.part_log doesn't exist" in node1.query_and_get_error( + "SELECT * FROM system.part_log" + ) + node1.query( + "CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() ORDER BY value" + ) + assert "Table system.part_log doesn't exist" in node1.query_and_get_error( + "SELECT * FROM system.part_log" + ) node1.query("INSERT INTO test_table VALUES ('name', 1)") node1.query("SYSTEM FLUSH LOGS") - assert "Table system.part_log doesn't exist" in node1.query_and_get_error("SELECT * FROM system.part_log") + assert "Table system.part_log doesn't exist" in node1.query_and_get_error( + "SELECT * FROM system.part_log" + ) # Note: if part_log is defined, we cannot say when the table will be created - because of metric_log, trace_log, text_log, query_log... + def test_config_with_standard_part_log(start_cluster): - node2.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") + node2.query( + "CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value" + ) node2.query("INSERT INTO test_table VALUES ('name', 1)") node2.query("SYSTEM FLUSH LOGS") assert node2.query("SELECT * FROM system.part_log") != "" def test_config_with_non_standard_part_log(start_cluster): - node3.query("CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value") + node3.query( + "CREATE TABLE test_table(word String, value UInt64) ENGINE=MergeTree() Order by value" + ) node3.query("INSERT INTO test_table VALUES ('name', 1)") node3.query("SYSTEM FLUSH LOGS") assert node3.query("SELECT * FROM system.own_part_log") != "" + + +def test_config_disk_name_test(start_cluster): + node4.query( + "CREATE TABLE test_table1(word String, value UInt64) ENGINE = MergeTree() ORDER BY word SETTINGS storage_policy = 'test1'" + ) + node4.query("INSERT INTO test_table1(*) VALUES ('test1', 2)") + node4.query( + "CREATE TABLE test_table2(word String, value UInt64) ENGINE = MergeTree() ORDER BY word SETTINGS storage_policy = 'test2'" + ) + node4.query("INSERT INTO test_table2(*) VALUES ('test2', 3)") + node4.query("SYSTEM FLUSH LOGS") + assert ( + node4.query("SELECT DISTINCT disk_name FROM system.part_log ORDER by disk_name") + == "test1\ntest2\n" + ) diff --git a/tests/integration/test_part_moves_between_shards/test.py b/tests/integration/test_part_moves_between_shards/test.py index ed7640e5f9e..1dbe5324124 100644 --- a/tests/integration/test_part_moves_between_shards/test.py +++ b/tests/integration/test_part_moves_between_shards/test.py @@ -12,28 +12,32 @@ transient_ch_errors = [23, 32, 210] cluster = ClickHouseCluster(__file__) s0r0 = cluster.add_instance( - 's0r0', - main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + "s0r0", + main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"], stay_alive=True, - with_zookeeper=True) + with_zookeeper=True, +) s0r1 = cluster.add_instance( - 's0r1', - main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + "s0r1", + main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"], stay_alive=True, - with_zookeeper=True) + with_zookeeper=True, +) s1r0 = cluster.add_instance( - 's1r0', - main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + "s1r0", + main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"], stay_alive=True, - with_zookeeper=True) + with_zookeeper=True, +) s1r1 = cluster.add_instance( - 's1r1', - main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + "s1r1", + main_configs=["configs/remote_servers.xml", "configs/merge_tree.xml"], stay_alive=True, - with_zookeeper=True) + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -48,12 +52,16 @@ def started_cluster(): def test_move(started_cluster): for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): for replica_ix, r in enumerate(rs): - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_move; CREATE TABLE test_move(v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_move', '{}') ORDER BY tuple() - """.format(shard_ix, r.name)) + """.format( + shard_ix, r.name + ) + ) s0r0.query("SYSTEM STOP MERGES test_move") s0r1.query("SYSTEM STOP MERGES test_move") @@ -64,7 +72,9 @@ def test_move(started_cluster): assert "2" == s0r0.query("SELECT count() FROM test_move").strip() assert "0" == s1r0.query("SELECT count() FROM test_move").strip() - s0r0.query("ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_move'") + s0r0.query( + "ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_move'" + ) print(s0r0.query("SELECT * FROM system.part_moves_between_shards")) @@ -80,7 +90,9 @@ def test_move(started_cluster): assert "1" == n.query("SELECT count() FROM test_move").strip() # Move part back - s1r0.query("ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/test_move'") + s1r0.query( + "ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/test_move'" + ) wait_for_state("DONE", s1r0, "test_move") @@ -94,18 +106,24 @@ def test_move(started_cluster): def test_deduplication_while_move(started_cluster): for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): for replica_ix, r in enumerate(rs): - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_deduplication; CREATE TABLE test_deduplication(v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_deduplication', '{}') ORDER BY tuple() - """.format(shard_ix, r.name)) + """.format( + shard_ix, r.name + ) + ) - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_deduplication_d; CREATE TABLE test_deduplication_d AS test_deduplication ENGINE Distributed('test_cluster', '', test_deduplication) - """) + """ + ) s0r0.query("SYSTEM STOP MERGES test_deduplication") s0r1.query("SYSTEM STOP MERGES test_deduplication") @@ -118,7 +136,8 @@ def test_deduplication_while_move(started_cluster): assert "0" == s1r0.query("SELECT count() FROM test_deduplication").strip() s0r0.query( - "ALTER TABLE test_deduplication MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_deduplication'") + "ALTER TABLE test_deduplication MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_deduplication'" + ) s0r0.query("SYSTEM START MERGES test_deduplication") expected = """ @@ -129,10 +148,30 @@ def test_deduplication_while_move(started_cluster): def deduplication_invariant_test(): n = random.choice(list(started_cluster.instances.values())) assert TSV( - n.query("SELECT * FROM test_deduplication_d ORDER BY v", - settings={"allow_experimental_query_deduplication": 1}) + n.query( + "SELECT * FROM test_deduplication_d ORDER BY v", + settings={"allow_experimental_query_deduplication": 1}, + ) ) == TSV(expected) + # https://github.com/ClickHouse/ClickHouse/issues/34089 + assert TSV( + n.query( + "SELECT count() FROM test_deduplication_d", + settings={"allow_experimental_query_deduplication": 1}, + ) + ) == TSV("2") + + assert TSV( + n.query( + "SELECT count() FROM test_deduplication_d", + settings={ + "allow_experimental_query_deduplication": 1, + "allow_experimental_projection_optimization": 1, + }, + ) + ) == TSV("2") + deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test) deduplication_invariant.start() @@ -144,18 +183,24 @@ def test_deduplication_while_move(started_cluster): def test_part_move_step_by_step(started_cluster): for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): for replica_ix, r in enumerate(rs): - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_part_move_step_by_step; CREATE TABLE test_part_move_step_by_step(v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_part_move_step_by_step', '{}') ORDER BY tuple() - """.format(shard_ix, r.name)) + """.format( + shard_ix, r.name + ) + ) - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_part_move_step_by_step_d; CREATE TABLE test_part_move_step_by_step_d AS test_part_move_step_by_step ENGINE Distributed('test_cluster', currentDatabase(), test_part_move_step_by_step) - """) + """ + ) s0r0.query("SYSTEM STOP MERGES test_part_move_step_by_step") s0r1.query("SYSTEM STOP MERGES test_part_move_step_by_step") @@ -176,8 +221,10 @@ def test_part_move_step_by_step(started_cluster): n = random.choice(list(started_cluster.instances.values())) try: assert TSV( - n.query("SELECT * FROM test_part_move_step_by_step_d ORDER BY v", - settings={"allow_experimental_query_deduplication": 1}) + n.query( + "SELECT * FROM test_part_move_step_by_step_d ORDER BY v", + settings={"allow_experimental_query_deduplication": 1}, + ) ) == TSV(expected) except QueryRuntimeException as e: # ignore transient errors that are caused by us restarting nodes @@ -191,10 +238,16 @@ def test_part_move_step_by_step(started_cluster): s0r1.stop_clickhouse() s0r0.query( - "ALTER TABLE test_part_move_step_by_step MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step'") + "ALTER TABLE test_part_move_step_by_step MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step'" + ) # Should hang on SYNC_SOURCE until all source replicas acknowledge new pinned UUIDs. - wait_for_state("SYNC_SOURCE", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + wait_for_state( + "SYNC_SOURCE", + s0r0, + "test_part_move_step_by_step", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Start all replicas in source shard but stop a replica in destination shard @@ -203,10 +256,19 @@ def test_part_move_step_by_step(started_cluster): s0r1.start_clickhouse() # After SYNC_SOURCE step no merges will be assigned. - s0r0.query("SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;") - s0r1.query("SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;") + s0r0.query( + "SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;" + ) + s0r1.query( + "SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;" + ) - wait_for_state("SYNC_DESTINATION", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + wait_for_state( + "SYNC_DESTINATION", + s0r0, + "test_part_move_step_by_step", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Start previously stopped replica in destination shard to let SYNC_DESTINATION @@ -214,7 +276,12 @@ def test_part_move_step_by_step(started_cluster): # Stop the other replica in destination shard to prevent DESTINATION_FETCH succeed. s1r0.stop_clickhouse() s1r1.start_clickhouse() - wait_for_state("DESTINATION_FETCH", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + wait_for_state( + "DESTINATION_FETCH", + s0r0, + "test_part_move_step_by_step", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Start previously stopped replica in destination shard to let DESTINATION_FETCH @@ -222,14 +289,24 @@ def test_part_move_step_by_step(started_cluster): # Stop the other replica in destination shard to prevent DESTINATION_ATTACH succeed. s1r1.stop_clickhouse() s1r0.start_clickhouse() - wait_for_state("DESTINATION_ATTACH", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + wait_for_state( + "DESTINATION_ATTACH", + s0r0, + "test_part_move_step_by_step", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Start all replicas in destination shard to let DESTINATION_ATTACH succeed. # Stop a source replica to prevent SOURCE_DROP succeeding. s0r0.stop_clickhouse() s1r1.start_clickhouse() - wait_for_state("SOURCE_DROP", s0r1, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + wait_for_state( + "SOURCE_DROP", + s0r1, + "test_part_move_step_by_step", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() s0r0.start_clickhouse() @@ -249,18 +326,24 @@ def test_part_move_step_by_step(started_cluster): def test_part_move_step_by_step_kill(started_cluster): for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): for replica_ix, r in enumerate(rs): - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_part_move_step_by_step_kill; CREATE TABLE test_part_move_step_by_step_kill(v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_part_move_step_by_step_kill', '{}') ORDER BY tuple() - """.format(shard_ix, r.name)) + """.format( + shard_ix, r.name + ) + ) - r.query(""" + r.query( + """ DROP TABLE IF EXISTS test_part_move_step_by_step_kill_d; CREATE TABLE test_part_move_step_by_step_kill_d AS test_part_move_step_by_step_kill ENGINE Distributed('test_cluster', currentDatabase(), test_part_move_step_by_step_kill) - """) + """ + ) s0r0.query("SYSTEM STOP MERGES test_part_move_step_by_step_kill") s0r1.query("SYSTEM STOP MERGES test_part_move_step_by_step_kill") @@ -269,8 +352,14 @@ def test_part_move_step_by_step_kill(started_cluster): s0r0.query("INSERT INTO test_part_move_step_by_step_kill VALUES (2)") s0r1.query("SYSTEM SYNC REPLICA test_part_move_step_by_step_kill", timeout=20) - assert "2" == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() - assert "0" == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + assert ( + "2" + == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + ) + assert ( + "0" + == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + ) expected = """ 1 @@ -281,10 +370,10 @@ def test_part_move_step_by_step_kill(started_cluster): n = random.choice(list(started_cluster.instances.values())) try: assert TSV( - n.query("SELECT * FROM test_part_move_step_by_step_kill_d ORDER BY v", - settings={ - "allow_experimental_query_deduplication": 1 - }) + n.query( + "SELECT * FROM test_part_move_step_by_step_kill_d ORDER BY v", + settings={"allow_experimental_query_deduplication": 1}, + ) ) == TSV(expected) except QueryRuntimeException as e: # ignore transient errors that are caused by us restarting nodes @@ -298,10 +387,16 @@ def test_part_move_step_by_step_kill(started_cluster): s0r1.stop_clickhouse() s0r0.query( - "ALTER TABLE test_part_move_step_by_step_kill MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step_kill'") + "ALTER TABLE test_part_move_step_by_step_kill MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step_kill'" + ) # Should hang on SYNC_SOURCE until all source replicas acknowledge new pinned UUIDs. - wait_for_state("SYNC_SOURCE", s0r0, "test_part_move_step_by_step_kill", "Some replicas haven\\'t processed event") + wait_for_state( + "SYNC_SOURCE", + s0r0, + "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Start all replicas in source shard but stop a replica in destination shard @@ -310,11 +405,19 @@ def test_part_move_step_by_step_kill(started_cluster): s0r1.start_clickhouse() # After SYNC_SOURCE step no merges will be assigned. - s0r0.query("SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;") - s0r1.query("SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;") + s0r0.query( + "SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;" + ) + s0r1.query( + "SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;" + ) - wait_for_state("SYNC_DESTINATION", s0r0, "test_part_move_step_by_step_kill", - "Some replicas haven\\'t processed event") + wait_for_state( + "SYNC_DESTINATION", + s0r0, + "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Start previously stopped replica in destination shard to let SYNC_DESTINATION @@ -322,39 +425,61 @@ def test_part_move_step_by_step_kill(started_cluster): # Stop the other replica in destination shard to prevent DESTINATION_FETCH succeed. s1r0.stop_clickhouse() s1r1.start_clickhouse() - wait_for_state("DESTINATION_FETCH", s0r0, "test_part_move_step_by_step_kill", - "Some replicas haven\\'t processed event") + wait_for_state( + "DESTINATION_FETCH", + s0r0, + "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event", + ) # Start previously stopped replica in destination shard to let DESTINATION_FETCH # succeed. # Stop the other replica in destination shard to prevent DESTINATION_ATTACH succeed. s1r1.stop_clickhouse() s1r0.start_clickhouse() - wait_for_state("DESTINATION_ATTACH", s0r0, "test_part_move_step_by_step_kill", - "Some replicas haven\\'t processed event") + wait_for_state( + "DESTINATION_ATTACH", + s0r0, + "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event", + ) deduplication_invariant.assert_no_exception() # Rollback here. - s0r0.query(""" + s0r0.query( + """ KILL PART_MOVE_TO_SHARD WHERE task_uuid = (SELECT task_uuid FROM system.part_moves_between_shards WHERE table = 'test_part_move_step_by_step_kill') - """) + """ + ) - wait_for_state("DESTINATION_ATTACH", s0r0, "test_part_move_step_by_step_kill", - assert_exception_msg="Some replicas haven\\'t processed event", - assert_rollback=True) + wait_for_state( + "DESTINATION_ATTACH", + s0r0, + "test_part_move_step_by_step_kill", + assert_exception_msg="Some replicas haven\\'t processed event", + assert_rollback=True, + ) s1r1.start_clickhouse() - wait_for_state("CANCELLED", s0r0, "test_part_move_step_by_step_kill", assert_rollback=True) + wait_for_state( + "CANCELLED", s0r0, "test_part_move_step_by_step_kill", assert_rollback=True + ) deduplication_invariant.assert_no_exception() # No hung tasks in replication queue. Would timeout otherwise. for instance in started_cluster.instances.values(): instance.query("SYSTEM SYNC REPLICA test_part_move_step_by_step_kill") - assert "2" == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() - assert "0" == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + assert ( + "2" + == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + ) + assert ( + "0" + == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + ) deduplication_invariant.stop_and_assert_no_exception() @@ -368,40 +493,69 @@ def test_move_not_permitted(started_cluster): s1r0.start_clickhouse() for ix, n in enumerate([s0r0, s1r0]): - n.query(""" + n.query( + """ DROP TABLE IF EXISTS not_permitted_columns; CREATE TABLE not_permitted_columns(v_{ix} UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{ix}/tables/not_permitted_columns', 'r') ORDER BY tuple(); - """.format(ix=ix)) + """.format( + ix=ix + ) + ) partition = "date" if ix > 0: partition = "v" - n.query(""" + n.query( + """ DROP TABLE IF EXISTS not_permitted_partition; CREATE TABLE not_permitted_partition(date Date, v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{ix}/tables/not_permitted_partition', 'r') PARTITION BY ({partition}) ORDER BY tuple(); - """.format(ix=ix, partition=partition)) + """.format( + ix=ix, partition=partition + ) + ) s0r0.query("INSERT INTO not_permitted_columns VALUES (1)") s0r0.query("INSERT INTO not_permitted_partition VALUES ('2021-09-03', 1)") - with pytest.raises(QueryRuntimeException, match="DB::Exception: Source and destination are the same"): - s0r0.query("ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/not_permitted_columns'") + with pytest.raises( + QueryRuntimeException, + match="DB::Exception: Source and destination are the same", + ): + s0r0.query( + "ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/not_permitted_columns'" + ) - with pytest.raises(QueryRuntimeException, match="DB::Exception: Table columns structure in ZooKeeper is different from local table structure."): - s0r0.query("ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_columns'") + with pytest.raises( + QueryRuntimeException, + match="DB::Exception: Table columns structure in ZooKeeper is different from local table structure.", + ): + s0r0.query( + "ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_columns'" + ) - with pytest.raises(QueryRuntimeException, match="DB::Exception: Existing table metadata in ZooKeeper differs in partition key expression."): - s0r0.query("ALTER TABLE not_permitted_partition MOVE PART '20210903_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_partition'") + with pytest.raises( + QueryRuntimeException, + match="DB::Exception: Existing table metadata in ZooKeeper differs in partition key expression.", + ): + s0r0.query( + "ALTER TABLE not_permitted_partition MOVE PART '20210903_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_partition'" + ) -def wait_for_state(desired_state, instance, test_table, assert_exception_msg=None, assert_rollback=False): +def wait_for_state( + desired_state, + instance, + test_table, + assert_exception_msg=None, + assert_rollback=False, +): last_debug_print_time = time.time() print("Waiting to reach state: {}".format(desired_state)) @@ -411,9 +565,13 @@ def wait_for_state(desired_state, instance, test_table, assert_exception_msg=Non print(" and rollback: {}".format(assert_rollback)) while True: - tasks = TSV.toMat(instance.query( - "SELECT state, num_tries, last_exception, rollback FROM system.part_moves_between_shards WHERE table = '{}'".format( - test_table))) + tasks = TSV.toMat( + instance.query( + "SELECT state, num_tries, last_exception, rollback FROM system.part_moves_between_shards WHERE table = '{}'".format( + test_table + ) + ) + ) assert len(tasks) == 1, "only one task expected in this test" if time.time() - last_debug_print_time > 30: @@ -437,7 +595,11 @@ def wait_for_state(desired_state, instance, test_table, assert_exception_msg=Non break elif state in ["DONE", "CANCELLED"]: - raise Exception("Reached terminal state {}, but was waiting for {}".format(state, desired_state)) + raise Exception( + "Reached terminal state {}, but was waiting for {}".format( + state, desired_state + ) + ) time.sleep(0.1) @@ -454,7 +616,7 @@ class ConcurrentInvariant: def start(self): if self.started: - raise Exception('invariant thread already started') + raise Exception("invariant thread already started") self.started = True self.thread.start() @@ -485,4 +647,4 @@ class ConcurrentInvariant: def _assert_started(self): if not self.started: - raise Exception('invariant thread not started, forgot to call start?') + raise Exception("invariant thread not started, forgot to call start?") diff --git a/tests/integration/test_part_uuid/test.py b/tests/integration/test_part_uuid/test.py index 0353bf9266d..b30dd884427 100644 --- a/tests/integration/test_part_uuid/test.py +++ b/tests/integration/test_part_uuid/test.py @@ -7,14 +7,20 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( - 'node1', - main_configs=['configs/remote_servers.xml', 'configs/merge_tree_uuids.xml'], - with_zookeeper=True) + "node1", + main_configs=["configs/remote_servers.xml", "configs/merge_tree_uuids.xml"], + with_zookeeper=True, +) node2 = cluster.add_instance( - 'node2', - main_configs=['configs/remote_servers.xml', 'configs/merge_tree_uuids.xml', 'configs/merge_tree_in_memory.xml'], - with_zookeeper=True) + "node2", + main_configs=[ + "configs/remote_servers.xml", + "configs/merge_tree_uuids.xml", + "configs/merge_tree_in_memory.xml", + ], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -30,11 +36,15 @@ def test_part_uuid(started_cluster): uuid_zero = uuid.UUID(bytes=b"\x00" * 16) for ix, n in enumerate([node1, node2]): - n.query(""" + n.query( + """ CREATE TABLE t(key UInt64, value UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple() - """.format(ix)) + """.format( + ix + ) + ) # Test insert assigns uuid to part. node1.query("INSERT INTO t VALUES (1, 1)") @@ -42,28 +52,46 @@ def test_part_uuid(started_cluster): uuids = set() for node in [node1, node2]: node.query("SYSTEM SYNC REPLICA t") - part_initial_uuid = uuid.UUID(node.query("SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name").strip()) + part_initial_uuid = uuid.UUID( + node.query( + "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name" + ).strip() + ) uuids.add(part_initial_uuid) assert uuid_zero != part_initial_uuid assert len(uuids) == 1, "expect the same uuid on all the replicas" # Test detach / attach. - node1.query("ALTER TABLE t DETACH PARTITION tuple(); ALTER TABLE t ATTACH PARTITION tuple()") + node1.query( + "ALTER TABLE t DETACH PARTITION tuple(); ALTER TABLE t ATTACH PARTITION tuple()" + ) for node in [node1, node2]: node.query("SYSTEM SYNC REPLICA t") - part_reattach_uuid = uuid.UUID(node.query( - "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name").strip()) + part_reattach_uuid = uuid.UUID( + node.query( + "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name" + ).strip() + ) assert part_initial_uuid == part_reattach_uuid # Test mutation assigns new non-zero uuids. - node1.query("ALTER TABLE t UPDATE value = 1 WHERE key = 1 SETTINGS mutations_sync = 2") - part_mutate_uuid = uuid.UUID(node1.query("SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name").strip()) + node1.query( + "ALTER TABLE t UPDATE value = 1 WHERE key = 1 SETTINGS mutations_sync = 2" + ) + part_mutate_uuid = uuid.UUID( + node1.query( + "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name" + ).strip() + ) assert part_mutate_uuid not in [uuid_zero, part_initial_uuid] node2.query("SYSTEM SYNC REPLICA t") - assert part_mutate_uuid == uuid.UUID(node2.query( - "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name").strip()) + assert part_mutate_uuid == uuid.UUID( + node2.query( + "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name" + ).strip() + ) # Test merge assigns new non-zero uuids. node2.query("INSERT INTO t VALUES (1, 1)") @@ -72,8 +100,11 @@ def test_part_uuid(started_cluster): uuids = set() for node in [node1, node2]: node.query("SYSTEM SYNC REPLICA t") - part_merge_uuid = uuid.UUID(node.query( - "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name").strip()) + part_merge_uuid = uuid.UUID( + node.query( + "SELECT uuid FROM system.parts WHERE table = 't' AND active ORDER BY name" + ).strip() + ) uuids.add(part_merge_uuid) assert part_mutate_uuid not in [uuid_zero, part_merge_uuid] assert len(uuids) == 1, "expect the same uuid on all the replicas" @@ -83,19 +114,32 @@ def test_part_uuid_wal(started_cluster): uuid_zero = uuid.UUID(bytes=b"\x00" * 16) for ix, n in enumerate([node1, node2]): - n.query(""" + n.query( + """ CREATE TABLE t_wal(key UInt64, value UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t_wal', '{}') ORDER BY tuple() - """.format(ix)) + """.format( + ix + ) + ) node2.query("INSERT INTO t_wal VALUES (1, 1)") uuids = set() for node in [node1, node2]: node.query("SYSTEM SYNC REPLICA t_wal") - part_initial_uuid = uuid.UUID(node.query("SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name").strip()) - assert "InMemory" == node.query("SELECT part_type FROM system.parts WHERE table = 't_wal' AND active ORDER BY name").strip() + part_initial_uuid = uuid.UUID( + node.query( + "SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name" + ).strip() + ) + assert ( + "InMemory" + == node.query( + "SELECT part_type FROM system.parts WHERE table = 't_wal' AND active ORDER BY name" + ).strip() + ) uuids.add(part_initial_uuid) assert uuid_zero != part_initial_uuid assert len(uuids) == 1, "expect the same uuid on all the replicas" @@ -103,6 +147,9 @@ def test_part_uuid_wal(started_cluster): # Test detach / attach table to trigger WAL processing. for node in [node1, node2]: node.query("DETACH TABLE t_wal; ATTACH TABLE t_wal") - part_reattach_uuid = uuid.UUID(node.query( - "SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name").strip()) + part_reattach_uuid = uuid.UUID( + node.query( + "SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name" + ).strip() + ) assert part_initial_uuid == part_reattach_uuid diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index 0a44ae332c2..b396b58df10 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -4,16 +4,18 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") q = instance.query -path_to_data = '/var/lib/clickhouse/' +path_to_data = "/var/lib/clickhouse/" @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - q('CREATE DATABASE test ENGINE = Ordinary') # Different path in shadow/ with Atomic + q( + "CREATE DATABASE test ENGINE = Ordinary" + ) # Different path in shadow/ with Atomic yield cluster @@ -24,15 +26,17 @@ def started_cluster(): @pytest.fixture def partition_table_simple(started_cluster): q("DROP TABLE IF EXISTS test.partition_simple") - q("CREATE TABLE test.partition_simple (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) " - "ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) " - "SETTINGS index_granularity=8192, index_granularity_bytes=0") + q( + "CREATE TABLE test.partition_simple (date MATERIALIZED toDate(0), x UInt64, sample_key MATERIALIZED intHash64(x)) " + "ENGINE=MergeTree PARTITION BY date SAMPLE BY sample_key ORDER BY (date,x,sample_key) " + "SETTINGS index_granularity=8192, index_granularity_bytes=0" + ) q("INSERT INTO test.partition_simple ( x ) VALUES ( now() )") q("INSERT INTO test.partition_simple ( x ) VALUES ( now()+1 )") yield - q('DROP TABLE test.partition_simple') + q("DROP TABLE test.partition_simple") def test_partition_simple(partition_table_simple): @@ -42,55 +46,70 @@ def test_partition_simple(partition_table_simple): def partition_complex_assert_columns_txt(): - path_to_parts = path_to_data + 'data/test/partition_complex/' - parts = TSV(q("SELECT name FROM system.parts WHERE database='test' AND table='partition_complex'")) + path_to_parts = path_to_data + "data/test/partition_complex/" + parts = TSV( + q( + "SELECT name FROM system.parts WHERE database='test' AND table='partition_complex'" + ) + ) assert len(parts) > 0 for part_name in parts.lines: - path_to_columns = path_to_parts + part_name + '/columns.txt' + path_to_columns = path_to_parts + part_name + "/columns.txt" # 2 header lines + 3 columns - assert instance.exec_in_container(['wc', '-l', path_to_columns]).split()[0] == '5' + assert ( + instance.exec_in_container(["wc", "-l", path_to_columns]).split()[0] == "5" + ) def partition_complex_assert_checksums(): # Do not check increment.txt - it can be changed by other tests with FREEZE - cmd = ["bash", "-c", f"cd {path_to_data} && find shadow -type f -exec" + " md5sum {} \\; | grep partition_complex" \ - " | sed 's shadow/[0-9]*/data/[a-z0-9_-]*/ shadow/1/data/test/ g' | sort | uniq"] + cmd = [ + "bash", + "-c", + f"cd {path_to_data} && find shadow -type f -exec" + + " md5sum {} \\; | grep partition_complex" + " | sed 's shadow/[0-9]*/data/[a-z0-9_-]*/ shadow/1/data/test/ g' | sort | uniq", + ] - checksums = "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.bin\n" \ - "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.bin\n" \ - "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition_complex/19700102_2_2_0/minmax_p.idx\n" \ - "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/partition.dat\n" \ - "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition_complex/19700201_1_1_0/partition.dat\n" \ - "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition_complex/19700102_2_2_0/checksums.txt\n" \ - "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.bin\n" \ - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.mrk\n" \ - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.mrk\n" \ - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.mrk\n" \ - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.mrk\n" \ - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.mrk\n" \ - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.mrk\n" \ - "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition_complex/19700201_1_1_0/primary.idx\n" \ - "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition_complex/19700201_1_1_0/checksums.txt\n" \ - "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700102_2_2_0/columns.txt\n" \ - "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700201_1_1_0/columns.txt\n" \ - "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.bin\n" \ - "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition_complex/19700102_2_2_0/primary.idx\n" \ - "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700102_2_2_0/default_compression_codec.txt\n" \ - "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700201_1_1_0/default_compression_codec.txt\n" \ - "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" \ - "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" \ - "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" \ - "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" \ - "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" + checksums = ( + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.bin\n" + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.bin\n" + "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition_complex/19700102_2_2_0/minmax_p.idx\n" + "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/partition.dat\n" + "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition_complex/19700201_1_1_0/partition.dat\n" + "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition_complex/19700102_2_2_0/checksums.txt\n" + "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.bin\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.mrk\n" + "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition_complex/19700201_1_1_0/primary.idx\n" + "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition_complex/19700201_1_1_0/checksums.txt\n" + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700102_2_2_0/columns.txt\n" + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700201_1_1_0/columns.txt\n" + "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.bin\n" + "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition_complex/19700102_2_2_0/primary.idx\n" + "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700102_2_2_0/default_compression_codec.txt\n" + "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700201_1_1_0/default_compression_codec.txt\n" + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" + "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" + "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" + "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" + ) - assert TSV(instance.exec_in_container(cmd).replace(' ', '\t')) == TSV(checksums) + assert TSV(instance.exec_in_container(cmd).replace(" ", "\t")) == TSV(checksums) @pytest.fixture def partition_table_complex(started_cluster): q("DROP TABLE IF EXISTS test.partition_complex") - q("CREATE TABLE test.partition_complex (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) " - "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0") + q( + "CREATE TABLE test.partition_complex (p Date, k Int8, v1 Int8 MATERIALIZED k + 1) " + "ENGINE = MergeTree PARTITION BY p ORDER BY k SETTINGS index_granularity=1, index_granularity_bytes=0" + ) q("INSERT INTO test.partition_complex (p, k) VALUES(toDate(31), 1)") q("INSERT INTO test.partition_complex (p, k) VALUES(toDate(1), 2)") @@ -118,16 +137,17 @@ def test_partition_complex(partition_table_complex): q("OPTIMIZE TABLE test.partition_complex") - expected = TSV('31\t1\t2\n' - '1\t2\t3') + expected = TSV("31\t1\t2\n" "1\t2\t3") res = q("SELECT toUInt16(p), k, v1 FROM test.partition_complex ORDER BY k") - assert (TSV(res) == expected) + assert TSV(res) == expected @pytest.fixture def cannot_attach_active_part_table(started_cluster): q("DROP TABLE IF EXISTS test.attach_active") - q("CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n") + q( + "CREATE TABLE test.attach_active (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 4) ORDER BY n" + ) q("INSERT INTO test.attach_active SELECT number FROM system.numbers LIMIT 16") yield @@ -136,22 +156,32 @@ def cannot_attach_active_part_table(started_cluster): def test_cannot_attach_active_part(cannot_attach_active_part_table): - error = instance.client.query_and_get_error("ALTER TABLE test.attach_active ATTACH PART '../1_2_2_0'") + error = instance.client.query_and_get_error( + "ALTER TABLE test.attach_active ATTACH PART '../1_2_2_0'" + ) print(error) - assert 0 <= error.find('Invalid part name') + assert 0 <= error.find("Invalid part name") - res = q("SElECT name FROM system.parts WHERE table='attach_active' AND database='test' ORDER BY name") - assert TSV(res) == TSV('0_1_1_0\n1_2_2_0\n2_3_3_0\n3_4_4_0') - assert TSV(q("SElECT count(), sum(n) FROM test.attach_active")) == TSV('16\t120') + res = q( + "SElECT name FROM system.parts WHERE table='attach_active' AND database='test' ORDER BY name" + ) + assert TSV(res) == TSV("0_1_1_0\n1_2_2_0\n2_3_3_0\n3_4_4_0") + assert TSV(q("SElECT count(), sum(n) FROM test.attach_active")) == TSV("16\t120") @pytest.fixture def attach_check_all_parts_table(started_cluster): q("SYSTEM STOP MERGES") q("DROP TABLE IF EXISTS test.attach_partition") - q("CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") - q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") - q("INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + q( + "CREATE TABLE test.attach_partition (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n" + ) + q( + "INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8" + ) + q( + "INSERT INTO test.attach_partition SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8" + ) yield @@ -162,40 +192,74 @@ def attach_check_all_parts_table(started_cluster): def test_attach_check_all_parts(attach_check_all_parts_table): q("ALTER TABLE test.attach_partition DETACH PARTITION 0") - path_to_detached = path_to_data + 'data/test/attach_partition/detached/' - instance.exec_in_container(['mkdir', '{}'.format(path_to_detached + '0_5_5_0')]) - instance.exec_in_container(['cp', '-pr', path_to_detached + '0_1_1_0', path_to_detached + 'attaching_0_6_6_0']) - instance.exec_in_container(['cp', '-pr', path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0']) + path_to_detached = path_to_data + "data/test/attach_partition/detached/" + instance.exec_in_container(["mkdir", "{}".format(path_to_detached + "0_5_5_0")]) + instance.exec_in_container( + [ + "cp", + "-pr", + path_to_detached + "0_1_1_0", + path_to_detached + "attaching_0_6_6_0", + ] + ) + instance.exec_in_container( + [ + "cp", + "-pr", + path_to_detached + "0_3_3_0", + path_to_detached + "deleting_0_7_7_0", + ] + ) - error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0") - assert 0 <= error.find('No columns in part 0_5_5_0') or 0 <= error.find('No columns.txt in part 0_5_5_0') + error = instance.client.query_and_get_error( + "ALTER TABLE test.attach_partition ATTACH PARTITION 0" + ) + assert 0 <= error.find("No columns in part 0_5_5_0") or 0 <= error.find( + "No columns.txt in part 0_5_5_0" + ) - parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") - assert TSV(parts) == TSV('1_2_2_0\n1_4_4_0') - detached = q("SELECT name FROM system.detached_parts " - "WHERE table='attach_partition' AND database='test' ORDER BY name") - assert TSV(detached) == TSV('0_1_1_0\n0_3_3_0\n0_5_5_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + parts = q( + "SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name" + ) + assert TSV(parts) == TSV("1_2_2_0\n1_4_4_0") + detached = q( + "SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name" + ) + assert TSV(detached) == TSV( + "0_1_1_0\n0_3_3_0\n0_5_5_0\nattaching_0_6_6_0\ndeleting_0_7_7_0" + ) - instance.exec_in_container(['rm', '-r', path_to_detached + '0_5_5_0']) + instance.exec_in_container(["rm", "-r", path_to_detached + "0_5_5_0"]) q("ALTER TABLE test.attach_partition ATTACH PARTITION 0") - parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name") - expected = '0_5_5_0\n0_6_6_0\n1_2_2_0\n1_4_4_0' + parts = q( + "SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name" + ) + expected = "0_5_5_0\n0_6_6_0\n1_2_2_0\n1_4_4_0" assert TSV(parts) == TSV(expected) - assert TSV(q("SElECT count(), sum(n) FROM test.attach_partition")) == TSV('16\t120') + assert TSV(q("SElECT count(), sum(n) FROM test.attach_partition")) == TSV("16\t120") - detached = q("SELECT name FROM system.detached_parts " - "WHERE table='attach_partition' AND database='test' ORDER BY name") - assert TSV(detached) == TSV('attaching_0_6_6_0\ndeleting_0_7_7_0') + detached = q( + "SELECT name FROM system.detached_parts " + "WHERE table='attach_partition' AND database='test' ORDER BY name" + ) + assert TSV(detached) == TSV("attaching_0_6_6_0\ndeleting_0_7_7_0") @pytest.fixture def drop_detached_parts_table(started_cluster): q("SYSTEM STOP MERGES") q("DROP TABLE IF EXISTS test.drop_detached") - q("CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n") - q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8") - q("INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8") + q( + "CREATE TABLE test.drop_detached (n UInt64) ENGINE = MergeTree() PARTITION BY intDiv(n, 8) ORDER BY n" + ) + q( + "INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 0 LIMIT 8" + ) + q( + "INSERT INTO test.drop_detached SELECT number FROM system.numbers WHERE number % 2 = 1 LIMIT 8" + ) yield @@ -208,126 +272,208 @@ def test_drop_detached_parts(drop_detached_parts_table): q("ALTER TABLE test.drop_detached DETACH PARTITION 0") q("ALTER TABLE test.drop_detached DETACH PARTITION 1") - path_to_detached = path_to_data + 'data/test/drop_detached/detached/' - instance.exec_in_container(['mkdir', '{}'.format(path_to_detached + 'attaching_0_6_6_0')]) - instance.exec_in_container(['mkdir', '{}'.format(path_to_detached + 'deleting_0_7_7_0')]) - instance.exec_in_container(['mkdir', '{}'.format(path_to_detached + 'any_other_name')]) - instance.exec_in_container(['mkdir', '{}'.format(path_to_detached + 'prefix_1_2_2_0_0')]) + path_to_detached = path_to_data + "data/test/drop_detached/detached/" + instance.exec_in_container( + ["mkdir", "{}".format(path_to_detached + "attaching_0_6_6_0")] + ) + instance.exec_in_container( + ["mkdir", "{}".format(path_to_detached + "deleting_0_7_7_0")] + ) + instance.exec_in_container( + ["mkdir", "{}".format(path_to_detached + "any_other_name")] + ) + instance.exec_in_container( + ["mkdir", "{}".format(path_to_detached + "prefix_1_2_2_0_0")] + ) - error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", - settings=s) - assert 0 <= error.find('Invalid part name') + error = instance.client.query_and_get_error( + "ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", settings=s + ) + assert 0 <= error.find("Invalid part name") q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0'", settings=s) - error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", - settings=s) - assert 0 <= error.find('Cannot drop part') + error = instance.client.query_and_get_error( + "ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", + settings=s, + ) + assert 0 <= error.find("Cannot drop part") - error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", - settings=s) - assert 0 <= error.find('Cannot drop part') + error = instance.client.query_and_get_error( + "ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", + settings=s, + ) + assert 0 <= error.find("Cannot drop part") q("ALTER TABLE test.drop_detached DROP DETACHED PART 'any_other_name'", settings=s) - detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") - assert TSV(detached) == TSV('0_3_3_0\n1_2_2_0\n1_4_4_0\nattaching_0_6_6_0\ndeleting_0_7_7_0\nprefix_1_2_2_0_0') + detached = q( + "SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name" + ) + assert TSV(detached) == TSV( + "0_3_3_0\n1_2_2_0\n1_4_4_0\nattaching_0_6_6_0\ndeleting_0_7_7_0\nprefix_1_2_2_0_0" + ) q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s) - detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name") - assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0') + detached = q( + "SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name" + ) + assert TSV(detached) == TSV("0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0") + def test_system_detached_parts(drop_detached_parts_table): q("create table sdp_0 (n int, x int) engine=MergeTree order by n") q("create table sdp_1 (n int, x int) engine=MergeTree order by n partition by x") q("create table sdp_2 (n int, x String) engine=MergeTree order by n partition by x") - q("create table sdp_3 (n int, x Enum('broken' = 0, 'all' = 1)) engine=MergeTree order by n partition by x") + q( + "create table sdp_3 (n int, x Enum('broken' = 0, 'all' = 1)) engine=MergeTree order by n partition by x" + ) for i in range(0, 4): q("system stop merges sdp_{}".format(i)) q("insert into sdp_{} values (0, 0)".format(i)) q("insert into sdp_{} values (1, 1)".format(i)) - for p in q("select distinct partition_id from system.parts where table='sdp_{}'".format(i))[:-1].split('\n'): + for p in q( + "select distinct partition_id from system.parts where table='sdp_{}'".format( + i + ) + )[:-1].split("\n"): q("alter table sdp_{} detach partition id '{}'".format(i, p)) - path_to_detached = path_to_data + 'data/default/sdp_{}/detached/{}' + path_to_detached = path_to_data + "data/default/sdp_{}/detached/{}" for i in range(0, 4): - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'attaching_0_6_6_0')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'deleting_0_7_7_0')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'any_other_name')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'prefix_1_2_2_0_0')]) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "attaching_0_6_6_0")] + ) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "deleting_0_7_7_0")] + ) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "any_other_name")] + ) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "prefix_1_2_2_0_0")] + ) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'ignored_202107_714380_714380_0')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'broken_202107_714380_714380_123')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'clone_all_714380_714380_42')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'clone_all_714380_714380_42_123')]) - instance.exec_in_container(['mkdir', path_to_detached.format(i, 'broken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123')]) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "ignored_202107_714380_714380_0")] + ) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "broken_202107_714380_714380_123")] + ) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "clone_all_714380_714380_42")] + ) + instance.exec_in_container( + ["mkdir", path_to_detached.format(i, "clone_all_714380_714380_42_123")] + ) + instance.exec_in_container( + [ + "mkdir", + path_to_detached.format( + i, + "broken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123", + ), + ] + ) - res = q("select * from system.detached_parts where table like 'sdp_%' order by table, name") - assert res == \ - "default\tsdp_0\tall\tall_1_1_0\tdefault\t\t1\t1\t0\n" \ - "default\tsdp_0\tall\tall_2_2_0\tdefault\t\t2\t2\t0\n" \ - "default\tsdp_0\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ - "default\tsdp_0\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ - "default\tsdp_0\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ - "default\tsdp_0\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ - "default\tsdp_0\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_0\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_0\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ - "default\tsdp_0\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ - "default\tsdp_0\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" \ - "default\tsdp_1\t0\t0_1_1_0\tdefault\t\t1\t1\t0\n" \ - "default\tsdp_1\t1\t1_2_2_0\tdefault\t\t2\t2\t0\n" \ - "default\tsdp_1\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ - "default\tsdp_1\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ - "default\tsdp_1\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ - "default\tsdp_1\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ - "default\tsdp_1\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_1\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_1\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ - "default\tsdp_1\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ - "default\tsdp_1\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" \ - "default\tsdp_2\t58ed7160db50ea45e1c6aa694c8cbfd1\t58ed7160db50ea45e1c6aa694c8cbfd1_1_1_0\tdefault\t\t1\t1\t0\n" \ - "default\tsdp_2\t6711e2b2592d86d18fc0f260cf33ef2b\t6711e2b2592d86d18fc0f260cf33ef2b_2_2_0\tdefault\t\t2\t2\t0\n" \ - "default\tsdp_2\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ - "default\tsdp_2\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ - "default\tsdp_2\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ - "default\tsdp_2\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ - "default\tsdp_2\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_2\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_2\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ - "default\tsdp_2\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ - "default\tsdp_2\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" \ - "default\tsdp_3\t0\t0_1_1_0\tdefault\t\t1\t1\t0\n" \ - "default\tsdp_3\t1\t1_2_2_0\tdefault\t\t2\t2\t0\n" \ - "default\tsdp_3\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" \ - "default\tsdp_3\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" \ - "default\tsdp_3\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" \ - "default\tsdp_3\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" \ - "default\tsdp_3\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_3\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" \ - "default\tsdp_3\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" \ - "default\tsdp_3\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" \ + res = q( + "select * from system.detached_parts where table like 'sdp_%' order by table, name" + ) + assert ( + res == "default\tsdp_0\tall\tall_1_1_0\tdefault\t\t1\t1\t0\n" + "default\tsdp_0\tall\tall_2_2_0\tdefault\t\t2\t2\t0\n" + "default\tsdp_0\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" + "default\tsdp_0\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" + "default\tsdp_0\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" + "default\tsdp_0\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" + "default\tsdp_0\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_0\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_0\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" + "default\tsdp_0\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" + "default\tsdp_0\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" + "default\tsdp_1\t0\t0_1_1_0\tdefault\t\t1\t1\t0\n" + "default\tsdp_1\t1\t1_2_2_0\tdefault\t\t2\t2\t0\n" + "default\tsdp_1\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" + "default\tsdp_1\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" + "default\tsdp_1\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" + "default\tsdp_1\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" + "default\tsdp_1\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_1\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_1\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" + "default\tsdp_1\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" + "default\tsdp_1\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" + "default\tsdp_2\t58ed7160db50ea45e1c6aa694c8cbfd1\t58ed7160db50ea45e1c6aa694c8cbfd1_1_1_0\tdefault\t\t1\t1\t0\n" + "default\tsdp_2\t6711e2b2592d86d18fc0f260cf33ef2b\t6711e2b2592d86d18fc0f260cf33ef2b_2_2_0\tdefault\t\t2\t2\t0\n" + "default\tsdp_2\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" + "default\tsdp_2\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" + "default\tsdp_2\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" + "default\tsdp_2\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" + "default\tsdp_2\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_2\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_2\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" + "default\tsdp_2\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" + "default\tsdp_2\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" + "default\tsdp_3\t0\t0_1_1_0\tdefault\t\t1\t1\t0\n" + "default\tsdp_3\t1\t1_2_2_0\tdefault\t\t2\t2\t0\n" + "default\tsdp_3\t\\N\tany_other_name\tdefault\t\\N\t\\N\t\\N\t\\N\n" + "default\tsdp_3\t0\tattaching_0_6_6_0\tdefault\tattaching\t6\t6\t0\n" + "default\tsdp_3\t6711e2b2592d86d18fc0f260cf33ef2b\tbroken-on-start_6711e2b2592d86d18fc0f260cf33ef2b_714380_714380_42_123\tdefault\tbroken-on-start\t714380\t714380\t42\n" + "default\tsdp_3\t202107\tbroken_202107_714380_714380_123\tdefault\tbroken\t714380\t714380\t123\n" + "default\tsdp_3\tall\tclone_all_714380_714380_42\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_3\tall\tclone_all_714380_714380_42_123\tdefault\tclone\t714380\t714380\t42\n" + "default\tsdp_3\t0\tdeleting_0_7_7_0\tdefault\tdeleting\t7\t7\t0\n" + "default\tsdp_3\t202107\tignored_202107_714380_714380_0\tdefault\tignored\t714380\t714380\t0\n" "default\tsdp_3\t1\tprefix_1_2_2_0_0\tdefault\tprefix\t2\t2\t0\n" + ) for i in range(0, 4): - for p in q("select distinct partition_id from system.detached_parts where table='sdp_{}' and partition_id is not null".format(i))[:-1].split('\n'): + for p in q( + "select distinct partition_id from system.detached_parts where table='sdp_{}' and partition_id is not null".format( + i + ) + )[:-1].split("\n"): q("alter table sdp_{} attach partition id '{}'".format(i, p)) - assert q("select n, x, count() from merge('default', 'sdp_') group by n, x") == "0\t0\t4\n1\t1\t4\n" + assert ( + q("select n, x, count() from merge('default', 'sdp_') group by n, x") + == "0\t0\t4\n1\t1\t4\n" + ) def test_detached_part_dir_exists(started_cluster): q("create table detached_part_dir_exists (n int) engine=MergeTree order by n") q("insert into detached_part_dir_exists select 1") # will create all_1_1_0 - q("alter table detached_part_dir_exists detach partition id 'all'") # will move all_1_1_0 to detached/all_1_1_0 + q( + "alter table detached_part_dir_exists detach partition id 'all'" + ) # will move all_1_1_0 to detached/all_1_1_0 q("detach table detached_part_dir_exists") q("attach table detached_part_dir_exists") q("insert into detached_part_dir_exists select 1") # will create all_1_1_0 q("insert into detached_part_dir_exists select 1") # will create all_2_2_0 - instance.exec_in_container(['bash', '-c', 'mkdir /var/lib/clickhouse/data/default/detached_part_dir_exists/detached/all_2_2_0'], privileged=True) - instance.exec_in_container(['bash', '-c', 'touch /var/lib/clickhouse/data/default/detached_part_dir_exists/detached/all_2_2_0/file'], privileged=True) - q("alter table detached_part_dir_exists detach partition id 'all'") # directories already exist, but it's ok - assert q("select name from system.detached_parts where table='detached_part_dir_exists' order by name") == \ - "all_1_1_0\nall_1_1_0_try1\nall_2_2_0\nall_2_2_0_try1\n" + instance.exec_in_container( + [ + "bash", + "-c", + "mkdir /var/lib/clickhouse/data/default/detached_part_dir_exists/detached/all_2_2_0", + ], + privileged=True, + ) + instance.exec_in_container( + [ + "bash", + "-c", + "touch /var/lib/clickhouse/data/default/detached_part_dir_exists/detached/all_2_2_0/file", + ], + privileged=True, + ) + q( + "alter table detached_part_dir_exists detach partition id 'all'" + ) # directories already exist, but it's ok + assert ( + q( + "select name from system.detached_parts where table='detached_part_dir_exists' order by name" + ) + == "all_1_1_0\nall_1_1_0_try1\nall_2_2_0\nall_2_2_0_try1\n" + ) q("drop table detached_part_dir_exists") diff --git a/tests/integration/test_parts_delete_zookeeper/test.py b/tests/integration/test_parts_delete_zookeeper/test.py index 62e14b68bd1..956f7ab21e2 100644 --- a/tests/integration/test_parts_delete_zookeeper/test.py +++ b/tests/integration/test_parts_delete_zookeeper/test.py @@ -6,7 +6,9 @@ from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -15,12 +17,12 @@ def start_cluster(): cluster.start() node1.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', 'node1') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS old_parts_lifetime=4, cleanup_delay_period=1; - ''' + """ ) yield cluster @@ -34,31 +36,69 @@ def start_cluster(): # Test that outdated parts are not removed when they cannot be removed from zookeeper def test_merge_doesnt_work_without_zookeeper(start_cluster): - node1.query("INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") - node1.query("INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)") - assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "2\n" + node1.query( + "INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)" + ) + node1.query( + "INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)" + ) + assert ( + node1.query("SELECT count(*) from system.parts where table = 'test_table'") + == "2\n" + ) node1.query("OPTIMIZE TABLE test_table FINAL") - assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "3\n" + assert ( + node1.query("SELECT count(*) from system.parts where table = 'test_table'") + == "3\n" + ) - assert_eq_with_retry(node1, "SELECT count(*) from system.parts where table = 'test_table' and active = 1", "1") + assert_eq_with_retry( + node1, + "SELECT count(*) from system.parts where table = 'test_table' and active = 1", + "1", + ) node1.query("TRUNCATE TABLE test_table") - assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "0\n" + assert ( + node1.query("SELECT count(*) from system.parts where table = 'test_table'") + == "0\n" + ) - node1.query("INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") - node1.query("INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)") - assert node1.query("SELECT count(*) from system.parts where table = 'test_table' and active") == "2\n" + node1.query( + "INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)" + ) + node1.query( + "INSERT INTO test_table VALUES ('2018-10-01', 4), ('2018-10-02', 5), ('2018-10-03', 6)" + ) + assert ( + node1.query( + "SELECT count(*) from system.parts where table = 'test_table' and active" + ) + == "2\n" + ) with PartitionManager() as pm: node1.query("OPTIMIZE TABLE test_table FINAL") pm.drop_instance_zk_connections(node1) # unfortunately we can be too fast and delete node before partition with ZK - if node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "1\n": + if ( + node1.query("SELECT count(*) from system.parts where table = 'test_table'") + == "1\n" + ): print("We were too fast and deleted parts before partition with ZK") else: time.sleep(10) # > old_parts_lifetime - assert node1.query("SELECT count(*) from system.parts where table = 'test_table'") == "3\n" + assert ( + node1.query( + "SELECT count(*) from system.parts where table = 'test_table'" + ) + == "3\n" + ) - assert_eq_with_retry(node1, "SELECT count(*) from system.parts where table = 'test_table' and active = 1", "1") + assert_eq_with_retry( + node1, + "SELECT count(*) from system.parts where table = 'test_table' and active = 1", + "1", + ) diff --git a/tests/integration/test_passing_max_partitions_to_read_remotely/test.py b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py index 45b3dd00b2a..e64ca7ece33 100644 --- a/tests/integration/test_passing_max_partitions_to_read_remotely/test.py +++ b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py @@ -23,6 +23,9 @@ def test_default_database_on_cluster(started_cluster): sql="CREATE TABLE test_local_table ENGINE MergeTree PARTITION BY i ORDER BY tuple() SETTINGS max_partitions_to_read = 1 AS SELECT arrayJoin([1, 2]) i;", ) - assert ch2.query( - sql="SELECT * FROM remote('ch1:9000', test_default_database, test_local_table) ORDER BY i FORMAT TSV SETTINGS max_partitions_to_read = 0;", - ) == "1\n2\n" + assert ( + ch2.query( + sql="SELECT * FROM remote('ch1:9000', test_default_database, test_local_table) ORDER BY i FORMAT TSV SETTINGS max_partitions_to_read = 0;", + ) + == "1\n2\n" + ) diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index 9fe3ef77da8..ba40b46c586 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -18,26 +18,33 @@ def get_random_array(): def get_random_string(): length = random.randint(0, 1000) - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) def insert_random_data(table, node, size): data = [ - '(' + ','.join(( - "'2019-10-11'", - str(i), - "'" + get_random_string() + "'", - str(get_random_array()))) + - ')' for i in range(size) + "(" + + ",".join( + ( + "'2019-10-11'", + str(i), + "'" + get_random_string() + "'", + str(get_random_array()), + ) + ) + + ")" + for i in range(size) ] - node.query("INSERT INTO {} VALUES {}".format(table, ','.join(data))) + node.query("INSERT INTO {} VALUES {}".format(table, ",".join(data))) def create_tables(name, nodes, node_settings, shard): for i, (node, settings) in enumerate(zip(nodes, node_settings)): node.query( - ''' + """ CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}') PARTITION BY toYYYYMM(date) @@ -46,46 +53,105 @@ def create_tables(name, nodes, node_settings, shard): min_rows_for_wide_part = {min_rows_for_wide_part}, min_rows_for_compact_part = {min_rows_for_compact_part}, min_bytes_for_wide_part = 0, min_bytes_for_compact_part = 0, in_memory_parts_enable_wal = 1 - '''.format(name=name, shard=shard, repl=i, **settings)) + """.format( + name=name, shard=shard, repl=i, **settings + ) + ) + def create_tables_old_format(name, nodes, shard): for i, node in enumerate(nodes): node.query( - ''' + """ CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/{name}', '{repl}', date, id, 64) - '''.format(name=name, shard=shard, repl=i)) + """.format( + name=name, shard=shard, repl=i + ) + ) -node1 = cluster.add_instance('node1', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], - with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], - with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=[], + user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=[], + user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True, +) -settings_default = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} -settings_compact_only = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 1000000, - 'min_rows_for_compact_part': 0} -settings_not_adaptive = {'index_granularity_bytes': 0, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} +settings_default = { + "index_granularity_bytes": 10485760, + "min_rows_for_wide_part": 512, + "min_rows_for_compact_part": 0, +} +settings_compact_only = { + "index_granularity_bytes": 10485760, + "min_rows_for_wide_part": 1000000, + "min_rows_for_compact_part": 0, +} +settings_not_adaptive = { + "index_granularity_bytes": 0, + "min_rows_for_wide_part": 512, + "min_rows_for_compact_part": 0, +} -node3 = cluster.add_instance('node3', main_configs=[], user_configs=["configs/users.d/not_optimize_count.xml"], - with_zookeeper=True) -node4 = cluster.add_instance('node4', user_configs=["configs/users.d/not_optimize_count.xml"], - main_configs=['configs/no_leader.xml'], with_zookeeper=True) +node3 = cluster.add_instance( + "node3", + main_configs=[], + user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + user_configs=["configs/users.d/not_optimize_count.xml"], + main_configs=["configs/no_leader.xml"], + with_zookeeper=True, +) -settings_compact = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} -settings_wide = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 0, 'min_rows_for_compact_part': 0} +settings_compact = { + "index_granularity_bytes": 10485760, + "min_rows_for_wide_part": 512, + "min_rows_for_compact_part": 0, +} +settings_wide = { + "index_granularity_bytes": 10485760, + "min_rows_for_wide_part": 0, + "min_rows_for_compact_part": 0, +} -node5 = cluster.add_instance('node5', main_configs=['configs/compact_parts.xml'], with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=['configs/compact_parts.xml'], with_zookeeper=True) +node5 = cluster.add_instance( + "node5", main_configs=["configs/compact_parts.xml"], with_zookeeper=True +) +node6 = cluster.add_instance( + "node6", main_configs=["configs/compact_parts.xml"], with_zookeeper=True +) -settings_in_memory = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, - 'min_rows_for_compact_part': 256} +settings_in_memory = { + "index_granularity_bytes": 10485760, + "min_rows_for_wide_part": 512, + "min_rows_for_compact_part": 256, +} -node9 = cluster.add_instance('node9', with_zookeeper=True, stay_alive=True) -node10 = cluster.add_instance('node10', with_zookeeper=True) +node9 = cluster.add_instance("node9", with_zookeeper=True, stay_alive=True) +node10 = cluster.add_instance("node10", with_zookeeper=True) -node11 = cluster.add_instance('node11', main_configs=['configs/do_not_merge.xml'], with_zookeeper=True, stay_alive=True) -node12 = cluster.add_instance('node12', main_configs=['configs/do_not_merge.xml'], with_zookeeper=True, stay_alive=True) +node11 = cluster.add_instance( + "node11", + main_configs=["configs/do_not_merge.xml"], + with_zookeeper=True, + stay_alive=True, +) +node12 = cluster.add_instance( + "node12", + main_configs=["configs/do_not_merge.xml"], + with_zookeeper=True, + stay_alive=True, +) @pytest.fixture(scope="module") @@ -93,18 +159,73 @@ def start_cluster(): try: cluster.start() - create_tables('polymorphic_table', [node1, node2], [settings_default, settings_default], "shard1") - create_tables('compact_parts_only', [node1, node2], [settings_compact_only, settings_compact_only], "shard1") - create_tables('non_adaptive_table', [node1, node2], [settings_not_adaptive, settings_not_adaptive], "shard1") - create_tables('polymorphic_table_compact', [node3, node4], [settings_compact, settings_wide], "shard2") - create_tables('polymorphic_table_wide', [node3, node4], [settings_wide, settings_compact], "shard2") - create_tables_old_format('polymorphic_table', [node5, node6], "shard3") - create_tables('in_memory_table', [node9, node10], [settings_in_memory, settings_in_memory], "shard4") - create_tables('wal_table', [node11, node12], [settings_in_memory, settings_in_memory], "shard4") - create_tables('restore_table', [node11, node12], [settings_in_memory, settings_in_memory], "shard5") - create_tables('deduplication_table', [node9, node10], [settings_in_memory, settings_in_memory], "shard5") - create_tables('sync_table', [node9, node10], [settings_in_memory, settings_in_memory], "shard5") - create_tables('alters_table', [node9, node10], [settings_in_memory, settings_in_memory], "shard5") + create_tables( + "polymorphic_table", + [node1, node2], + [settings_default, settings_default], + "shard1", + ) + create_tables( + "compact_parts_only", + [node1, node2], + [settings_compact_only, settings_compact_only], + "shard1", + ) + create_tables( + "non_adaptive_table", + [node1, node2], + [settings_not_adaptive, settings_not_adaptive], + "shard1", + ) + create_tables( + "polymorphic_table_compact", + [node3, node4], + [settings_compact, settings_wide], + "shard2", + ) + create_tables( + "polymorphic_table_wide", + [node3, node4], + [settings_wide, settings_compact], + "shard2", + ) + create_tables_old_format("polymorphic_table", [node5, node6], "shard3") + create_tables( + "in_memory_table", + [node9, node10], + [settings_in_memory, settings_in_memory], + "shard4", + ) + create_tables( + "wal_table", + [node11, node12], + [settings_in_memory, settings_in_memory], + "shard4", + ) + create_tables( + "restore_table", + [node11, node12], + [settings_in_memory, settings_in_memory], + "shard5", + ) + create_tables( + "deduplication_table", + [node9, node10], + [settings_in_memory, settings_in_memory], + "shard5", + ) + create_tables( + "sync_table", + [node9, node10], + [settings_in_memory, settings_in_memory], + "shard5", + ) + create_tables( + "alters_table", + [node9, node10], + [settings_in_memory, settings_in_memory], + "shard5", + ) yield cluster @@ -113,18 +234,18 @@ def start_cluster(): @pytest.mark.parametrize( - ('first_node', 'second_node'), + ("first_node", "second_node"), [ (node1, node2), # compact parts (node5, node6), # compact parts, old-format - ] + ], ) def test_polymorphic_parts_basics(start_cluster, first_node, second_node): first_node.query("SYSTEM STOP MERGES") second_node.query("SYSTEM STOP MERGES") for size in [300, 300, 600]: - insert_random_data('polymorphic_table', first_node, size) + insert_random_data("polymorphic_table", first_node, size) second_node.query("SYSTEM SYNC REPLICA polymorphic_table", timeout=20) assert first_node.query("SELECT count() FROM polymorphic_table") == "1200\n" @@ -132,19 +253,25 @@ def test_polymorphic_parts_basics(start_cluster, first_node, second_node): expected = "Compact\t2\nWide\t1\n" - assert TSV(first_node.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - expected) - assert TSV(second_node.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - expected) + assert TSV( + first_node.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV(expected) + assert TSV( + second_node.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'polymorphic_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV(expected) first_node.query("SYSTEM START MERGES") second_node.query("SYSTEM START MERGES") for _ in range(40): - insert_random_data('polymorphic_table', first_node, 10) - insert_random_data('polymorphic_table', second_node, 10) + insert_random_data("polymorphic_table", first_node, 10) + insert_random_data("polymorphic_table", second_node, 10) first_node.query("SYSTEM SYNC REPLICA polymorphic_table", timeout=20) second_node.query("SYSTEM SYNC REPLICA polymorphic_table", timeout=20) @@ -158,10 +285,18 @@ def test_polymorphic_parts_basics(start_cluster, first_node, second_node): assert first_node.query("SELECT count() FROM polymorphic_table") == "2000\n" assert second_node.query("SELECT count() FROM polymorphic_table") == "2000\n" - assert first_node.query( - "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active") == "Wide\n" - assert second_node.query( - "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active") == "Wide\n" + assert ( + first_node.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active" + ) + == "Wide\n" + ) + assert ( + second_node.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' AND active" + ) + == "Wide\n" + ) # Check alters and mutations also work first_node.query("ALTER TABLE polymorphic_table ADD COLUMN ss String") @@ -179,8 +314,8 @@ def test_polymorphic_parts_basics(start_cluster, first_node, second_node): # Checks mostly that merge from compact part to compact part works. def test_compact_parts_only(start_cluster): for i in range(20): - insert_random_data('compact_parts_only', node1, 100) - insert_random_data('compact_parts_only', node2, 100) + insert_random_data("compact_parts_only", node1, 100) + insert_random_data("compact_parts_only", node2, 100) node1.query("SYSTEM SYNC REPLICA compact_parts_only", timeout=20) node2.query("SYSTEM SYNC REPLICA compact_parts_only", timeout=20) @@ -188,38 +323,59 @@ def test_compact_parts_only(start_cluster): assert node1.query("SELECT count() FROM compact_parts_only") == "4000\n" assert node2.query("SELECT count() FROM compact_parts_only") == "4000\n" - assert node1.query( - "SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active") == "Compact\n" - assert node2.query( - "SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active") == "Compact\n" + assert ( + node1.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active" + ) + == "Compact\n" + ) + assert ( + node2.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'compact_parts_only' AND active" + ) + == "Compact\n" + ) node1.query("OPTIMIZE TABLE compact_parts_only FINAL") node2.query("SYSTEM SYNC REPLICA compact_parts_only", timeout=20) assert node2.query("SELECT count() FROM compact_parts_only") == "4000\n" expected = "Compact\t1\n" - assert TSV(node1.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - expected) - assert TSV(node2.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - expected) + assert TSV( + node1.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV(expected) + assert TSV( + node2.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'compact_parts_only' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV(expected) # Check that follower replicas create parts of the same type, which leader has chosen at merge. @pytest.mark.parametrize( - ('table', 'part_type'), - [ - ('polymorphic_table_compact', 'Compact'), - ('polymorphic_table_wide', 'Wide') - ] + ("table", "part_type"), + [("polymorphic_table_compact", "Compact"), ("polymorphic_table_wide", "Wide")], ) def test_different_part_types_on_replicas(start_cluster, table, part_type): leader = node3 follower = node4 - assert leader.query("SELECT is_leader FROM system.replicas WHERE table = '{}'".format(table)) == "1\n" - assert node4.query("SELECT is_leader FROM system.replicas WHERE table = '{}'".format(table)) == "0\n" + assert ( + leader.query( + "SELECT is_leader FROM system.replicas WHERE table = '{}'".format(table) + ) + == "1\n" + ) + assert ( + node4.query( + "SELECT is_leader FROM system.replicas WHERE table = '{}'".format(table) + ) + == "0\n" + ) for _ in range(3): insert_random_data(table, leader, 100) @@ -229,47 +385,75 @@ def test_different_part_types_on_replicas(start_cluster, table, part_type): expected = "{}\t1\n".format(part_type) - assert TSV(leader.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format( - table))) == TSV(expected) - assert TSV(follower.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format( - table))) == TSV(expected) + assert TSV( + leader.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format( + table + ) + ) + ) == TSV(expected) + assert TSV( + follower.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = '{}' AND active GROUP BY part_type ORDER BY part_type".format( + table + ) + ) + ) == TSV(expected) -node7 = cluster.add_instance('node7', user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, - image='yandex/clickhouse-server', tag='19.17.8.54', stay_alive=True, - with_installed_binary=True) -node8 = cluster.add_instance('node8', user_configs=["configs/users.d/not_optimize_count.xml"], with_zookeeper=True) +node7 = cluster.add_instance( + "node7", + user_configs=["configs_old/users.d/not_optimize_count.xml"], + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="19.17.8.54", + stay_alive=True, + with_installed_binary=True, +) +node8 = cluster.add_instance( + "node8", + user_configs=["configs/users.d/not_optimize_count.xml"], + with_zookeeper=True, +) -settings7 = {'index_granularity_bytes': 10485760} -settings8 = {'index_granularity_bytes': 10485760, 'min_rows_for_wide_part': 512, 'min_rows_for_compact_part': 0} +settings7 = {"index_granularity_bytes": 10485760} +settings8 = { + "index_granularity_bytes": 10485760, + "min_rows_for_wide_part": 512, + "min_rows_for_compact_part": 0, +} @pytest.fixture(scope="module") def start_cluster_diff_versions(): try: - for name in ['polymorphic_table', 'polymorphic_table_2']: + for name in ["polymorphic_table", "polymorphic_table_2"]: cluster.start() node7.query( - ''' + """ CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard5/{name}', '1') PARTITION BY toYYYYMM(date) ORDER BY id SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes} - '''.format(name=name, **settings7) + """.format( + name=name, **settings7 + ) ) node8.query( - ''' + """ CREATE TABLE {name}(date Date, id UInt32, s String, arr Array(Int32)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/shard5/{name}', '2') PARTITION BY toYYYYMM(date) ORDER BY id SETTINGS index_granularity = 64, index_granularity_bytes = {index_granularity_bytes}, min_rows_for_wide_part = {min_rows_for_wide_part}, min_bytes_for_wide_part = {min_bytes_for_wide_part} - '''.format(name=name, **settings8) + """.format( + name=name, **settings8 + ) ) yield cluster @@ -285,12 +469,16 @@ def test_polymorphic_parts_diff_versions(start_cluster_diff_versions): node_old = node7 node_new = node8 - insert_random_data('polymorphic_table', node7, 100) + insert_random_data("polymorphic_table", node7, 100) node8.query("SYSTEM SYNC REPLICA polymorphic_table", timeout=20) assert node8.query("SELECT count() FROM polymorphic_table") == "100\n" - assert node8.query( - "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' and active") == "Wide\n" + assert ( + node8.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table' and active" + ) + == "Wide\n" + ) @pytest.mark.skip(reason="compatability is temporary broken") @@ -301,7 +489,7 @@ def test_polymorphic_parts_diff_versions_2(start_cluster_diff_versions): node_old = node7 node_new = node8 - insert_random_data('polymorphic_table_2', node_new, 100) + insert_random_data("polymorphic_table_2", node_new, 100) assert node_new.query("SELECT count() FROM polymorphic_table_2") == "100\n" assert node_old.query("SELECT count() FROM polymorphic_table_2") == "0\n" @@ -314,29 +502,40 @@ def test_polymorphic_parts_diff_versions_2(start_cluster_diff_versions): # Works after update assert node_old.query("SELECT count() FROM polymorphic_table_2") == "100\n" - assert node_old.query( - "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table_2' and active") == "Compact\n" + assert ( + node_old.query( + "SELECT DISTINCT part_type FROM system.parts WHERE table = 'polymorphic_table_2' and active" + ) + == "Compact\n" + ) def test_polymorphic_parts_non_adaptive(start_cluster): node1.query("SYSTEM STOP MERGES") node2.query("SYSTEM STOP MERGES") - insert_random_data('non_adaptive_table', node1, 100) + insert_random_data("non_adaptive_table", node1, 100) node2.query("SYSTEM SYNC REPLICA non_adaptive_table", timeout=20) - insert_random_data('non_adaptive_table', node2, 100) + insert_random_data("non_adaptive_table", node2, 100) node1.query("SYSTEM SYNC REPLICA non_adaptive_table", timeout=20) - assert TSV(node1.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - "Wide\t2\n") - assert TSV(node2.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - "Wide\t2\n") + assert TSV( + node1.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV("Wide\t2\n") + assert TSV( + node2.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'non_adaptive_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV("Wide\t2\n") assert node1.contains_in_log( - " default.non_adaptive_table ([0-9a-f-]*): Table can't create parts with adaptive granularity") + " default.non_adaptive_table ([0-9a-f-]*): Table can't create parts with adaptive granularity" + ) def test_in_memory(start_cluster): @@ -344,7 +543,7 @@ def test_in_memory(start_cluster): node10.query("SYSTEM STOP MERGES") for size in [200, 200, 300, 600]: - insert_random_data('in_memory_table', node9, size) + insert_random_data("in_memory_table", node9, size) node10.query("SYSTEM SYNC REPLICA in_memory_table", timeout=20) assert node9.query("SELECT count() FROM in_memory_table") == "1300\n" @@ -352,81 +551,148 @@ def test_in_memory(start_cluster): expected = "Compact\t1\nInMemory\t2\nWide\t1\n" - assert TSV(node9.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - expected) - assert TSV(node10.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - expected) + assert TSV( + node9.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV(expected) + assert TSV( + node10.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV(expected) node9.query("SYSTEM START MERGES") node10.query("SYSTEM START MERGES") - assert_eq_with_retry(node9, "OPTIMIZE TABLE in_memory_table FINAL SETTINGS optimize_throw_if_noop = 1", "") + assert_eq_with_retry( + node9, + "OPTIMIZE TABLE in_memory_table FINAL SETTINGS optimize_throw_if_noop = 1", + "", + ) node10.query("SYSTEM SYNC REPLICA in_memory_table", timeout=20) assert node9.query("SELECT count() FROM in_memory_table") == "1300\n" assert node10.query("SELECT count() FROM in_memory_table") == "1300\n" - assert TSV(node9.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - "Wide\t1\n") - assert TSV(node10.query("SELECT part_type, count() FROM system.parts " \ - "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type")) == TSV( - "Wide\t1\n") + assert TSV( + node9.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV("Wide\t1\n") + assert TSV( + node10.query( + "SELECT part_type, count() FROM system.parts " + "WHERE table = 'in_memory_table' AND active GROUP BY part_type ORDER BY part_type" + ) + ) == TSV("Wide\t1\n") def test_in_memory_wal_rotate(start_cluster): # Write every part to single wal - node11.query("ALTER TABLE restore_table MODIFY SETTING write_ahead_log_max_bytes = 10") + node11.query( + "ALTER TABLE restore_table MODIFY SETTING write_ahead_log_max_bytes = 10" + ) for i in range(5): - insert_random_data('restore_table', node11, 50) + insert_random_data("restore_table", node11, 50) for i in range(5): # Check file exists - node11.exec_in_container(['bash', '-c', 'test -f /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin'.format(i)]) + node11.exec_in_container( + [ + "bash", + "-c", + "test -f /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin".format( + i + ), + ] + ) for node in [node11, node12]: node.query( - "ALTER TABLE restore_table MODIFY SETTING number_of_free_entries_in_pool_to_lower_max_size_of_merge = 0") - node.query("ALTER TABLE restore_table MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 10000000") + "ALTER TABLE restore_table MODIFY SETTING number_of_free_entries_in_pool_to_lower_max_size_of_merge = 0" + ) + node.query( + "ALTER TABLE restore_table MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 10000000" + ) - assert_eq_with_retry(node11, "OPTIMIZE TABLE restore_table FINAL SETTINGS optimize_throw_if_noop = 1", "") + assert_eq_with_retry( + node11, + "OPTIMIZE TABLE restore_table FINAL SETTINGS optimize_throw_if_noop = 1", + "", + ) # Restart to be sure, that clearing stale logs task was ran node11.restart_clickhouse(kill=True) for i in range(5): # check file doesn't exist - node11.exec_in_container(['bash', '-c', 'test ! -e /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin'.format(i)]) + node11.exec_in_container( + [ + "bash", + "-c", + "test ! -e /var/lib/clickhouse/data/default/restore_table/wal_{0}_{0}.bin".format( + i + ), + ] + ) # New wal file was created and ready to write part to it # Check file exists - node11.exec_in_container(['bash', '-c', 'test -f /var/lib/clickhouse/data/default/restore_table/wal.bin']) + node11.exec_in_container( + ["bash", "-c", "test -f /var/lib/clickhouse/data/default/restore_table/wal.bin"] + ) # Chech file empty - node11.exec_in_container(['bash', '-c', 'test ! -s /var/lib/clickhouse/data/default/restore_table/wal.bin']) + node11.exec_in_container( + [ + "bash", + "-c", + "test ! -s /var/lib/clickhouse/data/default/restore_table/wal.bin", + ] + ) def test_in_memory_deduplication(start_cluster): for i in range(3): # table can be in readonly node - exec_query_with_retry(node9, "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") - exec_query_with_retry(node10, "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") + exec_query_with_retry( + node9, + "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')", + ) + exec_query_with_retry( + node10, + "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')", + ) node9.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20) node10.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20) - assert node9.query("SELECT date, id, s FROM deduplication_table") == "2020-03-03\t1\tfoo\n" - assert node10.query("SELECT date, id, s FROM deduplication_table") == "2020-03-03\t1\tfoo\n" + assert ( + node9.query("SELECT date, id, s FROM deduplication_table") + == "2020-03-03\t1\tfoo\n" + ) + assert ( + node10.query("SELECT date, id, s FROM deduplication_table") + == "2020-03-03\t1\tfoo\n" + ) # Checks that restoring from WAL works after table schema changed def test_in_memory_alters(start_cluster): def check_parts_type(parts_num): - assert node9.query("SELECT part_type, count() FROM system.parts WHERE table = 'alters_table' \ - AND active GROUP BY part_type") == "InMemory\t{}\n".format(parts_num) + assert ( + node9.query( + "SELECT part_type, count() FROM system.parts WHERE table = 'alters_table' \ + AND active GROUP BY part_type" + ) + == "InMemory\t{}\n".format(parts_num) + ) node9.query( - "INSERT INTO alters_table (date, id, s) VALUES (toDate('2020-10-10'), 1, 'ab'), (toDate('2020-10-10'), 2, 'cd')") + "INSERT INTO alters_table (date, id, s) VALUES (toDate('2020-10-10'), 1, 'ab'), (toDate('2020-10-10'), 2, 'cd')" + ) node9.query("ALTER TABLE alters_table ADD COLUMN col1 UInt32") node9.restart_clickhouse(kill=True) @@ -434,7 +700,10 @@ def test_in_memory_alters(start_cluster): assert node9.query("SELECT id, s, col1 FROM alters_table ORDER BY id") == expected check_parts_type(1) # After hard restart table can be in readonly mode - exec_query_with_retry(node9, "INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)") + exec_query_with_retry( + node9, + "INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)", + ) node9.query("ALTER TABLE alters_table MODIFY COLUMN col1 String") node9.query("ALTER TABLE alters_table DROP COLUMN s") node9.restart_clickhouse(kill=True) @@ -446,26 +715,49 @@ def test_in_memory_alters(start_cluster): # Values of col1 was not materialized as integers, so they have # default string values after alter expected = "1\t_foo\n2\t_foo\n3\t100_foo\n" - assert node9.query("SELECT id, col1 || '_foo' FROM alters_table ORDER BY id") == expected + assert ( + node9.query("SELECT id, col1 || '_foo' FROM alters_table ORDER BY id") + == expected + ) def test_polymorphic_parts_index(start_cluster): - node1.query('CREATE DATABASE test_index ENGINE=Ordinary') # Different paths with Atomic - node1.query(''' + node1.query( + "CREATE DATABASE test_index ENGINE=Ordinary" + ) # Different paths with Atomic + node1.query( + """ CREATE TABLE test_index.index_compact(a UInt32, s String) ENGINE = MergeTree ORDER BY a - SETTINGS min_rows_for_wide_part = 1000, index_granularity = 128, merge_max_block_size = 100''') + SETTINGS min_rows_for_wide_part = 1000, index_granularity = 128, merge_max_block_size = 100""" + ) - node1.query("INSERT INTO test_index.index_compact SELECT number, toString(number) FROM numbers(100)") - node1.query("INSERT INTO test_index.index_compact SELECT number, toString(number) FROM numbers(30)") + node1.query( + "INSERT INTO test_index.index_compact SELECT number, toString(number) FROM numbers(100)" + ) + node1.query( + "INSERT INTO test_index.index_compact SELECT number, toString(number) FROM numbers(30)" + ) node1.query("OPTIMIZE TABLE test_index.index_compact FINAL") - assert node1.query("SELECT part_type FROM system.parts WHERE table = 'index_compact' AND active") == "Compact\n" - assert node1.query("SELECT marks FROM system.parts WHERE table = 'index_compact' AND active") == "2\n" + assert ( + node1.query( + "SELECT part_type FROM system.parts WHERE table = 'index_compact' AND active" + ) + == "Compact\n" + ) + assert ( + node1.query( + "SELECT marks FROM system.parts WHERE table = 'index_compact' AND active" + ) + == "2\n" + ) - index_path = os.path.join(node1.path, "database/data/test_index/index_compact/all_1_2_1/primary.idx") - f = open(index_path, 'rb') + index_path = os.path.join( + node1.path, "database/data/test_index/index_compact/all_1_2_1/primary.idx" + ) + f = open(index_path, "rb") assert os.path.getsize(index_path) == 8 - assert struct.unpack('I', f.read(4))[0] == 0 - assert struct.unpack('I', f.read(4))[0] == 99 + assert struct.unpack("I", f.read(4))[0] == 0 + assert struct.unpack("I", f.read(4))[0] == 99 diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index 7cd632cae6e..dd5b3a09ca5 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -6,7 +6,9 @@ from helpers.test_tools import assert_eq_with_retry from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/named_collections.xml"], with_postgres=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/named_collections.xml"], with_postgres=True +) postgres_table_template = """ CREATE TABLE {} ( @@ -17,6 +19,7 @@ postgres_drop_table_template = """ DROP TABLE {} """ + def get_postgres_conn(cluster, database=False): if database == True: conn_string = f"host={cluster.postgres_ip} port={cluster.postgres_port} dbname='test_database' user='postgres' password='mysecretpassword'" @@ -27,24 +30,28 @@ def get_postgres_conn(cluster, database=False): conn.autocommit = True return conn + def create_postgres_db(cursor, name): cursor.execute("CREATE DATABASE {}".format(name)) + def create_postgres_table(cursor, table_name): # database was specified in connection string cursor.execute(postgres_table_template.format(table_name)) + def drop_postgres_table(cursor, table_name): # database was specified in connection string cursor.execute(postgres_drop_table_template.format(table_name)) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() conn = get_postgres_conn(cluster) cursor = conn.cursor() - create_postgres_db(cursor, 'test_database') + create_postgres_db(cursor, "test_database") yield cluster finally: @@ -57,22 +64,27 @@ def test_postgres_database_engine_with_postgres_ddl(started_cluster): cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')") - assert 'test_database' in node1.query('SHOW DATABASES') + "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + ) + assert "test_database" in node1.query("SHOW DATABASES") - create_postgres_table(cursor, 'test_table') - assert 'test_table' in node1.query('SHOW TABLES FROM test_database') + create_postgres_table(cursor, "test_table") + assert "test_table" in node1.query("SHOW TABLES FROM test_database") - cursor.execute('ALTER TABLE test_table ADD COLUMN data Text') - assert 'data' in node1.query("SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + cursor.execute("ALTER TABLE test_table ADD COLUMN data Text") + assert "data" in node1.query( + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + ) - cursor.execute('ALTER TABLE test_table DROP COLUMN data') - assert 'data' not in node1.query("SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'") + cursor.execute("ALTER TABLE test_table DROP COLUMN data") + assert "data" not in node1.query( + "SELECT name FROM system.columns WHERE table = 'test_table' AND database = 'test_database'" + ) node1.query("DROP DATABASE test_database") - assert 'test_database' not in node1.query('SHOW DATABASES') + assert "test_database" not in node1.query("SHOW DATABASES") - drop_postgres_table(cursor, 'test_table') + drop_postgres_table(cursor, "test_table") def test_postgresql_database_engine_with_clickhouse_ddl(started_cluster): @@ -80,27 +92,28 @@ def test_postgresql_database_engine_with_clickhouse_ddl(started_cluster): cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')") + "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + ) - create_postgres_table(cursor, 'test_table') - assert 'test_table' in node1.query('SHOW TABLES FROM test_database') + create_postgres_table(cursor, "test_table") + assert "test_table" in node1.query("SHOW TABLES FROM test_database") node1.query("DROP TABLE test_database.test_table") - assert 'test_table' not in node1.query('SHOW TABLES FROM test_database') + assert "test_table" not in node1.query("SHOW TABLES FROM test_database") node1.query("ATTACH TABLE test_database.test_table") - assert 'test_table' in node1.query('SHOW TABLES FROM test_database') + assert "test_table" in node1.query("SHOW TABLES FROM test_database") node1.query("DETACH TABLE test_database.test_table") - assert 'test_table' not in node1.query('SHOW TABLES FROM test_database') + assert "test_table" not in node1.query("SHOW TABLES FROM test_database") node1.query("ATTACH TABLE test_database.test_table") - assert 'test_table' in node1.query('SHOW TABLES FROM test_database') + assert "test_table" in node1.query("SHOW TABLES FROM test_database") node1.query("DROP DATABASE test_database") - assert 'test_database' not in node1.query('SHOW DATABASES') + assert "test_database" not in node1.query("SHOW DATABASES") - drop_postgres_table(cursor, 'test_table') + drop_postgres_table(cursor, "test_table") def test_postgresql_database_engine_queries(started_cluster): @@ -108,19 +121,24 @@ def test_postgresql_database_engine_queries(started_cluster): cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')") + "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + ) - create_postgres_table(cursor, 'test_table') - assert node1.query("SELECT count() FROM test_database.test_table").rstrip() == '0' + create_postgres_table(cursor, "test_table") + assert node1.query("SELECT count() FROM test_database.test_table").rstrip() == "0" - node1.query("INSERT INTO test_database.test_table SELECT number, number from numbers(10000)") - assert node1.query("SELECT count() FROM test_database.test_table").rstrip() == '10000' + node1.query( + "INSERT INTO test_database.test_table SELECT number, number from numbers(10000)" + ) + assert ( + node1.query("SELECT count() FROM test_database.test_table").rstrip() == "10000" + ) - drop_postgres_table(cursor, 'test_table') - assert 'test_table' not in node1.query('SHOW TABLES FROM test_database') + drop_postgres_table(cursor, "test_table") + assert "test_table" not in node1.query("SHOW TABLES FROM test_database") node1.query("DROP DATABASE test_database") - assert 'test_database' not in node1.query('SHOW DATABASES') + assert "test_database" not in node1.query("SHOW DATABASES") def test_get_create_table_query_with_multidim_arrays(started_cluster): @@ -128,33 +146,40 @@ def test_get_create_table_query_with_multidim_arrays(started_cluster): cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')") + "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword')" + ) - cursor.execute(""" + cursor.execute( + """ CREATE TABLE array_columns ( b Integer[][][] NOT NULL, c Integer[][][] - )""") + )""" + ) node1.query("DETACH TABLE test_database.array_columns") node1.query("ATTACH TABLE test_database.array_columns") - node1.query("INSERT INTO test_database.array_columns " + node1.query( + "INSERT INTO test_database.array_columns " "VALUES (" "[[[1, 1], [1, 1]], [[3, 3], [3, 3]], [[4, 4], [5, 5]]], " "[[[1, NULL], [NULL, 1]], [[NULL, NULL], [NULL, NULL]], [[4, 4], [5, 5]]] " - ")") - result = node1.query(''' - SELECT * FROM test_database.array_columns''') + ")" + ) + result = node1.query( + """ + SELECT * FROM test_database.array_columns""" + ) expected = ( "[[[1,1],[1,1]],[[3,3],[3,3]],[[4,4],[5,5]]]\t" "[[[1,NULL],[NULL,1]],[[NULL,NULL],[NULL,NULL]],[[4,4],[5,5]]]\n" - ) - assert(result == expected) + ) + assert result == expected node1.query("DROP DATABASE test_database") - assert 'test_database' not in node1.query('SHOW DATABASES') - drop_postgres_table(cursor, 'array_columns') + assert "test_database" not in node1.query("SHOW DATABASES") + drop_postgres_table(cursor, "array_columns") def test_postgresql_database_engine_table_cache(started_cluster): @@ -162,95 +187,140 @@ def test_postgresql_database_engine_table_cache(started_cluster): cursor = conn.cursor() node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', '', 1)") + "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', '', 1)" + ) - create_postgres_table(cursor, 'test_table') - assert node1.query('DESCRIBE TABLE test_database.test_table').rstrip() == 'id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)' + create_postgres_table(cursor, "test_table") + assert ( + node1.query("DESCRIBE TABLE test_database.test_table").rstrip() + == "id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)" + ) - cursor.execute('ALTER TABLE test_table ADD COLUMN data Text') - assert node1.query('DESCRIBE TABLE test_database.test_table').rstrip() == 'id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)' + cursor.execute("ALTER TABLE test_table ADD COLUMN data Text") + assert ( + node1.query("DESCRIBE TABLE test_database.test_table").rstrip() + == "id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)" + ) node1.query("DETACH TABLE test_database.test_table") - assert 'test_table' not in node1.query('SHOW TABLES FROM test_database') + assert "test_table" not in node1.query("SHOW TABLES FROM test_database") node1.query("ATTACH TABLE test_database.test_table") - assert 'test_table' in node1.query('SHOW TABLES FROM test_database') + assert "test_table" in node1.query("SHOW TABLES FROM test_database") - assert node1.query('DESCRIBE TABLE test_database.test_table').rstrip() == 'id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)\t\t\t\t\t\ndata\tNullable(String)' + assert ( + node1.query("DESCRIBE TABLE test_database.test_table").rstrip() + == "id\tInt32\t\t\t\t\t\nvalue\tNullable(Int32)\t\t\t\t\t\ndata\tNullable(String)" + ) node1.query("DROP TABLE test_database.test_table") - assert 'test_table' not in node1.query('SHOW TABLES FROM test_database') + assert "test_table" not in node1.query("SHOW TABLES FROM test_database") node1.query("ATTACH TABLE test_database.test_table") - assert 'test_table' in node1.query('SHOW TABLES FROM test_database') + assert "test_table" in node1.query("SHOW TABLES FROM test_database") - node1.query("INSERT INTO test_database.test_table SELECT number, number, toString(number) from numbers(10000)") - assert node1.query("SELECT count() FROM test_database.test_table").rstrip() == '10000' + node1.query( + "INSERT INTO test_database.test_table SELECT number, number, toString(number) from numbers(10000)" + ) + assert ( + node1.query("SELECT count() FROM test_database.test_table").rstrip() == "10000" + ) - cursor.execute('DROP TABLE test_table;') - assert 'test_table' not in node1.query('SHOW TABLES FROM test_database') + cursor.execute("DROP TABLE test_table;") + assert "test_table" not in node1.query("SHOW TABLES FROM test_database") node1.query("DROP DATABASE test_database") - assert 'test_database' not in node1.query('SHOW DATABASES') + assert "test_database" not in node1.query("SHOW DATABASES") def test_postgresql_database_with_schema(started_cluster): conn = get_postgres_conn(started_cluster, True) cursor = conn.cursor() - cursor.execute('CREATE SCHEMA test_schema') - cursor.execute('CREATE TABLE test_schema.table1 (a integer)') - cursor.execute('CREATE TABLE test_schema.table2 (a integer)') - cursor.execute('CREATE TABLE table3 (a integer)') + cursor.execute("CREATE SCHEMA test_schema") + cursor.execute("CREATE TABLE test_schema.table1 (a integer)") + cursor.execute("CREATE TABLE test_schema.table2 (a integer)") + cursor.execute("CREATE TABLE table3 (a integer)") node1.query( - "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 'test_schema')") + "CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 'test_schema')" + ) - assert(node1.query('SHOW TABLES FROM test_database') == 'table1\ntable2\n') + assert node1.query("SHOW TABLES FROM test_database") == "table1\ntable2\n" node1.query("INSERT INTO test_database.table1 SELECT number from numbers(10000)") - assert node1.query("SELECT count() FROM test_database.table1").rstrip() == '10000' + assert node1.query("SELECT count() FROM test_database.table1").rstrip() == "10000" node1.query("DETACH TABLE test_database.table1") node1.query("ATTACH TABLE test_database.table1") - assert node1.query("SELECT count() FROM test_database.table1").rstrip() == '10000' + assert node1.query("SELECT count() FROM test_database.table1").rstrip() == "10000" node1.query("DROP DATABASE test_database") - cursor.execute('DROP SCHEMA test_schema CASCADE') - cursor.execute('DROP TABLE table3') + cursor.execute("DROP SCHEMA test_schema CASCADE") + cursor.execute("DROP TABLE table3") def test_predefined_connection_configuration(started_cluster): cursor = started_cluster.postgres_conn.cursor() - cursor.execute(f'DROP TABLE IF EXISTS test_table') - cursor.execute(f'CREATE TABLE test_table (a integer PRIMARY KEY, b integer)') + cursor.execute(f"DROP TABLE IF EXISTS test_table") + cursor.execute(f"CREATE TABLE test_table (a integer PRIMARY KEY, b integer)") node1.query("DROP DATABASE IF EXISTS postgres_database") node1.query("CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres1)") - node1.query("INSERT INTO postgres_database.test_table SELECT number, number from numbers(100)") - assert (node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() == '100') - cursor.execute('CREATE SCHEMA test_schema') - cursor.execute('CREATE TABLE test_schema.test_table (a integer)') + result = node1.query( + "select create_table_query from system.tables where database ='postgres_database'" + ) + assert result.strip().endswith( + "ENGINE = PostgreSQL(postgres1, table = \\'test_table\\')" + ) + + node1.query( + "INSERT INTO postgres_database.test_table SELECT number, number from numbers(100)" + ) + assert ( + node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() + == "100" + ) + + cursor.execute("CREATE SCHEMA test_schema") + cursor.execute("CREATE TABLE test_schema.test_table (a integer)") node1.query("DROP DATABASE IF EXISTS postgres_database") - node1.query("CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres1, schema='test_schema')") - node1.query("INSERT INTO postgres_database.test_table SELECT number from numbers(200)") - assert (node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() == '200') + node1.query( + "CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres1, schema='test_schema')" + ) + node1.query( + "INSERT INTO postgres_database.test_table SELECT number from numbers(200)" + ) + assert ( + node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() + == "200" + ) node1.query("DROP DATABASE IF EXISTS postgres_database") - node1.query_and_get_error("CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres1, 'test_schema')") - node1.query_and_get_error("CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres2)") - node1.query_and_get_error("CREATE DATABASE postgres_database ENGINE = PostgreSQL(unknown_collection)") - node1.query("CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres3, port=5432)") - assert (node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() == '100') + node1.query_and_get_error( + "CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres1, 'test_schema')" + ) + node1.query_and_get_error( + "CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres2)" + ) + node1.query_and_get_error( + "CREATE DATABASE postgres_database ENGINE = PostgreSQL(unknown_collection)" + ) + node1.query( + "CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres3, port=5432)" + ) + assert ( + node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() + == "100" + ) node1.query("DROP DATABASE postgres_database") - cursor.execute(f'DROP TABLE test_table ') - cursor.execute('DROP SCHEMA IF EXISTS test_schema CASCADE') + cursor.execute(f"DROP TABLE test_table ") + cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") - -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_postgresql_protocol/test.py b/tests/integration/test_postgresql_protocol/test.py index 7bea9569880..5c270fd9ca7 100644 --- a/tests/integration/test_postgresql_protocol/test.py +++ b/tests/integration/test_postgresql_protocol/test.py @@ -19,10 +19,19 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/postresql.xml", "configs/log.xml", "configs/ssl_conf.xml", - "configs/dhparam.pem", "configs/server.crt", "configs/server.key"], - user_configs=["configs/default_passwd.xml"], - env_variables={'UBSAN_OPTIONS': 'print_stacktrace=1'}) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/postresql.xml", + "configs/log.xml", + "configs/ssl_conf.xml", + "configs/dhparam.pem", + "configs/server.crt", + "configs/server.key", + ], + user_configs=["configs/default_passwd.xml"], + env_variables={"UBSAN_OPTIONS": "print_stacktrace=1"}, +) server_port = 5433 @@ -31,41 +40,75 @@ server_port = 5433 def server_address(): cluster.start() try: - yield cluster.get_instance_ip('node') + yield cluster.get_instance_ip("node") finally: cluster.shutdown() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def psql_client(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_postgresql.yml') + docker_compose = os.path.join(DOCKER_COMPOSE_PATH, "docker_compose_postgresql.yml") run_and_check( - ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) - yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=cluster.docker_api_version, timeout=600).containers.get(cluster.project_name + '_psql_1') + [ + "docker-compose", + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--build", + ] + ) + yield docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=cluster.docker_api_version, + timeout=600, + ).containers.get(cluster.project_name + "_psql_1") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def psql_server(psql_client): """Return PostgreSQL container when it is healthy.""" retries = 30 for i in range(retries): info = psql_client.client.api.inspect_container(psql_client.name) - if info['State']['Health']['Status'] == 'healthy': + if info["State"]["Health"]["Status"] == "healthy": break time.sleep(1) else: - print(info['State']) - raise Exception('PostgreSQL server has not started after {} retries.'.format(retries)) + print(info["State"]) + raise Exception( + "PostgreSQL server has not started after {} retries.".format(retries) + ) return psql_client -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def java_container(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, 'docker_compose_postgresql_java_client.yml') + docker_compose = os.path.join( + DOCKER_COMPOSE_PATH, "docker_compose_postgresql_java_client.yml" + ) run_and_check( - ['docker-compose', '-p', cluster.project_name, '-f', docker_compose, 'up', '--no-recreate', '-d', '--build']) - yield docker.DockerClient(base_url='unix:///var/run/docker.sock', version=cluster.docker_api_version, timeout=600).containers.get(cluster.project_name + '_java_1') + [ + "docker-compose", + "-p", + cluster.project_name, + "-f", + docker_compose, + "up", + "--no-recreate", + "-d", + "--build", + ] + ) + yield docker.DockerClient( + base_url="unix:///var/run/docker.sock", + version=cluster.docker_api_version, + timeout=600, + ).containers.get(cluster.project_name + "_java_1") def test_psql_is_ready(psql_server): @@ -73,85 +116,119 @@ def test_psql_is_ready(psql_server): def test_psql_client(psql_client, server_address): - cmd_prefix = 'psql "sslmode=require host={server_address} port={server_port} user=default dbname=default password=123" ' \ - .format(server_address=server_address, server_port=server_port) + cmd_prefix = 'psql "sslmode=require host={server_address} port={server_port} user=default dbname=default password=123" '.format( + server_address=server_address, server_port=server_port + ) cmd_prefix += "--no-align --field-separator=' ' " - code, (stdout, stderr) = psql_client.exec_run(cmd_prefix + '-c "SELECT 1 as a"', demux=True) - assert stdout.decode() == '\n'.join(['a', '1', '(1 row)', '']) - - code, (stdout, stderr) = psql_client.exec_run(cmd_prefix + '''-c "SELECT 'колонка' as a"''', demux=True) - assert stdout.decode() == '\n'.join(['a', 'колонка', '(1 row)', '']) + code, (stdout, stderr) = psql_client.exec_run( + cmd_prefix + '-c "SELECT 1 as a"', demux=True + ) + assert stdout.decode() == "\n".join(["a", "1", "(1 row)", ""]) code, (stdout, stderr) = psql_client.exec_run( - cmd_prefix + '-c ' + - ''' + cmd_prefix + '''-c "SELECT 'колонка' as a"''', demux=True + ) + assert stdout.decode() == "\n".join(["a", "колонка", "(1 row)", ""]) + + code, (stdout, stderr) = psql_client.exec_run( + cmd_prefix + + "-c " + + """ "CREATE DATABASE x; USE x; CREATE TABLE table1 (column UInt32) ENGINE = Memory; INSERT INTO table1 VALUES (0), (1), (5); INSERT INTO table1 VALUES (0), (1), (5); SELECT * FROM table1 ORDER BY column;" - ''', - demux=True + """, + demux=True, + ) + assert stdout.decode() == "\n".join( + ["column", "0", "0", "1", "1", "5", "5", "(6 rows)", ""] ) - assert stdout.decode() == '\n'.join(['column', '0', '0', '1', '1', '5', '5', '(6 rows)', '']) code, (stdout, stderr) = psql_client.exec_run( - cmd_prefix + '-c ' + - ''' + cmd_prefix + + "-c " + + """ "DROP DATABASE x; CREATE TEMPORARY TABLE tmp (tmp_column UInt32); INSERT INTO tmp VALUES (0), (1); SELECT * FROM tmp ORDER BY tmp_column;" - ''', - demux=True + """, + demux=True, ) - assert stdout.decode() == '\n'.join(['tmp_column', '0', '1', '(2 rows)', '']) + assert stdout.decode() == "\n".join(["tmp_column", "0", "1", "(2 rows)", ""]) def test_python_client(server_address): with pytest.raises(py_psql.InternalError) as exc_info: - ch = py_psql.connect(host=server_address, port=server_port, user='default', password='123', database='') + ch = py_psql.connect( + host=server_address, + port=server_port, + user="default", + password="123", + database="", + ) cur = ch.cursor() - cur.execute('select name from tables;') + cur.execute("select name from tables;") assert exc_info.value.args == ( - "Query execution failed.\nDB::Exception: Table default.tables doesn't exist\nSSL connection has been closed unexpectedly\n",) + "Query execution failed.\nDB::Exception: Table default.tables doesn't exist\nSSL connection has been closed unexpectedly\n", + ) - ch = py_psql.connect(host=server_address, port=server_port, user='default', password='123', database='') + ch = py_psql.connect( + host=server_address, + port=server_port, + user="default", + password="123", + database="", + ) cur = ch.cursor() - cur.execute('select 1 as a, 2 as b') - assert (cur.description[0].name, cur.description[1].name) == ('a', 'b') + cur.execute("select 1 as a, 2 as b") + assert (cur.description[0].name, cur.description[1].name) == ("a", "b") assert cur.fetchall() == [(1, 2)] - cur.execute('CREATE DATABASE x') - cur.execute('USE x') + cur.execute("CREATE DATABASE x") + cur.execute("USE x") cur.execute( - 'CREATE TEMPORARY TABLE tmp2 (ch Int8, i64 Int64, f64 Float64, str String, date Date, dec Decimal(19, 10), uuid UUID) ENGINE = Memory') + "CREATE TEMPORARY TABLE tmp2 (ch Int8, i64 Int64, f64 Float64, str String, date Date, dec Decimal(19, 10), uuid UUID) ENGINE = Memory" + ) cur.execute( - "insert into tmp2 (ch, i64, f64, str, date, dec, uuid) values (44, 534324234, 0.32423423, 'hello', '2019-01-23', 0.333333, '61f0c404-5cb3-11e7-907b-a6006ad3dba0')") - cur.execute('select * from tmp2') + "insert into tmp2 (ch, i64, f64, str, date, dec, uuid) values (44, 534324234, 0.32423423, 'hello', '2019-01-23', 0.333333, '61f0c404-5cb3-11e7-907b-a6006ad3dba0')" + ) + cur.execute("select * from tmp2") assert cur.fetchall()[0] == ( - '44', 534324234, 0.32423423, 'hello', datetime.date(2019, 1, 23), decimal.Decimal('0.3333330000'), - uuid.UUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')) + "44", + 534324234, + 0.32423423, + "hello", + datetime.date(2019, 1, 23), + decimal.Decimal("0.3333330000"), + uuid.UUID("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), + ) def test_java_client(server_address, java_container): - with open(os.path.join(SCRIPT_DIR, 'java.reference')) as fp: + with open(os.path.join(SCRIPT_DIR, "java.reference")) as fp: reference = fp.read() # database not exists exception. code, (stdout, stderr) = java_container.exec_run( - 'java JavaConnectorTest --host {host} --port {port} --user default --database ' - 'abc'.format(host=server_address, port=server_port), demux=True) + "java JavaConnectorTest --host {host} --port {port} --user default --database " + "abc".format(host=server_address, port=server_port), + demux=True, + ) assert code == 1 # non-empty password passed. code, (stdout, stderr) = java_container.exec_run( - 'java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database ' - 'default'.format(host=server_address, port=server_port), demux=True) + "java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database " + "default".format(host=server_address, port=server_port), + demux=True, + ) print(stdout, stderr, file=sys.stderr) assert code == 0 assert stdout.decode() == reference diff --git a/tests/integration/test_postgresql_replica_database_engine_1/test.py b/tests/integration/test_postgresql_replica_database_engine_1/test.py index 8b5d7f5f7b2..45aa59001ef 100644 --- a/tests/integration/test_postgresql_replica_database_engine_1/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_1/test.py @@ -20,15 +20,23 @@ from helpers.postgres_utility import check_tables_are_synchronized from helpers.postgres_utility import check_several_tables_are_synchronized from helpers.postgres_utility import assert_nested_table_is_created from helpers.postgres_utility import assert_number_of_columns -from helpers.postgres_utility import postgres_table_template, postgres_table_template_2, postgres_table_template_3, postgres_table_template_4 +from helpers.postgres_utility import ( + postgres_table_template, + postgres_table_template_2, + postgres_table_template_3, + postgres_table_template_4, +) from helpers.postgres_utility import queries cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs = ['configs/log_conf.xml'], - user_configs = ['configs/users.xml'], - with_postgres=True, stay_alive=True) +instance = cluster.add_instance( + "instance", + main_configs=["configs/log_conf.xml"], + user_configs=["configs/users.xml"], + with_postgres=True, + stay_alive=True, +) pg_manager = PostgresManager() @@ -54,54 +62,93 @@ def setup_teardown(): def test_load_and_sync_all_database_tables(started_cluster): NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables(NUM_TABLES) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) check_several_tables_are_synchronized(instance, NUM_TABLES) - result = instance.query("SELECT count() FROM system.tables WHERE database = 'test_database';") - assert(int(result) == NUM_TABLES) + result = instance.query( + "SELECT count() FROM system.tables WHERE database = 'test_database';" + ) + assert int(result) == NUM_TABLES def test_replicating_dml(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() NUM_TABLES = 5 for i in range(NUM_TABLES): - create_postgres_table(cursor, 'postgresql_replica_{}'.format(i)); - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT number, {} from numbers(50)".format(i, i)) + create_postgres_table(cursor, "postgresql_replica_{}".format(i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT number, {} from numbers(50)".format( + i, i + ) + ) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for i in range(NUM_TABLES): - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT 50 + number, {} from numbers(1000)".format(i, i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT 50 + number, {} from numbers(1000)".format( + i, i + ) + ) check_several_tables_are_synchronized(instance, NUM_TABLES) for i in range(NUM_TABLES): - cursor.execute('UPDATE postgresql_replica_{} SET value = {} * {} WHERE key < 50;'.format(i, i, i)) - cursor.execute('UPDATE postgresql_replica_{} SET value = {} * {} * {} WHERE key >= 50;'.format(i, i, i, i)) + cursor.execute( + "UPDATE postgresql_replica_{} SET value = {} * {} WHERE key < 50;".format( + i, i, i + ) + ) + cursor.execute( + "UPDATE postgresql_replica_{} SET value = {} * {} * {} WHERE key >= 50;".format( + i, i, i, i + ) + ) check_several_tables_are_synchronized(instance, NUM_TABLES) for i in range(NUM_TABLES): - cursor.execute('DELETE FROM postgresql_replica_{} WHERE (value*value + {}) % 2 = 0;'.format(i, i)) - cursor.execute('UPDATE postgresql_replica_{} SET value = value - (value % 7) WHERE key > 128 AND key < 512;'.format(i)) - cursor.execute('DELETE FROM postgresql_replica_{} WHERE key % 7 = 1;'.format(i, i)) + cursor.execute( + "DELETE FROM postgresql_replica_{} WHERE (value*value + {}) % 2 = 0;".format( + i, i + ) + ) + cursor.execute( + "UPDATE postgresql_replica_{} SET value = value - (value % 7) WHERE key > 128 AND key < 512;".format( + i + ) + ) + cursor.execute( + "DELETE FROM postgresql_replica_{} WHERE key % 7 = 1;".format(i, i) + ) check_several_tables_are_synchronized(instance, NUM_TABLES) def test_different_data_types(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - cursor.execute('drop table if exists test_data_types;') - cursor.execute('drop table if exists test_array_data_type;') + cursor.execute("drop table if exists test_data_types;") + cursor.execute("drop table if exists test_array_data_type;") cursor.execute( - '''CREATE TABLE test_data_types ( + """CREATE TABLE test_data_types ( id integer PRIMARY KEY, a smallint, b integer, c bigint, d real, e double precision, f serial, g bigserial, - h timestamp, i date, j decimal(5, 5), k numeric(5, 5))''') + h timestamp, i date, j decimal(5, 5), k numeric(5, 5))""" + ) cursor.execute( - '''CREATE TABLE test_array_data_type + """CREATE TABLE test_array_data_type ( key Integer NOT NULL PRIMARY KEY, a Date[] NOT NULL, -- Date @@ -114,27 +161,42 @@ def test_different_data_types(started_cluster): h Integer[][][], -- Nullable(Int32) i Char(2)[][][][], -- Nullable(String) k Char(2)[] -- Nullable(String) - )''') + )""" + ) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for i in range(10): - instance.query(''' + instance.query( + """ INSERT INTO postgres_database.test_data_types VALUES - ({}, -32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12.012345', '2000-05-12', 0.2, 0.2)'''.format(i)) + ({}, -32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12.012345', '2000-05-12', 0.2, 0.2)""".format( + i + ) + ) - check_tables_are_synchronized(instance, 'test_data_types', 'id'); - result = instance.query('SELECT * FROM test_database.test_data_types ORDER BY id LIMIT 1;') - assert(result == '0\t-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t0.2\t0.2\n') + check_tables_are_synchronized(instance, "test_data_types", "id") + result = instance.query( + "SELECT * FROM test_database.test_data_types ORDER BY id LIMIT 1;" + ) + assert ( + result + == "0\t-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t0.2\t0.2\n" + ) for i in range(10): - col = random.choice(['a', 'b', 'c']) - cursor.execute('UPDATE test_data_types SET {} = {};'.format(col, i)) - cursor.execute('''UPDATE test_data_types SET i = '2020-12-12';'''.format(col, i)) + col = random.choice(["a", "b", "c"]) + cursor.execute("UPDATE test_data_types SET {} = {};".format(col, i)) + cursor.execute( + """UPDATE test_data_types SET i = '2020-12-12';""".format(col, i) + ) - check_tables_are_synchronized(instance, 'test_data_types', 'id'); + check_tables_are_synchronized(instance, "test_data_types", "id") - instance.query("INSERT INTO postgres_database.test_array_data_type " + instance.query( + "INSERT INTO postgres_database.test_array_data_type " "VALUES (" "0, " "['2000-05-12', '2000-05-12'], " @@ -147,144 +209,203 @@ def test_different_data_types(started_cluster): "[[[1, NULL], [NULL, 1]], [[NULL, NULL], [NULL, NULL]], [[4, 4], [5, 5]]], " "[[[[NULL]]]], " "[]" - ")") + ")" + ) expected = ( - "0\t" + - "['2000-05-12','2000-05-12']\t" + - "['2000-05-12 12:12:12.012345','2000-05-12 12:12:12.012345']\t" + - "[[1.12345],[1.12345],[1.12345]]\t" + - "[[1.1234567891],[1.1234567891],[1.1234567891]]\t" + - "[[[0.11111,0.11111]],[[0.22222,0.22222]],[[0.33333,0.33333]]]\t" + "0\t" + + "['2000-05-12','2000-05-12']\t" + + "['2000-05-12 12:12:12.012345','2000-05-12 12:12:12.012345']\t" + + "[[1.12345],[1.12345],[1.12345]]\t" + + "[[1.1234567891],[1.1234567891],[1.1234567891]]\t" + + "[[[0.11111,0.11111]],[[0.22222,0.22222]],[[0.33333,0.33333]]]\t" "[[[1,1],[1,1]],[[3,3],[3,3]],[[4,4],[5,5]]]\t" "[[[[['winx','winx','winx']]]]]\t" "[[[1,NULL],[NULL,1]],[[NULL,NULL],[NULL,NULL]],[[4,4],[5,5]]]\t" "[[[[NULL]]]]\t" "[]\n" - ) + ) - check_tables_are_synchronized(instance, 'test_array_data_type'); - result = instance.query('SELECT * FROM test_database.test_array_data_type ORDER BY key;') - assert(result == expected) + check_tables_are_synchronized(instance, "test_array_data_type") + result = instance.query( + "SELECT * FROM test_database.test_array_data_type ORDER BY key;" + ) + assert result == expected pg_manager.drop_materialized_db() - cursor.execute('drop table if exists test_data_types;') - cursor.execute('drop table if exists test_array_data_type;') + cursor.execute("drop table if exists test_data_types;") + cursor.execute("drop table if exists test_array_data_type;") def test_load_and_sync_subset_of_database_tables(started_cluster): NUM_TABLES = 10 pg_manager.create_and_fill_postgres_tables(NUM_TABLES) - publication_tables = '' + publication_tables = "" for i in range(NUM_TABLES): - if i < int(NUM_TABLES/2): - if publication_tables != '': - publication_tables += ', ' - publication_tables += f'postgresql_replica_{i}' + if i < int(NUM_TABLES / 2): + if publication_tables != "": + publication_tables += ", " + publication_tables += f"postgresql_replica_{i}" pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=["materialized_postgresql_tables_list = '{}'".format(publication_tables)]) + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + "materialized_postgresql_tables_list = '{}'".format(publication_tables) + ], + ) time.sleep(1) - for i in range(int(NUM_TABLES/2)): - table_name = f'postgresql_replica_{i}' + for i in range(int(NUM_TABLES / 2)): + table_name = f"postgresql_replica_{i}" assert_nested_table_is_created(instance, table_name) - result = instance.query('''SELECT count() FROM system.tables WHERE database = 'test_database';''') - assert(int(result) == int(NUM_TABLES/2)) + result = instance.query( + """SELECT count() FROM system.tables WHERE database = 'test_database';""" + ) + assert int(result) == int(NUM_TABLES / 2) - database_tables = instance.query('SHOW TABLES FROM test_database') + database_tables = instance.query("SHOW TABLES FROM test_database") for i in range(NUM_TABLES): - table_name = 'postgresql_replica_{}'.format(i) - if i < int(NUM_TABLES/2): + table_name = "postgresql_replica_{}".format(i) + if i < int(NUM_TABLES / 2): assert table_name in database_tables else: assert table_name not in database_tables - instance.query("INSERT INTO postgres_database.{} SELECT 50 + number, {} from numbers(100)".format(table_name, i)) + instance.query( + "INSERT INTO postgres_database.{} SELECT 50 + number, {} from numbers(100)".format( + table_name, i + ) + ) for i in range(NUM_TABLES): - table_name = f'postgresql_replica_{i}' - if i < int(NUM_TABLES/2): - check_tables_are_synchronized(instance, table_name); + table_name = f"postgresql_replica_{i}" + if i < int(NUM_TABLES / 2): + check_tables_are_synchronized(instance, table_name) def test_changing_replica_identity_value(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, number from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, number from numbers(50)" + ) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 100 + number, number from numbers(50)") - check_tables_are_synchronized(instance, 'postgresql_replica'); + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 100 + number, number from numbers(50)" + ) + check_tables_are_synchronized(instance, "postgresql_replica") cursor.execute("UPDATE postgresql_replica SET key=key-25 WHERE key<100 ") - check_tables_are_synchronized(instance, 'postgresql_replica'); + check_tables_are_synchronized(instance, "postgresql_replica") def test_clickhouse_restart(started_cluster): NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables(NUM_TABLES) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) check_several_tables_are_synchronized(instance, NUM_TABLES) for i in range(NUM_TABLES): - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT 50 + number, {} from numbers(50000)".format(i, i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT 50 + number, {} from numbers(50000)".format( + i, i + ) + ) instance.restart_clickhouse() check_several_tables_are_synchronized(instance, NUM_TABLES) def test_replica_identity_index(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica', template=postgres_table_template_3); + create_postgres_table( + cursor, "postgresql_replica", template=postgres_table_template_3 + ) cursor.execute("CREATE unique INDEX idx on postgresql_replica(key1, key2);") cursor.execute("ALTER TABLE postgresql_replica REPLICA IDENTITY USING INDEX idx") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number, number, number from numbers(50, 10)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number, number, number from numbers(50, 10)" + ) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number, number, number from numbers(100, 10)") - check_tables_are_synchronized(instance, 'postgresql_replica', order_by='key1'); + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number, number, number from numbers(100, 10)" + ) + check_tables_are_synchronized(instance, "postgresql_replica", order_by="key1") cursor.execute("UPDATE postgresql_replica SET key1=key1-25 WHERE key1<100 ") cursor.execute("UPDATE postgresql_replica SET key2=key2-25 WHERE key2>100 ") cursor.execute("UPDATE postgresql_replica SET value1=value1+100 WHERE key1<100 ") cursor.execute("UPDATE postgresql_replica SET value2=value2+200 WHERE key2>100 ") - check_tables_are_synchronized(instance, 'postgresql_replica', order_by='key1'); + check_tables_are_synchronized(instance, "postgresql_replica", order_by="key1") - cursor.execute('DELETE FROM postgresql_replica WHERE key2<75;') - check_tables_are_synchronized(instance, 'postgresql_replica', order_by='key1'); + cursor.execute("DELETE FROM postgresql_replica WHERE key2<75;") + check_tables_are_synchronized(instance, "postgresql_replica", order_by="key1") def test_table_schema_changes(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() NUM_TABLES = 5 for i in range(NUM_TABLES): - create_postgres_table(cursor, 'postgresql_replica_{}'.format(i), template=postgres_table_template_2); - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT number, {}, {}, {} from numbers(25)".format(i, i, i, i)) + create_postgres_table( + cursor, + "postgresql_replica_{}".format(i), + template=postgres_table_template_2, + ) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT number, {}, {}, {} from numbers(25)".format( + i, i, i, i + ) + ) pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=["materialized_postgresql_allow_automatic_update = 1"]) + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=["materialized_postgresql_allow_automatic_update = 1"], + ) for i in range(NUM_TABLES): - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT 25 + number, {}, {}, {} from numbers(25)".format(i, i, i, i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT 25 + number, {}, {}, {} from numbers(25)".format( + i, i, i, i + ) + ) check_several_tables_are_synchronized(instance, NUM_TABLES) - expected = instance.query("SELECT key, value1, value3 FROM test_database.postgresql_replica_3 ORDER BY key"); + expected = instance.query( + "SELECT key, value1, value3 FROM test_database.postgresql_replica_3 ORDER BY key" + ) altered_idx = random.randint(0, 4) - altered_table = f'postgresql_replica_{altered_idx}' + altered_table = f"postgresql_replica_{altered_idx}" cursor.execute(f"ALTER TABLE {altered_table} DROP COLUMN value2") for i in range(NUM_TABLES): @@ -295,47 +416,62 @@ def test_table_schema_changes(started_cluster): assert_nested_table_is_created(instance, altered_table) assert_number_of_columns(instance, 3, altered_table) check_tables_are_synchronized(instance, altered_table) - print('check1 OK') + print("check1 OK") check_several_tables_are_synchronized(instance, NUM_TABLES) for i in range(NUM_TABLES): if i != altered_idx: - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT 51 + number, {}, {}, {} from numbers(49)".format(i, i, i, i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT 51 + number, {}, {}, {} from numbers(49)".format( + i, i, i, i + ) + ) else: - instance.query("INSERT INTO postgres_database.postgresql_replica_{} SELECT 51 + number, {}, {} from numbers(49)".format(i, i, i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT 51 + number, {}, {} from numbers(49)".format( + i, i, i + ) + ) - check_tables_are_synchronized(instance, altered_table); - print('check2 OK') + check_tables_are_synchronized(instance, altered_table) + print("check2 OK") check_several_tables_are_synchronized(instance, NUM_TABLES) def test_many_concurrent_queries(started_cluster): - query_pool = ['DELETE FROM postgresql_replica_{} WHERE (value*value) % 3 = 0;', - 'UPDATE postgresql_replica_{} SET value = value - 125 WHERE key % 2 = 0;', - 'DELETE FROM postgresql_replica_{} WHERE key % 10 = 0;', - 'UPDATE postgresql_replica_{} SET value = value*5 WHERE key % 2 = 1;', - 'DELETE FROM postgresql_replica_{} WHERE value % 2 = 0;', - 'UPDATE postgresql_replica_{} SET value = value + 2000 WHERE key % 5 = 0;', - 'DELETE FROM postgresql_replica_{} WHERE value % 3 = 0;', - 'UPDATE postgresql_replica_{} SET value = value * 2 WHERE key % 3 = 0;', - 'DELETE FROM postgresql_replica_{} WHERE value % 9 = 2;', - 'UPDATE postgresql_replica_{} SET value = value + 2 WHERE key % 3 = 1;', - 'DELETE FROM postgresql_replica_{} WHERE value%5 = 0;'] + query_pool = [ + "DELETE FROM postgresql_replica_{} WHERE (value*value) % 3 = 0;", + "UPDATE postgresql_replica_{} SET value = value - 125 WHERE key % 2 = 0;", + "DELETE FROM postgresql_replica_{} WHERE key % 10 = 0;", + "UPDATE postgresql_replica_{} SET value = value*5 WHERE key % 2 = 1;", + "DELETE FROM postgresql_replica_{} WHERE value % 2 = 0;", + "UPDATE postgresql_replica_{} SET value = value + 2000 WHERE key % 5 = 0;", + "DELETE FROM postgresql_replica_{} WHERE value % 3 = 0;", + "UPDATE postgresql_replica_{} SET value = value * 2 WHERE key % 3 = 0;", + "DELETE FROM postgresql_replica_{} WHERE value % 9 = 2;", + "UPDATE postgresql_replica_{} SET value = value + 2 WHERE key % 3 = 1;", + "DELETE FROM postgresql_replica_{} WHERE value%5 = 0;", + ] NUM_TABLES = 5 - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - pg_manager.create_and_fill_postgres_tables_from_cursor(cursor, NUM_TABLES, numbers=10000) + pg_manager.create_and_fill_postgres_tables_from_cursor( + cursor, NUM_TABLES, numbers=10000 + ) def attack(thread_id): - print('thread {}'.format(thread_id)) + print("thread {}".format(thread_id)) k = 10000 for i in range(20): - query_id = random.randrange(0, len(query_pool)-1) - table_id = random.randrange(0, 5) # num tables + query_id = random.randrange(0, len(query_pool) - 1) + table_id = random.randrange(0, 5) # num tables # random update / delete query cursor.execute(query_pool[query_id].format(table_id)) @@ -344,14 +480,22 @@ def test_many_concurrent_queries(started_cluster): # allow some thread to do inserts (not to violate key constraints) if thread_id < 5: print("try insert table {}".format(thread_id)) - instance.query('INSERT INTO postgres_database.postgresql_replica_{} SELECT {}*10000*({} + number), number from numbers(1000)'.format(i, thread_id, k)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT {}*10000*({} + number), number from numbers(1000)".format( + i, thread_id, k + ) + ) k += 1 print("insert table {} ok".format(thread_id)) if i == 5: # also change primary key value print("try update primary key {}".format(thread_id)) - cursor.execute("UPDATE postgresql_replica_{} SET key=key%100000+100000*{} WHERE key%{}=0".format(thread_id, i+1, i+1)) + cursor.execute( + "UPDATE postgresql_replica_{} SET key=key%100000+100000*{} WHERE key%{}=0".format( + thread_id, i + 1, i + 1 + ) + ) print("update primary key {} ok".format(thread_id)) n = [10000] @@ -361,7 +505,9 @@ def test_many_concurrent_queries(started_cluster): for i in range(threads_num): threads.append(threading.Thread(target=attack, args=(i,))) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for thread in threads: time.sleep(random.uniform(0, 1)) @@ -370,135 +516,222 @@ def test_many_concurrent_queries(started_cluster): n[0] = 50000 for table_id in range(NUM_TABLES): n[0] += 1 - instance.query('INSERT INTO postgres_database.postgresql_replica_{} SELECT {} + number, number from numbers(5000)'.format(table_id, n[0])) - #cursor.execute("UPDATE postgresql_replica_{} SET key=key%100000+100000*{} WHERE key%{}=0".format(table_id, table_id+1, table_id+1)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica_{} SELECT {} + number, number from numbers(5000)".format( + table_id, n[0] + ) + ) + # cursor.execute("UPDATE postgresql_replica_{} SET key=key%100000+100000*{} WHERE key%{}=0".format(table_id, table_id+1, table_id+1)) for thread in threads: thread.join() for i in range(NUM_TABLES): - check_tables_are_synchronized(instance, 'postgresql_replica_{}'.format(i)); - count1 = instance.query('SELECT count() FROM postgres_database.postgresql_replica_{}'.format(i)) - count2 = instance.query('SELECT count() FROM (SELECT * FROM test_database.postgresql_replica_{})'.format(i)) - assert(int(count1) == int(count2)) + check_tables_are_synchronized(instance, "postgresql_replica_{}".format(i)) + count1 = instance.query( + "SELECT count() FROM postgres_database.postgresql_replica_{}".format(i) + ) + count2 = instance.query( + "SELECT count() FROM (SELECT * FROM test_database.postgresql_replica_{})".format( + i + ) + ) + assert int(count1) == int(count2) print(count1, count2) def test_single_transaction(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, auto_commit=False) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + auto_commit=False, + ) cursor = conn.cursor() - table_name = 'postgresql_replica_0' - create_postgres_table(cursor, table_name); + table_name = "postgresql_replica_0" + create_postgres_table(cursor, table_name) conn.commit() - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) assert_nested_table_is_created(instance, table_name) for query in queries: - print('query {}'.format(query)) + print("query {}".format(query)) cursor.execute(query.format(0)) time.sleep(5) result = instance.query(f"select count() from test_database.{table_name}") # no commit yet - assert(int(result) == 0) + assert int(result) == 0 conn.commit() - check_tables_are_synchronized(instance, table_name); + check_tables_are_synchronized(instance, table_name) def test_virtual_columns(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - table_name = 'postgresql_replica_0' - create_postgres_table(cursor, table_name); + table_name = "postgresql_replica_0" + create_postgres_table(cursor, table_name) pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=["materialized_postgresql_allow_automatic_update = 1"]) + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=["materialized_postgresql_allow_automatic_update = 1"], + ) assert_nested_table_is_created(instance, table_name) - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(10)") - check_tables_are_synchronized(instance, table_name); + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(10)" + ) + check_tables_are_synchronized(instance, table_name) # just check that it works, no check with `expected` because _version is taken as LSN, which will be different each time. - result = instance.query(f'SELECT key, value, _sign, _version FROM test_database.{table_name};') + result = instance.query( + f"SELECT key, value, _sign, _version FROM test_database.{table_name};" + ) print(result) cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value2 integer") - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(10, 10)") + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(10, 10)" + ) assert_number_of_columns(instance, 3, table_name) - check_tables_are_synchronized(instance, table_name); + check_tables_are_synchronized(instance, table_name) - result = instance.query('SELECT key, value, value2, _sign, _version FROM test_database.postgresql_replica_0;') + result = instance.query( + "SELECT key, value, value2, _sign, _version FROM test_database.postgresql_replica_0;" + ) print(result) - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(20, 10)") - check_tables_are_synchronized(instance, table_name); + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number, number from numbers(20, 10)" + ) + check_tables_are_synchronized(instance, table_name) - result = instance.query(f'SELECT key, value, value2, _sign, _version FROM test_database.{table_name};') + result = instance.query( + f"SELECT key, value, value2, _sign, _version FROM test_database.{table_name};" + ) print(result) def test_multiple_databases(started_cluster): NUM_TABLES = 5 - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=False) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=False, + ) cursor = conn.cursor() - pg_manager.create_postgres_db(cursor, 'postgres_database_1') - pg_manager.create_postgres_db(cursor, 'postgres_database_2') + pg_manager.create_postgres_db(cursor, "postgres_database_1") + pg_manager.create_postgres_db(cursor, "postgres_database_2") - conn1 = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, database_name='postgres_database_1') - conn2 = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, database_name='postgres_database_2') + conn1 = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + database_name="postgres_database_1", + ) + conn2 = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + database_name="postgres_database_2", + ) cursor1 = conn1.cursor() cursor2 = conn2.cursor() - pg_manager.create_clickhouse_postgres_db(cluster.postgres_ip, cluster.postgres_port, 'postgres_database_1', 'postgres_database_1') - pg_manager.create_clickhouse_postgres_db(cluster.postgres_ip, cluster.postgres_port, 'postgres_database_2', 'postgres_database_2') + pg_manager.create_clickhouse_postgres_db( + cluster.postgres_ip, + cluster.postgres_port, + "postgres_database_1", + "postgres_database_1", + ) + pg_manager.create_clickhouse_postgres_db( + cluster.postgres_ip, + cluster.postgres_port, + "postgres_database_2", + "postgres_database_2", + ) cursors = [cursor1, cursor2] for cursor_id in range(len(cursors)): for i in range(NUM_TABLES): - table_name = 'postgresql_replica_{}'.format(i) - create_postgres_table(cursors[cursor_id], table_name); - instance.query("INSERT INTO postgres_database_{}.{} SELECT number, number from numbers(50)".format(cursor_id + 1, table_name)) - print('database 1 tables: ', instance.query('''SELECT name FROM system.tables WHERE database = 'postgres_database_1';''')) - print('database 2 tables: ', instance.query('''SELECT name FROM system.tables WHERE database = 'postgres_database_2';''')) + table_name = "postgresql_replica_{}".format(i) + create_postgres_table(cursors[cursor_id], table_name) + instance.query( + "INSERT INTO postgres_database_{}.{} SELECT number, number from numbers(50)".format( + cursor_id + 1, table_name + ) + ) + print( + "database 1 tables: ", + instance.query( + """SELECT name FROM system.tables WHERE database = 'postgres_database_1';""" + ), + ) + print( + "database 2 tables: ", + instance.query( + """SELECT name FROM system.tables WHERE database = 'postgres_database_2';""" + ), + ) - pg_manager.create_materialized_db(started_cluster.postgres_ip, started_cluster.postgres_port, - 'test_database_1', 'postgres_database_1') - pg_manager.create_materialized_db(started_cluster.postgres_ip, started_cluster.postgres_port, - 'test_database_2', 'postgres_database_2') + pg_manager.create_materialized_db( + started_cluster.postgres_ip, + started_cluster.postgres_port, + "test_database_1", + "postgres_database_1", + ) + pg_manager.create_materialized_db( + started_cluster.postgres_ip, + started_cluster.postgres_port, + "test_database_2", + "postgres_database_2", + ) cursors = [cursor1, cursor2] for cursor_id in range(len(cursors)): for i in range(NUM_TABLES): - table_name = 'postgresql_replica_{}'.format(i) - instance.query("INSERT INTO postgres_database_{}.{} SELECT 50 + number, number from numbers(50)".format(cursor_id + 1, table_name)) + table_name = "postgresql_replica_{}".format(i) + instance.query( + "INSERT INTO postgres_database_{}.{} SELECT 50 + number, number from numbers(50)".format( + cursor_id + 1, table_name + ) + ) for cursor_id in range(len(cursors)): for i in range(NUM_TABLES): - table_name = 'postgresql_replica_{}'.format(i) - check_tables_are_synchronized(instance, - table_name, 'key', 'postgres_database_{}'.format(cursor_id + 1), 'test_database_{}'.format(cursor_id + 1)); + table_name = "postgresql_replica_{}".format(i) + check_tables_are_synchronized( + instance, + table_name, + "key", + "postgres_database_{}".format(cursor_id + 1), + "test_database_{}".format(cursor_id + 1), + ) def test_concurrent_transactions(started_cluster): def transaction(thread_id): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True, auto_commit=False) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + auto_commit=False, + ) cursor = conn.cursor() for query in queries: cursor.execute(query.format(thread_id)) - print('thread {}, query {}'.format(thread_id, query)) + print("thread {}, query {}".format(thread_id, query)) conn.commit() NUM_TABLES = 6 @@ -509,7 +742,9 @@ def test_concurrent_transactions(started_cluster): for i in range(threads_num): threads.append(threading.Thread(target=transaction, args=(i,))) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for thread in threads: time.sleep(random.uniform(0, 0.5)) @@ -519,25 +754,37 @@ def test_concurrent_transactions(started_cluster): thread.join() for i in range(NUM_TABLES): - check_tables_are_synchronized(instance, f'postgresql_replica_{i}'); - count1 = instance.query(f'SELECT count() FROM postgres_database.postgresql_replica_{i}') - count2 = instance.query(f'SELECT count() FROM (SELECT * FROM test_database.postgresql_replica_{i})') - print(int(count1), int(count2), sep=' ') - assert(int(count1) == int(count2)) + check_tables_are_synchronized(instance, f"postgresql_replica_{i}") + count1 = instance.query( + f"SELECT count() FROM postgres_database.postgresql_replica_{i}" + ) + count2 = instance.query( + f"SELECT count() FROM (SELECT * FROM test_database.postgresql_replica_{i})" + ) + print(int(count1), int(count2), sep=" ") + assert int(count1) == int(count2) def test_abrupt_connection_loss_while_heavy_replication(started_cluster): def transaction(thread_id): if thread_id % 2: - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, auto_commit=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + auto_commit=True, + ) else: - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, auto_commit=False) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + auto_commit=False, + ) cursor = conn.cursor() for query in queries: cursor.execute(query.format(thread_id)) - print('thread {}, query {}'.format(thread_id, query)) + print("thread {}, query {}".format(thread_id, query)) if thread_id % 2 == 0: conn.commit() @@ -549,23 +796,25 @@ def test_abrupt_connection_loss_while_heavy_replication(started_cluster): for i in range(threads_num): threads.append(threading.Thread(target=transaction, args=(i,))) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for thread in threads: time.sleep(random.uniform(0, 0.5)) thread.start() for thread in threads: - thread.join() # Join here because it takes time for data to reach wal + thread.join() # Join here because it takes time for data to reach wal time.sleep(2) - started_cluster.pause_container('postgres1') + started_cluster.pause_container("postgres1") # for i in range(NUM_TABLES): # result = instance.query(f"SELECT count() FROM test_database.postgresql_replica_{i}") # print(result) # Just debug - started_cluster.unpause_container('postgres1') + started_cluster.unpause_container("postgres1") check_several_tables_are_synchronized(instance, NUM_TABLES) @@ -573,7 +822,9 @@ def test_drop_database_while_replication_startup_not_finished(started_cluster): NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables(NUM_TABLES, 100000) for i in range(6): - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) time.sleep(0.5 * i) pg_manager.drop_materialized_db() @@ -581,7 +832,9 @@ def test_drop_database_while_replication_startup_not_finished(started_cluster): def test_restart_server_while_replication_startup_not_finished(started_cluster): NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables(NUM_TABLES, 100000) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) time.sleep(1) instance.restart_clickhouse() check_several_tables_are_synchronized(instance, NUM_TABLES) @@ -590,15 +843,23 @@ def test_restart_server_while_replication_startup_not_finished(started_cluster): def test_abrupt_server_restart_while_heavy_replication(started_cluster): def transaction(thread_id): if thread_id % 2: - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, auto_commit=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + auto_commit=True, + ) else: - conn = get_postgres_conn(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, auto_commit=False) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + auto_commit=False, + ) cursor = conn.cursor() for query in queries: cursor.execute(query.format(thread_id)) - print('thread {}, query {}'.format(thread_id, query)) + print("thread {}, query {}".format(thread_id, query)) if thread_id % 2 == 0: conn.commit() @@ -610,65 +871,87 @@ def test_abrupt_server_restart_while_heavy_replication(started_cluster): for i in range(threads_num): threads.append(threading.Thread(target=transaction, args=(i,))) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for thread in threads: time.sleep(random.uniform(0, 0.5)) thread.start() for thread in threads: - thread.join() # Join here because it takes time for data to reach wal + thread.join() # Join here because it takes time for data to reach wal instance.restart_clickhouse() check_several_tables_are_synchronized(instance, NUM_TABLES) def test_quoting_1(started_cluster): - table_name = 'user' + table_name = "user" pg_manager.create_and_fill_postgres_table(table_name) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) - check_tables_are_synchronized(instance, table_name); + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) + check_tables_are_synchronized(instance, table_name) def test_quoting_2(started_cluster): - table_name = 'user' + table_name = "user" pg_manager.create_and_fill_postgres_table(table_name) pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=[f"materialized_postgresql_tables_list = '{table_name}'"]) - check_tables_are_synchronized(instance, table_name); + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[f"materialized_postgresql_tables_list = '{table_name}'"], + ) + check_tables_are_synchronized(instance, table_name) def test_user_managed_slots(started_cluster): - slot_name = 'user_slot' - table_name = 'test_table' + slot_name = "user_slot" + table_name = "test_table" pg_manager.create_and_fill_postgres_table(table_name) replication_connection = get_postgres_conn( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - database=True, replication=True, auto_commit=True) + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + replication=True, + auto_commit=True, + ) snapshot = create_replication_slot(replication_connection, slot_name=slot_name) pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=[f"materialized_postgresql_replication_slot = '{slot_name}'", - f"materialized_postgresql_snapshot = '{snapshot}'"]) - check_tables_are_synchronized(instance, table_name); + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + f"materialized_postgresql_replication_slot = '{slot_name}'", + f"materialized_postgresql_snapshot = '{snapshot}'", + ], + ) + check_tables_are_synchronized(instance, table_name) - instance.query("INSERT INTO postgres_database.{} SELECT number, number from numbers(10000, 10000)".format(table_name)) - check_tables_are_synchronized(instance, table_name); + instance.query( + "INSERT INTO postgres_database.{} SELECT number, number from numbers(10000, 10000)".format( + table_name + ) + ) + check_tables_are_synchronized(instance, table_name) instance.restart_clickhouse() - instance.query("INSERT INTO postgres_database.{} SELECT number, number from numbers(20000, 10000)".format(table_name)) - check_tables_are_synchronized(instance, table_name); + instance.query( + "INSERT INTO postgres_database.{} SELECT number, number from numbers(20000, 10000)".format( + table_name + ) + ) + check_tables_are_synchronized(instance, table_name) pg_manager.drop_materialized_db() drop_replication_slot(replication_connection, slot_name) replication_connection.close() -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 3226c040e8e..2fcd0374fa9 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -18,20 +18,32 @@ from helpers.postgres_utility import PostgresManager from helpers.postgres_utility import create_replication_slot, drop_replication_slot from helpers.postgres_utility import create_postgres_schema, drop_postgres_schema from helpers.postgres_utility import create_postgres_table, drop_postgres_table -from helpers.postgres_utility import create_postgres_table_with_schema, drop_postgres_table_with_schema +from helpers.postgres_utility import ( + create_postgres_table_with_schema, + drop_postgres_table_with_schema, +) from helpers.postgres_utility import check_tables_are_synchronized from helpers.postgres_utility import check_several_tables_are_synchronized from helpers.postgres_utility import assert_nested_table_is_created from helpers.postgres_utility import assert_number_of_columns -from helpers.postgres_utility import postgres_table_template, postgres_table_template_2, postgres_table_template_3, postgres_table_template_4, postgres_table_template_5 +from helpers.postgres_utility import ( + postgres_table_template, + postgres_table_template_2, + postgres_table_template_3, + postgres_table_template_4, + postgres_table_template_5, +) from helpers.postgres_utility import queries cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs = ['configs/log_conf.xml'], - user_configs = ['configs/users.xml'], - with_postgres=True, stay_alive=True) +instance = cluster.add_instance( + "instance", + main_configs=["configs/log_conf.xml"], + user_configs=["configs/users.xml"], + with_postgres=True, + stay_alive=True, +) pg_manager = PostgresManager() @@ -56,359 +68,586 @@ def setup_teardown(): def test_add_new_table_to_replication(started_cluster): cursor = pg_manager.get_db_cursor() - cursor.execute('DROP TABLE IF EXISTS test_table') + cursor.execute("DROP TABLE IF EXISTS test_table") NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables_from_cursor(cursor, NUM_TABLES, 10000) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) check_several_tables_are_synchronized(instance, NUM_TABLES) result = instance.query("SHOW TABLES FROM test_database") - assert(result == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n") + assert ( + result + == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n" + ) - table_name = 'postgresql_replica_5' + table_name = "postgresql_replica_5" pg_manager.create_and_fill_postgres_table_from_cursor(cursor, table_name) - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") # Check without ip - assert(result[-59:] == "\\'postgres_database\\', \\'postgres\\', \\'mysecretpassword\\')\n") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) # Check without ip + assert ( + result[-59:] + == "\\'postgres_database\\', \\'postgres\\', \\'mysecretpassword\\')\n" + ) - result = instance.query_and_get_error("ALTER DATABASE test_database MODIFY SETTING materialized_postgresql_tables_list='tabl1'") - assert('Changing setting `materialized_postgresql_tables_list` is not allowed' in result) + result = instance.query_and_get_error( + "ALTER DATABASE test_database MODIFY SETTING materialized_postgresql_tables_list='tabl1'" + ) + assert ( + "Changing setting `materialized_postgresql_tables_list` is not allowed" + in result + ) - result = instance.query_and_get_error("ALTER DATABASE test_database MODIFY SETTING materialized_postgresql_tables='tabl1'") - assert('Database engine MaterializedPostgreSQL does not support setting' in result) + result = instance.query_and_get_error( + "ALTER DATABASE test_database MODIFY SETTING materialized_postgresql_tables='tabl1'" + ) + assert "Database engine MaterializedPostgreSQL does not support setting" in result - instance.query(f"ATTACH TABLE test_database.{table_name}"); + instance.query(f"ATTACH TABLE test_database.{table_name}") result = instance.query("SHOW TABLES FROM test_database") - assert(result == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\npostgresql_replica_5\n") + assert ( + result + == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\npostgresql_replica_5\n" + ) - check_tables_are_synchronized(instance, table_name); - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(10000, 10000)") - check_tables_are_synchronized(instance, table_name); + check_tables_are_synchronized(instance, table_name) + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(10000, 10000)" + ) + check_tables_are_synchronized(instance, table_name) - result = instance.query_and_get_error(f"ATTACH TABLE test_database.{table_name}"); - assert('Table test_database.postgresql_replica_5 already exists' in result) + result = instance.query_and_get_error(f"ATTACH TABLE test_database.{table_name}") + assert "Table test_database.postgresql_replica_5 already exists" in result - result = instance.query_and_get_error("ATTACH TABLE test_database.unknown_table"); - assert('PostgreSQL table unknown_table does not exist' in result) + result = instance.query_and_get_error("ATTACH TABLE test_database.unknown_table") + assert "PostgreSQL table unknown_table does not exist" in result - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") - assert(result[-180:] == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4,postgresql_replica_5\\'\n") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) + assert ( + result[-180:] + == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4,postgresql_replica_5\\'\n" + ) - table_name = 'postgresql_replica_6' + table_name = "postgresql_replica_6" create_postgres_table(cursor, table_name) - instance.query("INSERT INTO postgres_database.{} SELECT number, number from numbers(10000)".format(table_name)) - instance.query(f"ATTACH TABLE test_database.{table_name}"); + instance.query( + "INSERT INTO postgres_database.{} SELECT number, number from numbers(10000)".format( + table_name + ) + ) + instance.query(f"ATTACH TABLE test_database.{table_name}") instance.restart_clickhouse() - table_name = 'postgresql_replica_7' + table_name = "postgresql_replica_7" create_postgres_table(cursor, table_name) - instance.query("INSERT INTO postgres_database.{} SELECT number, number from numbers(10000)".format(table_name)) - instance.query(f"ATTACH TABLE test_database.{table_name}"); + instance.query( + "INSERT INTO postgres_database.{} SELECT number, number from numbers(10000)".format( + table_name + ) + ) + instance.query(f"ATTACH TABLE test_database.{table_name}") - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") - assert(result[-222:] == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4,postgresql_replica_5,postgresql_replica_6,postgresql_replica_7\\'\n") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) + assert ( + result[-222:] + == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4,postgresql_replica_5,postgresql_replica_6,postgresql_replica_7\\'\n" + ) + + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(10000, 10000)" + ) result = instance.query("SHOW TABLES FROM test_database") - assert(result == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\npostgresql_replica_5\npostgresql_replica_6\npostgresql_replica_7\n") + assert ( + result + == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\npostgresql_replica_5\npostgresql_replica_6\npostgresql_replica_7\n" + ) check_several_tables_are_synchronized(instance, NUM_TABLES + 3) def test_remove_table_from_replication(started_cluster): NUM_TABLES = 5 pg_manager.create_and_fill_postgres_tables(NUM_TABLES, 10000) - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) check_several_tables_are_synchronized(instance, NUM_TABLES) result = instance.query("SHOW TABLES FROM test_database") - assert(result == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n") + assert ( + result + == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n" + ) - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") - assert(result[-59:] == "\\'postgres_database\\', \\'postgres\\', \\'mysecretpassword\\')\n") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) + assert ( + result[-59:] + == "\\'postgres_database\\', \\'postgres\\', \\'mysecretpassword\\')\n" + ) - table_name = 'postgresql_replica_4' - instance.query(f'DETACH TABLE test_database.{table_name}'); - result = instance.query_and_get_error(f'SELECT * FROM test_database.{table_name}') - assert("doesn't exist" in result) + table_name = "postgresql_replica_4" + instance.query(f"DETACH TABLE test_database.{table_name} PERMANENTLY") + result = instance.query_and_get_error(f"SELECT * FROM test_database.{table_name}") + assert "doesn't exist" in result result = instance.query("SHOW TABLES FROM test_database") - assert(result == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\n") + assert ( + result + == "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\n" + ) - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") - assert(result[-138:] == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3\\'\n") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) + assert ( + result[-138:] + == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3\\'\n" + ) - instance.query(f'ATTACH TABLE test_database.{table_name}'); - check_tables_are_synchronized(instance, table_name); + instance.query(f"ATTACH TABLE test_database.{table_name}") + check_tables_are_synchronized(instance, table_name) check_several_tables_are_synchronized(instance, NUM_TABLES) + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number from numbers(10000, 10000)" + ) + check_tables_are_synchronized(instance, table_name) - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") - assert(result[-159:] == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4\\'\n") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) + assert ( + result[-159:] + == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_1,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4\\'\n" + ) - table_name = 'postgresql_replica_1' - instance.query(f'DETACH TABLE test_database.{table_name}'); - result = instance.query('SHOW CREATE DATABASE test_database') - assert(result[:63] == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(") - assert(result[-138:] == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4\\'\n") + table_name = "postgresql_replica_1" + instance.query(f"DETACH TABLE test_database.{table_name} PERMANENTLY") + result = instance.query("SHOW CREATE DATABASE test_database") + assert ( + result[:63] + == "CREATE DATABASE test_database\\nENGINE = MaterializedPostgreSQL(" + ) + assert ( + result[-138:] + == ")\\nSETTINGS materialized_postgresql_tables_list = \\'postgresql_replica_0,postgresql_replica_2,postgresql_replica_3,postgresql_replica_4\\'\n" + ) cursor = pg_manager.get_db_cursor() - cursor.execute(f'drop table if exists postgresql_replica_0;') + cursor.execute(f"drop table if exists postgresql_replica_0;") # Removing from replication table which does not exist in PostgreSQL must be ok. - instance.query('DETACH TABLE test_database.postgresql_replica_0'); - assert instance.contains_in_log("from publication, because table does not exist in PostgreSQL") + instance.query("DETACH TABLE test_database.postgresql_replica_0 PERMANENTLY") + assert instance.contains_in_log( + "from publication, because table does not exist in PostgreSQL" + ) def test_predefined_connection_configuration(started_cluster): cursor = pg_manager.get_db_cursor() - cursor.execute(f'DROP TABLE IF EXISTS test_table') - cursor.execute(f'CREATE TABLE test_table (key integer PRIMARY KEY, value integer)') - cursor.execute(f'INSERT INTO test_table SELECT 1, 2') - instance.query("CREATE DATABASE test_database ENGINE = MaterializedPostgreSQL(postgres1) SETTINGS materialized_postgresql_tables_list='test_table'") - check_tables_are_synchronized(instance, "test_table"); + cursor.execute(f"DROP TABLE IF EXISTS test_table") + cursor.execute(f"CREATE TABLE test_table (key integer PRIMARY KEY, value integer)") + cursor.execute(f"INSERT INTO test_table SELECT 1, 2") + instance.query( + "CREATE DATABASE test_database ENGINE = MaterializedPostgreSQL(postgres1) SETTINGS materialized_postgresql_tables_list='test_table'" + ) + check_tables_are_synchronized(instance, "test_table") pg_manager.drop_materialized_db() insert_counter = 0 + def test_database_with_single_non_default_schema(started_cluster): cursor = pg_manager.get_db_cursor() - NUM_TABLES=5 - schema_name = 'test_schema' - materialized_db = 'test_database' - clickhouse_postgres_db = 'postgres_database_with_schema' + NUM_TABLES = 5 + schema_name = "test_schema" + materialized_db = "test_database" + clickhouse_postgres_db = "postgres_database_with_schema" global insert_counter insert_counter = 0 def insert_into_tables(): global insert_counter - clickhouse_postgres_db = 'postgres_database_with_schema' + clickhouse_postgres_db = "postgres_database_with_schema" for i in range(NUM_TABLES): - table_name = f'postgresql_replica_{i}' - instance.query(f"INSERT INTO {clickhouse_postgres_db}.{table_name} SELECT number, number from numbers(1000 * {insert_counter}, 1000)") + table_name = f"postgresql_replica_{i}" + instance.query( + f"INSERT INTO {clickhouse_postgres_db}.{table_name} SELECT number, number from numbers(1000 * {insert_counter}, 1000)" + ) insert_counter += 1 def assert_show_tables(expected): - result = instance.query('SHOW TABLES FROM test_database') - assert(result == expected) - print('assert show tables Ok') + result = instance.query("SHOW TABLES FROM test_database") + assert result == expected + print("assert show tables Ok") def check_all_tables_are_synchronized(): for i in range(NUM_TABLES): - print('checking table', i) - check_tables_are_synchronized(instance, f"postgresql_replica_{i}", postgres_database=clickhouse_postgres_db); - print('synchronization Ok') + print("checking table", i) + check_tables_are_synchronized( + instance, + f"postgresql_replica_{i}", + postgres_database=clickhouse_postgres_db, + ) + print("synchronization Ok") create_postgres_schema(cursor, schema_name) - pg_manager.create_clickhouse_postgres_db(ip=cluster.postgres_ip, port=cluster.postgres_port, name=clickhouse_postgres_db, schema_name=schema_name) + pg_manager.create_clickhouse_postgres_db( + ip=cluster.postgres_ip, + port=cluster.postgres_port, + name=clickhouse_postgres_db, + schema_name=schema_name, + ) for i in range(NUM_TABLES): - create_postgres_table_with_schema(cursor, schema_name, f'postgresql_replica_{i}'); + create_postgres_table_with_schema( + cursor, schema_name, f"postgresql_replica_{i}" + ) insert_into_tables() - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=[f"materialized_postgresql_schema = '{schema_name}'", "materialized_postgresql_allow_automatic_update = 1"]) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + f"materialized_postgresql_schema = '{schema_name}'", + "materialized_postgresql_allow_automatic_update = 1", + ], + ) insert_into_tables() check_all_tables_are_synchronized() - assert_show_tables("postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n") + assert_show_tables( + "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n" + ) instance.restart_clickhouse() check_all_tables_are_synchronized() - assert_show_tables("postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n") + assert_show_tables( + "postgresql_replica_0\npostgresql_replica_1\npostgresql_replica_2\npostgresql_replica_3\npostgresql_replica_4\n" + ) insert_into_tables() check_all_tables_are_synchronized() - print('ALTER') - altered_table = random.randint(0, NUM_TABLES-1) - cursor.execute("ALTER TABLE test_schema.postgresql_replica_{} ADD COLUMN value2 integer".format(altered_table)) + print("ALTER") + altered_table = random.randint(0, NUM_TABLES - 1) + cursor.execute( + "ALTER TABLE test_schema.postgresql_replica_{} ADD COLUMN value2 integer".format( + altered_table + ) + ) - instance.query(f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)") - assert_number_of_columns(instance, 3, f'postgresql_replica_{altered_table}') - check_tables_are_synchronized(instance, f"postgresql_replica_{altered_table}", postgres_database=clickhouse_postgres_db); + instance.query( + f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)" + ) + assert_number_of_columns(instance, 3, f"postgresql_replica_{altered_table}") + check_tables_are_synchronized( + instance, + f"postgresql_replica_{altered_table}", + postgres_database=clickhouse_postgres_db, + ) - print('DETACH-ATTACH') + print("DETACH-ATTACH") detached_table_name = "postgresql_replica_1" - instance.query(f"DETACH TABLE {materialized_db}.{detached_table_name}") - assert not instance.contains_in_log("from publication, because table does not exist in PostgreSQL") + instance.query(f"DETACH TABLE {materialized_db}.{detached_table_name} PERMANENTLY") + assert not instance.contains_in_log( + "from publication, because table does not exist in PostgreSQL" + ) instance.query(f"ATTACH TABLE {materialized_db}.{detached_table_name}") - check_tables_are_synchronized(instance, detached_table_name, postgres_database=clickhouse_postgres_db); + check_tables_are_synchronized( + instance, detached_table_name, postgres_database=clickhouse_postgres_db + ) def test_database_with_multiple_non_default_schemas_1(started_cluster): cursor = pg_manager.get_db_cursor() NUM_TABLES = 5 - schema_name = 'test_schema' - clickhouse_postgres_db = 'postgres_database_with_schema' - materialized_db = 'test_database' - publication_tables = '' + schema_name = "test_schema" + clickhouse_postgres_db = "postgres_database_with_schema" + materialized_db = "test_database" + publication_tables = "" global insert_counter insert_counter = 0 def insert_into_tables(): global insert_counter - clickhouse_postgres_db = 'postgres_database_with_schema' + clickhouse_postgres_db = "postgres_database_with_schema" for i in range(NUM_TABLES): - table_name = f'postgresql_replica_{i}' - instance.query(f"INSERT INTO {clickhouse_postgres_db}.{table_name} SELECT number, number from numbers(1000 * {insert_counter}, 1000)") + table_name = f"postgresql_replica_{i}" + instance.query( + f"INSERT INTO {clickhouse_postgres_db}.{table_name} SELECT number, number from numbers(1000 * {insert_counter}, 1000)" + ) insert_counter += 1 def assert_show_tables(expected): - result = instance.query('SHOW TABLES FROM test_database') - assert(result == expected) - print('assert show tables Ok') + result = instance.query("SHOW TABLES FROM test_database") + assert result == expected + print("assert show tables Ok") def check_all_tables_are_synchronized(): for i in range(NUM_TABLES): - print('checking table', i) - check_tables_are_synchronized(instance, "postgresql_replica_{}".format(i), schema_name=schema_name, postgres_database=clickhouse_postgres_db); - print('synchronization Ok') + print("checking table", i) + check_tables_are_synchronized( + instance, + "postgresql_replica_{}".format(i), + schema_name=schema_name, + postgres_database=clickhouse_postgres_db, + ) + print("synchronization Ok") create_postgres_schema(cursor, schema_name) - pg_manager.create_clickhouse_postgres_db(ip=cluster.postgres_ip, port=cluster.postgres_port, name=clickhouse_postgres_db, schema_name=schema_name) + pg_manager.create_clickhouse_postgres_db( + ip=cluster.postgres_ip, + port=cluster.postgres_port, + name=clickhouse_postgres_db, + schema_name=schema_name, + ) for i in range(NUM_TABLES): - table_name = 'postgresql_replica_{}'.format(i) - create_postgres_table_with_schema(cursor, schema_name, table_name); - if publication_tables != '': - publication_tables += ', ' - publication_tables += schema_name + '.' + table_name + table_name = "postgresql_replica_{}".format(i) + create_postgres_table_with_schema(cursor, schema_name, table_name) + if publication_tables != "": + publication_tables += ", " + publication_tables += schema_name + "." + table_name insert_into_tables() - pg_manager.create_materialized_db(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=[f"materialized_postgresql_tables_list = '{publication_tables}'", "materialized_postgresql_tables_list_with_schema=1", "materialized_postgresql_allow_automatic_update = 1"]) + pg_manager.create_materialized_db( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + f"materialized_postgresql_tables_list = '{publication_tables}'", + "materialized_postgresql_tables_list_with_schema=1", + "materialized_postgresql_allow_automatic_update = 1", + ], + ) check_all_tables_are_synchronized() - assert_show_tables("test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n") + assert_show_tables( + "test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n" + ) instance.restart_clickhouse() check_all_tables_are_synchronized() - assert_show_tables("test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n") + assert_show_tables( + "test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n" + ) insert_into_tables() check_all_tables_are_synchronized() - print('ALTER') - altered_table = random.randint(0, NUM_TABLES-1) - cursor.execute("ALTER TABLE test_schema.postgresql_replica_{} ADD COLUMN value2 integer".format(altered_table)) + print("ALTER") + altered_table = random.randint(0, NUM_TABLES - 1) + cursor.execute( + "ALTER TABLE test_schema.postgresql_replica_{} ADD COLUMN value2 integer".format( + altered_table + ) + ) - instance.query(f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)") - assert_number_of_columns(instance, 3, f'{schema_name}.postgresql_replica_{altered_table}') - check_tables_are_synchronized(instance, f"postgresql_replica_{altered_table}", schema_name=schema_name, postgres_database=clickhouse_postgres_db); + instance.query( + f"INSERT INTO {clickhouse_postgres_db}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(5000, 1000)" + ) + assert_number_of_columns( + instance, 3, f"{schema_name}.postgresql_replica_{altered_table}" + ) + check_tables_are_synchronized( + instance, + f"postgresql_replica_{altered_table}", + schema_name=schema_name, + postgres_database=clickhouse_postgres_db, + ) - print('DETACH-ATTACH') + print("DETACH-ATTACH") detached_table_name = "postgresql_replica_1" - instance.query(f"DETACH TABLE {materialized_db}.`{schema_name}.{detached_table_name}`") - assert not instance.contains_in_log("from publication, because table does not exist in PostgreSQL") - instance.query(f"ATTACH TABLE {materialized_db}.`{schema_name}.{detached_table_name}`") - assert_show_tables("test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n") - check_tables_are_synchronized(instance, detached_table_name, schema_name=schema_name, postgres_database=clickhouse_postgres_db); + instance.query( + f"DETACH TABLE {materialized_db}.`{schema_name}.{detached_table_name}` PERMANENTLY" + ) + assert not instance.contains_in_log( + "from publication, because table does not exist in PostgreSQL" + ) + instance.query( + f"ATTACH TABLE {materialized_db}.`{schema_name}.{detached_table_name}`" + ) + assert_show_tables( + "test_schema.postgresql_replica_0\ntest_schema.postgresql_replica_1\ntest_schema.postgresql_replica_2\ntest_schema.postgresql_replica_3\ntest_schema.postgresql_replica_4\n" + ) + check_tables_are_synchronized( + instance, + detached_table_name, + schema_name=schema_name, + postgres_database=clickhouse_postgres_db, + ) def test_database_with_multiple_non_default_schemas_2(started_cluster): cursor = pg_manager.get_db_cursor() NUM_TABLES = 2 schemas_num = 2 - schema_list = 'schema0, schema1' - materialized_db = 'test_database' + schema_list = "schema0, schema1" + materialized_db = "test_database" global insert_counter insert_counter = 0 def check_all_tables_are_synchronized(): for i in range(schemas_num): - schema_name = f'schema{i}' - clickhouse_postgres_db = f'clickhouse_postgres_db{i}' + schema_name = f"schema{i}" + clickhouse_postgres_db = f"clickhouse_postgres_db{i}" for ti in range(NUM_TABLES): - table_name = f'postgresql_replica_{ti}' - print(f'checking table {schema_name}.{table_name}') - check_tables_are_synchronized(instance, f'{table_name}', schema_name=schema_name, postgres_database=clickhouse_postgres_db); - print('synchronized Ok') + table_name = f"postgresql_replica_{ti}" + print(f"checking table {schema_name}.{table_name}") + check_tables_are_synchronized( + instance, + f"{table_name}", + schema_name=schema_name, + postgres_database=clickhouse_postgres_db, + ) + print("synchronized Ok") def insert_into_tables(): global insert_counter for i in range(schemas_num): - clickhouse_postgres_db = f'clickhouse_postgres_db{i}' + clickhouse_postgres_db = f"clickhouse_postgres_db{i}" for ti in range(NUM_TABLES): - table_name = f'postgresql_replica_{ti}' - instance.query(f'INSERT INTO {clickhouse_postgres_db}.{table_name} SELECT number, number from numbers(1000 * {insert_counter}, 1000)') + table_name = f"postgresql_replica_{ti}" + instance.query( + f"INSERT INTO {clickhouse_postgres_db}.{table_name} SELECT number, number from numbers(1000 * {insert_counter}, 1000)" + ) insert_counter += 1 def assert_show_tables(expected): - result = instance.query('SHOW TABLES FROM test_database') - assert(result == expected) - print('assert show tables Ok') + result = instance.query("SHOW TABLES FROM test_database") + assert result == expected + print("assert show tables Ok") for i in range(schemas_num): - schema_name = f'schema{i}' - clickhouse_postgres_db = f'clickhouse_postgres_db{i}' + schema_name = f"schema{i}" + clickhouse_postgres_db = f"clickhouse_postgres_db{i}" create_postgres_schema(cursor, schema_name) - pg_manager.create_clickhouse_postgres_db(ip=cluster.postgres_ip, port=cluster.postgres_port, name=clickhouse_postgres_db, schema_name=schema_name) + pg_manager.create_clickhouse_postgres_db( + ip=cluster.postgres_ip, + port=cluster.postgres_port, + name=clickhouse_postgres_db, + schema_name=schema_name, + ) for ti in range(NUM_TABLES): - table_name = f'postgresql_replica_{ti}' - create_postgres_table_with_schema(cursor, schema_name, table_name); + table_name = f"postgresql_replica_{ti}" + create_postgres_table_with_schema(cursor, schema_name, table_name) insert_into_tables() pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=[f"materialized_postgresql_schema_list = '{schema_list}'", - "materialized_postgresql_allow_automatic_update = 1"]) + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + f"materialized_postgresql_schema_list = '{schema_list}'", + "materialized_postgresql_allow_automatic_update = 1", + ], + ) check_all_tables_are_synchronized() insert_into_tables() - assert_show_tables("schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n") + assert_show_tables( + "schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n" + ) instance.restart_clickhouse() - assert_show_tables("schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n") + assert_show_tables( + "schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n" + ) check_all_tables_are_synchronized() insert_into_tables() check_all_tables_are_synchronized() - print('ALTER') - altered_schema = random.randint(0, schemas_num-1) - altered_table = random.randint(0, NUM_TABLES-1) - clickhouse_postgres_db = f'clickhouse_postgres_db{altered_schema}' - cursor.execute(f"ALTER TABLE schema{altered_schema}.postgresql_replica_{altered_table} ADD COLUMN value2 integer") + print("ALTER") + altered_schema = random.randint(0, schemas_num - 1) + altered_table = random.randint(0, NUM_TABLES - 1) + clickhouse_postgres_db = f"clickhouse_postgres_db{altered_schema}" + cursor.execute( + f"ALTER TABLE schema{altered_schema}.postgresql_replica_{altered_table} ADD COLUMN value2 integer" + ) - instance.query(f"INSERT INTO clickhouse_postgres_db{altered_schema}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(1000 * {insert_counter}, 1000)") - assert_number_of_columns(instance, 3, f'schema{altered_schema}.postgresql_replica_{altered_table}') - check_tables_are_synchronized(instance, f"postgresql_replica_{altered_table}", schema_name=f"schema{altered_schema}", postgres_database=clickhouse_postgres_db); + instance.query( + f"INSERT INTO clickhouse_postgres_db{altered_schema}.postgresql_replica_{altered_table} SELECT number, number, number from numbers(1000 * {insert_counter}, 1000)" + ) + assert_number_of_columns( + instance, 3, f"schema{altered_schema}.postgresql_replica_{altered_table}" + ) + check_tables_are_synchronized( + instance, + f"postgresql_replica_{altered_table}", + schema_name=f"schema{altered_schema}", + postgres_database=clickhouse_postgres_db, + ) - print('DETACH-ATTACH') + print("DETACH-ATTACH") detached_table_name = "postgresql_replica_1" detached_table_schema = "schema0" - clickhouse_postgres_db = f'clickhouse_postgres_db0' - instance.query(f"DETACH TABLE {materialized_db}.`{detached_table_schema}.{detached_table_name}`") - assert not instance.contains_in_log("from publication, because table does not exist in PostgreSQL") - instance.query(f"ATTACH TABLE {materialized_db}.`{detached_table_schema}.{detached_table_name}`") - assert_show_tables("schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n") - check_tables_are_synchronized(instance, f"postgresql_replica_{altered_table}", schema_name=detached_table_schema, postgres_database=clickhouse_postgres_db); + clickhouse_postgres_db = f"clickhouse_postgres_db0" + instance.query( + f"DETACH TABLE {materialized_db}.`{detached_table_schema}.{detached_table_name}` PERMANENTLY" + ) + assert not instance.contains_in_log( + "from publication, because table does not exist in PostgreSQL" + ) + instance.query( + f"ATTACH TABLE {materialized_db}.`{detached_table_schema}.{detached_table_name}`" + ) + assert_show_tables( + "schema0.postgresql_replica_0\nschema0.postgresql_replica_1\nschema1.postgresql_replica_0\nschema1.postgresql_replica_1\n" + ) + check_tables_are_synchronized( + instance, + f"postgresql_replica_{altered_table}", + schema_name=detached_table_schema, + postgres_database=clickhouse_postgres_db, + ) def test_table_override(started_cluster): cursor = pg_manager.get_db_cursor() - table_name = 'table_override' - materialized_database = 'test_database' - create_postgres_table(cursor, table_name, template=postgres_table_template_5); - instance.query(f"create table {table_name}(key Int32, value UUID) engine = PostgreSQL (postgres1, table={table_name})") - instance.query(f"insert into {table_name} select number, generateUUIDv4() from numbers(10)") - table_overrides = f" TABLE OVERRIDE {table_name} (COLUMNS (key Int32, value UUID))" + table_name = "table_override" + materialized_database = "test_database" + create_postgres_table(cursor, table_name, template=postgres_table_template_5) + instance.query( + f"create table {table_name}(key Int32, value UUID) engine = PostgreSQL (postgres1, table={table_name})" + ) + instance.query( + f"insert into {table_name} select number, generateUUIDv4() from numbers(10)" + ) + table_overrides = f" TABLE OVERRIDE {table_name} (COLUMNS (key Int32, value UUID) PARTITION BY key)" pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, settings=[f"materialized_postgresql_tables_list = '{table_name}'"], - table_overrides=table_overrides) + table_overrides=table_overrides, + ) assert_nested_table_is_created(instance, table_name, materialized_database) result = instance.query(f"show create table {materialized_database}.{table_name}") print(result) - expected = "CREATE TABLE test_database.table_override\\n(\\n `key` Int32,\\n `value` UUID,\\n `_sign` Int8() MATERIALIZED 1,\\n `_version` UInt64() MATERIALIZED 1\\n)\\nENGINE = ReplacingMergeTree(_version)\\nORDER BY tuple(key)" - assert(result.strip() == expected) + expected = "CREATE TABLE test_database.table_override\\n(\\n `key` Int32,\\n `value` UUID,\\n `_sign` Int8() MATERIALIZED 1,\\n `_version` UInt64() MATERIALIZED 1\\n)\\nENGINE = ReplacingMergeTree(_version)\\nPARTITION BY key\\nORDER BY tuple(key)" + assert result.strip() == expected time.sleep(5) query = f"select * from {materialized_database}.{table_name} order by key" expected = instance.query(f"select * from {table_name} order by key") @@ -420,15 +659,23 @@ def test_table_schema_changes_2(started_cluster): cursor = pg_manager.get_db_cursor() table_name = "test_table" - create_postgres_table(cursor, table_name, template=postgres_table_template_2); - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number, number, number from numbers(25)") + create_postgres_table(cursor, table_name, template=postgres_table_template_2) + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number, number, number from numbers(25)" + ) pg_manager.create_materialized_db( - ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, - settings=["materialized_postgresql_allow_automatic_update = 1, materialized_postgresql_tables_list='test_table'"]) + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + settings=[ + "materialized_postgresql_allow_automatic_update = 1, materialized_postgresql_tables_list='test_table'" + ], + ) - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, number, number, number from numbers(25, 25)") - check_tables_are_synchronized(instance, table_name); + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, number, number, number from numbers(25, 25)" + ) + check_tables_are_synchronized(instance, table_name) cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value1") cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value2") @@ -438,27 +685,35 @@ def test_table_schema_changes_2(started_cluster): cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value3 Text") cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value4 Text") cursor.execute(f"UPDATE {table_name} SET value3 = 'kek' WHERE key%2=0") - check_tables_are_synchronized(instance, table_name); - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number) from numbers(50, 25)") + check_tables_are_synchronized(instance, table_name) + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number) from numbers(50, 25)" + ) cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value5 Integer") cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value2") - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), number from numbers(75, 25)") - check_tables_are_synchronized(instance, table_name); + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), number from numbers(75, 25)" + ) + check_tables_are_synchronized(instance, table_name) instance.restart_clickhouse() - check_tables_are_synchronized(instance, table_name); + check_tables_are_synchronized(instance, table_name) cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value5") cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value5 Text") - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number) from numbers(100, 25)") - check_tables_are_synchronized(instance, table_name); + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number) from numbers(100, 25)" + ) + check_tables_are_synchronized(instance, table_name) cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value6 Text") cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value7 Integer") cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN value8 Integer") cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN value5") - instance.query(f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number), number, number from numbers(125, 25)") - check_tables_are_synchronized(instance, table_name); + instance.query( + f"INSERT INTO postgres_database.{table_name} SELECT number, toString(number), toString(number), toString(number), toString(number), number, number from numbers(125, 25)" + ) + check_tables_are_synchronized(instance, table_name) -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_profile_events_s3/test.py b/tests/integration/test_profile_events_s3/test.py index 15e2ff97d10..5171ea4ac0e 100644 --- a/tests/integration/test_profile_events_s3/test.py +++ b/tests/integration/test_profile_events_s3/test.py @@ -11,8 +11,16 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/log.xml", - "configs/query_log.xml", "configs/ssl_conf.xml"], with_minio=True) + cluster.add_instance( + "node", + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/log.xml", + "configs/query_log.xml", + "configs/ssl_conf.xml", + ], + with_minio=True, + ) logging.info("Starting cluster...") cluster.start() @@ -41,7 +49,9 @@ init_list = { def get_s3_events(instance): result = init_list.copy() - events = instance.query("SELECT event,value FROM system.events WHERE event LIKE 'S3%'").split("\n") + events = instance.query( + "SELECT event,value FROM system.events WHERE event LIKE 'S3%'" + ).split("\n") for event in events: ev = event.split("\t") if len(ev) == 2: @@ -57,12 +67,15 @@ def get_minio_stat(cluster): "rx_bytes": 0, "tx_bytes": 0, } - stat = requests.get(url="http://{}:{}/minio/prometheus/metrics".format(cluster.minio_ip, cluster.minio_port)).text.split( - "\n") + stat = requests.get( + url="http://{}:{}/minio/prometheus/metrics".format( + cluster.minio_ip, cluster.minio_port + ) + ).text.split("\n") for line in stat: x = re.search("s3_requests_total(\{.*\})?\s(\d+)(\s.*)?", line) if x != None: - y = re.search(".*api=\"(get|list|head|select).*", x.group(1)) + y = re.search('.*api="(get|list|head|select).*', x.group(1)) if y != None: result["get_requests"] += int(x.group(2)) else: @@ -82,12 +95,16 @@ def get_minio_stat(cluster): def get_query_stat(instance, hint): result = init_list.copy() instance.query("SYSTEM FLUSH LOGS") - events = instance.query(''' + events = instance.query( + """ SELECT ProfileEvents.keys, ProfileEvents.values FROM system.query_log ARRAY JOIN ProfileEvents WHERE type != 1 AND query LIKE '%{}%' - '''.format(hint.replace("'", "\\'"))).split("\n") + """.format( + hint.replace("'", "\\'") + ) + ).split("\n") for event in events: ev = event.split("\t") if len(ev) == 2: @@ -99,7 +116,7 @@ def get_query_stat(instance, hint): def get_minio_size(cluster): minio = cluster.minio_client size = 0 - for obj in minio.list_objects(cluster.minio_bucket, 'data/'): + for obj in minio.list_objects(cluster.minio_bucket, "data/"): size += obj.size return size @@ -123,10 +140,14 @@ def test_profile_events(cluster): metrics1 = get_s3_events(instance) minio1 = get_minio_stat(cluster) - assert metrics1["S3ReadRequestsCount"] - metrics0["S3ReadRequestsCount"] == minio1["get_requests"] - minio0[ - "get_requests"] - 1 # 1 from get_minio_size - assert metrics1["S3WriteRequestsCount"] - metrics0["S3WriteRequestsCount"] == minio1["set_requests"] - minio0[ - "set_requests"] + assert ( + metrics1["S3ReadRequestsCount"] - metrics0["S3ReadRequestsCount"] + == minio1["get_requests"] - minio0["get_requests"] - 1 + ) # 1 from get_minio_size + assert ( + metrics1["S3WriteRequestsCount"] - metrics0["S3WriteRequestsCount"] + == minio1["set_requests"] - minio0["set_requests"] + ) stat1 = get_query_stat(instance, query1) for metric in stat1: assert stat1[metric] == metrics1[metric] - metrics0[metric] @@ -139,10 +160,14 @@ def test_profile_events(cluster): metrics2 = get_s3_events(instance) minio2 = get_minio_stat(cluster) - assert metrics2["S3ReadRequestsCount"] - metrics1["S3ReadRequestsCount"] == minio2["get_requests"] - minio1[ - "get_requests"] - 1 # 1 from get_minio_size - assert metrics2["S3WriteRequestsCount"] - metrics1["S3WriteRequestsCount"] == minio2["set_requests"] - minio1[ - "set_requests"] + assert ( + metrics2["S3ReadRequestsCount"] - metrics1["S3ReadRequestsCount"] + == minio2["get_requests"] - minio1["get_requests"] - 1 + ) # 1 from get_minio_size + assert ( + metrics2["S3WriteRequestsCount"] - metrics1["S3WriteRequestsCount"] + == minio2["set_requests"] - minio1["set_requests"] + ) stat2 = get_query_stat(instance, query2) for metric in stat2: assert stat2[metric] == metrics2[metric] - metrics1[metric] @@ -154,12 +179,16 @@ def test_profile_events(cluster): metrics3 = get_s3_events(instance) minio3 = get_minio_stat(cluster) - assert metrics3["S3ReadRequestsCount"] - metrics2["S3ReadRequestsCount"] == minio3["get_requests"] - minio2[ - "get_requests"] - assert metrics3["S3WriteRequestsCount"] - metrics2["S3WriteRequestsCount"] == minio3["set_requests"] - minio2[ - "set_requests"] + assert ( + metrics3["S3ReadRequestsCount"] - metrics2["S3ReadRequestsCount"] + == minio3["get_requests"] - minio2["get_requests"] + ) + assert ( + metrics3["S3WriteRequestsCount"] - metrics2["S3WriteRequestsCount"] + == minio3["set_requests"] - minio2["set_requests"] + ) stat3 = get_query_stat(instance, query3) # With async reads profile events are not updated fully because reads are done in a separate thread. - #for metric in stat3: + # for metric in stat3: # print(metric) # assert stat3[metric] == metrics3[metric] - metrics2[metric] diff --git a/tests/integration/test_prometheus_endpoint/test.py b/tests/integration/test_prometheus_endpoint/test.py index 60d9164acd2..cf3d2ff2d98 100644 --- a/tests/integration/test_prometheus_endpoint/test.py +++ b/tests/integration/test_prometheus_endpoint/test.py @@ -1,5 +1,3 @@ - - import re import time @@ -8,7 +6,7 @@ import requests from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/prom_conf.xml']) +node = cluster.add_instance("node", main_configs=["configs/prom_conf.xml"]) @pytest.fixture(scope="module") @@ -30,7 +28,7 @@ def parse_response_line(line): if line.startswith("#"): return {} - match = re.match('^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? -?(\d)', line) + match = re.match("^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? -?(\d)", line) assert match, line name, _, val = match.groups() return {name: int(val)} @@ -39,8 +37,10 @@ def parse_response_line(line): def get_and_check_metrics(retries): while True: try: - response = requests.get("http://{host}:{port}/metrics".format( - host=node.ip_address, port=8001), allow_redirects=False) + response = requests.get( + "http://{host}:{port}/metrics".format(host=node.ip_address, port=8001), + allow_redirects=False, + ) if response.status_code != 200: response.raise_for_status() @@ -54,10 +54,10 @@ def get_and_check_metrics(retries): else: raise - assert response.headers['content-type'].startswith('text/plain') + assert response.headers["content-type"].startswith("text/plain") results = {} - for resp_line in response.text.split('\n'): + for resp_line in response.text.split("\n"): resp_line = resp_line.rstrip() if not resp_line: continue @@ -68,12 +68,12 @@ def get_and_check_metrics(retries): def test_prometheus_endpoint(start_cluster): metrics_dict = get_and_check_metrics(10) - assert metrics_dict['ClickHouseProfileEvents_Query'] >= 0 - prev_query_count = metrics_dict['ClickHouseProfileEvents_Query'] + assert metrics_dict["ClickHouseProfileEvents_Query"] >= 0 + prev_query_count = metrics_dict["ClickHouseProfileEvents_Query"] node.query("SELECT 1") node.query("SELECT 2") node.query("SELECT 3") metrics_dict = get_and_check_metrics(10) - assert metrics_dict['ClickHouseProfileEvents_Query'] >= prev_query_count + 3 + assert metrics_dict["ClickHouseProfileEvents_Query"] >= prev_query_count + 3 diff --git a/tests/integration/test_query_deduplication/configs/deduplication_settings.xml b/tests/integration/test_query_deduplication/configs/deduplication_settings.xml deleted file mode 100644 index 6552fd68468..00000000000 --- a/tests/integration/test_query_deduplication/configs/deduplication_settings.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - 1 - - diff --git a/tests/integration/test_query_deduplication/configs/profiles.xml b/tests/integration/test_query_deduplication/configs/profiles.xml deleted file mode 100644 index d0dc0943202..00000000000 --- a/tests/integration/test_query_deduplication/configs/profiles.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - 1 - - - \ No newline at end of file diff --git a/tests/integration/test_query_deduplication/test.py b/tests/integration/test_query_deduplication/test.py deleted file mode 100644 index 1088b539414..00000000000 --- a/tests/integration/test_query_deduplication/test.py +++ /dev/null @@ -1,168 +0,0 @@ -import uuid - -import pytest - -from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV - -DUPLICATED_UUID = uuid.uuid4() - -cluster = ClickHouseCluster(__file__) - -node1 = cluster.add_instance( - 'node1', - main_configs=['configs/remote_servers.xml', 'configs/deduplication_settings.xml'], - user_configs=['configs/profiles.xml']) - -node2 = cluster.add_instance( - 'node2', - main_configs=['configs/remote_servers.xml', 'configs/deduplication_settings.xml'], - user_configs=['configs/profiles.xml']) - -node3 = cluster.add_instance( - 'node3', - main_configs=['configs/remote_servers.xml', 'configs/deduplication_settings.xml'], - user_configs=['configs/profiles.xml']) - - -@pytest.fixture(scope="module") -def started_cluster(): - try: - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -def prepare_node(node, parts_uuid=None): - node.query(""" - CREATE TABLE t(_prefix UInt8 DEFAULT 0, key UInt64, value UInt64) - ENGINE MergeTree() - ORDER BY tuple() - PARTITION BY _prefix - SETTINGS index_granularity = 1 - """) - - node.query(""" - CREATE TABLE d AS t ENGINE=Distributed(test_cluster, default, t) - """) - - # Stop merges while populating test data - node.query("SYSTEM STOP MERGES") - - # Create 5 parts - for i in range(1, 6): - node.query("INSERT INTO t VALUES ({}, {}, {})".format(i, i, i)) - - node.query("DETACH TABLE t") - - if parts_uuid: - for part, part_uuid in parts_uuid: - script = """ - echo -n '{}' > /var/lib/clickhouse/data/default/t/{}/uuid.txt - """.format(part_uuid, part) - node.exec_in_container(["bash", "-c", script]) - - # Attach table back - node.query("ATTACH TABLE t") - - # NOTE: - # due to absence of the ability to lock part, need to operate on parts with preventin merges - # node.query("SYSTEM START MERGES") - # node.query("OPTIMIZE TABLE t FINAL") - - print(node.name) - print(node.query("SELECT name, uuid, partition FROM system.parts WHERE table = 't' AND active ORDER BY name")) - - assert '5' == node.query("SELECT count() FROM system.parts WHERE table = 't' AND active").strip() - if parts_uuid: - for part, part_uuid in parts_uuid: - assert '1' == node.query( - "SELECT count() FROM system.parts WHERE table = 't' AND uuid = '{}' AND active".format( - part_uuid)).strip() - - -@pytest.fixture(scope="module") -def prepared_cluster(started_cluster): - print("duplicated UUID: {}".format(DUPLICATED_UUID)) - prepare_node(node1, parts_uuid=[("3_3_3_0", DUPLICATED_UUID)]) - prepare_node(node2, parts_uuid=[("3_3_3_0", DUPLICATED_UUID)]) - prepare_node(node3) - - -def test_virtual_column(prepared_cluster): - # Part containing `key=3` has the same fingerprint on both nodes, - # we expect it to be included only once in the end result.; - # select query is using virtucal column _part_fingerprint to filter out part in one shard - expected = """ - 1 2 - 2 2 - 3 1 - 4 2 - 5 2 - """ - assert TSV(expected) == TSV(node1.query(""" - SELECT - key, - count() AS c - FROM d - WHERE ((_shard_num = 1) AND (_part_uuid != '{}')) OR (_shard_num = 2) - GROUP BY key - ORDER BY - key ASC - """.format(DUPLICATED_UUID))) - - -def test_with_deduplication(prepared_cluster): - # Part containing `key=3` has the same fingerprint on both nodes, - # we expect it to be included only once in the end result - expected = """ -1 3 -2 3 -3 2 -4 3 -5 3 -""" - assert TSV(expected) == TSV(node1.query( - "SET allow_experimental_query_deduplication=1; SELECT key, count() c FROM d GROUP BY key ORDER BY key")) - - -def test_no_merge_with_deduplication(prepared_cluster): - # Part containing `key=3` has the same fingerprint on both nodes, - # we expect it to be included only once in the end result. - # even with distributed_group_by_no_merge=1 the duplicated part should be excluded from the final result - expected = """ -1 1 -2 1 -3 1 -4 1 -5 1 -1 1 -2 1 -3 1 -4 1 -5 1 -1 1 -2 1 -4 1 -5 1 -""" - assert TSV(expected) == TSV(node1.query("SELECT key, count() c FROM d GROUP BY key ORDER BY key", settings={ - "allow_experimental_query_deduplication": 1, - "distributed_group_by_no_merge": 1, - })) - - -def test_without_deduplication(prepared_cluster): - # Part containing `key=3` has the same fingerprint on both nodes, - # but allow_experimental_query_deduplication is disabled, - # so it will not be excluded - expected = """ -1 3 -2 3 -3 3 -4 3 -5 3 -""" - assert TSV(expected) == TSV(node1.query( - "SET allow_experimental_query_deduplication=0; SELECT key, count() c FROM d GROUP BY key ORDER BY key")) diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index 2211333bb26..779d1a69dcc 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -6,20 +6,29 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -zero = cluster.add_instance("zero", user_configs=["configs/users.d/settings.xml"], - main_configs=["configs/config.d/remote_servers.xml"], - macros={"cluster": "anime", "shard": "0", "replica": "zero"}, - with_zookeeper=True) +zero = cluster.add_instance( + "zero", + user_configs=["configs/users.d/settings.xml"], + main_configs=["configs/config.d/remote_servers.xml"], + macros={"cluster": "anime", "shard": "0", "replica": "zero"}, + with_zookeeper=True, +) -first = cluster.add_instance("first", user_configs=["configs/users.d/settings.xml"], - main_configs=["configs/config.d/remote_servers.xml"], - macros={"cluster": "anime", "shard": "0", "replica": "first"}, - with_zookeeper=True) +first = cluster.add_instance( + "first", + user_configs=["configs/users.d/settings.xml"], + main_configs=["configs/config.d/remote_servers.xml"], + macros={"cluster": "anime", "shard": "0", "replica": "first"}, + with_zookeeper=True, +) -second = cluster.add_instance("second", user_configs=["configs/users.d/settings.xml"], - main_configs=["configs/config.d/remote_servers.xml"], - macros={"cluster": "anime", "shard": "0", "replica": "second"}, - with_zookeeper=True) +second = cluster.add_instance( + "second", + user_configs=["configs/users.d/settings.xml"], + main_configs=["configs/config.d/remote_servers.xml"], + macros={"cluster": "anime", "shard": "0", "replica": "second"}, + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -36,45 +45,54 @@ def started_cluster(): def test_simple_add_replica(started_cluster): zero.query("DROP TABLE IF EXISTS test_simple ON CLUSTER cluster") - create_query = "CREATE TABLE test_simple " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ - "PARTITION BY d ORDER BY a" + create_query = ( + "CREATE TABLE test_simple " + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " + "PARTITION BY d ORDER BY a" + ) zero.query(create_query) first.query(create_query) first.query("SYSTEM STOP FETCHES test_simple") - zero.query("INSERT INTO test_simple VALUES (1, '2011-01-01')", settings={'insert_quorum': 1}) + zero.query( + "INSERT INTO test_simple VALUES (1, '2011-01-01')", + settings={"insert_quorum": 1}, + ) - assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") - assert '' == first.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == zero.query("SELECT * from test_simple") + assert "" == first.query("SELECT * from test_simple") first.query("SYSTEM START FETCHES test_simple") first.query("SYSTEM SYNC REPLICA test_simple", timeout=20) - assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") - assert '1\t2011-01-01\n' == first.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == zero.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == first.query("SELECT * from test_simple") second.query(create_query) second.query("SYSTEM SYNC REPLICA test_simple", timeout=20) - assert '1\t2011-01-01\n' == zero.query("SELECT * from test_simple") - assert '1\t2011-01-01\n' == first.query("SELECT * from test_simple") - assert '1\t2011-01-01\n' == second.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == zero.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == first.query("SELECT * from test_simple") + assert "1\t2011-01-01\n" == second.query("SELECT * from test_simple") zero.query("DROP TABLE IF EXISTS test_simple ON CLUSTER cluster") def test_drop_replica_and_achieve_quorum(started_cluster): - zero.query("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster" + ) - create_query = "CREATE TABLE test_drop_replica_and_achieve_quorum " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " \ - "PARTITION BY d ORDER BY a" + create_query = ( + "CREATE TABLE test_drop_replica_and_achieve_quorum " + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree('/clickhouse/tables/{shard}/{table}', '{replica}') " + "PARTITION BY d ORDER BY a" + ) print("Create Replicated table with two replicas") zero.query(create_query) @@ -86,14 +104,23 @@ def test_drop_replica_and_achieve_quorum(started_cluster): print("Insert to other replica. This query will fail.") quorum_timeout = zero.query_and_get_error( "INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (1, '2011-01-01')", - settings={'insert_quorum_timeout': 5000}) + settings={"insert_quorum_timeout": 5000}, + ) assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." - assert TSV("1\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum", - settings={'select_sequential_consistency': 0})) + assert TSV("1\t2011-01-01\n") == TSV( + zero.query( + "SELECT * FROM test_drop_replica_and_achieve_quorum", + settings={"select_sequential_consistency": 0}, + ) + ) - assert TSV("") == TSV(zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum", - settings={'select_sequential_consistency': 1})) + assert TSV("") == TSV( + zero.query( + "SELECT * FROM test_drop_replica_and_achieve_quorum", + settings={"select_sequential_consistency": 1}, + ) + ) # TODO:(Mikhaylov) begin; maybe delete this lines. I want clickhouse to fetch parts and update quorum. print("START FETCHES first replica") @@ -110,36 +137,45 @@ def test_drop_replica_and_achieve_quorum(started_cluster): second.query("SYSTEM SYNC REPLICA test_drop_replica_and_achieve_quorum", timeout=20) print("Quorum for previous insert achieved.") - assert TSV("1\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_drop_replica_and_achieve_quorum", - settings={'select_sequential_consistency': 1})) + assert TSV("1\t2011-01-01\n") == TSV( + second.query( + "SELECT * FROM test_drop_replica_and_achieve_quorum", + settings={"select_sequential_consistency": 1}, + ) + ) print("Now we can insert some other data.") - zero.query("INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (2, '2012-02-02')") + zero.query( + "INSERT INTO test_drop_replica_and_achieve_quorum(a,d) VALUES (2, '2012-02-02')" + ) assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( - zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + zero.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a") + ) assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( - first.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + first.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a") + ) assert TSV("1\t2011-01-01\n2\t2012-02-02\n") == TSV( - second.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a")) + second.query("SELECT * FROM test_drop_replica_and_achieve_quorum ORDER BY a") + ) - zero.query("DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_drop_replica_and_achieve_quorum ON CLUSTER cluster" + ) -@pytest.mark.parametrize( - ('add_new_data'), - [ - False, - True - ] -) +@pytest.mark.parametrize(("add_new_data"), [False, True]) def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): - zero.query("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster" + ) - create_query = "CREATE TABLE test_quorum_insert_with_drop_partition ON CLUSTER cluster " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree " \ - "PARTITION BY d ORDER BY a " + create_query = ( + "CREATE TABLE test_quorum_insert_with_drop_partition ON CLUSTER cluster " + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree " + "PARTITION BY d ORDER BY a " + ) print("Create Replicated table with three replicas") zero.query(create_query) @@ -148,14 +184,20 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): first.query("SYSTEM STOP FETCHES test_quorum_insert_with_drop_partition") print("Insert with quorum. (zero and second)") - zero.query("INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(1, '2011-01-01')") + zero.query( + "INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(1, '2011-01-01')" + ) print("Drop partition.") - zero.query("ALTER TABLE test_quorum_insert_with_drop_partition DROP PARTITION '2011-01-01'") + zero.query( + "ALTER TABLE test_quorum_insert_with_drop_partition DROP PARTITION '2011-01-01'" + ) - if (add_new_data): + if add_new_data: print("Insert to deleted partition") - zero.query("INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(2, '2011-01-01')") + zero.query( + "INSERT INTO test_quorum_insert_with_drop_partition(a,d) VALUES(2, '2011-01-01')" + ) print("Resume fetches for test_quorum_insert_with_drop_partition at first replica.") first.query("SYSTEM START FETCHES test_quorum_insert_with_drop_partition") @@ -163,43 +205,57 @@ def test_insert_quorum_with_drop_partition(started_cluster, add_new_data): print("Sync first replica with others.") first.query("SYSTEM SYNC REPLICA test_quorum_insert_with_drop_partition") - assert "20110101" not in first.query(""" + assert "20110101" not in first.query( + """ WITH (SELECT toString(uuid) FROM system.tables WHERE name = 'test_quorum_insert_with_drop_partition') AS uuid, '/clickhouse/tables/' || uuid || '/0/quorum/last_part' AS p SELECT * FROM system.zookeeper WHERE path = p FORMAT Vertical - """) + """ + ) print("Select from updated partition.") - if (add_new_data): - assert TSV("2\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")) - assert TSV("2\t2011-01-01\n") == TSV(second.query("SELECT * FROM test_quorum_insert_with_drop_partition")) + if add_new_data: + assert TSV("2\t2011-01-01\n") == TSV( + zero.query("SELECT * FROM test_quorum_insert_with_drop_partition") + ) + assert TSV("2\t2011-01-01\n") == TSV( + second.query("SELECT * FROM test_quorum_insert_with_drop_partition") + ) else: - assert TSV("") == TSV(zero.query("SELECT * FROM test_quorum_insert_with_drop_partition")) - assert TSV("") == TSV(second.query("SELECT * FROM test_quorum_insert_with_drop_partition")) + assert TSV("") == TSV( + zero.query("SELECT * FROM test_quorum_insert_with_drop_partition") + ) + assert TSV("") == TSV( + second.query("SELECT * FROM test_quorum_insert_with_drop_partition") + ) - zero.query("DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_quorum_insert_with_drop_partition ON CLUSTER cluster" + ) -@pytest.mark.parametrize( - ('add_new_data'), - [ - False, - True - ] -) +@pytest.mark.parametrize(("add_new_data"), [False, True]) def test_insert_quorum_with_move_partition(started_cluster, add_new_data): - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster") - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster" + ) + zero.query( + "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster" + ) - create_source = "CREATE TABLE test_insert_quorum_with_move_partition_source ON CLUSTER cluster " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree " \ - "PARTITION BY d ORDER BY a " + create_source = ( + "CREATE TABLE test_insert_quorum_with_move_partition_source ON CLUSTER cluster " + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree " + "PARTITION BY d ORDER BY a " + ) - create_destination = "CREATE TABLE test_insert_quorum_with_move_partition_destination ON CLUSTER cluster " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree " \ - "PARTITION BY d ORDER BY a " + create_destination = ( + "CREATE TABLE test_insert_quorum_with_move_partition_destination ON CLUSTER cluster " + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree " + "PARTITION BY d ORDER BY a " + ) print("Create source Replicated table with three replicas") zero.query(create_source) @@ -207,54 +263,78 @@ def test_insert_quorum_with_move_partition(started_cluster, add_new_data): print("Create destination Replicated table with three replicas") zero.query(create_destination) - print("Stop fetches for test_insert_quorum_with_move_partition_source at first replica.") + print( + "Stop fetches for test_insert_quorum_with_move_partition_source at first replica." + ) first.query("SYSTEM STOP FETCHES test_insert_quorum_with_move_partition_source") print("Insert with quorum. (zero and second)") - zero.query("INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(1, '2011-01-01')") + zero.query( + "INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(1, '2011-01-01')" + ) print("Drop partition.") zero.query( - "ALTER TABLE test_insert_quorum_with_move_partition_source MOVE PARTITION '2011-01-01' TO TABLE test_insert_quorum_with_move_partition_destination") + "ALTER TABLE test_insert_quorum_with_move_partition_source MOVE PARTITION '2011-01-01' TO TABLE test_insert_quorum_with_move_partition_destination" + ) - if (add_new_data): + if add_new_data: print("Insert to deleted partition") - zero.query("INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(2, '2011-01-01')") + zero.query( + "INSERT INTO test_insert_quorum_with_move_partition_source(a,d) VALUES(2, '2011-01-01')" + ) - print("Resume fetches for test_insert_quorum_with_move_partition_source at first replica.") + print( + "Resume fetches for test_insert_quorum_with_move_partition_source at first replica." + ) first.query("SYSTEM START FETCHES test_insert_quorum_with_move_partition_source") print("Sync first replica with others.") first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_move_partition_source") - assert "20110101" not in first.query(""" + assert "20110101" not in first.query( + """ WITH (SELECT toString(uuid) FROM system.tables WHERE name = 'test_insert_quorum_with_move_partition_source') AS uuid, '/clickhouse/tables/' || uuid || '/0/quorum/last_part' AS p SELECT * FROM system.zookeeper WHERE path = p FORMAT Vertical - """) + """ + ) print("Select from updated partition.") - if (add_new_data): - assert TSV("2\t2011-01-01\n") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + if add_new_data: assert TSV("2\t2011-01-01\n") == TSV( - second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source") + ) + assert TSV("2\t2011-01-01\n") == TSV( + second.query("SELECT * FROM test_insert_quorum_with_move_partition_source") + ) else: - assert TSV("") == TSV(zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) - assert TSV("") == TSV(second.query("SELECT * FROM test_insert_quorum_with_move_partition_source")) + assert TSV("") == TSV( + zero.query("SELECT * FROM test_insert_quorum_with_move_partition_source") + ) + assert TSV("") == TSV( + second.query("SELECT * FROM test_insert_quorum_with_move_partition_source") + ) - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster") - zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster") + zero.query( + "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_source ON CLUSTER cluster" + ) + zero.query( + "DROP TABLE IF EXISTS test_insert_quorum_with_move_partition_destination ON CLUSTER cluster" + ) def test_insert_quorum_with_ttl(started_cluster): zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") - create_query = "CREATE TABLE test_insert_quorum_with_ttl " \ - "(a Int8, d Date) " \ - "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " \ - "PARTITION BY d ORDER BY a " \ - "TTL d + INTERVAL 5 second DELETE WHERE toYear(d) = 2011 " \ - "SETTINGS merge_with_ttl_timeout=2 " + create_query = ( + "CREATE TABLE test_insert_quorum_with_ttl " + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " + "PARTITION BY d ORDER BY a " + "TTL d + INTERVAL 5 second DELETE WHERE toYear(d) = 2011 " + "SETTINGS merge_with_ttl_timeout=2 " + ) print("Create Replicated table with two replicas") zero.query(create_query) @@ -264,14 +344,22 @@ def test_insert_quorum_with_ttl(started_cluster): first.query("SYSTEM STOP FETCHES test_insert_quorum_with_ttl") print("Insert should fail since it can not reach the quorum.") - quorum_timeout = zero.query_and_get_error("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", - settings={'insert_quorum_timeout': 5000}) + quorum_timeout = zero.query_and_get_error( + "INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", + settings={"insert_quorum_timeout": 5000}, + ) assert "Timeout while waiting for quorum" in quorum_timeout, "Query must fail." - print("Wait 10 seconds and TTL merge have to be executed. But it won't delete data.") + print( + "Wait 10 seconds and TTL merge have to be executed. But it won't delete data." + ) time.sleep(10) assert TSV("1\t2011-01-01\n") == TSV( - zero.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 0})) + zero.query( + "SELECT * FROM test_insert_quorum_with_ttl", + settings={"select_sequential_consistency": 0}, + ) + ) print("Resume fetches for test_insert_quorum_with_ttl at first replica.") first.query("SYSTEM START FETCHES test_insert_quorum_with_ttl") @@ -279,8 +367,10 @@ def test_insert_quorum_with_ttl(started_cluster): print("Sync first replica.") first.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") - zero.query("INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", - settings={'insert_quorum_timeout': 5000}) + zero.query( + "INSERT INTO test_insert_quorum_with_ttl(a,d) VALUES(1, '2011-01-01')", + settings={"insert_quorum_timeout": 5000}, + ) print("Inserts should resume.") zero.query("INSERT INTO test_insert_quorum_with_ttl(a, d) VALUES(2, '2012-02-02')") @@ -290,8 +380,16 @@ def test_insert_quorum_with_ttl(started_cluster): zero.query("SYSTEM SYNC REPLICA test_insert_quorum_with_ttl") assert TSV("2\t2012-02-02\n") == TSV( - first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 0})) + first.query( + "SELECT * FROM test_insert_quorum_with_ttl", + settings={"select_sequential_consistency": 0}, + ) + ) assert TSV("2\t2012-02-02\n") == TSV( - first.query("SELECT * FROM test_insert_quorum_with_ttl", settings={'select_sequential_consistency': 1})) + first.query( + "SELECT * FROM test_insert_quorum_with_ttl", + settings={"select_sequential_consistency": 1}, + ) + ) zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") diff --git a/tests/integration/test_quorum_inserts_parallel/test.py b/tests/integration/test_quorum_inserts_parallel/test.py index c89f1a03df7..99548e37a54 100644 --- a/tests/integration/test_quorum_inserts_parallel/test.py +++ b/tests/integration/test_quorum_inserts_parallel/test.py @@ -14,6 +14,7 @@ node1 = cluster.add_instance("node1", with_zookeeper=True) node2 = cluster.add_instance("node2", with_zookeeper=True) node3 = cluster.add_instance("node3", with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): global cluster @@ -28,12 +29,19 @@ def started_cluster(): def test_parallel_quorum_actually_parallel(started_cluster): settings = {"insert_quorum": "3", "insert_quorum_parallel": "1"} for i, node in enumerate([node1, node2, node3]): - node.query("CREATE TABLE r (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/r', '{num}') ORDER BY tuple()".format(num=i)) + node.query( + "CREATE TABLE r (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/r', '{num}') ORDER BY tuple()".format( + num=i + ) + ) p = Pool(10) def long_insert(node): - node.query("INSERT INTO r SELECT number, toString(number) FROM numbers(5) where sleepEachRow(1) == 0", settings=settings) + node.query( + "INSERT INTO r SELECT number, toString(number) FROM numbers(5) where sleepEachRow(1) == 0", + settings=settings, + ) job = p.apply_async(long_insert, (node1,)) @@ -58,19 +66,37 @@ def test_parallel_quorum_actually_parallel(started_cluster): def test_parallel_quorum_actually_quorum(started_cluster): for i, node in enumerate([node1, node2, node3]): - node.query("CREATE TABLE q (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/q', '{num}') ORDER BY tuple()".format(num=i)) + node.query( + "CREATE TABLE q (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/q', '{num}') ORDER BY tuple()".format( + num=i + ) + ) with PartitionManager() as pm: pm.partition_instances(node2, node1, port=9009) pm.partition_instances(node2, node3, port=9009) with pytest.raises(QueryRuntimeException): - node1.query("INSERT INTO q VALUES(1, 'Hello')", settings={"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "3000"}) + node1.query( + "INSERT INTO q VALUES(1, 'Hello')", + settings={ + "insert_quorum": "3", + "insert_quorum_parallel": "1", + "insert_quorum_timeout": "3000", + }, + ) assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "1") assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "0") assert_eq_with_retry(node3, "SELECT COUNT() FROM q", "1") - node1.query("INSERT INTO q VALUES(2, 'wlrd')", settings={"insert_quorum": "2", "insert_quorum_parallel": "1", "insert_quorum_timeout": "3000"}) + node1.query( + "INSERT INTO q VALUES(2, 'wlrd')", + settings={ + "insert_quorum": "2", + "insert_quorum_parallel": "1", + "insert_quorum_timeout": "3000", + }, + ) assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "2") assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "0") @@ -80,14 +106,38 @@ def test_parallel_quorum_actually_quorum(started_cluster): node.query("INSERT INTO q VALUES(3, 'Hi')", settings=settings) p = Pool(2) - res = p.apply_async(insert_value_to_node, (node1, {"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "60000"})) + res = p.apply_async( + insert_value_to_node, + ( + node1, + { + "insert_quorum": "3", + "insert_quorum_parallel": "1", + "insert_quorum_timeout": "60000", + }, + ), + ) - assert_eq_with_retry(node1, "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", "3") - assert_eq_with_retry(node3, "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", "3") - assert_eq_with_retry(node2, "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", "0") + assert_eq_with_retry( + node1, + "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", + "3", + ) + assert_eq_with_retry( + node3, + "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", + "3", + ) + assert_eq_with_retry( + node2, + "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", + "0", + ) # Insert to the second to satisfy quorum - insert_value_to_node(node2, {"insert_quorum": "3", "insert_quorum_parallel": "1"}) + insert_value_to_node( + node2, {"insert_quorum": "3", "insert_quorum_parallel": "1"} + ) res.get() diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index 83ee32bd7dd..651726f30c0 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -7,10 +7,15 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', user_configs=["configs/users.d/assign_myquota_to_default_user.xml", - "configs/users.d/drop_default_quota.xml", - "configs/users.d/myquota.xml", - "configs/users.d/user_with_no_quota.xml"]) +instance = cluster.add_instance( + "instance", + user_configs=[ + "configs/users.d/assign_myquota_to_default_user.xml", + "configs/users.d/drop_default_quota.xml", + "configs/users.d/myquota.xml", + "configs/users.d/user_with_no_quota.xml", + ], +) def check_system_quotas(canonical): @@ -22,16 +27,22 @@ def check_system_quotas(canonical): def system_quota_limits(canonical): canonical_tsv = TSV(canonical) - r = TSV(instance.query("SELECT * FROM system.quota_limits ORDER BY quota_name, duration")) + r = TSV( + instance.query( + "SELECT * FROM system.quota_limits ORDER BY quota_name, duration" + ) + ) print(("system_quota_limits: {},\ncanonical: {}".format(r, TSV(canonical_tsv)))) assert r == canonical_tsv def system_quota_usage(canonical): canonical_tsv = TSV(canonical) - query = "SELECT quota_name, quota_key, duration, queries, max_queries, query_selects, max_query_selects, query_inserts, max_query_inserts, errors, max_errors, result_rows, max_result_rows," \ - "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time " \ - "FROM system.quota_usage ORDER BY duration" + query = ( + "SELECT quota_name, quota_key, duration, queries, max_queries, query_selects, max_query_selects, query_inserts, max_query_inserts, errors, max_errors, result_rows, max_result_rows," + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time " + "FROM system.quota_usage ORDER BY duration" + ) r = TSV(instance.query(query)) print(("system_quota_usage: {},\ncanonical: {}".format(r, TSV(canonical_tsv)))) assert r == canonical_tsv @@ -39,9 +50,11 @@ def system_quota_usage(canonical): def system_quotas_usage(canonical): canonical_tsv = TSV(canonical) - query = "SELECT quota_name, quota_key, is_current, duration, queries, max_queries, query_selects, max_query_selects, query_inserts, max_query_inserts, errors, max_errors, result_rows, max_result_rows, " \ - "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time " \ - "FROM system.quotas_usage ORDER BY quota_name, quota_key, duration" + query = ( + "SELECT quota_name, quota_key, is_current, duration, queries, max_queries, query_selects, max_query_selects, query_inserts, max_query_inserts, errors, max_errors, result_rows, max_result_rows, " + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time " + "FROM system.quotas_usage ORDER BY quota_name, quota_key, duration" + ) r = TSV(instance.query(query)) print(("system_quotas_usage: {},\ncanonical: {}".format(r, TSV(canonical_tsv)))) assert r == canonical_tsv @@ -49,12 +62,14 @@ def system_quotas_usage(canonical): def copy_quota_xml(local_file_name, reload_immediately=True): script_dir = os.path.dirname(os.path.realpath(__file__)) - instance.copy_file_to_container(os.path.join(script_dir, local_file_name), - '/etc/clickhouse-server/users.d/myquota.xml') + instance.copy_file_to_container( + os.path.join(script_dir, local_file_name), + "/etc/clickhouse-server/users.d/myquota.xml", + ) if reload_immediately: - # We use the special user 'user_with_no_quota' here because - # we don't want SYSTEM RELOAD CONFIG to mess our quota consuming checks. - instance.query("SYSTEM RELOAD CONFIG", user='user_with_no_quota') + # We use the special user 'user_with_no_quota' here because + # we don't want SYSTEM RELOAD CONFIG to mess our quota consuming checks. + instance.query("SYSTEM RELOAD CONFIG", user="user_with_no_quota") @pytest.fixture(scope="module", autouse=True) @@ -62,7 +77,9 @@ def started_cluster(): try: cluster.start() instance.query("DROP TABLE IF EXISTS test_table") - instance.query("CREATE TABLE test_table(x UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE test_table(x UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) instance.query("INSERT INTO test_table SELECT number FROM numbers(50)") yield cluster @@ -75,320 +92,1592 @@ def started_cluster(): def reset_quotas_and_usage_info(): try: instance.query("DROP QUOTA IF EXISTS qA, qB") - copy_quota_xml('simpliest.xml') # To reset usage info. - copy_quota_xml('normal_limits.xml') + copy_quota_xml("simpliest.xml") # To reset usage info. + copy_quota_xml("normal_limits.xml") yield finally: pass def test_quota_from_users_xml(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + [31556952], + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) system_quotas_usage( - [["myQuota", "default", 1, 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 1, + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 1, 1000, 1, 500, 0, 500, 0, "\\N", 50, "\\N", 200, "\\N", 50, 1000, 200, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 1, + 1000, + 1, + 500, + 0, + 500, + 0, + "\\N", + 50, + "\\N", + 200, + "\\N", + 50, + 1000, + 200, + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT SUM(x) from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 2, 1000, 2, 500, 0, 500, 0, "\\N", 51, "\\N", 208, "\\N", 100, 1000, 400, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 2, + 1000, + 2, + 500, + 0, + 500, + 0, + "\\N", + 51, + "\\N", + 208, + "\\N", + 100, + 1000, + 400, + "\\N", + "\\N", + ] + ] + ) def test_simpliest_quota(): # Simpliest quota doesn't even track usage. - copy_quota_xml('simpliest.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, - "['default']", "[]"]]) + copy_quota_xml("simpliest.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[]", + 0, + "['default']", + "[]", + ] + ] + ) system_quota_limits("") system_quota_usage( - [["myQuota", "default", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ] + ] + ) def test_tracking_quota(): # Now we're tracking usage. - copy_quota_xml('tracking.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", "\\N"]]) + copy_quota_xml("tracking.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ] + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 1, "\\N", 1, "\\N", 0, "\\N", 0, "\\N", 50, "\\N", 200, "\\N", 50, "\\N", 200, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 1, + "\\N", + 1, + "\\N", + 0, + "\\N", + 0, + "\\N", + 50, + "\\N", + 200, + "\\N", + 50, + "\\N", + 200, + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT SUM(x) from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 2, "\\N", 2, "\\N", 0, "\\N", 0, "\\N", 51, "\\N", 208, "\\N", 100, "\\N", 400, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 2, + "\\N", + 2, + "\\N", + 0, + "\\N", + 0, + "\\N", + 51, + "\\N", + 208, + "\\N", + 100, + "\\N", + 400, + "\\N", + "\\N", + ] + ] + ) def test_exceed_quota(): # Change quota, now the limits are tiny so we will exceed the quota. - copy_quota_xml('tiny_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1, 1, 1, 1, 1, "\\N", 1, "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, "\\N", 0, 1, 0, "\\N", "\\N"]]) + copy_quota_xml("tiny_limits.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [["myQuota", 31556952, 0, 1, 1, 1, 1, 1, "\\N", 1, "\\N", "\\N"]] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + "\\N", + 0, + 1, + 0, + "\\N", + "\\N", + ] + ] + ) - assert re.search("Quota.*has\ been\ exceeded", instance.query_and_get_error("SELECT * from test_table")) - system_quota_usage([["myQuota", "default", 31556952, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, "\\N", 50, 1, 0, "\\N", "\\N"]]) + assert re.search( + "Quota.*has\ been\ exceeded", + instance.query_and_get_error("SELECT * from test_table"), + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 0, + 1, + 0, + "\\N", + 50, + 1, + 0, + "\\N", + "\\N", + ] + ] + ) # Change quota, now the limits are enough to execute queries. - copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 1, 1000, 1, 500, 0, 500, 1, "\\N", 0, "\\N", 0, "\\N", 50, 1000, 0, "\\N", "\\N"]]) + copy_quota_xml("normal_limits.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 1, + 1000, + 1, + 500, + 0, + 500, + 1, + "\\N", + 0, + "\\N", + 0, + "\\N", + 50, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 2, 1000, 2, 500, 0, 500, 1, "\\N", 50, "\\N", 200, "\\N", 100, 1000, 200, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 2, + 1000, + 2, + 500, + 0, + 500, + 1, + "\\N", + 50, + "\\N", + 200, + "\\N", + 100, + 1000, + 200, + "\\N", + "\\N", + ] + ] + ) def test_add_remove_interval(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + [31556952], + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) # Add interval. - copy_quota_xml('two_intervals.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", - "[31556952,63113904]", 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, "\\N", "\\N", "\\N", "\\N", "\\N", 1000, "\\N", "\\N"], - ["myQuota", 63113904, 1, "\\N", "\\N", "\\N", "\\N", "\\N", 30000, "\\N", 20000, 120]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"], - ["myQuota", "default", 63113904, 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, 30000, 0, "\\N", 0, 20000, 120]]) + copy_quota_xml("two_intervals.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952,63113904]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ], + [ + "myQuota", + 63113904, + 1, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + 30000, + "\\N", + 20000, + 120, + ], + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + 1000, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ], + [ + "myQuota", + "default", + 63113904, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 30000, + 0, + "\\N", + 0, + 20000, + 120, + ], + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 1, 1000, 1, "\\N", 0, "\\N", 0, "\\N", 50, "\\N", 200, "\\N", 50, 1000, 200, "\\N", "\\N"], - ["myQuota", "default", 63113904, 1, "\\N", 1, "\\N", 0, "\\N", 0, "\\N", 50, "\\N", 200, 30000, 50, "\\N", 200, 20000, 120]]) + [ + [ + "myQuota", + "default", + 31556952, + 1, + 1000, + 1, + "\\N", + 0, + "\\N", + 0, + "\\N", + 50, + "\\N", + 200, + "\\N", + 50, + 1000, + 200, + "\\N", + "\\N", + ], + [ + "myQuota", + "default", + 63113904, + 1, + "\\N", + 1, + "\\N", + 0, + "\\N", + 0, + "\\N", + 50, + "\\N", + 200, + 30000, + 50, + "\\N", + 200, + 20000, + 120, + ], + ] + ) # Remove interval. - copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) + copy_quota_xml("normal_limits.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + [31556952], + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) system_quota_usage( - [["myQuota", "default", 31556952, 1, 1000, 1, 500, 0, 500, 0, "\\N", 50, "\\N", 200, "\\N", 50, 1000, 200, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 1, + 1000, + 1, + 500, + 0, + 500, + 0, + "\\N", + 50, + "\\N", + 200, + "\\N", + 50, + 1000, + 200, + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 2, 1000, 2, 500, 0, 500, 0, "\\N", 100, "\\N", 400, "\\N", 100, 1000, 400, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 2, + 1000, + 2, + 500, + 0, + 500, + 0, + "\\N", + 100, + "\\N", + 400, + "\\N", + 100, + 1000, + 400, + "\\N", + "\\N", + ] + ] + ) # Remove all intervals. - copy_quota_xml('simpliest.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[]", 0, - "['default']", "[]"]]) + copy_quota_xml("simpliest.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[]", + 0, + "['default']", + "[]", + ] + ] + ) system_quota_limits("") system_quota_usage( - [["myQuota", "default", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ] + ] + ) instance.query("SELECT * from test_table") system_quota_usage( - [["myQuota", "default", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + ] + ] + ) # Add one interval back. - copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + copy_quota_xml("normal_limits.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + [31556952], + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) def test_add_remove_quota(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + [31556952], + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) system_quotas_usage( - [["myQuota", "default", 1, 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 1, + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) # Add quota. - copy_quota_xml('two_quotas.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"], - ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "['client_key','user_name']", - "[3600,2629746]", 0, "[]", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, "\\N", "\\N", "\\N", "\\N", "\\N", 1000, "\\N", "\\N"], - ["myQuota2", 3600, 1, "\\N", "\\N", "\\N", "\\N", 4000, 400000, 4000, 400000, 60], - ["myQuota2", 2629746, 0, "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", "\\N", 1800]]) + copy_quota_xml("two_quotas.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ], + [ + "myQuota2", + "4590510c-4d13-bf21-ec8a-c2187b092e73", + "users.xml", + "['client_key','user_name']", + "[3600,2629746]", + 0, + "[]", + "[]", + ], + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ], + [ + "myQuota2", + 3600, + 1, + "\\N", + "\\N", + "\\N", + "\\N", + 4000, + 400000, + 4000, + 400000, + 60, + ], + [ + "myQuota2", + 2629746, + 0, + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + "\\N", + 1800, + ], + ] + ) system_quotas_usage( - [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 1, + 31556952, + 0, + 1000, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) # Drop quota. - copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) + copy_quota_xml("normal_limits.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) system_quotas_usage( - [["myQuota", "default", 1, 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 1, + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) # Drop all quotas. - copy_quota_xml('no_quotas.xml') + copy_quota_xml("no_quotas.xml") check_system_quotas("") system_quota_limits("") system_quotas_usage("") # Add one quota back. - copy_quota_xml('normal_limits.xml') - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) + copy_quota_xml("normal_limits.xml") + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) system_quotas_usage( - [["myQuota", "default", 1, 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 1, + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) def test_reload_users_xml_by_timer(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", "[31556952]", - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + "[31556952]", + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) time.sleep(1) # The modification time of the 'quota.xml' file should be different, # because config files are reload by timer only when the modification time is changed. - copy_quota_xml('tiny_limits.xml', reload_immediately=False) - assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [ - ["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", ['user_name'], "[31556952]", 0, "['default']", - "[]"]]) - assert_eq_with_retry(instance, "SELECT * FROM system.quota_limits", - [["myQuota", 31556952, 0, 1, 1, 1, 1, 1, "\\N", 1, "\\N", "\\N"]]) + copy_quota_xml("tiny_limits.xml", reload_immediately=False) + assert_eq_with_retry( + instance, + "SELECT * FROM system.quotas", + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + ["user_name"], + "[31556952]", + 0, + "['default']", + "[]", + ] + ], + ) + assert_eq_with_retry( + instance, + "SELECT * FROM system.quota_limits", + [["myQuota", 31556952, 0, 1, 1, 1, 1, 1, "\\N", 1, "\\N", "\\N"]], + ) def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query( - "SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, query_selects = 500, query_inserts = 500, read_rows = 1000 TO default\n" - assert instance.query( - "SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, query_selects = 500, query_inserts = 500, read_rows = 1000 TO default\n" + assert ( + instance.query("SHOW CREATE QUOTA") + == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, query_selects = 500, query_inserts = 500, read_rows = 1000 TO default\n" + ) + assert ( + instance.query("SHOW CREATE QUOTAS") + == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, query_selects = 500, query_inserts = 500, read_rows = 1000 TO default\n" + ) assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t500\\t0\\t500\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("SELECT * from test_table") assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t500\\t0\\t500\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) expected_access = "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, query_selects = 500, query_inserts = 500, read_rows = 1000 TO default\n" assert expected_access in instance.query("SHOW ACCESS") # Add interval. - copy_quota_xml('two_intervals.xml') + copy_quota_xml("two_intervals.xml") assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query( - "SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000, FOR RANDOMIZED INTERVAL 2 year MAX result_bytes = 30000, read_bytes = 20000, execution_time = 120 TO default\n" + assert ( + instance.query("SHOW CREATE QUOTA") + == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000, FOR RANDOMIZED INTERVAL 2 year MAX result_bytes = 30000, read_bytes = 20000, execution_time = 120 TO default\n" + ) assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n" "myQuota\\tdefault\\t.*\\t63113904\\t0\\t\\\\N\t0\\t\\\\N\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t30000\\t0\\t\\\\N\\t0\\t20000\\t.*\\t120", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) # Drop interval, add quota. - copy_quota_xml('two_quotas.xml') + copy_quota_xml("two_quotas.xml") assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" - assert instance.query( - "SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" - assert instance.query( - "SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" - assert instance.query( - "SHOW CREATE QUOTAS") == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" \ - "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" + assert ( + instance.query("SHOW CREATE QUOTA myQuota") + == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + ) + assert ( + instance.query("SHOW CREATE QUOTA myQuota2") + == "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" + ) + assert ( + instance.query("SHOW CREATE QUOTAS") + == "CREATE QUOTA myQuota KEYED BY user_name FOR INTERVAL 1 year MAX queries = 1000, read_rows = 1000 TO default\n" + "CREATE QUOTA myQuota2 KEYED BY client_key, user_name FOR RANDOMIZED INTERVAL 1 hour MAX result_rows = 4000, result_bytes = 400000, read_rows = 4000, read_bytes = 400000, execution_time = 60, FOR INTERVAL 1 month MAX execution_time = 1800\n" + ) assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) # Drop all quotas. - copy_quota_xml('no_quotas.xml') + copy_quota_xml("no_quotas.xml") assert instance.query("SHOW QUOTAS") == "" assert instance.query("SHOW CREATE QUOTA") == "" assert instance.query("SHOW QUOTA") == "" def test_dcl_management(): - copy_quota_xml('no_quotas.xml') + copy_quota_xml("no_quotas.xml") assert instance.query("SHOW QUOTA") == "" - instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER") - assert instance.query( - "SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 5 quarter MAX queries = 123 TO default\n" + instance.query( + "CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER" + ) + assert ( + instance.query("SHOW CREATE QUOTA qA") + == "CREATE QUOTA qA FOR INTERVAL 5 quarter MAX queries = 123 TO default\n" + ) assert re.match( "qA\\t\\t.*\\t39446190\\t0\\t123\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("SELECT * from test_table") assert re.match( "qA\\t\\t.*\\t39446190\\t1\\t123\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query( - "ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") - assert instance.query( - "SHOW CREATE QUOTA qA") == "CREATE QUOTA qA FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default\n" + "ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5" + ) + assert ( + instance.query("SHOW CREATE QUOTA qA") + == "CREATE QUOTA qA FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default\n" + ) assert re.match( "qA\\t\\t.*\\t1800\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t0.5\n" "qA\\t\\t.*\\t39446190\\t1\\t321\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t10\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("SELECT * from test_table") assert re.match( "qA\\t\\t.*\\t1800\\t1\\t\\\\N\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t0.5\n" "qA\\t\\t.*\\t39446190\\t2\\t321\\t2\\t\\\\N\\t0\\t\\\\N\\t0\\t10\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query( - "ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") + "ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS" + ) assert re.match( "qA\\t\\t.*\\t42075936\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("SELECT * from test_table") assert re.match( "qA\\t\\t.*\\t42075936\\t1\\t\\\\N\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("ALTER QUOTA qA RENAME TO qB") - assert instance.query( - "SHOW CREATE QUOTA qB") == "CREATE QUOTA qB FOR RANDOMIZED INTERVAL 16 month TRACKING ONLY TO default\n" + assert ( + instance.query("SHOW CREATE QUOTA qB") + == "CREATE QUOTA qB FOR RANDOMIZED INTERVAL 16 month TRACKING ONLY TO default\n" + ) assert re.match( "qB\\t\\t.*\\t42075936\\t1\\t\\\\N\\t1\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("SELECT * from test_table") assert re.match( "qB\\t\\t.*\\t42075936\\t2\\t\\\\N\\t2\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) instance.query("DROP QUOTA qB") assert instance.query("SHOW QUOTA") == "" def test_users_xml_is_readonly(): - assert re.search("storage is readonly", instance.query_and_get_error("DROP QUOTA myQuota")) + assert re.search( + "storage is readonly", instance.query_and_get_error("DROP QUOTA myQuota") + ) def test_query_inserts(): - check_system_quotas([["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "['user_name']", [31556952], - 0, "['default']", "[]"]]) - system_quota_limits([["myQuota", 31556952, 0, 1000, 500, 500, "\\N", "\\N", "\\N", 1000, "\\N", "\\N"]]) - system_quota_usage([["myQuota", "default", 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + check_system_quotas( + [ + [ + "myQuota", + "e651da9c-a748-8703-061a-7e5e5096dae7", + "users.xml", + "['user_name']", + [31556952], + 0, + "['default']", + "[]", + ] + ] + ) + system_quota_limits( + [ + [ + "myQuota", + 31556952, + 0, + 1000, + 500, + 500, + "\\N", + "\\N", + "\\N", + 1000, + "\\N", + "\\N", + ] + ] + ) + system_quota_usage( + [ + [ + "myQuota", + "default", + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) system_quotas_usage( - [["myQuota", "default", 1, 31556952, 0, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 1, + 31556952, + 0, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) instance.query("DROP TABLE IF EXISTS test_table_ins") - instance.query("CREATE TABLE test_table_ins(x UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE test_table_ins(x UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) system_quota_usage( - [["myQuota", "default", 31556952, 2, 1000, 0, 500, 0, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) - + [ + [ + "myQuota", + "default", + 31556952, + 2, + 1000, + 0, + 500, + 0, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) + instance.query("INSERT INTO test_table_ins values(1)") system_quota_usage( - [["myQuota", "default", 31556952, 3, 1000, 0, 500, 1, 500, 0, "\\N", 0, "\\N", 0, "\\N", 0, 1000, 0, "\\N", "\\N"]]) + [ + [ + "myQuota", + "default", + 31556952, + 3, + 1000, + 0, + 500, + 1, + 500, + 0, + "\\N", + 0, + "\\N", + 0, + "\\N", + 0, + 1000, + 0, + "\\N", + "\\N", + ] + ] + ) instance.query("DROP TABLE test_table_ins") @@ -396,28 +1685,40 @@ def test_consumption_of_show_tables(): assert instance.query("SHOW TABLES") == "test_table\n" assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t500\\t0\\t500\\t0\\t\\\\N\\t1\\t\\\\N.*", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) + def test_consumption_of_show_databases(): - assert instance.query("SHOW DATABASES") == "INFORMATION_SCHEMA\ndefault\ninformation_schema\nsystem\n" + assert ( + instance.query("SHOW DATABASES") + == "INFORMATION_SCHEMA\ndefault\ninformation_schema\nsystem\n" + ) assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t500\\t0\\t500\\t0\\t\\\\N\\t4\\t\\\\N.*", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) + def test_consumption_of_show_clusters(): assert len(instance.query("SHOW CLUSTERS")) > 0 assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t500\\t0\\t500\\t0\\t\\\\N.*", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) + def test_consumption_of_show_processlist(): instance.query("SHOW PROCESSLIST") assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t500\\t0\\t500\\t0\\t\\\\N\\t0\\t\\\\N.*", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) + def test_consumption_of_show_privileges(): assert len(instance.query("SHOW PRIVILEGES")) > 0 assert re.match( "myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t1\\t500\\t0\\t500\\t0\\t\\\\N.*", - instance.query("SHOW QUOTA")) + instance.query("SHOW QUOTA"), + ) diff --git a/tests/integration/test_random_inserts/test.py b/tests/integration/test_random_inserts/test.py index a06649dba52..4d6aaa9276d 100644 --- a/tests/integration/test_random_inserts/test.py +++ b/tests/integration/test_random_inserts/test.py @@ -11,12 +11,18 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], - with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 1}) -node2 = cluster.add_instance('node2', - main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], - with_zookeeper=True, macros={"layer": 0, "shard": 0, "replica": 2}) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, + macros={"layer": 0, "shard": 0, "replica": 1}, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/conf.d/merge_tree.xml", "configs/conf.d/remote_servers.xml"], + with_zookeeper=True, + macros={"layer": 0, "shard": 0, "replica": 2}, +) nodes = [node1, node2] @@ -35,9 +41,11 @@ def test_random_inserts(started_cluster): # Duration of the test, reduce it if don't want to wait DURATION_SECONDS = 10 # * 60 - node1.query(""" + node1.query( + """ CREATE TABLE simple ON CLUSTER test_cluster (date Date, i UInt32, s String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, i, 8192)""") + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/simple', '{replica}', date, i, 8192)""" + ) with PartitionManager() as pm_random_drops: for sacrifice in nodes: @@ -52,21 +60,38 @@ def test_random_inserts(started_cluster): bash_script = os.path.join(os.path.dirname(__file__), "test.sh") inserters = [] for node in nodes: - cmd = ['/bin/bash', bash_script, node.ip_address, str(min_timestamp), str(max_timestamp), - str(cluster.get_client_cmd())] - inserters.append(CommandRequest(cmd, timeout=DURATION_SECONDS * 2, stdin='')) + cmd = [ + "/bin/bash", + bash_script, + node.ip_address, + str(min_timestamp), + str(max_timestamp), + str(cluster.get_client_cmd()), + ] + inserters.append( + CommandRequest(cmd, timeout=DURATION_SECONDS * 2, stdin="") + ) print(node.name, node.ip_address) for inserter in inserters: inserter.get_answer() - answer = "{}\t{}\t{}\t{}\n".format(num_timestamps, num_timestamps, min_timestamp, max_timestamp) + answer = "{}\t{}\t{}\t{}\n".format( + num_timestamps, num_timestamps, min_timestamp, max_timestamp + ) for node in nodes: - res = node.query_with_retry("SELECT count(), uniqExact(i), min(i), max(i) FROM simple", - check_callback=lambda res: TSV(res) == TSV(answer)) - assert TSV(res) == TSV(answer), node.name + " : " + node.query( - "SELECT groupArray(_part), i, count() AS c FROM simple GROUP BY i ORDER BY c DESC LIMIT 1") + res = node.query_with_retry( + "SELECT count(), uniqExact(i), min(i), max(i) FROM simple", + check_callback=lambda res: TSV(res) == TSV(answer), + ) + assert TSV(res) == TSV(answer), ( + node.name + + " : " + + node.query( + "SELECT groupArray(_part), i, count() AS c FROM simple GROUP BY i ORDER BY c DESC LIMIT 1" + ) + ) node1.query("""DROP TABLE simple ON CLUSTER test_cluster""") @@ -84,14 +109,16 @@ class Runner: self.stop_ev.wait(random.random()) year = 2000 - month = '01' + month = "01" day = str(thread_num + 1).zfill(2) x = 1 while not self.stop_ev.is_set(): payload = """ {year}-{month}-{day} {x1} {year}-{month}-{day} {x2} -""".format(year=year, month=month, day=day, x1=x, x2=(x + 1)).strip() +""".format( + year=year, month=month, day=day, x1=x, x2=(x + 1) + ).strip() try: random.choice(nodes).query("INSERT INTO repl_test FORMAT TSV", payload) @@ -106,7 +133,7 @@ class Runner: self.mtx.release() except Exception as e: - print('Exception:', e) + print("Exception:", e) x += 2 self.stop_ev.wait(0.1 + random.random() / 10) @@ -120,7 +147,8 @@ def test_insert_multithreaded(started_cluster): for node in nodes: node.query( - "CREATE TABLE repl_test(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/repl_test', '{replica}') ORDER BY x PARTITION BY toYYYYMM(d)") + "CREATE TABLE repl_test(d Date, x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/repl_test', '{replica}') ORDER BY x PARTITION BY toYYYYMM(d)" + ) runner = Runner() @@ -145,7 +173,11 @@ def test_insert_multithreaded(started_cluster): time.sleep(0.5) def get_delay(node): - return int(node.query("SELECT absolute_delay FROM system.replicas WHERE table = 'repl_test'").rstrip()) + return int( + node.query( + "SELECT absolute_delay FROM system.replicas WHERE table = 'repl_test'" + ).rstrip() + ) if all([get_delay(n) == 0 for n in nodes]): all_replicated = True diff --git a/tests/integration/test_range_hashed_dictionary_types/test.py b/tests/integration/test_range_hashed_dictionary_types/test.py index 198e2e27db8..91b0184c791 100644 --- a/tests/integration/test_range_hashed_dictionary_types/test.py +++ b/tests/integration/test_range_hashed_dictionary_types/test.py @@ -4,7 +4,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1') +node1 = cluster.add_instance("node1") @pytest.fixture(scope="module") @@ -19,7 +19,8 @@ def started_cluster(): def test_range_hashed_dict(started_cluster): script = "echo '4990954156238030839\t2018-12-31 21:00:00\t2020-12-30 20:59:59\t0.1\tRU' > /var/lib/clickhouse/user_files/rates.tsv" node1.exec_in_container(["bash", "-c", script]) - node1.query(""" + node1.query( + """ CREATE DICTIONARY rates ( hash_id UInt64, @@ -36,8 +37,13 @@ def test_range_hashed_dict(started_cluster): LAYOUT(RANGE_HASHED()) RANGE(MIN start_date MAX end_date) LIFETIME(60); - """) + """ + ) node1.query("SYSTEM RELOAD DICTIONARY default.rates") - assert node1.query( - "SELECT dictGetString('default.rates', 'currency', toUInt64(4990954156238030839), toDateTime('2019-10-01 00:00:00'))") == "RU\n" + assert ( + node1.query( + "SELECT dictGetString('default.rates', 'currency', toUInt64(4990954156238030839), toDateTime('2019-10-01 00:00:00'))" + ) + == "RU\n" + ) diff --git a/tests/integration/test_read_temporary_tables_on_failure/test.py b/tests/integration/test_read_temporary_tables_on_failure/test.py index ae59fb31641..fd1d92eff92 100644 --- a/tests/integration/test_read_temporary_tables_on_failure/test.py +++ b/tests/integration/test_read_temporary_tables_on_failure/test.py @@ -4,7 +4,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') +node = cluster.add_instance("node") @pytest.fixture(scope="module") @@ -21,7 +21,7 @@ def test_different_versions(start_cluster): with pytest.raises(QueryTimeoutExceedException): node.query("SELECT sleepEachRow(3) FROM numbers(10)", timeout=5) with pytest.raises(QueryRuntimeException): - node.query("SELECT 1", settings={'max_concurrent_queries_for_user': 1}) - assert node.contains_in_log('Too many simultaneous queries for user') - assert not node.contains_in_log('Unknown packet') - assert not node.contains_in_log('Unexpected packet') + node.query("SELECT 1", settings={"max_concurrent_queries_for_user": 1}) + assert node.contains_in_log("Too many simultaneous queries for user") + assert not node.contains_in_log("Unknown packet") + assert not node.contains_in_log("Unexpected packet") diff --git a/tests/integration/test_recompression_ttl/test.py b/tests/integration/test_recompression_ttl/test.py index e74ae928b51..851e3bb4eb8 100644 --- a/tests/integration/test_recompression_ttl/test.py +++ b/tests/integration/test_recompression_ttl/test.py @@ -4,8 +4,12 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/background_pool_config.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/background_pool_config.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/background_pool_config.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/background_pool_config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -24,7 +28,11 @@ def started_cluster(): def wait_part_in_parts(node, table, part_name, retries=40): for i in range(retries): - result = node.query("SELECT name FROM system.parts where name = '{}' and table = '{}'".format(part_name, table)) + result = node.query( + "SELECT name FROM system.parts where name = '{}' and table = '{}'".format( + part_name, table + ) + ) if result: return True time.sleep(0.5) @@ -35,7 +43,10 @@ def wait_part_in_parts(node, table, part_name, retries=40): def optimize_final_table_until_success(node, table_name, retries=40): for i in range(retries): try: - node.query("OPTIMIZE TABLE {} FINAL".format(table_name), settings={"optimize_throw_if_noop": "1"}) + node.query( + "OPTIMIZE TABLE {} FINAL".format(table_name), + settings={"optimize_throw_if_noop": "1"}, + ) return True except: time.sleep(0.5) @@ -46,19 +57,29 @@ def optimize_final_table_until_success(node, table_name, retries=40): def wait_part_and_get_compression_codec(node, table, part_name, retries=40): if wait_part_in_parts(node, table, part_name, retries): return node.query( - "SELECT default_compression_codec FROM system.parts where name = '{}' and table = '{}'".format(part_name, - table)).strip() + "SELECT default_compression_codec FROM system.parts where name = '{}' and table = '{}'".format( + part_name, table + ) + ).strip() return None def test_recompression_simple(started_cluster): node1.query( - "CREATE TABLE table_for_recompression (d DateTime, key UInt64, data String) ENGINE MergeTree() ORDER BY tuple() TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(10)) SETTINGS merge_with_recompression_ttl_timeout = 0") + "CREATE TABLE table_for_recompression (d DateTime, key UInt64, data String) ENGINE MergeTree() ORDER BY tuple() TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(10)) SETTINGS merge_with_recompression_ttl_timeout = 0" + ) node1.query("INSERT INTO table_for_recompression VALUES (now(), 1, '1')") - assert node1.query("SELECT default_compression_codec FROM system.parts where name = 'all_1_1_0'") == "LZ4\n" + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_1_1_0'" + ) + == "LZ4\n" + ) - codec = wait_part_and_get_compression_codec(node1, "table_for_recompression", "all_1_1_1") + codec = wait_part_and_get_compression_codec( + node1, "table_for_recompression", "all_1_1_1" + ) if not codec: assert False, "Part all_1_1_1 doesn't appeared in system.parts" @@ -69,32 +90,50 @@ def test_recompression_simple(started_cluster): optimize_final_table_until_success(node1, "table_for_recompression") - assert node1.query("SELECT default_compression_codec FROM system.parts where name = 'all_1_1_2'") == "ZSTD(10)\n" + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_1_1_2'" + ) + == "ZSTD(10)\n" + ) def test_recompression_multiple_ttls(started_cluster): - node2.query("CREATE TABLE table_for_recompression (d DateTime, key UInt64, data String) ENGINE MergeTree() ORDER BY tuple() \ + node2.query( + "CREATE TABLE table_for_recompression (d DateTime, key UInt64, data String) ENGINE MergeTree() ORDER BY tuple() \ TTL d + INTERVAL 5 SECOND RECOMPRESS CODEC(ZSTD(10)), \ d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(11)), \ - d + INTERVAL 15 SECOND RECOMPRESS CODEC(ZSTD(12)) SETTINGS merge_with_recompression_ttl_timeout = 0") + d + INTERVAL 15 SECOND RECOMPRESS CODEC(ZSTD(12)) SETTINGS merge_with_recompression_ttl_timeout = 0" + ) node2.query("INSERT INTO table_for_recompression VALUES (now(), 1, '1')") - assert node2.query("SELECT default_compression_codec FROM system.parts where name = 'all_1_1_0'") == "LZ4\n" + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_1_1_0'" + ) + == "LZ4\n" + ) - codec = wait_part_and_get_compression_codec(node2, "table_for_recompression", "all_1_1_1") + codec = wait_part_and_get_compression_codec( + node2, "table_for_recompression", "all_1_1_1" + ) if not codec: assert False, "Part all_1_1_1 doesn't appeared in system.parts" assert codec == "ZSTD(10)" - codec = wait_part_and_get_compression_codec(node2, "table_for_recompression", "all_1_1_2") + codec = wait_part_and_get_compression_codec( + node2, "table_for_recompression", "all_1_1_2" + ) if not codec: assert False, "Part all_1_1_2 doesn't appeared in system.parts" assert codec == "ZSTD(11)" - codec = wait_part_and_get_compression_codec(node2, "table_for_recompression", "all_1_1_3") + codec = wait_part_and_get_compression_codec( + node2, "table_for_recompression", "all_1_1_3" + ) if not codec: assert False, "Part all_1_1_3 doesn't appeared in system.parts" @@ -105,32 +144,56 @@ def test_recompression_multiple_ttls(started_cluster): optimize_final_table_until_success(node2, "table_for_recompression") - assert node2.query("SELECT default_compression_codec FROM system.parts where name = 'all_1_1_4'") == "ZSTD(12)\n" + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_1_1_4'" + ) + == "ZSTD(12)\n" + ) - assert node2.query( - "SELECT recompression_ttl_info.expression FROM system.parts where name = 'all_1_1_4'") == "['plus(d, toIntervalSecond(10))','plus(d, toIntervalSecond(15))','plus(d, toIntervalSecond(5))']\n" + assert ( + node2.query( + "SELECT recompression_ttl_info.expression FROM system.parts where name = 'all_1_1_4'" + ) + == "['plus(d, toIntervalSecond(10))','plus(d, toIntervalSecond(15))','plus(d, toIntervalSecond(5))']\n" + ) def test_recompression_replicated(started_cluster): for i, node in enumerate([node1, node2]): - node.query("CREATE TABLE recompression_replicated (d DateTime, key UInt64, data String) \ + node.query( + "CREATE TABLE recompression_replicated (d DateTime, key UInt64, data String) \ ENGINE ReplicatedMergeTree('/test/rr', '{}') ORDER BY tuple() \ TTL d + INTERVAL 10 SECOND RECOMPRESS CODEC(ZSTD(13)) SETTINGS merge_with_recompression_ttl_timeout = 0".format( - i + 1)) + i + 1 + ) + ) node1.query("INSERT INTO recompression_replicated VALUES (now(), 1, '1')") node2.query("SYSTEM SYNC REPLICA recompression_replicated", timeout=5) - assert node1.query( - "SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'") == "LZ4\n" - assert node2.query( - "SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'") == "LZ4\n" + assert ( + node1.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'" + ) + == "LZ4\n" + ) + assert ( + node2.query( + "SELECT default_compression_codec FROM system.parts where name = 'all_0_0_0' and table = 'recompression_replicated'" + ) + == "LZ4\n" + ) - codec1 = wait_part_and_get_compression_codec(node1, "recompression_replicated", "all_0_0_1") + codec1 = wait_part_and_get_compression_codec( + node1, "recompression_replicated", "all_0_0_1" + ) if not codec1: assert False, "Part all_0_0_1 doesn't appeared in system.parts on node1" - codec2 = wait_part_and_get_compression_codec(node2, "recompression_replicated", "all_0_0_1") + codec2 = wait_part_and_get_compression_codec( + node2, "recompression_replicated", "all_0_0_1" + ) if not codec2: assert False, "Part all_0_0_1 doesn't appeared in system.parts on node2" diff --git a/tests/integration/test_recovery_replica/test.py b/tests/integration/test_recovery_replica/test.py index bf869d0de31..4a1298162da 100644 --- a/tests/integration/test_recovery_replica/test.py +++ b/tests/integration/test_recovery_replica/test.py @@ -6,26 +6,32 @@ from helpers.test_tools import assert_eq_with_retry SETTINGS = "SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0" + def fill_nodes(nodes): for node in nodes: node.query( - ''' + """ CREATE TABLE test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) {settings}; - '''.format(replica=node.name, settings=SETTINGS)) + """.format( + replica=node.name, settings=SETTINGS + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) +node3 = cluster.add_instance("node3", with_zookeeper=True) nodes = [node1, node2, node3] + def sync_replicas(table): for node in nodes: node.query("SYSTEM SYNC REPLICA {}".format(table)) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -50,35 +56,53 @@ def test_recovery(start_cluster): for i in range(1, 11): node1.query("INSERT INTO test_table VALUES (1, {})".format(i)) - node2.query_with_retry("ATTACH TABLE test_table", - check_callback=lambda x: len(node2.query("select * from test_table")) > 0) + node2.query_with_retry( + "ATTACH TABLE test_table", + check_callback=lambda x: len(node2.query("select * from test_table")) > 0, + ) - assert_eq_with_retry(node2, "SELECT count(*) FROM test_table", node1.query("SELECT count(*) FROM test_table")) + assert_eq_with_retry( + node2, + "SELECT count(*) FROM test_table", + node1.query("SELECT count(*) FROM test_table"), + ) lost_marker = "Will mark replica node2 as lost" assert node1.contains_in_log(lost_marker) or node3.contains_in_log(lost_marker) sync_replicas("test_table") for node in nodes: - assert node.query("SELECT count(), sum(id) FROM test_table WHERE date=toDate(1)") == "11\t55\n" + assert ( + node.query("SELECT count(), sum(id) FROM test_table WHERE date=toDate(1)") + == "11\t55\n" + ) + def test_choose_source_replica(start_cluster): node3.query("INSERT INTO test_table VALUES (2, 0)") sync_replicas("test_table") node2.query("DETACH TABLE test_table") - node1.query("SYSTEM STOP FETCHES test_table") # node1 will have many entries in queue, so node2 will clone node3 + node1.query( + "SYSTEM STOP FETCHES test_table" + ) # node1 will have many entries in queue, so node2 will clone node3 for i in range(1, 11): node3.query("INSERT INTO test_table VALUES (2, {})".format(i)) - node2.query_with_retry("ATTACH TABLE test_table", - check_callback=lambda x: len(node2.query("select * from test_table")) > 0) + node2.query_with_retry( + "ATTACH TABLE test_table", + check_callback=lambda x: len(node2.query("select * from test_table")) > 0, + ) node1.query("SYSTEM START FETCHES test_table") node1.query("SYSTEM SYNC REPLICA test_table") node2.query("SYSTEM SYNC REPLICA test_table") - assert node1.query("SELECT count(*) FROM test_table") == node3.query("SELECT count(*) FROM test_table") - assert node2.query("SELECT count(*) FROM test_table") == node3.query("SELECT count(*) FROM test_table") + assert node1.query("SELECT count(*) FROM test_table") == node3.query( + "SELECT count(*) FROM test_table" + ) + assert node2.query("SELECT count(*) FROM test_table") == node3.query( + "SELECT count(*) FROM test_table" + ) lost_marker = "Will mark replica node2 as lost" assert node1.contains_in_log(lost_marker) or node3.contains_in_log(lost_marker) @@ -86,17 +110,23 @@ def test_choose_source_replica(start_cluster): sync_replicas("test_table") for node in nodes: - assert node.query("SELECT count(), sum(id) FROM test_table WHERE date=toDate(2)") == "11\t55\n" + assert ( + node.query("SELECT count(), sum(id) FROM test_table WHERE date=toDate(2)") + == "11\t55\n" + ) def test_update_metadata(start_cluster): for node in nodes: node.query( - ''' + """ CREATE TABLE update_metadata(key UInt32) ENGINE = ReplicatedMergeTree('/test/update_metadata', '{replica}') ORDER BY key PARTITION BY key % 10 {settings}; - '''.format(replica=node.name, settings=SETTINGS)) + """.format( + replica=node.name, settings=SETTINGS + ) + ) for i in range(1, 11): node1.query("INSERT INTO update_metadata VALUES ({})".format(i)) @@ -106,17 +136,26 @@ def test_update_metadata(start_cluster): node1.query("ALTER TABLE update_metadata ADD COLUMN col1 UInt32") for i in range(1, 11): - node1.query("INSERT INTO update_metadata VALUES ({}, {})".format(i * 10, i * 10)) + node1.query( + "INSERT INTO update_metadata VALUES ({}, {})".format(i * 10, i * 10) + ) lost_marker = "Will mark replica node2 as lost" assert node1.contains_in_log(lost_marker) or node3.contains_in_log(lost_marker) node2.query("ATTACH TABLE update_metadata") sync_replicas("update_metadata") - assert node1.query("DESC TABLE update_metadata") == node2.query("DESC TABLE update_metadata") - assert node1.query("DESC TABLE update_metadata") == node3.query("DESC TABLE update_metadata") + assert node1.query("DESC TABLE update_metadata") == node2.query( + "DESC TABLE update_metadata" + ) + assert node1.query("DESC TABLE update_metadata") == node3.query( + "DESC TABLE update_metadata" + ) for node in nodes: - assert node.query("SELECT count(), sum(key), sum(col1) FROM update_metadata") == "20\t605\t550\n" + assert ( + node.query("SELECT count(), sum(key), sum(col1) FROM update_metadata") + == "20\t605\t550\n" + ) node2.query("DETACH TABLE update_metadata") # alter with mutation @@ -129,14 +168,21 @@ def test_update_metadata(start_cluster): node2.query("ATTACH TABLE update_metadata") sync_replicas("update_metadata") - assert node1.query("DESC TABLE update_metadata") == node2.query("DESC TABLE update_metadata") - assert node1.query("DESC TABLE update_metadata") == node3.query("DESC TABLE update_metadata") + assert node1.query("DESC TABLE update_metadata") == node2.query( + "DESC TABLE update_metadata" + ) + assert node1.query("DESC TABLE update_metadata") == node3.query( + "DESC TABLE update_metadata" + ) # check that it's possible to execute alter on cloned replica node2.query("ALTER TABLE update_metadata ADD COLUMN col1 UInt32") sync_replicas("update_metadata") for node in nodes: - assert node.query("SELECT count(), sum(key), sum(col1) FROM update_metadata") == "30\t6105\t0\n" + assert ( + node.query("SELECT count(), sum(key), sum(col1) FROM update_metadata") + == "30\t6105\t0\n" + ) # more complex case with multiple alters node2.query("TRUNCATE TABLE update_metadata") @@ -144,21 +190,31 @@ def test_update_metadata(start_cluster): node1.query("INSERT INTO update_metadata VALUES ({}, {})".format(i, i)) # The following alters hang because of "No active replica has part ... or covering part" - #node2.query("SYSTEM STOP REPLICATED SENDS update_metadata") - #node2.query("INSERT INTO update_metadata VALUES (42, 42)") # this part will be lost + # node2.query("SYSTEM STOP REPLICATED SENDS update_metadata") + # node2.query("INSERT INTO update_metadata VALUES (42, 42)") # this part will be lost node2.query("DETACH TABLE update_metadata") node1.query("ALTER TABLE update_metadata MODIFY COLUMN col1 String") node1.query("ALTER TABLE update_metadata ADD COLUMN col2 INT") for i in range(1, 11): - node3.query("INSERT INTO update_metadata VALUES ({}, '{}', {})".format(i * 10, i * 10, i * 10)) + node3.query( + "INSERT INTO update_metadata VALUES ({}, '{}', {})".format( + i * 10, i * 10, i * 10 + ) + ) node1.query("ALTER TABLE update_metadata DROP COLUMN col1") node1.query("ALTER TABLE update_metadata ADD COLUMN col3 Date") node2.query("ATTACH TABLE update_metadata") sync_replicas("update_metadata") - assert node1.query("DESC TABLE update_metadata") == node2.query("DESC TABLE update_metadata") - assert node1.query("DESC TABLE update_metadata") == node3.query("DESC TABLE update_metadata") + assert node1.query("DESC TABLE update_metadata") == node2.query( + "DESC TABLE update_metadata" + ) + assert node1.query("DESC TABLE update_metadata") == node3.query( + "DESC TABLE update_metadata" + ) for node in nodes: - assert node.query("SELECT count(), sum(key), sum(col2) FROM update_metadata") == "20\t605\t550\n" - + assert ( + node.query("SELECT count(), sum(key), sum(col2) FROM update_metadata") + == "20\t605\t550\n" + ) diff --git a/tests/integration/test_redirect_url_storage/test.py b/tests/integration/test_redirect_url_storage/test.py index 061920954b6..06ff78707d7 100644 --- a/tests/integration/test_redirect_url_storage/test.py +++ b/tests/integration/test_redirect_url_storage/test.py @@ -6,7 +6,12 @@ import threading import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/named_collections.xml'], with_zookeeper=False, with_hdfs=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/named_collections.xml"], + with_zookeeper=False, + with_hdfs=True, +) @pytest.fixture(scope="module") @@ -27,7 +32,8 @@ def test_url_without_redirect(started_cluster): # access datanode port directly node1.query( - "create table WebHDFSStorage (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") + "create table WebHDFSStorage (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50075/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')" + ) assert node1.query("select * from WebHDFSStorage") == "1\tMark\t72.53\n" @@ -42,7 +48,8 @@ def test_url_with_globs(started_cluster): hdfs_api.write_data("/simple_storage_2_3", "6\n") result = node1.query( - "select * from url('http://hdfs1:50075/webhdfs/v1/simple_storage_{1..2}_{1..3}?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'data String') as data order by data") + "select * from url('http://hdfs1:50075/webhdfs/v1/simple_storage_{1..2}_{1..3}?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'data String') as data order by data" + ) assert result == "1\n2\n3\n4\n5\n6\n" @@ -57,7 +64,8 @@ def test_url_with_globs_and_failover(started_cluster): hdfs_api.write_data("/simple_storage_3_3", "6\n") result = node1.query( - "select * from url('http://hdfs1:50075/webhdfs/v1/simple_storage_{0|1|2|3}_{1..3}?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'data String') as data order by data") + "select * from url('http://hdfs1:50075/webhdfs/v1/simple_storage_{0|1|2|3}_{1..3}?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'data String') as data order by data" + ) assert result == "1\n2\n3\n" or result == "4\n5\n6\n" @@ -69,9 +77,13 @@ def test_url_with_redirect_not_allowed(started_cluster): # access proxy port without allowing redirects node1.query( - "create table WebHDFSStorageWithoutRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") + "create table WebHDFSStorageWithoutRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')" + ) with pytest.raises(Exception): - assert node1.query("select * from WebHDFSStorageWithoutRedirect") == "1\tMark\t72.53\n" + assert ( + node1.query("select * from WebHDFSStorageWithoutRedirect") + == "1\tMark\t72.53\n" + ) def test_url_with_redirect_allowed(started_cluster): @@ -83,10 +95,17 @@ def test_url_with_redirect_allowed(started_cluster): # access proxy port with allowing redirects # http://localhost:50070/webhdfs/v1/b?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0 node1.query( - "create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')") - assert node1.query("SET max_http_get_redirects=1; select * from WebHDFSStorageWithRedirect") == "1\tMark\t72.53\n" + "create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL('http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV')" + ) + assert ( + node1.query( + "SET max_http_get_redirects=1; select * from WebHDFSStorageWithRedirect" + ) + == "1\tMark\t72.53\n" + ) node1.query("drop table WebHDFSStorageWithRedirect") + def test_predefined_connection_configuration(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -94,29 +113,45 @@ def test_predefined_connection_configuration(started_cluster): assert hdfs_api.read_data("/simple_storage") == "1\tMark\t72.53\n" node1.query( - "create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL(url1, url='http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', format='TSV')") - assert node1.query("SET max_http_get_redirects=1; select * from WebHDFSStorageWithRedirect") == "1\tMark\t72.53\n" - result = node1.query("SET max_http_get_redirects=1; select * from url(url1, url='http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', format='TSV', structure='id UInt32, name String, weight Float64')") - assert(result == "1\tMark\t72.53\n") + "create table WebHDFSStorageWithRedirect (id UInt32, name String, weight Float64) ENGINE = URL(url1, url='http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', format='TSV')" + ) + assert ( + node1.query( + "SET max_http_get_redirects=1; select * from WebHDFSStorageWithRedirect" + ) + == "1\tMark\t72.53\n" + ) + result = node1.query( + "SET max_http_get_redirects=1; select * from url(url1, url='http://hdfs1:50070/webhdfs/v1/simple_storage?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', format='TSV', structure='id UInt32, name String, weight Float64')" + ) + assert result == "1\tMark\t72.53\n" node1.query("drop table WebHDFSStorageWithRedirect") -result = '' +result = "" + + def test_url_reconnect(started_cluster): hdfs_api = started_cluster.hdfs_api with PartitionManager() as pm: node1.query( - "insert into table function hdfs('hdfs://hdfs1:9000/storage_big', 'TSV', 'id Int32') select number from numbers(500000)") + "insert into table function hdfs('hdfs://hdfs1:9000/storage_big', 'TSV', 'id Int32') select number from numbers(500000)" + ) - pm_rule = {'destination': node1.ip_address, 'source_port': 50075, 'action': 'REJECT'} + pm_rule = { + "destination": node1.ip_address, + "source_port": 50075, + "action": "REJECT", + } pm._add_rule(pm_rule) def select(): global result result = node1.query( - "select sum(cityHash64(id)) from url('http://hdfs1:50075/webhdfs/v1/storage_big?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'id Int32') settings http_max_tries = 10, http_retry_max_backoff_ms=1000") - assert(int(result), 6581218782194912115) + "select sum(cityHash64(id)) from url('http://hdfs1:50075/webhdfs/v1/storage_big?op=OPEN&namenoderpcaddress=hdfs1:9000&offset=0', 'TSV', 'id Int32') settings http_max_tries = 10, http_retry_max_backoff_ms=1000" + ) + assert (int(result), 6581218782194912115) thread = threading.Thread(target=select) thread.start() @@ -126,5 +161,5 @@ def test_url_reconnect(started_cluster): thread.join() - assert(int(result), 6581218782194912115) - assert node1.contains_in_log("Error: Timeout: connect timed out") + assert (int(result), 6581218782194912115) + assert node1.contains_in_log("Timeout: connect timed out") diff --git a/tests/integration/test_relative_filepath/test.py b/tests/integration/test_relative_filepath/test.py index 45c969b86f5..a9701092b65 100644 --- a/tests/integration/test_relative_filepath/test.py +++ b/tests/integration/test_relative_filepath/test.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/config.xml']) +node = cluster.add_instance("node", main_configs=["configs/config.xml"]) path_to_userfiles_from_defaut_config = "user_files" @@ -20,19 +20,41 @@ def test_filepath(start_cluster): # 2 rows data some_data = "Test\t111.222\nData\t333.444" - node.exec_in_container(['bash', '-c', 'mkdir -p {}'.format( - path_to_userfiles_from_defaut_config - )], privileged=True, user='root') + node.exec_in_container( + ["bash", "-c", "mkdir -p {}".format(path_to_userfiles_from_defaut_config)], + privileged=True, + user="root", + ) - node.exec_in_container(['bash', '-c', 'echo "{}" > {}'.format( - some_data, - path_to_userfiles_from_defaut_config + "/relative_user_file_test" - )], privileged=True, user='root') + node.exec_in_container( + [ + "bash", + "-c", + 'echo "{}" > {}'.format( + some_data, + path_to_userfiles_from_defaut_config + "/relative_user_file_test", + ), + ], + privileged=True, + user="root", + ) - test_requests = [("relative_user_file_test", "2"), - ("../" + path_to_userfiles_from_defaut_config + "/relative_user_file_test", "2")] + test_requests = [ + ("relative_user_file_test", "2"), + ( + "../" + path_to_userfiles_from_defaut_config + "/relative_user_file_test", + "2", + ), + ] for pattern, value in test_requests: - assert node.query(''' + assert ( + node.query( + """ select count() from file('{}', 'TSV', 'text String, number Float64') - '''.format(pattern)) == '{}\n'.format(value) + """.format( + pattern + ) + ) + == "{}\n".format(value) + ) diff --git a/tests/integration/test_reload_auxiliary_zookeepers/test.py b/tests/integration/test_reload_auxiliary_zookeepers/test.py index a52f21b5e02..bb1455333fc 100644 --- a/tests/integration/test_reload_auxiliary_zookeepers/test.py +++ b/tests/integration/test_reload_auxiliary_zookeepers/test.py @@ -60,7 +60,9 @@ def test_reload_auxiliary_zookeepers(start_cluster): """ - node.replace_config("/etc/clickhouse-server/conf.d/zookeeper_config.xml", new_config) + node.replace_config( + "/etc/clickhouse-server/conf.d/zookeeper_config.xml", new_config + ) node.query("SYSTEM RELOAD CONFIG") @@ -81,7 +83,9 @@ def test_reload_auxiliary_zookeepers(start_cluster): 2000 """ - node.replace_config("/etc/clickhouse-server/conf.d/zookeeper_config.xml", new_config) + node.replace_config( + "/etc/clickhouse-server/conf.d/zookeeper_config.xml", new_config + ) node.query("SYSTEM RELOAD CONFIG") time.sleep(5) diff --git a/tests/integration/test_reload_certificate/configs/ECcert.crt b/tests/integration/test_reload_certificate/configs/ECcert.crt new file mode 100644 index 00000000000..b87ce0099dc --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/ECcert.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICkzCCAhigAwIBAgIUcrahhUuSDdw60Wyfo2E4kVUWWQ8wCgYIKoZIzj0EAwIw +fzELMAkGA1UEBhMCQ04xEzARBgNVBAgMClNvbWUtU3RhdGUxDTALBgNVBAcMBGNp +dHkxEDAOBgNVBAoMB2NvbXBhbnkxEDAOBgNVBAsMB3NlY3Rpb24xEjAQBgNVBAMM +CWxvY2FsaG9zdDEUMBIGCSqGSIb3DQEJARYFZW1haWwwIBcNMjIwMjI3MTg1NzQz +WhgPMjEyMjAyMDMxODU3NDNaMH8xCzAJBgNVBAYTAkNOMRMwEQYDVQQIDApTb21l +LVN0YXRlMQ0wCwYDVQQHDARjaXR5MRAwDgYDVQQKDAdjb21wYW55MRAwDgYDVQQL +DAdzZWN0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QxFDASBgkqhkiG9w0BCQEWBWVt +YWlsMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEgoPY/r89/83zHzmpbsA+kW3YflVQ +tKXO8Kl7ki5q+v1qUu3xmr4HttNxvHLOCfK798KMGg9y+NO5y4D4D2ZgLGxkNt8X +yWvhkbe3xKdGSqBpplbLT+M9FtmQ6tzzzFJVo1MwUTAdBgNVHQ4EFgQUmpLPeJBD +ID5s1AeWsVIEt6Z/ca0wHwYDVR0jBBgwFoAUmpLPeJBDID5s1AeWsVIEt6Z/ca0w +DwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNpADBmAjEAv4uNU4NgprBgNQxk +fIZpJCf/TpisuVsLUHXl8JrMVKKVUf7zr59GH2yiOoukfD5hAjEAlCohSA6/Ken4 +JWkKPCrfnsBZ7VX8Y+4ZqLKuG+IGAu2vQTg+Jc6M23M1vEgi1dqf +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_certificate/configs/ECcert.key b/tests/integration/test_reload_certificate/configs/ECcert.key new file mode 100644 index 00000000000..b127f8a53fe --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/ECcert.key @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAJbfB78wfRHn5A4x3e +EAqrFk/hbBD+c8snbFgjQqxg4qTcp154Rc01B9V0US27MJuhZANiAASCg9j+vz3/ +zfMfOaluwD6Rbdh+VVC0pc7wqXuSLmr6/WpS7fGavge203G8cs4J8rv3wowaD3L4 +07nLgPgPZmAsbGQ23xfJa+GRt7fEp0ZKoGmmVstP4z0W2ZDq3PPMUlU= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_certificate/test.py b/tests/integration/test_reload_certificate/test.py index dc0c391d6f0..0f2579f4544 100644 --- a/tests/integration/test_reload_certificate/test.py +++ b/tests/integration/test_reload_certificate/test.py @@ -4,9 +4,19 @@ from helpers.cluster import ClickHouseCluster SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/first.crt", "configs/first.key", - "configs/second.crt", "configs/second.key", - "configs/cert.xml"]) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/first.crt", + "configs/first.key", + "configs/second.crt", + "configs/second.key", + "configs/ECcert.crt", + "configs/ECcert.key", + "configs/cert.xml", + ], +) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): @@ -16,12 +26,17 @@ def started_cluster(): finally: cluster.shutdown() + def change_config_to_key(name): - ''' - * Generate config with certificate/key name from args. - * Reload config. - ''' - node.exec_in_container(["bash", "-c" , """cat > /etc/clickhouse-server/config.d/cert.xml << EOF + """ + * Generate config with certificate/key name from args. + * Reload config. + """ + node.exec_in_container( + [ + "bash", + "-c", + """cat > /etc/clickhouse-server/config.d/cert.xml << EOF 8443 @@ -36,40 +51,145 @@ def change_config_to_key(name): -EOF""".format(cur_name=name)]) +EOF""".format( + cur_name=name + ), + ] + ) node.query("SYSTEM RELOAD CONFIG") + def test_first_than_second_cert(): - ''' Consistently set first key and check that only it will be accepted, then repeat same for second key. ''' + """Consistently set first key and check that only it will be accepted, then repeat same for second key.""" # Set first key - change_config_to_key('first') + change_config_to_key("first") # Command with correct certificate - assert node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='first'), - 'https://localhost:8443/']) == 'Ok.\n' + assert ( + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="first" + ), + "https://localhost:8443/", + ] + ) + == "Ok.\n" + ) # Command with wrong certificate - # This command don't use option '-k', so it will lead to error while execution. + # This command don't use option '-k', so it will lead to error while execution. # That's why except will always work try: - node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='second'), - 'https://localhost:8443/']) + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="second" + ), + "https://localhost:8443/", + ] + ) assert False except: assert True - + # Change to other key - change_config_to_key('second') + change_config_to_key("second") # Command with correct certificate - assert node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='second'), - 'https://localhost:8443/']) == 'Ok.\n' + assert ( + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="second" + ), + "https://localhost:8443/", + ] + ) + == "Ok.\n" + ) # Command with wrong certificate # Same as previous try: - node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='first'), - 'https://localhost:8443/']) + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="first" + ), + "https://localhost:8443/", + ] + ) + assert False + except: + assert True + + +def test_ECcert_reload(): + # Set first key + change_config_to_key("first") + + # Command with correct certificate + assert ( + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="first" + ), + "https://localhost:8443/", + ] + ) + == "Ok.\n" + ) + + # Change to other key + change_config_to_key("ECcert") + + # Command with correct certificate + assert ( + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="ECcert" + ), + "https://localhost:8443/", + ] + ) + == "Ok.\n" + ) + + # Command with wrong certificate + # Same as previous + try: + node.exec_in_container( + [ + "curl", + "--silent", + "--cacert", + "/etc/clickhouse-server/config.d/{cur_name}.crt".format( + cur_name="first" + ), + "https://localhost:8443/", + ] + ) assert False except: assert True diff --git a/tests/integration/test_reload_clusters_config/test.py b/tests/integration/test_reload_clusters_config/test.py index 048b704034b..6979fd5565b 100644 --- a/tests/integration/test_reload_clusters_config/test.py +++ b/tests/integration/test_reload_clusters_config/test.py @@ -11,23 +11,30 @@ from helpers.network import PartitionManager from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) -node_1 = cluster.add_instance('node_1', with_zookeeper=True) -node_2 = cluster.add_instance('node_2', with_zookeeper=True) +node = cluster.add_instance( + "node", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) +node_1 = cluster.add_instance("node_1", with_zookeeper=True) +node_2 = cluster.add_instance("node_2", with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node.query('''CREATE TABLE distributed (id UInt32) ENGINE = - Distributed('test_cluster', 'default', 'replicated')''') - - node.query('''CREATE TABLE distributed2 (id UInt32) ENGINE = - Distributed('test_cluster2', 'default', 'replicated')''') + node.query( + """CREATE TABLE distributed (id UInt32) ENGINE = + Distributed('test_cluster', 'default', 'replicated')""" + ) - cluster.pause_container('node_1') - cluster.pause_container('node_2') + node.query( + """CREATE TABLE distributed2 (id UInt32) ENGINE = + Distributed('test_cluster2', 'default', 'replicated')""" + ) + + cluster.pause_container("node_1") + cluster.pause_container("node_2") yield cluster @@ -35,7 +42,7 @@ def started_cluster(): cluster.shutdown() -base_config = ''' +base_config = """ @@ -66,9 +73,9 @@ base_config = ''' -''' +""" -test_config1 = ''' +test_config1 = """ @@ -95,9 +102,9 @@ test_config1 = ''' -''' +""" -test_config2 = ''' +test_config2 = """ @@ -115,9 +122,9 @@ test_config2 = ''' -''' +""" -test_config3 = ''' +test_config3 = """ @@ -157,16 +164,24 @@ test_config3 = ''' -''' +""" def send_repeated_query(table, count=5): for i in range(count): - node.query_and_get_error("SELECT count() FROM {} SETTINGS receive_timeout=1".format(table)) + node.query_and_get_error( + "SELECT count() FROM {} SETTINGS receive_timeout=1".format(table) + ) def get_errors_count(cluster, host_name="node_1"): - return int(node.query("SELECT errors_count FROM system.clusters WHERE cluster='{}' and host_name='{}'".format(cluster, host_name))) + return int( + node.query( + "SELECT errors_count FROM system.clusters WHERE cluster='{}' and host_name='{}'".format( + cluster, host_name + ) + ) + ) def set_config(config): @@ -178,7 +193,7 @@ def test_simple_reload(started_cluster): send_repeated_query("distributed") assert get_errors_count("test_cluster") > 0 - + node.query("SYSTEM RELOAD CONFIG") assert get_errors_count("test_cluster") > 0 @@ -209,9 +224,9 @@ def test_delete_cluster(started_cluster): set_config(test_config2) assert get_errors_count("test_cluster") > 0 - + result = node.query("SELECT * FROM system.clusters WHERE cluster='test_cluster2'") - assert result == '' + assert result == "" set_config(base_config) @@ -229,7 +244,6 @@ def test_add_cluster(started_cluster): assert get_errors_count("test_cluster2") > 0 result = node.query("SELECT * FROM system.clusters WHERE cluster='test_cluster3'") - assert result != '' + assert result != "" set_config(base_config) - diff --git a/tests/integration/test_reload_max_table_size_to_drop/test.py b/tests/integration/test_reload_max_table_size_to_drop/test.py index 7e7219088b8..da7dba12fa0 100644 --- a/tests/integration/test_reload_max_table_size_to_drop/test.py +++ b/tests/integration/test_reload_max_table_size_to_drop/test.py @@ -5,18 +5,23 @@ import pytest from helpers.cluster import ClickHouseCluster, get_instances_dir cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/max_table_size_to_drop.xml"]) +node = cluster.add_instance("node", main_configs=["configs/max_table_size_to_drop.xml"]) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/max_table_size_to_drop.xml'.format(get_instances_dir())) +CONFIG_PATH = os.path.join( + SCRIPT_DIR, + "./{}/node/configs/config.d/max_table_size_to_drop.xml".format(get_instances_dir()), +) @pytest.fixture(scope="module") def start_cluster(): try: cluster.start() - node.query("CREATE TABLE test(date Date, id UInt32) ENGINE = MergeTree() PARTITION BY date ORDER BY id") + node.query( + "CREATE TABLE test(date Date, id UInt32) ENGINE = MergeTree() PARTITION BY date ORDER BY id" + ) yield cluster finally: cluster.shutdown() @@ -32,11 +37,14 @@ def test_reload_max_table_size_to_drop(start_cluster): assert out == "" assert err != "" - config = open(CONFIG_PATH, 'r') + config = open(CONFIG_PATH, "r") config_lines = config.readlines() config.close() - config_lines = [line.replace("1", "1000000") for line in config_lines] - config = open(CONFIG_PATH, 'w') + config_lines = [ + line.replace("1", "1000000") + for line in config_lines + ] + config = open(CONFIG_PATH, "w") config.writelines(config_lines) config.close() diff --git a/tests/integration/test_reload_zookeeper/test.py b/tests/integration/test_reload_zookeeper/test.py index 73ef42a86f6..8924376d6fd 100644 --- a/tests/integration/test_reload_zookeeper/test.py +++ b/tests/integration/test_reload_zookeeper/test.py @@ -7,8 +7,8 @@ from helpers.client import QueryRuntimeException from helpers.test_tools import assert_eq_with_retry -cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper.xml') -node = cluster.add_instance('node', with_zookeeper=True) +cluster = ClickHouseCluster(__file__, zookeeper_config_path="configs/zookeeper.xml") +node = cluster.add_instance("node", with_zookeeper=True) @pytest.fixture(scope="module") @@ -16,33 +16,36 @@ def start_cluster(): try: cluster.start() node.query( - ''' + """ CREATE TABLE test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/shard1/test/test_table', '1') PARTITION BY toYYYYMM(date) ORDER BY id - ''') + """ + ) yield cluster finally: cluster.shutdown() -def test_reload_zookeeper(start_cluster): +def test_reload_zookeeper(start_cluster): def wait_zookeeper_node_to_start(zk_nodes, timeout=60): start = time.time() while time.time() - start < timeout: try: for instance in zk_nodes: conn = start_cluster.get_kazoo_client(instance) - conn.get_children('/') + conn.get_children("/") print("All instances of ZooKeeper started") return except Exception as ex: print(("Can't connect to ZooKeeper " + str(ex))) time.sleep(0.5) - node.query("INSERT INTO test_table(date, id) select today(), number FROM numbers(1000)") + node.query( + "INSERT INTO test_table(date, id) select today(), number FROM numbers(1000)" + ) ## remove zoo2, zoo3 from configs new_config = """ @@ -59,23 +62,41 @@ def test_reload_zookeeper(start_cluster): node.replace_config("/etc/clickhouse-server/conf.d/zookeeper.xml", new_config) node.query("SYSTEM RELOAD CONFIG") ## config reloads, but can still work - assert_eq_with_retry(node, "SELECT COUNT() FROM test_table", '1000', retry_count=120, sleep_time=0.5) + assert_eq_with_retry( + node, "SELECT COUNT() FROM test_table", "1000", retry_count=120, sleep_time=0.5 + ) ## stop all zookeepers, table will be readonly cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) node.query("SELECT COUNT() FROM test_table") with pytest.raises(QueryRuntimeException): - node.query("SELECT COUNT() FROM test_table", settings={"select_sequential_consistency" : 1}) + node.query( + "SELECT COUNT() FROM test_table", + settings={"select_sequential_consistency": 1}, + ) ## start zoo2, zoo3, table will be readonly too, because it only connect to zoo1 cluster.start_zookeeper_nodes(["zoo2", "zoo3"]) wait_zookeeper_node_to_start(["zoo2", "zoo3"]) node.query("SELECT COUNT() FROM test_table") with pytest.raises(QueryRuntimeException): - node.query("SELECT COUNT() FROM test_table", settings={"select_sequential_consistency" : 1}) + node.query( + "SELECT COUNT() FROM test_table", + settings={"select_sequential_consistency": 1}, + ) def get_active_zk_connections(): - return str(node.exec_in_container(['bash', '-c', 'lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l'], privileged=True, user='root')).strip() + return str( + node.exec_in_container( + [ + "bash", + "-c", + "lsof -a -i4 -i6 -itcp -w | grep 2181 | grep ESTABLISHED | wc -l", + ], + privileged=True, + user="root", + ) + ).strip() ## set config to zoo2, server will be normal new_config = """ @@ -93,9 +114,15 @@ def test_reload_zookeeper(start_cluster): node.query("SYSTEM RELOAD CONFIG") active_zk_connections = get_active_zk_connections() - assert active_zk_connections == '1', "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) + assert ( + active_zk_connections == "1" + ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) - assert_eq_with_retry(node, "SELECT COUNT() FROM test_table", '1000', retry_count=120, sleep_time=0.5) + assert_eq_with_retry( + node, "SELECT COUNT() FROM test_table", "1000", retry_count=120, sleep_time=0.5 + ) active_zk_connections = get_active_zk_connections() - assert active_zk_connections == '1', "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) + assert ( + active_zk_connections == "1" + ), "Total connections to ZooKeeper not equal to 1, {}".format(active_zk_connections) diff --git a/tests/integration/test_reloading_settings_from_users_xml/test.py b/tests/integration/test_reloading_settings_from_users_xml/test.py index b45568ee904..3b95796ab9c 100644 --- a/tests/integration/test_reloading_settings_from_users_xml/test.py +++ b/tests/integration/test_reloading_settings_from_users_xml/test.py @@ -6,7 +6,8 @@ from helpers.test_tools import assert_eq_with_retry, assert_logs_contain_with_re SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', user_configs=["configs/normal_settings.xml"]) +node = cluster.add_instance("node", user_configs=["configs/normal_settings.xml"]) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): @@ -20,7 +21,10 @@ def started_cluster(): @pytest.fixture(autouse=True) def reset_to_normal_settings_after_test(): try: - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/normal_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/normal_settings.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) node.query("SYSTEM RELOAD CONFIG") yield finally: @@ -30,8 +34,11 @@ def reset_to_normal_settings_after_test(): def test_force_reload(): assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" - - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/changed_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/changed_settings.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) node.query("SYSTEM RELOAD CONFIG") assert node.query("SELECT getSetting('max_memory_usage')") == "20000000000\n" @@ -42,16 +49,24 @@ def test_reload_on_timeout(): assert node.query("SELECT getSetting('max_memory_usage')") == "10000000000\n" assert node.query("SELECT getSetting('load_balancing')") == "first_or_random\n" - time.sleep(1) # The modification time of the 'z.xml' file should be different, - # because config files are reload by timer only when the modification time is changed. - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/changed_settings.xml"), '/etc/clickhouse-server/users.d/z.xml') + time.sleep(1) # The modification time of the 'z.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/changed_settings.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) assert_eq_with_retry(node, "SELECT getSetting('max_memory_usage')", "20000000000") - assert_eq_with_retry(node, "SELECT getSetting('load_balancing')", "nearest_hostname") + assert_eq_with_retry( + node, "SELECT getSetting('load_balancing')", "nearest_hostname" + ) def test_unknown_setting_force_reload(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unknown_setting.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/unknown_setting.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) error_message = "Setting xyz is neither a builtin setting nor started with the prefix 'custom_' registered for user-defined settings" assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") @@ -61,9 +76,12 @@ def test_unknown_setting_force_reload(): def test_unknown_setting_reload_on_timeout(): - time.sleep(1) # The modification time of the 'z.xml' file should be different, - # because config files are reload by timer only when the modification time is changed. - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unknown_setting.xml"), '/etc/clickhouse-server/users.d/z.xml') + time.sleep(1) # The modification time of the 'z.xml' file should be different, + # because config files are reload by timer only when the modification time is changed. + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/unknown_setting.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) error_message = "Setting xyz is neither a builtin setting nor started with the prefix 'custom_' registered for user-defined settings" assert_logs_contain_with_retry(node, error_message) @@ -73,7 +91,10 @@ def test_unknown_setting_reload_on_timeout(): def test_unexpected_setting_int(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unexpected_setting_int.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/unexpected_setting_int.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) error_message = "Cannot parse" assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") @@ -82,7 +103,10 @@ def test_unexpected_setting_int(): def test_unexpected_setting_enum(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/unexpected_setting_int.xml"), '/etc/clickhouse-server/users.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/unexpected_setting_int.xml"), + "/etc/clickhouse-server/users.d/z.xml", + ) error_message = "Cannot parse" assert error_message in node.query_and_get_error("SYSTEM RELOAD CONFIG") diff --git a/tests/integration/test_reloading_storage_configuration/test.py b/tests/integration/test_reloading_storage_configuration/test.py index e9fba6012f7..4b21919ab3d 100644 --- a/tests/integration/test_reloading_storage_configuration/test.py +++ b/tests/integration/test_reloading_storage_configuration/test.py @@ -13,25 +13,41 @@ import pytest cluster = helpers.cluster.ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', - '/external:size=200M'], - macros={"shard": 0, "replica": 1}) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/logs_config.xml"], + with_zookeeper=True, + stay_alive=True, + tmpfs=[ + "/jbod1:size=40M", + "/jbod2:size=40M", + "/jbod3:size=40M", + "/jbod4:size=40M", + "/external:size=200M", + ], + macros={"shard": 0, "replica": 1}, +) -node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - stay_alive=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/jbod3:size=40M', '/jbod4:size=40M', - '/external:size=200M'], - macros={"shard": 0, "replica": 2}) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/logs_config.xml"], + with_zookeeper=True, + stay_alive=True, + tmpfs=[ + "/jbod1:size=40M", + "/jbod2:size=40M", + "/jbod3:size=40M", + "/jbod4:size=40M", + "/external:size=200M", + ], + macros={"shard": 0, "replica": 2}, +) def get_log(node): - return node.exec_in_container(["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"]) + return node.exec_in_container( + ["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"] + ) @pytest.fixture(scope="module") @@ -45,11 +61,17 @@ def started_cluster(): def start_over(): - shutil.copy(os.path.join(os.path.dirname(__file__), "configs/config.d/storage_configuration.xml"), - os.path.join(node1.config_d_dir, "storage_configuration.xml")) + shutil.copy( + os.path.join( + os.path.dirname(__file__), "configs/config.d/storage_configuration.xml" + ), + os.path.join(node1.config_d_dir, "storage_configuration.xml"), + ) for node in (node1, node2): - separate_configuration_path = os.path.join(node.config_d_dir, "separate_configuration.xml") + separate_configuration_path = os.path.join( + node.config_d_dir, "separate_configuration.xml" + ) try: os.remove(separate_configuration_path) except: @@ -57,16 +79,23 @@ def start_over(): def add_disk(node, name, path, separate_file=False): - separate_configuration_path = os.path.join(node.config_d_dir, "separate_configuration.xml") + separate_configuration_path = os.path.join( + node.config_d_dir, "separate_configuration.xml" + ) try: if separate_file: tree = ET.parse(separate_configuration_path) else: - tree = ET.parse(os.path.join(node.config_d_dir, "storage_configuration.xml")) + tree = ET.parse( + os.path.join(node.config_d_dir, "storage_configuration.xml") + ) except: tree = ET.ElementTree( - ET.fromstring('')) + ET.fromstring( + "" + ) + ) root = tree.getroot() new_disk = ET.Element(name) new_path = ET.Element("path") @@ -78,19 +107,25 @@ def add_disk(node, name, path, separate_file=False): else: tree.write(os.path.join(node.config_d_dir, "storage_configuration.xml")) + def update_disk(node, name, path, keep_free_space_bytes, separate_file=False): - separate_configuration_path = os.path.join(node.config_d_dir, - "separate_configuration.xml") + separate_configuration_path = os.path.join( + node.config_d_dir, "separate_configuration.xml" + ) try: if separate_file: tree = ET.parse(separate_configuration_path) else: tree = ET.parse( - os.path.join(node.config_d_dir, "storage_configuration.xml")) + os.path.join(node.config_d_dir, "storage_configuration.xml") + ) except: tree = ET.ElementTree( - ET.fromstring('')) + ET.fromstring( + "" + ) + ) root = tree.getroot() disk = root.find("storage_configuration").find("disks").find(name) @@ -136,15 +171,21 @@ def test_add_disk(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - assert "jbod3" not in set(node1.query("SELECT name FROM system.disks").splitlines()) + assert "jbod3" not in set( + node1.query("SELECT name FROM system.disks").splitlines() + ) add_disk(node1, "jbod3", "/jbod3/") node1.query("SYSTEM RELOAD CONFIG") @@ -156,6 +197,7 @@ def test_add_disk(started_cluster): except: """""" + def test_update_disk(started_cluster): try: name = "test_update_disk" @@ -165,28 +207,35 @@ def test_update_disk(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - assert node1.query("SELECT path, keep_free_space FROM system.disks where name = 'jbod2'") == TSV([ - ["/jbod2/", "10485760"]]) + assert node1.query( + "SELECT path, keep_free_space FROM system.disks where name = 'jbod2'" + ) == TSV([["/jbod2/", "10485760"]]) update_disk(node1, "jbod2", "/jbod2/", "20971520") node1.query("SYSTEM RELOAD CONFIG") - assert node1.query("SELECT path, keep_free_space FROM system.disks where name = 'jbod2'") == TSV([ - ["/jbod2/", "20971520"]]) + assert node1.query( + "SELECT path, keep_free_space FROM system.disks where name = 'jbod2'" + ) == TSV([["/jbod2/", "20971520"]]) finally: try: node1.query("DROP TABLE IF EXISTS {}".format(name)) except: """""" + def test_add_disk_to_separate_config(started_cluster): try: name = "test_add_disk" @@ -196,15 +245,21 @@ def test_add_disk_to_separate_config(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - assert "jbod3" not in set(node1.query("SELECT name FROM system.disks").splitlines()) + assert "jbod3" not in set( + node1.query("SELECT name FROM system.disks").splitlines() + ) add_disk(node1, "jbod3", "/jbod3/", separate_file=True) node1.query("SYSTEM RELOAD CONFIG") @@ -230,23 +285,35 @@ def test_add_policy(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) add_policy(node1, "cool_policy", {"volume1": ["jbod3", "jbod4"]}) node1.query("SYSTEM RELOAD CONFIG") disks = set(node1.query("SELECT name FROM system.disks").splitlines()) - assert "cool_policy" in set(node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) - assert {"volume1"} == set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + assert "cool_policy" in set( + node1.query("SELECT policy_name FROM system.storage_policies").splitlines() + ) + assert {"volume1"} == set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'" + ).splitlines() + ) assert {"['jbod3','jbod4']"} == set( - node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'" + ).splitlines() + ) finally: try: @@ -265,39 +332,69 @@ def test_new_policy_works(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) node1.query("SYSTEM RELOAD CONFIG") # Incompatible storage policy. with pytest.raises(helpers.client.QueryRuntimeException): - node1.query(""" + node1.query( + """ ALTER TABLE {name} MODIFY SETTING storage_policy='cool_policy' - """.format(name=name)) + """.format( + name=name + ) + ) start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "cool_policy", collections.OrderedDict( - [("volume1", ["jbod3"]), ("main", ["jbod1", "jbod2"]), ("external", ["external"])])) + add_policy( + node1, + "cool_policy", + collections.OrderedDict( + [ + ("volume1", ["jbod3"]), + ("main", ["jbod1", "jbod2"]), + ("external", ["external"]), + ] + ), + ) node1.query("SYSTEM RELOAD CONFIG") - node1.query(""" + node1.query( + """ ALTER TABLE {name} MODIFY SETTING storage_policy='cool_policy' - """.format(name=name)) + """.format( + name=name + ) + ) - node1.query(""" + node1.query( + """ INSERT INTO TABLE {name} VALUES (1) - """.format(name=name)) - assert {"jbod3"} == set(node1.query( - "SELECT disk_name FROM system.parts WHERE active = 1 AND table = '{name}'".format(name=name)).splitlines()) + """.format( + name=name + ) + ) + assert {"jbod3"} == set( + node1.query( + "SELECT disk_name FROM system.parts WHERE active = 1 AND table = '{name}'".format( + name=name + ) + ).splitlines() + ) finally: try: @@ -318,24 +415,38 @@ def test_add_volume_to_policy(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "cool_policy", collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) + add_policy( + node1, + "cool_policy", + collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])]), + ) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + volumes = set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'" + ).splitlines() + ) disks_sets = set( - node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'" + ).splitlines() + ) assert {"volume1", "volume2"} == volumes assert {"['jbod3']", "['jbod4']"} == disks_sets @@ -358,13 +469,17 @@ def test_add_disk_to_policy(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) start_over() add_disk(node1, "jbod3", "/jbod3/") @@ -372,10 +487,16 @@ def test_add_disk_to_policy(started_cluster): add_policy(node1, "cool_policy", {"volume1": ["jbod3", "jbod4"]}) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + volumes = set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'cool_policy'" + ).splitlines() + ) disks_sets = set( - node1.query("SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'").splitlines()) + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'cool_policy'" + ).splitlines() + ) assert {"volume1"} == volumes assert {"['jbod3','jbod4']"} == disks_sets @@ -396,20 +517,28 @@ def test_remove_disk(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - assert "remove_disk_jbod3" in set(node1.query("SELECT name FROM system.disks").splitlines()) + assert "remove_disk_jbod3" in set( + node1.query("SELECT name FROM system.disks").splitlines() + ) start_over() node1.query("SYSTEM RELOAD CONFIG") - assert "remove_disk_jbod3" in set(node1.query("SELECT name FROM system.disks").splitlines()) + assert "remove_disk_jbod3" in set( + node1.query("SELECT name FROM system.disks").splitlines() + ) assert re.search("Warning.*remove_disk_jbod3", get_log(node1)) finally: try: @@ -430,16 +559,21 @@ def test_remove_policy(started_cluster): node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) assert "remove_policy_cool_policy" in set( - node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + node1.query("SELECT policy_name FROM system.storage_policies").splitlines() + ) start_over() add_disk(node1, "jbod3", "/jbod3/") @@ -447,7 +581,8 @@ def test_remove_policy(started_cluster): node1.query("SYSTEM RELOAD CONFIG") assert "remove_policy_cool_policy" in set( - node1.query("SELECT policy_name FROM system.storage_policies").splitlines()) + node1.query("SELECT policy_name FROM system.storage_policies").splitlines() + ) assert re.search("Error.*remove_policy_cool_policy", get_log(node1)) finally: @@ -465,23 +600,36 @@ def test_remove_volume_from_policy(started_cluster): start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "test_remove_volume_from_policy_cool_policy", - collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])])) + add_policy( + node1, + "test_remove_volume_from_policy_cool_policy", + collections.OrderedDict([("volume1", ["jbod3"]), ("volume2", ["jbod4"])]), + ) node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - volumes = set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query( - "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + volumes = set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'" + ).splitlines() + ) + disks_sets = set( + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'" + ).splitlines() + ) assert {"volume1", "volume2"} == volumes assert {"['jbod3']", "['jbod4']"} == disks_sets @@ -491,13 +639,21 @@ def test_remove_volume_from_policy(started_cluster): add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query( - "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'").splitlines()) + volumes = set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'" + ).splitlines() + ) + disks_sets = set( + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_volume_from_policy_cool_policy'" + ).splitlines() + ) assert {"volume1", "volume2"} == volumes assert {"['jbod3']", "['jbod4']"} == disks_sets - assert re.search("Error.*test_remove_volume_from_policy_cool_policy", get_log(node1)) + assert re.search( + "Error.*test_remove_volume_from_policy_cool_policy", get_log(node1) + ) finally: try: @@ -514,22 +670,36 @@ def test_remove_disk_from_policy(started_cluster): start_over() add_disk(node1, "jbod3", "/jbod3/") add_disk(node1, "jbod4", "/jbod4/") - add_policy(node1, "test_remove_disk_from_policy_cool_policy", {"volume1": ["jbod3", "jbod4"]}) + add_policy( + node1, + "test_remove_disk_from_policy_cool_policy", + {"volume1": ["jbod3", "jbod4"]}, + ) node1.restart_clickhouse(kill=True) time.sleep(2) - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( d UInt64 ) ENGINE = {engine} ORDER BY d SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - volumes = set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query( - "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + volumes = set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'" + ).splitlines() + ) + disks_sets = set( + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'" + ).splitlines() + ) assert {"volume1"} == volumes assert {"['jbod3','jbod4']"} == disks_sets @@ -539,13 +709,21 @@ def test_remove_disk_from_policy(started_cluster): add_policy(node1, "cool_policy", {"volume1": ["jbod3"]}) node1.query("SYSTEM RELOAD CONFIG") - volumes = set(node1.query( - "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) - disks_sets = set(node1.query( - "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'").splitlines()) + volumes = set( + node1.query( + "SELECT volume_name FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'" + ).splitlines() + ) + disks_sets = set( + node1.query( + "SELECT disks FROM system.storage_policies WHERE policy_name = 'test_remove_disk_from_policy_cool_policy'" + ).splitlines() + ) assert {"volume1"} == volumes assert {"['jbod3','jbod4']"} == disks_sets - assert re.search("Error.*test_remove_disk_from_policy_cool_policy", get_log(node1)) + assert re.search( + "Error.*test_remove_disk_from_policy_cool_policy", get_log(node1) + ) finally: try: diff --git a/tests/integration/test_remote_prewhere/test.py b/tests/integration/test_remote_prewhere/test.py index 907a9d43d2a..60372b3028e 100644 --- a/tests/integration/test_remote_prewhere/test.py +++ b/tests/integration/test_remote_prewhere/test.py @@ -3,8 +3,8 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1') -node2 = cluster.add_instance('node2') +node1 = cluster.add_instance("node1") +node2 = cluster.add_instance("node2") @pytest.fixture(scope="module") @@ -13,7 +13,8 @@ def start_cluster(): cluster.start() for node in [node1, node2]: - node.query(""" + node.query( + """ CREATE TABLE test_table( APIKey UInt32, CustomAttributeId UInt64, @@ -22,7 +23,8 @@ def start_cluster(): Data String) ENGINE = SummingMergeTree() ORDER BY (APIKey, CustomAttributeId, ProfileIDHash, DeviceIDHash, intHash32(DeviceIDHash)) - """) + """ + ) yield cluster finally: @@ -30,5 +32,9 @@ def start_cluster(): def test_remote(start_cluster): - assert node1.query( - "SELECT 1 FROM remote('node{1,2}', default.test_table) WHERE (APIKey = 137715) AND (CustomAttributeId IN (45, 66)) AND (ProfileIDHash != 0) LIMIT 1") == "" + assert ( + node1.query( + "SELECT 1 FROM remote('node{1,2}', default.test_table) WHERE (APIKey = 137715) AND (CustomAttributeId IN (45, 66)) AND (ProfileIDHash != 0) LIMIT 1" + ) + == "" + ) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 7269ee73d8e..33343da8f6d 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -1,5 +1,3 @@ - - import random import time from multiprocessing.dummy import Pool @@ -11,16 +9,21 @@ from helpers.cluster import ClickHouseCluster node_options = dict( with_zookeeper=True, - main_configs=["configs/remote_servers.xml", "configs/config.d/instant_moves.xml", - "configs/config.d/part_log.xml", "configs/config.d/zookeeper_session_timeout.xml", - "configs/config.d/storage_configuration.xml"], - tmpfs=['/external:size=200M', '/internal:size=1M']) + main_configs=[ + "configs/remote_servers.xml", + "configs/config.d/instant_moves.xml", + "configs/config.d/part_log.xml", + "configs/config.d/zookeeper_session_timeout.xml", + "configs/config.d/storage_configuration.xml", + ], + tmpfs=["/external:size=200M", "/internal:size=1M"], +) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', macros={"shard": 0, "replica": 1}, **node_options) -node2 = cluster.add_instance('node2', macros={"shard": 0, "replica": 2}, **node_options) -node3 = cluster.add_instance('node3', macros={"shard": 1, "replica": 1}, **node_options) -node4 = cluster.add_instance('node4', macros={"shard": 1, "replica": 2}, **node_options) +node1 = cluster.add_instance("node1", macros={"shard": 0, "replica": 1}, **node_options) +node2 = cluster.add_instance("node2", macros={"shard": 0, "replica": 2}, **node_options) +node3 = cluster.add_instance("node3", macros={"shard": 1, "replica": 1}, **node_options) +node4 = cluster.add_instance("node4", macros={"shard": 1, "replica": 2}, **node_options) nodes = [node1, node2, node3, node4] @@ -40,8 +43,14 @@ def drop_table(nodes, table_name): node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) -def create_table(nodes, table_name, with_storage_policy=False, with_time_column=False, - with_ttl_move=False, with_ttl_delete=False): +def create_table( + nodes, + table_name, + with_storage_policy=False, + with_time_column=False, + with_ttl_move=False, + with_ttl_delete=False, +): extra_columns = "" settings = [] @@ -71,13 +80,19 @@ def create_table(nodes, table_name, with_storage_policy=False, with_time_column= if settings: sql += """ SETTINGS {} - """.format(", ".join(settings)) + """.format( + ", ".join(settings) + ) if with_time_column: extra_columns = """, time DateTime """ - node.query(sql.format(table_name=table_name, replica=node.name, extra_columns=extra_columns)) + node.query( + sql.format( + table_name=table_name, replica=node.name, extra_columns=extra_columns + ) + ) def create_distributed_table(node, table_name): @@ -89,25 +104,45 @@ def create_distributed_table(node, table_name): ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/%(table_name)s_replicated', '{replica}') ORDER BY num PARTITION BY num %% 100; - """ % dict(table_name=table_name) + """ % dict( + table_name=table_name + ) node.query(sql) sql = """ CREATE TABLE %(table_name)s ON CLUSTER test_cluster AS %(table_name)s_replicated ENGINE = Distributed(test_cluster, default, %(table_name)s_replicated, rand()) - """ % dict(table_name=table_name) + """ % dict( + table_name=table_name + ) node.query(sql) def drop_distributed_table(node, table_name): - node.query("DROP TABLE IF EXISTS {} ON CLUSTER test_cluster SYNC".format(table_name)) - node.query("DROP TABLE IF EXISTS {}_replicated ON CLUSTER test_cluster SYNC".format(table_name)) + node.query( + "DROP TABLE IF EXISTS {} ON CLUSTER test_cluster SYNC".format(table_name) + ) + node.query( + "DROP TABLE IF EXISTS {}_replicated ON CLUSTER test_cluster SYNC".format( + table_name + ) + ) time.sleep(1) -def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, - slow=False, with_many_parts=False, offset=0, with_time_column=False): +def insert( + node, + table_name, + chunk=1000, + col_names=None, + iterations=1, + ignore_exception=False, + slow=False, + with_many_parts=False, + offset=0, + with_time_column=False, +): if col_names is None: - col_names = ['num', 'num2'] + col_names = ["num", "num2"] for i in range(iterations): try: query = ["SET max_partitions_per_insert_block = 10000000"] @@ -115,25 +150,48 @@ def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_ex query.append("SET max_insert_block_size = 256") if with_time_column: query.append( - "INSERT INTO {table_name} ({col0}, {col1}, time) SELECT number AS {col0}, number + 1 AS {col1}, now() + 10 AS time FROM numbers_mt({chunk})" - .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + "INSERT INTO {table_name} ({col0}, {col1}, time) SELECT number AS {col0}, number + 1 AS {col1}, now() + 10 AS time FROM numbers_mt({chunk})".format( + table_name=table_name, + chunk=chunk, + col0=col_names[0], + col1=col_names[1], + ) + ) elif slow: query.append( - "INSERT INTO {table_name} ({col0}, {col1}) SELECT number + sleepEachRow(0.001) AS {col0}, number + 1 AS {col1} FROM numbers_mt({chunk})" - .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + "INSERT INTO {table_name} ({col0}, {col1}) SELECT number + sleepEachRow(0.001) AS {col0}, number + 1 AS {col1} FROM numbers_mt({chunk})".format( + table_name=table_name, + chunk=chunk, + col0=col_names[0], + col1=col_names[1], + ) + ) else: query.append( - "INSERT INTO {table_name} ({col0},{col1}) SELECT number + {offset} AS {col0}, number + 1 + {offset} AS {col1} FROM numbers_mt({chunk})" - .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1], - offset=str(offset))) + "INSERT INTO {table_name} ({col0},{col1}) SELECT number + {offset} AS {col0}, number + 1 + {offset} AS {col1} FROM numbers_mt({chunk})".format( + table_name=table_name, + chunk=chunk, + col0=col_names[0], + col1=col_names[1], + offset=str(offset), + ) + ) node.query(";\n".join(query)) except QueryRuntimeException as ex: if not ignore_exception: raise -def select(node, table_name, col_name="num", expected_result=None, iterations=1, ignore_exception=False, slow=False, - poll=None): +def select( + node, + table_name, + col_name="num", + expected_result=None, + iterations=1, + ignore_exception=False, + slow=False, + poll=None, +): for i in range(iterations): start_time = time.time() while True: @@ -141,11 +199,21 @@ def select(node, table_name, col_name="num", expected_result=None, iterations=1, if slow: r = node.query( "SELECT count() FROM (SELECT num2, sleepEachRow(0.5) FROM {} WHERE {} % 1000 > 0)".format( - table_name, col_name)) + table_name, col_name + ) + ) else: - r = node.query("SELECT count() FROM {} WHERE {} % 1000 > 0".format(table_name, col_name)) + r = node.query( + "SELECT count() FROM {} WHERE {} % 1000 > 0".format( + table_name, col_name + ) + ) if expected_result: - if r != expected_result and poll and time.time() - start_time < poll: + if ( + r != expected_result + and poll + and time.time() - start_time < poll + ): continue assert r == expected_result except QueryRuntimeException as ex: @@ -154,23 +222,31 @@ def select(node, table_name, col_name="num", expected_result=None, iterations=1, break -def rename_column(node, table_name, name, new_name, iterations=1, ignore_exception=False): +def rename_column( + node, table_name, name, new_name, iterations=1, ignore_exception=False +): for i in range(iterations): try: - node.query("ALTER TABLE {table_name} RENAME COLUMN {name} to {new_name}".format( - table_name=table_name, name=name, new_name=new_name - )) + node.query( + "ALTER TABLE {table_name} RENAME COLUMN {name} to {new_name}".format( + table_name=table_name, name=name, new_name=new_name + ) + ) except QueryRuntimeException as ex: if not ignore_exception: raise -def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ignore_exception=False): +def rename_column_on_cluster( + node, table_name, name, new_name, iterations=1, ignore_exception=False +): for i in range(iterations): try: - node.query("ALTER TABLE {table_name} ON CLUSTER test_cluster RENAME COLUMN {name} to {new_name}".format( - table_name=table_name, name=name, new_name=new_name - )) + node.query( + "ALTER TABLE {table_name} ON CLUSTER test_cluster RENAME COLUMN {name} to {new_name}".format( + table_name=table_name, name=name, new_name=new_name + ) + ) except QueryRuntimeException as ex: if not ignore_exception: raise @@ -179,10 +255,13 @@ def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ign def alter_move(node, table_name, iterations=1, ignore_exception=False): for i in range(iterations): move_part = random.randint(0, 99) - move_volume = 'external' + move_volume = "external" try: - node.query("ALTER TABLE {table_name} MOVE PARTITION '{move_part}' TO VOLUME '{move_volume}'" - .format(table_name=table_name, move_part=move_part, move_volume=move_volume)) + node.query( + "ALTER TABLE {table_name} MOVE PARTITION '{move_part}' TO VOLUME '{move_volume}'".format( + table_name=table_name, move_part=move_part, move_volume=move_volume + ) + ) except QueryRuntimeException as ex: if not ignore_exception: raise @@ -198,9 +277,21 @@ def test_rename_parallel_same_node(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) - tasks.append(p.apply_async(rename_column, (node1, table_name, "foo2", "foo3", 5, True))) - tasks.append(p.apply_async(rename_column, (node1, table_name, "foo3", "num2", 5, True))) + tasks.append( + p.apply_async( + rename_column, (node1, table_name, "num2", "foo2", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node1, table_name, "foo2", "foo3", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node1, table_name, "foo3", "num2", 5, True) + ) + ) for task in tasks: task.get(timeout=240) @@ -224,9 +315,21 @@ def test_rename_parallel(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append( + p.apply_async( + rename_column, (node1, table_name, "num2", "foo2", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node2, table_name, "foo2", "foo3", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node3, table_name, "foo3", "num2", 5, True) + ) + ) for task in tasks: task.get(timeout=240) @@ -254,12 +357,30 @@ def test_rename_with_parallel_select(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) - tasks.append(p.apply_async(select, (node1, table_name, "foo3", "999\n", 5, True))) - tasks.append(p.apply_async(select, (node2, table_name, "num2", "999\n", 5, True))) - tasks.append(p.apply_async(select, (node3, table_name, "foo2", "999\n", 5, True))) + tasks.append( + p.apply_async( + rename_column, (node1, table_name, "num2", "foo2", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node2, table_name, "foo2", "foo3", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node3, table_name, "foo3", "num2", 5, True) + ) + ) + tasks.append( + p.apply_async(select, (node1, table_name, "foo3", "999\n", 5, True)) + ) + tasks.append( + p.apply_async(select, (node2, table_name, "num2", "999\n", 5, True)) + ) + tasks.append( + p.apply_async(select, (node3, table_name, "foo2", "999\n", 5, True)) + ) for task in tasks: task.get(timeout=240) @@ -283,12 +404,36 @@ def test_rename_with_parallel_insert(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) - tasks.append(p.apply_async(insert, (node1, table_name, 100, ["num", "foo3"], 5, True))) - tasks.append(p.apply_async(insert, (node2, table_name, 100, ["num", "num2"], 5, True))) - tasks.append(p.apply_async(insert, (node3, table_name, 100, ["num", "foo2"], 5, True))) + tasks.append( + p.apply_async( + rename_column, (node1, table_name, "num2", "foo2", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node2, table_name, "foo2", "foo3", 5, True) + ) + ) + tasks.append( + p.apply_async( + rename_column, (node3, table_name, "foo3", "num2", 5, True) + ) + ) + tasks.append( + p.apply_async( + insert, (node1, table_name, 100, ["num", "foo3"], 5, True) + ) + ) + tasks.append( + p.apply_async( + insert, (node2, table_name, 100, ["num", "num2"], 5, True) + ) + ) + tasks.append( + p.apply_async( + insert, (node3, table_name, 100, ["num", "foo2"], 5, True) + ) + ) for task in tasks: task.get(timeout=240) @@ -309,7 +454,17 @@ def test_rename_with_parallel_merges(started_cluster): print("Creating tables", datetime.datetime.now()) create_table(nodes, table_name) for i in range(5): - insert(node1, table_name, 100, ["num", "num2"], 1, False, False, True, offset=i * 100) + insert( + node1, + table_name, + 100, + ["num", "num2"], + 1, + False, + False, + True, + offset=i * 100, + ) print("Data inserted", datetime.datetime.now()) @@ -323,9 +478,15 @@ def test_rename_with_parallel_merges(started_cluster): print("Creating pool") p = Pool(15) tasks = [] - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 2, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 2, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 2, True))) + tasks.append( + p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 2, True)) + ) + tasks.append( + p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 2, True)) + ) + tasks.append( + p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 2, True)) + ) tasks.append(p.apply_async(merge_parts, (node1, table_name, 2))) tasks.append(p.apply_async(merge_parts, (node2, table_name, 2))) tasks.append(p.apply_async(merge_parts, (node3, table_name, 2))) @@ -358,8 +519,16 @@ def test_rename_with_parallel_slow_insert(started_cluster): p = Pool(15) tasks = [] - tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, True))) - tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, True, True))) # deduplicated + tasks.append( + p.apply_async( + insert, (node1, table_name, 10000, ["num", "num2"], 1, False, True) + ) + ) + tasks.append( + p.apply_async( + insert, (node1, table_name, 10000, ["num", "num2"], 1, True, True) + ) + ) # deduplicated time.sleep(0.5) tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) @@ -380,30 +549,64 @@ def test_rename_with_parallel_slow_insert(started_cluster): def test_rename_with_parallel_ttl_move(started_cluster): - table_name = 'test_rename_with_parallel_ttl_move' + table_name = "test_rename_with_parallel_ttl_move" try: - create_table(nodes, table_name, with_storage_policy=True, with_time_column=True, with_ttl_move=True) + create_table( + nodes, + table_name, + with_storage_policy=True, + with_time_column=True, + with_ttl_move=True, + ) rename_column(node1, table_name, "time", "time2", 1, False) rename_column(node1, table_name, "time2", "time", 1, False) p = Pool(15) tasks = [] - tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) + tasks.append( + p.apply_async( + insert, + ( + node1, + table_name, + 10000, + ["num", "num2"], + 1, + False, + False, + True, + 0, + True, + ), + ) + ) time.sleep(5) rename_column(node1, table_name, "time", "time2", 1, False) time.sleep(4) - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 5, True))) + tasks.append( + p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True)) + ) + tasks.append( + p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True)) + ) + tasks.append( + p.apply_async(rename_column, (node3, table_name, "num3", "num2", 5, True)) + ) for task in tasks: task.get(timeout=240) # check some parts got moved - assert "external" in set(node1.query( - "SELECT disk_name FROM system.parts WHERE table == '{}' AND active=1 ORDER BY modification_time".format( - table_name)).strip().splitlines()) + assert "external" in set( + node1.query( + "SELECT disk_name FROM system.parts WHERE table == '{}' AND active=1 ORDER BY modification_time".format( + table_name + ) + ) + .strip() + .splitlines() + ) # rename column back to original rename_column(node1, table_name, "foo2", "num2", 1, True) @@ -416,7 +619,7 @@ def test_rename_with_parallel_ttl_move(started_cluster): def test_rename_with_parallel_ttl_delete(started_cluster): - table_name = 'test_rename_with_parallel_ttl_delete' + table_name = "test_rename_with_parallel_ttl_delete" try: create_table(nodes, table_name, with_time_column=True, with_ttl_delete=True) rename_column(node1, table_name, "time", "time2", 1, False) @@ -429,11 +632,33 @@ def test_rename_with_parallel_ttl_delete(started_cluster): p = Pool(15) tasks = [] - tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) + tasks.append( + p.apply_async( + insert, + ( + node1, + table_name, + 10000, + ["num", "num2"], + 1, + False, + False, + True, + 0, + True, + ), + ) + ) time.sleep(15) - tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) - tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) - tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 5, True))) + tasks.append( + p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True)) + ) + tasks.append( + p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True)) + ) + tasks.append( + p.apply_async(rename_column, (node3, table_name, "num3", "num2", 5, True)) + ) tasks.append(p.apply_async(merge_parts, (node1, table_name, 3))) tasks.append(p.apply_async(merge_parts, (node2, table_name, 3))) tasks.append(p.apply_async(merge_parts, (node3, table_name, 3))) @@ -445,29 +670,32 @@ def test_rename_with_parallel_ttl_delete(started_cluster): rename_column(node1, table_name, "foo2", "num2", 1, True) rename_column(node1, table_name, "foo3", "num2", 1, True) - assert int(node1.query("SELECT count() FROM {}".format(table_name)).strip()) < 10000 + assert ( + int(node1.query("SELECT count() FROM {}".format(table_name)).strip()) + < 10000 + ) finally: drop_table(nodes, table_name) def test_rename_distributed(started_cluster): - table_name = 'test_rename_distributed' + table_name = "test_rename_distributed" try: create_distributed_table(node1, table_name) insert(node1, table_name, 1000) - rename_column_on_cluster(node1, table_name, 'num2', 'foo2') - rename_column_on_cluster(node1, '%s_replicated' % table_name, 'num2', 'foo2') + rename_column_on_cluster(node1, table_name, "num2", "foo2") + rename_column_on_cluster(node1, "%s_replicated" % table_name, "num2", "foo2") - insert(node1, table_name, 1000, col_names=['num', 'foo2']) + insert(node1, table_name, 1000, col_names=["num", "foo2"]) - select(node1, table_name, "foo2", '1998\n', poll=30) + select(node1, table_name, "foo2", "1998\n", poll=30) finally: drop_distributed_table(node1, table_name) def test_rename_distributed_parallel_insert_and_select(started_cluster): - table_name = 'test_rename_distributed_parallel_insert_and_select' + table_name = "test_rename_distributed_parallel_insert_and_select" try: create_distributed_table(node1, table_name) insert(node1, table_name, 1000) @@ -475,30 +703,73 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): p = Pool(15) tasks = [] for i in range(1): - tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'num2', 'foo2', 3, True))) tasks.append( - p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 3, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo2', 'foo3', 3, True))) + p.apply_async( + rename_column_on_cluster, + (node1, table_name, "num2", "foo2", 3, True), + ) + ) tasks.append( - p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 3, True))) - tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo3', 'num2', 3, True))) + p.apply_async( + rename_column_on_cluster, + (node1, "%s_replicated" % table_name, "num2", "foo2", 3, True), + ) + ) tasks.append( - p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 3, True))) - tasks.append(p.apply_async(insert, (node1, table_name, 10, ["num", "foo3"], 5, True))) - tasks.append(p.apply_async(insert, (node2, table_name, 10, ["num", "num2"], 5, True))) - tasks.append(p.apply_async(insert, (node3, table_name, 10, ["num", "foo2"], 5, True))) - tasks.append(p.apply_async(select, (node1, table_name, "foo2", None, 5, True))) - tasks.append(p.apply_async(select, (node2, table_name, "foo3", None, 5, True))) - tasks.append(p.apply_async(select, (node3, table_name, "num2", None, 5, True))) + p.apply_async( + rename_column_on_cluster, + (node1, table_name, "foo2", "foo3", 3, True), + ) + ) + tasks.append( + p.apply_async( + rename_column_on_cluster, + (node1, "%s_replicated" % table_name, "foo2", "foo3", 3, True), + ) + ) + tasks.append( + p.apply_async( + rename_column_on_cluster, + (node1, table_name, "foo3", "num2", 3, True), + ) + ) + tasks.append( + p.apply_async( + rename_column_on_cluster, + (node1, "%s_replicated" % table_name, "foo3", "num2", 3, True), + ) + ) + tasks.append( + p.apply_async(insert, (node1, table_name, 10, ["num", "foo3"], 5, True)) + ) + tasks.append( + p.apply_async(insert, (node2, table_name, 10, ["num", "num2"], 5, True)) + ) + tasks.append( + p.apply_async(insert, (node3, table_name, 10, ["num", "foo2"], 5, True)) + ) + tasks.append( + p.apply_async(select, (node1, table_name, "foo2", None, 5, True)) + ) + tasks.append( + p.apply_async(select, (node2, table_name, "foo3", None, 5, True)) + ) + tasks.append( + p.apply_async(select, (node3, table_name, "num2", None, 5, True)) + ) for task in tasks: task.get(timeout=240) - rename_column_on_cluster(node1, table_name, 'foo2', 'num2', 1, True) - rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo2', 'num2', 1, True) - rename_column_on_cluster(node1, table_name, 'foo3', 'num2', 1, True) - rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo3', 'num2', 1, True) + rename_column_on_cluster(node1, table_name, "foo2", "num2", 1, True) + rename_column_on_cluster( + node1, "%s_replicated" % table_name, "foo2", "num2", 1, True + ) + rename_column_on_cluster(node1, table_name, "foo3", "num2", 1, True) + rename_column_on_cluster( + node1, "%s_replicated" % table_name, "foo3", "num2", 1, True + ) - insert(node1, table_name, 1000, col_names=['num', 'num2']) + insert(node1, table_name, 1000, col_names=["num", "num2"]) select(node1, table_name, "num2") select(node2, table_name, "num2") select(node3, table_name, "num2") diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index d30a038825f..7ce79d9aca8 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -16,7 +16,7 @@ cluster = ClickHouseCluster(__file__) def _fill_nodes(nodes, shard): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE real_table(date Date, id UInt32, dummy UInt32) @@ -27,11 +27,18 @@ def _fill_nodes(nodes, shard): CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -51,18 +58,22 @@ def test_normal_work(normal_work): node1.query("insert into test_table values ('2017-06-16', 111, 0)") node1.query("insert into real_table values ('2017-06-16', 222, 0)") - assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node1, "SELECT id FROM real_table order by id", '222') - assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node1, "SELECT id FROM real_table order by id", "222") + assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", "111") node1.query("ALTER TABLE test_table REPLACE PARTITION 201706 FROM real_table") - assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", '222') - assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", '222') + assert_eq_with_retry(node1, "SELECT id FROM test_table order by id", "222") + assert_eq_with_retry(node2, "SELECT id FROM test_table order by id", "222") -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node3 = cluster.add_instance( + "node3", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node4 = cluster.add_instance( + "node4", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -82,9 +93,9 @@ def test_drop_failover(drop_failover): node3.query("insert into test_table values ('2017-06-16', 111, 0)") node3.query("insert into real_table values ('2017-06-16', 222, 0)") - assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node3, "SELECT id FROM real_table order by id", '222') - assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node3, "SELECT id FROM real_table order by id", "222") + assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", "111") with PartitionManager() as pm: # Hinder replication between replicas @@ -95,9 +106,9 @@ def test_drop_failover(drop_failover): node3.query("ALTER TABLE test_table REPLACE PARTITION 201706 FROM real_table") # Node3 replace is ok - assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", '222') + assert_eq_with_retry(node3, "SELECT id FROM test_table order by id", "222") # Network interrupted -- replace is not ok, but it's ok - assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", "111") # Drop partition on source node node3.query("ALTER TABLE test_table DROP PARTITION 201706") @@ -107,13 +118,19 @@ def test_drop_failover(drop_failover): msg = node4.query_with_retry( "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", - check_callback=lambda x: 'Not found part' not in x, sleep_time=1) - assert 'Not found part' not in msg - assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '') + check_callback=lambda x: "Not found part" not in x, + sleep_time=1, + ) + assert "Not found part" not in msg + assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", "") -node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node5 = cluster.add_instance( + "node5", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node6 = cluster.add_instance( + "node6", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -134,10 +151,10 @@ def test_replace_after_replace_failover(replace_after_replace_failover): node5.query("insert into real_table values ('2017-06-16', 222, 0)") node5.query("insert into other_table values ('2017-06-16', 333, 0)") - assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '111') - assert_eq_with_retry(node5, "SELECT id FROM real_table order by id", '222') - assert_eq_with_retry(node5, "SELECT id FROM other_table order by id", '333') - assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", "111") + assert_eq_with_retry(node5, "SELECT id FROM real_table order by id", "222") + assert_eq_with_retry(node5, "SELECT id FROM other_table order by id", "333") + assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", "111") with PartitionManager() as pm: # Hinder replication between replicas @@ -148,20 +165,22 @@ def test_replace_after_replace_failover(replace_after_replace_failover): node5.query("ALTER TABLE test_table REPLACE PARTITION 201706 FROM real_table") # Node5 replace is ok - assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '222') + assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", "222") # Network interrupted -- replace is not ok, but it's ok - assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '111') + assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", "111") # Replace partition on source node node5.query("ALTER TABLE test_table REPLACE PARTITION 201706 FROM other_table") - assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '333') + assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", "333") # Wait few seconds for connection to zookeeper to be restored time.sleep(5) msg = node6.query_with_retry( "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", - check_callback=lambda x: 'Not found part' not in x, sleep_time=1) - assert 'Not found part' not in msg - assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '333') + check_callback=lambda x: "Not found part" not in x, + sleep_time=1, + ) + assert "Not found part" not in msg + assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", "333") diff --git a/tests/integration/test_replica_can_become_leader/test.py b/tests/integration/test_replica_can_become_leader/test.py index fae4fa28226..58e7b6f6e19 100644 --- a/tests/integration/test_replica_can_become_leader/test.py +++ b/tests/integration/test_replica_can_become_leader/test.py @@ -3,9 +3,13 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/notleader.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/notleaderignorecase.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/notleader.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/notleaderignorecase.xml"], with_zookeeper=True +) +node3 = cluster.add_instance("node3", with_zookeeper=True) @pytest.fixture(scope="module") @@ -15,20 +19,24 @@ def start_cluster(): for i, node in enumerate((node1, node2)): node.query( - ''' + """ CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table', '{}') PARTITION BY date ORDER BY id - '''.format(i) + """.format( + i + ) ) with pytest.raises(QueryRuntimeException): node3.query( - ''' + """ CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table', '{}') PARTITION BY date ORDER BY id SETTINGS replicated_can_become_leader=0sad - '''.format(3) + """.format( + 3 + ) ) yield cluster @@ -38,5 +46,15 @@ def start_cluster(): def test_can_become_leader(start_cluster): - assert node1.query("select can_become_leader from system.replicas where table = 'test_table'") == '0\n' - assert node2.query("select can_become_leader from system.replicas where table = 'test_table'") == '0\n' + assert ( + node1.query( + "select can_become_leader from system.replicas where table = 'test_table'" + ) + == "0\n" + ) + assert ( + node2.query( + "select can_become_leader from system.replicas where table = 'test_table'" + ) + == "0\n" + ) diff --git a/tests/integration/test_replica_is_active/test.py b/tests/integration/test_replica_is_active/test.py index f786ff71958..d5e0931dff2 100644 --- a/tests/integration/test_replica_is_active/test.py +++ b/tests/integration/test_replica_is_active/test.py @@ -4,9 +4,10 @@ from helpers.cluster import ClickHouseCluster from ast import literal_eval cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) +node3 = cluster.add_instance("node3", with_zookeeper=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -14,13 +15,15 @@ def start_cluster(): cluster.start() for i, node in enumerate((node1, node2, node3)): - node_name = 'node' + str(i + 1) + node_name = "node" + str(i + 1) node.query( - ''' + """ CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_table', '{}') PARTITION BY date ORDER BY id - '''.format(node_name) + """.format( + node_name + ) ) yield cluster @@ -30,13 +33,19 @@ def start_cluster(): def test_replica_is_active(start_cluster): - query_result = node1.query("select replica_is_active from system.replicas where table = 'test_table'") - assert literal_eval(query_result) == {'node1': 1, 'node2': 1, 'node3': 1} + query_result = node1.query( + "select replica_is_active from system.replicas where table = 'test_table'" + ) + assert literal_eval(query_result) == {"node1": 1, "node2": 1, "node3": 1} node3.stop() - query_result = node1.query("select replica_is_active from system.replicas where table = 'test_table'") - assert literal_eval(query_result) == {'node1': 1, 'node2': 1, 'node3': 0} + query_result = node1.query( + "select replica_is_active from system.replicas where table = 'test_table'" + ) + assert literal_eval(query_result) == {"node1": 1, "node2": 1, "node3": 0} node2.stop() - query_result = node1.query("select replica_is_active from system.replicas where table = 'test_table'") - assert literal_eval(query_result) == {'node1': 1, 'node2': 0, 'node3': 0} + query_result = node1.query( + "select replica_is_active from system.replicas where table = 'test_table'" + ) + assert literal_eval(query_result) == {"node1": 1, "node2": 0, "node3": 0} diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 171ae24b98d..13e9c225a61 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -12,21 +12,61 @@ test_recover_staled_replica_run = 1 cluster = ClickHouseCluster(__file__) -main_node = cluster.add_instance('main_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, stay_alive=True, macros={"shard": 1, "replica": 1}) -dummy_node = cluster.add_instance('dummy_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, stay_alive=True, macros={"shard": 1, "replica": 2}) -competing_node = cluster.add_instance('competing_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, macros={"shard": 1, "replica": 3}) -snapshotting_node = cluster.add_instance('snapshotting_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True, macros={"shard": 2, "replica": 1}) -snapshot_recovering_node = cluster.add_instance('snapshot_recovering_node', main_configs=['configs/config.xml'], user_configs=['configs/settings.xml'], with_zookeeper=True) +main_node = cluster.add_instance( + "main_node", + main_configs=["configs/config.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 1, "replica": 1}, +) +dummy_node = cluster.add_instance( + "dummy_node", + main_configs=["configs/config.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 1, "replica": 2}, +) +competing_node = cluster.add_instance( + "competing_node", + main_configs=["configs/config.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + macros={"shard": 1, "replica": 3}, +) +snapshotting_node = cluster.add_instance( + "snapshotting_node", + main_configs=["configs/config.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + macros={"shard": 2, "replica": 1}, +) +snapshot_recovering_node = cluster.add_instance( + "snapshot_recovering_node", + main_configs=["configs/config.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, +) -all_nodes = [main_node, dummy_node, competing_node, snapshotting_node, snapshot_recovering_node] +all_nodes = [ + main_node, + dummy_node, + competing_node, + snapshotting_node, + snapshot_recovering_node, +] uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + def assert_create_query(nodes, table_name, expected): replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) query = "show create table {}".format(table_name) for node in nodes: assert_eq_with_retry(node, query, expected, get_result=replace_uuid) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -36,103 +76,182 @@ def started_cluster(): finally: cluster.shutdown() + def test_create_replicated_table(started_cluster): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica' || '1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") - assert "Explicit zookeeper_path and replica_name are specified" in \ - main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " - "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica' || '1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) + assert ( + "Explicit zookeeper_path and replica_name are specified" + in main_node.query_and_get_error( + "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" + ) + ) - assert "Explicit zookeeper_path and replica_name are specified" in \ - main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " - "ENGINE=ReplicatedMergeTree('/test/tmp', 'r', d, k, 8192);") + assert ( + "Explicit zookeeper_path and replica_name are specified" + in main_node.query_and_get_error( + "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "ENGINE=ReplicatedMergeTree('/test/tmp', 'r', d, k, 8192);" + ) + ) - assert "Old syntax is not allowed" in \ - main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " - "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);") + assert "Old syntax is not allowed" in main_node.query_and_get_error( + "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) " + "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" + ) - main_node.query("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);") + main_node.query( + "CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" + ) - expected = "CREATE TABLE testdb.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" \ - "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\n" \ - "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" + expected = ( + "CREATE TABLE testdb.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" + "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\n" + "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" + ) assert_create_query([main_node, dummy_node], "testdb.replicated_table", expected) # assert without replacing uuid - assert main_node.query("show create testdb.replicated_table") == dummy_node.query("show create testdb.replicated_table") + assert main_node.query("show create testdb.replicated_table") == dummy_node.query( + "show create testdb.replicated_table" + ) main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") -@pytest.mark.parametrize("engine", ['MergeTree', 'ReplicatedMergeTree']) + +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_simple_alter_table(started_cluster, engine): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) # test_simple_alter_table - name = "testdb.alter_test_{}".format(engine) - main_node.query("CREATE TABLE {} " - "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " - "ENGINE = {} PARTITION BY StartDate ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);".format(name, engine)) + name = "testdb.alter_test_{}".format(engine) + main_node.query( + "CREATE TABLE {} " + "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " + "ENGINE = {} PARTITION BY StartDate ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);".format( + name, engine + ) + ) main_node.query("ALTER TABLE {} ADD COLUMN Added0 UInt32;".format(name)) main_node.query("ALTER TABLE {} ADD COLUMN Added2 UInt32;".format(name)) - main_node.query("ALTER TABLE {} ADD COLUMN Added1 UInt32 AFTER Added0;".format(name)) - main_node.query("ALTER TABLE {} ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;".format(name)) - main_node.query("ALTER TABLE {} ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;".format(name)) - main_node.query("ALTER TABLE {} ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;".format(name)) + main_node.query( + "ALTER TABLE {} ADD COLUMN Added1 UInt32 AFTER Added0;".format(name) + ) + main_node.query( + "ALTER TABLE {} ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;".format( + name + ) + ) + main_node.query( + "ALTER TABLE {} ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;".format( + name + ) + ) + main_node.query( + "ALTER TABLE {} ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;".format( + name + ) + ) - full_engine = engine if not "Replicated" in engine else engine + "(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')" - expected = "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" \ - " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" \ - " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n" \ - " `AddedNested1.A` Array(UInt32),\\n `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n" \ - " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64)\\n)\\n" \ - "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" \ - "SETTINGS index_granularity = 8192".format(name, full_engine) + full_engine = ( + engine + if not "Replicated" in engine + else engine + "(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')" + ) + expected = ( + "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" + " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n" + " `AddedNested1.A` Array(UInt32),\\n `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n" + " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64)\\n)\\n" + "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" + "SETTINGS index_granularity = 8192".format(name, full_engine) + ) assert_create_query([main_node, dummy_node], name, expected) # test_create_replica_after_delay - competing_node.query("CREATE DATABASE IF NOT EXISTS testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');") + competing_node.query( + "CREATE DATABASE IF NOT EXISTS testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');" + ) - name = "testdb.alter_test_{}".format(engine) + name = "testdb.alter_test_{}".format(engine) main_node.query("ALTER TABLE {} ADD COLUMN Added3 UInt32;".format(name)) main_node.query("ALTER TABLE {} DROP COLUMN AddedNested1;".format(name)) main_node.query("ALTER TABLE {} RENAME COLUMN Added1 TO AddedNested1;".format(name)) - full_engine = engine if not "Replicated" in engine else engine + "(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')" - expected = "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" \ - " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" \ - " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `AddedNested1` UInt32,\\n `Added2` UInt32,\\n" \ - " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64),\\n `Added3` UInt32\\n)\\n" \ - "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" \ - "SETTINGS index_granularity = 8192".format(name, full_engine) + full_engine = ( + engine + if not "Replicated" in engine + else engine + "(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')" + ) + expected = ( + "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" + " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `AddedNested1` UInt32,\\n `Added2` UInt32,\\n" + " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64),\\n `Added3` UInt32\\n)\\n" + "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" + "SETTINGS index_granularity = 8192".format(name, full_engine) + ) assert_create_query([main_node, dummy_node, competing_node], name, expected) main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") competing_node.query("DROP DATABASE testdb SYNC") + def get_table_uuid(database, name): - return main_node.query(f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'").strip() + return main_node.query( + f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'" + ).strip() + @pytest.fixture(scope="module", name="attachable_part") def fixture_attachable_part(started_cluster): main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") - main_node.query(f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)") + main_node.query( + f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)" + ) main_node.query(f"INSERT INTO testdb_attach_atomic.test VALUES (123)") - main_node.query(f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'") + main_node.query( + f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'" + ) table_uuid = get_table_uuid("testdb_attach_atomic", "test") - return os.path.join(main_node.path, f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0") + return os.path.join( + main_node.path, + f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0", + ) + @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_attach(started_cluster, attachable_part, engine): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) - name = "alter_attach_test_{}".format(engine) - main_node.query(f"CREATE TABLE testdb.{name} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + name = "alter_attach_test_{}".format(engine) + main_node.query( + f"CREATE TABLE testdb.{name} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) table_uuid = get_table_uuid("testdb", name) # Provide and attach a part to the main node shutil.copytree( - attachable_part, os.path.join(main_node.path, f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0") + attachable_part, + os.path.join( + main_node.path, + f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0", + ), ) main_node.query(f"ALTER TABLE testdb.{name} ATTACH PART 'all_1_1_0'") # On the main node, data is attached @@ -145,14 +264,21 @@ def test_alter_attach(started_cluster, attachable_part, engine): main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") + @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_part(started_cluster, engine): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) table = f"alter_drop_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" - main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + main_node.query( + f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") if engine == "MergeTree": dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") @@ -166,14 +292,21 @@ def test_alter_drop_part(started_cluster, engine): main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") + @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_detach_part(started_cluster, engine): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) table = f"alter_detach_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" - main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + main_node.query( + f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") if engine == "MergeTree": dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") @@ -188,14 +321,21 @@ def test_alter_detach_part(started_cluster, engine): main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") + @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) def test_alter_drop_detached_part(started_cluster, engine): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) table = f"alter_drop_detached_{engine}" part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" - main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + main_node.query( + f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" + ) main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") main_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") if engine == "MergeTree": @@ -211,14 +351,24 @@ def test_alter_drop_detached_part(started_cluster, engine): def test_alter_fetch(started_cluster): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) - main_node.query("CREATE TABLE testdb.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") - main_node.query("CREATE TABLE testdb.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") + main_node.query( + "CREATE TABLE testdb.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" + ) + main_node.query( + "CREATE TABLE testdb.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" + ) main_node.query("INSERT INTO testdb.fetch_source VALUES (123)") table_uuid = get_table_uuid("testdb", "fetch_source") - main_node.query(f"ALTER TABLE testdb.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' ") + main_node.query( + f"ALTER TABLE testdb.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " + ) detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='testdb' AND table='fetch_target'" assert main_node.query(detached_parts_query) == "all_0_0_0\n" assert dummy_node.query(detached_parts_query) == "" @@ -226,91 +376,153 @@ def test_alter_fetch(started_cluster): main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") + def test_alters_from_different_replicas(started_cluster): - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") - dummy_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) + dummy_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica2');" + ) # test_alters_from_different_replicas - competing_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');") + competing_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');" + ) - main_node.query("CREATE TABLE testdb.concurrent_test " - "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " - "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);") + main_node.query( + "CREATE TABLE testdb.concurrent_test " + "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " + "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);" + ) - main_node.query("CREATE TABLE testdb.dist AS testdb.concurrent_test ENGINE = Distributed(testdb, testdb, concurrent_test, CounterID)") + main_node.query( + "CREATE TABLE testdb.dist AS testdb.concurrent_test ENGINE = Distributed(testdb, testdb, concurrent_test, CounterID)" + ) dummy_node.stop_clickhouse(kill=True) settings = {"distributed_ddl_task_timeout": 5} - assert "There are 1 unfinished hosts (0 of them are currently active)" in \ - competing_node.query_and_get_error("ALTER TABLE testdb.concurrent_test ADD COLUMN Added0 UInt32;", settings=settings) - settings = {"distributed_ddl_task_timeout": 5, "distributed_ddl_output_mode": "null_status_on_timeout"} - assert "shard1|replica2\t\\N\t\\N" in \ - main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN Added2 UInt32;", settings=settings) - settings = {"distributed_ddl_task_timeout": 5, "distributed_ddl_output_mode": "never_throw"} - assert "shard1|replica2\t\\N\t\\N" in \ - competing_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", settings=settings) + assert ( + "There are 1 unfinished hosts (0 of them are currently active)" + in competing_node.query_and_get_error( + "ALTER TABLE testdb.concurrent_test ADD COLUMN Added0 UInt32;", + settings=settings, + ) + ) + settings = { + "distributed_ddl_task_timeout": 5, + "distributed_ddl_output_mode": "null_status_on_timeout", + } + assert "shard1|replica2\t\\N\t\\N" in main_node.query( + "ALTER TABLE testdb.concurrent_test ADD COLUMN Added2 UInt32;", + settings=settings, + ) + settings = { + "distributed_ddl_task_timeout": 5, + "distributed_ddl_output_mode": "never_throw", + } + assert "shard1|replica2\t\\N\t\\N" in competing_node.query( + "ALTER TABLE testdb.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", + settings=settings, + ) dummy_node.start_clickhouse() - main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;") - competing_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;") - main_node.query("ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;") + main_node.query( + "ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" + ) + competing_node.query( + "ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" + ) + main_node.query( + "ALTER TABLE testdb.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" + ) - expected = "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" \ - " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" \ - " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" \ - " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" \ - " `AddedNested2.B` Array(UInt64)\\n)\\n" \ - "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192)" + expected = ( + "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" + " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" + " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" + " `AddedNested2.B` Array(UInt64)\\n)\\n" + "ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192)" + ) assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) # test_create_replica_after_delay main_node.query("DROP TABLE testdb.concurrent_test SYNC") - main_node.query("CREATE TABLE testdb.concurrent_test " - "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " - "ENGINE = ReplicatedMergeTree ORDER BY CounterID;") + main_node.query( + "CREATE TABLE testdb.concurrent_test " + "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " + "ENGINE = ReplicatedMergeTree ORDER BY CounterID;" + ) - expected = "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" \ - " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" \ - "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" + expected = ( + "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" + "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" + ) assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) - main_node.query("INSERT INTO testdb.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)") + main_node.query( + "INSERT INTO testdb.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" + ) # test_replica_restart main_node.restart_clickhouse() - expected = "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" \ - " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" \ - "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" - + expected = ( + "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" + "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" + ) # test_snapshot_and_snapshot_recover - snapshotting_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica1');") - snapshot_recovering_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica2');") + snapshotting_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica1');" + ) + snapshot_recovering_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard2', 'replica2');" + ) assert_create_query(all_nodes, "testdb.concurrent_test", expected) main_node.query("SYSTEM FLUSH DISTRIBUTED testdb.dist") - main_node.query("ALTER TABLE testdb.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1") + main_node.query( + "ALTER TABLE testdb.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" + ) res = main_node.query("ALTER TABLE testdb.concurrent_test DELETE WHERE UserID % 2") - assert "shard1|replica1" in res and "shard1|replica2" in res and "shard1|replica3" in res + assert ( + "shard1|replica1" in res + and "shard1|replica2" in res + and "shard1|replica3" in res + ) assert "shard2|replica1" in res and "shard2|replica2" in res - expected = "1\t1\tmain_node\n" \ - "1\t2\tdummy_node\n" \ - "1\t3\tcompeting_node\n" \ - "2\t1\tsnapshotting_node\n" \ - "2\t2\tsnapshot_recovering_node\n" - assert main_node.query("SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='testdb'") == expected + expected = ( + "1\t1\tmain_node\n" + "1\t2\tdummy_node\n" + "1\t3\tcompeting_node\n" + "2\t1\tsnapshotting_node\n" + "2\t2\tsnapshot_recovering_node\n" + ) + assert ( + main_node.query( + "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='testdb'" + ) + == expected + ) # test_drop_and_create_replica main_node.query("DROP DATABASE testdb SYNC") - main_node.query("CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');") + main_node.query( + "CREATE DATABASE testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica1');" + ) - expected = "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" \ - " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" \ - "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" + expected = ( + "CREATE TABLE testdb.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" + " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" + "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/uuid/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" + ) assert_create_query([main_node, competing_node], "testdb.concurrent_test", expected) assert_create_query(all_nodes, "testdb.concurrent_test", expected) @@ -318,112 +530,242 @@ def test_alters_from_different_replicas(started_cluster): for node in all_nodes: node.query("SYSTEM SYNC REPLICA testdb.concurrent_test") - expected = "0\t2021-02-02\t4249604106\n" \ - "1\t2021-02-03\t1343103100\n" \ - "4\t2021-02-06\t3902320246\n" \ - "7\t2021-02-09\t3844986530\n" \ - "9\t2021-02-11\t1241149650\n" + expected = ( + "0\t2021-02-02\t4249604106\n" + "1\t2021-02-03\t1343103100\n" + "4\t2021-02-06\t3902320246\n" + "7\t2021-02-09\t3844986530\n" + "9\t2021-02-11\t1241149650\n" + ) - assert_eq_with_retry(dummy_node, "SELECT CounterID, StartDate, UserID FROM testdb.dist ORDER BY CounterID", expected) + assert_eq_with_retry( + dummy_node, + "SELECT CounterID, StartDate, UserID FROM testdb.dist ORDER BY CounterID", + expected, + ) main_node.query("DROP DATABASE testdb SYNC") dummy_node.query("DROP DATABASE testdb SYNC") competing_node.query("DROP DATABASE testdb SYNC") snapshotting_node.query("DROP DATABASE testdb SYNC") snapshot_recovering_node.query("DROP DATABASE testdb SYNC") + def test_recover_staled_replica(started_cluster): - main_node.query("CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');") - started_cluster.get_kazoo_client('zoo1').set('/clickhouse/databases/recover/logs_to_keep', b'10') - dummy_node.query("CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica2');") + main_node.query( + "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');" + ) + started_cluster.get_kazoo_client("zoo1").set( + "/clickhouse/databases/recover/logs_to_keep", b"10" + ) + dummy_node.query( + "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica2');" + ) settings = {"distributed_ddl_task_timeout": 0} main_node.query("CREATE TABLE recover.t1 (n int) ENGINE=Memory", settings=settings) - dummy_node.query("CREATE TABLE recover.t2 (s String) ENGINE=Memory", settings=settings) - main_node.query("CREATE TABLE recover.mt1 (n int) ENGINE=MergeTree order by n", settings=settings) - dummy_node.query("CREATE TABLE recover.mt2 (n int) ENGINE=MergeTree order by n", settings=settings) - main_node.query("CREATE TABLE recover.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) - dummy_node.query("CREATE TABLE recover.rmt2 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) - main_node.query("CREATE TABLE recover.rmt3 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) - dummy_node.query("CREATE TABLE recover.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", settings=settings) - main_node.query("CREATE MATERIALIZED VIEW recover.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", settings=settings) - dummy_node.query("CREATE MATERIALIZED VIEW recover.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", settings=settings) - main_node.query("CREATE DICTIONARY recover.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " - "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())") - dummy_node.query("CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " - "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())") + dummy_node.query( + "CREATE TABLE recover.t2 (s String) ENGINE=Memory", settings=settings + ) + main_node.query( + "CREATE TABLE recover.mt1 (n int) ENGINE=MergeTree order by n", + settings=settings, + ) + dummy_node.query( + "CREATE TABLE recover.mt2 (n int) ENGINE=MergeTree order by n", + settings=settings, + ) + main_node.query( + "CREATE TABLE recover.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", + settings=settings, + ) + dummy_node.query( + "CREATE TABLE recover.rmt2 (n int) ENGINE=ReplicatedMergeTree order by n", + settings=settings, + ) + main_node.query( + "CREATE TABLE recover.rmt3 (n int) ENGINE=ReplicatedMergeTree order by n", + settings=settings, + ) + dummy_node.query( + "CREATE TABLE recover.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", + settings=settings, + ) + main_node.query( + "CREATE MATERIALIZED VIEW recover.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", + settings=settings, + ) + dummy_node.query( + "CREATE MATERIALIZED VIEW recover.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", + settings=settings, + ) + main_node.query( + "CREATE DICTIONARY recover.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" + ) + dummy_node.query( + "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" + ) - for table in ['t1', 't2', 'mt1', 'mt2', 'rmt1', 'rmt2', 'rmt3', 'rmt5']: + for table in ["t1", "t2", "mt1", "mt2", "rmt1", "rmt2", "rmt3", "rmt5"]: main_node.query("INSERT INTO recover.{} VALUES (42)".format(table)) - for table in ['t1', 't2', 'mt1', 'mt2']: + for table in ["t1", "t2", "mt1", "mt2"]: dummy_node.query("INSERT INTO recover.{} VALUES (42)".format(table)) - for table in ['rmt1', 'rmt2', 'rmt3', 'rmt5']: + for table in ["rmt1", "rmt2", "rmt3", "rmt5"]: main_node.query("SYSTEM SYNC REPLICA recover.{}".format(table)) with PartitionManager() as pm: pm.drop_instance_zk_connections(dummy_node) dummy_node.query_and_get_error("RENAME TABLE recover.t1 TO recover.m1") - main_node.query_with_retry("RENAME TABLE recover.t1 TO recover.m1", settings=settings) - main_node.query_with_retry("ALTER TABLE recover.mt1 ADD COLUMN m int", settings=settings) - main_node.query_with_retry("ALTER TABLE recover.rmt1 ADD COLUMN m int", settings=settings) - main_node.query_with_retry("RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings) + main_node.query_with_retry( + "RENAME TABLE recover.t1 TO recover.m1", settings=settings + ) + main_node.query_with_retry( + "ALTER TABLE recover.mt1 ADD COLUMN m int", settings=settings + ) + main_node.query_with_retry( + "ALTER TABLE recover.rmt1 ADD COLUMN m int", settings=settings + ) + main_node.query_with_retry( + "RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings + ) main_node.query_with_retry("DROP TABLE recover.rmt5", settings=settings) main_node.query_with_retry("DROP DICTIONARY recover.d2", settings=settings) - main_node.query_with_retry("CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " - "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " - "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", settings=settings) + main_node.query_with_retry( + "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " + "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " + "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", + settings=settings, + ) - inner_table = ".inner_id." + dummy_node.query_with_retry("SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'").strip() - main_node.query_with_retry("ALTER TABLE recover.`{}` MODIFY COLUMN n int DEFAULT 42".format(inner_table), settings=settings) - main_node.query_with_retry("ALTER TABLE recover.mv1 MODIFY QUERY SELECT m FROM recover.rmt1".format(inner_table), settings=settings) - main_node.query_with_retry("RENAME TABLE recover.mv2 TO recover.mv3".format(inner_table), settings=settings) + inner_table = ( + ".inner_id." + + dummy_node.query_with_retry( + "SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'" + ).strip() + ) + main_node.query_with_retry( + "ALTER TABLE recover.`{}` MODIFY COLUMN n int DEFAULT 42".format( + inner_table + ), + settings=settings, + ) + main_node.query_with_retry( + "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m FROM recover.rmt1".format( + inner_table + ), + settings=settings, + ) + main_node.query_with_retry( + "RENAME TABLE recover.mv2 TO recover.mv3".format(inner_table), + settings=settings, + ) - main_node.query_with_retry("CREATE TABLE recover.tmp AS recover.m1", settings=settings) + main_node.query_with_retry( + "CREATE TABLE recover.tmp AS recover.m1", settings=settings + ) main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) - main_node.query_with_retry("CREATE TABLE recover.tmp AS recover.m1", settings=settings) + main_node.query_with_retry( + "CREATE TABLE recover.tmp AS recover.m1", settings=settings + ) main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) - main_node.query_with_retry("CREATE TABLE recover.tmp AS recover.m1", settings=settings) + main_node.query_with_retry( + "CREATE TABLE recover.tmp AS recover.m1", settings=settings + ) - assert main_node.query("SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name") == \ - "d1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" - query = "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " \ - "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" + assert ( + main_node.query( + "SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name" + ) + == "d1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" + ) + query = ( + "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " + "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" + ) expected = main_node.query(query) assert_eq_with_retry(dummy_node, query, expected) - assert main_node.query("SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'") == "2\n" - assert dummy_node.query("SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'") == "2\n" + assert ( + main_node.query( + "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" + ) + == "2\n" + ) + assert ( + dummy_node.query( + "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" + ) + == "2\n" + ) - for table in ['m1', 't2', 'mt1', 'mt2', 'rmt1', 'rmt2', 'rmt4', 'd1', 'd2', 'mv1', 'mv3']: + for table in [ + "m1", + "t2", + "mt1", + "mt2", + "rmt1", + "rmt2", + "rmt4", + "d1", + "d2", + "mv1", + "mv3", + ]: assert main_node.query("SELECT (*,).1 FROM recover.{}".format(table)) == "42\n" - for table in ['t2', 'rmt1', 'rmt2', 'rmt4', 'd1', 'd2', 'mt2', 'mv1', 'mv3']: + for table in ["t2", "rmt1", "rmt2", "rmt4", "d1", "d2", "mt2", "mv1", "mv3"]: assert dummy_node.query("SELECT (*,).1 FROM recover.{}".format(table)) == "42\n" - for table in ['m1', 'mt1']: + for table in ["m1", "mt1"]: assert dummy_node.query("SELECT count() FROM recover.{}".format(table)) == "0\n" global test_recover_staled_replica_run - assert dummy_node.query("SELECT count() FROM system.tables WHERE database='recover_broken_tables'") == f"{2*test_recover_staled_replica_run}\n" + assert ( + dummy_node.query( + "SELECT count() FROM system.tables WHERE database='recover_broken_tables'" + ) + == f"{2*test_recover_staled_replica_run}\n" + ) test_recover_staled_replica_run += 1 - table = dummy_node.query("SHOW TABLES FROM recover_broken_tables LIKE 'mt1_29_%' LIMIT 1").strip() - assert dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) == "42\n" - table = dummy_node.query("SHOW TABLES FROM recover_broken_tables LIKE 'rmt5_29_%' LIMIT 1").strip() - assert dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) == "42\n" + table = dummy_node.query( + "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_29_%' LIMIT 1" + ).strip() + assert ( + dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) + == "42\n" + ) + table = dummy_node.query( + "SHOW TABLES FROM recover_broken_tables LIKE 'rmt5_29_%' LIMIT 1" + ).strip() + assert ( + dummy_node.query("SELECT (*,).1 FROM recover_broken_tables.{}".format(table)) + == "42\n" + ) expected = "Cleaned 6 outdated objects: dropped 1 dictionaries and 3 tables, moved 2 tables" assert_logs_contain(dummy_node, expected) dummy_node.query("DROP TABLE recover.tmp") - assert_eq_with_retry(main_node, "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", "0\n") + assert_eq_with_retry( + main_node, + "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", + "0\n", + ) main_node.query("DROP DATABASE recover SYNC") dummy_node.query("DROP DATABASE recover SYNC") + def test_startup_without_zk(started_cluster): with PartitionManager() as pm: pm.drop_instance_zk_connections(main_node) - err = main_node.query_and_get_error("CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');") + err = main_node.query_and_get_error( + "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" + ) assert "ZooKeeper" in err - main_node.query("CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');") - #main_node.query("CREATE TABLE startup.rmt (n int) ENGINE=ReplicatedMergeTree order by n") + main_node.query( + "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" + ) + # main_node.query("CREATE TABLE startup.rmt (n int) ENGINE=ReplicatedMergeTree order by n") main_node.query("CREATE TABLE startup.rmt (n int) ENGINE=MergeTree order by n") main_node.query("INSERT INTO startup.rmt VALUES (42)") with PartitionManager() as pm: @@ -442,6 +784,7 @@ def test_startup_without_zk(started_cluster): assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" main_node.query("DROP DATABASE startup SYNC") + def test_server_uuid(started_cluster): uuid1 = main_node.query("select serverUUID()") uuid2 = dummy_node.query("select serverUUID()") diff --git a/tests/integration/test_replicated_fetches_bandwidth/test.py b/tests/integration/test_replicated_fetches_bandwidth/test.py index f39baea064c..059102f8683 100644 --- a/tests/integration/test_replicated_fetches_bandwidth/test.py +++ b/tests/integration/test_replicated_fetches_bandwidth/test.py @@ -9,9 +9,12 @@ import time import statistics cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) -node3 = cluster.add_instance('node3', user_configs=['configs/limit_replication_config.xml'], with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) +node3 = cluster.add_instance( + "node3", user_configs=["configs/limit_replication_config.xml"], with_zookeeper=True +) + @pytest.fixture(scope="module") def start_cluster(): @@ -22,19 +25,29 @@ def start_cluster(): finally: cluster.shutdown() + def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) + def test_limited_fetch_single_table(start_cluster): print("Limited fetches single table") try: for i, node in enumerate([node1, node2]): - node.query(f"CREATE TABLE limited_fetch_table(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_fetch_table', '{i}') ORDER BY tuple() PARTITION BY key SETTINGS max_replicated_fetches_network_bandwidth=10485760") + node.query( + f"CREATE TABLE limited_fetch_table(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_fetch_table', '{i}') ORDER BY tuple() PARTITION BY key SETTINGS max_replicated_fetches_network_bandwidth=10485760" + ) node2.query("SYSTEM STOP FETCHES limited_fetch_table") for i in range(5): - node1.query("INSERT INTO limited_fetch_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(300)".format(i)) + node1.query( + "INSERT INTO limited_fetch_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(300)".format( + i + ) + ) n1_net = NetThroughput(node1) n2_net = NetThroughput(node2) @@ -42,31 +55,41 @@ def test_limited_fetch_single_table(start_cluster): node2.query("SYSTEM START FETCHES limited_fetch_table") n2_fetch_speed = [] for i in range(10): - n1_in, n1_out = n1_net.measure_speed('megabytes') - n2_in, n2_out = n2_net.measure_speed('megabytes') - print("[N1] input:", n1_in, 'MB/s', "output:", n1_out, "MB/s") - print("[N2] input:", n2_in, 'MB/s', "output:", n2_out, "MB/s") + n1_in, n1_out = n1_net.measure_speed("megabytes") + n2_in, n2_out = n2_net.measure_speed("megabytes") + print("[N1] input:", n1_in, "MB/s", "output:", n1_out, "MB/s") + print("[N2] input:", n2_in, "MB/s", "output:", n2_out, "MB/s") n2_fetch_speed.append(n2_in) time.sleep(0.5) median_speed = statistics.median(n2_fetch_speed) # approximate border. Without limit we will have more than 100 MB/s for very slow builds. - assert median_speed <= 15, "We exceeded max fetch speed for more than 10MB/s. Must be around 10 (+- 5), got " + str(median_speed) + assert median_speed <= 15, ( + "We exceeded max fetch speed for more than 10MB/s. Must be around 10 (+- 5), got " + + str(median_speed) + ) finally: for node in [node1, node2]: node.query("DROP TABLE IF EXISTS limited_fetch_table SYNC") + def test_limited_send_single_table(start_cluster): print("Limited sends single table") try: for i, node in enumerate([node1, node2]): - node.query(f"CREATE TABLE limited_send_table(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_fetch_table', '{i}') ORDER BY tuple() PARTITION BY key SETTINGS max_replicated_sends_network_bandwidth=5242880") + node.query( + f"CREATE TABLE limited_send_table(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_fetch_table', '{i}') ORDER BY tuple() PARTITION BY key SETTINGS max_replicated_sends_network_bandwidth=5242880" + ) node2.query("SYSTEM STOP FETCHES limited_send_table") for i in range(5): - node1.query("INSERT INTO limited_send_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(150)".format(i)) + node1.query( + "INSERT INTO limited_send_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(150)".format( + i + ) + ) n1_net = NetThroughput(node1) n2_net = NetThroughput(node2) @@ -74,16 +97,19 @@ def test_limited_send_single_table(start_cluster): node2.query("SYSTEM START FETCHES limited_send_table") n1_sends_speed = [] for i in range(10): - n1_in, n1_out = n1_net.measure_speed('megabytes') - n2_in, n2_out = n2_net.measure_speed('megabytes') - print("[N1] input:", n1_in, 'MB/s', "output:", n1_out, "MB/s") - print("[N2] input:", n2_in, 'MB/s', "output:", n2_out, "MB/s") + n1_in, n1_out = n1_net.measure_speed("megabytes") + n2_in, n2_out = n2_net.measure_speed("megabytes") + print("[N1] input:", n1_in, "MB/s", "output:", n1_out, "MB/s") + print("[N2] input:", n2_in, "MB/s", "output:", n2_out, "MB/s") n1_sends_speed.append(n1_out) time.sleep(0.5) median_speed = statistics.median(n1_sends_speed) # approximate border. Without limit we will have more than 100 MB/s for very slow builds. - assert median_speed <= 10, "We exceeded max send speed for more than 5MB/s. Must be around 5 (+- 5), got " + str(median_speed) + assert median_speed <= 10, ( + "We exceeded max send speed for more than 5MB/s. Must be around 5 (+- 5), got " + + str(median_speed) + ) finally: for node in [node1, node2]: @@ -95,12 +121,18 @@ def test_limited_fetches_for_server(start_cluster): try: for i, node in enumerate([node1, node3]): for j in range(5): - node.query(f"CREATE TABLE limited_fetches{j}(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_fetches{j}', '{i}') ORDER BY tuple() PARTITION BY key") + node.query( + f"CREATE TABLE limited_fetches{j}(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_fetches{j}', '{i}') ORDER BY tuple() PARTITION BY key" + ) for j in range(5): node3.query(f"SYSTEM STOP FETCHES limited_fetches{j}") for i in range(5): - node1.query("INSERT INTO limited_fetches{} SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(50)".format(j, i)) + node1.query( + "INSERT INTO limited_fetches{} SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(50)".format( + j, i + ) + ) n1_net = NetThroughput(node1) n3_net = NetThroughput(node3) @@ -110,16 +142,19 @@ def test_limited_fetches_for_server(start_cluster): n3_fetches_speed = [] for i in range(5): - n1_in, n1_out = n1_net.measure_speed('megabytes') - n3_in, n3_out = n3_net.measure_speed('megabytes') - print("[N1] input:", n1_in, 'MB/s', "output:", n1_out, "MB/s") - print("[N3] input:", n3_in, 'MB/s', "output:", n3_out, "MB/s") + n1_in, n1_out = n1_net.measure_speed("megabytes") + n3_in, n3_out = n3_net.measure_speed("megabytes") + print("[N1] input:", n1_in, "MB/s", "output:", n1_out, "MB/s") + print("[N3] input:", n3_in, "MB/s", "output:", n3_out, "MB/s") n3_fetches_speed.append(n3_in) time.sleep(0.5) median_speed = statistics.median(n3_fetches_speed) # approximate border. Without limit we will have more than 100 MB/s for very slow builds. - assert median_speed <= 15, "We exceeded max fetch speed for more than 15MB/s. Must be around 5 (+- 10), got " + str(median_speed) + assert median_speed <= 15, ( + "We exceeded max fetch speed for more than 15MB/s. Must be around 5 (+- 10), got " + + str(median_speed) + ) finally: for node in [node1, node3]: @@ -132,12 +167,18 @@ def test_limited_sends_for_server(start_cluster): try: for i, node in enumerate([node1, node3]): for j in range(5): - node.query(f"CREATE TABLE limited_sends{j}(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_sends{j}', '{i}') ORDER BY tuple() PARTITION BY key") + node.query( + f"CREATE TABLE limited_sends{j}(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/limited_sends{j}', '{i}') ORDER BY tuple() PARTITION BY key" + ) for j in range(5): node1.query(f"SYSTEM STOP FETCHES limited_sends{j}") for i in range(5): - node3.query("INSERT INTO limited_sends{} SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(50)".format(j, i)) + node3.query( + "INSERT INTO limited_sends{} SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(50)".format( + j, i + ) + ) n1_net = NetThroughput(node1) n3_net = NetThroughput(node3) @@ -147,16 +188,19 @@ def test_limited_sends_for_server(start_cluster): n3_sends_speed = [] for i in range(5): - n1_in, n1_out = n1_net.measure_speed('megabytes') - n3_in, n3_out = n3_net.measure_speed('megabytes') - print("[N1] input:", n1_in, 'MB/s', "output:", n1_out, "MB/s") - print("[N3] input:", n3_in, 'MB/s', "output:", n3_out, "MB/s") + n1_in, n1_out = n1_net.measure_speed("megabytes") + n3_in, n3_out = n3_net.measure_speed("megabytes") + print("[N1] input:", n1_in, "MB/s", "output:", n1_out, "MB/s") + print("[N3] input:", n3_in, "MB/s", "output:", n3_out, "MB/s") n3_sends_speed.append(n3_out) time.sleep(0.5) median_speed = statistics.median(n3_sends_speed) # approximate border. Without limit we will have more than 100 MB/s for very slow builds. - assert median_speed <= 20, "We exceeded max send speed for more than 20MB/s. Must be around 5 (+- 10), got " + str(median_speed) + assert median_speed <= 20, ( + "We exceeded max send speed for more than 20MB/s. Must be around 5 (+- 10), got " + + str(median_speed) + ) finally: for node in [node1, node3]: @@ -168,12 +212,18 @@ def test_should_execute_fetch(start_cluster): print("Should execute fetch") try: for i, node in enumerate([node1, node2]): - node.query(f"CREATE TABLE should_execute_table(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/should_execute_table', '{i}') ORDER BY tuple() PARTITION BY key SETTINGS max_replicated_fetches_network_bandwidth=3505253") + node.query( + f"CREATE TABLE should_execute_table(key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/should_execute_table', '{i}') ORDER BY tuple() PARTITION BY key SETTINGS max_replicated_fetches_network_bandwidth=3505253" + ) node2.query("SYSTEM STOP FETCHES should_execute_table") for i in range(3): - node1.query("INSERT INTO should_execute_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(200)".format(i)) + node1.query( + "INSERT INTO should_execute_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(200)".format( + i + ) + ) n1_net = NetThroughput(node1) n2_net = NetThroughput(node2) @@ -181,19 +231,27 @@ def test_should_execute_fetch(start_cluster): node2.query("SYSTEM START FETCHES should_execute_table") for i in range(10): - node1.query("INSERT INTO should_execute_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(3)".format(i)) + node1.query( + "INSERT INTO should_execute_table SELECT {}, (select randomPrintableASCII(104857)) FROM numbers(3)".format( + i + ) + ) n2_fetch_speed = [] replication_queue_data = [] for i in range(10): - n1_in, n1_out = n1_net.measure_speed('megabytes') - n2_in, n2_out = n2_net.measure_speed('megabytes') + n1_in, n1_out = n1_net.measure_speed("megabytes") + n2_in, n2_out = n2_net.measure_speed("megabytes") fetches_count = node2.query("SELECT count() FROM system.replicated_fetches") if fetches_count == "0\n": break print("Fetches count", fetches_count) - replication_queue_data.append(node2.query("SELECT count() FROM system.replication_queue WHERE postpone_reason like '%fetches have already throttled%'")) + replication_queue_data.append( + node2.query( + "SELECT count() FROM system.replication_queue WHERE postpone_reason like '%fetches have already throttled%'" + ) + ) n2_fetch_speed.append(n2_in) time.sleep(0.5) diff --git a/tests/integration/test_replicated_fetches_timeouts/test.py b/tests/integration/test_replicated_fetches_timeouts/test.py index 88763265270..7d5da55549c 100644 --- a/tests/integration/test_replicated_fetches_timeouts/test.py +++ b/tests/integration/test_replicated_fetches_timeouts/test.py @@ -10,12 +10,12 @@ from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( - 'node1', with_zookeeper=True, - main_configs=['configs/server.xml']) + "node1", with_zookeeper=True, main_configs=["configs/server.xml"] +) node2 = cluster.add_instance( - 'node2', with_zookeeper=True, - main_configs=['configs/server.xml']) + "node2", with_zookeeper=True, main_configs=["configs/server.xml"] +) @pytest.fixture(scope="module") @@ -30,22 +30,34 @@ def started_cluster(): def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) def test_no_stall(started_cluster): for instance in started_cluster.instances.values(): - instance.query(""" + instance.query( + """ CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '{instance}') ORDER BY tuple() - PARTITION BY key""") + PARTITION BY key""" + ) # Pause node3 until the test setup is prepared node2.query("SYSTEM STOP FETCHES t") - node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(500)".format(get_random_string(104857))) - node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(500)".format(get_random_string(104857))) + node1.query( + "INSERT INTO t SELECT 1, '{}' FROM numbers(500)".format( + get_random_string(104857) + ) + ) + node1.query( + "INSERT INTO t SELECT 2, '{}' FROM numbers(500)".format( + get_random_string(104857) + ) + ) with PartitionManager() as pm: pm.add_network_delay(node1, 2000) @@ -53,12 +65,15 @@ def test_no_stall(started_cluster): # Wait for timeout exceptions to confirm that timeout is triggered. while True: - conn_timeout_exceptions = int(node2.query( - """ + conn_timeout_exceptions = int( + node2.query( + """ SELECT count() FROM system.replication_queue WHERE last_exception LIKE '%connect timed out%' - """)) + """ + ) + ) if conn_timeout_exceptions >= 2: break @@ -68,19 +83,24 @@ def test_no_stall(started_cluster): print("Connection timeouts tested!") # Increase connection timeout and wait for receive timeouts. - node2.query(""" + node2.query( + """ ALTER TABLE t MODIFY SETTING replicated_fetches_http_connection_timeout = 30, - replicated_fetches_http_receive_timeout = 1""") + replicated_fetches_http_receive_timeout = 1""" + ) while True: - timeout_exceptions = int(node2.query( - """ + timeout_exceptions = int( + node2.query( + """ SELECT count() FROM system.replication_queue WHERE last_exception LIKE '%Timeout%' AND last_exception NOT LIKE '%connect timed out%' - """).strip()) + """ + ).strip() + ) if timeout_exceptions >= 2: break diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index b56aa5706c9..00367daad33 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -2,8 +2,23 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.12.4.5', stay_alive=True, with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.12.4.5', stay_alive=True, with_installed_binary=True) +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.12.4.5", + stay_alive=True, + with_installed_binary=True, +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.12.4.5", + stay_alive=True, + with_installed_binary=True, +) + @pytest.fixture(scope="module") def started_cluster(): @@ -18,13 +33,14 @@ def started_cluster(): finally: cluster.shutdown() + def test_replicated_merge_tree_defaults_compatibility(started_cluster): # This test checks, that result of parsing list of columns with defaults # from 'CREATE/ATTACH' is compatible with parsing from zookeeper metadata on different versions. # We create table and write 'columns' node in zookeeper with old version, than restart with new version # drop and try recreate one replica. During startup of table structure is checked between 'CREATE' query and zookeeper. - create_query = ''' + create_query = """ CREATE TABLE test.table ( a UInt32, @@ -33,7 +49,7 @@ def test_replicated_merge_tree_defaults_compatibility(started_cluster): ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/table', '{replica}') ORDER BY a - ''' + """ for node in (node1, node2): node.query("CREATE DATABASE test ENGINE = Ordinary") @@ -41,10 +57,12 @@ def test_replicated_merge_tree_defaults_compatibility(started_cluster): node1.query("DETACH TABLE test.table") node2.query("SYSTEM DROP REPLICA 'node1' FROM TABLE test.table") - node1.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/metadata/test/table.sql"]) + node1.exec_in_container( + ["bash", "-c", "rm /var/lib/clickhouse/metadata/test/table.sql"] + ) node1.exec_in_container(["bash", "-c", "rm -r /var/lib/clickhouse/data/test/table"]) - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") exists_replica_1 = zk.exists("/clickhouse/tables/test/table/replicas/node1") assert exists_replica_1 == None diff --git a/tests/integration/test_replicated_merge_tree_config/test.py b/tests/integration/test_replicated_merge_tree_config/test.py index 2a7725960bf..b5c033032ba 100644 --- a/tests/integration/test_replicated_merge_tree_config/test.py +++ b/tests/integration/test_replicated_merge_tree_config/test.py @@ -3,7 +3,9 @@ from helpers.cluster import ClickHouseCluster import logging cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["configs/config.xml"], with_zookeeper=True) +node = cluster.add_instance( + "node", main_configs=["configs/config.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") diff --git a/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py b/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py index bc5a419aaf2..aea41fc0684 100644 --- a/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py +++ b/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py @@ -7,17 +7,22 @@ import os SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", - main_configs=["configs/remote_servers.xml", "configs/storage.xml"], - tmpfs=["/disk:size=100M"], - macros={'replica': 'node1'}, - with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/storage.xml"], + tmpfs=["/disk:size=100M"], + macros={"replica": "node1"}, + with_zookeeper=True, +) + +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml", "configs/storage.xml"], + tmpfs=["/disk:size=100M"], + macros={"replica": "node2"}, + with_zookeeper=True, +) -node2 = cluster.add_instance("node2", - main_configs=["configs/remote_servers.xml", "configs/storage.xml"], - tmpfs=["/disk:size=100M"], - macros={'replica': 'node2'}, - with_zookeeper=True) @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -29,9 +34,13 @@ def start_cluster(): def copy_keys(instance, keys_file_name): - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, f"configs/{keys_file_name}.xml"), "/etc/clickhouse-server/config.d/z_keys.xml") + instance.copy_file_to_container( + os.path.join(SCRIPT_DIR, f"configs/{keys_file_name}.xml"), + "/etc/clickhouse-server/config.d/z_keys.xml", + ) instance.query("SYSTEM RELOAD CONFIG") + def create_table(): node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY") node1.query( @@ -45,16 +54,19 @@ def create_table(): """ ) + def insert_data(): node1.query("INSERT INTO tbl VALUES (1, 'str1')") - node2.query("INSERT INTO tbl VALUES (1, 'str1')") # Test deduplication + node2.query("INSERT INTO tbl VALUES (1, 'str1')") # Test deduplication node2.query("INSERT INTO tbl VALUES (2, 'str2')") + def optimize_table(): node1.query("OPTIMIZE TABLE tbl ON CLUSTER 'cluster' FINAL") + def check_table(): - expected=[[1, 'str1'], [2, 'str2']] + expected = [[1, "str1"], [2, "str2"]] assert node1.query("SELECT * FROM tbl ORDER BY id") == TSV(expected) assert node2.query("SELECT * FROM tbl ORDER BY id") == TSV(expected) assert node1.query("CHECK TABLE tbl") == "1\n" @@ -63,9 +75,10 @@ def check_table(): # Actual tests: + def test_same_keys(): - copy_keys(node1, 'key_a') - copy_keys(node2, 'key_a') + copy_keys(node1, "key_a") + copy_keys(node2, "key_a") create_table() insert_data() @@ -76,8 +89,8 @@ def test_same_keys(): def test_different_keys(): - copy_keys(node1, 'key_a') - copy_keys(node2, 'key_b') + copy_keys(node1, "key_a") + copy_keys(node2, "key_b") create_table() insert_data() diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/test.py b/tests/integration/test_replicated_merge_tree_encryption_codec/test.py index 3aec2259703..6f08daae4cf 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/test.py +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/test.py @@ -7,15 +7,20 @@ import os SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", - main_configs=["configs/remote_servers.xml", "configs/encryption_codec.xml"], - macros={'replica': 'node1'}, - with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/encryption_codec.xml"], + macros={"replica": "node1"}, + with_zookeeper=True, +) + +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml", "configs/encryption_codec.xml"], + macros={"replica": "node2"}, + with_zookeeper=True, +) -node2 = cluster.add_instance("node2", - main_configs=["configs/remote_servers.xml", "configs/encryption_codec.xml"], - macros={'replica': 'node2'}, - with_zookeeper=True) @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -27,9 +32,13 @@ def start_cluster(): def copy_keys(instance, keys_file_name): - instance.copy_file_to_container(os.path.join(SCRIPT_DIR, f"configs/{keys_file_name}.xml"), "/etc/clickhouse-server/config.d/z_keys.xml") + instance.copy_file_to_container( + os.path.join(SCRIPT_DIR, f"configs/{keys_file_name}.xml"), + "/etc/clickhouse-server/config.d/z_keys.xml", + ) instance.query("SYSTEM RELOAD CONFIG") + def create_table(): node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY") node1.query( @@ -42,16 +51,19 @@ def create_table(): """ ) + def insert_data(): node1.query("INSERT INTO tbl VALUES (1, 'str1')") - node2.query("INSERT INTO tbl VALUES (1, 'str1')") # Test deduplication + node2.query("INSERT INTO tbl VALUES (1, 'str1')") # Test deduplication node2.query("INSERT INTO tbl VALUES (2, 'str2')") + def optimize_table(): node1.query("OPTIMIZE TABLE tbl ON CLUSTER 'cluster' FINAL") + def check_table(): - expected=[[1, 'str1'], [2, 'str2']] + expected = [[1, "str1"], [2, "str2"]] assert node1.query("SELECT * FROM tbl ORDER BY id") == TSV(expected) assert node2.query("SELECT * FROM tbl ORDER BY id") == TSV(expected) assert node1.query("CHECK TABLE tbl") == "1\n" @@ -60,9 +72,10 @@ def check_table(): # Actual tests: + def test_same_keys(): - copy_keys(node1, 'key_a') - copy_keys(node2, 'key_a') + copy_keys(node1, "key_a") + copy_keys(node2, "key_a") create_table() insert_data() @@ -73,8 +86,8 @@ def test_same_keys(): def test_different_keys(): - copy_keys(node1, 'key_a') - copy_keys(node2, 'key_b') + copy_keys(node1, "key_a") + copy_keys(node2, "key_b") create_table() insert_data() @@ -82,13 +95,13 @@ def test_different_keys(): assert "BAD_DECRYPT" in node2.query_and_get_error("SELECT * FROM tbl") # Hang? - #optimize_table() - #check_table() + # optimize_table() + # check_table() def test_different_current_key_ids(): - copy_keys(node1, 'key_a_and_b_current_a') - copy_keys(node2, 'key_a_and_b_current_b') + copy_keys(node1, "key_a_and_b_current_a") + copy_keys(node2, "key_a_and_b_current_b") create_table() insert_data() @@ -99,8 +112,8 @@ def test_different_current_key_ids(): def test_different_nonces(): - copy_keys(node1, 'key_a_and_nonce_x') - copy_keys(node2, 'key_a_and_nonce_y') + copy_keys(node1, "key_a_and_nonce_x") + copy_keys(node2, "key_a_and_nonce_y") create_table() insert_data() diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py index f557a69569a..1e34a924e39 100644 --- a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py @@ -8,7 +8,7 @@ from helpers.cluster import ClickHouseCluster from pyhdfs import HdfsClient SHARDS = 2 -FILES_OVERHEAD_PER_TABLE = 1 # format_version.txt +FILES_OVERHEAD_PER_TABLE = 1 # format_version.txt FILES_OVERHEAD_PER_PART_COMPACT = 7 @@ -20,31 +20,39 @@ def wait_for_hdfs_objects(cluster, fp, expected, num_tries=30): break num_tries -= 1 time.sleep(1) - assert(len(fs.listdir(fp)) == expected) + assert len(fs.listdir(fp)) == expected @pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], - macros={'replica': 'node1'}, - with_zookeeper=True, - with_hdfs=True) - cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], - macros={'replica': 'node2'}, - with_zookeeper=True, - with_hdfs=True) + cluster.add_instance( + "node1", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "node1"}, + with_zookeeper=True, + with_hdfs=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "node2"}, + with_zookeeper=True, + with_hdfs=True, + ) logging.info("Starting cluster...") cluster.start() if cluster.instances["node1"].is_debug_build(): # https://github.com/ClickHouse/ClickHouse/issues/27814 - pytest.skip("libhdfs3 calls rand function which does not pass harmful check in debug build") + pytest.skip( + "libhdfs3 calls rand function which does not pass harmful check in debug build" + ) logging.info("Cluster started") fs = HdfsClient(hosts=cluster.hdfs_ip) - fs.mkdirs('/clickhouse1') - fs.mkdirs('/clickhouse2') + fs.mkdirs("/clickhouse1") + fs.mkdirs("/clickhouse2") logging.info("Created HDFS directory") yield cluster @@ -64,111 +72,190 @@ def test_hdfs_zero_copy_replication_insert(cluster): SETTINGS storage_policy='hdfs_only' """ ) - wait_for_hdfs_objects(cluster, "/clickhouse1", SHARDS * FILES_OVERHEAD_PER_TABLE) + wait_for_hdfs_objects( + cluster, "/clickhouse1", SHARDS * FILES_OVERHEAD_PER_TABLE + ) node1.query("INSERT INTO hdfs_test VALUES (now() - INTERVAL 3 DAY, 10)") node2.query("SYSTEM SYNC REPLICA hdfs_test") assert node1.query("SELECT count() FROM hdfs_test FORMAT Values") == "(1)" assert node2.query("SELECT count() FROM hdfs_test FORMAT Values") == "(1)" - assert node1.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)" - assert node2.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)" - assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values") == "('all','hdfs1')" - assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values") == "('all','hdfs1')" - wait_for_hdfs_objects(cluster, "/clickhouse1", SHARDS * FILES_OVERHEAD_PER_TABLE + FILES_OVERHEAD_PER_PART_COMPACT) + assert ( + node1.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)" + ) + assert ( + node2.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)" + ) + assert ( + node1.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values" + ) + == "('all','hdfs1')" + ) + assert ( + node2.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values" + ) + == "('all','hdfs1')" + ) + wait_for_hdfs_objects( + cluster, + "/clickhouse1", + SHARDS * FILES_OVERHEAD_PER_TABLE + FILES_OVERHEAD_PER_PART_COMPACT, + ) finally: node1.query("DROP TABLE IF EXISTS hdfs_test NO DELAY") node2.query("DROP TABLE IF EXISTS hdfs_test NO DELAY") - @pytest.mark.parametrize( ("storage_policy", "init_objects"), - [("hybrid", 0), - ("tiered", 0), - ("tiered_copy", FILES_OVERHEAD_PER_TABLE)] + [("hybrid", 0), ("tiered", 0), ("tiered_copy", FILES_OVERHEAD_PER_TABLE)], ) def test_hdfs_zero_copy_replication_single_move(cluster, storage_policy, init_objects): node1 = cluster.instances["node1"] try: node1.query( - Template(""" + Template( + """ CREATE TABLE single_node_move_test (dt DateTime, id Int64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/single_node_move_test', '{replica}') ORDER BY (dt, id) SETTINGS storage_policy='$policy' - """).substitute(policy=storage_policy) + """ + ).substitute(policy=storage_policy) ) wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects) - node1.query("INSERT INTO single_node_move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)") - assert node1.query("SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values") == "(10),(11)" + node1.query( + "INSERT INTO single_node_move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)" + ) + assert ( + node1.query( + "SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values" + ) + == "(10),(11)" + ) - node1.query("ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'external'") - assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='single_node_move_test' FORMAT Values") == "('all','hdfs1')" - assert node1.query("SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values") == "(10),(11)" - wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT) + node1.query( + "ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'external'" + ) + assert ( + node1.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='single_node_move_test' FORMAT Values" + ) + == "('all','hdfs1')" + ) + assert ( + node1.query( + "SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values" + ) + == "(10),(11)" + ) + wait_for_hdfs_objects( + cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT + ) - node1.query("ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'main'") - assert node1.query("SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values") == "(10),(11)" + node1.query( + "ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'main'" + ) + assert ( + node1.query( + "SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values" + ) + == "(10),(11)" + ) finally: node1.query("DROP TABLE IF EXISTS single_node_move_test NO DELAY") @pytest.mark.parametrize( ("storage_policy", "init_objects"), - [("hybrid", 0), - ("tiered", 0), - ("tiered_copy", SHARDS * FILES_OVERHEAD_PER_TABLE)] + [("hybrid", 0), ("tiered", 0), ("tiered_copy", SHARDS * FILES_OVERHEAD_PER_TABLE)], ) def test_hdfs_zero_copy_replication_move(cluster, storage_policy, init_objects): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] try: node1.query( - Template(""" + Template( + """ CREATE TABLE move_test ON CLUSTER test_cluster (dt DateTime, id Int64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/move_test', '{replica}') ORDER BY (dt, id) SETTINGS storage_policy='$policy' - """).substitute(policy=storage_policy) + """ + ).substitute(policy=storage_policy) ) wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects) - node1.query("INSERT INTO move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)") + node1.query( + "INSERT INTO move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)" + ) node2.query("SYSTEM SYNC REPLICA move_test") - assert node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" - assert node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" + assert ( + node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") + == "(10),(11)" + ) + assert ( + node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") + == "(10),(11)" + ) - node1.query("ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'") - wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT) + node1.query( + "ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'" + ) + wait_for_hdfs_objects( + cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT + ) - node2.query("ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'") - assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values") == "('all','hdfs1')" - assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values") == "('all','hdfs1')" - assert node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" - assert node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" - wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT) + node2.query( + "ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'" + ) + assert ( + node1.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values" + ) + == "('all','hdfs1')" + ) + assert ( + node2.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values" + ) + == "('all','hdfs1')" + ) + assert ( + node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") + == "(10),(11)" + ) + assert ( + node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") + == "(10),(11)" + ) + wait_for_hdfs_objects( + cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT + ) finally: node1.query("DROP TABLE IF EXISTS move_test NO DELAY") node2.query("DROP TABLE IF EXISTS move_test NO DELAY") -@pytest.mark.parametrize( - ("storage_policy"), ["hybrid", "tiered", "tiered_copy"] -) +@pytest.mark.parametrize(("storage_policy"), ["hybrid", "tiered", "tiered_copy"]) def test_hdfs_zero_copy_with_ttl_move(cluster, storage_policy): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] try: node1.query( - Template(""" + Template( + """ CREATE TABLE ttl_move_test ON CLUSTER test_cluster (dt DateTime, id Int64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/ttl_move_test', '{replica}') ORDER BY (dt, id) TTL dt + INTERVAL 2 DAY TO VOLUME 'external' SETTINGS storage_policy='$policy' - """).substitute(policy=storage_policy) + """ + ).substitute(policy=storage_policy) ) node1.query("INSERT INTO ttl_move_test VALUES (now() - INTERVAL 3 DAY, 10)") @@ -179,8 +266,14 @@ def test_hdfs_zero_copy_with_ttl_move(cluster, storage_policy): assert node1.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" assert node2.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" - assert node1.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values") == "(10),(11)" - assert node2.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values") == "(10),(11)" + assert ( + node1.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values") + == "(10),(11)" + ) + assert ( + node2.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values") + == "(10),(11)" + ) finally: node1.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") node2.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") @@ -208,8 +301,14 @@ def test_hdfs_zero_copy_with_ttl_delete(cluster): assert node1.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1)" assert node2.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1)" - assert node1.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values") == "(11)" - assert node2.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values") == "(11)" + assert ( + node1.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values") + == "(11)" + ) + assert ( + node2.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values") + == "(11)" + ) finally: node1.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") node2.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") diff --git a/tests/integration/test_replicated_merge_tree_s3/test.py b/tests/integration/test_replicated_merge_tree_s3/test.py index d04bdae36e2..cc85a4eab02 100644 --- a/tests/integration/test_replicated_merge_tree_s3/test.py +++ b/tests/integration/test_replicated_merge_tree_s3/test.py @@ -11,12 +11,25 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '1'}, - with_minio=True, with_zookeeper=True) - cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '2'}, - with_zookeeper=True) - cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '3'}, - with_zookeeper=True) + cluster.add_instance( + "node1", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "1"}, + with_minio=True, + with_zookeeper=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "2"}, + with_zookeeper=True, + ) + cluster.add_instance( + "node3", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "3"}, + with_zookeeper=True, + ) logging.info("Starting cluster...") cluster.start() @@ -35,7 +48,7 @@ FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def random_string(length): letters = string.ascii_letters - return ''.join(random.choice(letters) for i in range(length)) + return "".join(random.choice(letters) for i in range(length)) def generate_values(date_str, count, sign=1): @@ -71,32 +84,43 @@ def drop_table(cluster): minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) + @pytest.mark.parametrize( "min_rows_for_wide_part,files_per_part", - [ - (0, FILES_OVERHEAD_PER_PART_WIDE), - (8192, FILES_OVERHEAD_PER_PART_COMPACT) - ] + [(0, FILES_OVERHEAD_PER_PART_WIDE), (8192, FILES_OVERHEAD_PER_PART_COMPACT)], ) def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part): - create_table(cluster, additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part)) + create_table( + cluster, + additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part), + ) all_values = "" for node_idx in range(1, 4): node = cluster.instances["node" + str(node_idx)] values = generate_values("2020-01-0" + str(node_idx), 4096) - node.query("INSERT INTO s3_test VALUES {}".format(values), settings={"insert_quorum": 3}) + node.query( + "INSERT INTO s3_test VALUES {}".format(values), + settings={"insert_quorum": 3}, + ) if node_idx != 1: all_values += "," all_values += values for node_idx in range(1, 4): node = cluster.instances["node" + str(node_idx)] - assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", - settings={"select_sequential_consistency": 1}) == all_values + assert ( + node.query( + "SELECT * FROM s3_test order by dt, id FORMAT Values", + settings={"select_sequential_consistency": 1}, + ) + == all_values + ) minio = cluster.minio_client - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == 3 * (FILES_OVERHEAD + files_per_part * 3) + assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == 3 * ( + FILES_OVERHEAD + files_per_part * 3 + ) diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py index edf39969b47..73b611ad169 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -14,12 +14,25 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '1'}, - with_minio=True, with_zookeeper=True) - cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '2'}, - with_zookeeper=True) - cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '3'}, - with_zookeeper=True) + cluster.add_instance( + "node1", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "1"}, + with_minio=True, + with_zookeeper=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "2"}, + with_zookeeper=True, + ) + cluster.add_instance( + "node3", + main_configs=["configs/config.d/storage_conf.xml"], + macros={"replica": "3"}, + with_zookeeper=True, + ) logging.info("Starting cluster...") cluster.start() @@ -38,7 +51,7 @@ FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 def random_string(length): letters = string.ascii_letters - return ''.join(random.choice(letters) for i in range(length)) + return "".join(random.choice(letters) for i in range(length)) def generate_values(date_str, count, sign=1): @@ -65,6 +78,7 @@ def create_table(cluster, additional_settings=None): list(cluster.instances.values())[0].query(create_table_statement) + @pytest.fixture(autouse=True) def drop_table(cluster): yield @@ -73,32 +87,43 @@ def drop_table(cluster): minio = cluster.minio_client # Remove extra objects to prevent tests cascade failing - for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + for obj in list(minio.list_objects(cluster.minio_bucket, "data/")): minio.remove_object(cluster.minio_bucket, obj.object_name) + @pytest.mark.parametrize( "min_rows_for_wide_part,files_per_part", - [ - (0, FILES_OVERHEAD_PER_PART_WIDE), - (8192, FILES_OVERHEAD_PER_PART_COMPACT) - ] + [(0, FILES_OVERHEAD_PER_PART_WIDE), (8192, FILES_OVERHEAD_PER_PART_COMPACT)], ) def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part): - create_table(cluster, additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part)) + create_table( + cluster, + additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part), + ) all_values = "" for node_idx in range(1, 4): node = cluster.instances["node" + str(node_idx)] values = generate_values("2020-01-0" + str(node_idx), 4096) - node.query("INSERT INTO s3_test VALUES {}".format(values), settings={"insert_quorum": 3}) + node.query( + "INSERT INTO s3_test VALUES {}".format(values), + settings={"insert_quorum": 3}, + ) if node_idx != 1: all_values += "," all_values += values for node_idx in range(1, 4): node = cluster.instances["node" + str(node_idx)] - assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", - settings={"select_sequential_consistency": 1}) == all_values + assert ( + node.query( + "SELECT * FROM s3_test order by dt, id FORMAT Values", + settings={"select_sequential_consistency": 1}, + ) + == all_values + ) minio = cluster.minio_client - assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == (3 * FILES_OVERHEAD) + (files_per_part * 3) + assert len(list(minio.list_objects(cluster.minio_bucket, "data/"))) == ( + 3 * FILES_OVERHEAD + ) + (files_per_part * 3) diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py index 4644790ff94..c46e6840153 100644 --- a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py @@ -7,8 +7,16 @@ from helpers.client import QueryRuntimeException from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", main_configs=["configs/zookeeper_config.xml", "configs/remote_servers.xml"], with_zookeeper=True) -node2 = cluster.add_instance("node2", main_configs=["configs/zookeeper_config.xml", "configs/remote_servers.xml"], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/zookeeper_config.xml", "configs/remote_servers.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/zookeeper_config.xml", "configs/remote_servers.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -29,16 +37,20 @@ def drop_table(nodes, table_name): for node in nodes: node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + # Create table with default zookeeper. def test_create_replicated_merge_tree_with_default_zookeeper(started_cluster): drop_table([node1, node2], "test_default_zookeeper") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_default_zookeeper(a Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_default_zookeeper', '{replica}') ORDER BY a; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) # Insert data into node1, and query it from node2. node1.query("INSERT INTO test_default_zookeeper VALUES (1)") @@ -48,16 +60,20 @@ def test_create_replicated_merge_tree_with_default_zookeeper(started_cluster): assert TSV(node1.query("SELECT a FROM test_default_zookeeper")) == TSV(expected) assert TSV(node2.query("SELECT a FROM test_default_zookeeper")) == TSV(expected) + # Create table with auxiliary zookeeper. def test_create_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): drop_table([node1, node2], "test_auxiliary_zookeeper") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_auxiliary_zookeeper(a Int32) ENGINE = ReplicatedMergeTree('zookeeper2:/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') ORDER BY a; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) # Insert data into node1, and query it from node2. node1.query("INSERT INTO test_auxiliary_zookeeper VALUES (1)") @@ -67,27 +83,37 @@ def test_create_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): assert TSV(node1.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) assert TSV(node2.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) + # Create table with auxiliary zookeeper. -def test_create_replicated_merge_tree_with_not_exists_auxiliary_zookeeper(started_cluster): +def test_create_replicated_merge_tree_with_not_exists_auxiliary_zookeeper( + started_cluster, +): drop_table([node1], "test_auxiliary_zookeeper") with pytest.raises(QueryRuntimeException): node1.query( - ''' + """ CREATE TABLE test_auxiliary_zookeeper(a Int32) ENGINE = ReplicatedMergeTree('zookeeper_not_exits:/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') ORDER BY a; - '''.format(replica=node1.name)) + """.format( + replica=node1.name + ) + ) + # Drop table with auxiliary zookeeper. def test_drop_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): drop_table([node1, node2], "test_auxiliary_zookeeper") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_auxiliary_zookeeper(a Int32) ENGINE = ReplicatedMergeTree('zookeeper2:/clickhouse/tables/test/test_auxiliary_zookeeper', '{replica}') ORDER BY a; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) # Insert data into node1, and query it from node2. node1.query("INSERT INTO test_auxiliary_zookeeper VALUES (1)") @@ -97,17 +123,26 @@ def test_drop_replicated_merge_tree_with_auxiliary_zookeeper(started_cluster): assert TSV(node1.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) assert TSV(node2.query("SELECT a FROM test_auxiliary_zookeeper")) == TSV(expected) - zk = cluster.get_kazoo_client('zoo1') - assert zk.exists('/clickhouse/tables/test/test_auxiliary_zookeeper') + zk = cluster.get_kazoo_client("zoo1") + assert zk.exists("/clickhouse/tables/test/test_auxiliary_zookeeper") drop_table([node1, node2], "test_auxiliary_zookeeper") - assert zk.exists('/clickhouse/tables/test/test_auxiliary_zookeeper') is None + assert zk.exists("/clickhouse/tables/test/test_auxiliary_zookeeper") is None + def test_path_ambiguity(started_cluster): drop_table([node1, node2], "test_path_ambiguity1") drop_table([node1, node2], "test_path_ambiguity2") - node1.query("create table test_path_ambiguity1 (n int) engine=ReplicatedMergeTree('/test:bad:/path', '1') order by n") - assert "Invalid auxiliary ZooKeeper name" in node1.query_and_get_error("create table test_path_ambiguity2 (n int) engine=ReplicatedMergeTree('test:bad:/path', '1') order by n") - assert "ZooKeeper path must starts with '/'" in node1.query_and_get_error("create table test_path_ambiguity2 (n int) engine=ReplicatedMergeTree('test/bad:/path', '1') order by n") - node1.query("create table test_path_ambiguity2 (n int) engine=ReplicatedMergeTree('zookeeper2:/bad:/path', '1') order by n") + node1.query( + "create table test_path_ambiguity1 (n int) engine=ReplicatedMergeTree('/test:bad:/path', '1') order by n" + ) + assert "Invalid auxiliary ZooKeeper name" in node1.query_and_get_error( + "create table test_path_ambiguity2 (n int) engine=ReplicatedMergeTree('test:bad:/path', '1') order by n" + ) + assert "ZooKeeper path must starts with '/'" in node1.query_and_get_error( + "create table test_path_ambiguity2 (n int) engine=ReplicatedMergeTree('test/bad:/path', '1') order by n" + ) + node1.query( + "create table test_path_ambiguity2 (n int) engine=ReplicatedMergeTree('zookeeper2:/bad:/path', '1') order by n" + ) drop_table([node1, node2], "test_path_ambiguity1") drop_table([node1, node2], "test_path_ambiguity2") diff --git a/tests/integration/test_replicated_mutations/test.py b/tests/integration/test_replicated_mutations/test.py index 5efc022cf36..7479f082b06 100644 --- a/tests/integration/test_replicated_mutations/test.py +++ b/tests/integration/test_replicated_mutations/test.py @@ -9,35 +9,55 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', macros={'cluster': 'test1'}, with_zookeeper=True) +node1 = cluster.add_instance("node1", macros={"cluster": "test1"}, with_zookeeper=True) # Check, that limits on max part size for merges doesn`t affect mutations -node2 = cluster.add_instance('node2', macros={'cluster': 'test1'}, main_configs=["configs/merge_tree.xml"], - with_zookeeper=True) +node2 = cluster.add_instance( + "node2", + macros={"cluster": "test1"}, + main_configs=["configs/merge_tree.xml"], + with_zookeeper=True, +) -node3 = cluster.add_instance('node3', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], - with_zookeeper=True) -node4 = cluster.add_instance('node4', macros={'cluster': 'test2'}, main_configs=["configs/merge_tree_max_parts.xml"], - with_zookeeper=True) +node3 = cluster.add_instance( + "node3", + macros={"cluster": "test2"}, + main_configs=["configs/merge_tree_max_parts.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + macros={"cluster": "test2"}, + main_configs=["configs/merge_tree_max_parts.xml"], + with_zookeeper=True, +) -node5 = cluster.add_instance('node5', macros={'cluster': 'test3'}, main_configs=["configs/merge_tree_max_parts.xml"]) +node5 = cluster.add_instance( + "node5", + macros={"cluster": "test3"}, + main_configs=["configs/merge_tree_max_parts.xml"], +) all_nodes = [node1, node2, node3, node4, node5] + def prepare_cluster(): for node in all_nodes: node.query("DROP TABLE IF EXISTS test_mutations SYNC") for node in [node1, node2, node3, node4]: - node.query(""" + node.query( + """ CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/test/test_mutations', '{instance}') ORDER BY x PARTITION BY toYYYYMM(d) SETTINGS number_of_free_entries_in_pool_to_execute_mutation=0 - """) + """ + ) node5.query( - "CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)") + "CREATE TABLE test_mutations(d Date, x UInt32, i UInt32) ENGINE MergeTree() ORDER BY x PARTITION BY toYYYYMM(d)" + ) @pytest.fixture(scope="module") @@ -76,7 +96,7 @@ class Runner: # Each thread inserts a small random number of rows with random year, month 01 and day determined # by the thread number. The idea is to avoid spurious duplicates and to insert into a # nontrivial number of partitions. - month = '01' + month = "01" day = str(thread_num + 1).zfill(2) i = 1 while not self.stop_ev.is_set(): @@ -89,15 +109,17 @@ class Runner: self.currently_inserting_xs[x] += 1 year = 2000 + random.randint(0, partitions_num) - date_str = '{year}-{month}-{day}'.format(year=year, month=month, day=day) - payload = '' + date_str = "{year}-{month}-{day}".format(year=year, month=month, day=day) + payload = "" for x in xs: - payload += '{date_str} {x} {i}\n'.format(date_str=date_str, x=x, i=i) + payload += "{date_str} {x} {i}\n".format(date_str=date_str, x=x, i=i) i += 1 try: logging.debug(f"thread {thread_num}: insert for {date_str}: {xs}") - random.choice(self.nodes).query("INSERT INTO test_mutations FORMAT TSV", payload) + random.choice(self.nodes).query( + "INSERT INTO test_mutations FORMAT TSV", payload + ) with self.mtx: for x in xs: @@ -124,7 +146,10 @@ class Runner: if self.current_xs: x = random.choice(list(self.current_xs.elements())) - if self.currently_inserting_xs[x] == 0 and x not in self.currently_deleting_xs: + if ( + self.currently_inserting_xs[x] == 0 + and x not in self.currently_deleting_xs + ): chosen = True self.currently_deleting_xs.add(x) to_delete_count = self.current_xs[x] @@ -135,7 +160,9 @@ class Runner: try: logging.debug(f"thread {thread_num}: delete {to_delete_count} * {x}") - random.choice(self.nodes).query("ALTER TABLE test_mutations DELETE WHERE x = {}".format(x)) + random.choice(self.nodes).query( + "ALTER TABLE test_mutations DELETE WHERE x = {}".format(x) + ) with self.mtx: self.total_mutations += 1 @@ -157,7 +184,11 @@ def wait_for_mutations(nodes, number_of_mutations): time.sleep(0.8) def get_done_mutations(node): - return int(node.query("SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations'").rstrip()) + return int( + node.query( + "SELECT sum(is_done) FROM system.mutations WHERE table = 'test_mutations'" + ).rstrip() + ) if all([get_done_mutations(n) == number_of_mutations for n in nodes]): return True @@ -195,32 +226,41 @@ def test_mutations(started_cluster): all_done = wait_for_mutations(nodes, runner.total_mutations) logging.debug(f"Total mutations: {runner.total_mutations}") for node in nodes: - logging.debug(node.query( - "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames")) + logging.debug( + node.query( + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames" + ) + ) assert all_done expected_sum = runner.total_inserted_xs - runner.total_deleted_xs actual_sums = [] for i, node in enumerate(nodes): - actual_sums.append(int(node.query("SELECT sum(x) FROM test_mutations").rstrip())) + actual_sums.append( + int(node.query("SELECT sum(x) FROM test_mutations").rstrip()) + ) assert actual_sums[i] == expected_sum @pytest.mark.parametrize( - ('nodes',), + ("nodes",), [ - ([node5, ],), # MergeTree + ( + [ + node5, + ], + ), # MergeTree ([node3, node4],), # ReplicatedMergeTree - ] + ], ) def test_mutations_dont_prevent_merges(started_cluster, nodes): prepare_cluster() for year in range(2000, 2016): - rows = '' - date_str = '{}-01-{}'.format(year, random.randint(1, 10)) + rows = "" + date_str = "{}-01-{}".format(year, random.randint(1, 10)) for i in range(10): - rows += '{} {} {}\n'.format(date_str, random.randint(1, 10), i) + rows += "{} {} {}\n".format(date_str, random.randint(1, 10), i) nodes[0].query("INSERT INTO test_mutations FORMAT TSV", rows) # will run mutations of 16 parts in parallel, mutations will sleep for about 20 seconds @@ -242,10 +282,16 @@ def test_mutations_dont_prevent_merges(started_cluster, nodes): t.join() for node in nodes: - logging.debug(node.query( - "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames")) - logging.debug(node.query( - "SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames")) + logging.debug( + node.query( + "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames" + ) + ) + logging.debug( + node.query( + "SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames" + ) + ) assert all_done, "All done" assert all([str(e).find("Too many parts") < 0 for e in runner.exceptions]) diff --git a/tests/integration/test_replicated_parse_zk_metadata/test.py b/tests/integration/test_replicated_parse_zk_metadata/test.py index d8b6685ddcd..4646d502b70 100644 --- a/tests/integration/test_replicated_parse_zk_metadata/test.py +++ b/tests/integration/test_replicated_parse_zk_metadata/test.py @@ -3,10 +3,10 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', with_zookeeper=True) +node = cluster.add_instance("node", with_zookeeper=True) -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def started_cluster(): try: cluster.start() @@ -17,7 +17,7 @@ def started_cluster(): def test_replicated_engine_parse_metadata_on_attach(): node.query( - ''' + """ CREATE TABLE data ( key Int, INDEX key_idx0 key+0 TYPE minmax GRANULARITY 1, @@ -25,15 +25,18 @@ def test_replicated_engine_parse_metadata_on_attach(): ) ENGINE = ReplicatedMergeTree('/ch/tables/default/data', 'node') ORDER BY key; - ''') - node.query('DETACH TABLE data') + """ + ) + node.query("DETACH TABLE data") - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") # Add **extra space between indices**, to check that it will be re-parsed # and successfully accepted by the server. # # This metadata was obtain from the server without #11325 - zk.set('/ch/tables/default/data/replicas/node/metadata', b""" + zk.set( + "/ch/tables/default/data/replicas/node/metadata", + b""" metadata format version: 1 date column: sampling expression: @@ -46,5 +49,6 @@ partition key: indices: key_idx0 key + 0 TYPE minmax GRANULARITY 1, key_idx1 key + 1 TYPE minmax GRANULARITY 1 granularity bytes: 10485760 -""".lstrip()) - node.query('ATTACH TABLE data') +""".lstrip(), + ) + node.query("ATTACH TABLE data") diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py index 75bc93921be..add45d262e6 100644 --- a/tests/integration/test_replicated_users/test.py +++ b/tests/integration/test_replicated_users/test.py @@ -5,11 +5,16 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/config.xml'], with_zookeeper=True, stay_alive=True) -node2 = cluster.add_instance('node2', main_configs=['configs/config.xml'], with_zookeeper=True, stay_alive=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True +) all_nodes = [node1, node2] + @pytest.fixture(scope="module") def started_cluster(): try: @@ -31,9 +36,10 @@ entities = [ Entity(keyword="ROLE", name="therole"), Entity(keyword="ROW POLICY", name="thepolicy", options=" ON default.t1"), Entity(keyword="QUOTA", name="thequota"), - Entity(keyword="SETTINGS PROFILE", name="theprofile") + Entity(keyword="SETTINGS PROFILE", name="theprofile"), ] + def get_entity_id(entity): return entity.keyword @@ -41,8 +47,12 @@ def get_entity_id(entity): @pytest.mark.parametrize("entity", entities, ids=get_entity_id) def test_create_replicated(started_cluster, entity): node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") - assert f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" in \ - node2.query_and_get_error(f"CREATE {entity.keyword} {entity.name} {entity.options}") + assert ( + f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" + in node2.query_and_get_error( + f"CREATE {entity.keyword} {entity.name} {entity.options}" + ) + ) node1.query(f"DROP {entity.keyword} {entity.name} {entity.options}") @@ -54,20 +64,27 @@ def test_create_and_delete_replicated(started_cluster, entity): @pytest.mark.parametrize("entity", entities, ids=get_entity_id) def test_create_replicated_on_cluster(started_cluster, entity): - assert f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" in \ - node1.query_and_get_error(f"CREATE {entity.keyword} {entity.name} ON CLUSTER default {entity.options}") + assert ( + f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" + in node1.query_and_get_error( + f"CREATE {entity.keyword} {entity.name} ON CLUSTER default {entity.options}" + ) + ) node1.query(f"DROP {entity.keyword} {entity.name} {entity.options}") @pytest.mark.parametrize("entity", entities, ids=get_entity_id) def test_create_replicated_if_not_exists_on_cluster(started_cluster, entity): - node1.query(f"CREATE {entity.keyword} IF NOT EXISTS {entity.name} ON CLUSTER default {entity.options}") + node1.query( + f"CREATE {entity.keyword} IF NOT EXISTS {entity.name} ON CLUSTER default {entity.options}" + ) node1.query(f"DROP {entity.keyword} {entity.name} {entity.options}") @pytest.mark.parametrize("entity", entities, ids=get_entity_id) def test_rename_replicated(started_cluster, entity): node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") - node2.query(f"ALTER {entity.keyword} {entity.name} {entity.options} RENAME TO {entity.name}2") + node2.query( + f"ALTER {entity.keyword} {entity.name} {entity.options} RENAME TO {entity.name}2" + ) node1.query(f"DROP {entity.keyword} {entity.name}2 {entity.options}") - diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index 13a605f2650..82cc5e757f1 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -4,9 +4,14 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='19.1.14', - with_installed_binary=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="19.1.14", + with_installed_binary=True, +) @pytest.fixture(scope="module") @@ -20,4 +25,9 @@ def start_cluster(): def test_different_versions(start_cluster): - assert node1.query("SELECT uniqExact(x) FROM (SELECT version() as x from remote('node{1,2}', system.one))") == "2\n" + assert ( + node1.query( + "SELECT uniqExact(x) FROM (SELECT version() as x from remote('node{1,2}', system.one))" + ) + == "2\n" + ) diff --git a/tests/integration/test_replication_credentials/test.py b/tests/integration/test_replication_credentials/test.py index 359a8fc4b0d..e5313cb6bd4 100644 --- a/tests/integration/test_replication_credentials/test.py +++ b/tests/integration/test_replication_credentials/test.py @@ -7,18 +7,27 @@ from helpers.cluster import ClickHouseCluster def _fill_nodes(nodes, shard): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], - with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], - with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/credentials1.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml", "configs/credentials1.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -38,20 +47,26 @@ def test_same_credentials(same_credentials_cluster): node1.query("insert into test_table values ('2017-06-16', 111, 0)") time.sleep(1) - assert node1.query("SELECT id FROM test_table order by id") == '111\n' - assert node2.query("SELECT id FROM test_table order by id") == '111\n' + assert node1.query("SELECT id FROM test_table order by id") == "111\n" + assert node2.query("SELECT id FROM test_table order by id") == "111\n" node2.query("insert into test_table values ('2017-06-17', 222, 1)") time.sleep(1) - assert node1.query("SELECT id FROM test_table order by id") == '111\n222\n' - assert node2.query("SELECT id FROM test_table order by id") == '111\n222\n' + assert node1.query("SELECT id FROM test_table order by id") == "111\n222\n" + assert node2.query("SELECT id FROM test_table order by id") == "111\n222\n" -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], - with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], - with_zookeeper=True) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/remote_servers.xml", "configs/no_credentials.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + main_configs=["configs/remote_servers.xml", "configs/no_credentials.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -71,20 +86,26 @@ def test_no_credentials(no_credentials_cluster): node3.query("insert into test_table values ('2017-06-18', 111, 0)") time.sleep(1) - assert node3.query("SELECT id FROM test_table order by id") == '111\n' - assert node4.query("SELECT id FROM test_table order by id") == '111\n' + assert node3.query("SELECT id FROM test_table order by id") == "111\n" + assert node4.query("SELECT id FROM test_table order by id") == "111\n" node4.query("insert into test_table values ('2017-06-19', 222, 1)") time.sleep(1) - assert node3.query("SELECT id FROM test_table order by id") == '111\n222\n' - assert node4.query("SELECT id FROM test_table order by id") == '111\n222\n' + assert node3.query("SELECT id FROM test_table order by id") == "111\n222\n" + assert node4.query("SELECT id FROM test_table order by id") == "111\n222\n" -node5 = cluster.add_instance('node5', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], - with_zookeeper=True) -node6 = cluster.add_instance('node6', main_configs=['configs/remote_servers.xml', 'configs/credentials2.xml'], - with_zookeeper=True) +node5 = cluster.add_instance( + "node5", + main_configs=["configs/remote_servers.xml", "configs/credentials1.xml"], + with_zookeeper=True, +) +node6 = cluster.add_instance( + "node6", + main_configs=["configs/remote_servers.xml", "configs/credentials2.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -104,14 +125,14 @@ def test_different_credentials(different_credentials_cluster): node5.query("insert into test_table values ('2017-06-20', 111, 0)") time.sleep(1) - assert node5.query("SELECT id FROM test_table order by id") == '111\n' - assert node6.query("SELECT id FROM test_table order by id") == '' + assert node5.query("SELECT id FROM test_table order by id") == "111\n" + assert node6.query("SELECT id FROM test_table order by id") == "" node6.query("insert into test_table values ('2017-06-21', 222, 1)") time.sleep(1) - assert node5.query("SELECT id FROM test_table order by id") == '111\n' - assert node6.query("SELECT id FROM test_table order by id") == '222\n' + assert node5.query("SELECT id FROM test_table order by id") == "111\n" + assert node6.query("SELECT id FROM test_table order by id") == "222\n" add_old = """ @@ -137,13 +158,19 @@ def test_different_credentials(different_credentials_cluster): node5.query("INSERT INTO test_table values('2017-06-21', 333, 1)") node6.query("SYSTEM SYNC REPLICA test_table", timeout=10) - assert node6.query("SELECT id FROM test_table order by id") == '111\n222\n333\n' + assert node6.query("SELECT id FROM test_table order by id") == "111\n222\n333\n" -node7 = cluster.add_instance('node7', main_configs=['configs/remote_servers.xml', 'configs/credentials1.xml'], - with_zookeeper=True) -node8 = cluster.add_instance('node8', main_configs=['configs/remote_servers.xml', 'configs/no_credentials.xml'], - with_zookeeper=True) +node7 = cluster.add_instance( + "node7", + main_configs=["configs/remote_servers.xml", "configs/credentials1.xml"], + with_zookeeper=True, +) +node8 = cluster.add_instance( + "node8", + main_configs=["configs/remote_servers.xml", "configs/no_credentials.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -163,14 +190,14 @@ def test_credentials_and_no_credentials(credentials_and_no_credentials_cluster): node7.query("insert into test_table values ('2017-06-21', 111, 0)") time.sleep(1) - assert node7.query("SELECT id FROM test_table order by id") == '111\n' - assert node8.query("SELECT id FROM test_table order by id") == '' + assert node7.query("SELECT id FROM test_table order by id") == "111\n" + assert node8.query("SELECT id FROM test_table order by id") == "" node8.query("insert into test_table values ('2017-06-22', 222, 1)") time.sleep(1) - assert node7.query("SELECT id FROM test_table order by id") == '111\n' - assert node8.query("SELECT id FROM test_table order by id") == '222\n' + assert node7.query("SELECT id FROM test_table order by id") == "111\n" + assert node8.query("SELECT id FROM test_table order by id") == "222\n" allow_empty = """ @@ -184,10 +211,11 @@ def test_credentials_and_no_credentials(credentials_and_no_credentials_cluster): """ # change state: Flip node7 to mixed auth/non-auth (allow node8) - node7.replace_config("/etc/clickhouse-server/config.d/credentials1.xml", - allow_empty) + node7.replace_config( + "/etc/clickhouse-server/config.d/credentials1.xml", allow_empty + ) node7.query("SYSTEM RELOAD CONFIG") node7.query("insert into test_table values ('2017-06-22', 333, 1)") node8.query("SYSTEM SYNC REPLICA test_table", timeout=10) - assert node8.query("SELECT id FROM test_table order by id") == '111\n222\n333\n' + assert node8.query("SELECT id FROM test_table order by id") == "111\n222\n333\n" diff --git a/tests/integration/test_replication_without_zookeeper/test.py b/tests/integration/test_replication_without_zookeeper/test.py index 26347b47d36..1b2bb6ef517 100644 --- a/tests/integration/test_replication_without_zookeeper/test.py +++ b/tests/integration/test_replication_without_zookeeper/test.py @@ -4,7 +4,12 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True, stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, + stay_alive=True, +) @pytest.fixture(scope="module") @@ -13,11 +18,11 @@ def start_cluster(): cluster.start() node1.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/replicated', 'node1') ORDER BY id PARTITION BY toYYYYMM(date); - ''' + """ ) yield cluster @@ -34,18 +39,28 @@ def drop_zk(zk): def test_startup_without_zookeeper(start_cluster): - node1.query("INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") + node1.query( + "INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)" + ) assert node1.query("SELECT COUNT(*) from test_table") == "3\n" - assert node1.query("SELECT is_readonly from system.replicas where table='test_table'") == "0\n" + assert ( + node1.query("SELECT is_readonly from system.replicas where table='test_table'") + == "0\n" + ) cluster.run_kazoo_commands_with_retries(drop_zk) time.sleep(5) assert node1.query("SELECT COUNT(*) from test_table") == "3\n" with pytest.raises(Exception): - node1.query("INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)") + node1.query( + "INSERT INTO test_table VALUES ('2018-10-01', 1), ('2018-10-02', 2), ('2018-10-03', 3)" + ) node1.restart_clickhouse() assert node1.query("SELECT COUNT(*) from test_table") == "3\n" - assert node1.query("SELECT is_readonly from system.replicas where table='test_table'") == "1\n" + assert ( + node1.query("SELECT is_readonly from system.replicas where table='test_table'") + == "1\n" + ) diff --git a/tests/integration/test_restart_server/test.py b/tests/integration/test_restart_server/test.py index 47797f7c4a5..180f8240d01 100755 --- a/tests/integration/test_restart_server/test.py +++ b/tests/integration/test_restart_server/test.py @@ -2,7 +2,8 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True) +node = cluster.add_instance("node", stay_alive=True) + @pytest.fixture(scope="module") def start_cluster(): @@ -19,4 +20,3 @@ def test_drop_memory_database(start_cluster): node.query("DROP DATABASE test") node.restart_clickhouse(kill=True) assert node.query("SHOW DATABASES LIKE 'test'").strip() == "" - diff --git a/tests/integration/test_restore_replica/test.py b/tests/integration/test_restore_replica/test.py index 4013b5b474c..0b11cdf7512 100644 --- a/tests/integration/test_restore_replica/test.py +++ b/tests/integration/test_restore_replica/test.py @@ -5,23 +5,29 @@ from helpers.cluster import ClickHouseCluster from helpers.cluster import ClickHouseKiller from helpers.test_tools import assert_eq_with_retry + def fill_nodes(nodes): for node in nodes: node.query( - ''' + """ CREATE TABLE test(n UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}') ORDER BY n PARTITION BY n % 10; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) + cluster = ClickHouseCluster(__file__) -configs =["configs/remote_servers.xml"] +configs = ["configs/remote_servers.xml"] -node_1 = cluster.add_instance('replica1', with_zookeeper=True, main_configs=configs) -node_2 = cluster.add_instance('replica2', with_zookeeper=True, main_configs=configs) -node_3 = cluster.add_instance('replica3', with_zookeeper=True, main_configs=configs) +node_1 = cluster.add_instance("replica1", with_zookeeper=True, main_configs=configs) +node_2 = cluster.add_instance("replica2", with_zookeeper=True, main_configs=configs) +node_3 = cluster.add_instance("replica3", with_zookeeper=True, main_configs=configs) nodes = [node_1, node_2, node_3] + def fill_table(): node_1.query("TRUNCATE TABLE test") @@ -38,6 +44,7 @@ def fill_table(): node_1.query("INSERT INTO test SELECT number + 800 FROM numbers(200)") check_data(499500, 1000) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -51,26 +58,30 @@ def start_cluster(): finally: cluster.shutdown() + def check_data(_sum: int, count: int) -> None: res = "{}\t{}\n".format(_sum, count) assert_eq_with_retry(node_1, "SELECT sum(n), count() FROM test", res) assert_eq_with_retry(node_2, "SELECT sum(n), count() FROM test", res) assert_eq_with_retry(node_3, "SELECT sum(n), count() FROM test", res) + def check_after_restoration(): check_data(1999000, 2000) for node in nodes: node.query_and_get_error("SYSTEM RESTORE REPLICA test") + def test_restore_replica_invalid_tables(start_cluster): print("Checking the invocation on non-existent and non-replicated tables") node_1.query_and_get_error("SYSTEM RESTORE REPLICA i_dont_exist_42") node_1.query_and_get_error("SYSTEM RESTORE REPLICA no_db.i_dont_exist_42") node_1.query_and_get_error("SYSTEM RESTORE REPLICA system.numbers") + def test_restore_replica_sequential(start_cluster): - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") fill_table() print("Deleting root ZK path metadata") @@ -78,7 +89,9 @@ def test_restore_replica_sequential(start_cluster): assert zk.exists("/clickhouse/tables/test") is None node_1.query("SYSTEM RESTART REPLICA test") - node_1.query_and_get_error("INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0") + node_1.query_and_get_error( + "INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0" + ) print("Restoring replica1") @@ -101,8 +114,9 @@ def test_restore_replica_sequential(start_cluster): check_after_restoration() + def test_restore_replica_parallel(start_cluster): - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") fill_table() print("Deleting root ZK path metadata") @@ -110,7 +124,9 @@ def test_restore_replica_parallel(start_cluster): assert zk.exists("/clickhouse/tables/test") is None node_1.query("SYSTEM RESTART REPLICA test") - node_1.query_and_get_error("INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0") + node_1.query_and_get_error( + "INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0" + ) print("Restoring replicas in parallel") @@ -126,8 +142,9 @@ def test_restore_replica_parallel(start_cluster): check_after_restoration() + def test_restore_replica_alive_replicas(start_cluster): - zk = cluster.get_kazoo_client('zoo1') + zk = cluster.get_kazoo_client("zoo1") fill_table() print("Deleting replica2 path, trying to restore replica1") diff --git a/tests/integration/test_rocksdb_options/test.py b/tests/integration/test_rocksdb_options/test.py index e8542749d8d..a00d3528eed 100644 --- a/tests/integration/test_rocksdb_options/test.py +++ b/tests/integration/test_rocksdb_options/test.py @@ -9,10 +9,12 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/rocksdb.xml'], stay_alive=True) +node = cluster.add_instance( + "node", main_configs=["configs/rocksdb.xml"], stay_alive=True +) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def start_cluster(): try: cluster.start() @@ -20,66 +22,138 @@ def start_cluster(): finally: cluster.shutdown() + def test_valid_options(start_cluster): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); DROP TABLE test; - """) + """ + ) + def test_invalid_options(start_cluster): - node.exec_in_container(['bash', '-c', "sed -i 's/max_background_jobs/no_such_option/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/max_background_jobs/no_such_option/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() with pytest.raises(QueryRuntimeException): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); - """) - node.exec_in_container(['bash', '-c', "sed -i 's/no_such_option/max_background_jobs/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + """ + ) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/no_such_option/max_background_jobs/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() + def test_table_valid_options(start_cluster): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); DROP TABLE test; - """) + """ + ) + def test_table_invalid_options(start_cluster): - node.exec_in_container(['bash', '-c', "sed -i 's/max_open_files/no_such_table_option/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/max_open_files/no_such_table_option/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() with pytest.raises(QueryRuntimeException): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); - """) - node.exec_in_container(['bash', '-c', "sed -i 's/no_such_table_option/max_open_files/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + """ + ) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/no_such_table_option/max_open_files/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() + def test_valid_column_family_options(start_cluster): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); DROP TABLE test; - """) + """ + ) + def test_invalid_column_family_options(start_cluster): - node.exec_in_container(['bash', '-c', "sed -i 's/num_levels/no_such_column_family_option/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/num_levels/no_such_column_family_option/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() with pytest.raises(QueryRuntimeException): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); - """) - node.exec_in_container(['bash', '-c', "sed -i 's/no_such_column_family_option/num_levels/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + """ + ) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/no_such_column_family_option/num_levels/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() + def test_table_valid_column_family_options(start_cluster): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); DROP TABLE test; - """) + """ + ) + def test_table_invalid_column_family_options(start_cluster): - node.exec_in_container(['bash', '-c', "sed -i 's/max_bytes_for_level_base/no_such_table_column_family_option/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/max_bytes_for_level_base/no_such_table_column_family_option/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() with pytest.raises(QueryRuntimeException): - node.query(""" + node.query( + """ CREATE TABLE test (key UInt64, value String) Engine=EmbeddedRocksDB PRIMARY KEY(key); - """) - node.exec_in_container(['bash', '-c', "sed -i 's/no_such_table_column_family_option/max_bytes_for_level_base/g' /etc/clickhouse-server/config.d/rocksdb.xml"]) + """ + ) + node.exec_in_container( + [ + "bash", + "-c", + "sed -i 's/no_such_table_column_family_option/max_bytes_for_level_base/g' /etc/clickhouse-server/config.d/rocksdb.xml", + ] + ) node.restart_clickhouse() diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 7600bc73b16..44ce0e13f2f 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -3,14 +3,16 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") session_id_counter = 0 + + def new_session_id(): global session_id_counter session_id_counter += 1 - return 'session #' + str(session_id_counter) + return "session #" + str(session_id_counter) @pytest.fixture(scope="module", autouse=True) @@ -18,7 +20,9 @@ def started_cluster(): try: cluster.start() - instance.query("CREATE TABLE test_table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE test_table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) instance.query("INSERT INTO test_table VALUES (1,5), (2,10)") yield cluster @@ -38,69 +42,91 @@ def cleanup_after_test(): def test_create_role(): instance.query("CREATE USER A") - instance.query('CREATE ROLE R1') + instance.query("CREATE ROLE R1") - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) - instance.query('GRANT SELECT ON test_table TO R1') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + instance.query("GRANT SELECT ON test_table TO R1") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) - instance.query('GRANT R1 TO A') - assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" + instance.query("GRANT R1 TO A") + assert instance.query("SELECT * FROM test_table", user="A") == "1\t5\n2\t10\n" - instance.query('REVOKE R1 FROM A') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + instance.query("REVOKE R1 FROM A") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) def test_grant_role_to_role(): instance.query("CREATE USER A") - instance.query('CREATE ROLE R1') - instance.query('CREATE ROLE R2') + instance.query("CREATE ROLE R1") + instance.query("CREATE ROLE R2") - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) - instance.query('GRANT R1 TO A') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + instance.query("GRANT R1 TO A") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) - instance.query('GRANT R2 TO R1') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + instance.query("GRANT R2 TO R1") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) - instance.query('GRANT SELECT ON test_table TO R2') - assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" + instance.query("GRANT SELECT ON test_table TO R2") + assert instance.query("SELECT * FROM test_table", user="A") == "1\t5\n2\t10\n" def test_combine_privileges(): instance.query("CREATE USER A ") - instance.query('CREATE ROLE R1') - instance.query('CREATE ROLE R2') + instance.query("CREATE ROLE R1") + instance.query("CREATE ROLE R2") - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) - instance.query('GRANT R1 TO A') - instance.query('GRANT SELECT(x) ON test_table TO R1') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='A') - assert instance.query("SELECT x FROM test_table", user='A') == "1\n2\n" + instance.query("GRANT R1 TO A") + instance.query("GRANT SELECT(x) ON test_table TO R1") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="A" + ) + assert instance.query("SELECT x FROM test_table", user="A") == "1\n2\n" - instance.query('GRANT SELECT(y) ON test_table TO R2') - instance.query('GRANT R2 TO A') - assert instance.query("SELECT * FROM test_table", user='A') == "1\t5\n2\t10\n" + instance.query("GRANT SELECT(y) ON test_table TO R2") + instance.query("GRANT R2 TO A") + assert instance.query("SELECT * FROM test_table", user="A") == "1\t5\n2\t10\n" def test_admin_option(): instance.query("CREATE USER A") instance.query("CREATE USER B") - instance.query('CREATE ROLE R1') + instance.query("CREATE ROLE R1") - instance.query('GRANT SELECT ON test_table TO R1') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='B') + instance.query("GRANT SELECT ON test_table TO R1") + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="B" + ) - instance.query('GRANT R1 TO A') - assert "Not enough privileges" in instance.query_and_get_error("GRANT R1 TO B", user='A') - assert "Not enough privileges" in instance.query_and_get_error("SELECT * FROM test_table", user='B') + instance.query("GRANT R1 TO A") + assert "Not enough privileges" in instance.query_and_get_error( + "GRANT R1 TO B", user="A" + ) + assert "Not enough privileges" in instance.query_and_get_error( + "SELECT * FROM test_table", user="B" + ) - instance.query('GRANT R1 TO A WITH ADMIN OPTION') - instance.query("GRANT R1 TO B", user='A') - assert instance.query("SELECT * FROM test_table", user='B') == "1\t5\n2\t10\n" + instance.query("GRANT R1 TO A WITH ADMIN OPTION") + instance.query("GRANT R1 TO B", user="A") + assert instance.query("SELECT * FROM test_table", user="B") == "1\t5\n2\t10\n" def test_revoke_requires_admin_option(): @@ -111,37 +137,37 @@ def test_revoke_requires_admin_option(): assert instance.query("SHOW GRANTS FOR B") == "GRANT R1 TO B\n" expected_error = "necessary to have the role R1 granted" - assert expected_error in instance.query_and_get_error("REVOKE R1 FROM B", user='A') + assert expected_error in instance.query_and_get_error("REVOKE R1 FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1 TO B\n" instance.query("GRANT R1 TO A") expected_error = "granted, but without ADMIN option" - assert expected_error in instance.query_and_get_error("REVOKE R1 FROM B", user='A') + assert expected_error in instance.query_and_get_error("REVOKE R1 FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1 TO B\n" instance.query("GRANT R1 TO A WITH ADMIN OPTION") - instance.query("REVOKE R1 FROM B", user='A') + instance.query("REVOKE R1 FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("GRANT R1 TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1 TO B\n" - instance.query("REVOKE ALL FROM B", user='A') + instance.query("REVOKE ALL FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("GRANT R1, R2 TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1, R2 TO B\n" expected_error = "necessary to have the role R2 granted" - assert expected_error in instance.query_and_get_error("REVOKE ALL FROM B", user='A') + assert expected_error in instance.query_and_get_error("REVOKE ALL FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1, R2 TO B\n" - instance.query("REVOKE ALL EXCEPT R2 FROM B", user='A') + instance.query("REVOKE ALL EXCEPT R2 FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "GRANT R2 TO B\n" instance.query("GRANT R2 TO A WITH ADMIN OPTION") - instance.query("REVOKE ALL FROM B", user='A') + instance.query("REVOKE ALL FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" instance.query("GRANT R1, R2 TO B") assert instance.query("SHOW GRANTS FOR B") == "GRANT R1, R2 TO B\n" - instance.query("REVOKE ALL FROM B", user='A') + instance.query("REVOKE ALL FROM B", user="A") assert instance.query("SHOW GRANTS FOR B") == "" @@ -151,19 +177,29 @@ def test_set_role(): instance.query("GRANT R1, R2 TO A") session_id = new_session_id() - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([["R1", 0, 1], ["R2", 0, 1]]) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([["R1", 0, 1], ["R2", 0, 1]]) - instance.http_query('SET ROLE R1', user='A', params={'session_id':session_id}) - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([["R1", 0, 1]]) + instance.http_query("SET ROLE R1", user="A", params={"session_id": session_id}) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([["R1", 0, 1]]) - instance.http_query('SET ROLE R2', user='A', params={'session_id':session_id}) - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([["R2", 0, 1]]) + instance.http_query("SET ROLE R2", user="A", params={"session_id": session_id}) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([["R2", 0, 1]]) - instance.http_query('SET ROLE NONE', user='A', params={'session_id':session_id}) - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([]) + instance.http_query("SET ROLE NONE", user="A", params={"session_id": session_id}) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([]) - instance.http_query('SET ROLE DEFAULT', user='A', params={'session_id':session_id}) - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([["R1", 0, 1], ["R2", 0, 1]]) + instance.http_query("SET ROLE DEFAULT", user="A", params={"session_id": session_id}) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([["R1", 0, 1], ["R2", 0, 1]]) def test_changing_default_roles_affects_new_sessions_only(): @@ -172,105 +208,201 @@ def test_changing_default_roles_affects_new_sessions_only(): instance.query("GRANT R1, R2 TO A") session_id = new_session_id() - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([["R1", 0, 1], ["R2", 0, 1]]) - instance.query('SET DEFAULT ROLE R2 TO A') - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':session_id}) == TSV([["R1", 0, 0], ["R2", 0, 1]]) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([["R1", 0, 1], ["R2", 0, 1]]) + instance.query("SET DEFAULT ROLE R2 TO A") + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": session_id} + ) == TSV([["R1", 0, 0], ["R2", 0, 1]]) other_session_id = new_session_id() - assert instance.http_query('SHOW CURRENT ROLES', user='A', params={'session_id':other_session_id}) == TSV([["R2", 0, 1]]) + assert instance.http_query( + "SHOW CURRENT ROLES", user="A", params={"session_id": other_session_id} + ) == TSV([["R2", 0, 1]]) def test_introspection(): instance.query("CREATE USER A") instance.query("CREATE USER B") - instance.query('CREATE ROLE R1') - instance.query('CREATE ROLE R2') - instance.query('GRANT R1 TO A') - instance.query('GRANT R2 TO B WITH ADMIN OPTION') - instance.query('GRANT SELECT ON test.table TO A, R2') - instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') - instance.query('REVOKE SELECT(x) ON test.table FROM R2') + instance.query("CREATE ROLE R1") + instance.query("CREATE ROLE R2") + instance.query("GRANT R1 TO A") + instance.query("GRANT R2 TO B WITH ADMIN OPTION") + instance.query("GRANT SELECT ON test.table TO A, R2") + instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") + instance.query("REVOKE SELECT(x) ON test.table FROM R2") assert instance.query("SHOW ROLES") == TSV(["R1", "R2"]) assert instance.query("SHOW CREATE ROLE R1") == TSV(["CREATE ROLE R1"]) assert instance.query("SHOW CREATE ROLE R2") == TSV(["CREATE ROLE R2"]) - assert instance.query("SHOW CREATE ROLES R1, R2") == TSV(["CREATE ROLE R1", "CREATE ROLE R2"]) - assert instance.query("SHOW CREATE ROLES") == TSV(["CREATE ROLE R1", "CREATE ROLE R2"]) + assert instance.query("SHOW CREATE ROLES R1, R2") == TSV( + ["CREATE ROLE R1", "CREATE ROLE R2"] + ) + assert instance.query("SHOW CREATE ROLES") == TSV( + ["CREATE ROLE R1", "CREATE ROLE R2"] + ) - assert instance.query("SHOW GRANTS FOR A") == TSV(["GRANT SELECT ON test.table TO A", "GRANT R1 TO A"]) + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT SELECT ON test.table TO A", "GRANT R1 TO A"] + ) assert instance.query("SHOW GRANTS FOR B") == TSV( - ["GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION"]) + [ + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT R2 TO B WITH ADMIN OPTION", + ] + ) assert instance.query("SHOW GRANTS FOR R1") == "" assert instance.query("SHOW GRANTS FOR R2") == TSV( - ["GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2"]) + ["GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2"] + ) - assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A", "GRANT R1 TO A"]) - assert instance.query("SHOW GRANTS", user='B') == TSV( - ["GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION"]) - assert instance.query("SHOW CURRENT ROLES", user='A') == TSV([["R1", 0, 1]]) - assert instance.query("SHOW CURRENT ROLES", user='B') == TSV([["R2", 1, 1]]) - assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([["R1", 0, 1, 1]]) - assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([["R2", 1, 1, 1]]) + assert instance.query("SHOW GRANTS", user="A") == TSV( + ["GRANT SELECT ON test.table TO A", "GRANT R1 TO A"] + ) + assert instance.query("SHOW GRANTS", user="B") == TSV( + [ + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT R2 TO B WITH ADMIN OPTION", + ] + ) + assert instance.query("SHOW CURRENT ROLES", user="A") == TSV([["R1", 0, 1]]) + assert instance.query("SHOW CURRENT ROLES", user="B") == TSV([["R2", 1, 1]]) + assert instance.query("SHOW ENABLED ROLES", user="A") == TSV([["R1", 0, 1, 1]]) + assert instance.query("SHOW ENABLED ROLES", user="B") == TSV([["R2", 1, 1, 1]]) - expected_access1 = "CREATE ROLE R1\n" \ - "CREATE ROLE R2\n" + expected_access1 = "CREATE ROLE R1\n" "CREATE ROLE R2\n" expected_access2 = "GRANT R1 TO A\n" expected_access3 = "GRANT R2 TO B WITH ADMIN OPTION" assert expected_access1 in instance.query("SHOW ACCESS") assert expected_access2 in instance.query("SHOW ACCESS") assert expected_access3 in instance.query("SHOW ACCESS") - assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") == \ - TSV([["R1", "local directory"], - ["R2", "local directory"]]) + assert instance.query( + "SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name" + ) == TSV([["R1", "local directory"], ["R2", "local directory"]]) assert instance.query( - "SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, database, table, column, is_partial_revoke, grant_option") == \ - TSV([["A", "\\N", "SELECT", "test", "table", "\\N", 0, 0], - ["B", "\\N", "CREATE", "\\N", "\\N", "\\N", 0, 1], - ["\\N", "R2", "SELECT", "test", "table", "x", 1, 0], - ["\\N", "R2", "SELECT", "test", "table", "\\N", 0, 0]]) + "SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, database, table, column, is_partial_revoke, grant_option" + ) == TSV( + [ + ["A", "\\N", "SELECT", "test", "table", "\\N", 0, 0], + ["B", "\\N", "CREATE", "\\N", "\\N", "\\N", 0, 1], + ["\\N", "R2", "SELECT", "test", "table", "x", 1, 0], + ["\\N", "R2", "SELECT", "test", "table", "\\N", 0, 0], + ] + ) assert instance.query( - "SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name") == \ - TSV([["A", "\\N", "R1", 1, 0], - ["B", "\\N", "R2", 1, 1]]) + "SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name" + ) == TSV([["A", "\\N", "R1", 1, 0], ["B", "\\N", "R2", 1, 1]]) - assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='A') == TSV([["R1", 0, 1]]) - assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='B') == TSV([["R2", 1, 1]]) - assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='A') == TSV([["R1", 0, 1, 1]]) - assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='B') == TSV([["R2", 1, 1, 1]]) + assert instance.query( + "SELECT * from system.current_roles ORDER BY role_name", user="A" + ) == TSV([["R1", 0, 1]]) + assert instance.query( + "SELECT * from system.current_roles ORDER BY role_name", user="B" + ) == TSV([["R2", 1, 1]]) + assert instance.query( + "SELECT * from system.enabled_roles ORDER BY role_name", user="A" + ) == TSV([["R1", 0, 1, 1]]) + assert instance.query( + "SELECT * from system.enabled_roles ORDER BY role_name", user="B" + ) == TSV([["R2", 1, 1, 1]]) def test_function_current_roles(): instance.query("CREATE USER A") - instance.query('CREATE ROLE R1, R2, R3, R4') - instance.query('GRANT R4 TO R2') - instance.query('GRANT R1,R2,R3 TO A') + instance.query("CREATE ROLE R1, R2, R3, R4") + instance.query("GRANT R4 TO R2") + instance.query("GRANT R1,R2,R3 TO A") session_id = new_session_id() - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R1','R2','R3']\t['R1','R2','R3']\t['R1','R2','R3','R4']\n" + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R1','R2','R3']\t['R1','R2','R3']\t['R1','R2','R3','R4']\n" + ) - instance.http_query('SET ROLE R1', user='A', params={'session_id':session_id}) - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R1','R2','R3']\t['R1']\t['R1']\n" + instance.http_query("SET ROLE R1", user="A", params={"session_id": session_id}) + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R1','R2','R3']\t['R1']\t['R1']\n" + ) - instance.http_query('SET ROLE R2', user='A', params={'session_id':session_id}) - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R1','R2','R3']\t['R2']\t['R2','R4']\n" + instance.http_query("SET ROLE R2", user="A", params={"session_id": session_id}) + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R1','R2','R3']\t['R2']\t['R2','R4']\n" + ) - instance.http_query('SET ROLE NONE', user='A', params={'session_id':session_id}) - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R1','R2','R3']\t[]\t[]\n" + instance.http_query("SET ROLE NONE", user="A", params={"session_id": session_id}) + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R1','R2','R3']\t[]\t[]\n" + ) - instance.http_query('SET ROLE DEFAULT', user='A', params={'session_id':session_id}) - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R1','R2','R3']\t['R1','R2','R3']\t['R1','R2','R3','R4']\n" + instance.http_query("SET ROLE DEFAULT", user="A", params={"session_id": session_id}) + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R1','R2','R3']\t['R1','R2','R3']\t['R1','R2','R3','R4']\n" + ) - instance.query('SET DEFAULT ROLE R2 TO A') - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R2']\t['R1','R2','R3']\t['R1','R2','R3','R4']\n" + instance.query("SET DEFAULT ROLE R2 TO A") + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R2']\t['R1','R2','R3']\t['R1','R2','R3','R4']\n" + ) - instance.query('REVOKE R3 FROM A') - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R2']\t['R1','R2']\t['R1','R2','R4']\n" + instance.query("REVOKE R3 FROM A") + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R2']\t['R1','R2']\t['R1','R2','R4']\n" + ) - instance.query('REVOKE R2 FROM A') - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "[]\t['R1']\t['R1']\n" + instance.query("REVOKE R2 FROM A") + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "[]\t['R1']\t['R1']\n" + ) - instance.query('SET DEFAULT ROLE ALL TO A') - assert instance.http_query('SELECT defaultRoles(), currentRoles(), enabledRoles()', user='A', params={'session_id':session_id}) == "['R1']\t['R1']\t['R1']\n" + instance.query("SET DEFAULT ROLE ALL TO A") + assert ( + instance.http_query( + "SELECT defaultRoles(), currentRoles(), enabledRoles()", + user="A", + params={"session_id": session_id}, + ) + == "['R1']\t['R1']\t['R1']\n" + ) diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index 66a35bea06b..c0ebfc7a070 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -7,22 +7,36 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, TSV cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/row_policy.xml", "configs/users.d/another_user.xml", - "configs/users.d/any_join_distinct_right_table_keys.xml"], - with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/row_policy.xml", "configs/users.d/another_user.xml", - "configs/users.d/any_join_distinct_right_table_keys.xml"], - with_zookeeper=True) +node = cluster.add_instance( + "node", + main_configs=["configs/config.d/remote_servers.xml"], + user_configs=[ + "configs/users.d/row_policy.xml", + "configs/users.d/another_user.xml", + "configs/users.d/any_join_distinct_right_table_keys.xml", + ], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/config.d/remote_servers.xml"], + user_configs=[ + "configs/users.d/row_policy.xml", + "configs/users.d/another_user.xml", + "configs/users.d/any_join_distinct_right_table_keys.xml", + ], + with_zookeeper=True, +) nodes = [node, node2] def copy_policy_xml(local_file_name, reload_immediately=True): script_dir = os.path.dirname(os.path.realpath(__file__)) for current_node in nodes: - current_node.copy_file_to_container(os.path.join(script_dir, local_file_name), - '/etc/clickhouse-server/users.d/row_policy.xml') + current_node.copy_file_to_container( + os.path.join(script_dir, local_file_name), + "/etc/clickhouse-server/users.d/row_policy.xml", + ) if reload_immediately: current_node.query("SYSTEM RELOAD CONFIG") @@ -33,7 +47,8 @@ def started_cluster(): cluster.start() for current_node in nodes: - current_node.query(''' + current_node.query( + """ CREATE DATABASE mydb; CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; @@ -52,7 +67,8 @@ def started_cluster(): INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - ''') + """ + ) node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") @@ -68,303 +84,602 @@ def reset_policies(): try: yield finally: - copy_policy_xml('normal_filters.xml') + copy_policy_xml("normal_filters.xml") for current_node in nodes: current_node.query("DROP POLICY IF EXISTS pA, pB ON mydb.filtered_table1") def test_smoke(): assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) assert node.query("SELECT a FROM mydb.filtered_table1") == TSV([[1], [1]]) assert node.query("SELECT b FROM mydb.filtered_table1") == TSV([[0], [1]]) - assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a = 1") == TSV([[1], [1]]) - assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a IN (1)") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a = 1") == TSV( + [[1], [1]] + ) + assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a IN (1)") == TSV( + [[1], [1]] + ) assert node.query("SELECT a = 1 FROM mydb.filtered_table1") == TSV([[1], [1]]) assert node.query("SELECT a FROM mydb.filtered_table3") == TSV([[0], [1]]) assert node.query("SELECT b FROM mydb.filtered_table3") == TSV([[1], [0]]) assert node.query("SELECT c FROM mydb.filtered_table3") == TSV([[1], [1]]) assert node.query("SELECT a + b FROM mydb.filtered_table3") == TSV([[1], [1]]) - assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV( + [[0], [1]] + ) assert node.query("SELECT c = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) def test_join(): assert node.query( - "SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == TSV( - [[1, 0, 1, 1], [1, 1, 1, 1]]) + "SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b" + ) == TSV([[1, 0, 1, 1], [1, 1, 1, 1]]) assert node.query( - "SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == TSV( - [[1, 1, 1, 0]]) + "SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a" + ) == TSV([[1, 1, 1, 0]]) def test_cannot_trick_row_policy_with_keyword_with(): - assert node.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == TSV([[0], [0]]) - assert node.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == TSV([[0], [1]]) + assert node.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == TSV( + [[0], [0]] + ) + assert node.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == TSV( + [[0], [1]] + ) - assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1 WHERE a >= 0 AND b >= 0 SETTINGS optimize_move_to_prewhere = 0") == TSV([[1, 0], [1, 1]]) - assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1 PREWHERE a >= 0 AND b >= 0") == TSV([[1, 0], [1, 1]]) - assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1 PREWHERE a >= 0 WHERE b >= 0") == TSV([[1, 0], [1, 1]]) - assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1 PREWHERE b >= 0 WHERE a >= 0") == TSV([[1, 0], [1, 1]]) + assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == TSV( + [[1, 0], [1, 1]] + ) + assert node.query( + "WITH 0 AS a SELECT * FROM mydb.filtered_table1 WHERE a >= 0 AND b >= 0 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[1, 0], [1, 1]]) + assert node.query( + "WITH 0 AS a SELECT * FROM mydb.filtered_table1 PREWHERE a >= 0 AND b >= 0" + ) == TSV([[1, 0], [1, 1]]) + assert node.query( + "WITH 0 AS a SELECT * FROM mydb.filtered_table1 PREWHERE a >= 0 WHERE b >= 0" + ) == TSV([[1, 0], [1, 1]]) + assert node.query( + "WITH 0 AS a SELECT * FROM mydb.filtered_table1 PREWHERE b >= 0 WHERE a >= 0" + ) == TSV([[1, 0], [1, 1]]) - assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1]]) - assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 WHERE a >= 0 AND b >= 0 SETTINGS optimize_move_to_prewhere = 0") == TSV([[0, 0], [0, 1]]) - assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 PREWHERE a >= 0 AND b >= 0") == TSV([[0, 0], [0, 1]]) - assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 PREWHERE a >= 0 WHERE b >= 0") == TSV([[0, 0], [0, 1]]) - assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 PREWHERE b >= 0 WHERE a >= 0") == TSV([[0, 0], [0, 1]]) + assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == TSV( + [[0, 0], [0, 1]] + ) + assert node.query( + "WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 WHERE a >= 0 AND b >= 0 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[0, 0], [0, 1]]) + assert node.query( + "WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 PREWHERE a >= 0 AND b >= 0" + ) == TSV([[0, 0], [0, 1]]) + assert node.query( + "WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 PREWHERE a >= 0 WHERE b >= 0" + ) == TSV([[0, 0], [0, 1]]) + assert node.query( + "WITH 0 AS a SELECT a, b FROM mydb.filtered_table1 PREWHERE b >= 0 WHERE a >= 0" + ) == TSV([[0, 0], [0, 1]]) - assert node.query("WITH 0 AS c SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - assert node.query("WITH 0 AS c SELECT * FROM mydb.filtered_table3 WHERE c >= 0 AND a >= 0 SETTINGS optimize_move_to_prewhere = 0") == TSV([[0, 1], [1, 0]]) - assert node.query("WITH 0 AS c SELECT * FROM mydb.filtered_table3 PREWHERE c >= 0 AND a >= 0") == TSV([[0, 1], [1, 0]]) - assert node.query("WITH 0 AS c SELECT * FROM mydb.filtered_table3 PREWHERE c >= 0 WHERE a >= 0") == TSV([[0, 1], [1, 0]]) - assert node.query("WITH 0 AS c SELECT * FROM mydb.filtered_table3 PREWHERE a >= 0 WHERE c >= 0") == TSV([[0, 1], [1, 0]]) + assert node.query("WITH 0 AS c SELECT * FROM mydb.filtered_table3") == TSV( + [[0, 1], [1, 0]] + ) + assert node.query( + "WITH 0 AS c SELECT * FROM mydb.filtered_table3 WHERE c >= 0 AND a >= 0 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[0, 1], [1, 0]]) + assert node.query( + "WITH 0 AS c SELECT * FROM mydb.filtered_table3 PREWHERE c >= 0 AND a >= 0" + ) == TSV([[0, 1], [1, 0]]) + assert node.query( + "WITH 0 AS c SELECT * FROM mydb.filtered_table3 PREWHERE c >= 0 WHERE a >= 0" + ) == TSV([[0, 1], [1, 0]]) + assert node.query( + "WITH 0 AS c SELECT * FROM mydb.filtered_table3 PREWHERE a >= 0 WHERE c >= 0" + ) == TSV([[0, 1], [1, 0]]) - assert node.query("WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3") == TSV([[0, 1, 0], [1, 0, 0]]) - assert node.query("WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 WHERE c >= 0 AND a >= 0 SETTINGS optimize_move_to_prewhere = 0") == TSV([[0, 1, 0], [1, 0, 0]]) - assert node.query("WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 PREWHERE c >= 0 AND a >= 0") == TSV([[0, 1, 0], [1, 0, 0]]) - assert node.query("WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 PREWHERE c >= 0 WHERE a >= 0") == TSV([[0, 1, 0], [1, 0, 0]]) - assert node.query("WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 PREWHERE a >= 0 WHERE c >= 0") == TSV([[0, 1, 0], [1, 0, 0]]) + assert node.query("WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3") == TSV( + [[0, 1, 0], [1, 0, 0]] + ) + assert node.query( + "WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 WHERE c >= 0 AND a >= 0 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[0, 1, 0], [1, 0, 0]]) + assert node.query( + "WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 PREWHERE c >= 0 AND a >= 0" + ) == TSV([[0, 1, 0], [1, 0, 0]]) + assert node.query( + "WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 PREWHERE c >= 0 WHERE a >= 0" + ) == TSV([[0, 1, 0], [1, 0, 0]]) + assert node.query( + "WITH 0 AS c SELECT a, b, c FROM mydb.filtered_table3 PREWHERE a >= 0 WHERE c >= 0" + ) == TSV([[0, 1, 0], [1, 0, 0]]) def test_policy_from_users_xml_affects_only_user_assigned(): assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1", user="another") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert node.query("SELECT * FROM mydb.filtered_table2", user="another") == TSV( - [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]] + ) - assert node.query("SELECT * FROM mydb.local") == TSV([[1, 0], [1, 1], [2, 0], [2, 1]]) - assert node.query("SELECT * FROM mydb.local", user="another") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.local") == TSV( + [[1, 0], [1, 1], [2, 0], [2, 1]] + ) + assert node.query("SELECT * FROM mydb.local", user="another") == TSV( + [[1, 0], [1, 1]] + ) def test_with_prewhere(): - copy_policy_xml('normal_filter2_table2.xml') - assert node.query("SELECT * FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0") == TSV([[4, 3, 2, 1]]) - assert node.query("SELECT a FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0") == TSV([[4]]) - assert node.query("SELECT a, b FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0") == TSV([[4, 3]]) - assert node.query("SELECT b, c FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0") == TSV([[3, 2]]) - assert node.query("SELECT d FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0") == TSV([[1]]) + copy_policy_xml("normal_filter2_table2.xml") + assert node.query( + "SELECT * FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[4, 3, 2, 1]]) + assert node.query( + "SELECT a FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[4]]) + assert node.query( + "SELECT a, b FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[4, 3]]) + assert node.query( + "SELECT b, c FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[3, 2]]) + assert node.query( + "SELECT d FROM mydb.filtered_table2 WHERE a > 1 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[1]]) - assert node.query("SELECT * FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[4, 3, 2, 1]]) - assert node.query("SELECT a FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[4]]) - assert node.query("SELECT a, b FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[4, 3]]) - assert node.query("SELECT b, c FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[3, 2]]) - assert node.query("SELECT d FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[1]]) + assert node.query("SELECT * FROM mydb.filtered_table2 PREWHERE a > 1") == TSV( + [[4, 3, 2, 1]] + ) + assert node.query("SELECT a FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[4]]) + assert node.query("SELECT a, b FROM mydb.filtered_table2 PREWHERE a > 1") == TSV( + [[4, 3]] + ) + assert node.query("SELECT b, c FROM mydb.filtered_table2 PREWHERE a > 1") == TSV( + [[3, 2]] + ) + assert node.query("SELECT d FROM mydb.filtered_table2 PREWHERE a > 1") == TSV([[1]]) - assert node.query("SELECT * FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[1, 2, 3, 4]]) - assert node.query("SELECT a FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[1]]) - assert node.query("SELECT b FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[2]]) - assert node.query("SELECT a, b FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[1, 2]]) - assert node.query("SELECT a, c FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[1, 3]]) - assert node.query("SELECT b, d FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[2, 4]]) - assert node.query("SELECT c, d FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10") == TSV([[3, 4]]) + assert node.query( + "SELECT * FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[1, 2, 3, 4]]) + assert node.query( + "SELECT a FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[1]]) + assert node.query( + "SELECT b FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[2]]) + assert node.query( + "SELECT a, b FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[1, 2]]) + assert node.query( + "SELECT a, c FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[1, 3]]) + assert node.query( + "SELECT b, d FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[2, 4]]) + assert node.query( + "SELECT c, d FROM mydb.filtered_table2 PREWHERE a < 4 WHERE b < 10" + ) == TSV([[3, 4]]) def test_throwif_error_in_where_with_same_condition_as_filter(): - copy_policy_xml('normal_filter2_table2.xml') - assert 'expected' in node.query_and_get_error("SELECT * FROM mydb.filtered_table2 WHERE throwIf(a > 0, 'expected') = 0 SETTINGS optimize_move_to_prewhere = 0") + copy_policy_xml("normal_filter2_table2.xml") + assert "expected" in node.query_and_get_error( + "SELECT * FROM mydb.filtered_table2 WHERE throwIf(a > 0, 'expected') = 0 SETTINGS optimize_move_to_prewhere = 0" + ) def test_throwif_error_in_prewhere_with_same_condition_as_filter(): - copy_policy_xml('normal_filter2_table2.xml') - assert 'expected' in node.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE throwIf(a > 0, 'expected') = 0") + copy_policy_xml("normal_filter2_table2.xml") + assert "expected" in node.query_and_get_error( + "SELECT * FROM mydb.filtered_table2 PREWHERE throwIf(a > 0, 'expected') = 0" + ) def test_throwif_in_where_doesnt_expose_restricted_data(): - copy_policy_xml('no_filters.xml') - assert 'expected' in node.query_and_get_error("SELECT * FROM mydb.filtered_table2 WHERE throwIf(a = 0, 'expected') = 0 SETTINGS optimize_move_to_prewhere = 0") + copy_policy_xml("no_filters.xml") + assert "expected" in node.query_and_get_error( + "SELECT * FROM mydb.filtered_table2 WHERE throwIf(a = 0, 'expected') = 0 SETTINGS optimize_move_to_prewhere = 0" + ) - copy_policy_xml('normal_filter2_table2.xml') - assert node.query("SELECT * FROM mydb.filtered_table2 WHERE throwIf(a = 0, 'pwned') = 0 SETTINGS optimize_move_to_prewhere = 0") == TSV([ - [1, 2, 3, 4], [4, 3, 2, 1]]) + copy_policy_xml("normal_filter2_table2.xml") + assert node.query( + "SELECT * FROM mydb.filtered_table2 WHERE throwIf(a = 0, 'pwned') = 0 SETTINGS optimize_move_to_prewhere = 0" + ) == TSV([[1, 2, 3, 4], [4, 3, 2, 1]]) def test_throwif_in_prewhere_doesnt_expose_restricted_data(): - copy_policy_xml('no_filters.xml') - assert 'expected' in node.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE throwIf(a = 0, 'expected') = 0") + copy_policy_xml("no_filters.xml") + assert "expected" in node.query_and_get_error( + "SELECT * FROM mydb.filtered_table2 PREWHERE throwIf(a = 0, 'expected') = 0" + ) - copy_policy_xml('normal_filter2_table2.xml') - assert node.query("SELECT * FROM mydb.filtered_table2 PREWHERE throwIf(a = 0, 'pwned') = 0") == TSV([ - [1, 2, 3, 4], [4, 3, 2, 1]]) + copy_policy_xml("normal_filter2_table2.xml") + assert node.query( + "SELECT * FROM mydb.filtered_table2 PREWHERE throwIf(a = 0, 'pwned') = 0" + ) == TSV([[1, 2, 3, 4], [4, 3, 2, 1]]) def test_change_of_users_xml_changes_row_policies(): - copy_policy_xml('normal_filters.xml') + copy_policy_xml("normal_filters.xml") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - copy_policy_xml('all_rows.xml') - assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + copy_policy_xml("all_rows.xml") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( - [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]] + ) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) - copy_policy_xml('no_rows.xml') + copy_policy_xml("no_rows.xml") assert node.query("SELECT * FROM mydb.filtered_table1") == "" assert node.query("SELECT * FROM mydb.filtered_table2") == "" assert node.query("SELECT * FROM mydb.filtered_table3") == "" - copy_policy_xml('normal_filters.xml') + copy_policy_xml("normal_filters.xml") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - copy_policy_xml('normal_filter2_table2.xml') - assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[1, 2, 3, 4], [4, 3, 2, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) - - copy_policy_xml('no_filters.xml') - assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + copy_policy_xml("normal_filter2_table2.xml") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( - [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + [[1, 2, 3, 4], [4, 3, 2, 1]] + ) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) - copy_policy_xml('normal_filters.xml') + copy_policy_xml("no_filters.xml") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]] + ) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) + + copy_policy_xml("normal_filters.xml") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) def test_reload_users_xml_by_timer(): - copy_policy_xml('normal_filters.xml') + copy_policy_xml("normal_filters.xml") assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) - assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV( + [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. - copy_policy_xml('all_rows.xml', False) - assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[0, 0], [0, 1], [1, 0], [1, 1]]) - assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", - [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) - assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 0], [0, 1], [1, 0], [1, 1]]) + time.sleep( + 1 + ) # The modification time of the 'row_policy.xml' file should be different. + copy_policy_xml("all_rows.xml", False) + assert_eq_with_retry( + node, "SELECT * FROM mydb.filtered_table1", [[0, 0], [0, 1], [1, 0], [1, 1]] + ) + assert_eq_with_retry( + node, + "SELECT * FROM mydb.filtered_table2", + [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]], + ) + assert_eq_with_retry( + node, "SELECT * FROM mydb.filtered_table3", [[0, 0], [0, 1], [1, 0], [1, 1]] + ) - time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. - copy_policy_xml('normal_filters.xml', False) + time.sleep( + 1 + ) # The modification time of the 'row_policy.xml' file should be different. + copy_policy_xml("normal_filters.xml", False) assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[1, 0], [1, 1]]) - assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0]]) + assert_eq_with_retry( + node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0]] + ) assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 1], [1, 0]]) def test_introspection(): policies = [ - ["another ON mydb.filtered_table1", "another", "mydb", "filtered_table1", - "6068883a-0e9d-f802-7e22-0144f8e66d3c", "users.xml", "1", 0, 0, "['another']", "[]"], - ["another ON mydb.filtered_table2", "another", "mydb", "filtered_table2", - "c019e957-c60b-d54e-cc52-7c90dac5fb01", "users.xml", "1", 0, 0, "['another']", "[]"], - ["another ON mydb.filtered_table3", "another", "mydb", "filtered_table3", - "4cb080d0-44e8-dbef-6026-346655143628", "users.xml", "1", 0, 0, "['another']", "[]"], - ["another ON mydb.local", "another", "mydb", "local", "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97", "users.xml", - "a = 1", 0, 0, "['another']", "[]"], - ["default ON mydb.filtered_table1", "default", "mydb", "filtered_table1", - "9e8a8f62-4965-2b5e-8599-57c7b99b3549", "users.xml", "a = 1", 0, 0, "['default']", "[]"], - ["default ON mydb.filtered_table2", "default", "mydb", "filtered_table2", - "cffae79d-b9bf-a2ef-b798-019c18470b25", "users.xml", "a + b < 1 or c - d > 5", 0, 0, "['default']", "[]"], - ["default ON mydb.filtered_table3", "default", "mydb", "filtered_table3", - "12fc5cef-e3da-3940-ec79-d8be3911f42b", "users.xml", "c = 1", 0, 0, "['default']", "[]"], - ["default ON mydb.local", "default", "mydb", "local", "cdacaeb5-1d97-f99d-2bb0-4574f290629c", "users.xml", "1", - 0, 0, "['default']", "[]"] + [ + "another ON mydb.filtered_table1", + "another", + "mydb", + "filtered_table1", + "6068883a-0e9d-f802-7e22-0144f8e66d3c", + "users.xml", + "1", + 0, + 0, + "['another']", + "[]", + ], + [ + "another ON mydb.filtered_table2", + "another", + "mydb", + "filtered_table2", + "c019e957-c60b-d54e-cc52-7c90dac5fb01", + "users.xml", + "1", + 0, + 0, + "['another']", + "[]", + ], + [ + "another ON mydb.filtered_table3", + "another", + "mydb", + "filtered_table3", + "4cb080d0-44e8-dbef-6026-346655143628", + "users.xml", + "1", + 0, + 0, + "['another']", + "[]", + ], + [ + "another ON mydb.local", + "another", + "mydb", + "local", + "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97", + "users.xml", + "a = 1", + 0, + 0, + "['another']", + "[]", + ], + [ + "default ON mydb.filtered_table1", + "default", + "mydb", + "filtered_table1", + "9e8a8f62-4965-2b5e-8599-57c7b99b3549", + "users.xml", + "a = 1", + 0, + 0, + "['default']", + "[]", + ], + [ + "default ON mydb.filtered_table2", + "default", + "mydb", + "filtered_table2", + "cffae79d-b9bf-a2ef-b798-019c18470b25", + "users.xml", + "a + b < 1 or c - d > 5", + 0, + 0, + "['default']", + "[]", + ], + [ + "default ON mydb.filtered_table3", + "default", + "mydb", + "filtered_table3", + "12fc5cef-e3da-3940-ec79-d8be3911f42b", + "users.xml", + "c = 1", + 0, + 0, + "['default']", + "[]", + ], + [ + "default ON mydb.local", + "default", + "mydb", + "local", + "cdacaeb5-1d97-f99d-2bb0-4574f290629c", + "users.xml", + "1", + 0, + 0, + "['default']", + "[]", + ], ] - assert node.query("SELECT * from system.row_policies ORDER BY short_name, database, table") == TSV(policies) + assert node.query( + "SELECT * from system.row_policies ORDER BY short_name, database, table" + ) == TSV(policies) def test_dcl_introspection(): assert node.query("SHOW POLICIES") == TSV( - ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", - "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", - "default ON mydb.filtered_table3", "default ON mydb.local"]) + [ + "another ON mydb.filtered_table1", + "another ON mydb.filtered_table2", + "another ON mydb.filtered_table3", + "another ON mydb.local", + "default ON mydb.filtered_table1", + "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", + "default ON mydb.local", + ] + ) - assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV(["another", "default"]) + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV( + ["another", "default"] + ) assert node.query("SHOW POLICIES ON mydb.local") == TSV(["another", "default"]) assert node.query("SHOW POLICIES ON mydb.*") == TSV( - ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", - "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", - "default ON mydb.filtered_table3", "default ON mydb.local"]) + [ + "another ON mydb.filtered_table1", + "another ON mydb.filtered_table2", + "another ON mydb.filtered_table3", + "another ON mydb.local", + "default ON mydb.filtered_table1", + "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", + "default ON mydb.local", + ] + ) assert node.query("SHOW POLICIES default") == TSV( - ["default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", - "default ON mydb.local"]) + [ + "default ON mydb.filtered_table1", + "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", + "default ON mydb.local", + ] + ) - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") + == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") + == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") + == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.local") + == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + ) assert node.query("SHOW CREATE POLICY default") == TSV( - ["CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", - "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", - "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default"]) + [ + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default", + ] + ) assert node.query("SHOW CREATE POLICIES ON mydb.filtered_table1") == TSV( - ["CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default"]) + [ + "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + ] + ) assert node.query("SHOW CREATE POLICIES ON mydb.*") == TSV( - ["CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", - "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", - "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", - "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default"]) + [ + "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default", + ] + ) assert node.query("SHOW CREATE POLICIES") == TSV( - ["CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", - "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", - "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", - "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", - "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default"]) + [ + "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another", + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another", + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default", + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default", + ] + ) - expected_access = "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another\n" \ - "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another\n" \ - "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another\n" \ - "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another\n" \ - "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" \ - "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" \ - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" \ - "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + expected_access = ( + "CREATE ROW POLICY another ON mydb.filtered_table1 FOR SELECT USING 1 TO another\n" + "CREATE ROW POLICY another ON mydb.filtered_table2 FOR SELECT USING 1 TO another\n" + "CREATE ROW POLICY another ON mydb.filtered_table3 FOR SELECT USING 1 TO another\n" + "CREATE ROW POLICY another ON mydb.local FOR SELECT USING a = 1 TO another\n" + "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" + "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + ) assert expected_access in node.query("SHOW ACCESS") - copy_policy_xml('all_rows.xml') + copy_policy_xml("all_rows.xml") assert node.query("SHOW POLICIES") == TSV( - ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", - "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" + [ + "another ON mydb.filtered_table1", + "another ON mydb.filtered_table2", + "another ON mydb.filtered_table3", + "default ON mydb.filtered_table1", + "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", + ] + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") + == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") + == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") + == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" + ) - copy_policy_xml('no_rows.xml') + copy_policy_xml("no_rows.xml") assert node.query("SHOW POLICIES") == TSV( - ["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", - "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" - assert node.query( - "SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" + [ + "another ON mydb.filtered_table1", + "another ON mydb.filtered_table2", + "another ON mydb.filtered_table3", + "default ON mydb.filtered_table1", + "default ON mydb.filtered_table2", + "default ON mydb.filtered_table3", + ] + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") + == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") + == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" + ) + assert ( + node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") + == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" + ) - copy_policy_xml('no_filters.xml') + copy_policy_xml("no_filters.xml") assert node.query("SHOW POLICIES") == "" def test_dcl_management(): - copy_policy_xml('no_filters.xml') + copy_policy_xml("no_filters.xml") assert node.query("SHOW POLICIES") == "" node.query("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a b TO default\n" + assert ( + node.query("SHOW CREATE POLICY pB ON mydb.filtered_table1") + == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" + ) node.query("DROP POLICY pB ON mydb.filtered_table1") - assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV( + [[0, 0], [0, 1], [1, 0], [1, 1]] + ) assert node.query("SHOW POLICIES") == "" +def test_grant_create_row_policy(): + copy_policy_xml("no_filters.xml") + assert node.query("SHOW POLICIES") == "" + node.query("CREATE USER X") + + expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table1" + assert expected_error in node.query_and_get_error( + "CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a (d + 5) TO default", - "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 0 TO default", - "CREATE ROW POLICY default ON mydb.table FOR SELECT USING a = 0 TO default"]) + [ + "CREATE ROW POLICY default ON mydb.`.filtered_table4` FOR SELECT USING c = 2 TO default", + "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING c > (d + 5) TO default", + "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 0 TO default", + "CREATE ROW POLICY default ON mydb.table FOR SELECT USING a = 0 TO default", + ] + ) def test_miscellaneous_engines(): - node.query("CREATE ROW POLICY OR REPLACE pC ON mydb.other_table FOR SELECT USING a = 1 TO default") + node.query( + "CREATE ROW POLICY OR REPLACE pC ON mydb.other_table FOR SELECT USING a = 1 TO default" + ) assert node.query("SHOW ROW POLICIES ON mydb.other_table") == "pC\n" # ReplicatedMergeTree node.query("DROP TABLE IF EXISTS mydb.other_table") - node.query("CREATE TABLE mydb.other_table (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") + node.query( + "CREATE TABLE mydb.other_table (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a" + ) node.query("INSERT INTO mydb.other_table values (0, 0), (0, 1), (1, 0), (1, 1)") assert node.query("SELECT * FROM mydb.other_table") == TSV([[1, 0], [1, 1]]) # CollapsingMergeTree node.query("DROP TABLE mydb.other_table") - node.query("CREATE TABLE mydb.other_table (a UInt8, b Int8) ENGINE CollapsingMergeTree(b) ORDER BY a") + node.query( + "CREATE TABLE mydb.other_table (a UInt8, b Int8) ENGINE CollapsingMergeTree(b) ORDER BY a" + ) node.query("INSERT INTO mydb.other_table values (0, 1), (0, 1), (1, 1), (1, 1)") assert node.query("SELECT * FROM mydb.other_table") == TSV([[1, 1], [1, 1]]) # ReplicatedCollapsingMergeTree node.query("DROP TABLE mydb.other_table") - node.query("CREATE TABLE mydb.other_table (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-01/filtered_table1', 'replica1', b) ORDER BY a") + node.query( + "CREATE TABLE mydb.other_table (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-01/filtered_table1', 'replica1', b) ORDER BY a" + ) node.query("INSERT INTO mydb.other_table values (0, 1), (0, 1), (1, 1), (1, 1)") assert node.query("SELECT * FROM mydb.other_table") == TSV([[1, 1], [1, 1]]) @@ -434,17 +839,27 @@ def test_miscellaneous_engines(): # DistributedMergeTree node.query("DROP TABLE IF EXISTS mydb.other_table") - node.query("CREATE TABLE mydb.other_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") - assert node.query("SELECT * FROM mydb.other_table", user="another") == TSV([[1, 0], [1, 1], [1, 0], [1, 1]]) - assert node.query("SELECT sum(a), b FROM mydb.other_table GROUP BY b ORDER BY b", user="another") == TSV([[2, 0], [2, 1]]) + node.query( + "CREATE TABLE mydb.other_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)" + ) + assert node.query("SELECT * FROM mydb.other_table", user="another") == TSV( + [[1, 0], [1, 1], [1, 0], [1, 1]] + ) + assert node.query( + "SELECT sum(a), b FROM mydb.other_table GROUP BY b ORDER BY b", user="another" + ) == TSV([[2, 0], [2, 1]]) def test_policy_on_distributed_table_via_role(): node.query("DROP TABLE IF EXISTS local_tbl") node.query("DROP TABLE IF EXISTS dist_tbl") - node.query("CREATE TABLE local_tbl engine=MergeTree ORDER BY tuple() as select * FROM numbers(10)") - node.query("CREATE TABLE dist_tbl ENGINE=Distributed( 'test_cluster_two_shards_localhost', default, local_tbl) AS local_tbl") + node.query( + "CREATE TABLE local_tbl engine=MergeTree ORDER BY tuple() as select * FROM numbers(10)" + ) + node.query( + "CREATE TABLE dist_tbl ENGINE=Distributed( 'test_cluster_two_shards_localhost', default, local_tbl) AS local_tbl" + ) node.query("CREATE ROLE OR REPLACE 'role1'") node.query("CREATE USER OR REPLACE 'user1' DEFAULT ROLE 'role1'") @@ -452,8 +867,16 @@ def test_policy_on_distributed_table_via_role(): node.query("GRANT SELECT ON dist_tbl TO 'role1'") node.query("GRANT SELECT ON local_tbl TO 'role1'") - node.query("CREATE ROW POLICY OR REPLACE 'all_data' ON dist_tbl, local_tbl USING 1 TO ALL EXCEPT 'role1'") - node.query("CREATE ROW POLICY OR REPLACE 'role1_data' ON dist_tbl, local_tbl USING number % 2 = 0 TO 'role1'") + node.query( + "CREATE ROW POLICY OR REPLACE 'all_data' ON dist_tbl, local_tbl USING 1 TO ALL EXCEPT 'role1'" + ) + node.query( + "CREATE ROW POLICY OR REPLACE 'role1_data' ON dist_tbl, local_tbl USING number % 2 = 0 TO 'role1'" + ) - assert node.query("SELECT * FROM local_tbl SETTINGS prefer_localhost_replica=0", user="user1") == TSV([[0], [2], [4], [6], [8]]) - assert node.query("SELECT * FROM dist_tbl SETTINGS prefer_localhost_replica=0", user="user1") == TSV([[0], [2], [4], [6], [8], [0], [2], [4], [6], [8]]) + assert node.query( + "SELECT * FROM local_tbl SETTINGS prefer_localhost_replica=0", user="user1" + ) == TSV([[0], [2], [4], [6], [8]]) + assert node.query( + "SELECT * FROM dist_tbl SETTINGS prefer_localhost_replica=0", user="user1" + ) == TSV([[0], [2], [4], [6], [8], [0], [2], [4], [6], [8]]) diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index f60e6e6862f..561d3e3ed28 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -9,12 +9,22 @@ logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -S3_DATA = ['data/clickhouse/part1.csv', 'data/clickhouse/part123.csv', 'data/database/part2.csv', 'data/database/partition675.csv'] +S3_DATA = [ + "data/clickhouse/part1.csv", + "data/clickhouse/part123.csv", + "data/database/part2.csv", + "data/database/partition675.csv", +] + def create_buckets_s3(cluster): minio = cluster.minio_client for file in S3_DATA: - minio.fput_object(bucket_name=cluster.minio_bucket, object_name=file, file_path=os.path.join(SCRIPT_DIR, file)) + minio.fput_object( + bucket_name=cluster.minio_bucket, + object_name=file, + file_path=os.path.join(SCRIPT_DIR, file), + ) for obj in minio.list_objects(cluster.minio_bucket, recursive=True): print(obj.object_name) @@ -23,10 +33,12 @@ def create_buckets_s3(cluster): def started_cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance('s0_0_0', main_configs=["configs/cluster.xml"], with_minio=True) - cluster.add_instance('s0_0_1', main_configs=["configs/cluster.xml"]) - cluster.add_instance('s0_1_0', main_configs=["configs/cluster.xml"]) - + cluster.add_instance( + "s0_0_0", main_configs=["configs/cluster.xml"], with_minio=True + ) + cluster.add_instance("s0_0_1", main_configs=["configs/cluster.xml"]) + cluster.add_instance("s0_1_0", main_configs=["configs/cluster.xml"]) + logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -39,45 +51,54 @@ def started_cluster(): def test_select_all(started_cluster): - node = started_cluster.instances['s0_0_0'] - pure_s3 = node.query(""" + node = started_cluster.instances["s0_0_0"] + pure_s3 = node.query( + """ SELECT * from s3( 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') - ORDER BY (name, value, polygon)""") + ORDER BY (name, value, polygon)""" + ) # print(pure_s3) - s3_distibuted = node.query(""" + s3_distibuted = node.query( + """ SELECT * from s3Cluster( 'cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', - 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon)""") + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon)""" + ) # print(s3_distibuted) assert TSV(pure_s3) == TSV(s3_distibuted) def test_count(started_cluster): - node = started_cluster.instances['s0_0_0'] - pure_s3 = node.query(""" + node = started_cluster.instances["s0_0_0"] + pure_s3 = node.query( + """ SELECT count(*) from s3( 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', - 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))')""") + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))')""" + ) # print(pure_s3) - s3_distibuted = node.query(""" + s3_distibuted = node.query( + """ SELECT count(*) from s3Cluster( 'cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', - 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))')""") + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))')""" + ) # print(s3_distibuted) assert TSV(pure_s3) == TSV(s3_distibuted) def test_union_all(started_cluster): - node = started_cluster.instances['s0_0_0'] - pure_s3 = node.query(""" + node = started_cluster.instances["s0_0_0"] + pure_s3 = node.query( + """ SELECT * FROM ( SELECT * from s3( @@ -91,9 +112,11 @@ def test_union_all(started_cluster): 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ) ORDER BY (name, value, polygon) - """) + """ + ) # print(pure_s3) - s3_distibuted = node.query(""" + s3_distibuted = node.query( + """ SELECT * FROM ( SELECT * from s3Cluster( @@ -107,15 +130,17 @@ def test_union_all(started_cluster): 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ) ORDER BY (name, value, polygon) - """) + """ + ) # print(s3_distibuted) assert TSV(pure_s3) == TSV(s3_distibuted) def test_wrong_cluster(started_cluster): - node = started_cluster.instances['s0_0_0'] - error = node.query_and_get_error(""" + node = started_cluster.instances["s0_0_0"] + error = node.query_and_get_error( + """ SELECT count(*) from s3Cluster( 'non_existent_cluster', 'http://minio1:9001/root/data/{clickhouse,database}/*', @@ -124,6 +149,7 @@ def test_wrong_cluster(started_cluster): SELECT count(*) from s3Cluster( 'non_existent_cluster', 'http://minio1:9001/root/data/{clickhouse,database}/*', - 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))')""") - - assert "not found" in error \ No newline at end of file + 'minio', 'minio123', 'CSV', 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))')""" + ) + + assert "not found" in error diff --git a/tests/integration/test_s3_low_cardinality_right_border/test.py b/tests/integration/test_s3_low_cardinality_right_border/test.py index 056c3e4430f..babe25fa899 100644 --- a/tests/integration/test_s3_low_cardinality_right_border/test.py +++ b/tests/integration/test_s3_low_cardinality_right_border/test.py @@ -65,6 +65,7 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance("node1", main_configs=["configs/s3.xml"], with_minio=True) + @pytest.fixture(scope="module") def started_cluster(): try: @@ -76,7 +77,8 @@ def started_cluster(): def test_s3_right_border(started_cluster): - node1.query(""" + node1.query( + """ CREATE TABLE s3_low_cardinality ( str_column LowCardinality(String) @@ -84,12 +86,17 @@ CREATE TABLE s3_low_cardinality ENGINE = MergeTree() ORDER BY tuple() SETTINGS storage_policy = 's3', min_bytes_for_wide_part = 0, index_granularity = 1024; - """) + """ + ) node1.query("INSERT INTO s3_low_cardinality SELECT 'aaaaaa' FROM numbers(600000)") - node1.query("INSERT INTO s3_low_cardinality SELECT toString(number) FROM numbers(100000)") + node1.query( + "INSERT INTO s3_low_cardinality SELECT toString(number) FROM numbers(100000)" + ) node1.query("INSERT INTO s3_low_cardinality SELECT 'bbbbbb' FROM numbers(500000)") - node1.query("INSERT INTO s3_low_cardinality SELECT toString(number + 100000000) FROM numbers(100000)") + node1.query( + "INSERT INTO s3_low_cardinality SELECT toString(number + 100000000) FROM numbers(100000)" + ) node1.query("OPTIMIZE TABLE s3_low_cardinality FINAL") @@ -98,4 +105,10 @@ SETTINGS storage_policy = 's3', min_bytes_for_wide_part = 0, index_granularity "merge_tree_min_rows_for_concurrent_read": "0", "max_threads": "2", } - assert node1.query("SELECT COUNT() FROM s3_low_cardinality WHERE not ignore(str_column)", settings=settings) == "1300000\n" + assert ( + node1.query( + "SELECT COUNT() FROM s3_low_cardinality WHERE not ignore(str_column)", + settings=settings, + ) + == "1300000\n" + ) diff --git a/tests/integration/test_s3_with_https/test.py b/tests/integration/test_s3_with_https/test.py index 4fa8260ed2e..46e281251a0 100644 --- a/tests/integration/test_s3_with_https/test.py +++ b/tests/integration/test_s3_with_https/test.py @@ -15,8 +15,15 @@ def check_proxy_logs(cluster, proxy_instance): def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", main_configs=["configs/config.d/storage_conf.xml", "configs/config.d/ssl.xml"], - with_minio=True, minio_certs_dir="minio_certs") + cluster.add_instance( + "node", + main_configs=[ + "configs/config.d/storage_conf.xml", + "configs/config.d/ssl.xml", + ], + with_minio=True, + minio_certs_dir="minio_certs", + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -26,9 +33,7 @@ def cluster(): cluster.shutdown() -@pytest.mark.parametrize( - "policy", ["s3_secure", "s3_secure_with_proxy"] -) +@pytest.mark.parametrize("policy", ["s3_secure", "s3_secure_with_proxy"]) def test_s3_with_https(cluster, policy): node = cluster.instances["node"] @@ -40,12 +45,16 @@ def test_s3_with_https(cluster, policy): ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='{}' - """ - .format(policy) + """.format( + policy + ) ) node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") - assert node.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" + assert ( + node.query("SELECT * FROM s3_test order by id FORMAT Values") + == "(0,'data'),(1,'data')" + ) node.query("DROP TABLE IF EXISTS s3_test NO DELAY") diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py index 87fe4ce30f6..eaea4c1dab2 100644 --- a/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py +++ b/tests/integration/test_s3_with_proxy/proxy-resolver/resolver.py @@ -3,12 +3,12 @@ import random import bottle -@bottle.route('/hostname') +@bottle.route("/hostname") def index(): if random.randrange(2) == 0: - return 'proxy1' + return "proxy1" else: - return 'proxy2' + return "proxy2" -bottle.run(host='0.0.0.0', port=8080) +bottle.run(host="0.0.0.0", port=8080) diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 33ad981d18d..1102d190a87 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -7,10 +7,13 @@ from helpers.cluster import ClickHouseCluster # Runs simple proxy resolver in python env container. def run_resolver(cluster): - container_id = cluster.get_container_id('resolver') + container_id = cluster.get_container_id("resolver") current_dir = os.path.dirname(__file__) - cluster.copy_file_to_container(container_id, os.path.join(current_dir, "proxy-resolver", "resolver.py"), - "resolver.py") + cluster.copy_file_to_container( + container_id, + os.path.join(current_dir, "proxy-resolver", "resolver.py"), + "resolver.py", + ) cluster.exec_in_container(container_id, ["python", "resolver.py"], detach=True) @@ -18,9 +21,9 @@ def run_resolver(cluster): def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node", - main_configs=["configs/config.d/storage_conf.xml"], - with_minio=True) + cluster.add_instance( + "node", main_configs=["configs/config.d/storage_conf.xml"], with_minio=True + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -45,9 +48,7 @@ def check_proxy_logs(cluster, proxy_instance, http_methods={"POST", "PUT", "GET" assert False, "http method not found in logs" -@pytest.mark.parametrize( - "policy", ["s3", "s3_with_resolver"] -) +@pytest.mark.parametrize("policy", ["s3", "s3_with_resolver"]) def test_s3_with_proxy_list(cluster, policy): node = cluster.instances["node"] @@ -59,12 +60,16 @@ def test_s3_with_proxy_list(cluster, policy): ) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='{}' - """ - .format(policy) + """.format( + policy + ) ) node.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") - assert node.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" + assert ( + node.query("SELECT * FROM s3_test order by id FORMAT Values") + == "(0,'data'),(1,'data')" + ) node.query("DROP TABLE IF EXISTS s3_test NO DELAY") diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index fb30a83877b..d7aa4feb1d2 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -13,12 +13,20 @@ logging.getLogger().addHandler(logging.StreamHandler()) def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/config.d/s3.xml"], macros={'replica': '1'}, - with_minio=True, - with_zookeeper=True) - cluster.add_instance("node2", main_configs=["configs/config.d/s3.xml"], macros={'replica': '2'}, - with_minio=True, - with_zookeeper=True) + cluster.add_instance( + "node1", + main_configs=["configs/config.d/s3.xml"], + macros={"replica": "1"}, + with_minio=True, + with_zookeeper=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/config.d/s3.xml"], + macros={"replica": "2"}, + with_minio=True, + with_zookeeper=True, + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -28,28 +36,28 @@ def cluster(): cluster.shutdown() -def get_large_objects_count(cluster, size=100, folder='data'): +def get_large_objects_count(cluster, size=100, folder="data"): minio = cluster.minio_client counter = 0 - for obj in minio.list_objects(cluster.minio_bucket, '{}/'.format(folder)): + for obj in minio.list_objects(cluster.minio_bucket, "{}/".format(folder)): if obj.size is not None and obj.size >= size: counter = counter + 1 return counter -def check_objects_exisis(cluster, object_list, folder='data'): +def check_objects_exisis(cluster, object_list, folder="data"): minio = cluster.minio_client for obj in object_list: if obj: - minio.stat_object(cluster.minio_bucket, '{}/{}'.format(folder, obj)) + minio.stat_object(cluster.minio_bucket, "{}/{}".format(folder, obj)) -def check_objects_not_exisis(cluster, object_list, folder='data'): +def check_objects_not_exisis(cluster, object_list, folder="data"): minio = cluster.minio_client for obj in object_list: if obj: try: - minio.stat_object(cluster.minio_bucket, '{}/{}'.format(folder, obj)) + minio.stat_object(cluster.minio_bucket, "{}/{}".format(folder, obj)) except Exception as error: assert "NoSuchKey" in str(error) else: @@ -69,7 +77,11 @@ def wait_for_active_parts(node, num_expected_parts, table_name, timeout=30): deadline = time.monotonic() + timeout num_parts = 0 while time.monotonic() < deadline: - num_parts_str = node.query("select count() from system.parts where table = '{}' and active".format(table_name)) + num_parts_str = node.query( + "select count() from system.parts where table = '{}' and active".format( + table_name + ) + ) num_parts = int(num_parts_str.strip()) if num_parts == num_expected_parts: return @@ -81,9 +93,7 @@ def wait_for_active_parts(node, num_expected_parts, table_name, timeout=30): # Result of `get_large_objects_count` can be changed in other tests, so run this case at the beginning @pytest.mark.order(0) -@pytest.mark.parametrize( - "policy", ["s3"] -) +@pytest.mark.parametrize("policy", ["s3"]) def test_s3_zero_copy_replication(cluster, policy): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] @@ -94,14 +104,21 @@ def test_s3_zero_copy_replication(cluster, policy): ENGINE=ReplicatedMergeTree('/clickhouse/tables/s3_test', '{}') ORDER BY id SETTINGS storage_policy='{}' - """ - .format('{replica}', policy) + """.format( + "{replica}", policy + ) ) node1.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") node2.query("SYSTEM SYNC REPLICA s3_test") - assert node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" - assert node2.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" + assert ( + node1.query("SELECT * FROM s3_test order by id FORMAT Values") + == "(0,'data'),(1,'data')" + ) + assert ( + node2.query("SELECT * FROM s3_test order by id FORMAT Values") + == "(0,'data'),(1,'data')" + ) # Based on version 21.x - should be only 1 file with size 100+ (checksums.txt), used by both nodes assert get_large_objects_count(cluster) == 1 @@ -109,8 +126,14 @@ def test_s3_zero_copy_replication(cluster, policy): node2.query("INSERT INTO s3_test VALUES (2,'data'),(3,'data')") node1.query("SYSTEM SYNC REPLICA s3_test") - assert node2.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" - assert node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + assert ( + node2.query("SELECT * FROM s3_test order by id FORMAT Values") + == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + ) + assert ( + node1.query("SELECT * FROM s3_test order by id FORMAT Values") + == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + ) # Based on version 21.x - two parts wait_for_large_objects_count(cluster, 2) @@ -137,44 +160,90 @@ def test_s3_zero_copy_on_hybrid_storage(cluster): ENGINE=ReplicatedMergeTree('/clickhouse/tables/hybrid_test', '{}') ORDER BY id SETTINGS storage_policy='hybrid' - """ - .format('{replica}') + """.format( + "{replica}" + ) ) node1.query("INSERT INTO hybrid_test VALUES (0,'data'),(1,'data')") node2.query("SYSTEM SYNC REPLICA hybrid_test") - assert node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") == "(0,'data'),(1,'data')" - assert node2.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") == "(0,'data'),(1,'data')" + assert ( + node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") + == "(0,'data'),(1,'data')" + ) + assert ( + node2.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") + == "(0,'data'),(1,'data')" + ) - assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values") == "('all','default')" - assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values") == "('all','default')" + assert ( + node1.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values" + ) + == "('all','default')" + ) + assert ( + node2.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values" + ) + == "('all','default')" + ) node1.query("ALTER TABLE hybrid_test MOVE PARTITION ID 'all' TO DISK 's31'") - assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values") == "('all','s31')" - assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values") == "('all','default')" + assert ( + node1.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values" + ) + == "('all','s31')" + ) + assert ( + node2.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values" + ) + == "('all','default')" + ) # Total objects in S3 s3_objects = get_large_objects_count(cluster, size=0) node2.query("ALTER TABLE hybrid_test MOVE PARTITION ID 'all' TO DISK 's31'") - assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values") == "('all','s31')" - assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values") == "('all','s31')" + assert ( + node1.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values" + ) + == "('all','s31')" + ) + assert ( + node2.query( + "SELECT partition_id,disk_name FROM system.parts WHERE table='hybrid_test' FORMAT Values" + ) + == "('all','s31')" + ) # Check that after moving partition on node2 no new obects on s3 wait_for_large_objects_count(cluster, s3_objects, size=0) - assert node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") == "(0,'data'),(1,'data')" - assert node2.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") == "(0,'data'),(1,'data')" + assert ( + node1.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") + == "(0,'data'),(1,'data')" + ) + assert ( + node2.query("SELECT * FROM hybrid_test ORDER BY id FORMAT Values") + == "(0,'data'),(1,'data')" + ) node1.query("DROP TABLE IF EXISTS hybrid_test NO DELAY") node2.query("DROP TABLE IF EXISTS hybrid_test NO DELAY") def insert_data_time(node, table, number_of_mb, time, start=0): - values = ','.join(f"({x},{time})" for x in range(start, int((1024 * 1024 * number_of_mb) / 8) + start + 1)) + values = ",".join( + f"({x},{time})" + for x in range(start, int((1024 * 1024 * number_of_mb) / 8) + start + 1) + ) node.query(f"INSERT INTO {table} VALUES {values}") @@ -182,9 +251,9 @@ def insert_large_data(node, table): tm = time.mktime((datetime.date.today() - datetime.timedelta(days=7)).timetuple()) insert_data_time(node, table, 1, tm, 0) tm = time.mktime((datetime.date.today() - datetime.timedelta(days=3)).timetuple()) - insert_data_time(node, table, 1, tm, 1024*1024) + insert_data_time(node, table, 1, tm, 1024 * 1024) tm = time.mktime(datetime.date.today().timetuple()) - insert_data_time(node, table, 10, tm, 1024*1024*2) + insert_data_time(node, table, 10, tm, 1024 * 1024 * 2) @pytest.mark.parametrize( @@ -194,7 +263,7 @@ def insert_large_data(node, table): ("tiered_copy", False, 10), ("tiered", True, 3), ("tiered_copy", True, 3), - ] + ], ) def test_s3_zero_copy_with_ttl_move(cluster, storage_policy, large_data, iterations): node1 = cluster.instances["node1"] @@ -211,12 +280,13 @@ def test_s3_zero_copy_with_ttl_move(cluster, storage_policy, large_data, iterati ORDER BY d TTL d1 + INTERVAL 2 DAY TO VOLUME 'external' SETTINGS storage_policy='{}' - """ - .format('{replica}', storage_policy) + """.format( + "{replica}", storage_policy + ) ) if large_data: - insert_large_data(node1, 'ttl_move_test') + insert_large_data(node1, "ttl_move_test") else: node1.query("INSERT INTO ttl_move_test VALUES (10, now() - INTERVAL 3 DAY)") node1.query("INSERT INTO ttl_move_test VALUES (11, now() - INTERVAL 1 DAY)") @@ -225,13 +295,29 @@ def test_s3_zero_copy_with_ttl_move(cluster, storage_policy, large_data, iterati node2.query("SYSTEM SYNC REPLICA ttl_move_test") if large_data: - assert node1.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(1572867)" - assert node2.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(1572867)" + assert ( + node1.query("SELECT count() FROM ttl_move_test FORMAT Values") + == "(1572867)" + ) + assert ( + node2.query("SELECT count() FROM ttl_move_test FORMAT Values") + == "(1572867)" + ) else: - assert node1.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" - assert node2.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" - assert node1.query("SELECT d FROM ttl_move_test ORDER BY d FORMAT Values") == "(10),(11)" - assert node2.query("SELECT d FROM ttl_move_test ORDER BY d FORMAT Values") == "(10),(11)" + assert ( + node1.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" + ) + assert ( + node2.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" + ) + assert ( + node1.query("SELECT d FROM ttl_move_test ORDER BY d FORMAT Values") + == "(10),(11)" + ) + assert ( + node2.query("SELECT d FROM ttl_move_test ORDER BY d FORMAT Values") + == "(10),(11)" + ) node1.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") node2.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") @@ -242,7 +328,7 @@ def test_s3_zero_copy_with_ttl_move(cluster, storage_policy, large_data, iterati [ (False, 10), (True, 3), - ] + ], ) def test_s3_zero_copy_with_ttl_delete(cluster, large_data, iterations): node1 = cluster.instances["node1"] @@ -259,27 +345,50 @@ def test_s3_zero_copy_with_ttl_delete(cluster, large_data, iterations): ORDER BY d TTL d1 + INTERVAL 2 DAY SETTINGS storage_policy='tiered' - """ - .format('{replica}') + """.format( + "{replica}" + ) ) if large_data: - insert_large_data(node1, 'ttl_delete_test') + insert_large_data(node1, "ttl_delete_test") else: - node1.query("INSERT INTO ttl_delete_test VALUES (10, now() - INTERVAL 3 DAY)") - node1.query("INSERT INTO ttl_delete_test VALUES (11, now() - INTERVAL 1 DAY)") + node1.query( + "INSERT INTO ttl_delete_test VALUES (10, now() - INTERVAL 3 DAY)" + ) + node1.query( + "INSERT INTO ttl_delete_test VALUES (11, now() - INTERVAL 1 DAY)" + ) node1.query("OPTIMIZE TABLE ttl_delete_test FINAL") node2.query("SYSTEM SYNC REPLICA ttl_delete_test") if large_data: - assert node1.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1310721)" - assert node2.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1310721)" + assert ( + node1.query("SELECT count() FROM ttl_delete_test FORMAT Values") + == "(1310721)" + ) + assert ( + node2.query("SELECT count() FROM ttl_delete_test FORMAT Values") + == "(1310721)" + ) else: - assert node1.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1)" - assert node2.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1)" - assert node1.query("SELECT d FROM ttl_delete_test ORDER BY d FORMAT Values") == "(11)" - assert node2.query("SELECT d FROM ttl_delete_test ORDER BY d FORMAT Values") == "(11)" + assert ( + node1.query("SELECT count() FROM ttl_delete_test FORMAT Values") + == "(1)" + ) + assert ( + node2.query("SELECT count() FROM ttl_delete_test FORMAT Values") + == "(1)" + ) + assert ( + node1.query("SELECT d FROM ttl_delete_test ORDER BY d FORMAT Values") + == "(11)" + ) + assert ( + node2.query("SELECT d FROM ttl_delete_test ORDER BY d FORMAT Values") + == "(11)" + ) node1.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") node2.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") @@ -289,12 +398,16 @@ def wait_mutations(node, table, seconds): time.sleep(1) while seconds > 0: seconds -= 1 - mutations = node.query(f"SELECT count() FROM system.mutations WHERE table='{table}' AND is_done=0") - if mutations == '0\n': + mutations = node.query( + f"SELECT count() FROM system.mutations WHERE table='{table}' AND is_done=0" + ) + if mutations == "0\n": return time.sleep(1) - mutations = node.query(f"SELECT count() FROM system.mutations WHERE table='{table}' AND is_done=0") - assert mutations == '0\n' + mutations = node.query( + f"SELECT count() FROM system.mutations WHERE table='{table}' AND is_done=0" + ) + assert mutations == "0\n" def test_s3_zero_copy_unfreeze(cluster): @@ -310,8 +423,9 @@ def test_s3_zero_copy_unfreeze(cluster): ENGINE=ReplicatedMergeTree('/clickhouse/tables/unfreeze_test', '{}') ORDER BY d SETTINGS storage_policy='s3' - """ - .format('{replica}') + """.format( + "{replica}" + ) ) node1.query("INSERT INTO unfreeze_test VALUES (0)") @@ -329,6 +443,7 @@ def test_s3_zero_copy_unfreeze(cluster): check_objects_exisis(cluster, objects01) node1.query("TRUNCATE TABLE unfreeze_test") + node2.query("SYSTEM SYNC REPLICA unfreeze_test") objects11 = node1.get_backuped_s3_objects("s31", "freeze_backup1") objects12 = node2.get_backuped_s3_objects("s31", "freeze_backup2") @@ -365,14 +480,16 @@ def test_s3_zero_copy_drop_detached(cluster): ENGINE=ReplicatedMergeTree('/clickhouse/tables/drop_detached_test', '{}') ORDER BY d PARTITION BY d SETTINGS storage_policy='s3' - """ - .format('{replica}') + """.format( + "{replica}" + ) ) node1.query("INSERT INTO drop_detached_test VALUES (0)") node1.query("ALTER TABLE drop_detached_test FREEZE WITH NAME 'detach_backup1'") node1.query("INSERT INTO drop_detached_test VALUES (1)") node1.query("ALTER TABLE drop_detached_test FREEZE WITH NAME 'detach_backup2'") + node2.query("SYSTEM SYNC REPLICA drop_detached_test") objects1 = node1.get_backuped_s3_objects("s31", "detach_backup1") objects2 = node1.get_backuped_s3_objects("s31", "detach_backup2") @@ -384,33 +501,51 @@ def test_s3_zero_copy_drop_detached(cluster): node1.query("ALTER TABLE drop_detached_test DETACH PARTITION '0'") node1.query("ALTER TABLE drop_detached_test DETACH PARTITION '1'") + node2.query("SYSTEM SYNC REPLICA drop_detached_test") + wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) check_objects_exisis(cluster, objects1) check_objects_exisis(cluster, objects2) - node2.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", settings={"allow_drop_detached": 1}) + node2.query( + "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", + settings={"allow_drop_detached": 1}, + ) + node1.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) check_objects_exisis(cluster, objects1) check_objects_exisis(cluster, objects2) - node1.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", settings={"allow_drop_detached": 1}) + node1.query( + "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", + settings={"allow_drop_detached": 1}, + ) + node2.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) check_objects_exisis(cluster, objects1) check_objects_not_exisis(cluster, objects_diff) - node1.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", settings={"allow_drop_detached": 1}) + node1.query( + "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", + settings={"allow_drop_detached": 1}, + ) + node2.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) check_objects_exisis(cluster, objects1) - node2.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", settings={"allow_drop_detached": 1}) + node2.query( + "ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", + settings={"allow_drop_detached": 1}, + ) + node1.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -426,13 +561,13 @@ def test_s3_zero_copy_concurrent_merge(cluster): for node in (node1, node2): node.query( - """ + """ CREATE TABLE concurrent_merge (id UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/concurrent_merge', '{replica}') ORDER BY id SETTINGS index_granularity=2, storage_policy='s3', remote_fs_execute_merges_on_single_replica_time_threshold=1 """ - ) + ) node1.query("system stop merges") node2.query("system stop merges") @@ -441,7 +576,7 @@ def test_s3_zero_copy_concurrent_merge(cluster): node1.query("insert into concurrent_merge select number from numbers(40)") node1.query("insert into concurrent_merge select number + 1 from numbers(40)") - wait_for_active_parts(node2, 2, 'concurrent_merge') + wait_for_active_parts(node2, 2, "concurrent_merge") # Merge will materialize default column, it should sleep every granule and take 20 * 2 * 0.1 = 4 sec. node1.query("alter table concurrent_merge add column x UInt32 default sleep(0.1)") @@ -457,8 +592,8 @@ def test_s3_zero_copy_concurrent_merge(cluster): # For now, it does not happen (every blob has a random name, and we just have a duplicating data) node1.query("optimize table concurrent_merge final") - wait_for_active_parts(node1, 1, 'concurrent_merge') - wait_for_active_parts(node2, 1, 'concurrent_merge') + wait_for_active_parts(node1, 1, "concurrent_merge") + wait_for_active_parts(node2, 1, "concurrent_merge") for node in (node1, node2): - assert node.query('select sum(id) from concurrent_merge').strip() == '1600' + assert node.query("select sum(id) from concurrent_merge").strip() == "1600" diff --git a/tests/integration/test_s3_zero_copy_ttl/test.py b/tests/integration/test_s3_zero_copy_ttl/test.py index 5f63bfbfdff..14b4664fcc1 100644 --- a/tests/integration/test_s3_zero_copy_ttl/test.py +++ b/tests/integration/test_s3_zero_copy_ttl/test.py @@ -5,9 +5,16 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True) -node2 = cluster.add_instance("node2", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True) -node3 = cluster.add_instance("node3", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/s3.xml"], with_minio=True, with_zookeeper=True +) + @pytest.fixture(scope="module") def started_cluster(): @@ -18,6 +25,7 @@ def started_cluster(): finally: cluster.shutdown() + def test_ttl_move_and_s3(started_cluster): for i, node in enumerate([node1, node2, node3]): node.query( @@ -28,7 +36,10 @@ def test_ttl_move_and_s3(started_cluster): PARTITION BY id TTL date TO DISK 's3_disk' SETTINGS storage_policy='s3_and_default' - """.format(i)) + """.format( + i + ) + ) node1.query("SYSTEM STOP MOVES s3_test_with_ttl") @@ -40,7 +51,9 @@ def test_ttl_move_and_s3(started_cluster): else: node = node2 - node.query(f"INSERT INTO s3_test_with_ttl SELECT now() + 5, {i}, randomPrintableASCII(1048570)") + node.query( + f"INSERT INTO s3_test_with_ttl SELECT now() + 5, {i}, randomPrintableASCII(1048570)" + ) node1.query("SYSTEM SYNC REPLICA s3_test_with_ttl") node2.query("SYSTEM SYNC REPLICA s3_test_with_ttl") @@ -57,10 +70,14 @@ def test_ttl_move_and_s3(started_cluster): time.sleep(5) - print(node1.query("SELECT * FROM system.parts WHERE table = 's3_test_with_ttl' FORMAT Vertical")) + print( + node1.query( + "SELECT * FROM system.parts WHERE table = 's3_test_with_ttl' FORMAT Vertical" + ) + ) minio = cluster.minio_client - objects = minio.list_objects(cluster.minio_bucket, 'data/', recursive=True) + objects = minio.list_objects(cluster.minio_bucket, "data/", recursive=True) counter = 0 for obj in objects: print("Objectname:", obj.object_name, "metadata:", obj.metadata) diff --git a/tests/integration/test_secure_socket/test.py b/tests/integration/test_secure_socket/test.py index c542b855258..2dffbed03d6 100644 --- a/tests/integration/test_secure_socket/test.py +++ b/tests/integration/test_secure_socket/test.py @@ -7,15 +7,15 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -NODES = {'node' + str(i): None for i in (1, 2)} +NODES = {"node" + str(i): None for i in (1, 2)} -config = ''' +config = """ {sleep_in_send_data_ms} -''' +""" @pytest.fixture(scope="module") @@ -29,14 +29,22 @@ def started_cluster(): "configs_secure/config.d/ssl_conf.xml", ] - NODES['node1'] = cluster.add_instance('node1', main_configs=main_configs) - NODES['node2'] = cluster.add_instance('node2', main_configs=main_configs, user_configs=["configs_secure/users.d/users.xml"]) + NODES["node1"] = cluster.add_instance("node1", main_configs=main_configs) + NODES["node2"] = cluster.add_instance( + "node2", + main_configs=main_configs, + user_configs=["configs_secure/users.d/users.xml"], + ) try: cluster.start() - NODES['node2'].query("CREATE TABLE base_table (x UInt64) ENGINE = MergeTree ORDER BY x;") - NODES['node2'].query("INSERT INTO base_table VALUES (5);") - NODES['node1'].query("CREATE TABLE distributed_table (x UInt64) ENGINE = Distributed(test_cluster, default, base_table);") + NODES["node2"].query( + "CREATE TABLE base_table (x UInt64) ENGINE = MergeTree ORDER BY x;" + ) + NODES["node2"].query("INSERT INTO base_table VALUES (5);") + NODES["node1"].query( + "CREATE TABLE distributed_table (x UInt64) ENGINE = Distributed(test_cluster, default, base_table);" + ) yield cluster @@ -45,11 +53,16 @@ def started_cluster(): def test(started_cluster): - NODES['node2'].replace_config('/etc/clickhouse-server/users.d/users.xml', config.format(sleep_in_send_data_ms=1000000)) - + NODES["node2"].replace_config( + "/etc/clickhouse-server/users.d/users.xml", + config.format(sleep_in_send_data_ms=1000000), + ) + attempts = 0 while attempts < 1000: - setting = NODES['node2'].http_query("SELECT value FROM system.settings WHERE name='sleep_in_send_data_ms'") + setting = NODES["node2"].http_query( + "SELECT value FROM system.settings WHERE name='sleep_in_send_data_ms'" + ) if int(setting) == 1000000: break time.sleep(0.1) @@ -57,28 +70,31 @@ def test(started_cluster): assert attempts < 1000 - start = time.time() - NODES['node1'].query_and_get_error('SELECT * FROM distributed_table settings receive_timeout=5, send_timeout=5, use_hedged_requests=0, async_socket_for_remote=0;') + NODES["node1"].query_and_get_error( + "SELECT * FROM distributed_table settings receive_timeout=5, send_timeout=5, use_hedged_requests=0, async_socket_for_remote=0;" + ) end = time.time() assert end - start < 10 start = time.time() - error = NODES['node1'].query_and_get_error('SELECT * FROM distributed_table settings receive_timeout=5, send_timeout=5, use_hedged_requests=0, async_socket_for_remote=1;') + error = NODES["node1"].query_and_get_error( + "SELECT * FROM distributed_table settings receive_timeout=5, send_timeout=5, use_hedged_requests=0, async_socket_for_remote=1;" + ) end = time.time() assert end - start < 10 # Check that exception about timeout wasn't thrown from DB::ReadBufferFromPocoSocket::nextImpl(). - assert error.find('DB::ReadBufferFromPocoSocket::nextImpl()') == -1 + assert error.find("DB::ReadBufferFromPocoSocket::nextImpl()") == -1 start = time.time() - error = NODES['node1'].query_and_get_error('SELECT * FROM distributed_table settings receive_timeout=5, send_timeout=5, use_hedged_requests=1, async_socket_for_remote=1;') + error = NODES["node1"].query_and_get_error( + "SELECT * FROM distributed_table settings receive_timeout=5, send_timeout=5, use_hedged_requests=1, async_socket_for_remote=1;" + ) end = time.time() assert end - start < 10 # Check that exception about timeout wasn't thrown from DB::ReadBufferFromPocoSocket::nextImpl(). - assert error.find('DB::ReadBufferFromPocoSocket::nextImpl()') == -1 - - + assert error.find("DB::ReadBufferFromPocoSocket::nextImpl()") == -1 diff --git a/tests/integration/test_select_access_rights/test.py b/tests/integration/test_select_access_rights/test.py index 0272eac5fa1..76940cdadb4 100644 --- a/tests/integration/test_select_access_rights/test.py +++ b/tests/integration/test_select_access_rights/test.py @@ -3,7 +3,7 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -24,214 +24,334 @@ def cleanup_after_test(): def test_select_single_column(): - instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + instance.query( + "CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d" + ) select_query = "SELECT a FROM table1" - assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(a) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" instance.query("REVOKE SELECT(a) ON default.table1 FROM A") - assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) def test_select_single_column_with_table_grant(): - instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + instance.query( + "CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d" + ) select_query = "SELECT a FROM table1" - assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" instance.query("REVOKE SELECT(a) ON default.table1 FROM A") - assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) def test_select_all_columns(): - instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + instance.query( + "CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d" + ) select_query = "SELECT * FROM table1" - assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(d) ON default.table1 TO A") - assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(a) ON default.table1 TO A") - assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(b) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" def test_select_all_columns_with_table_grant(): - instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") + instance.query( + "CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d" + ) select_query = "SELECT * FROM table1" - assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" def test_alias(): - instance.query("CREATE TABLE table1(x Int32, y Int32) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE table1(x Int32, y Int32) ENGINE = MergeTree ORDER BY tuple()" + ) select_query = "SELECT x, y, x + y AS s FROM table1" - assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(x, y) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(x, y) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" def test_alias_columns(): - instance.query("CREATE TABLE table1(x Int32, y Int32, s Int32 ALIAS x + y) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE table1(x Int32, y Int32, s Int32 ALIAS x + y) ENGINE = MergeTree ORDER BY tuple()" + ) select_query = "SELECT * FROM table1" - assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(x, y) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(x,y) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" select_query = "SELECT s FROM table1" - assert "it's necessary to have grant SELECT(s) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(s) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(s) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" instance.query("REVOKE SELECT(x,y) ON default.table1 FROM A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" def test_materialized_columns(): - instance.query("CREATE TABLE table1(x Int32, y Int32, p Int32 MATERIALIZED x * y) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE table1(x Int32, y Int32, p Int32 MATERIALIZED x * y) ENGINE = MergeTree ORDER BY tuple()" + ) select_query = "SELECT * FROM table1" - assert "it's necessary to have grant SELECT(x, y) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(x, y) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(x,y) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" select_query = "SELECT p FROM table1" - assert "it's necessary to have grant SELECT(p) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') - + assert ( + "it's necessary to have grant SELECT(p) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) + instance.query("GRANT SELECT(p) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" instance.query("REVOKE SELECT(x,y) ON default.table1 FROM A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" def test_select_join(): - instance.query("CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d") - instance.query("CREATE TABLE table2(d DATE, x UInt32, y UInt8) ENGINE = MergeTree ORDER BY d") + instance.query( + "CREATE TABLE table1(d DATE, a String, b UInt8) ENGINE = MergeTree ORDER BY d" + ) + instance.query( + "CREATE TABLE table2(d DATE, x UInt32, y UInt8) ENGINE = MergeTree ORDER BY d" + ) select_query = "SELECT * FROM table1 JOIN table2 USING(d)" - assert "it's necessary to have grant SELECT(d, x, y) ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, x, y) ON default.table2" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(d, x, y) ON default.table2 TO A") - assert "it's necessary to have grant SELECT(d, a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(d, a, b) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" instance.query("REVOKE SELECT ON default.table2 FROM A") - assert "it's necessary to have grant SELECT(d, x, y) ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(d, x, y) ON default.table2" + in instance.query_and_get_error(select_query, user="A") + ) def test_select_union(): - instance.query("CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()") - instance.query("CREATE TABLE table2(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()" + ) + instance.query( + "CREATE TABLE table2(a String, b UInt8) ENGINE = MergeTree ORDER BY tuple()" + ) select_query = "SELECT * FROM table1 UNION ALL SELECT * FROM table2" - assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(a, b) ON default.table1 TO A") - assert "it's necessary to have grant SELECT(a, b) ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table2" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(a, b) ON default.table2 TO A") - assert instance.query(select_query, user = 'A') == "" + assert instance.query(select_query, user="A") == "" instance.query("REVOKE SELECT ON default.table1 FROM A") - assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) def test_select_count(): - instance.query("CREATE TABLE table1(x String, y UInt8) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE table1(x String, y UInt8) ENGINE = MergeTree ORDER BY tuple()" + ) select_query = "SELECT count() FROM table1" - assert "it's necessary to have grant SELECT for at least one column on default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT for at least one column on default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(x) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "0\n" + assert instance.query(select_query, user="A") == "0\n" instance.query("REVOKE SELECT(x) ON default.table1 FROM A") - assert "it's necessary to have grant SELECT for at least one column on default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT for at least one column on default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(y) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "0\n" + assert instance.query(select_query, user="A") == "0\n" instance.query("REVOKE SELECT(y) ON default.table1 FROM A") - assert "it's necessary to have grant SELECT for at least one column on default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT for at least one column on default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "0\n" + assert instance.query(select_query, user="A") == "0\n" def test_select_where(): # User should have grants for the columns used in WHERE. - instance.query("CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY b") + instance.query( + "CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY b" + ) instance.query("INSERT INTO table1 VALUES ('xxx', 0), ('yyy', 1), ('zzz', 0)") instance.query("GRANT SELECT(a) ON default.table1 TO A") select_query = "SELECT a FROM table1 WHERE b = 0" - assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(b) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "xxx\nzzz\n" + assert instance.query(select_query, user="A") == "xxx\nzzz\n" instance.query("REVOKE SELECT ON default.table1 FROM A") - assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "xxx\nzzz\n" + assert instance.query(select_query, user="A") == "xxx\nzzz\n" def test_select_prewhere(): # User should have grants for the columns used in PREWHERE. - instance.query("CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY b") + instance.query( + "CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY b" + ) instance.query("INSERT INTO table1 VALUES ('xxx', 0), ('yyy', 1), ('zzz', 0)") instance.query("GRANT SELECT(a) ON default.table1 TO A") select_query = "SELECT a FROM table1 PREWHERE b = 0" - assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT(b) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "xxx\nzzz\n" + assert instance.query(select_query, user="A") == "xxx\nzzz\n" instance.query("REVOKE SELECT ON default.table1 FROM A") - assert "it's necessary to have grant SELECT(a, b) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT(a, b) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT SELECT ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "xxx\nzzz\n" + assert instance.query(select_query, user="A") == "xxx\nzzz\n" def test_select_with_row_policy(): # Normal users should not aware of the existence of row policy filters. - instance.query("CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY b") + instance.query( + "CREATE TABLE table1(a String, b UInt8) ENGINE = MergeTree ORDER BY b" + ) instance.query("INSERT INTO table1 VALUES ('xxx', 0), ('yyy', 1), ('zzz', 0)") instance.query("CREATE ROW POLICY pol1 ON table1 USING b = 0 TO A") select_query = "SELECT a FROM table1" select_query2 = "SELECT count() FROM table1" - assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') - assert "it's necessary to have grant SELECT for at least one column on default.table1" in instance.query_and_get_error(select_query2, user = 'A') + assert ( + "it's necessary to have grant SELECT(a) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) + assert ( + "it's necessary to have grant SELECT for at least one column on default.table1" + in instance.query_and_get_error(select_query2, user="A") + ) instance.query("GRANT SELECT(a) ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "xxx\nzzz\n" - assert instance.query(select_query2, user = 'A') == "2\n" + assert instance.query(select_query, user="A") == "xxx\nzzz\n" + assert instance.query(select_query2, user="A") == "2\n" instance.query("REVOKE SELECT(a) ON default.table1 FROM A") - assert "it's necessary to have grant SELECT(a) ON default.table1" in instance.query_and_get_error(select_query, user = 'A') - assert "it's necessary to have grant SELECT for at least one column on default.table1" in instance.query_and_get_error(select_query2, user = 'A') + assert ( + "it's necessary to have grant SELECT(a) ON default.table1" + in instance.query_and_get_error(select_query, user="A") + ) + assert ( + "it's necessary to have grant SELECT for at least one column on default.table1" + in instance.query_and_get_error(select_query2, user="A") + ) diff --git a/tests/integration/test_send_crash_reports/fake_sentry_server.py b/tests/integration/test_send_crash_reports/fake_sentry_server.py index fa40f642e41..37d733cc005 100644 --- a/tests/integration/test_send_crash_reports/fake_sentry_server.py +++ b/tests/integration/test_send_crash_reports/fake_sentry_server.py @@ -1,19 +1,27 @@ import http.server -RESULT_PATH = '/result.txt' +RESULT_PATH = "/result.txt" class SentryHandler(http.server.BaseHTTPRequestHandler): def do_POST(self): post_data = self.__read_and_decode_post_data() - with open(RESULT_PATH, 'w') as f: + with open(RESULT_PATH, "w") as f: content_length = self.headers.get("content-length") if self.headers.get("content-type") != "application/x-sentry-envelope": f.write("INCORRECT_CONTENT_TYPE") elif int(content_length) < 200: - f.write("INCORRECT_CONTENT_LENGTH:" + content_length + '\n' + post_data.decode()) - elif b'"http://6f33034cfe684dd7a3ab9875e57b1c8d@localhost:9500/5226277"' not in post_data: - f.write('INCORRECT_POST_DATA') + f.write( + "INCORRECT_CONTENT_LENGTH:" + + content_length + + "\n" + + post_data.decode() + ) + elif ( + b'"http://6f33034cfe684dd7a3ab9875e57b1c8d@localhost:9500/5226277"' + not in post_data + ): + f.write("INCORRECT_POST_DATA") else: f.write("OK") self.send_response(200) @@ -36,9 +44,15 @@ class SentryHandler(http.server.BaseHTTPRequestHandler): if __name__ == "__main__": - with open(RESULT_PATH, 'w') as f: + with open(RESULT_PATH, "w") as f: f.write("INITIAL_STATE") - httpd = http.server.HTTPServer(("localhost", 9500,), SentryHandler) + httpd = http.server.HTTPServer( + ( + "localhost", + 9500, + ), + SentryHandler, + ) try: httpd.serve_forever() finally: diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index 55c63c3fe12..90a6c684de7 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -19,9 +19,12 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) def started_node(): cluster = helpers.cluster.ClickHouseCluster(__file__) try: - node = cluster.add_instance("node", main_configs=[ - os.path.join(SCRIPT_DIR, "configs", "config_send_crash_reports.xml") - ]) + node = cluster.add_instance( + "node", + main_configs=[ + os.path.join(SCRIPT_DIR, "configs", "config_send_crash_reports.xml") + ], + ) cluster.start() yield node finally: @@ -33,23 +36,36 @@ def started_node(): def test_send_segfault(started_node): - if started_node.is_built_with_thread_sanitizer() or started_node.is_built_with_memory_sanitizer(): + if ( + started_node.is_built_with_thread_sanitizer() + or started_node.is_built_with_memory_sanitizer() + ): pytest.skip("doesn't fit in timeouts for stacktrace generation") - started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py") - started_node.exec_in_container(["bash", "-c", "python3 /fake_sentry_server.py > /fake_sentry_server.log 2>&1"], detach=True, user="root") + started_node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py" + ) + started_node.exec_in_container( + ["bash", "-c", "python3 /fake_sentry_server.py > /fake_sentry_server.log 2>&1"], + detach=True, + user="root", + ) time.sleep(1) - started_node.exec_in_container(["bash", "-c", "pkill -SEGV clickhouse"], user="root") + started_node.exec_in_container( + ["bash", "-c", "pkill -SEGV clickhouse"], user="root" + ) result = None for attempt in range(1, 6): time.sleep(attempt) - result = started_node.exec_in_container(['cat', fake_sentry_server.RESULT_PATH], user='root') - if result == 'OK': + result = started_node.exec_in_container( + ["cat", fake_sentry_server.RESULT_PATH], user="root" + ) + if result == "OK": break - if result == 'INITIAL_STATE': + if result == "INITIAL_STATE": continue if result: - assert False, 'Unexpected state: ' + result + assert False, "Unexpected state: " + result - assert result == 'OK', 'Crash report not sent' + assert result == "OK", "Crash report not sent" diff --git a/tests/integration/test_send_request_to_leader_replica/test.py b/tests/integration/test_send_request_to_leader_replica/test.py index 721e446ff82..60df18bf7d3 100644 --- a/tests/integration/test_send_request_to_leader_replica/test.py +++ b/tests/integration/test_send_request_to_leader_replica/test.py @@ -5,14 +5,30 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/user_good_restricted.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], - user_configs=['configs/user_good_allowed.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/user_good_restricted.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/user_good_restricted.xml"], + with_zookeeper=True, +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/user_good_allowed.xml"], + with_zookeeper=True, +) +node4 = cluster.add_instance( + "node4", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/user_good_allowed.xml"], + with_zookeeper=True, +) @pytest.fixture(scope="module") @@ -21,16 +37,26 @@ def started_cluster(): cluster.start() for node in [node1, node2]: - node.query(''' + node.query( + """ CREATE TABLE sometable(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/sometable', '{replica}', date, id, 8192); - '''.format(replica=node.name), user='awesome') + """.format( + replica=node.name + ), + user="awesome", + ) for node in [node3, node4]: - node.query(''' + node.query( + """ CREATE TABLE someothertable(date Date, id UInt32, value Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/someothertable', '{replica}', date, id, 8192); - '''.format(replica=node.name), user='good') + """.format( + replica=node.name + ), + user="good", + ) yield cluster @@ -38,39 +64,82 @@ def started_cluster(): cluster.shutdown() -@pytest.mark.parametrize("table,query,expected,n1,n2", [ - pytest.param("sometable", "ALTER TABLE sometable DROP PARTITION 201706", '1', node1, node2, id="case1"), - pytest.param("sometable", "TRUNCATE TABLE sometable", '0', node1, node2, id="case2"), - pytest.param("sometable", "OPTIMIZE TABLE sometable", '4', node1, node2, id="case3"), - pytest.param("someothertable", "ALTER TABLE someothertable DROP PARTITION 201706", '1', node3, node4, id="case4"), - pytest.param("someothertable", "TRUNCATE TABLE someothertable", '0', node3, node4, id="case5"), - pytest.param("someothertable", "OPTIMIZE TABLE someothertable", '4', node3, node4, id="case6"), -]) +@pytest.mark.parametrize( + "table,query,expected,n1,n2", + [ + pytest.param( + "sometable", + "ALTER TABLE sometable DROP PARTITION 201706", + "1", + node1, + node2, + id="case1", + ), + pytest.param( + "sometable", "TRUNCATE TABLE sometable", "0", node1, node2, id="case2" + ), + pytest.param( + "sometable", "OPTIMIZE TABLE sometable", "4", node1, node2, id="case3" + ), + pytest.param( + "someothertable", + "ALTER TABLE someothertable DROP PARTITION 201706", + "1", + node3, + node4, + id="case4", + ), + pytest.param( + "someothertable", + "TRUNCATE TABLE someothertable", + "0", + node3, + node4, + id="case5", + ), + pytest.param( + "someothertable", + "OPTIMIZE TABLE someothertable", + "4", + node3, + node4, + id="case6", + ), + ], +) def test_alter_table_drop_partition(started_cluster, table, query, expected, n1, n2): - to_insert = '''\ + to_insert = """\ 2017-06-16 111 0 2017-06-16 222 1 2017-06-16 333 2 2017-07-16 444 3 -''' - n1.query("INSERT INTO {} FORMAT TSV".format(table), stdin=to_insert, user='good') +""" + n1.query("INSERT INTO {} FORMAT TSV".format(table), stdin=to_insert, user="good") - assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), '4', user='good') - assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), '4', user='good') + assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), "4", user="good") + assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), "4", user="good") ### It maybe leader and everything will be ok - n1.query(query, user='good') + n1.query(query, user="good") - assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), expected, user='good') - assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), expected, user='good') + assert_eq_with_retry( + n1, "SELECT COUNT(*) from {}".format(table), expected, user="good" + ) + assert_eq_with_retry( + n2, "SELECT COUNT(*) from {}".format(table), expected, user="good" + ) - n1.query("INSERT INTO {} FORMAT TSV".format(table), stdin=to_insert, user='good') + n1.query("INSERT INTO {} FORMAT TSV".format(table), stdin=to_insert, user="good") - assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), '4', user='good') - assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), '4', user='good') + assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), "4", user="good") + assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), "4", user="good") ### If node1 is leader than node2 will be slave - n2.query(query, user='good') + n2.query(query, user="good") - assert_eq_with_retry(n1, "SELECT COUNT(*) from {}".format(table), expected, user='good') - assert_eq_with_retry(n2, "SELECT COUNT(*) from {}".format(table), expected, user='good') + assert_eq_with_retry( + n1, "SELECT COUNT(*) from {}".format(table), expected, user="good" + ) + assert_eq_with_retry( + n2, "SELECT COUNT(*) from {}".format(table), expected, user="good" + ) diff --git a/tests/integration/test_server_initialization/test.py b/tests/integration/test_server_initialization/test.py index 08032436982..1b57e14a51b 100644 --- a/tests/integration/test_server_initialization/test.py +++ b/tests/integration/test_server_initialization/test.py @@ -7,11 +7,15 @@ from helpers.cluster import ClickHouseCluster def started_cluster(): try: cluster = ClickHouseCluster(__file__) - instance = cluster.add_instance('dummy', clickhouse_path_dir='clickhouse_path', stay_alive=True) + instance = cluster.add_instance( + "dummy", clickhouse_path_dir="clickhouse_path", stay_alive=True + ) cluster.start() - cluster_fail = ClickHouseCluster(__file__, name='fail') - instance_fail = cluster_fail.add_instance('dummy_fail', clickhouse_path_dir='clickhouse_path_fail') + cluster_fail = ClickHouseCluster(__file__, name="fail") + instance_fail = cluster_fail.add_instance( + "dummy_fail", clickhouse_path_dir="clickhouse_path_fail" + ) with pytest.raises(Exception): cluster_fail.start() cluster_fail.shutdown() # cleanup @@ -23,26 +27,40 @@ def started_cluster(): def test_sophisticated_default(started_cluster): - instance = started_cluster.instances['dummy'] + instance = started_cluster.instances["dummy"] instance.query("INSERT INTO sophisticated_default (c) VALUES (0)") assert instance.query("SELECT a, b, c FROM sophisticated_default") == "3\t9\t0\n" def test_partially_dropped_tables(started_cluster): - instance = started_cluster.instances['dummy'] - assert instance.exec_in_container(['bash', '-c', 'find /var/lib/clickhouse/*/default -name *.sql* | sort'], - privileged=True, user='root') \ - == "/var/lib/clickhouse/metadata/default/should_be_restored.sql\n" \ - "/var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" + instance = started_cluster.instances["dummy"] + assert ( + instance.exec_in_container( + ["bash", "-c", "find /var/lib/clickhouse/*/default -name *.sql* | sort"], + privileged=True, + user="root", + ) + == "/var/lib/clickhouse/metadata/default/should_be_restored.sql\n" + "/var/lib/clickhouse/metadata/default/sophisticated_default.sql\n" + ) assert instance.query("SELECT n FROM should_be_restored") == "1\n2\n3\n" - assert instance.query("SELECT count() FROM system.tables WHERE name='should_be_dropped'") == "0\n" + assert ( + instance.query( + "SELECT count() FROM system.tables WHERE name='should_be_dropped'" + ) + == "0\n" + ) def test_live_view_dependency(started_cluster): - instance = started_cluster.instances['dummy'] + instance = started_cluster.instances["dummy"] instance.query("CREATE DATABASE a_load_first") instance.query("CREATE DATABASE b_load_second") - instance.query("CREATE TABLE b_load_second.mt (a Int32) Engine=MergeTree order by tuple()") - instance.query("CREATE LIVE VIEW a_load_first.lv AS SELECT sum(a) FROM b_load_second.mt", - settings={'allow_experimental_live_view': 1}) + instance.query( + "CREATE TABLE b_load_second.mt (a Int32) Engine=MergeTree order by tuple()" + ) + instance.query( + "CREATE LIVE VIEW a_load_first.lv AS SELECT sum(a) FROM b_load_second.mt", + settings={"allow_experimental_live_view": 1}, + ) instance.restart_clickhouse() diff --git a/tests/integration/test_server_reload/test.py b/tests/integration/test_server_reload/test.py index 3c22b476f64..5cda659b5c4 100644 --- a/tests/integration/test_server_reload/test.py +++ b/tests/integration/test_server_reload/test.py @@ -17,10 +17,15 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( "instance", main_configs=[ - "configs/ports_from_zk.xml", "configs/ssl_conf.xml", "configs/dhparam.pem", "configs/server.crt", "configs/server.key" + "configs/ports_from_zk.xml", + "configs/ssl_conf.xml", + "configs/dhparam.pem", + "configs/server.crt", + "configs/server.key", ], user_configs=["configs/default_passwd.xml"], - with_zookeeper=True) + with_zookeeper=True, +) LOADS_QUERY = "SELECT value FROM system.events WHERE event = 'MainConfigLoads'" @@ -33,7 +38,9 @@ gen_dir = Path(__file__).parent / "_gen" gen_dir.mkdir(exist_ok=True) run_and_check( f"python3 -m grpc_tools.protoc -I{proto_dir!s} --python_out={gen_dir!s} --grpc_python_out={gen_dir!s} \ - {proto_dir!s}/clickhouse_grpc.proto", shell=True) + {proto_dir!s}/clickhouse_grpc.proto", + shell=True, +) sys.path.append(str(gen_dir)) import clickhouse_grpc_pb2 @@ -56,7 +63,11 @@ def fixture_zk(cluster): def get_client(cluster, port): - return Client(host=cluster.get_instance_ip("instance"), port=port, command=cluster.client_bin_path) + return Client( + host=cluster.get_instance_ip("instance"), + port=port, + command=cluster.client_bin_path, + ) def get_mysql_client(cluster, port): @@ -64,7 +75,12 @@ def get_mysql_client(cluster, port): while True: try: return pymysql.connections.Connection( - host=cluster.get_instance_ip("instance"), user="default", password="", database="default", port=port) + host=cluster.get_instance_ip("instance"), + user="default", + password="", + database="default", + port=port, + ) except pymysql.err.OperationalError: if time.monotonic() - start_time > 10: raise @@ -76,7 +92,12 @@ def get_pgsql_client(cluster, port): while True: try: return psycopg2.connect( - host=cluster.get_instance_ip("instance"), user="postgresql", password="123", database="default", port=port) + host=cluster.get_instance_ip("instance"), + user="postgresql", + password="123", + database="default", + port=port, + ) except psycopg2.OperationalError: if time.monotonic() - start_time > 10: raise @@ -212,7 +233,9 @@ def test_change_grpc_port(cluster, zk): assert grpc_query(grpc_channel, "SELECT 1") == "1\n" with sync_loaded_config(client.query): zk.set("/clickhouse/ports/grpc", b"9090") - with pytest.raises(grpc._channel._InactiveRpcError, match="StatusCode.UNAVAILABLE"): + with pytest.raises( + grpc._channel._InactiveRpcError, match="StatusCode.UNAVAILABLE" + ): grpc_query(grpc_channel, "SELECT 1") grpc_channel_on_new_port = get_grpc_channel(cluster, port=9090) assert grpc_query(grpc_channel_on_new_port, "SELECT 1") == "1\n" @@ -264,13 +287,22 @@ def test_remove_grpc_port(cluster, zk): assert grpc_query(grpc_channel, "SELECT 1") == "1\n" with sync_loaded_config(client.query): zk.delete("/clickhouse/ports/grpc") - with pytest.raises(grpc._channel._InactiveRpcError, match="StatusCode.UNAVAILABLE"): + with pytest.raises( + grpc._channel._InactiveRpcError, match="StatusCode.UNAVAILABLE" + ): grpc_query(grpc_channel, "SELECT 1") def test_change_listen_host(cluster, zk): - localhost_client = Client(host="127.0.0.1", port=9000, command="/usr/bin/clickhouse") - localhost_client.command = ["docker", "exec", "-i", instance.docker_id] + localhost_client.command + localhost_client = Client( + host="127.0.0.1", port=9000, command="/usr/bin/clickhouse" + ) + localhost_client.command = [ + "docker", + "exec", + "-i", + instance.docker_id, + ] + localhost_client.command try: client = get_client(cluster, port=9000) with sync_loaded_config(localhost_client.query): @@ -281,4 +313,3 @@ def test_change_listen_host(cluster, zk): finally: with sync_loaded_config(localhost_client.query): configure_ports_from_zk(zk) - diff --git a/tests/integration/test_settings_constraints/test.py b/tests/integration/test_settings_constraints/test.py index 18c80d9c1da..f6490c60407 100644 --- a/tests/integration/test_settings_constraints/test.py +++ b/tests/integration/test_settings_constraints/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', user_configs=["configs/users.xml"]) +instance = cluster.add_instance("instance", user_configs=["configs/users.xml"]) @pytest.fixture(scope="module") @@ -16,80 +16,122 @@ def started_cluster(): def test_system_settings(started_cluster): - assert instance.query( - "SELECT name, value, min, max, readonly from system.settings WHERE name = 'force_index_by_date'") == \ - "force_index_by_date\t0\t\\N\t\\N\t1\n" + assert ( + instance.query( + "SELECT name, value, min, max, readonly from system.settings WHERE name = 'force_index_by_date'" + ) + == "force_index_by_date\t0\t\\N\t\\N\t1\n" + ) - assert instance.query( - "SELECT name, value, min, max, readonly from system.settings WHERE name = 'max_memory_usage'") == \ - "max_memory_usage\t10000000000\t5000000000\t20000000000\t0\n" + assert ( + instance.query( + "SELECT name, value, min, max, readonly from system.settings WHERE name = 'max_memory_usage'" + ) + == "max_memory_usage\t10000000000\t5000000000\t20000000000\t0\n" + ) - assert instance.query("SELECT name, value, min, max, readonly from system.settings WHERE name = 'readonly'") == \ - "readonly\t0\t\\N\t\\N\t0\n" + assert ( + instance.query( + "SELECT name, value, min, max, readonly from system.settings WHERE name = 'readonly'" + ) + == "readonly\t0\t\\N\t\\N\t0\n" + ) def test_system_constraints(started_cluster): - assert_query_settings(instance, "SELECT 1", - settings={'readonly': 0}, - exception="Cannot modify 'readonly'", - user="readonly_user") + assert_query_settings( + instance, + "SELECT 1", + settings={"readonly": 0}, + exception="Cannot modify 'readonly'", + user="readonly_user", + ) - assert_query_settings(instance, "SELECT 1", - settings={'allow_ddl': 1}, - exception="Cannot modify 'allow_ddl'", - user="no_dll_user") + assert_query_settings( + instance, + "SELECT 1", + settings={"allow_ddl": 1}, + exception="Cannot modify 'allow_ddl'", + user="no_dll_user", + ) def test_read_only_constraint(started_cluster): # Default value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='force_index_by_date'", - settings={}, - result="0") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='force_index_by_date'", + settings={}, + result="0", + ) # Invalid value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='force_index_by_date'", - settings={'force_index_by_date': 1}, - result=None, - exception="Setting force_index_by_date should not be changed") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='force_index_by_date'", + settings={"force_index_by_date": 1}, + result=None, + exception="Setting force_index_by_date should not be changed", + ) def test_min_constraint(started_cluster): # Default value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", - {}, - result="10000000000") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='max_memory_usage'", + {}, + result="10000000000", + ) # Valid value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", - settings={'max_memory_usage': 5000000000}, - result="5000000000") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={"max_memory_usage": 5000000000}, + result="5000000000", + ) # Invalid value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", - settings={'max_memory_usage': 4999999999}, - result=None, - exception="Setting max_memory_usage shouldn't be less than 5000000000") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={"max_memory_usage": 4999999999}, + result=None, + exception="Setting max_memory_usage shouldn't be less than 5000000000", + ) def test_max_constraint(started_cluster): # Default value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", - {}, - result="10000000000") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='max_memory_usage'", + {}, + result="10000000000", + ) # Valid value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", - settings={'max_memory_usage': 20000000000}, - result="20000000000") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={"max_memory_usage": 20000000000}, + result="20000000000", + ) # Invalid value - assert_query_settings(instance, "SELECT value FROM system.settings WHERE name='max_memory_usage'", - settings={'max_memory_usage': 20000000001}, - result=None, - exception="Setting max_memory_usage shouldn't be greater than 20000000000") + assert_query_settings( + instance, + "SELECT value FROM system.settings WHERE name='max_memory_usage'", + settings={"max_memory_usage": 20000000001}, + result=None, + exception="Setting max_memory_usage shouldn't be greater than 20000000000", + ) -def assert_query_settings(instance, query, settings, result=None, exception=None, user=None): +def assert_query_settings( + instance, query, settings, result=None, exception=None, user=None +): """ Try and send the query with custom settings via all available methods: 1. TCP Protocol with settings packet @@ -103,13 +145,17 @@ def assert_query_settings(instance, query, settings, result=None, exception=None # tcp level settings if exception: - assert exception in instance.query_and_get_error(query, settings=settings, user=user) + assert exception in instance.query_and_get_error( + query, settings=settings, user=user + ) else: assert instance.query(query, settings=settings, user=user).strip() == result # http level settings if exception: - assert exception in instance.http_query_and_get_error(query, params=settings, user=user) + assert exception in instance.http_query_and_get_error( + query, params=settings, user=user + ) else: assert instance.http_query(query, params=settings, user=user).strip() == result diff --git a/tests/integration/test_settings_constraints_distributed/test.py b/tests/integration/test_settings_constraints_distributed/test.py index eed4e66131d..75d4732ffc4 100644 --- a/tests/integration/test_settings_constraints_distributed/test.py +++ b/tests/integration/test_settings_constraints_distributed/test.py @@ -5,12 +5,22 @@ from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/allow_introspection_functions.xml"]) -node2 = cluster.add_instance('node2', main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/allow_introspection_functions.xml"]) -distributed = cluster.add_instance('distributed', main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/allow_introspection_functions.xml"], stay_alive=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml"], +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml"], +) +distributed = cluster.add_instance( + "distributed", + main_configs=["configs/config.d/remote_servers.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml"], + stay_alive=True, +) @pytest.fixture(scope="module", autouse=True) @@ -24,7 +34,8 @@ def started_cluster(): distributed.query("CREATE ROLE admin") distributed.query("GRANT ALL ON *.* TO admin") distributed.query( - "CREATE TABLE shard_settings (name String, value String) ENGINE = Distributed(test_cluster, system, settings);") + "CREATE TABLE shard_settings (name String, value String) ENGINE = Distributed(test_cluster, system, settings);" + ) yield cluster @@ -42,81 +53,126 @@ def restart_distributed(): def test_select_clamps_settings(): for node in [node1, node2]: - node.query("CREATE TABLE sometable_select (date Date, id UInt32, value Int32) ENGINE = MergeTree() ORDER BY id;") + node.query( + "CREATE TABLE sometable_select (date Date, id UInt32, value Int32) ENGINE = MergeTree() ORDER BY id;" + ) node.query("INSERT INTO sometable_select VALUES (toDate('2010-01-10'), 1, 1)") distributed.query( - "CREATE TABLE proxy_select (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable_select, toUInt64(date));") + "CREATE TABLE proxy_select (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable_select, toUInt64(date));" + ) - - distributed.query("CREATE USER normal DEFAULT ROLE admin SETTINGS max_memory_usage = 80000000") - distributed.query("CREATE USER wasteful DEFAULT ROLE admin SETTINGS max_memory_usage = 2000000000") + distributed.query( + "CREATE USER normal DEFAULT ROLE admin SETTINGS max_memory_usage = 80000000" + ) + distributed.query( + "CREATE USER wasteful DEFAULT ROLE admin SETTINGS max_memory_usage = 2000000000" + ) distributed.query("CREATE USER readonly DEFAULT ROLE admin SETTINGS readonly = 1") - node1.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") + node1.query( + "ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999" + ) node2.query("ALTER USER shard SETTINGS readonly = 1") # Check that shards doesn't throw exceptions on constraints violation query = "SELECT COUNT() FROM proxy_select" - assert distributed.query(query) == '2\n' - assert distributed.query(query, user='normal') == '2\n' - assert distributed.query(query, user='wasteful') == '2\n' - assert distributed.query(query, user='readonly') == '2\n' + assert distributed.query(query) == "2\n" + assert distributed.query(query, user="normal") == "2\n" + assert distributed.query(query, user="wasteful") == "2\n" + assert distributed.query(query, user="readonly") == "2\n" - assert distributed.query(query, settings={"max_memory_usage": 40000000, "readonly": 2}) == '2\n' - assert distributed.query(query, settings={"max_memory_usage": 3000000000, "readonly": 2}) == '2\n' + assert ( + distributed.query(query, settings={"max_memory_usage": 40000000, "readonly": 2}) + == "2\n" + ) + assert ( + distributed.query( + query, settings={"max_memory_usage": 3000000000, "readonly": 2} + ) + == "2\n" + ) query = "SELECT COUNT() FROM remote('node{1,2}', 'default', 'sometable_select')" - assert distributed.query(query) == '2\n' - assert distributed.query(query, user='normal') == '2\n' - assert distributed.query(query, user='wasteful') == '2\n' + assert distributed.query(query) == "2\n" + assert distributed.query(query, user="normal") == "2\n" + assert distributed.query(query, user="wasteful") == "2\n" # Check that shards clamp passed settings. query = "SELECT hostName() as host, name, value FROM shard_settings WHERE name = 'max_memory_usage' OR name = 'readonly' ORDER BY host, name, value" - assert distributed.query(query) == 'node1\tmax_memory_usage\t99999999\n' \ - 'node1\treadonly\t0\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' - assert distributed.query(query, user='normal') == 'node1\tmax_memory_usage\t80000000\n' \ - 'node1\treadonly\t0\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' - assert distributed.query(query, user='wasteful') == 'node1\tmax_memory_usage\t99999999\n' \ - 'node1\treadonly\t0\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' - assert distributed.query(query, user='readonly') == 'node1\tmax_memory_usage\t99999999\n' \ - 'node1\treadonly\t1\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' + assert ( + distributed.query(query) == "node1\tmax_memory_usage\t99999999\n" + "node1\treadonly\t0\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) + assert ( + distributed.query(query, user="normal") == "node1\tmax_memory_usage\t80000000\n" + "node1\treadonly\t0\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) + assert ( + distributed.query(query, user="wasteful") + == "node1\tmax_memory_usage\t99999999\n" + "node1\treadonly\t0\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) + assert ( + distributed.query(query, user="readonly") + == "node1\tmax_memory_usage\t99999999\n" + "node1\treadonly\t1\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) - assert distributed.query(query, settings={"max_memory_usage": 1}) == 'node1\tmax_memory_usage\t11111111\n' \ - 'node1\treadonly\t0\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' - assert distributed.query(query, settings={"max_memory_usage": 40000000, - "readonly": 2}) == 'node1\tmax_memory_usage\t40000000\n' \ - 'node1\treadonly\t2\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' - assert distributed.query(query, settings={"max_memory_usage": 3000000000, - "readonly": 2}) == 'node1\tmax_memory_usage\t99999999\n' \ - 'node1\treadonly\t2\n' \ - 'node2\tmax_memory_usage\t10000000000\n' \ - 'node2\treadonly\t1\n' + assert ( + distributed.query(query, settings={"max_memory_usage": 1}) + == "node1\tmax_memory_usage\t11111111\n" + "node1\treadonly\t0\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) + assert ( + distributed.query(query, settings={"max_memory_usage": 40000000, "readonly": 2}) + == "node1\tmax_memory_usage\t40000000\n" + "node1\treadonly\t2\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) + assert ( + distributed.query( + query, settings={"max_memory_usage": 3000000000, "readonly": 2} + ) + == "node1\tmax_memory_usage\t99999999\n" + "node1\treadonly\t2\n" + "node2\tmax_memory_usage\t10000000000\n" + "node2\treadonly\t1\n" + ) def test_insert_clamps_settings(): for node in [node1, node2]: - node.query("CREATE TABLE sometable_insert (date Date, id UInt32, value Int32) ENGINE = MergeTree() ORDER BY id;") + node.query( + "CREATE TABLE sometable_insert (date Date, id UInt32, value Int32) ENGINE = MergeTree() ORDER BY id;" + ) node.query("INSERT INTO sometable_insert VALUES (toDate('2010-01-10'), 1, 1)") distributed.query( - "CREATE TABLE proxy_insert (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable_insert, toUInt64(date));") + "CREATE TABLE proxy_insert (date Date, id UInt32, value Int32) ENGINE = Distributed(test_cluster, default, sometable_insert, toUInt64(date));" + ) - node1.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") - node2.query("ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999") + node1.query( + "ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999" + ) + node2.query( + "ALTER USER shard SETTINGS max_memory_usage = 50000000 MIN 11111111 MAX 99999999" + ) distributed.query("INSERT INTO proxy_insert VALUES (toDate('2020-02-20'), 2, 2)") - distributed.query("INSERT INTO proxy_insert VALUES (toDate('2020-02-21'), 2, 2)", settings={"max_memory_usage": 5000000}) + distributed.query( + "INSERT INTO proxy_insert VALUES (toDate('2020-02-21'), 2, 2)", + settings={"max_memory_usage": 5000000}, + ) distributed.query("SYSTEM FLUSH DISTRIBUTED proxy_insert") assert_eq_with_retry(distributed, "SELECT COUNT() FROM proxy_insert", "4") diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 7be0b395764..b4c0cec9f48 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -3,12 +3,17 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") def system_settings_profile(profile_name): - return TSV(instance.query( - "SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name='" + profile_name + "'")) + return TSV( + instance.query( + "SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name='" + + profile_name + + "'" + ) + ) def system_settings_profile_elements(profile_name=None, user_name=None, role_name=None): @@ -23,10 +28,12 @@ def system_settings_profile_elements(profile_name=None, user_name=None, role_nam session_id_counter = 0 + + def new_session_id(): global session_id_counter session_id_counter += 1 - return 'session #' + str(session_id_counter) + return "session #" + str(session_id_counter) @pytest.fixture(scope="module", autouse=True) @@ -49,51 +56,110 @@ def reset_after_test(): finally: instance.query("CREATE USER OR REPLACE robin") instance.query("DROP ROLE IF EXISTS worker") - instance.query("DROP SETTINGS PROFILE IF EXISTS xyz, alpha, P1, P2, P3, P4, P5, P6") + instance.query( + "DROP SETTINGS PROFILE IF EXISTS xyz, alpha, P1, P2, P3, P4, P5, P6" + ) def test_smoke(): # Set settings and constraints via CREATE SETTINGS PROFILE ... TO user instance.query( - "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin") - assert instance.query( - "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error( - "SET max_memory_usage = 80000000", user="robin") - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( - "SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [["xyz", "local directory", 1, 0, "['robin']", "[]"]] + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin" + ) + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE xyz") + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000001\n" + ) + assert ( + "Setting max_memory_usage shouldn't be less than 90000000" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) + assert system_settings_profile("xyz") == [ + ["xyz", "local directory", 1, 0, "['robin']", "[]"] + ] assert system_settings_profile_elements(profile_name="xyz") == [ - ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000001, 90000000, 110000000, "\\N", "\\N"]] + [ + "xyz", + "\\N", + "\\N", + 0, + "max_memory_usage", + 100000001, + 90000000, + 110000000, + "\\N", + "\\N", + ] + ] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") - assert instance.query( - "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "10000000000\n" + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE xyz") + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "10000000000\n" + ) instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [["xyz", "local directory", 1, 0, "[]", "[]"]] + assert system_settings_profile("xyz") == [ + ["xyz", "local directory", 1, 0, "[]", "[]"] + ] assert system_settings_profile_elements(user_name="robin") == [] # Set settings and constraints via CREATE USER ... SETTINGS PROFILE instance.query("ALTER USER robin SETTINGS PROFILE xyz") - assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin SETTINGS PROFILE xyz\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error( - "SET max_memory_usage = 80000000", user="robin") - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( - "SET max_memory_usage = 120000000", user="robin") + assert ( + instance.query("SHOW CREATE USER robin") + == "CREATE USER robin SETTINGS PROFILE xyz\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000001\n" + ) + assert ( + "Setting max_memory_usage shouldn't be less than 90000000" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) assert system_settings_profile_elements(user_name="robin") == [ - ["\\N", "robin", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"]] + ["\\N", "robin", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ] instance.query("ALTER USER robin SETTINGS NONE") assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "10000000000\n" + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "10000000000\n" + ) instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") assert system_settings_profile_elements(user_name="robin") == [] @@ -102,94 +168,200 @@ def test_smoke(): def test_settings_from_granted_role(): # Set settings and constraints via granted role instance.query( - "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000") + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000" + ) instance.query("CREATE ROLE worker SETTINGS PROFILE xyz") instance.query("GRANT worker TO robin") - assert instance.query( - "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" - assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker SETTINGS PROFILE xyz\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "100000001\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_ast_depth'", user="robin") == "2000\n" - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( - "SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [["xyz", "local directory", 2, 0, "[]", "[]"]] + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE xyz") + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" + ) + assert ( + instance.query("SHOW CREATE ROLE worker") + == "CREATE ROLE worker SETTINGS PROFILE xyz\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000001\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_ast_depth'", + user="robin", + ) + == "2000\n" + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) + assert system_settings_profile("xyz") == [ + ["xyz", "local directory", 2, 0, "[]", "[]"] + ] assert system_settings_profile_elements(profile_name="xyz") == [ - ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000001, "\\N", 110000000, "\\N", "\\N"], - ["xyz", "\\N", "\\N", 1, "max_ast_depth", 2000, "\\N", "\\N", "\\N", "\\N"]] + [ + "xyz", + "\\N", + "\\N", + 0, + "max_memory_usage", + 100000001, + "\\N", + 110000000, + "\\N", + "\\N", + ], + ["xyz", "\\N", "\\N", 1, "max_ast_depth", 2000, "\\N", "\\N", "\\N", "\\N"], + ] assert system_settings_profile_elements(role_name="worker") == [ - ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"]] + ["\\N", "\\N", "worker", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ] instance.query("REVOKE worker FROM robin") - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "10000000000\n" + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "10000000000\n" + ) instance.query("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER ROLE worker SETTINGS NONE") instance.query("GRANT worker TO robin") assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "10000000000\n" + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "10000000000\n" + ) instance.query("SET max_memory_usage = 120000000", user="robin") assert system_settings_profile_elements(role_name="worker") == [] # Set settings and constraints via CREATE SETTINGS PROFILE ... TO granted role instance.query("ALTER SETTINGS PROFILE xyz TO worker") - assert instance.query( - "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000 TO worker\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( - "SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [["xyz", "local directory", 2, 0, "['worker']", "[]"]] + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE xyz") + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000 TO worker\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000001\n" + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) + assert system_settings_profile("xyz") == [ + ["xyz", "local directory", 2, 0, "['worker']", "[]"] + ] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") - assert instance.query( - "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "10000000000\n" + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE xyz") + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "10000000000\n" + ) instance.query("SET max_memory_usage = 120000000", user="robin") - assert system_settings_profile("xyz") == [["xyz", "local directory", 2, 0, "[]", "[]"]] + assert system_settings_profile("xyz") == [ + ["xyz", "local directory", 2, 0, "[]", "[]"] + ] def test_inheritance(): - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY") + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY" + ) instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") - assert instance.query( - "SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" - assert instance.query( - "SHOW CREATE SETTINGS PROFILE alpha") == "CREATE SETTINGS PROFILE alpha SETTINGS INHERIT xyz TO robin\n" - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "100000002\n" - assert "Setting max_memory_usage should not be changed" in instance.query_and_get_error( - "SET max_memory_usage = 80000000", user="robin") + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE xyz") + == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" + ) + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE alpha") + == "CREATE SETTINGS PROFILE alpha SETTINGS INHERIT xyz TO robin\n" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000002\n" + ) + assert ( + "Setting max_memory_usage should not be changed" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) - assert system_settings_profile("xyz") == [["xyz", "local directory", 1, 0, "[]", "[]"]] + assert system_settings_profile("xyz") == [ + ["xyz", "local directory", 1, 0, "[]", "[]"] + ] assert system_settings_profile_elements(profile_name="xyz") == [ - ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000002, "\\N", "\\N", 1, "\\N"]] - assert system_settings_profile("alpha") == [["alpha", "local directory", 1, 0, "['robin']", "[]"]] + ["xyz", "\\N", "\\N", 0, "max_memory_usage", 100000002, "\\N", "\\N", 1, "\\N"] + ] + assert system_settings_profile("alpha") == [ + ["alpha", "local directory", 1, 0, "['robin']", "[]"] + ] assert system_settings_profile_elements(profile_name="alpha") == [ - ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"]] + ["alpha", "\\N", "\\N", 0, "\\N", "\\N", "\\N", "\\N", "\\N", "xyz"] + ] assert system_settings_profile_elements(user_name="robin") == [] def test_alter_and_drop(): instance.query( - "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 TO robin") - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "100000003\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error( - "SET max_memory_usage = 80000000", user="robin") - assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error( - "SET max_memory_usage = 120000000", user="robin") + "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 TO robin" + ) + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "100000003\n" + ) + assert ( + "Setting max_memory_usage shouldn't be less than 90000000" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) + assert ( + "Setting max_memory_usage shouldn't be greater than 110000000" + in instance.query_and_get_error( + "SET max_memory_usage = 120000000", user="robin" + ) + ) instance.query("ALTER SETTINGS PROFILE xyz SETTINGS readonly=1") - assert "Cannot modify 'max_memory_usage' setting in readonly mode" in instance.query_and_get_error( - "SET max_memory_usage = 80000000", user="robin") + assert ( + "Cannot modify 'max_memory_usage' setting in readonly mode" + in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + ) instance.query("DROP SETTINGS PROFILE xyz") - assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", - user="robin") == "10000000000\n" + assert ( + instance.query( + "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", + user="robin", + ) + == "10000000000\n" + ) instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") @@ -200,28 +372,49 @@ def test_show_profiles(): assert instance.query("SHOW PROFILES") == "default\nreadonly\nxyz\n" assert instance.query("SHOW CREATE PROFILE xyz") == "CREATE SETTINGS PROFILE xyz\n" - assert instance.query( - "SHOW CREATE SETTINGS PROFILE default") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, load_balancing = \\'random\\'\n" - assert instance.query( - "SHOW CREATE PROFILES") == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, load_balancing = \\'random\\'\n" \ - "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n" \ - "CREATE SETTINGS PROFILE xyz\n" + assert ( + instance.query("SHOW CREATE SETTINGS PROFILE default") + == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, load_balancing = \\'random\\'\n" + ) + assert ( + instance.query("SHOW CREATE PROFILES") + == "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, load_balancing = \\'random\\'\n" + "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n" + "CREATE SETTINGS PROFILE xyz\n" + ) - expected_access = "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, load_balancing = \\'random\\'\n" \ - "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n" \ - "CREATE SETTINGS PROFILE xyz\n" + expected_access = ( + "CREATE SETTINGS PROFILE default SETTINGS max_memory_usage = 10000000000, load_balancing = \\'random\\'\n" + "CREATE SETTINGS PROFILE readonly SETTINGS readonly = 1\n" + "CREATE SETTINGS PROFILE xyz\n" + ) assert expected_access in instance.query("SHOW ACCESS") def test_set_profile(): - instance.query("CREATE SETTINGS PROFILE P1 SETTINGS max_memory_usage=10000000001 MAX 20000000002") + instance.query( + "CREATE SETTINGS PROFILE P1 SETTINGS max_memory_usage=10000000001 MAX 20000000002" + ) session_id = new_session_id() - instance.http_query("SET profile='P1'", user='robin', params={'session_id':session_id}) - assert instance.http_query("SELECT getSetting('max_memory_usage')", user='robin', params={'session_id':session_id}) == "10000000001\n" + instance.http_query( + "SET profile='P1'", user="robin", params={"session_id": session_id} + ) + assert ( + instance.http_query( + "SELECT getSetting('max_memory_usage')", + user="robin", + params={"session_id": session_id}, + ) + == "10000000001\n" + ) expected_error = "max_memory_usage shouldn't be greater than 20000000002" - assert expected_error in instance.http_query_and_get_error("SET max_memory_usage=20000000003", user='robin', params={'session_id':session_id}) + assert expected_error in instance.http_query_and_get_error( + "SET max_memory_usage=20000000003", + user="robin", + params={"session_id": session_id}, + ) def test_changing_default_profiles_affects_new_sessions_only(): @@ -230,12 +423,33 @@ def test_changing_default_profiles_affects_new_sessions_only(): instance.query("ALTER USER robin SETTINGS PROFILE P1") session_id = new_session_id() - assert instance.http_query("SELECT getSetting('max_memory_usage')", user='robin', params={'session_id':session_id}) == "10000000001\n" + assert ( + instance.http_query( + "SELECT getSetting('max_memory_usage')", + user="robin", + params={"session_id": session_id}, + ) + == "10000000001\n" + ) instance.query("ALTER USER robin SETTINGS PROFILE P2") - assert instance.http_query("SELECT getSetting('max_memory_usage')", user='robin', params={'session_id':session_id}) == "10000000001\n" + assert ( + instance.http_query( + "SELECT getSetting('max_memory_usage')", + user="robin", + params={"session_id": session_id}, + ) + == "10000000001\n" + ) other_session_id = new_session_id() - assert instance.http_query("SELECT getSetting('max_memory_usage')", user='robin', params={'session_id':other_session_id}) == "10000000002\n" + assert ( + instance.http_query( + "SELECT getSetting('max_memory_usage')", + user="robin", + params={"session_id": other_session_id}, + ) + == "10000000002\n" + ) def test_function_current_profiles(): @@ -249,22 +463,60 @@ def test_function_current_profiles(): instance.query("CREATE SETTINGS PROFILE P6") session_id = new_session_id() - assert instance.http_query('SELECT defaultProfiles(), currentProfiles(), enabledProfiles()', user='robin', params={'session_id':session_id}) == "['P1','P2']\t['P1','P2']\t['default','P3','P4','P5','P1','P2']\n" + assert ( + instance.http_query( + "SELECT defaultProfiles(), currentProfiles(), enabledProfiles()", + user="robin", + params={"session_id": session_id}, + ) + == "['P1','P2']\t['P1','P2']\t['default','P3','P4','P5','P1','P2']\n" + ) - instance.http_query("SET profile='P6'", user='robin', params={'session_id':session_id}) - assert instance.http_query('SELECT defaultProfiles(), currentProfiles(), enabledProfiles()', user='robin', params={'session_id':session_id}) == "['P1','P2']\t['P6']\t['default','P3','P4','P5','P1','P2','P6']\n" + instance.http_query( + "SET profile='P6'", user="robin", params={"session_id": session_id} + ) + assert ( + instance.http_query( + "SELECT defaultProfiles(), currentProfiles(), enabledProfiles()", + user="robin", + params={"session_id": session_id}, + ) + == "['P1','P2']\t['P6']\t['default','P3','P4','P5','P1','P2','P6']\n" + ) - instance.http_query("SET profile='P5'", user='robin', params={'session_id':session_id}) - assert instance.http_query('SELECT defaultProfiles(), currentProfiles(), enabledProfiles()', user='robin', params={'session_id':session_id}) == "['P1','P2']\t['P5']\t['default','P3','P1','P2','P6','P4','P5']\n" + instance.http_query( + "SET profile='P5'", user="robin", params={"session_id": session_id} + ) + assert ( + instance.http_query( + "SELECT defaultProfiles(), currentProfiles(), enabledProfiles()", + user="robin", + params={"session_id": session_id}, + ) + == "['P1','P2']\t['P5']\t['default','P3','P1','P2','P6','P4','P5']\n" + ) instance.query("ALTER USER robin SETTINGS PROFILE P2") - assert instance.http_query('SELECT defaultProfiles(), currentProfiles(), enabledProfiles()', user='robin', params={'session_id':session_id}) == "['P2']\t['P5']\t['default','P3','P1','P2','P6','P4','P5']\n" + assert ( + instance.http_query( + "SELECT defaultProfiles(), currentProfiles(), enabledProfiles()", + user="robin", + params={"session_id": session_id}, + ) + == "['P2']\t['P5']\t['default','P3','P1','P2','P6','P4','P5']\n" + ) def test_allow_ddl(): - assert "it's necessary to have grant" in instance.query_and_get_error("CREATE TABLE tbl(a Int32) ENGINE=Log", user="robin") - assert "it's necessary to have grant" in instance.query_and_get_error("GRANT CREATE ON tbl TO robin", user="robin") - assert "DDL queries are prohibited" in instance.query_and_get_error("CREATE TABLE tbl(a Int32) ENGINE=Log", settings={"allow_ddl": 0}) + assert "it's necessary to have grant" in instance.query_and_get_error( + "CREATE TABLE tbl(a Int32) ENGINE=Log", user="robin" + ) + assert "it's necessary to have grant" in instance.query_and_get_error( + "GRANT CREATE ON tbl TO robin", user="robin" + ) + assert "DDL queries are prohibited" in instance.query_and_get_error( + "CREATE TABLE tbl(a Int32) ENGINE=Log", settings={"allow_ddl": 0} + ) instance.query("GRANT CREATE ON tbl TO robin") instance.query("CREATE TABLE tbl(a Int32) ENGINE=Log", user="robin") @@ -272,27 +524,60 @@ def test_allow_ddl(): def test_allow_introspection(): - assert instance.query("SELECT demangle('a')", settings={"allow_introspection_functions": 1}) == "signed char\n" + assert ( + instance.query( + "SELECT demangle('a')", settings={"allow_introspection_functions": 1} + ) + == "signed char\n" + ) - assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')") - assert "it's necessary to have grant" in instance.query_and_get_error("SELECT demangle('a')", user="robin") - assert "it's necessary to have grant" in instance.query_and_get_error("SELECT demangle('a')", user="robin", settings={"allow_introspection_functions": 1}) + assert "Introspection functions are disabled" in instance.query_and_get_error( + "SELECT demangle('a')" + ) + assert "it's necessary to have grant" in instance.query_and_get_error( + "SELECT demangle('a')", user="robin" + ) + assert "it's necessary to have grant" in instance.query_and_get_error( + "SELECT demangle('a')", + user="robin", + settings={"allow_introspection_functions": 1}, + ) instance.query("GRANT demangle ON *.* TO robin") - assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") - assert instance.query("SELECT demangle('a')", user="robin", settings={"allow_introspection_functions": 1}) == "signed char\n" + assert "Introspection functions are disabled" in instance.query_and_get_error( + "SELECT demangle('a')", user="robin" + ) + assert ( + instance.query( + "SELECT demangle('a')", + user="robin", + settings={"allow_introspection_functions": 1}, + ) + == "signed char\n" + ) instance.query("ALTER USER robin SETTINGS allow_introspection_functions=1") assert instance.query("SELECT demangle('a')", user="robin") == "signed char\n" instance.query("ALTER USER robin SETTINGS NONE") - assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") + assert "Introspection functions are disabled" in instance.query_and_get_error( + "SELECT demangle('a')", user="robin" + ) - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS allow_introspection_functions=1 TO robin") + instance.query( + "CREATE SETTINGS PROFILE xyz SETTINGS allow_introspection_functions=1 TO robin" + ) assert instance.query("SELECT demangle('a')", user="robin") == "signed char\n" instance.query("DROP SETTINGS PROFILE xyz") - assert "Introspection functions are disabled" in instance.query_and_get_error("SELECT demangle('a')", user="robin") + assert "Introspection functions are disabled" in instance.query_and_get_error( + "SELECT demangle('a')", user="robin" + ) - instance.query("REVOKE demangle ON *.* FROM robin", settings={"allow_introspection_functions": 1}) - assert "it's necessary to have grant" in instance.query_and_get_error("SELECT demangle('a')", user="robin") + instance.query( + "REVOKE demangle ON *.* FROM robin", + settings={"allow_introspection_functions": 1}, + ) + assert "it's necessary to have grant" in instance.query_and_get_error( + "SELECT demangle('a')", user="robin" + ) diff --git a/tests/integration/test_sharding_key_from_default_column/test.py b/tests/integration/test_sharding_key_from_default_column/test.py index 1717a1ee14a..1ecf96305a4 100644 --- a/tests/integration/test_sharding_key_from_default_column/test.py +++ b/tests/integration/test_sharding_key_from_default_column/test.py @@ -5,8 +5,12 @@ from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/test_cluster.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/test_cluster.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/test_cluster.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/test_cluster.xml"], with_zookeeper=True +) @pytest.fixture(scope="module", autouse=True) @@ -29,88 +33,162 @@ def cleanup_after_test(): # A default column is used in the sharding key expression. def test_default_column(): - node1.query("CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 DEFAULT x + 100, z Int32 DEFAULT x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)") - node1.query("CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 DEFAULT x + 200, z Int32 DEFAULT x - y) ENGINE = MergeTree() ORDER BY y") + node1.query( + "CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 DEFAULT x + 100, z Int32 DEFAULT x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)" + ) + node1.query( + "CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 DEFAULT x + 200, z Int32 DEFAULT x - y) ENGINE = MergeTree() ORDER BY y" + ) for insert_sync in [0, 1]: - settings = {'insert_distributed_sync': insert_sync} - + settings = {"insert_distributed_sync": insert_sync} + # INSERT INTO TABLE dist (x) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") - node1.query("INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings) + node1.query( + "INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings + ) node1.query("SYSTEM FLUSH DISTRIBUTED dist") - assert node1.query("SELECT x, y, z FROM local") == TSV([[2, 102, 104], [4, 104, 108]]) - assert node2.query("SELECT x, y, z FROM local") == TSV([[1, 101, 102], [3, 103, 106]]) - assert node1.query("SELECT x, y, z FROM dist") == TSV([[2, 102, 104], [4, 104, 108], [1, 101, 102], [3, 103, 106]]) + assert node1.query("SELECT x, y, z FROM local") == TSV( + [[2, 102, 104], [4, 104, 108]] + ) + assert node2.query("SELECT x, y, z FROM local") == TSV( + [[1, 101, 102], [3, 103, 106]] + ) + assert node1.query("SELECT x, y, z FROM dist") == TSV( + [[2, 102, 104], [4, 104, 108], [1, 101, 102], [3, 103, 106]] + ) # INSERT INTO TABLE dist (x, y) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") - node1.query("INSERT INTO TABLE dist (x, y) VALUES (1, 11), (2, 22), (3, 33)", settings=settings) + node1.query( + "INSERT INTO TABLE dist (x, y) VALUES (1, 11), (2, 22), (3, 33)", + settings=settings, + ) node1.query("SYSTEM FLUSH DISTRIBUTED dist") assert node1.query("SELECT x, y, z FROM local") == TSV([[2, 22, 24]]) - assert node2.query("SELECT x, y, z FROM local") == TSV([[1, 11, 12], [3, 33, 36]]) - assert node1.query("SELECT x, y, z FROM dist") == TSV([[2, 22, 24], [1, 11, 12], [3, 33, 36]]) + assert node2.query("SELECT x, y, z FROM local") == TSV( + [[1, 11, 12], [3, 33, 36]] + ) + assert node1.query("SELECT x, y, z FROM dist") == TSV( + [[2, 22, 24], [1, 11, 12], [3, 33, 36]] + ) # A materialized column is used in the sharding key expression and `insert_allow_materialized_columns` set to 1. def test_materialized_column_allow_insert_materialized(): - node1.query("CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 100, z Int32 MATERIALIZED x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)") - node1.query("CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 200, z Int32 MATERIALIZED x - y) ENGINE = MergeTree() ORDER BY y") + node1.query( + "CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 100, z Int32 MATERIALIZED x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)" + ) + node1.query( + "CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 200, z Int32 MATERIALIZED x - y) ENGINE = MergeTree() ORDER BY y" + ) for insert_sync in [0, 1]: - settings = {'insert_distributed_sync': insert_sync, 'insert_allow_materialized_columns': 1} - + settings = { + "insert_distributed_sync": insert_sync, + "insert_allow_materialized_columns": 1, + } + # INSERT INTO TABLE dist (x) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") - node1.query("INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings) + node1.query( + "INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings + ) node1.query("SYSTEM FLUSH DISTRIBUTED dist") - assert node1.query("SELECT x, y, z FROM local") == TSV([[2, 102, 104], [4, 104, 108]]) - assert node2.query("SELECT x, y, z FROM local") == TSV([[1, 101, 102], [3, 103, 106]]) - assert node1.query("SELECT x, y, z FROM dist") == TSV([[2, 102, 104], [4, 104, 108], [1, 101, 102], [3, 103, 106]]) + assert node1.query("SELECT x, y, z FROM local") == TSV( + [[2, 102, 104], [4, 104, 108]] + ) + assert node2.query("SELECT x, y, z FROM local") == TSV( + [[1, 101, 102], [3, 103, 106]] + ) + assert node1.query("SELECT x, y, z FROM dist") == TSV( + [[2, 102, 104], [4, 104, 108], [1, 101, 102], [3, 103, 106]] + ) # INSERT INTO TABLE dist (x, y) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") - node1.query("INSERT INTO TABLE dist (x, y) VALUES (1, 11), (2, 22), (3, 33)", settings=settings) + node1.query( + "INSERT INTO TABLE dist (x, y) VALUES (1, 11), (2, 22), (3, 33)", + settings=settings, + ) node1.query("SYSTEM FLUSH DISTRIBUTED dist") assert node1.query("SELECT x, y, z FROM local") == TSV([[2, 22, 24]]) - assert node2.query("SELECT x, y, z FROM local") == TSV([[1, 11, 12], [3, 33, 36]]) - assert node1.query("SELECT x, y, z FROM dist") == TSV([[2, 22, 24], [1, 11, 12], [3, 33, 36]]) + assert node2.query("SELECT x, y, z FROM local") == TSV( + [[1, 11, 12], [3, 33, 36]] + ) + assert node1.query("SELECT x, y, z FROM dist") == TSV( + [[2, 22, 24], [1, 11, 12], [3, 33, 36]] + ) # A materialized column is used in the sharding key expression and `insert_allow_materialized_columns` set to 0. def test_materialized_column_disallow_insert_materialized(): - node1.query("CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 100, z Int32 MATERIALIZED x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)") - node1.query("CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 200, z Int32 MATERIALIZED x - y) ENGINE = MergeTree() ORDER BY y") + node1.query( + "CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 100, z Int32 MATERIALIZED x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)" + ) + node1.query( + "CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 200, z Int32 MATERIALIZED x - y) ENGINE = MergeTree() ORDER BY y" + ) for insert_sync in [0, 1]: - settings = {'insert_distributed_sync': insert_sync, 'insert_allow_materialized_columns': 0} - + settings = { + "insert_distributed_sync": insert_sync, + "insert_allow_materialized_columns": 0, + } + # INSERT INTO TABLE dist (x) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") - node1.query("INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings) + node1.query( + "INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings + ) node1.query("SYSTEM FLUSH DISTRIBUTED dist") - assert node1.query("SELECT x, y, z FROM local") == TSV([[2, 202, -200], [4, 204, -200]]) - assert node2.query("SELECT x, y, z FROM local") == TSV([[1, 201, -200], [3, 203, -200]]) - assert node1.query("SELECT x, y, z FROM dist") == TSV([[2, 202, -200], [4, 204, -200], [1, 201, -200], [3, 203, -200]]) + assert node1.query("SELECT x, y, z FROM local") == TSV( + [[2, 202, -200], [4, 204, -200]] + ) + assert node2.query("SELECT x, y, z FROM local") == TSV( + [[1, 201, -200], [3, 203, -200]] + ) + assert node1.query("SELECT x, y, z FROM dist") == TSV( + [[2, 202, -200], [4, 204, -200], [1, 201, -200], [3, 203, -200]] + ) # INSERT INTO TABLE dist (x, y) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") expected_error = "Cannot insert column y, because it is MATERIALIZED column" - assert expected_error in node1.query_and_get_error("INSERT INTO TABLE dist (x, y) VALUES (1, 11), (2, 22), (3, 33)", settings=settings) + assert expected_error in node1.query_and_get_error( + "INSERT INTO TABLE dist (x, y) VALUES (1, 11), (2, 22), (3, 33)", + settings=settings, + ) # Almost the same as the previous test `test_materialized_column_disallow_insert_materialized`, but the sharding key has different values. def test_materialized_column_disallow_insert_materialized_different_shards(): - node1.query("CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 101, z Int32 MATERIALIZED x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)") - node1.query("CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 200, z Int32 MATERIALIZED x - y) ENGINE = MergeTree() ORDER BY y") + node1.query( + "CREATE TABLE dist ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 101, z Int32 MATERIALIZED x + y) ENGINE = Distributed('test_cluster', currentDatabase(), local, y)" + ) + node1.query( + "CREATE TABLE local ON CLUSTER 'test_cluster' (x Int32, y Int32 MATERIALIZED x + 200, z Int32 MATERIALIZED x - y) ENGINE = MergeTree() ORDER BY y" + ) for insert_sync in [0, 1]: - settings = {'insert_distributed_sync': insert_sync, 'insert_allow_materialized_columns': 0} - + settings = { + "insert_distributed_sync": insert_sync, + "insert_allow_materialized_columns": 0, + } + # INSERT INTO TABLE dist (x) node1.query("TRUNCATE TABLE local ON CLUSTER 'test_cluster'") - node1.query("INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings) + node1.query( + "INSERT INTO TABLE dist (x) VALUES (1), (2), (3), (4)", settings=settings + ) node1.query("SYSTEM FLUSH DISTRIBUTED dist") - assert node1.query("SELECT x, y, z FROM local") == TSV([[1, 201, -200], [3, 203, -200]]) - assert node2.query("SELECT x, y, z FROM local") == TSV([[2, 202, -200], [4, 204, -200]]) - assert node1.query("SELECT x, y, z FROM dist") == TSV([[1, 201, -200], [3, 203, -200], [2, 202, -200], [4, 204, -200]]) + assert node1.query("SELECT x, y, z FROM local") == TSV( + [[1, 201, -200], [3, 203, -200]] + ) + assert node2.query("SELECT x, y, z FROM local") == TSV( + [[2, 202, -200], [4, 204, -200]] + ) + assert node1.query("SELECT x, y, z FROM dist") == TSV( + [[1, 201, -200], [3, 203, -200], [2, 202, -200], [4, 204, -200]] + ) diff --git a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py index d5c74a99622..c940998ec42 100644 --- a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py +++ b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py @@ -2,9 +2,15 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) -ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) -ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) +ch1 = cluster.add_instance( + "ch1", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True +) +ch2 = cluster.add_instance( + "ch2", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True +) +ch3 = cluster.add_instance( + "ch3", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True +) @pytest.fixture(scope="module", autouse=True) @@ -18,17 +24,31 @@ def started_cluster(): def test_sql_user_defined_functions_on_cluster(): - assert "Unknown function test_function" in ch1.query_and_get_error("SELECT test_function(1);") - assert "Unknown function test_function" in ch2.query_and_get_error("SELECT test_function(1);") - assert "Unknown function test_function" in ch3.query_and_get_error("SELECT test_function(1);") + assert "Unknown function test_function" in ch1.query_and_get_error( + "SELECT test_function(1);" + ) + assert "Unknown function test_function" in ch2.query_and_get_error( + "SELECT test_function(1);" + ) + assert "Unknown function test_function" in ch3.query_and_get_error( + "SELECT test_function(1);" + ) - ch1.query_with_retry("CREATE FUNCTION test_function ON CLUSTER 'cluster' AS x -> x + 1;") + ch1.query_with_retry( + "CREATE FUNCTION test_function ON CLUSTER 'cluster' AS x -> x + 1;" + ) assert ch1.query("SELECT test_function(1);") == "2\n" assert ch2.query("SELECT test_function(1);") == "2\n" assert ch3.query("SELECT test_function(1);") == "2\n" ch2.query_with_retry("DROP FUNCTION test_function ON CLUSTER 'cluster'") - assert "Unknown function test_function" in ch1.query_and_get_error("SELECT test_function(1);") - assert "Unknown function test_function" in ch2.query_and_get_error("SELECT test_function(1);") - assert "Unknown function test_function" in ch3.query_and_get_error("SELECT test_function(1);") + assert "Unknown function test_function" in ch1.query_and_get_error( + "SELECT test_function(1);" + ) + assert "Unknown function test_function" in ch2.query_and_get_error( + "SELECT test_function(1);" + ) + assert "Unknown function test_function" in ch3.query_and_get_error( + "SELECT test_function(1);" + ) diff --git a/tests/integration/test_ssl_cert_authentication/__init__.py b/tests/integration/test_ssl_cert_authentication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem new file mode 100644 index 00000000000..293e1c7f564 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhTCCA22gAwIBAgIUVRNcr0jCH3vSTxg8QYQH6CCtyF4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjIwMjE4 +MDk0MzA2WhcNMzIwMjE2MDk0MzA2WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALwojNvu +fXQYQ4tucqNOEDHf2sNgxwxqY6QdtJ+zNfVjsK4I3Vqo8TtzxfDYGolkYem/bYJM +xQar9ehUm9ok/0kJgIo8vDXxxDJtvjz5Fd5oFWJLMxojLE9NSa0A4m18jGfbFNsF +XoU0njiInyzNaU9d4bMpaweseCZdt9Y4LR93FkuhSU/v18lWQPob8SSIij059IZP +sEUxpDOTxclAmG/Knd/6v3ecVFiQgexZM0gCtf7kcw41mxsAaP/mOexodIZDR70Y +LYjL7R2ZGhpClfQc8SO5NSpfEqsfreDX7XoaCTsy7/rqr3Nfiby6sc//awG0Ww/f +FRf2+2BU2xEwOVa3i5wU5raYY6eqFLK9q9c2IWPSqYzAmvhK2pqWQ/iaCU/Q89ow +SbKudJTLK8Y6v9LW4Q8ZLZF+CzS5cI+QEfIYqTLFdInH1BLoxx7cymEv07CDkcTo +2WtV8GdMph2P3U/9NoXQDonjCSj0lQUjgUdcrBPaIIVbIn6/5vfw8LQa8PoGDhIx +AYQkqPR+LHxCqIMzdqKZ+OXD/HPhiigpxLhF7mVRLvvoyrOZVJbcu1qmgCcQw0IE +fWzvWne+9cYC9lgt8+/k6d6B1uhYsIwwhgoj0dffFjc0sF6zfceGK+H1K2JCE0aY +zT1HlvSoZdA7lEs5xbGJnkBHqlOvQ63ynXCzAgMBAAGjUzBRMB0GA1UdDgQWBBTn +AtgFU20JF7kTZCKlY7/hi0kYRzAfBgNVHSMEGDAWgBTnAtgFU20JF7kTZCKlY7/h +i0kYRzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCpiWgJ1XUw +a8Bdeznsa57oy+5mqQZWpRVkzTQRHEGV850OGh7WQ6u9kVAHefaHH9hsVxyggton +6/MDsu4KL5jqKmJaIAepPIOw6DTc2zs044I7W/rxRp+w1hL2TS+EahMrSPwdzCcl +NNAM0dXocGylf6qwwMqiYAR1K3UIrlyq4QTr1oEPIqJBkDg1JDYrt4T2DroPjW20 +5hlCQ/tft5ddGL0EFEaKWwAcPFm7jAwJiz2eUqmT6PcmaZ24qPn5RXVkaBAkrSga +1WgM8r3LGu2EKhdiDc5hRJKjS8RZyLvZNNzlL3+N42nGmGZkND5bV6u82OD+qn17 +LRZOt0Cr70HqszSYk/67ijjaa4n/fuuAqorV+yYB8accRXtoi00nxykT+H+yI1rD +swvcrfDvhUgY5zmunWyQUYh0q/2Hj75GbLup3Cd0B4MrBwqyCqcEugM4OSf6aRMr +e/vjeggTVPN08xE1LUkugalx0B0aoO6qFahJ2CmkAcYLLlS2N+F7TMuPavc0kVxD +I3qA5G9zvNCliSLX2+kM+LzslI8+pP/A98bvh6nW4HtZkI0jq1ks7XR0GeOhCI8E +0l/YuElxxgKhN4INKhhMoDKqPib4z8gbmkenR2CenQCpfLMIrhTXZgtw+gvEgpIE +/QK97G8XPqga6zn471wrYJnuyJli+sP7aw== +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl new file mode 100644 index 00000000000..c02cd0a4526 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl @@ -0,0 +1 @@ +05F10C67567FE30795D77AF2540F6AC8D4CF2461 diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem new file mode 100644 index 00000000000..e85dca8553e --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC8KIzb7n10GEOL +bnKjThAx39rDYMcMamOkHbSfszX1Y7CuCN1aqPE7c8Xw2BqJZGHpv22CTMUGq/Xo +VJvaJP9JCYCKPLw18cQybb48+RXeaBViSzMaIyxPTUmtAOJtfIxn2xTbBV6FNJ44 +iJ8szWlPXeGzKWsHrHgmXbfWOC0fdxZLoUlP79fJVkD6G/EkiIo9OfSGT7BFMaQz +k8XJQJhvyp3f+r93nFRYkIHsWTNIArX+5HMONZsbAGj/5jnsaHSGQ0e9GC2Iy+0d +mRoaQpX0HPEjuTUqXxKrH63g1+16Ggk7Mu/66q9zX4m8urHP/2sBtFsP3xUX9vtg +VNsRMDlWt4ucFOa2mGOnqhSyvavXNiFj0qmMwJr4StqalkP4mglP0PPaMEmyrnSU +yyvGOr/S1uEPGS2Rfgs0uXCPkBHyGKkyxXSJx9QS6Mce3MphL9Owg5HE6NlrVfBn +TKYdj91P/TaF0A6J4wko9JUFI4FHXKwT2iCFWyJ+v+b38PC0GvD6Bg4SMQGEJKj0 +fix8QqiDM3aimfjlw/xz4YooKcS4Re5lUS776MqzmVSW3LtapoAnEMNCBH1s71p3 +vvXGAvZYLfPv5OnegdboWLCMMIYKI9HX3xY3NLBes33Hhivh9StiQhNGmM09R5b0 +qGXQO5RLOcWxiZ5AR6pTr0Ot8p1wswIDAQABAoICAQCO/c4Wccb7TFlAhD4wpumd +zX5GDq0WXV+94CldWGdARnOFvwzhkhRJ1zDtWH3KPfQ/HJBPfqIY8OQfnPUYMhej +3MnHxGJQKJyuqkHxumYJMFZX7cg3K9XHqne8NzjcddOKNa9Cx3DOkG9RjVpSRQSs +IS+d5XMGUOa6WWyVKvn3uJvD/B1n12DJDHiy2jtHRVCxOPMAg1z1KMWdwMaFrEZs +ZrHV/ow1jSN4btGd2SgkqJLA08IwYUKvoX8qQj9wzu0G/+hr5wzrsfZQEQMKQ+IL +s1b6jAzAV6IrVBbjEZXSviiXyZ0gteuCJW/acpMg+/3JPNQbWrCAFt1wluwowto/ +JAFIvlh29hfE5c+HEMpQNa0tdj7jepBn/0YEbgwpayMikKiLZXEpgheWCGypAQWp +Hm+N0Ym7HSGe82obxi8EjKRnNwFUtotWzUBKeo9aFwPZHLFlspljd+5ynDvKqXnk +txYZj6K3TtMs30HAG6fqxSPyiZ5W+5yF7nt6qLODs6m4Os+lrk1GnoqC0/uLMzIU +CRJKulrJOK4/Z2tPn9IAhcREbS4oROUeNqqo0Cfs3ssvkV7JTHF4IsKhCmElMmGa +bevOI+pvdjfECShy0Jnbtni6ece/II4/edfUp9kWN45xZLpzDjfqCVD66JS9g6ZU +i/EVll+d5zaI2TzzwZgHUQKCAQEA3d8siwXbq7x0cAB013+tvkvGMJ2EuS1TWdLk +a2P6CAnlZMWvv2cPSd2WpimHjqKxrbn6VE79mOc2l9Y1NOUUWWZATrhN7V8xMapQ +0YiYCHeaMERUAUKdzCgRN2/mRbZCBzpPBbWbb6NtKfRFJsD9zAe2JBwDVh9hvAL8 +YVBoczrEfj1ILnmtPhAJVI6s6rDsA4MgKjLs0Tt7Cc7rQxqNSpHEvwv1yLQmjp0N +L5b1TEt7fqVJ9dirykJquBYEKf55Z1qZhQzmnbu9OPnzeqGDakl5F/UsXDB5Bokp +ilcV+nFbh175Q+gTEhaSacGW8gzRw6j18PuciBjeWVEM5hhxOwKCAQEA2RnRMjv9 +46jQarJTFbIHg1SqrR87GSLnt6672M5TX9frzxMCuVDjKgdecstvLjm6X+/cPQKT +Q3javJnJXT4cx//1J7RLO6ZBVSCZf3//XntdHdFVJf5ySQtK+MJyfxjpzP6KBPfb +WPrva8p29ejbBdtsOT0M6gY5tPfadU2XEaf+BoyX9NUmu1U46Iqi+eCOjR+GVvhP +pJzGgLeOsaRVCfc9I7XPoVu3AEx5Kt55yRYm4fyGPsAd+mRDbIXMXdL0k8CfWWDr +8TT5rqKI+gFPFQCwToBW3DwHIGY+3RmoXFfQ0IJaKwOk4AB7m6HC3mv1crtjTFSM +9p74oQzNX7UG6QKCAQBEs2cygRTdH5SaXbnQRKvC4emzggLn5/4IMUIjcqioNpA+ +XOwngzz7rU6JkxBzfTMxTQYTdwYVg3qnF2AQSeK8L+o3teADYVd1PnyZ9QbGkGpB +CddNMJh17+4s0UxnR6E4Zbi0VuCTd/JEbGvBLT8pHzYqBjaOQ1dbBT2q0GAXVhoj +0Mv6ABlBv2t0MF2gqjnaeI7MIkqsGxPlHJpChAU+EtbuJUDs7cOGo2DC3KaGAlVy +CLJXGslO7rPm3oJZkn97HlWtGiqKquhTrSnUThDIJ4oEfhlHTocbG/ut53tZuiIS +T7k1arYFAtJBRv17Y7bMNBQ7k12L0s9+rpck5GqjAoIBAQCVBPSkj6tZbpII+viu +5rHjguVYyhwtx9jYK1eDnTR7kGGrlPgErjIPslkxYNSjHTsCCUnakv70jGtQlBs1 +JqJo4hesNkSB4D/uJ99VNk3a08D566uP1dUqsFa44/flp/ssG/gvKtbkf/KBwcrg +RwK4RYJG09IefUF1J8BLToQIuZBTfIP9qaXZZskWTbtK28ndsqrq3a0FaBuVVOnc +o9k/avcLoQuxTZwS12tAcs+TqOHtswGO5x5stg/V2Q2LxXbeSJTYq/+oZN2R8r0l +JmrbFsruR4fXylh189jouWjoYdrSlPdBmVG99HbkQCfbtq0XIOsrBMpxqnMtUPVT +4ZWpAoIBAQCrao4XHpRM3KsfPqdkg0vqFDBA+LzKlWu1fl8w5TGpFK8H1tv5kHNv +h0XmeU5cXwiweri3KjQz7h+qVBHZeAvyrEoxJQdImax+5OUy7lysDs+SL02gLZb3 +Z7g+u4Buwx+cv4O7hwUEDDk/5X3NBFk7/iwztKUtM+Fl8jt9K3K24s87xXp9YevI +UEawden9jVcuXQjEUrwz8cjoz/y25vK5pQ6k82PVImkMZK99/PmgmGyOl7hdRA3F +ff0Kb8pRGmV/cWRKzHaC8QchW8jdU2EGxMkrFl1DvoVKLbyDf1glRncKP9iozHAR ++s184IJCUvyMxH83uKKAiBGaDRC+Lbm7 +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem new file mode 100644 index 00000000000..bd6eea62094 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRfMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMBU0fao +RrITeF4kpN81p7qirX/Gc56+Cux6u7RF1O6WU9v+V5jLw8chQZ87z4QSrFiT1ZnT +pwWYPwJ+pDk6AWEoiKuOaceOh0bjZCuxADHs+qQrye5D8GXvyFvWE2cT1pD5JNEZ +DSl2YHqNs4uTGRP9BP817iRDcuvdxpanaWxfXGfehJRMiEVgKDs+RUpoW4aVNivI +InrUWc4RXXkzaJKqhpCU3jAJBV4jSD5ZnA8PUfcoAj6z6T3I6phuDfRP5ldA3br8 +yg0hCB7Y5QrO5lRAgEoIuNnC+U6/AIwWPI36Rjiwg3EUwI/BIiL4AWjzkjSdr0mn +zyHPRk4pcn01T0GTpQi6tfZZpumDD3LkPuEy9svMpJ8ntqDnAsIJVjbg1S60hHes +yYHoQw1HxU0vrncxwcQkVaPLx0uGlioaLlvu83AVnWXbylZXsV/pLy6dE3H51GBF +DX3Zj6nkuJitk8/hNp440/Lve7SaKFPo5NdH+8ACWGdFdz3zxgPuhBDoxEeqj4c1 +FQA1ABXx2akW3lQ5VxTAg5AYORvVhJTozosr+Kn3MlRdZjl94tnVByD8MGLLE0C4 +L/qXR/IlbkOCz5LHapdC5j62ZEBwiElmMO/tMGl4ORV9tdTBrRZ9DMmKek2E8Qwz +y770PGkhp1cTzZt6UfZEympowmfjtiZfHIq1AgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAHwRpqnpcD3EW588GSDZhZrVf3nS9M06ljQGtDUqNSI4XJp1cVT1sMaa4LjM +cWgHtayFw+jbDLwioXHjMlV+8tERH+0x+qsADG349caDYT/OF13v/jyuboUZ9AqE +KpfOQH7jCLU7rEbEl6kvT3F3xaHJg8mE7msyRFfguB2JrqZkKIj4HANxJUJo4PwB +5bq9nE3AVNAgUeQEwfu0r5SjroNpcHfm7xWqMK2mDMCsy/DvI7n97Q7vZajcTT0x +UXfgx+3CLEvLMpa2myE5OIMOeLzfZwxrxyNH7BdZsROnkGv1cX+9HZpYcue/UDxp +P2OApbTuZKaTJOyMADc17s0seE0DTAHnHAWrJwVhf8wYKKtEs+i+Sw5LNSkh5fgS +hTzGF93yClDYzWEqMSKhKPeimtpz4ZBNuGf471KbpVbUKJJvJmOxqoZ5S0kpFILL +YMALf652uf5or5d0cDNvcJTwvMi6evchIV17d/jH+MxyJQs9VCkMpJxFbMrXb3YB +b57K3Z25P6w3Qfj4zuKQFANari7Gs6qSiaUBiEhEdTQlGspkq+FLndtX818sbMk5 +LAK6JaUH0ywV2jn5XSW0irQLDXqb6Q0bSyw6pdpDjk0o4UW67JCE4kGagRDnfSqL +ZODvO/dEtVLyAsjmOx8MkqLyseI7VESVd8eiJAyL0sifh+/E +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem new file mode 100644 index 00000000000..8bc1e656566 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDAVNH2qEayE3he +JKTfNae6oq1/xnOevgrseru0RdTullPb/leYy8PHIUGfO8+EEqxYk9WZ06cFmD8C +fqQ5OgFhKIirjmnHjodG42QrsQAx7PqkK8nuQ/Bl78hb1hNnE9aQ+STRGQ0pdmB6 +jbOLkxkT/QT/Ne4kQ3Lr3caWp2lsX1xn3oSUTIhFYCg7PkVKaFuGlTYryCJ61FnO +EV15M2iSqoaQlN4wCQVeI0g+WZwPD1H3KAI+s+k9yOqYbg30T+ZXQN26/MoNIQge +2OUKzuZUQIBKCLjZwvlOvwCMFjyN+kY4sINxFMCPwSIi+AFo85I0na9Jp88hz0ZO +KXJ9NU9Bk6UIurX2Wabpgw9y5D7hMvbLzKSfJ7ag5wLCCVY24NUutIR3rMmB6EMN +R8VNL653McHEJFWjy8dLhpYqGi5b7vNwFZ1l28pWV7Ff6S8unRNx+dRgRQ192Y+p +5LiYrZPP4TaeONPy73u0mihT6OTXR/vAAlhnRXc988YD7oQQ6MRHqo+HNRUANQAV +8dmpFt5UOVcUwIOQGDkb1YSU6M6LK/ip9zJUXWY5feLZ1Qcg/DBiyxNAuC/6l0fy +JW5Dgs+Sx2qXQuY+tmRAcIhJZjDv7TBpeDkVfbXUwa0WfQzJinpNhPEMM8u+9Dxp +IadXE82belH2RMpqaMJn47YmXxyKtQIDAQABAoICAAEBsKOg19XgwjWD7ZT5e+o/ +JbdQe5RuHDKGperYnres871oBF9ZWan2I5jIwFpJmrtP8sM+V1ZxKItDzGo8QnuW +sbhsI2OW/GBDmmecIosgWWN4kzL7CgwOiDbq1OkqMmpJ04aAohAAfZrGmRT27R+s +qFUJnDh2XeicHYj2UVfu29XzVTBNgj0StsMwnT45c5ktuL3b60pHSD0K3DlhKn/y +AohJLyyDL5MBjkQ9RdLSWrR3ciOP332iSpAHq20G6ga04TQ0VH5jGN7IddJrqMry +F3nLt+Pz4EgoOcGB8Ekx8SIk0ltKJ4PZF+uk7qT0+WPrG1rAVRYxNoX8M4wyNjr4 +TcAZsV2DnGdnp+2u0SSzMczeop5hPTJKxaLaPw1JOoIk5fqW94MbEHqGnEXEIN+D +OWeUKWZ/B1YubavOeR+c3STZrh2SgmhKk6g5NMFlfnyvolPu47H8NOrewOhVG+TZ +gsQoGxSyOXwZTQ/Jd6Yg9lek8nKJBc4Res7ia/x3H+gjjRoNFI+L2HQnWztx5YMZ +H9M6hcpclZubO/w4iLq9OB2QUHn7aIT3lWRV/xS0Yh2zGCufasaMA1KSKC5zq0Fk +gCzAkYDq/ymrJs3LQQ0wegKd1akL4z5fxmXTn2v2BGoEd52uuxhL0mM/9zzRxdR2 +IsOgAym+siLXMCHTDbdVAoIBAQDuMcea66WKidS+A9frCEsabYccKzrdMEhs6Mle +orFieMC+3ZpzFIBkXPZ522I+M4nIdBKuRw9PnYTE5t30euOj60Oq905j2a+Ho4ki +kW6dC+tNDF49Hqxn9e99xbvTUi97dREcERlHA+AnRektEciyD17bi88aUy9w83Mw +G5Z+ej+9o40w8+TDopE2SIJhUAHR6LOAMq1v5y1lmTn0sbTuxZFLA0qWX9aGLi+T +4RD0MzJAtKJDbr3yPTLHAXmaMSKHhWYYgWTH9iwEhGQAm5VJy3oNJUkM7ej7Yfs7 +aTDOk61egCKhEHdWavP68MqmNOPHgnq4/edmvQnhfKtI8SMnAoIBAQDOtWDi/OnU +ZjZPnmJwwoPuXe6IjYg47bFRGv94xEpSesCAYdXNaNLPl0f/Ut9y3nXr+j+XqJWo +UqtRGFu2i9lUK3cu90GLXEaLbYWGcgL8YnJu0senLxkqxPWcGxoKmbo3xMjqk/pF +EVZ5e1qqVTlrB4q7QWmLKrS8YlcaTnChPeSBRFfryg/xvQ11Hxtq89SKkTH4ps16 +0KtiCxvfQHVASyRLIKLdyabPInB+yP3Fsn4BIx8jGtOQ/OCY01TXq9OyaRu2hJTk +qsjOLnqf6huM2so3X0Tw8AdgNoF96JJvfhwiPI5CSo9UKjhuvus1Ip5ZFFNo4Ngy +n3Zlgp1HxZzDAoIBAQC9ffqmo3sxqI8Hj3UxdIqS/rlyzm1o0+V6RwMT92gYx6nG +7fLWRGQT8+TdcotIoqWlQ7oszTlABDdAkc3XlgANQre1hkLlqqM6y/3n8zzFUVsj +E4jRJNrRZdTeAPV4mzRNCgfPhUbPuSSU+cgT48b+6L10+VeMQMtIF1T226uw+L5G +tps3a3/9pxHQ1oRquESKYo6SmT5i/M2fuvNhWBJxtdjtjTPER4AZhRqykWV0cFo1 +Ib7I2Ivh74+6w9Ciux4WJCjhq+aqMYw5F72awitU5rw1QwlHcOldO0irrfZ3EQLm +YBesfLYDmNh6NR9ydDcVXBcXnl593DvFF/IH+FYXAoIBAQCQZydLCzHy3oC8eEH+ +0fRGljooDO+IDYzcwwaLgF0HZ5eJWE97EuqKeP2kAWn2HjC07Hp2YSBDmZTyrxiK +2wG1CjRVjAeu6oShrJ4mAQnS9JdKkldFlOJ4/WUza79yflgX05IkRcIFdAo8DY+W +BLl66qbhD95CiU//dpew2fFWwx0ZrPvazar7zn1TP6rwuWvWbX5CXYyYaqP/dxE+ +khIXGyc8kI0WcWPlugJqn9CgxoO+GaIL7Ra1Z+MjACd6DyBxt3nTtKUrZZ+oYdHq +Wypp6QJxUk2gH56XeRxXMBz0ZF4VEMa0ys98FY6c1yULVqbWRhvK3aBLJRkZ6vgj +BorvAoIBAASy89mnP7d9jY7pSg/8znsUF8fQwKpRJZKS+8xgbzsZP+zT7CjxCbPL +xcNK0fl6pRBv+gyIM013R7J1uvZJ3W6rspVxlXOvofvwYSuLOjwsZA26RM8s7Do5 +e62Bg7PUHbbaD+C8HzbJlyXeQ++oddWPbIkxJMwhP1Uvy3wA6c7E7w/UACZvv20J +KriU33QmW/o0YpOX8xBVwgsCld+IfUIYm1S1mpU6k3oUfGIA5iyKx1XLTMhlaYUG +dTdExwxQp73Jk585qWSpaiQ05OrgYyzZ8OHA2kRTPK+54HSwRfn6senf3TakZHBi +zjy/DZmOU/a/EiR7MCGg+jS1x9GBxOE= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem new file mode 100644 index 00000000000..b821609068b --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMBU0faoRrITeF4k +pN81p7qirX/Gc56+Cux6u7RF1O6WU9v+V5jLw8chQZ87z4QSrFiT1ZnTpwWYPwJ+ +pDk6AWEoiKuOaceOh0bjZCuxADHs+qQrye5D8GXvyFvWE2cT1pD5JNEZDSl2YHqN +s4uTGRP9BP817iRDcuvdxpanaWxfXGfehJRMiEVgKDs+RUpoW4aVNivIInrUWc4R +XXkzaJKqhpCU3jAJBV4jSD5ZnA8PUfcoAj6z6T3I6phuDfRP5ldA3br8yg0hCB7Y +5QrO5lRAgEoIuNnC+U6/AIwWPI36Rjiwg3EUwI/BIiL4AWjzkjSdr0mnzyHPRk4p +cn01T0GTpQi6tfZZpumDD3LkPuEy9svMpJ8ntqDnAsIJVjbg1S60hHesyYHoQw1H +xU0vrncxwcQkVaPLx0uGlioaLlvu83AVnWXbylZXsV/pLy6dE3H51GBFDX3Zj6nk +uJitk8/hNp440/Lve7SaKFPo5NdH+8ACWGdFdz3zxgPuhBDoxEeqj4c1FQA1ABXx +2akW3lQ5VxTAg5AYORvVhJTozosr+Kn3MlRdZjl94tnVByD8MGLLE0C4L/qXR/Il +bkOCz5LHapdC5j62ZEBwiElmMO/tMGl4ORV9tdTBrRZ9DMmKek2E8Qwzy770PGkh +p1cTzZt6UfZEympowmfjtiZfHIq1AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +fGx/D6rNeaVO/vSUGX5q1iJKd8Gnw+/8NRgbuvCDuDOSy8LyqnLmVntj8q9FHpJM +SRH3LnylMVFZdybso2ZbhR1UDReGvHCtKICG3LLP1uWwy5nS3mkGBHFm9COyFP21 +kWOit1+106gEhg2f/NXh31HFmh+myepLjPEj5KxvnQhQfaQESsDYDZAs6/qT1mqp +A7GixOXh7hIFBJ97cU7fKby0Wtv7GqKAYQkaf26ImoGijtMPIlzvwJboJWmOYzIH +zrOHqspFkJD8YvYOwLIKdahViqXU7POL9uRn0vFyaXVcyXRq83Pz+bPSW9AFYsYG +ukSZiJs1yCINZI/Mk1vlfaZWYPIbBkJZ0Ny0vw112dIEilWAkVdsmJyV95aBddQI +Md64CYWZbV5P7/0QOX+v2ZQpWVnaV0m07K6VVuTL3bw6BQ9fcj7vaql6wl8jl/9l +nEotaZiY1f1pUUko3XzXpZEFB1lGBHupuS/Plz8pfFefN/7sOZoWn1VhD9I1A8uh +b2mg6hyQ7pe2NrHOTY1+L1xxxKKHt01kvDhws09qxRXtNsLrL8tl94i1ndLjHIwD +/VRnVU04E/VoTKaEXuETLZwOZu8pLwdiejrWEAmtsbmmcKq/Bk42wa+Wrmge2Chs +V8EOAtq91AjUcQeh7s2fV6yWweMGm1J6pdkNWckCsUs= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem new file mode 100644 index 00000000000..886cc533fcc --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRgMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOGIanwq +rZCqMT+ePwRkiQnD0gyVt5+kwkb8X+fdBJRF0kr70YfzMpKdZP4l4W6C0Jv/ysIH +usrI5pQxcFAIe/7DLW0JPkMLKgXsOtPNZPIkc7WYkq3cbzB0ZTsK8O3IYhwn0dAY +O49T//YqM3TLTFsG89B6uCEg7dQiP9hh6boic8M/WyAseOkJNfw+wYcTWhl1toKc +dLbo8ehESUtVhCOPVT602zBUYFkleqKPeHJ/gzl3/mTnqfeUBljGI2aXwOl7r6rI +D/or7wew2HZ81dTGDqB+yqUhBIVNseJPHOuKbke2E2qWVzAkRnX4b2ehsSaSknpC +KGWyLibaQyR0/Gt8Duu1XIsZKeFjCw27yogSTQ6xTUhLDF1anQyoJX9btSQZsTbD +3vtHbD1O07KSfiG0Z1p8LaR10RAFA7f3HLwwy6c9ExpGu5ED+co8aO5Xp5wysg8X +fYZYx4CaY3moQPJPDS6eOpUXd/6h27Fm34h9VdSj2p6j9JYsmTeEgb0x+JjAQyRS ++Koj/tbSbBqjbvO+FUaldRlHCHYCQTnjsSNBf7SxqE9lfgFitcgiHKSdD7QIfwNB +EK1o7L8OugC/SQtHGe3ngUGuNmHI9w6ItGuVqoJYP3Hwa6ClGmYlTRLoAj8NkBib +toxwGIspTlTzmmLXpqeZTPaA2K5eiq8O5DKvAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBALp4L1aky2jfgk18tney56sUL2Us2aHqyOz9LlowWFdNMtCKo0WKpZ1qXGfQ +92QE+zc/MEdmv3V/H1MmSr7trTq1u7E5vVVI9Lq2lNbRLDQLi1+qd9E7Kdl6Oxw/ +Ecc8oxIbg86p83HhzPfJG64m3x6S6m2c4sNrHRAO/gxxJex6ZSFfQwYJZFlcvvBX +CH70RBtBG/ggasVtwqBuuIRNJ2gAtiWG2RtyGlOjPiAg7nUQiYlXLHVOjvrKDvrI +KTjzRdEUMqKtIrNUBHSbWZlxKZ2Ddavshg/0T0reAN/u5KTDxiGaQxlVEA7xfm+j +etqjzTz7LnKuRsA+Z8UUYaV6mKYfKObDoUs/12IomRCUTQi1K8MP3fGmmk+4Xiyu ++t15EqWJzhjuT2RjCAL47X6ksdOtonX9t29l6ykCvYpK1mlzG+EhqDyMIn62TNfx +OFjWwhIFgyEUWtwkihIKtv3ZVtrJVO/j+HCUfq+6IpjYHdlpdb4OaHgBtpokOtM8 +PmTHJbP2bxmNIMAU1WTfV+e/JkdTKHJclC5DTGF48yRgdKSOTq0G1eJYh4DhlEIM +vOw2rXeWR6VSkvA5vF7HANEptl1tkT3dsKR4BXkSIO16ldWBEHMM4UeXx85GGM0k +TRON4FWBMi6PXX6mrmPXcUW7AyKG2JL9gNlxRgWHVK7xmZyp +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem new file mode 100644 index 00000000000..462916c0670 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDhiGp8Kq2QqjE/ +nj8EZIkJw9IMlbefpMJG/F/n3QSURdJK+9GH8zKSnWT+JeFugtCb/8rCB7rKyOaU +MXBQCHv+wy1tCT5DCyoF7DrTzWTyJHO1mJKt3G8wdGU7CvDtyGIcJ9HQGDuPU//2 +KjN0y0xbBvPQerghIO3UIj/YYem6InPDP1sgLHjpCTX8PsGHE1oZdbaCnHS26PHo +RElLVYQjj1U+tNswVGBZJXqij3hyf4M5d/5k56n3lAZYxiNml8Dpe6+qyA/6K+8H +sNh2fNXUxg6gfsqlIQSFTbHiTxzrim5HthNqllcwJEZ1+G9nobEmkpJ6Qihlsi4m +2kMkdPxrfA7rtVyLGSnhYwsNu8qIEk0OsU1ISwxdWp0MqCV/W7UkGbE2w977R2w9 +TtOykn4htGdafC2kddEQBQO39xy8MMunPRMaRruRA/nKPGjuV6ecMrIPF32GWMeA +mmN5qEDyTw0unjqVF3f+oduxZt+IfVXUo9qeo/SWLJk3hIG9MfiYwEMkUviqI/7W +0mwao27zvhVGpXUZRwh2AkE547EjQX+0sahPZX4BYrXIIhyknQ+0CH8DQRCtaOy/ +DroAv0kLRxnt54FBrjZhyPcOiLRrlaqCWD9x8GugpRpmJU0S6AI/DZAYm7aMcBiL +KU5U85pi16anmUz2gNiuXoqvDuQyrwIDAQABAoICAHZuu3RuuOxB41DEGdWFsczV +7wS6zk1gKME8IGTS1GfEbpT/vd1FYaZKTtGDNOlieoehAGl5w6Zfb24ctBzjB7IV +7lHWy8JLJ4sqrQ2ySzM43yZac5QnMKBiTxJ9QV2sn5CnfG9pekVe2Af9yz2m0Hbw +pLIy72Q+NYXzYlGPwTwEgYPjTkgL8oZ1VssabWgwSl0aSng2DrhKhVXyHgcYZiaC +S0J9mKi9dkb5/ndFHfwKZ++Syp1UZhXjvp15lvd181DoqavmGTXHQmNog5NdJLDy +PJYdXu7t8sDJtwLfhpFOBXFU9MdBIZHfSr0CdAYYi710tMTM3wfgVIoEjcOkRzRx +36O66ehHfcyNsK52Z+DZ6uR4c+MOG0kzTiHQhyxjiu+3nYMGw1XdyE+k+eZDMPd3 +vTaR7kYOQvVvdOVAUuFZG9mK2p0mpofb9cFxFD0vJUqTYXxSdKUNIexR4mWQJw/h +rWOg/42GK4iLY2X6/CsDh6pTsM+HCzwmTGGkL54FvDsB2AhAhXPz/kGiBRTrh9/p +QBxacSPoqN+kF3u2qZRPEmjuimiW2AaXARbTABNSBQJIEmWzWOVdgUBVetGoN/ML +8mcYDmXhAc6F96eqPj0dX8cHfqYPguPhtzLj5V6XGym7hYQyOLBcE7tr2BcdjUfM +V6OFHsPNmsYWZ9F6zCv5AoIBAQD3M6gziCA0G0cG05ef0C3D9OVGWpHqr0yiR3MO +ZKsYbJJn4WOtWWvo8N5oqZBQ8VIoyGd1eiSIDuxXEWniFWjn57QN2nrDNTsEQPgk +HzomgFzuDZ7V4JsjJ9F2nAG5i2HoEwKNHdzfni6mhwGaapd+4GlET0jlC71p+h0X +CPsD6Jwabp6OUyT+xm8XW3mTWskBzKfq0OPbsdv8UB1dPt6jVrkjoe76TlTsWXWi +U9p9/h6kI984R9T10J61c21dokuL/KlHqb6TIQY3RcCgm2bfucmuawIq6vs1PBrK +VCvMX1BuTva9CYg/+hxm9Ky08jFWSCEEtzaORyN+4mmf4maFAoIBAQDpj1NoI7RP +mYqG9vHyXSDUUNbchpLOFKIaeh2DGk0sFmLi/obglsxOKu8K3r/EobNt+vpDTBxI +1EjPWdKuaXNYYjNjrVmPHdHPoHD8JmXzJDbZnXSylV9MVYSMNF+7BWUiPg3/QC7b +1a+ljJH/KEWFb0xrIfNPxVzyq8dyFOxcmLfRVLYlEW+fRYeaZ3QApxGi/BoYK8KN +vG8f/a8jpPwYCVa3JJ7/donEtsbxTkm66aacn8Vo2Y/tdo0nxyqC9PyBU+tV0u4w +aYtEZ28kpC9QheRx8D7WzhvsFc/KsshiB6jddjOVR6VgiUFCo+b/5PqpyZVTVrcs +tj8062A3KvyjAoIBAGRPn/eZS4gZcY8BmcuODKQx4j/UTNXw4KYRXE0A6LT2icqB +mZMkcDeMVpQeCqPt6SsHd4QiVmSnuZvzQwYtLe69BUGB4MMJ/LLTMl5mFZC+Efe/ +qy6bABkZ9VOuJr0GJGqqHCTrc0+CvudwbWQd0O/5XH4NtkTLqMcyaU+Jo2KIp5/K +N6kFcEO6fiX6RrFW665BP/p3XZ8u41fVorTN6EZb0LD26yTDWI64FpYSdN0fm4t7 +yv7ply9QwrZa6oxOaV2a345nASBvDDito2cI6IvstjyCy9RimiGWDEECOuup2deJ +T3KSRanAcnoM23Bpvz+F8XAacJb3ox2//qCUnIkCggEBAJHl2XllTF6pEFLs8giv +SjG26fFKE2yukPCvNb5O8MRIm68mxkSHjsqJoVeN/Act57MdI7ZkVgrcqTr15ljT +QJ2GgomSoS54tzbXB51Ls0XmamkYJezkyGobxbf7g42Fej6guwenJV5oJtfobs8Q +bhVDiF4oECDVrhFdYzKNhXT2ZWVbYIjZUnwQ5/t5Aorh0m+Ywgg1VcxKWLSIOR6w +ElZFhyjStIvqlXcPokjc2cvr5wtR9vRfa7wv4U9m59R0i0OSk6DCKc6OL9QkNNaT +xYasjR7rr6VpjSG2Il6BvhEWrdLh4qku30zlkKG7VzKk7Dyh0ykDM1u34NYC7tCn +hrcCggEBAO+Rnkk5eYYqGk/64+Qy5qA7djvvZ8AgihwJL3+ZUDSOxh0W+Er4NB6n +j0kI22N//D2j6hg93TNj9jI6lISfmY+TSikr/P+bQPGXl8wvekQxpjT5JhCYI93M +LXnSULuy7J1ujkMGdxEvfOTjvmD0ejtnuaGd+jM7hx4QNBbJj4VdV+r5BQOJAlfY +gk6n3RgAnu86szquWM6dObIz9BWtIcMVGlxA7yDmxjVDDHLwGpcwG+MTQRcHoeT6 +2+b7FtVN1NFLazfgPS3bxKs5jaUB+Ibm9BD8B7THviNikqRYqwoJMWpJgdWo/lOQ +X0ueOR40kfa077G7jNfb03qOPUR1mFw= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem new file mode 100644 index 00000000000..846f6db84dc --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOGIanwqrZCqMT+e +PwRkiQnD0gyVt5+kwkb8X+fdBJRF0kr70YfzMpKdZP4l4W6C0Jv/ysIHusrI5pQx +cFAIe/7DLW0JPkMLKgXsOtPNZPIkc7WYkq3cbzB0ZTsK8O3IYhwn0dAYO49T//Yq +M3TLTFsG89B6uCEg7dQiP9hh6boic8M/WyAseOkJNfw+wYcTWhl1toKcdLbo8ehE +SUtVhCOPVT602zBUYFkleqKPeHJ/gzl3/mTnqfeUBljGI2aXwOl7r6rID/or7wew +2HZ81dTGDqB+yqUhBIVNseJPHOuKbke2E2qWVzAkRnX4b2ehsSaSknpCKGWyLiba +QyR0/Gt8Duu1XIsZKeFjCw27yogSTQ6xTUhLDF1anQyoJX9btSQZsTbD3vtHbD1O +07KSfiG0Z1p8LaR10RAFA7f3HLwwy6c9ExpGu5ED+co8aO5Xp5wysg8XfYZYx4Ca +Y3moQPJPDS6eOpUXd/6h27Fm34h9VdSj2p6j9JYsmTeEgb0x+JjAQyRS+Koj/tbS +bBqjbvO+FUaldRlHCHYCQTnjsSNBf7SxqE9lfgFitcgiHKSdD7QIfwNBEK1o7L8O +ugC/SQtHGe3ngUGuNmHI9w6ItGuVqoJYP3Hwa6ClGmYlTRLoAj8NkBibtoxwGIsp +TlTzmmLXpqeZTPaA2K5eiq8O5DKvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +3DJlf7AkZklzzswgm487f+y2bB7IYr55JwENASDxQEOdVcdgLC3IWu3hLiFwdqac +0Sw2OHZuETwJiIX3fD+qUT6TgbsP21W7wEQ4jfKg/bsXFMbrvw/ILkOW2JLTH4Cc +9ylCN+46dQ9heATkiF/Co+uASz9IoSDdtoycA3BuKGBZI8VGa56QmJOOsMM5NgxT +RTh2r23tV4E8AGYj3HC+b1rzK1RTlsj/m5nM9Jv0/NqoV1cprS1ONr8CBhN0ttuA +WLrG+DUZTMJYFabqTptlgejQFhiFp5HT5A+eXgZ8uEUX1I3q5jq1BEWtLdmJNZ45 +QViSJOokH/+1kfRSWiAH7pdBz4URLBcsDhAag4J7kV38t7fgdaIizY8R2Ss82iEP +xqa4A0PA065wB44zng/VrPrHoH1YnGRugXEnrqgcipC0FxUl3oQjvwOSR/E7yFU0 +GIr1MpRcyrd0z4p16783qnMpE1Aa0msED2SBKIK13WcNY+CtDF/wO47ZNywl1hBo +VkM+ohPpmonaVXNGdpdoZpeGjkBUbqkn+so4aYkX/WuZ6vY2vwdV0prD1vdAFfD2 +AeJx5ypu5aeKn6nK0eMy6W/VEJx6RLCiYVOCIcssgy31rmk4iLQJP2StYVK2mZKp +5aSR4eTv1/XlMujq+ZqcuUqA1id9wP7908Xr0DzdNdA= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem new file mode 100644 index 00000000000..ce9a472cb9a --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRhMA0GCSqGSIb3DQEBCwUAMFIx +CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTIyMDIxODA5NDMw +OVoXDTMyMDIxNjA5NDMwOVowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE +AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN8Bt8gv +50J66lQ+l/NUW+lqW4DesmSLv1BnjDd5SSA8tfczt999/l1epAGeEN/Pl4dAxXP/ +cxpx+J+xF6SKNxQ0RP+PHQMiDzCUgBq4OKs09kDQ/uvycUZlQuWPtR610TWjZR5r +VrNSwJQp3VGDdNyEbKj/yd6Yi5NC1iLuqPC20fw5/9BVTm1P2wWX7nv1AWs235s2 +yAG7pLNcgPiTfSmXyyT31YBjb9Onun7gv7exI/3K9mS+aWq6ci1xAXtykVCs551T +OQmDAUxda041YghEThO4MrZa6uSZqVwnoUcXTla+8biLYb3+9CnIjM5whAOTR+9r +jpsuuXEUOsrX9Mgb1HTS+ksmrA+Eka7MdVi60Hoon09uNvcTM8CSKNgnTzcPCM6t +J4NHDiimJM5WA/eY8i3NNCTa1HUGEeIK51UOdjIFKsvzG0TCI2FM7jQLJK5S38tI +deZ98iQbguVGhoCvRotLEAwW1M2rSOu7bxAZU4QJ93IuUfkLn2BipOuyuR55Z/6F +z5Jij/1lK2/pKWhntUHTIpG+bBHDF++0LN0aB29uIwYRkoz9JUgnNz4FDVbLvJ+z +5Ywr61t8AujZdfMZDpRYlzfWPGej8pm7/Eux5jgx/3jcLtqfqkfZLSuFjBKfkUU1 +eGsC80RupMJKIeppv541W6nQJlmJYKv7DCvrAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAD+YMVntBdeq7xJEL7xU4QEHzUGhDWodGMJfmswcxe7gf5Nztcq5YIug+akL +ewg0wzgCA5YGz00J92sKDF16RmYyPfkxmrCYdNGwISjNJyEEcPEVkdAzwILjv2Lq +0shFlSsf+Zp/M4XhHeirmzz/jJ9KHlzEYoCz1WOn+UGF12KgV2oQOamJSWOMCoMh +81oy90V5IlCBqnYfZCYj7cbYLBd5jZMZ+7lsVnxttzPTg1gIoP6vrLT32Ubnzx9N +IoAeiUg7az/fbnuOkJtu0cjz9aSdpjm2h2giyVAFJ8DkQ9C92tdr9DWZKn7rDO16 +TMdv0q8NFjRGhqdmqWUG6o2cUmQsJ/ZiIcHx5X1b7j7PYSS+ae9zi1tcpHAN6kCw +WHguIf5I8MIZxE741ZMBokFSIqd6Bh1EP/TUx1+g2a/nH3ZaNd4/KKADxfUU2Y58 +UwdKeX9YpcRz+NNO+1h3NoE1a/i0dhwiBf4OzBiV0WpAjQHT95IlQxTxfHFp42IH +GrbqIS3qK5DKlNFkBBk1beKxBGKmTH+Pw6fhjkuPYQzjmGo4xluivfeT8SiBT2iO +uIGLd+sitIooom0KEjHuHS9cdZ5XEPIUDAFhmIt7Y5K8J2fs+xtYzhibg3n0Q6qh +xTx7GzhTA1HSUE/467af5J3CSfpGAjZQZo/t2/A6tCumzk9F +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem new file mode 100644 index 00000000000..b7464eb2866 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDfAbfIL+dCeupU +PpfzVFvpaluA3rJki79QZ4w3eUkgPLX3M7ffff5dXqQBnhDfz5eHQMVz/3Macfif +sRekijcUNET/jx0DIg8wlIAauDirNPZA0P7r8nFGZULlj7UetdE1o2Uea1azUsCU +Kd1Rg3TchGyo/8nemIuTQtYi7qjwttH8Of/QVU5tT9sFl+579QFrNt+bNsgBu6Sz +XID4k30pl8sk99WAY2/Tp7p+4L+3sSP9yvZkvmlqunItcQF7cpFQrOedUzkJgwFM +XWtONWIIRE4TuDK2WurkmalcJ6FHF05WvvG4i2G9/vQpyIzOcIQDk0fva46bLrlx +FDrK1/TIG9R00vpLJqwPhJGuzHVYutB6KJ9Pbjb3EzPAkijYJ083DwjOrSeDRw4o +piTOVgP3mPItzTQk2tR1BhHiCudVDnYyBSrL8xtEwiNhTO40CySuUt/LSHXmffIk +G4LlRoaAr0aLSxAMFtTNq0jru28QGVOECfdyLlH5C59gYqTrsrkeeWf+hc+SYo/9 +ZStv6SloZ7VB0yKRvmwRwxfvtCzdGgdvbiMGEZKM/SVIJzc+BQ1Wy7yfs+WMK+tb +fALo2XXzGQ6UWJc31jxno/KZu/xLseY4Mf943C7an6pH2S0rhYwSn5FFNXhrAvNE +bqTCSiHqab+eNVup0CZZiWCr+wwr6wIDAQABAoIB/0I0QFst3XnfA7H+4x1Z7e9d +o8yeUFeJJUK5eub9Grh3TY4VzICM5vbRId9ZDalj95gvom7NZ15yd1zxNhOi9LcK +zXERC4vikJ/bdix4hFpPXsvfP87MKtS7OyDriNmVIIbL+zkMpLCX4JQb2ZhZblgI ++DkztrpejxEoxmmYcI8Ft1Ep5sfyi1XoXx1J/YLPOZyarcdme/oHut2EmMUzA/VV +GvnemYOEAa7UHImOL1xZOlYd6wf9f04wC7Vx1v7PBFTu/9O04TnxqnEBStns/y11 +GbjA9k0ssI8tDxpMqZRxVtBp31jqCBpflhzRbPvca1SkZLavN6baODNZzhpqAkDX +3R4lU5C7wu4jtzydUyEsCFNdtkGKlxpZRbRZk+keUC+HeCmXPED7p9egwF6Zi8VI +oaXl1KvHZO2W5x/BV9I1taEPhmOuRR49KxkU4e+IjqaWYN1qsqYqCs/od22Rah72 +KT+thr0mdxC4lb+pvteafricUQuq/dSbEY/lva7PhPQRKVX/VxOaAxBnhA1LHVgZ +imsW8W3eOQYJbxniTrz9EblWAg4dCcupsjMDUDUyACB/E6isDtYU1J2im6p4gbqw +tXg3bRh7KruIHbPSJyrFm1uqe+v97TLhpwPHKCsxE4HiJgRzaQDRckLJQebqNp3Y +e7kLLjg6uGsjAl6OwKECggEBAP5bLGVrmBmAz8RYPnG1MQWlsFg/eIhMFCqMjT3P +swPUU2VJKC3TC3OwFLxlAr0lkXol+8L8aEvxGjHksleA+1z0lav43b1/2jKgLgI6 +Ym5BxMJa+sUJpI6K7CedJ6wf2ozbpVXazvNBZ3o2l0QbC/KpX886CZH9YJgn7N0M +TfPe9er5zmETdHGTWtA0sDI8fZ8XndKmnWG9KTQCGur6gemF8SKuzGv/BnL+BZnv +bDqSvyN8Wjk35KPNeKVW78ROxRuEdB5brryGk955hX50PRRoofW8GSmLJNKNYvIj +VRkKrDKpz8gW1C2/xa9j5tQkGRFMDAptmk+yvtmDxfZz38UCggEBAOByrXLMTcwR +bz4MYcSmEdLv2VA/bZ+y0kW0frUU5il2fyQseoFbunVbTDiXYf40uueMbOONZktM +w04CXKRaTbnS/s6SGU5VW19jv+xzwrzpB2Shm08APwgFnSw40bKCpN4ZWQbOyFVq +QIMXfA0+Go3zJz37MsSgY+mzhHp4WITobVFpdlhaLvrLPCB78uInZrFsvNN6NP+K +OIbOoTA9u+BP73THHkpQdrRJaJWowpqejz8kzQ/Xu0Xe6AG1EGVp39phKpWH9TPF +8xoxjbdIGPkzCzYO3hgz6PlnWVj8iyTxklnaUblqKkY2mOlMA00ujcdF3d3IHvaM +Xolej+XeZ+8CggEBAKeZDdzaE4Oic8RtXN/xwxZ0gYj0cYhlkNgkeqCi7dL1IepY +VQg0ypP1DwTADhjx2zTAOG7XgCWh/V+o0LaFv5sVclW5iuplhzHah9ZiAB+kaHCk +IB6a5vohoc/MZqqs5oXv6LZ0ke6JRxSpSezPYYUIg5/5Hvs6GF7J1/IjPG4XmLS2 +23zto8l+jdUpEnxXjXK5zf1SWdtgF/kz9ealH9rurd/ri7kRdn9oz+oJb6f8r8ND +GfQf1yDzr65KZXxVZt1l3llukemZR2/NZN/Y2bJL64QO6AmOrLmr/emMzHLOrH5J +lCbEnBR1C14xFpTsIDRchoaMh6RCJC0Q/e0Rlv0CggEAAOIysJsBS2ZeK75cvCtz +MoNjNZ+qTNClZ0TYotncNhmTUo8iRFQaHdAoMqjV5+xJOBQjcZni5zT8J9h2iOca +GzsraaDFnLtVSsDXxpSGFbxNHSZNuDfmB6AOCFiI6sz83Sr4YMB7pWpvqpRzFpJC +BIEKjIHqpz+CZS8hvGGw54UKuSFTJ/Hi8XXPXMlgIWfKTbSB4cs/XiorIsy5cbks +fiuSY8FM6zn53afUU5KAgZ9SLQt2CzPsNtAz1Z3i3KNYEEIFquUIIBYNaPL8/dW4 +03JR/vp8AVhi+Ghhv6nu2kxhKR1k6Pf0Bqa8X16/PJSMVlZ+Extwk8Pls2C97Ee9 +3QKCAQEAgjcbHKBjd7AeyNpPSzNpv81Rry5qqOc+Cxx8LtOHBl1wc5VB5FPxfbuX +MX2skvWPnokDoXcI1a1WQwdjaZUsSoqdeyPtw8pFWiNLJZkYImiP3zMCZXYUEkzk +3EXQZryWEqBYBqxlEvTyjbBmnrAwOPOUKARFi1l9JKJ4QpdELXo9Yl+w2IQEQ5N9 +jrSY7LwS/cb25rhEc6oh/89aY83HPyABh4lC9bsciXki54YIeS+y9ijN8yCRxikr +mVGfQ0Y/qcY9spAj05yr/vnlENBB5ohxwKKsemOnH93E2GFxc1dzmWCGvISjUduB +I68TOg71OfCKgfeixNgcOvQoN+xngA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem new file mode 100644 index 00000000000..7b4445b3609 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN8Bt8gv50J66lQ+ +l/NUW+lqW4DesmSLv1BnjDd5SSA8tfczt999/l1epAGeEN/Pl4dAxXP/cxpx+J+x +F6SKNxQ0RP+PHQMiDzCUgBq4OKs09kDQ/uvycUZlQuWPtR610TWjZR5rVrNSwJQp +3VGDdNyEbKj/yd6Yi5NC1iLuqPC20fw5/9BVTm1P2wWX7nv1AWs235s2yAG7pLNc +gPiTfSmXyyT31YBjb9Onun7gv7exI/3K9mS+aWq6ci1xAXtykVCs551TOQmDAUxd +a041YghEThO4MrZa6uSZqVwnoUcXTla+8biLYb3+9CnIjM5whAOTR+9rjpsuuXEU +OsrX9Mgb1HTS+ksmrA+Eka7MdVi60Hoon09uNvcTM8CSKNgnTzcPCM6tJ4NHDiim +JM5WA/eY8i3NNCTa1HUGEeIK51UOdjIFKsvzG0TCI2FM7jQLJK5S38tIdeZ98iQb +guVGhoCvRotLEAwW1M2rSOu7bxAZU4QJ93IuUfkLn2BipOuyuR55Z/6Fz5Jij/1l +K2/pKWhntUHTIpG+bBHDF++0LN0aB29uIwYRkoz9JUgnNz4FDVbLvJ+z5Ywr61t8 +AujZdfMZDpRYlzfWPGej8pm7/Eux5jgx/3jcLtqfqkfZLSuFjBKfkUU1eGsC80Ru +pMJKIeppv541W6nQJlmJYKv7DCvrAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +Rggrols8hXGEcWeIEGn66kY9IVTzaTUf3oMfEbdf/2Q1QzHzmqp53yamHl5ioMgX +o5UBVxthgh1VOxkvCxIzlKDJprzVFkfwwc7h9c0HGt3No/ERobHDT6YRaGukAL5g +muIGBUseyBAOIfyqc5kbCRWfPrAOttAH4gd8XMBgO8XdfHAvyXBC8Ha55O6oriX9 +IAKL5+3nVJkBle+62OmROnstbcdKyK4UtOeki/6ptYVE0d9I+NfKjuk3eKtICW8Q +Pn3IEcNEZoFG2UQ19ENWwYEZyMZJt0aunqnm7L4RYiZT5w4meeendzXSKLKR6+Ye +ULt1sDRskgKoNRzmeCVzci05BG48jv/E7Az6aV/qhGiU2qIAPMdVXncWUhR3fj+E +CL/uLifOvfC6SnKw/7qQmgjUvEe4Duvi670a5QuImpm/mAIN22cXPc+QquSdR5xy +loz/o3JJQZemPAOM0CMIHZ+cGESxH30QCBNn5HfcOf5fRZVCss4Hl6JxHR2G4yN3 +RKEIUXR03qgSK91WHl3WvqwXgmIAiUuvPjo2i7kSuaUUHilZiXK1ngIqYfUTB5SQ +O8pG0fx3fbhVDA3RQfXeJE6FA2AyLvqOcsseRzvcQjQm4MU7p+RVaY17rI6/EkS8 +ac3E7BPwnXqSAkPSEgoiezv/Z0Hkmrcu6fIsUuf4ETU= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh new file mode 100755 index 00000000000..d6126d361f5 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# 1. Generate CA's private key and self-signed certificate +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca" + +# 2. Generate server's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout server-key.pem -out server-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" + +# 3. Use CA's private key to sign server's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile server-ext.cnf -out server-cert.pem + +# 4. Generate client's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout client1-key.pem -out client1-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client1" +openssl req -newkey rsa:4096 -nodes -batch -keyout client2-key.pem -out client2-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client2" +openssl req -newkey rsa:4096 -nodes -batch -keyout client3-key.pem -out client3-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client3" + +# 5. Use CA's private key to sign client's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in client1-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client1-cert.pem +openssl x509 -req -days 3650 -in client2-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client2-cert.pem +openssl x509 -req -days 3650 -in client3-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client3-cert.pem + +# 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem new file mode 100644 index 00000000000..6f8e5a3c6b1 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFSTCCAzGgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJF4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjIwMjE4 +MDk0MzA2WhcNMzIwMjE2MDk0MzA2WjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC8 +jV8igQGgCvu/7BJDI5VQl43VGAFjH2Na/E9P4E5uwkSlJVED1WKvIlxRWhOaQOfC +587nZVhQtHpdbCvBdKrHml4SVbTchs5SN2kZsHeqaQzcGnejnczE0SYo4xNyniSv +GiQ1M8G3fiZNflEIPM/+Ob2oI3YnVWFGy0a5rQcHZWS45KuGILMP0aRHyzyh/31c +K3i2xA7A3V2jBNuD4kHG8TLgfDeoCecTI0iU/LJnDOolX5XdpyeoJ6YyYOGg3F9e +bRmbNlJN3Iky3Vzyc4jYG7y6f5DqfebYMW6hCvLpf9lN6/gPNOb2KjL3hvJ+hbj+ +b9EkVAzpw7mW1VHEy+WbtYMPoKy08JTc7zr1tv/vQGr3XExwlC9iixZXMaVt1kP1 +TEVHv2FiUOiZsVaqtoFpS/wBvKeQdkzNy+66pRpG9bLuOnL4hlz+rwHkdBmHGk+q +cXdwglqIDqXKlCpIMSkFPH1364KLdJ2qBgWWoWCJjUmgbrA8/LU6DX+GBbEiw45T +PQKP//RMkOrHOYRD33WTU0iKP61zn5+9RD5OLxEUOtCvL7AfB+jt4vYrMTT2U3Kl +OckWxNx55bYLdLfGKtepGV2r5xzce0UMbWQrXQRuka3a/j5VJUTuUgcwgd6FoP4N +4ObW2H1YEtE5M30xpa1kcqJ1RGEWagakISgn2Z3TywIDAQABoxMwETAPBgNVHREE +CDAGhwQKBaxNMA0GCSqGSIb3DQEBCwUAA4ICAQCE2eJVcvsMmJu6xAfoE6/u6BrD +opMicCtlC2qt0BgSIzzDQ/iWjnWKGM1C+pO+2G0WTczj7ugsxjPzhkyBpuEZaWt0 +9/tJTKIrgaRZvEe0ifsJxyqL5LJgfxK7TbDPcUBKr1v+TOxPVRq0FuG16x+yka4C +rwxfBHU43FmtEFfgu13r515F3ggXcdlojkce8ZKtTAGEcN0MpbJ6XS90BHU0sy5A +APTm0fR0vM3kg1nuBLbSGF5KfASdw13gb6QsDbll0IqK8LvXYiX5CaVfkAe/pFkO +/2iIxYW74yC2gV+DcFdRPVfFxSKrdg0tDER35OYg1/vXRjV5BWr1EjE3qjrCcUZy +rlF3fms7Arr20ka2nSa8avn4ALpyJZmKasoxNAAsxivingNVZkql48OqsJ3n0qGk +LI6Yu+UM/pc78a3NHsdsCbnf8qvae4oJa1kyiochJu+gUOzHvs4Ydti9iTQn2Byo +2A2LzyVPBmSOhzdQ7SwpvHA4A2ftao+dZoA/+o4rmBtbmgxjpBPyPJTN0ZfKlpKl +Oyi57ov+cJmZctSUbP3M11gBva7aYu1Rd7/eXeCEl1FHhmKL/Ee+UrNZLiwspb2E +Sa+pOHdJX8VgsIYXku2UKaGT2QFITxO7fnxghioxgsyCKrQ+m1gL9vgXj/gJu+48 +c+5CZ9SobLdMkVOtQQ== +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-ext.cnf b/tests/integration/test_ssl_cert_authentication/certs/server-ext.cnf new file mode 100644 index 00000000000..83d9b03ccb7 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-ext.cnf @@ -0,0 +1 @@ +subjectAltName=IP:10.5.172.77 diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-key.pem b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem new file mode 100644 index 00000000000..065a2290749 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC8jV8igQGgCvu/ +7BJDI5VQl43VGAFjH2Na/E9P4E5uwkSlJVED1WKvIlxRWhOaQOfC587nZVhQtHpd +bCvBdKrHml4SVbTchs5SN2kZsHeqaQzcGnejnczE0SYo4xNyniSvGiQ1M8G3fiZN +flEIPM/+Ob2oI3YnVWFGy0a5rQcHZWS45KuGILMP0aRHyzyh/31cK3i2xA7A3V2j +BNuD4kHG8TLgfDeoCecTI0iU/LJnDOolX5XdpyeoJ6YyYOGg3F9ebRmbNlJN3Iky +3Vzyc4jYG7y6f5DqfebYMW6hCvLpf9lN6/gPNOb2KjL3hvJ+hbj+b9EkVAzpw7mW +1VHEy+WbtYMPoKy08JTc7zr1tv/vQGr3XExwlC9iixZXMaVt1kP1TEVHv2FiUOiZ +sVaqtoFpS/wBvKeQdkzNy+66pRpG9bLuOnL4hlz+rwHkdBmHGk+qcXdwglqIDqXK +lCpIMSkFPH1364KLdJ2qBgWWoWCJjUmgbrA8/LU6DX+GBbEiw45TPQKP//RMkOrH +OYRD33WTU0iKP61zn5+9RD5OLxEUOtCvL7AfB+jt4vYrMTT2U3KlOckWxNx55bYL +dLfGKtepGV2r5xzce0UMbWQrXQRuka3a/j5VJUTuUgcwgd6FoP4N4ObW2H1YEtE5 +M30xpa1kcqJ1RGEWagakISgn2Z3TywIDAQABAoICAQC11lTwLp/Fm7IL9fvquc9P +CMmkv2DfGi80WO2YJ8ccM8gFyEYoP0rLgYSshAUxlvSr1+iG6grQ0izMGfzctcnZ +c3rTjco9fthNG9kFCFVvh536SqAkr5MCIH3/onZn7DGOmNRgZoikkEkaJP66xgME +tuS72W8iIcoNfw63FDIaJOONGCJ+2Nw3HkOjZVIVHRLlp5rkD5H218Vs6MtWlgY/ +eO9K5SC7sskhgL6HyGe40BCjeFpMh97L4Wj7XslZ3A0xQGAYervHES9TWX5A58EK +QT2yUkIMktzklE+PicKYA08rQa1Z5Pf0YOAELSWBdS7iWi3FLjXB35sE5rbT5puH +9hZXSDWLggbefuaUJyowDEZy2aHP5pvXKBDhEANRbU8VaDyHhlNLqVNquE5Cn4HO +zPeH+KLFbbABUok7yYZmIC9Bfn+rXvNzTX6A13AdJI/HcKA5RBGtpAY/168Pt/Aq +dzuqepu42rFAvS45RNevp72IIavx/QdAA1OTgKxh3c2Mf85pIXJ51aWWLnn+EZ5/ +EsE0crfwkuKJvjubNC4oOwMTFMIBI2WsjvaAw8pQw0Kb0ksExKd0wz9mKcqR/v0I +K9oYsaHkx5je0NOZds385+zCoQHYaw1aKUd7ZLqr5G/Nf/2TEYpMWco4ETA8lzu3 +Ty/8XkNw8jd4p+7bUuz1mQKCAQEA4MNU7GWDPwUKNNSz335nGH2oBvSGbYiwLcRM +D+x2+RTfOAFSSJ+Q5tQ+327ZkAB5dK2mxmDYKB+Ln1UBIneViUflkMyh4fuutIXI +wYo+BL71r89MqhRvvMK9hWnCGtJTJedf2iQENJzVn4J76BvTPRYywBv9pofPOlj1 +MtwwMA4CZAmQpCUaF5NQr4nliYx+slkcKwlm+cOxeZGa8mkNgQdmCcTZkRz6qsiR +vQDEDiS1+5lCJ6nWW4L2tOPejNN//hVlbPGMaA0oiu7I7w4aSxnTlLhDgJzJwmN8 +NFYl+u5AcPq9iRtBnzfPmd87S9bg10zcIiMKxw898sU24Pa0jQKCAQEA1sG5hO3c +4API//k7NEWXsx5Ns2JE/AV1LtmBgqXkn1DAJ+b6V1nIUppTs0zspEWrae9KrsAk +z47qIbPaTLHuptLrvEXk2LVfzcK32a7fXXDOB5KkBhzlJM1J3PTRQFR9lr7qX6vr +EDc4p7p55IDEGnJdXa7x+z56QjpAZaHlzexQxvoWWoLBkDuoT389sdU7CbgTa4A+ +CR6D6qKd6H6tfmv5sPlvp+aje+ObacP9I4WyVjscWkzBHxS3n/fTLjY6OFv+o8PM +TdytN4+HZnu4MDJlF3vx9P6CbnnVCaScXDxPGcoSJPcoEQqoyxuvUQLDUQkzWF14 +02EvXW0dbgiPtwKCAQA0EUwFD2ceHD7HClc4+QFNDR71rYPOsBGQKJ8uOSs+fHVR +dgznwf9BWf3OqNFBqLp6KxgtcJXihZxEpt6Ca416pesqZh1CSpmoPC3LmAjR9KLZ +vX4XEHDqG3roAx3yNLMKXtU3pYxL2+Eo+INXu8ptpkzPcCyMfX2mGKGEzLllCHnJ +TuXxAJ9QwtG4OIuyF5fqHPaHicAPMCRW80If0fJM57fdn3p/QWVYVupcDGdel2aJ +CHHo2lFMFcStFvShTwWhiLdcS4CpQhMYTETEDFJO/4aiNyV8D9Y1b/J/9U0LGlJX +Wd66elPzXGx9StdjtD2V4rpENjXy8zb4nHMgHkapAoIBACvvtmTbxTSPka/M7a/k +DQU4Te1FTZfCBhdvqG9yQTPW8Xk4aD82vyUnLbihJEj3d/pUWpMl/GH6eywp/59x +R8IZpOD/67HqaY9PJw4CGPClA4HJHoWho7/DwDjUXXsrzgXpSUoJgi3vHkgyfn2h +Wn2OqEtiX19niNvDzyj71mgq0Nvkjm42EiPQEL8y6QxY85spbc+wjQCQnayDWIsY +X6ZdsNfkMFPJe+j8x+77ie6ai8HYlhRjX59cPbUcnrf1oDOnnpEincnQPCAB3VG6 +PhSeOtBzKy1UZJr1kgBHDTZRoF1GWi/14NybsazcHSIVzp/lofuSJAYa+/XBPSQl +3EECggEBALSLZPdg13906LEyznYnjgq+nMh88usegvU9qsBAFExClLLfr6Ak77og +boNoOwbaFn+xiz5M8BTJIPizJcm5GjYaqg58zotTtG51h6VgMri+fb/BUpVr7p7n +aSq3kXDZlrwZnmooCT+KcGx++w2N2SYSyZX1TELt/dpfuWJvph+E37PkONEEiHPF +ZtSA/f9lpfP5/nx1pLmv4ksKdXqpz3/kNqaf9zbhQLgOm/VoBHL4NVPYRylGpCJb +R68/7yvHBd2EskZoJB53TlJmwu+fC6ee1UiG6aqTULfEsiGidi6jIt56Gz52ox66 +BHL/JsJ0Be5xM3V4x4PtihQ3Dw546FY= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-req.pem b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem new file mode 100644 index 00000000000..be2f756cc7b --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2Vy +dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvI1fIoEBoAr7v+wS +QyOVUJeN1RgBYx9jWvxPT+BObsJEpSVRA9ViryJcUVoTmkDnwufO52VYULR6XWwr +wXSqx5peElW03IbOUjdpGbB3qmkM3Bp3o53MxNEmKOMTcp4krxokNTPBt34mTX5R +CDzP/jm9qCN2J1VhRstGua0HB2VkuOSrhiCzD9GkR8s8of99XCt4tsQOwN1dowTb +g+JBxvEy4Hw3qAnnEyNIlPyyZwzqJV+V3acnqCemMmDhoNxfXm0ZmzZSTdyJMt1c +8nOI2Bu8un+Q6n3m2DFuoQry6X/ZTev4DzTm9ioy94byfoW4/m/RJFQM6cO5ltVR +xMvlm7WDD6CstPCU3O869bb/70Bq91xMcJQvYosWVzGlbdZD9UxFR79hYlDombFW +qraBaUv8AbynkHZMzcvuuqUaRvWy7jpy+IZc/q8B5HQZhxpPqnF3cIJaiA6lypQq +SDEpBTx9d+uCi3SdqgYFlqFgiY1JoG6wPPy1Og1/hgWxIsOOUz0Cj//0TJDqxzmE +Q991k1NIij+tc5+fvUQ+Ti8RFDrQry+wHwfo7eL2KzE09lNypTnJFsTceeW2C3S3 +xirXqRldq+cc3HtFDG1kK10EbpGt2v4+VSVE7lIHMIHehaD+DeDm1th9WBLROTN9 +MaWtZHKidURhFmoGpCEoJ9md08sCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQAb +FDegAoUBz9O4JR1u68IMnGkO5nINGAPQOqf9a2BxGujnSB7Lw6SHukjkUqqgnfQ0 +x/aWOI8JVAi/ptscojgMQUDsVNsij5v+jbJE+ZAobxnTmKP0wTc2ktpf4d8UMVc8 +gyM85jLHZ8caCcuy0D97W81vgIv33dNHWtP+sfbQhX9wJ2YQTahIC8NpuQfLAOUH +EFxWil0mfN+9vRQ1C5naKtvrOPqyM0RPrWiudIJ5QjI4aSXxUCupxxnaQMoI0Y50 +MvVVT3VwWgP+hL4b+yEJFHRpE7BwCZijsLIXkXmVZoveHhiSMYen1HWIP1VMDEHP +CUtG5UQcA78CBS8qg4nyFbDU4hWClAkAt96O8Y2epJYepIoYuBBSEfrgupESMLjS +E9Hfq/H6Ac/Q3zWa320udvA+ysfS8pagkoiH9+TarrsDjhxLjg2h2bGcXKlrsP1R +mRVZwfNOl3/ZNq5HBPb9Z5WXKvcsTCQAlnHJdaSmzdyArB0guwUHg8ZZNZqCdVgL +TPsfE84yI/HlwRfuQILfGxq99p/UYFwnee5CoM/PPvaAT+9z/lykMWZA7osuBcK6 +zP8XneGmZOkmez5+YJgSC0xeaDxr2R52eQXlQEJGDbFDtQap/X+cJDGyqmGnbhSu +6XkGy0l8mAkpcurMcy3wWf6+joskZAN4Joi4ZjKsQA== +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem new file mode 100644 index 00000000000..ef95a73deba --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIUL2Y/QpwqqHyi43PwPeA6ygdPYK4wDQYJKoZIhvcNAQEL +BQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MB4XDTIy +MDIxODA5NDMxMFoXDTMyMDIxNjA5NDMxMFowVjELMAkGA1UEBhMCUlUxEzARBgNV +BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 +ZDEPMA0GA1UEAwwGY2xpZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +AgEAxO2PSeaiNFMRRiFXpnMw07u6EIdEc1Jx3cPvZjEUg/pdEmMYkrSxr2MeqRkl +tWH8TcIIoiWDLIcM6IU0mF6a5ULu84hFb9b20qRG3wRNb5yO86HnoyzU99t98++a +9iaY1QAt03k8wq4jRjU2k/eoVSoLT5uVP5KxiNzdS2BTHFSsxrt/xcwdgkfJouHN +p+MYUekk6qaQy5fTqTpqdkgO2v/JoYCi0whBNj205d+WnS7xfeyVSJP1OJWHRZ7K +Y+LU6hz6wHIng4s/ag7VdAk0PArWs50BmH5g2zJfvt7VeTQebaJWUtSEY05odOqt +KZteUmmhxW/2M73wGVF3WAJCnaxypsjcmMZFCpMXpwyTFrqobvC3APl6SOP+Ev1M +LxhhCIDuLFu46P55KKEKjUCsYigd1VsHjjvoajGcqlPlMsVHJc9VChsQDz6agzDP +Fb/LyYbrDTTmsI57/s1jAZyemq2SEYPApJvcdZ/ucl741jI0671EZPlip9iUQgt3 +MHlc3t53/GtF2W6GF5Fogch7c+0c2BhMupAHAXwfLABvv5X8GDyjsNlwB6ea9jeC +Hw+0rEotZzCXId3daFytGNm1jI216kXLSbvz6uz1wMGS6Hrhk87whgvQ58RMNs1K +SGDFw1WFv+QZeTO7wqcn8Y/eqF7q9RBhOpPMJMX8Sx/UXuECAwEAAaNTMFEwHQYD +VR0OBBYEFCI7Iy7tY0D4HPa9BZCZxYuJ51mZMB8GA1UdIwQYMBaAFCI7Iy7tY0D4 +HPa9BZCZxYuJ51mZMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +AIKYtBwTp3yvUGSXorV32dnU0Hp0MOie/itgx/la6b3h2bZSoCigKmcmvMaAaNzA +pxeYSsf5wPnONpWfo9hsGrUDMT4ETnXdzA1dbidIrhJbGsY8CN217Qt3YZWNWkrz +xLwxEwAovQZqnGDvtx+tRE8i6YJO6/kca+GB7liHFvUx8zaQ6gCwfloduG8rOAeq +noeCpW/zqYQSQGK35ntQ8MTTRbi7jMOTCikvRlldS73ODQcAR7jywgBYf/i8ANtz +NoWa4KbWuqKsQKMIGOi1fMLMaNlDSzJyw6UJ2GVCcL1NxkCZi0yudfAAxWlRis9G +zLjm7YdNBiC6RVZudGhvzjlsLZpE9DgiwXqcDv3Y1dpstD5ikrNhlQo6THH1YeFy +B8vjVGZZZu4B2JEo+QWH+zFGJosD66YoaKMVuwRPwoGDQoO0Pfbpq41A4KUhB3cf +X49/rbInqwsN5MuGp4l4+T7k7Wm0Y1Qo4FXDVbFxHvvniyHUsZk9Llzf5wBLl84m +xheUGgCHSflfXuuWi76yoADHCv+Eqi4/aLJmkUewKXJlm+XYs9bdBHUI+Y10KmhA +hgcHXF56L+N4mLRwUuLxa5qwQIqNX32+piQoO9opxnVKKCptpATLE30TOMLEXBlp +J+6b1e4BIasAAEGUhTgPj/SLL0u59Bv0K5SlSn7VZ0gI +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem new file mode 100644 index 00000000000..b2213cd2675 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDE7Y9J5qI0UxFG +IVemczDTu7oQh0RzUnHdw+9mMRSD+l0SYxiStLGvYx6pGSW1YfxNwgiiJYMshwzo +hTSYXprlQu7ziEVv1vbSpEbfBE1vnI7zoeejLNT3233z75r2JpjVAC3TeTzCriNG +NTaT96hVKgtPm5U/krGI3N1LYFMcVKzGu3/FzB2CR8mi4c2n4xhR6STqppDLl9Op +Omp2SA7a/8mhgKLTCEE2PbTl35adLvF97JVIk/U4lYdFnspj4tTqHPrAcieDiz9q +DtV0CTQ8CtaznQGYfmDbMl++3tV5NB5tolZS1IRjTmh06q0pm15SaaHFb/YzvfAZ +UXdYAkKdrHKmyNyYxkUKkxenDJMWuqhu8LcA+XpI4/4S/UwvGGEIgO4sW7jo/nko +oQqNQKxiKB3VWweOO+hqMZyqU+UyxUclz1UKGxAPPpqDMM8Vv8vJhusNNOawjnv+ +zWMBnJ6arZIRg8Ckm9x1n+5yXvjWMjTrvURk+WKn2JRCC3cweVze3nf8a0XZboYX +kWiByHtz7RzYGEy6kAcBfB8sAG+/lfwYPKOw2XAHp5r2N4IfD7SsSi1nMJch3d1o +XK0Y2bWMjbXqRctJu/Pq7PXAwZLoeuGTzvCGC9DnxEw2zUpIYMXDVYW/5Bl5M7vC +pyfxj96oXur1EGE6k8wkxfxLH9Re4QIDAQABAoICAQCjj/CAX/f/X7MsPYtQa8J1 +Sinbio42/pYmrJPNnBw/FhZxrC7/wucGFlyj9IgWZCEr8Go9SsztkeoNwn2RxJoA +q5xOV7PclX4CLIHUv/0VI8Kz5pi/NgBZMUwm7K8Xna041OI7ECqARCR2LsJ7GasN +uVMVttK6r7uXQmLnNUUydb3ffmI8xjEIQVnfWI74z60mc2+/GcOP5jXeC+/a+DSm +fudYpcAXaXbId24ls5SkTxYzEepYEtQNQFzPXXkah49yN8mpR+c74c805scxjmd9 +Kz9yhYiKwQTvaqKNpQVHmxte0gPC3lJrLPejjDtxIGOyLZw4oaqrBSpDzR9D0PTE +C+BR6VlXpVCTcAoiweuoDIxNTiJ5IbIJme3iMWxsAIJ4n10rSFFl9Cmmqbphp/6/ +XInB0X7Zyr1kBrwf+DH6DJhje5NXgGKVR9oe9jjW5v8V2tg1RrkzNU8iKBSxpvcI +x4mKhhRLYgoq/iNeYBVQrwJYktIbweVCQ5Spj7/20IrMkn3FAmMsZxGMZmLisJ9t +B0vvUkUgWxuJTsPJ2j+ytpGT0E2xIDYCpbG2EopDc8WvHcVNhagBvLC6xIjIKm7N +2zpBU2W3fPNXoToCAmaLDPYeRRpG6XaGFQAfvKUQRLBDGTfQ177qr34UBnmgvxDq +J2gA9rQm3XziLMuSlJexAQKCAQEA+yz49Ah7FFq0QffsoRb0qOJbfcmMGTRkaafb +ztto4EFSnjH2EwoSShu4DfqWw+ws1KxHlItNHHko5pVNpS4lj1OpnobW3QD7kEIV +mYKa3KowYUcCq1Gzq2RNDZqsC2BSXwx1MG0VVKYOahnu5bvzQq2Ft8W7CWBnbTbY +0Jxjs4KaOza+bH7Vfb5Yre0tlW7U5vI/YO8+YKxpxfOU9kVo8ZLQ/9r/YH8nnLa+ +Fd91+WjcUW8CTKU+Oz3lb/Vwcs6YOoAraq/wtOCqWURunBXkQtzOIn0bgBh3WEk1 +EQ+MVDHshlVVjv/rfnL571ZTT1amCJuEIwQRzLSvbso883srMQKCAQEAyLXaG3Pp +LYiRKu7Bqr5PPuqdT72UFabPpfgd5EtcFOL0xUpfRya6HyFdM25FWI8haXeg4e8N +0cIs3gMG+RRgm1xISJIZi92L0Cwj+kLFu2U5SkvAKMqZFh5q350FRi4Bp7ae4YrL +aguWLZCxhznh4D5xQGM6c8ObRfUUEMT+dnLPcj4zn9KHhoUudXjLKjPNw5v6nkbw +xtRdwANlHx/LX/d4+iwt2plDWmT+d2OLvqZcPyyghTMqV45L0p9XAXBsLnz4Zipx +7VJ8iH3jL5oaQ6YAFY+cXIrWBN0q3UYbXdkaA2ve6voioeF3KQNRmU10k7GKNRWl +pRNn62+rAR8isQKCAQAZnPVqFS9P3QwCqiCEMM4UJrkDs7jInTIcIBTnHDKuo5qk +LR4VxPImgnsbWdFj+0J7EXJfMHFVlPlZwiHf1TvZSMPEOaXRdZcxl7uSIuJd3DEA +ynf4NmWm9Zxx5bLjmhfsP1336TfCoQhZQ3m8DZV52C4Jlm1DQIRre6tSYpA8LvZB +UYzLjYeBwhZS7hu24E1vm4ZhASSQQSSsHfGzx1IzSDBt1swx7+V/MpdhrZ7fJxVI +bJSEcllNOzuZViL4Yh7d4FINGBHor/xPDA5ndkgHlXKjy7QxNM1+wEBcFATQVSL0 +c+E8qtY918Wq5VergH9/4zPvSivyfv5gwtjCT24RAoIBABP6HbJb0BqrHB/U0cvn +00Vk3rGAIgwhpUtUrcz6PzkI+enlJCSV0zKkBH3I/Pf6jw3LTWUPgSWemQ6j6H7E +K3VrMvqeKBLGw1K+Afq3yKyFP7WIYqDswV31Oxf0rgC1NY7220uBoAt3CcSRQUo/ +VZ8XN/h7p+a70mmdIhklMlqhxMoPLN48eybFfMFOe5JAw7szfDdiwjZYDti8vcTi +SkDMBeuImCvI025c3QMPEmqwbkAPdg6r8Av06tEU8PkAspPR9ntcwCgp7KE9Pm6P +fQu8qwd6WsrPOswTI2AQyUqHAFLU2sQyj13jbhPT87w5fF/y7NmpxOnwS4igfbnH +2pECggEBALO0FiJClb0GSqMXYJ+nbtRObD4AynYNVMEqYdZu5DBb6vb4T7uumTD5 +I1fKOa5SSELngUj23p2G6sVBsDyDHotGJYJwDGejHOFnEpY+J0Das0pGS40FsFC7 +qABIUaMoLKcIR9Ofcm9uu2n+koNULV2aaXj7A4OYhRCQi2PqiEx1wimdrLfGqTXn +O4rSf826ODch87vuPbfFPCaIFG28R3nByp/ZBH5QNiB3NBmc3A0tiHFnZW3cpOfW +Jm/Vu0PcNVVw32SroS2FCroR7qSWsvt61UzJtliLUiFHoUAxrXXiAxcZW1D2Hmpq +neUhT/t9hHdcMJgoxm2IITf6ip8nTnY= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml new file mode 100644 index 00000000000..163449872be --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml @@ -0,0 +1,36 @@ + + + 8443 + + + + + + + + /etc/clickhouse-server/config.d/server-cert.pem + /etc/clickhouse-server/config.d/server-key.pem + /etc/clickhouse-server/config.d/ca-cert.pem + relaxed + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + \ No newline at end of file diff --git a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml new file mode 100644 index 00000000000..c41776f9e78 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml @@ -0,0 +1,22 @@ + + + + + + client1 + + + + + client2 + client3 + + + + + + + qwe123 + + + diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py new file mode 100644 index 00000000000..74bd08e9b35 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -0,0 +1,235 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import urllib.request, urllib.parse +import ssl +import os.path + +HTTPS_PORT = 8443 +NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "node", + ipv4_address=NODE_IP, + main_configs=[ + "configs/ssl_config.xml", + "certs/server-key.pem", + "certs/server-cert.pem", + "certs/ca-cert.pem", + ], + user_configs=["configs/users_with_ssl_auth.xml"], +) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def get_ssl_context(cert_name): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(cafile=f"{SCRIPT_DIR}/certs/ca-cert.pem") + if cert_name: + context.load_cert_chain( + f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", + f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", + ) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + return context + + +def execute_query_https( + query, user, enable_ssl_auth=True, cert_name=None, password=None +): + url = f"https://{NODE_IP_WITH_HTTPS_PORT}/?query={urllib.parse.quote(query)}" + request = urllib.request.Request(url) + request.add_header("X-ClickHouse-User", user) + if enable_ssl_auth: + request.add_header("X-ClickHouse-SSL-Certificate-Auth", "on") + if password: + request.add_header("X-ClickHouse-Key", password) + response = urllib.request.urlopen( + request, context=get_ssl_context(cert_name) + ).read() + return response.decode("utf-8") + + +def test_https(): + assert ( + execute_query_https("SELECT currentUser()", user="john", cert_name="client1") + == "john\n" + ) + assert ( + execute_query_https("SELECT currentUser()", user="lucy", cert_name="client2") + == "lucy\n" + ) + assert ( + execute_query_https("SELECT currentUser()", user="lucy", cert_name="client3") + == "lucy\n" + ) + + +def test_https_wrong_cert(): + # Wrong certificate: different user's certificate + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="client2") + assert "HTTP Error 403" in str(err.value) + + # Wrong certificate: self-signed certificate. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="wrong") + assert "unknown ca" in str(err.value) + + # No certificate. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john") + assert "HTTP Error 403" in str(err.value) + + # No header enabling SSL authentication. + with pytest.raises(Exception) as err: + execute_query_https( + "SELECT currentUser()", + user="john", + enable_ssl_auth=False, + cert_name="client1", + ) + + +def test_https_non_ssl_auth(): + # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). + # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + ) + == "jane\n" + ) + + # But we still can send a certificate if we want. + assert ( + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="client1", + ) + == "peter\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="client2", + ) + == "peter\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="client3", + ) + == "peter\n" + ) + + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="client1", + ) + == "jane\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="client2", + ) + == "jane\n" + ) + assert ( + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="client3", + ) + == "jane\n" + ) + + # However if we send a certificate it must not be wrong. + with pytest.raises(Exception) as err: + execute_query_https( + "SELECT currentUser()", + user="peter", + enable_ssl_auth=False, + cert_name="wrong", + ) + assert "unknown ca" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https( + "SELECT currentUser()", + user="jane", + enable_ssl_auth=False, + password="qwe123", + cert_name="wrong", + ) + assert "unknown ca" in str(err.value) + + +def test_create_user(): + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") + assert ( + execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") + == "emma\n" + ) + assert ( + instance.query("SHOW CREATE USER emma") + == "CREATE USER emma IDENTIFIED WITH ssl_certificate CN \\'client3\\'\n" + ) + + instance.query("ALTER USER emma IDENTIFIED WITH ssl_certificate CN 'client2'") + assert ( + execute_query_https("SELECT currentUser()", user="emma", cert_name="client2") + == "emma\n" + ) + assert ( + instance.query("SHOW CREATE USER emma") + == "CREATE USER emma IDENTIFIED WITH ssl_certificate CN \\'client2\\'\n" + ) + + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") + assert "HTTP Error 403" in str(err.value) + + assert ( + instance.query("SHOW CREATE USER lucy") + == "CREATE USER lucy IDENTIFIED WITH ssl_certificate CN \\'client2\\', \\'client3\\'\n" + ) + + assert ( + instance.query( + "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" + ) + == 'emma\tssl_certificate\t{"common_names":["client2"]}\n' + 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' + ) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index b0836a38c9e..7f340424ccf 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -5,7 +5,7 @@ from helpers.cluster import ClickHouseCluster from pyhdfs import HdfsClient cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_hdfs=True) +node1 = cluster.add_instance("node1", with_hdfs=True) @pytest.fixture(scope="module") @@ -16,11 +16,13 @@ def started_cluster(): finally: cluster.shutdown() + def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api node1.query("drop table if exists SimpleHDFSStorage SYNC") node1.query( - "create table SimpleHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/simple_storage', 'TSV')") + "create table SimpleHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/simple_storage', 'TSV')" + ) node1.query("insert into SimpleHDFSStorage values (1, 'Mark', 72.53)") assert hdfs_api.read_data("/simple_storage") == "1\tMark\t72.53\n" assert node1.query("select * from SimpleHDFSStorage") == "1\tMark\t72.53\n" @@ -30,13 +32,17 @@ def test_read_write_storage_with_globs(started_cluster): hdfs_api = started_cluster.hdfs_api node1.query( - "create table HDFSStorageWithRange (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1..5}', 'TSV')") + "create table HDFSStorageWithRange (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1..5}', 'TSV')" + ) node1.query( - "create table HDFSStorageWithEnum (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1,2,3,4,5}', 'TSV')") + "create table HDFSStorageWithEnum (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage{1,2,3,4,5}', 'TSV')" + ) node1.query( - "create table HDFSStorageWithQuestionMark (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage?', 'TSV')") + "create table HDFSStorageWithQuestionMark (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage?', 'TSV')" + ) node1.query( - "create table HDFSStorageWithAsterisk (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage*', 'TSV')") + "create table HDFSStorageWithAsterisk (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage*', 'TSV')" + ) for i in ["1", "2", "3"]: hdfs_api.write_data("/storage" + i, i + "\tMark\t72.53\n") @@ -72,23 +78,28 @@ def test_read_write_storage_with_globs(started_cluster): def test_read_write_table(started_cluster): hdfs_api = started_cluster.hdfs_api - data = "1\tSerialize\t555.222\n2\tData\t777.333\n" hdfs_api.write_data("/simple_table_function", data) assert hdfs_api.read_data("/simple_table_function") == data - assert node1.query( - "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')") == data + assert ( + node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')" + ) + == data + ) def test_write_table(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query( - "create table OtherHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/other_storage', 'TSV')") - node1.query("insert into OtherHDFSStorage values (10, 'tomas', 55.55), (11, 'jack', 32.54)") + "create table OtherHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/other_storage', 'TSV')" + ) + node1.query( + "insert into OtherHDFSStorage values (10, 'tomas', 55.55), (11, 'jack', 32.54)" + ) result = "10\ttomas\t55.55\n11\tjack\t32.54\n" assert hdfs_api.read_data("/other_storage") == result @@ -98,117 +109,154 @@ def test_write_table(started_cluster): def test_bad_hdfs_uri(started_cluster): try: node1.query( - "create table BadStorage1 (id UInt32, name String, weight Float64) ENGINE = HDFS('hads:hgsdfs100500:9000/other_storage', 'TSV')") + "create table BadStorage1 (id UInt32, name String, weight Float64) ENGINE = HDFS('hads:hgsdfs100500:9000/other_storage', 'TSV')" + ) except Exception as ex: print(ex) assert "Bad hdfs url" in str(ex) try: node1.query( - "create table BadStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs100500:9000/other_storage', 'TSV')") + "create table BadStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs100500:9000/other_storage', 'TSV')" + ) except Exception as ex: print(ex) assert "Unable to create builder to connect to HDFS" in str(ex) try: node1.query( - "create table BadStorage3 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/<>', 'TSV')") + "create table BadStorage3 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/<>', 'TSV')" + ) except Exception as ex: print(ex) assert "Unable to open HDFS file" in str(ex) + @pytest.mark.timeout(800) def test_globs_in_read_table(started_cluster): hdfs_api = started_cluster.hdfs_api - some_data = "1\tSerialize\t555.222\n2\tData\t777.333\n" globs_dir = "/dir_for_test_with_globs/" - files = ["dir1/dir_dir/file1", "dir2/file2", "simple_table_function", "dir/file", "some_dir/dir1/file", - "some_dir/dir2/file", "some_dir/file", "table1_function", "table2_function", "table3_function"] + files = [ + "dir1/dir_dir/file1", + "dir2/file2", + "simple_table_function", + "dir/file", + "some_dir/dir1/file", + "some_dir/dir2/file", + "some_dir/file", + "table1_function", + "table2_function", + "table3_function", + ] for filename in files: hdfs_api.write_data(globs_dir + filename, some_data) - test_requests = [("dir{1..5}/dir_dir/file1", 1, 1), - ("*_table_functio?", 1, 1), - ("dir/fil?", 1, 1), - ("table{3..8}_function", 1, 1), - ("table{2..8}_function", 2, 2), - ("dir/*", 1, 1), - ("dir/*?*?*?*?*", 1, 1), - ("dir/*?*?*?*?*?*", 0, 0), - ("some_dir/*/file", 2, 1), - ("some_dir/dir?/*", 2, 1), - ("*/*/*", 3, 2), - ("?", 0, 0)] + test_requests = [ + ("dir{1..5}/dir_dir/file1", 1, 1), + ("*_table_functio?", 1, 1), + ("dir/fil?", 1, 1), + ("table{3..8}_function", 1, 1), + ("table{2..8}_function", 2, 2), + ("dir/*", 1, 1), + ("dir/*?*?*?*?*", 1, 1), + ("dir/*?*?*?*?*?*", 0, 0), + ("some_dir/*/file", 2, 1), + ("some_dir/dir?/*", 2, 1), + ("*/*/*", 3, 2), + ("?", 0, 0), + ] for pattern, paths_amount, files_amount in test_requests: - inside_table_func = "'hdfs://hdfs1:9000" + globs_dir + pattern + "', 'TSV', 'id UInt64, text String, number Float64'" + inside_table_func = ( + "'hdfs://hdfs1:9000" + + globs_dir + + pattern + + "', 'TSV', 'id UInt64, text String, number Float64'" + ) print("inside_table_func ", inside_table_func) - assert node1.query("select * from hdfs(" + inside_table_func + ")") == paths_amount * some_data - assert node1.query("select count(distinct _path) from hdfs(" + inside_table_func + ")").rstrip() == str( - paths_amount) - assert node1.query("select count(distinct _file) from hdfs(" + inside_table_func + ")").rstrip() == str( - files_amount) + assert ( + node1.query("select * from hdfs(" + inside_table_func + ")") + == paths_amount * some_data + ) + assert node1.query( + "select count(distinct _path) from hdfs(" + inside_table_func + ")" + ).rstrip() == str(paths_amount) + assert node1.query( + "select count(distinct _file) from hdfs(" + inside_table_func + ")" + ).rstrip() == str(files_amount) def test_read_write_gzip_table(started_cluster): hdfs_api = started_cluster.hdfs_api - data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" hdfs_api.write_gzip_data("/simple_table_function.gz", data) assert hdfs_api.read_gzip_data("/simple_table_function.gz") == data - assert node1.query( - "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64')") == data + assert ( + node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64')" + ) + == data + ) def test_read_write_gzip_table_with_parameter_gzip(started_cluster): hdfs_api = started_cluster.hdfs_api - data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" hdfs_api.write_gzip_data("/simple_table_function", data) assert hdfs_api.read_gzip_data("/simple_table_function") == data - assert node1.query( - "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64', 'gzip')") == data + assert ( + node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64', 'gzip')" + ) + == data + ) def test_read_write_table_with_parameter_none(started_cluster): hdfs_api = started_cluster.hdfs_api - data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" hdfs_api.write_data("/simple_table_function.gz", data) assert hdfs_api.read_data("/simple_table_function.gz") == data - assert node1.query( - "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'none')") == data + assert ( + node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'none')" + ) + == data + ) def test_read_write_gzip_table_with_parameter_auto_gz(started_cluster): hdfs_api = started_cluster.hdfs_api - data = "1\tHello Jessica\t555.222\n2\tI rolled a joint\t777.333\n" hdfs_api.write_gzip_data("/simple_table_function.gz", data) assert hdfs_api.read_gzip_data("/simple_table_function.gz") == data - assert node1.query( - "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'auto')") == data + assert ( + node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function.gz', 'TSV', 'id UInt64, text String, number Float64', 'auto')" + ) + == data + ) def test_write_gz_storage(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query( - "create table GZHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage.gz', 'TSV')") + "create table GZHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/storage.gz', 'TSV')" + ) node1.query("insert into GZHDFSStorage values (1, 'Mark', 72.53)") assert hdfs_api.read_gzip_data("/storage.gz") == "1\tMark\t72.53\n" assert node1.query("select * from GZHDFSStorage") == "1\tMark\t72.53\n" @@ -217,9 +265,9 @@ def test_write_gz_storage(started_cluster): def test_write_gzip_storage(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query( - "create table GZIPHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/gzip_storage', 'TSV', 'gzip')") + "create table GZIPHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/gzip_storage', 'TSV', 'gzip')" + ) node1.query("insert into GZIPHDFSStorage values (1, 'Mark', 72.53)") assert hdfs_api.read_gzip_data("/gzip_storage") == "1\tMark\t72.53\n" assert node1.query("select * from GZIPHDFSStorage") == "1\tMark\t72.53\n" @@ -228,19 +276,26 @@ def test_write_gzip_storage(started_cluster): def test_virtual_columns(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query("create table virtual_cols (id UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/file*', 'TSV')") + node1.query( + "create table virtual_cols (id UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/file*', 'TSV')" + ) hdfs_api.write_data("/file1", "1\n") hdfs_api.write_data("/file2", "2\n") hdfs_api.write_data("/file3", "3\n") expected = "1\tfile1\thdfs://hdfs1:9000//file1\n2\tfile2\thdfs://hdfs1:9000//file2\n3\tfile3\thdfs://hdfs1:9000//file3\n" - assert node1.query("select id, _file as file_name, _path as file_path from virtual_cols order by id") == expected + assert ( + node1.query( + "select id, _file as file_name, _path as file_path from virtual_cols order by id" + ) + == expected + ) def test_read_files_with_spaces(started_cluster): hdfs_api = started_cluster.hdfs_api fs = HdfsClient(hosts=started_cluster.hdfs_ip) - dir = '/test_spaces' + dir = "/test_spaces" exists = fs.exists(dir) if exists: fs.delete(dir, recursive=True) @@ -250,16 +305,18 @@ def test_read_files_with_spaces(started_cluster): hdfs_api.write_data(f"{dir}/test test test 2.txt", "2\n") hdfs_api.write_data(f"{dir}/test test test 3.txt", "3\n") - node1.query(f"create table test (id UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{dir}/test*', 'TSV')") + node1.query( + f"create table test (id UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{dir}/test*', 'TSV')" + ) assert node1.query("select * from test order by id") == "1\n2\n3\n" fs.delete(dir, recursive=True) - def test_truncate_table(started_cluster): hdfs_api = started_cluster.hdfs_api node1.query( - "create table test_truncate (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/tr', 'TSV')") + "create table test_truncate (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/tr', 'TSV')" + ) node1.query("insert into test_truncate values (1, 'Mark', 72.53)") assert hdfs_api.read_data("/tr") == "1\tMark\t72.53\n" assert node1.query("select * from test_truncate") == "1\tMark\t72.53\n" @@ -277,38 +334,60 @@ def test_partition_by(started_cluster): values = "(1, 2, 3), (3, 2, 1), (1, 3, 2)" table_function = f"hdfs('hdfs://hdfs1:9000/{file_name}', 'TSV', '{table_format}')" - node1.query(f"insert into table function {table_function} PARTITION BY {partition_by} values {values}") - result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test_1', 'TSV', '{table_format}')") - assert(result.strip() == "3\t2\t1") - result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test_2', 'TSV', '{table_format}')") - assert(result.strip() == "1\t3\t2") - result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test_3', 'TSV', '{table_format}')") - assert(result.strip() == "1\t2\t3") + node1.query( + f"insert into table function {table_function} PARTITION BY {partition_by} values {values}" + ) + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/test_1', 'TSV', '{table_format}')" + ) + assert result.strip() == "3\t2\t1" + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/test_2', 'TSV', '{table_format}')" + ) + assert result.strip() == "1\t3\t2" + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/test_3', 'TSV', '{table_format}')" + ) + assert result.strip() == "1\t2\t3" file_name = "test2_{_partition_id}" - node1.query(f"create table p(column1 UInt32, column2 UInt32, column3 UInt32) engine = HDFS('hdfs://hdfs1:9000/{file_name}', 'TSV') partition by column3") + node1.query( + f"create table p(column1 UInt32, column2 UInt32, column3 UInt32) engine = HDFS('hdfs://hdfs1:9000/{file_name}', 'TSV') partition by column3" + ) node1.query(f"insert into p values {values}") - result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test2_1', 'TSV', '{table_format}')") - assert(result.strip() == "3\t2\t1") - result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test2_2', 'TSV', '{table_format}')") - assert(result.strip() == "1\t3\t2") - result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test2_3', 'TSV', '{table_format}')") - assert(result.strip() == "1\t2\t3") + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/test2_1', 'TSV', '{table_format}')" + ) + assert result.strip() == "3\t2\t1" + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/test2_2', 'TSV', '{table_format}')" + ) + assert result.strip() == "1\t3\t2" + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/test2_3', 'TSV', '{table_format}')" + ) + assert result.strip() == "1\t2\t3" def test_seekable_formats(started_cluster): hdfs_api = started_cluster.hdfs_api - table_function = f"hdfs('hdfs://hdfs1:9000/parquet', 'Parquet', 'a Int32, b String')" - node1.query(f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000)") + table_function = ( + f"hdfs('hdfs://hdfs1:9000/parquet', 'Parquet', 'a Int32, b String')" + ) + node1.query( + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000)" + ) result = node1.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 table_function = f"hdfs('hdfs://hdfs1:9000/orc', 'ORC', 'a Int32, b String')" - node1.query(f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000)") + node1.query( + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000)" + ) result = node1.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 def test_read_table_with_default(started_cluster): @@ -319,31 +398,41 @@ def test_read_table_with_default(started_cluster): assert hdfs_api.read_data("/simple_table_function") == data output = "n\tm\n100\t200\n" - assert node1.query( - "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSVWithNames', 'n UInt32, m UInt32 DEFAULT n * 2') FORMAT TSVWithNames") == output + assert ( + node1.query( + "select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSVWithNames', 'n UInt32, m UInt32 DEFAULT n * 2') FORMAT TSVWithNames" + ) + == output + ) def test_schema_inference(started_cluster): - node1.query(f"insert into table function hdfs('hdfs://hdfs1:9000/native', 'Native', 'a Int32, b String') SELECT number, randomString(100) FROM numbers(5000000)") + node1.query( + f"insert into table function hdfs('hdfs://hdfs1:9000/native', 'Native', 'a Int32, b String') SELECT number, randomString(100) FROM numbers(5000000)" + ) result = node1.query(f"desc hdfs('hdfs://hdfs1:9000/native', 'Native')") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" - result = node1.query(f"select count(*) from hdfs('hdfs://hdfs1:9000/native', 'Native')") - assert(int(result) == 5000000) + result = node1.query( + f"select count(*) from hdfs('hdfs://hdfs1:9000/native', 'Native')" + ) + assert int(result) == 5000000 - node1.query(f"create table schema_inference engine=HDFS('hdfs://hdfs1:9000/native', 'Native')") + node1.query( + f"create table schema_inference engine=HDFS('hdfs://hdfs1:9000/native', 'Native')" + ) result = node1.query(f"desc schema_inference") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" result = node1.query(f"select count(*) from schema_inference") - assert(int(result) == 5000000) + assert int(result) == 5000000 def test_hdfsCluster(started_cluster): hdfs_api = started_cluster.hdfs_api fs = HdfsClient(hosts=started_cluster.hdfs_ip) - dir = '/test_hdfsCluster' + dir = "/test_hdfsCluster" exists = fs.exists(dir) if exists: fs.delete(dir, recursive=True) @@ -352,31 +441,43 @@ def test_hdfsCluster(started_cluster): hdfs_api.write_data("/test_hdfsCluster/file2", "2\n") hdfs_api.write_data("/test_hdfsCluster/file3", "3\n") - actual = node1.query("select id, _file as file_name, _path as file_path from hdfs('hdfs://hdfs1:9000/test_hdfsCluster/file*', 'TSV', 'id UInt32') order by id") + actual = node1.query( + "select id, _file as file_name, _path as file_path from hdfs('hdfs://hdfs1:9000/test_hdfsCluster/file*', 'TSV', 'id UInt32') order by id" + ) expected = "1\tfile1\thdfs://hdfs1:9000/test_hdfsCluster/file1\n2\tfile2\thdfs://hdfs1:9000/test_hdfsCluster/file2\n3\tfile3\thdfs://hdfs1:9000/test_hdfsCluster/file3\n" assert actual == expected - actual = node1.query("select id, _file as file_name, _path as file_path from hdfsCluster('test_cluster_two_shards', 'hdfs://hdfs1:9000/test_hdfsCluster/file*', 'TSV', 'id UInt32') order by id") + actual = node1.query( + "select id, _file as file_name, _path as file_path from hdfsCluster('test_cluster_two_shards', 'hdfs://hdfs1:9000/test_hdfsCluster/file*', 'TSV', 'id UInt32') order by id" + ) expected = "1\tfile1\thdfs://hdfs1:9000/test_hdfsCluster/file1\n2\tfile2\thdfs://hdfs1:9000/test_hdfsCluster/file2\n3\tfile3\thdfs://hdfs1:9000/test_hdfsCluster/file3\n" assert actual == expected fs.delete(dir, recursive=True) + def test_hdfs_directory_not_exist(started_cluster): - ddl ="create table HDFSStorageWithNotExistDir (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/data/not_eixst', 'TSV')"; + ddl = "create table HDFSStorageWithNotExistDir (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/data/not_eixst', 'TSV')" node1.query(ddl) assert "" == node1.query("select * from HDFSStorageWithNotExistDir") + def test_overwrite(started_cluster): hdfs_api = started_cluster.hdfs_api table_function = f"hdfs('hdfs://hdfs1:9000/data', 'Parquet', 'a Int32, b String')" node1.query(f"create table test_overwrite as {table_function}") - node1.query(f"insert into test_overwrite select number, randomString(100) from numbers(5)") - node1.query_and_get_error(f"insert into test_overwrite select number, randomString(100) FROM numbers(10)") - node1.query(f"insert into test_overwrite select number, randomString(100) from numbers(10) settings hdfs_truncate_on_insert=1") + node1.query( + f"insert into test_overwrite select number, randomString(100) from numbers(5)" + ) + node1.query_and_get_error( + f"insert into test_overwrite select number, randomString(100) FROM numbers(10)" + ) + node1.query( + f"insert into test_overwrite select number, randomString(100) from numbers(10) settings hdfs_truncate_on_insert=1" + ) result = node1.query(f"select count() from test_overwrite") - assert(int(result) == 10) + assert int(result) == 10 def test_multiple_inserts(started_cluster): @@ -384,33 +485,76 @@ def test_multiple_inserts(started_cluster): table_function = f"hdfs('hdfs://hdfs1:9000/data_multiple_inserts', 'Parquet', 'a Int32, b String')" node1.query(f"create table test_multiple_inserts as {table_function}") - node1.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)") - node1.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings hdfs_create_new_file_on_insert=1") - node1.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings hdfs_create_new_file_on_insert=1") + node1.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)" + ) + node1.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings hdfs_create_new_file_on_insert=1" + ) + node1.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings hdfs_create_new_file_on_insert=1" + ) result = node1.query(f"select count() from test_multiple_inserts") - assert(int(result) == 60) + assert int(result) == 60 result = node1.query(f"drop table test_multiple_inserts") table_function = f"hdfs('hdfs://hdfs1:9000/data_multiple_inserts.gz', 'Parquet', 'a Int32, b String')" node1.query(f"create table test_multiple_inserts as {table_function}") - node1.query(f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(10)") - node1.query(f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(20) settings hdfs_create_new_file_on_insert=1") - node1.query(f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(30) settings hdfs_create_new_file_on_insert=1") + node1.query( + f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(10)" + ) + node1.query( + f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(20) settings hdfs_create_new_file_on_insert=1" + ) + node1.query( + f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(30) settings hdfs_create_new_file_on_insert=1" + ) result = node1.query(f"select count() from test_multiple_inserts") - assert(int(result) == 60) + assert int(result) == 60 + - def test_format_detection(started_cluster): - node1.query(f"create table arrow_table (x UInt64) engine=HDFS('hdfs://hdfs1:9000/data.arrow')") + node1.query( + f"create table arrow_table (x UInt64) engine=HDFS('hdfs://hdfs1:9000/data.arrow')" + ) node1.query(f"insert into arrow_table select 1") result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/data.arrow')") - assert(int(result) == 1) + assert int(result) == 1 -if __name__ == '__main__': +def test_schema_inference_with_globs(started_cluster): + node1.query( + f"insert into table function hdfs('hdfs://hdfs1:9000/data1.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL" + ) + node1.query( + f"insert into table function hdfs('hdfs://hdfs1:9000/data2.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select 0" + ) + + result = node1.query(f"desc hdfs('hdfs://hdfs1:9000/data*.jsoncompacteachrow')") + assert result.strip() == "c1\tNullable(Float64)" + + result = node1.query( + f"select * from hdfs('hdfs://hdfs1:9000/data*.jsoncompacteachrow')" + ) + assert sorted(result.split()) == ["0", "\\N"] + + +def test_insert_select_schema_inference(started_cluster): + node1.query( + f"insert into table function hdfs('hdfs://hdfs1:9000/test.native.zst') select toUInt64(1) as x" + ) + + result = node1.query(f"desc hdfs('hdfs://hdfs1:9000/test.native.zst')") + assert result.strip() == "x\tUInt64" + + result = node1.query(f"select * from hdfs('hdfs://hdfs1:9000/test.native.zst')") + assert int(result) == 1 + + +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_kafka/kafka_pb2.py b/tests/integration/test_storage_kafka/kafka_pb2.py index a9dcab1a85a..d29bc7e8541 100644 --- a/tests/integration/test_storage_kafka/kafka_pb2.py +++ b/tests/integration/test_storage_kafka/kafka_pb2.py @@ -3,7 +3,7 @@ import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -14,58 +14,80 @@ from google.protobuf import symbol_database as _symbol_database _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( - name='clickhouse_path/format_schemas/kafka.proto', - package='', - syntax='proto3', + name="clickhouse_path/format_schemas/kafka.proto", + package="", + syntax="proto3", serialized_pb=_b( - '\n*clickhouse_path/format_schemas/kafka.proto\"*\n\x0cKeyValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3') + '\n*clickhouse_path/format_schemas/kafka.proto"*\n\x0cKeyValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3' + ), ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _KEYVALUEPAIR = _descriptor.Descriptor( - name='KeyValuePair', - full_name='KeyValuePair', + name="KeyValuePair", + full_name="KeyValuePair", filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='KeyValuePair.key', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), + name="key", + full_name="KeyValuePair.key", + index=0, + number=1, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( - name='value', full_name='KeyValuePair.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ + name="value", + full_name="KeyValuePair.value", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], + extensions=[], nested_types=[], - enum_types=[ - ], + enum_types=[], options=None, is_extendable=False, - syntax='proto3', + syntax="proto3", extension_ranges=[], - oneofs=[ - ], + oneofs=[], serialized_start=46, serialized_end=88, ) -DESCRIPTOR.message_types_by_name['KeyValuePair'] = _KEYVALUEPAIR +DESCRIPTOR.message_types_by_name["KeyValuePair"] = _KEYVALUEPAIR -KeyValuePair = _reflection.GeneratedProtocolMessageType('KeyValuePair', (_message.Message,), dict( - DESCRIPTOR=_KEYVALUEPAIR, - __module__='clickhouse_path.format_schemas.kafka_pb2' - # @@protoc_insertion_point(class_scope:KeyValuePair) -)) +KeyValuePair = _reflection.GeneratedProtocolMessageType( + "KeyValuePair", + (_message.Message,), + dict( + DESCRIPTOR=_KEYVALUEPAIR, + __module__="clickhouse_path.format_schemas.kafka_pb2" + # @@protoc_insertion_point(class_scope:KeyValuePair) + ), +) _sym_db.RegisterMessage(KeyValuePair) # @@protoc_insertion_point(module_scope) diff --git a/tests/integration/test_storage_kafka/message_with_repeated_pb2.py b/tests/integration/test_storage_kafka/message_with_repeated_pb2.py index 69702307e7f..b0755a121ae 100644 --- a/tests/integration/test_storage_kafka/message_with_repeated_pb2.py +++ b/tests/integration/test_storage_kafka/message_with_repeated_pb2.py @@ -2,177 +2,311 @@ # source: clickhouse_path/format_schemas/message_with_repeated.proto import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor.FileDescriptor( - name='clickhouse_path/format_schemas/message_with_repeated.proto', - package='', - syntax='proto3', - serialized_options=_b('H\001'), - serialized_pb=_b('\n:clickhouse_path/format_schemas/message_with_repeated.proto\"t\n\x07Message\x12\x0c\n\x04tnow\x18\x01 \x01(\r\x12\x0e\n\x06server\x18\x02 \x01(\t\x12\r\n\x05\x63lien\x18\x03 \x01(\t\x12\r\n\x05sPort\x18\x04 \x01(\r\x12\r\n\x05\x63Port\x18\x05 \x01(\r\x12\x0e\n\x01r\x18\x06 \x03(\x0b\x32\x03.dd\x12\x0e\n\x06method\x18\x07 \x01(\t\"J\n\x02\x64\x64\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x63lass\x18\x02 \x01(\r\x12\x0c\n\x04type\x18\x03 \x01(\r\x12\x0b\n\x03ttl\x18\x04 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\x42\x02H\x01\x62\x06proto3') + name="clickhouse_path/format_schemas/message_with_repeated.proto", + package="", + syntax="proto3", + serialized_options=_b("H\001"), + serialized_pb=_b( + '\n:clickhouse_path/format_schemas/message_with_repeated.proto"t\n\x07Message\x12\x0c\n\x04tnow\x18\x01 \x01(\r\x12\x0e\n\x06server\x18\x02 \x01(\t\x12\r\n\x05\x63lien\x18\x03 \x01(\t\x12\r\n\x05sPort\x18\x04 \x01(\r\x12\r\n\x05\x63Port\x18\x05 \x01(\r\x12\x0e\n\x01r\x18\x06 \x03(\x0b\x32\x03.dd\x12\x0e\n\x06method\x18\x07 \x01(\t"J\n\x02\x64\x64\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x63lass\x18\x02 \x01(\r\x12\x0c\n\x04type\x18\x03 \x01(\r\x12\x0b\n\x03ttl\x18\x04 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\x42\x02H\x01\x62\x06proto3' + ), ) - - _MESSAGE = _descriptor.Descriptor( - name='Message', - full_name='Message', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='tnow', full_name='Message.tnow', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='server', full_name='Message.server', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='clien', full_name='Message.clien', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='sPort', full_name='Message.sPort', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='cPort', full_name='Message.cPort', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='r', full_name='Message.r', index=5, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='method', full_name='Message.method', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=62, - serialized_end=178, + name="Message", + full_name="Message", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="tnow", + full_name="Message.tnow", + index=0, + number=1, + type=13, + cpp_type=3, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="server", + full_name="Message.server", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="clien", + full_name="Message.clien", + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="sPort", + full_name="Message.sPort", + index=3, + number=4, + type=13, + cpp_type=3, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="cPort", + full_name="Message.cPort", + index=4, + number=5, + type=13, + cpp_type=3, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="r", + full_name="Message.r", + index=5, + number=6, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="method", + full_name="Message.method", + index=6, + number=7, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=62, + serialized_end=178, ) _DD = _descriptor.Descriptor( - name='dd', - full_name='dd', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='dd.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='class', full_name='dd.class', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='type', full_name='dd.type', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='ttl', full_name='dd.ttl', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='data', full_name='dd.data', index=4, - number=5, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=180, - serialized_end=254, + name="dd", + full_name="dd", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="dd.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="class", + full_name="dd.class", + index=1, + number=2, + type=13, + cpp_type=3, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="type", + full_name="dd.type", + index=2, + number=3, + type=13, + cpp_type=3, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="ttl", + full_name="dd.ttl", + index=3, + number=4, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="data", + full_name="dd.data", + index=4, + number=5, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=180, + serialized_end=254, ) -_MESSAGE.fields_by_name['r'].message_type = _DD -DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE -DESCRIPTOR.message_types_by_name['dd'] = _DD +_MESSAGE.fields_by_name["r"].message_type = _DD +DESCRIPTOR.message_types_by_name["Message"] = _MESSAGE +DESCRIPTOR.message_types_by_name["dd"] = _DD _sym_db.RegisterFileDescriptor(DESCRIPTOR) -Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( - DESCRIPTOR = _MESSAGE, - __module__ = 'clickhouse_path.format_schemas.message_with_repeated_pb2' - # @@protoc_insertion_point(class_scope:Message) - )) +Message = _reflection.GeneratedProtocolMessageType( + "Message", + (_message.Message,), + dict( + DESCRIPTOR=_MESSAGE, + __module__="clickhouse_path.format_schemas.message_with_repeated_pb2" + # @@protoc_insertion_point(class_scope:Message) + ), +) _sym_db.RegisterMessage(Message) -dd = _reflection.GeneratedProtocolMessageType('dd', (_message.Message,), dict( - DESCRIPTOR = _DD, - __module__ = 'clickhouse_path.format_schemas.message_with_repeated_pb2' - # @@protoc_insertion_point(class_scope:dd) - )) +dd = _reflection.GeneratedProtocolMessageType( + "dd", + (_message.Message,), + dict( + DESCRIPTOR=_DD, + __module__="clickhouse_path.format_schemas.message_with_repeated_pb2" + # @@protoc_insertion_point(class_scope:dd) + ), +) _sym_db.RegisterMessage(dd) diff --git a/tests/integration/test_storage_kafka/social_pb2.py b/tests/integration/test_storage_kafka/social_pb2.py index eeba5efc8b1..429572a0b45 100644 --- a/tests/integration/test_storage_kafka/social_pb2.py +++ b/tests/integration/test_storage_kafka/social_pb2.py @@ -6,69 +6,89 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor.FileDescriptor( - name='social.proto', - package='', - syntax='proto3', - serialized_options=None, - serialized_pb=b'\n\x0csocial.proto\"+\n\x04User\x12\x10\n\x08username\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x05\x62\x06proto3' + name="social.proto", + package="", + syntax="proto3", + serialized_options=None, + serialized_pb=b'\n\x0csocial.proto"+\n\x04User\x12\x10\n\x08username\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x05\x62\x06proto3', ) - - _USER = _descriptor.Descriptor( - name='User', - full_name='User', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='username', full_name='User.username', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='timestamp', full_name='User.timestamp', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=16, - serialized_end=59, + name="User", + full_name="User", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="username", + full_name="User.username", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="timestamp", + full_name="User.timestamp", + index=1, + number=2, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=16, + serialized_end=59, ) -DESCRIPTOR.message_types_by_name['User'] = _USER +DESCRIPTOR.message_types_by_name["User"] = _USER _sym_db.RegisterFileDescriptor(DESCRIPTOR) -User = _reflection.GeneratedProtocolMessageType('User', (_message.Message,), { - 'DESCRIPTOR' : _USER, - '__module__' : 'social_pb2' - # @@protoc_insertion_point(class_scope:User) - }) +User = _reflection.GeneratedProtocolMessageType( + "User", + (_message.Message,), + { + "DESCRIPTOR": _USER, + "__module__": "social_pb2" + # @@protoc_insertion_point(class_scope:User) + }, +) _sym_db.RegisterMessage(User) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 8326797f96d..e451e15a5d6 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -13,7 +13,9 @@ import math import avro.schema import avro.io import avro.datafile -from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient +from confluent_kafka.avro.cached_schema_registry_client import ( + CachedSchemaRegistryClient, +) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer import kafka.errors @@ -43,26 +45,33 @@ from . import message_with_repeated_pb2 # TODO: add test for SELECT LIMIT is working. cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs=['configs/kafka.xml', 'configs/named_collection.xml'], - user_configs=['configs/users.xml'], - with_kafka=True, - with_zookeeper=True, # For Replicated Table - macros={"kafka_broker":"kafka1", - "kafka_topic_old":"old", - "kafka_group_name_old":"old", - "kafka_topic_new":"new", - "kafka_group_name_new":"new", - "kafka_client_id":"instance", - "kafka_format_json_each_row":"JSONEachRow"}, - clickhouse_path_dir='clickhouse_path') +instance = cluster.add_instance( + "instance", + main_configs=["configs/kafka.xml", "configs/named_collection.xml"], + user_configs=["configs/users.xml"], + with_kafka=True, + with_zookeeper=True, # For Replicated Table + macros={ + "kafka_broker": "kafka1", + "kafka_topic_old": "old", + "kafka_group_name_old": "old", + "kafka_topic_new": "new", + "kafka_group_name_new": "new", + "kafka_client_id": "instance", + "kafka_format_json_each_row": "JSONEachRow", + }, + clickhouse_path_dir="clickhouse_path", +) def get_kafka_producer(port, serializer, retries): errors = [] for _ in range(retries): try: - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(port), value_serializer=serializer) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(port), + value_serializer=serializer, + ) logging.debug("Kafka Connection establised: localhost:{}".format(port)) return producer except Exception as e: @@ -71,12 +80,30 @@ def get_kafka_producer(port, serializer, retries): raise Exception("Connection not establised, {}".format(errors)) + def producer_serializer(x): return x.encode() if isinstance(x, str) else x -def kafka_create_topic(admin_client, topic_name, num_partitions=1, replication_factor=1, max_retries=50, config=None): - logging.debug(f"Kafka create topic={topic_name}, num_partitions={num_partitions}, replication_factor={replication_factor}") - topics_list = [NewTopic(name=topic_name, num_partitions=num_partitions, replication_factor=replication_factor, topic_configs=config)] + +def kafka_create_topic( + admin_client, + topic_name, + num_partitions=1, + replication_factor=1, + max_retries=50, + config=None, +): + logging.debug( + f"Kafka create topic={topic_name}, num_partitions={num_partitions}, replication_factor={replication_factor}" + ) + topics_list = [ + NewTopic( + name=topic_name, + num_partitions=num_partitions, + replication_factor=replication_factor, + topic_configs=config, + ) + ] retries = 0 while True: try: @@ -91,6 +118,7 @@ def kafka_create_topic(admin_client, topic_name, num_partitions=1, replication_f else: raise + def kafka_delete_topic(admin_client, topic, max_retries=50): result = admin_client.delete_topics([topic]) for (topic, e) in result.topic_error_codes: @@ -111,19 +139,31 @@ def kafka_delete_topic(admin_client, topic, max_retries=50): if retries > max_retries: raise Exception(f"Failed to delete topics {topic}, {result}") + def kafka_produce(kafka_cluster, topic, messages, timestamp=None, retries=15): - logging.debug("kafka_produce server:{}:{} topic:{}".format("localhost", kafka_cluster.kafka_port, topic)) - producer = get_kafka_producer(kafka_cluster.kafka_port, producer_serializer, retries) + logging.debug( + "kafka_produce server:{}:{} topic:{}".format( + "localhost", kafka_cluster.kafka_port, topic + ) + ) + producer = get_kafka_producer( + kafka_cluster.kafka_port, producer_serializer, retries + ) for message in messages: producer.send(topic=topic, value=message, timestamp_ms=timestamp) producer.flush() + ## just to ensure the python client / producer is working properly def kafka_producer_send_heartbeat_msg(max_retries=50): - kafka_produce(kafka_cluster, 'test_heartbeat_topic', ['test'], retries=max_retries) + kafka_produce(kafka_cluster, "test_heartbeat_topic", ["test"], retries=max_retries) -def kafka_consume(kafka_cluster, topic, needDecode = True, timestamp = 0): - consumer = KafkaConsumer(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), auto_offset_reset="earliest") + +def kafka_consume(kafka_cluster, topic, needDecode=True, timestamp=0): + consumer = KafkaConsumer( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), + auto_offset_reset="earliest", + ) consumer.subscribe(topics=(topic)) for toppar, messages in list(consumer.poll(5000).items()): if toppar.topic == topic: @@ -136,22 +176,31 @@ def kafka_consume(kafka_cluster, topic, needDecode = True, timestamp = 0): consumer.unsubscribe() consumer.close() + def kafka_produce_protobuf_messages(kafka_cluster, topic, start_index, num_messages): - data = b'' + data = b"" for i in range(start_index, start_index + num_messages): msg = kafka_pb2.KeyValuePair() msg.key = i msg.value = str(i) serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), value_serializer=producer_serializer) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), + value_serializer=producer_serializer, + ) producer.send(topic=topic, value=data) producer.flush() logging.debug(("Produced {} messages for topic {}".format(num_messages, topic))) -def kafka_produce_protobuf_messages_no_delimeters(kafka_cluster, topic, start_index, num_messages): - data = '' - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + +def kafka_produce_protobuf_messages_no_delimeters( + kafka_cluster, topic, start_index, num_messages +): + data = "" + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) for i in range(start_index, start_index + num_messages): msg = kafka_pb2.KeyValuePair() msg.key = i @@ -161,37 +210,43 @@ def kafka_produce_protobuf_messages_no_delimeters(kafka_cluster, topic, start_in producer.flush() logging.debug("Produced {} messages for topic {}".format(num_messages, topic)) -def kafka_produce_protobuf_social(kafka_cluster,topic, start_index, num_messages): - data = b'' + +def kafka_produce_protobuf_social(kafka_cluster, topic, start_index, num_messages): + data = b"" for i in range(start_index, start_index + num_messages): msg = social_pb2.User() - msg.username='John Doe {}'.format(i) - msg.timestamp=1000000+i + msg.username = "John Doe {}".format(i) + msg.timestamp = 1000000 + i serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), value_serializer=producer_serializer) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), + value_serializer=producer_serializer, + ) producer.send(topic=topic, value=data) producer.flush() logging.debug(("Produced {} messages for topic {}".format(num_messages, topic))) + def avro_message(value): - schema = avro.schema.make_avsc_object({ - 'name': 'row', - 'type': 'record', - 'fields': [ - {'name': 'id', 'type': 'long'}, - {'name': 'blockNo', 'type': 'int'}, - {'name': 'val1', 'type': 'string'}, - {'name': 'val2', 'type': 'float'}, - {'name': 'val3', 'type': 'int'} - ] - }) + schema = avro.schema.make_avsc_object( + { + "name": "row", + "type": "record", + "fields": [ + {"name": "id", "type": "long"}, + {"name": "blockNo", "type": "int"}, + {"name": "val1", "type": "string"}, + {"name": "val2", "type": "float"}, + {"name": "val3", "type": "int"}, + ], + } + ) bytes_writer = io.BytesIO() # writer = avro.io.DatumWriter(schema) # encoder = avro.io.BinaryEncoder(bytes_writer) # writer.write(value, encoder) - # DataFileWrite seems to be mandatory to get schema encoded writer = avro.datafile.DataFileWriter(bytes_writer, avro.io.DatumWriter(), schema) if isinstance(value, list): @@ -206,64 +261,78 @@ def avro_message(value): bytes_writer.close() return raw_bytes + def avro_confluent_message(schema_registry_client, value): # type: (CachedSchemaRegistryClient, dict) -> str serializer = MessageSerializer(schema_registry_client) - schema = avro.schema.make_avsc_object({ - 'name': 'row', - 'type': 'record', - 'fields': [ - {'name': 'id', 'type': 'long'}, - {'name': 'blockNo', 'type': 'int'}, - {'name': 'val1', 'type': 'string'}, - {'name': 'val2', 'type': 'float'}, - {'name': 'val3', 'type': 'int'} - ] - }) - return serializer.encode_record_with_schema('test_subject', schema, value) + schema = avro.schema.make_avsc_object( + { + "name": "row", + "type": "record", + "fields": [ + {"name": "id", "type": "long"}, + {"name": "blockNo", "type": "int"}, + {"name": "val1", "type": "string"}, + {"name": "val2", "type": "float"}, + {"name": "val3", "type": "int"}, + ], + } + ) + return serializer.encode_record_with_schema("test_subject", schema, value) + # Tests + def test_kafka_settings_old_syntax(kafka_cluster): - assert TSV(instance.query("SELECT * FROM system.macros WHERE macro like 'kafka%' ORDER BY macro", - ignore_error=True)) == TSV('''kafka_broker kafka1 + assert TSV( + instance.query( + "SELECT * FROM system.macros WHERE macro like 'kafka%' ORDER BY macro", + ignore_error=True, + ) + ) == TSV( + """kafka_broker kafka1 kafka_client_id instance kafka_format_json_each_row JSONEachRow kafka_group_name_new new kafka_group_name_old old kafka_topic_new new kafka_topic_old old -''') +""" + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_old}', '{kafka_group_name_old}', '{kafka_format_json_each_row}', '\\n') SETTINGS kafka_commit_on_select = 1; - ''') + """ + ) # Don't insert malformed messages since old settings syntax # doesn't support skipping of broken messages. messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'old', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "old", messages) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break kafka_check_result(result, True) - members = describe_consumer_group(kafka_cluster, 'old') - assert members[0]['client_id'] == 'ClickHouse-instance-test-kafka' + members = describe_consumer_group(kafka_cluster, "old") + assert members[0]["client_id"] == "ClickHouse-instance-test-kafka" # text_desc = kafka_cluster.exec_in_container(kafka_cluster.get_container_id('kafka1'),"kafka-consumer-groups --bootstrap-server localhost:9092 --describe --members --group old --verbose")) def test_kafka_settings_new_syntax(kafka_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = '{kafka_broker}:19092', @@ -274,44 +343,58 @@ def test_kafka_settings_new_syntax(kafka_cluster): kafka_commit_on_select = 1, kafka_client_id = '{kafka_client_id} test 1234', kafka_skip_broken_messages = 1; - ''') + """ + ) messages = [] for i in range(25): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'new', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "new", messages) # Insert couple of malformed messages. - kafka_produce(kafka_cluster, 'new', ['}{very_broken_message,']) - kafka_produce(kafka_cluster, 'new', ['}another{very_broken_message,']) + kafka_produce(kafka_cluster, "new", ["}{very_broken_message,"]) + kafka_produce(kafka_cluster, "new", ["}another{very_broken_message,"]) messages = [] for i in range(25, 50): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'new', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "new", messages) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break kafka_check_result(result, True) - members = describe_consumer_group(kafka_cluster, 'new') - assert members[0]['client_id'] == 'instance test 1234' + members = describe_consumer_group(kafka_cluster, "new") + assert members[0]["client_id"] == "instance test 1234" def test_kafka_json_as_string(kafka_cluster): - kafka_produce(kafka_cluster, 'kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', - '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + kafka_produce( + kafka_cluster, + "kafka_json_as_string", + [ + '{"t": 123, "e": {"x": "woof"} }', + "", + '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}', + ], + ) # 'tombstone' record (null value) = marker of deleted record - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(cluster.kafka_port), value_serializer=producer_serializer, key_serializer=producer_serializer) - producer.send(topic='kafka_json_as_string', key='xxx') + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(cluster.kafka_port), + value_serializer=producer_serializer, + key_serializer=producer_serializer, + ) + producer.send(topic="kafka_json_as_string", key="xxx") producer.flush() - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (field String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -320,22 +403,28 @@ def test_kafka_json_as_string(kafka_cluster): kafka_commit_on_select = 1, kafka_format = 'JSONAsString', kafka_flush_interval_ms=1000; - ''') + """ + ) - result = instance.query('SELECT * FROM test.kafka;') - expected = '''\ + result = instance.query("SELECT * FROM test.kafka;") + expected = """\ {"t": 123, "e": {"x": "woof"} } {"t": 124, "e": {"x": "test"} } {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} -''' +""" assert TSV(result) == TSV(expected) assert instance.contains_in_log( - "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: [0-9]*) return no rows") + "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: [0-9]*) return no rows" + ) def test_kafka_formats(kafka_cluster): - schema_registry_client = CachedSchemaRegistryClient('http://localhost:{}'.format(kafka_cluster.schema_registry_port)) - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + schema_registry_client = CachedSchemaRegistryClient( + "http://localhost:{}".format(kafka_cluster.schema_registry_port) + ) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) # data was dumped from clickhouse itself in a following manner # clickhouse-client --format=Native --query='SELECT toInt64(number) as id, toUInt16( intDiv( id, 65536 ) ) as blockNo, reinterpretAsString(19777) as val1, toFloat32(0.5) as val2, toUInt8(1) as val3 from numbers(100) ORDER BY id' | xxd -ps | tr -d '\n' | sed 's/\(..\)/\\x\1/g' @@ -343,25 +432,25 @@ def test_kafka_formats(kafka_cluster): all_formats = { ## Text formats ## # dumped with clickhouse-client ... | perl -pe 's/\n/\\n/; s/\t/\\t/g;' - 'JSONEachRow': { - 'data_sample': [ + "JSONEachRow": { + "data_sample": [ '{"id":"0","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', '{"id":"1","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"2","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"3","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"4","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"5","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"6","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"7","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"8","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"9","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"10","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"11","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"12","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"13","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"14","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"15","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', '{"id":"0","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', ], - 'supports_empty_value': True, + "supports_empty_value": True, }, # JSONAsString doesn't fit to that test, and tested separately - 'JSONCompactEachRow': { - 'data_sample': [ + "JSONCompactEachRow": { + "data_sample": [ '["0", 0, "AM", 0.5, 1]\n', '["1", 0, "AM", 0.5, 1]\n["2", 0, "AM", 0.5, 1]\n["3", 0, "AM", 0.5, 1]\n["4", 0, "AM", 0.5, 1]\n["5", 0, "AM", 0.5, 1]\n["6", 0, "AM", 0.5, 1]\n["7", 0, "AM", 0.5, 1]\n["8", 0, "AM", 0.5, 1]\n["9", 0, "AM", 0.5, 1]\n["10", 0, "AM", 0.5, 1]\n["11", 0, "AM", 0.5, 1]\n["12", 0, "AM", 0.5, 1]\n["13", 0, "AM", 0.5, 1]\n["14", 0, "AM", 0.5, 1]\n["15", 0, "AM", 0.5, 1]\n', '["0", 0, "AM", 0.5, 1]\n', ], - 'supports_empty_value': True, + "supports_empty_value": True, }, - 'JSONCompactEachRowWithNamesAndTypes': { - 'data_sample': [ + "JSONCompactEachRowWithNamesAndTypes": { + "data_sample": [ '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["0", 0, "AM", 0.5, 1]\n', '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["1", 0, "AM", 0.5, 1]\n["2", 0, "AM", 0.5, 1]\n["3", 0, "AM", 0.5, 1]\n["4", 0, "AM", 0.5, 1]\n["5", 0, "AM", 0.5, 1]\n["6", 0, "AM", 0.5, 1]\n["7", 0, "AM", 0.5, 1]\n["8", 0, "AM", 0.5, 1]\n["9", 0, "AM", 0.5, 1]\n["10", 0, "AM", 0.5, 1]\n["11", 0, "AM", 0.5, 1]\n["12", 0, "AM", 0.5, 1]\n["13", 0, "AM", 0.5, 1]\n["14", 0, "AM", 0.5, 1]\n["15", 0, "AM", 0.5, 1]\n', '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["0", 0, "AM", 0.5, 1]\n', @@ -372,11 +461,11 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/IRowInputFormat.cpp:0: DB::IRowInputFormat::generate() @ 0x1de72710 in /usr/bin/clickhouse ], }, - 'TSKV': { - 'data_sample': [ - 'id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', - 'id=1\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=2\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=3\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=4\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=5\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=6\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=7\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=8\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=9\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=10\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=11\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=12\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=13\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=14\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=15\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', - 'id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', + "TSKV": { + "data_sample": [ + "id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n", + "id=1\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=2\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=3\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=4\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=5\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=6\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=7\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=8\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=9\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=10\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=11\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=12\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=13\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=14\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=15\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n", + "id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n", # '' # On empty message exception: Unexpected end of stream while reading key name from TSKV format # /src/Processors/Formats/Impl/TSKVRowInputFormat.cpp:88: DB::readName(DB::ReadBuffer&, StringRef&, std::__1::basic_string, std::__1::allocator >&) @ 0x1df8c098 in /usr/bin/clickhouse @@ -384,24 +473,24 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/IRowInputFormat.cpp:64: DB::IRowInputFormat::generate() @ 0x1de727cf in /usr/bin/clickhouse ], }, - 'CSV': { - 'data_sample': [ + "CSV": { + "data_sample": [ '0,0,"AM",0.5,1\n', '1,0,"AM",0.5,1\n2,0,"AM",0.5,1\n3,0,"AM",0.5,1\n4,0,"AM",0.5,1\n5,0,"AM",0.5,1\n6,0,"AM",0.5,1\n7,0,"AM",0.5,1\n8,0,"AM",0.5,1\n9,0,"AM",0.5,1\n10,0,"AM",0.5,1\n11,0,"AM",0.5,1\n12,0,"AM",0.5,1\n13,0,"AM",0.5,1\n14,0,"AM",0.5,1\n15,0,"AM",0.5,1\n', '0,0,"AM",0.5,1\n', ], - 'supports_empty_value': True, + "supports_empty_value": True, }, - 'TSV': { - 'data_sample': [ - '0\t0\tAM\t0.5\t1\n', - '1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - '0\t0\tAM\t0.5\t1\n', + "TSV": { + "data_sample": [ + "0\t0\tAM\t0.5\t1\n", + "1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "0\t0\tAM\t0.5\t1\n", ], - 'supports_empty_value': True, + "supports_empty_value": True, }, - 'CSVWithNames': { - 'data_sample': [ + "CSVWithNames": { + "data_sample": [ '"id","blockNo","val1","val2","val3"\n0,0,"AM",0.5,1\n', '"id","blockNo","val1","val2","val3"\n1,0,"AM",0.5,1\n2,0,"AM",0.5,1\n3,0,"AM",0.5,1\n4,0,"AM",0.5,1\n5,0,"AM",0.5,1\n6,0,"AM",0.5,1\n7,0,"AM",0.5,1\n8,0,"AM",0.5,1\n9,0,"AM",0.5,1\n10,0,"AM",0.5,1\n11,0,"AM",0.5,1\n12,0,"AM",0.5,1\n13,0,"AM",0.5,1\n14,0,"AM",0.5,1\n15,0,"AM",0.5,1\n', '"id","blockNo","val1","val2","val3"\n0,0,"AM",0.5,1\n', @@ -415,27 +504,27 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, - 'Values': { - 'data_sample': [ + "Values": { + "data_sample": [ "(0,0,'AM',0.5,1)", "(1,0,'AM',0.5,1),(2,0,'AM',0.5,1),(3,0,'AM',0.5,1),(4,0,'AM',0.5,1),(5,0,'AM',0.5,1),(6,0,'AM',0.5,1),(7,0,'AM',0.5,1),(8,0,'AM',0.5,1),(9,0,'AM',0.5,1),(10,0,'AM',0.5,1),(11,0,'AM',0.5,1),(12,0,'AM',0.5,1),(13,0,'AM',0.5,1),(14,0,'AM',0.5,1),(15,0,'AM',0.5,1)", "(0,0,'AM',0.5,1)", ], - 'supports_empty_value': True, + "supports_empty_value": True, }, - 'TSVWithNames': { - 'data_sample': [ - 'id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n', + "TSVWithNames": { + "data_sample": [ + "id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n", ], - 'supports_empty_value': True, + "supports_empty_value": True, }, - 'TSVWithNamesAndTypes': { - 'data_sample': [ - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n', + "TSVWithNamesAndTypes": { + "data_sample": [ + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n", # '', # On empty message exception happens: Cannot parse input: expected '\n' at end of stream. # /src/IO/ReadHelpers.cpp:84: DB::throwAtAssertionFailed(char const*, DB::ReadBuffer&) @ 0x15c8d8ec in /usr/bin/clickhouse @@ -445,23 +534,23 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/IRowInputFormat.cpp:0: DB::IRowInputFormat::generate() @ 0x1de72710 in /usr/bin/clickhouse ], }, - 'CustomSeparated' : { - 'data_sample' : [ - '0\t0\tAM\t0.5\t1\n', - '1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - '0\t0\tAM\t0.5\t1\n', + "CustomSeparated": { + "data_sample": [ + "0\t0\tAM\t0.5\t1\n", + "1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "0\t0\tAM\t0.5\t1\n", ], }, - 'Template' : { - 'data_sample' : [ + "Template": { + "data_sample": [ + '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', + '(id = 1, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 2, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 3, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 4, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 5, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 6, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 7, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 8, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 9, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 10, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 11, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 12, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 13, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 14, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 15, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', - '(id = 1, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 2, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 3, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 4, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 5, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 6, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 7, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 8, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 9, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 10, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 11, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 12, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 13, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 14, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 15, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', - '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', ], - 'extra_settings': ", format_template_row='template_row.format'" + "extra_settings": ", format_template_row='template_row.format'", }, - 'Regexp': { - 'data_sample': [ + "Regexp": { + "data_sample": [ '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', '(id = 1, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 2, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 3, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 4, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 5, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 6, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 7, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 8, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 9, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 10, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 11, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 12, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 13, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 14, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)\n(id = 15, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', '(id = 0, blockNo = 0, val1 = "AM", val2 = 0.5, val3 = 1)', @@ -469,17 +558,16 @@ def test_kafka_formats(kafka_cluster): # On empty message exception happens: Line "" doesn't match the regexp.: (at row 1) # /src/Processors/Formats/Impl/RegexpRowInputFormat.cpp:140: DB::RegexpRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1df82fcb in /usr/bin/clickhouse ], - 'extra_settings': r", format_regexp='\(id = (.+?), blockNo = (.+?), val1 = \"(.+?)\", val2 = (.+?), val3 = (.+?)\)', format_regexp_escaping_rule='Escaped'" + "extra_settings": r", format_regexp='\(id = (.+?), blockNo = (.+?), val1 = \"(.+?)\", val2 = (.+?), val3 = (.+?)\)', format_regexp_escaping_rule='Escaped'", }, - ## BINARY FORMATS # dumped with # clickhouse-client ... | xxd -ps -c 200 | tr -d '\n' | sed 's/\(..\)/\\x\1/g' - 'Native': { - 'data_sample': [ - b'\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01', - b'\x05\x0f\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01', - b'\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01', + "Native": { + "data_sample": [ + b"\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01", + b"\x05\x0f\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01", + b"\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01", # '' # On empty message exception happens: DB::Exception: Attempt to read after eof # /src/IO/VarInt.h:122: DB::throwReadAfterEOF() @ 0x15c34487 in /usr/bin/clickhouse @@ -491,21 +579,21 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, - 'MsgPack': { - 'data_sample': [ - b'\x00\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01', - b'\x01\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x02\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x03\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x04\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x05\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x06\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x07\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x08\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x09\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0a\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0b\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0c\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0d\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0e\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0f\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01', - b'\x00\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01', + "MsgPack": { + "data_sample": [ + b"\x00\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01", + b"\x01\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x02\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x03\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x04\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x05\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x06\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x07\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x08\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x09\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0a\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0b\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0c\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0d\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0e\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01\x0f\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01", + b"\x00\x00\xa2\x41\x4d\xca\x3f\x00\x00\x00\x01", # '' # On empty message exception happens: Unexpected end of file while parsing msgpack object.: (at row 1) # coming from Processors/Formats/Impl/MsgPackRowInputFormat.cpp:170 ], }, - 'RowBinary': { - 'data_sample': [ - b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', + "RowBinary": { + "data_sample": [ + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", # '' # On empty message exception happens: DB::Exception: Cannot read all data. Bytes read: 0. Bytes expected: 8. # /src/IO/ReadBuffer.h:157: DB::ReadBuffer::readStrict(char*, unsigned long) @ 0x15c6894d in /usr/bin/clickhouse @@ -515,11 +603,11 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/Impl/BinaryRowInputFormat.cpp:22: DB::BinaryRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1dea2c0b in /usr/bin/clickhouse ], }, - 'RowBinaryWithNamesAndTypes': { - 'data_sample': [ - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', + "RowBinaryWithNamesAndTypes": { + "data_sample": [ + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", # '' # !!! On empty message segfault: Address not mapped to object # /contrib/FastMemcpy/FastMemcpy.h:666: memcpy_fast @ 0x21742d65 in /usr/bin/clickhouse @@ -530,11 +618,11 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/Impl/BinaryRowInputFormat.cpp:22: DB::BinaryRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1dea2c0b in /usr/bin/clickhouse ], }, - 'Protobuf': { - 'data_sample': [ - b'\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01', - b'\x0d\x08\x01\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x02\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x03\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x04\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x05\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x06\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x07\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x08\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x09\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0a\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0c\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0d\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0e\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0f\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01', - b'\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01', + "Protobuf": { + "data_sample": [ + b"\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01", + b"\x0d\x08\x01\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x02\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x03\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x04\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x05\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x06\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x07\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x08\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x09\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0a\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0c\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0d\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0e\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01\x0d\x08\x0f\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01", + b"\x0b\x1a\x02\x41\x4d\x25\x00\x00\x00\x3f\x28\x01", # '' # On empty message exception: Attempt to read after eof # /src/IO/ReadBuffer.h:184: DB::ReadBuffer::throwReadAfterEOF() @ 0x15c9699b in /usr/bin/clickhouse @@ -543,96 +631,122 @@ def test_kafka_formats(kafka_cluster): # /src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp:25: DB::ProtobufRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1df4cc71 in /usr/bin/clickhouse # /src/Processors/Formats/IRowInputFormat.cpp:64: DB::IRowInputFormat::generate() @ 0x1de727cf in /usr/bin/clickhouse ], - 'extra_settings': ", kafka_schema='test:TestMessage'" + "extra_settings": ", kafka_schema='test:TestMessage'", }, - 'ORC': { - 'data_sample': [ - b'\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', - b'\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x0f\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x7e\x25\x0e\x2e\x46\x43\x21\x46\x4b\x09\xad\x00\x06\x00\x33\x00\x00\x0a\x17\x0a\x03\x00\x00\x00\x12\x10\x08\x0f\x22\x0a\x0a\x02\x41\x4d\x12\x02\x41\x4d\x18\x3c\x50\x00\x3a\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x7e\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x66\x73\x3d\xd3\x00\x06\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x02\x10\x02\x18\x1e\x50\x00\x05\x00\x00\x0c\x00\x2b\x00\x00\x31\x32\x33\x34\x35\x36\x37\x38\x39\x31\x30\x31\x31\x31\x32\x31\x33\x31\x34\x31\x35\x09\x00\x00\x06\x01\x03\x02\x09\x00\x00\xc0\x0e\x00\x00\x07\x00\x00\x42\x00\x80\x05\x00\x00\x41\x4d\x0a\x00\x00\xe3\xe2\x42\x01\x00\x09\x00\x00\xc0\x0e\x02\x00\x05\x00\x00\x0c\x01\x94\x00\x00\x2d\xca\xc1\x0e\x80\x30\x08\x03\xd0\xc1\x60\x2e\xf3\x62\x76\x6a\xe2\x0e\xfe\xff\x57\x5a\x3b\x0f\xe4\x51\xe8\x68\xbd\x5d\x05\xe7\xf8\x34\x40\x3a\x6e\x59\xb1\x64\xe0\x91\xa9\xbf\xb1\x97\xd2\x95\x9d\x1e\xca\x55\x3a\x6d\xb4\xd2\xdd\x0b\x74\x9a\x74\xf7\x12\x39\xbd\x97\x7f\x7c\x06\xbb\xa6\x8d\x97\x17\xb4\x00\x00\xe3\x4a\xe6\x62\xe1\xe0\x0f\x60\xe0\xe2\xe3\xe0\x17\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\xe0\x57\xe2\xe0\x62\x34\x14\x62\xb4\x94\xd0\x02\x8a\xc8\x73\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\xc2\x06\x28\x26\xc4\x25\xca\xc1\x6f\xc4\xcb\xc5\x68\x20\xc4\x6c\xa0\x67\x2a\xc5\x6c\xae\x67\x0a\x14\xe6\x87\x1a\xc6\x24\xc0\x24\x21\x07\x32\x0c\x00\x4a\x01\x00\xe3\x60\x16\x58\xc3\x24\xc5\xcd\xc1\x2c\x30\x89\x51\xc2\x4b\xc1\x57\x83\x5f\x49\x83\x83\x47\x88\x95\x91\x89\x99\x85\x55\x8a\x3d\x29\x27\x3f\x39\xdb\x2f\x5f\x8a\x29\x33\x45\x8a\xa5\x2c\x31\xc7\x10\x4c\x1a\x81\x49\x63\x25\x26\x0e\x46\x20\x66\x07\x63\x36\x0e\x3e\x0d\x26\x03\x10\x9f\xd1\x80\xdf\x8a\x85\x83\x3f\x80\xc1\x8a\x8f\x83\x5f\x88\x8d\x83\x41\x80\x41\x82\x21\x80\x21\x82\xd5\x4a\x80\x83\x5f\x89\x83\x8b\xd1\x50\x88\xd1\x52\x42\x0b\x28\x22\x6f\x25\x04\x14\xe1\xe2\x62\x72\xf4\x15\x02\x62\x09\x1b\xa0\x98\x90\x95\x28\x07\xbf\x11\x2f\x17\xa3\x81\x10\xb3\x81\x9e\xa9\x14\xb3\xb9\x9e\x29\x50\x98\x1f\x6a\x18\x93\x00\x93\x84\x1c\xc8\x30\x87\x09\x7e\x1e\x0c\x00\x08\xa8\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x5d\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', - b'\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', + "ORC": { + "data_sample": [ + b"\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", + b"\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x0f\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x7e\x25\x0e\x2e\x46\x43\x21\x46\x4b\x09\xad\x00\x06\x00\x33\x00\x00\x0a\x17\x0a\x03\x00\x00\x00\x12\x10\x08\x0f\x22\x0a\x0a\x02\x41\x4d\x12\x02\x41\x4d\x18\x3c\x50\x00\x3a\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x7e\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x66\x73\x3d\xd3\x00\x06\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x02\x10\x02\x18\x1e\x50\x00\x05\x00\x00\x0c\x00\x2b\x00\x00\x31\x32\x33\x34\x35\x36\x37\x38\x39\x31\x30\x31\x31\x31\x32\x31\x33\x31\x34\x31\x35\x09\x00\x00\x06\x01\x03\x02\x09\x00\x00\xc0\x0e\x00\x00\x07\x00\x00\x42\x00\x80\x05\x00\x00\x41\x4d\x0a\x00\x00\xe3\xe2\x42\x01\x00\x09\x00\x00\xc0\x0e\x02\x00\x05\x00\x00\x0c\x01\x94\x00\x00\x2d\xca\xc1\x0e\x80\x30\x08\x03\xd0\xc1\x60\x2e\xf3\x62\x76\x6a\xe2\x0e\xfe\xff\x57\x5a\x3b\x0f\xe4\x51\xe8\x68\xbd\x5d\x05\xe7\xf8\x34\x40\x3a\x6e\x59\xb1\x64\xe0\x91\xa9\xbf\xb1\x97\xd2\x95\x9d\x1e\xca\x55\x3a\x6d\xb4\xd2\xdd\x0b\x74\x9a\x74\xf7\x12\x39\xbd\x97\x7f\x7c\x06\xbb\xa6\x8d\x97\x17\xb4\x00\x00\xe3\x4a\xe6\x62\xe1\xe0\x0f\x60\xe0\xe2\xe3\xe0\x17\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\xe0\x57\xe2\xe0\x62\x34\x14\x62\xb4\x94\xd0\x02\x8a\xc8\x73\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\xc2\x06\x28\x26\xc4\x25\xca\xc1\x6f\xc4\xcb\xc5\x68\x20\xc4\x6c\xa0\x67\x2a\xc5\x6c\xae\x67\x0a\x14\xe6\x87\x1a\xc6\x24\xc0\x24\x21\x07\x32\x0c\x00\x4a\x01\x00\xe3\x60\x16\x58\xc3\x24\xc5\xcd\xc1\x2c\x30\x89\x51\xc2\x4b\xc1\x57\x83\x5f\x49\x83\x83\x47\x88\x95\x91\x89\x99\x85\x55\x8a\x3d\x29\x27\x3f\x39\xdb\x2f\x5f\x8a\x29\x33\x45\x8a\xa5\x2c\x31\xc7\x10\x4c\x1a\x81\x49\x63\x25\x26\x0e\x46\x20\x66\x07\x63\x36\x0e\x3e\x0d\x26\x03\x10\x9f\xd1\x80\xdf\x8a\x85\x83\x3f\x80\xc1\x8a\x8f\x83\x5f\x88\x8d\x83\x41\x80\x41\x82\x21\x80\x21\x82\xd5\x4a\x80\x83\x5f\x89\x83\x8b\xd1\x50\x88\xd1\x52\x42\x0b\x28\x22\x6f\x25\x04\x14\xe1\xe2\x62\x72\xf4\x15\x02\x62\x09\x1b\xa0\x98\x90\x95\x28\x07\xbf\x11\x2f\x17\xa3\x81\x10\xb3\x81\x9e\xa9\x14\xb3\xb9\x9e\x29\x50\x98\x1f\x6a\x18\x93\x00\x93\x84\x1c\xc8\x30\x87\x09\x7e\x1e\x0c\x00\x08\xa8\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x5d\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", + b"\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", # '' # On empty message exception: IOError: File size too small, Stack trace (when copying this message, always include the lines below): # /src/Processors/Formats/Impl/ORCBlockInputFormat.cpp:36: DB::ORCBlockInputFormat::generate() @ 0x1df282a6 in /usr/bin/clickhouse # /src/Processors/ISource.cpp:48: DB::ISource::work() @ 0x1dd79737 in /usr/bin/clickhouse ], }, - 'CapnProto': { - 'data_sample': [ - b'\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00', - b'\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00', - b'\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00', + "CapnProto": { + "data_sample": [ + b"\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00", + b"\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00", + b"\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x3f\x01\x00\x00\x00\x1a\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00", # '' # On empty message exception: Cannot read all data. Bytes read: 0. Bytes expected: 4. # /src/IO/ReadBuffer.h:157: DB::ReadBuffer::readStrict(char*, unsigned long) @ 0x15c6894d in /usr/bin/clickhouse # /src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp:212: DB::CapnProtoRowInputFormat::readMessage() @ 0x1ded1cab in /usr/bin/clickhouse # /src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp:241: DB::CapnProtoRowInputFormat::readRow(std::__1::vector::mutable_ptr, std::__1::allocator::mutable_ptr > >&, DB::RowReadExtension&) @ 0x1ded205d in /usr/bin/clickhouse ], - 'extra_settings': ", kafka_schema='test:TestRecordStruct'" + "extra_settings": ", kafka_schema='test:TestRecordStruct'", }, - 'Parquet' : { - 'data_sample': [ - b'\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31', - b'\x50\x41\x52\x31\x15\x04\x15\xf0\x01\x15\x90\x01\x4c\x15\x1e\x15\x04\x12\x00\x00\x78\x04\x01\x00\x09\x01\x00\x02\x09\x07\x04\x00\x03\x0d\x08\x00\x04\x0d\x08\x00\x05\x0d\x08\x00\x06\x0d\x08\x00\x07\x0d\x08\x00\x08\x0d\x08\x00\x09\x0d\x08\x00\x0a\x0d\x08\x00\x0b\x0d\x08\x00\x0c\x0d\x08\x00\x0d\x0d\x08\x3c\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x14\x15\x18\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x24\x04\x05\x10\x32\x54\x76\x98\xba\xdc\x0e\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x1e\x19\x1c\x19\x5c\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\xa6\x06\x16\x1e\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc5\x01\x00\x00\x50\x41\x52\x31', - b'\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31', + "Parquet": { + "data_sample": [ + b"\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31", + b"\x50\x41\x52\x31\x15\x04\x15\xf0\x01\x15\x90\x01\x4c\x15\x1e\x15\x04\x12\x00\x00\x78\x04\x01\x00\x09\x01\x00\x02\x09\x07\x04\x00\x03\x0d\x08\x00\x04\x0d\x08\x00\x05\x0d\x08\x00\x06\x0d\x08\x00\x07\x0d\x08\x00\x08\x0d\x08\x00\x09\x0d\x08\x00\x0a\x0d\x08\x00\x0b\x0d\x08\x00\x0c\x0d\x08\x00\x0d\x0d\x08\x3c\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x14\x15\x18\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x24\x04\x05\x10\x32\x54\x76\x98\xba\xdc\x0e\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x1e\x19\x1c\x19\x5c\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\xa6\x06\x16\x1e\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc5\x01\x00\x00\x50\x41\x52\x31", + b"\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31", ], }, - 'AvroConfluent': { - 'data_sample': [ - avro_confluent_message(schema_registry_client, - {'id': 0, 'blockNo': 0, 'val1': str('AM'), 'val2': 0.5, "val3": 1}), - - b''.join([avro_confluent_message(schema_registry_client, - {'id': id, 'blockNo': 0, 'val1': str('AM'), - 'val2': 0.5, "val3": 1}) for id in range(1, 16)]), - - avro_confluent_message(schema_registry_client, - {'id': 0, 'blockNo': 0, 'val1': str('AM'), 'val2': 0.5, "val3": 1}), + "AvroConfluent": { + "data_sample": [ + avro_confluent_message( + schema_registry_client, + {"id": 0, "blockNo": 0, "val1": str("AM"), "val2": 0.5, "val3": 1}, + ), + b"".join( + [ + avro_confluent_message( + schema_registry_client, + { + "id": id, + "blockNo": 0, + "val1": str("AM"), + "val2": 0.5, + "val3": 1, + }, + ) + for id in range(1, 16) + ] + ), + avro_confluent_message( + schema_registry_client, + {"id": 0, "blockNo": 0, "val1": str("AM"), "val2": 0.5, "val3": 1}, + ), ], - 'extra_settings': ", format_avro_schema_registry_url='http://{}:{}'".format( - kafka_cluster.schema_registry_host, - 8081 + "extra_settings": ", format_avro_schema_registry_url='http://{}:{}'".format( + kafka_cluster.schema_registry_host, 8081 ), - 'supports_empty_value': True, + "supports_empty_value": True, }, - 'Avro': { + "Avro": { # It seems impossible to send more than one avro file per a message # because of nature of Avro: blocks go one after another - 'data_sample': [ - avro_message({'id': 0, 'blockNo': 0, 'val1': str('AM'), 'val2': 0.5, "val3": 1}), - - avro_message([{'id': id, 'blockNo': 0, 'val1': str('AM'), - 'val2': 0.5, "val3": 1} for id in range(1, 16)]), - - avro_message({'id': 0, 'blockNo': 0, 'val1': str('AM'), 'val2': 0.5, "val3": 1}), + "data_sample": [ + avro_message( + {"id": 0, "blockNo": 0, "val1": str("AM"), "val2": 0.5, "val3": 1} + ), + avro_message( + [ + { + "id": id, + "blockNo": 0, + "val1": str("AM"), + "val2": 0.5, + "val3": 1, + } + for id in range(1, 16) + ] + ), + avro_message( + {"id": 0, "blockNo": 0, "val1": str("AM"), "val2": 0.5, "val3": 1} + ), ], - 'supports_empty_value': False, + "supports_empty_value": False, }, - 'Arrow' : { - 'data_sample' : [ - b'\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', - b'\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', - b'\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31', + "Arrow": { + "data_sample": [ + b"\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31", + b"\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31", + b"\x41\x52\x52\x4f\x57\x31\x00\x00\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\x0c\x00\x14\x00\x06\x00\x08\x00\x0c\x00\x10\x00\x0c\x00\x00\x00\x00\x00\x03\x00\x3c\x00\x00\x00\x28\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00\x58\x01\x00\x00\x00\x00\x00\x00\x60\x01\x00\x00\x00\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\x78\x01\x00\x00\x41\x52\x52\x4f\x57\x31", ], }, - 'ArrowStream' : { - 'data_sample' : [ - b'\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00', - b'\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00', - b'\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00', + "ArrowStream": { + "data_sample": [ + b"\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", + b"\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x48\x01\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x78\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x38\x01\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x06\x00\x00\x00\x08\x00\x00\x00\x0a\x00\x00\x00\x0c\x00\x00\x00\x0e\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x14\x00\x00\x00\x16\x00\x00\x00\x18\x00\x00\x00\x1a\x00\x00\x00\x1c\x00\x00\x00\x1e\x00\x00\x00\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x41\x4d\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\xff\xff\xff\xff\x00\x00\x00\x00", + b"\xff\xff\xff\xff\x48\x01\x00\x00\x10\x00\x00\x00\x00\x00\x0a\x00\x0c\x00\x06\x00\x05\x00\x08\x00\x0a\x00\x00\x00\x00\x01\x03\x00\x0c\x00\x00\x00\x08\x00\x08\x00\x00\x00\x04\x00\x08\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xe4\x00\x00\x00\x9c\x00\x00\x00\x6c\x00\x00\x00\x34\x00\x00\x00\x04\x00\x00\x00\x40\xff\xff\xff\x00\x00\x00\x02\x18\x00\x00\x00\x0c\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x72\xff\xff\xff\x08\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x33\x00\x00\x00\x00\x6c\xff\xff\xff\x00\x00\x00\x03\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x06\x00\x06\x00\x00\x00\x00\x00\x01\x00\x04\x00\x00\x00\x76\x61\x6c\x32\x00\x00\x00\x00\xa0\xff\xff\xff\x00\x00\x00\x05\x18\x00\x00\x00\x10\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00\x76\x61\x6c\x31\x00\x00\x00\x00\xcc\xff\xff\xff\x00\x00\x00\x02\x20\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x08\x00\x04\x00\x06\x00\x00\x00\x10\x00\x00\x00\x07\x00\x00\x00\x62\x6c\x6f\x63\x6b\x4e\x6f\x00\x10\x00\x14\x00\x08\x00\x00\x00\x07\x00\x0c\x00\x00\x00\x10\x00\x10\x00\x00\x00\x00\x00\x00\x02\x24\x00\x00\x00\x14\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08\x00\x0c\x00\x08\x00\x07\x00\x08\x00\x00\x00\x00\x00\x00\x01\x40\x00\x00\x00\x02\x00\x00\x00\x69\x64\x00\x00\xff\xff\xff\xff\x58\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x16\x00\x06\x00\x05\x00\x08\x00\x0c\x00\x0c\x00\x00\x00\x00\x03\x03\x00\x18\x00\x00\x00\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x18\x00\x0c\x00\x04\x00\x08\x00\x0a\x00\x00\x00\xcc\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x41\x4d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3f\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", ], }, } for format_name, format_opts in list(all_formats.items()): - logging.debug(('Set up {}'.format(format_name))) - topic_name = 'format_tests_{}'.format(format_name) - data_sample = format_opts['data_sample'] + logging.debug(("Set up {}".format(format_name))) + topic_name = "format_tests_{}".format(format_name) + data_sample = format_opts["data_sample"] data_prefix = [] # prepend empty value when supported - if format_opts.get('supports_empty_value', False): - data_prefix = data_prefix + [''] + if format_opts.get("supports_empty_value", False): + data_prefix = data_prefix + [""] kafka_produce(kafka_cluster, topic_name, data_prefix + data_sample) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka_{format_name}; CREATE TABLE test.kafka_{format_name} ( @@ -652,18 +766,30 @@ def test_kafka_formats(kafka_cluster): CREATE MATERIALIZED VIEW test.kafka_{format_name}_mv Engine=Log AS SELECT *, _topic, _partition, _offset FROM test.kafka_{format_name}; - '''.format(topic_name=topic_name, format_name=format_name, - extra_settings=format_opts.get('extra_settings') or '')) + """.format( + topic_name=topic_name, + format_name=format_name, + extra_settings=format_opts.get("extra_settings") or "", + ) + ) - instance.wait_for_log_line('kafka.*Committed offset [0-9]+.*format_tests_', repetitions=len(all_formats.keys()), look_behind_lines=12000) + instance.wait_for_log_line( + "kafka.*Committed offset [0-9]+.*format_tests_", + repetitions=len(all_formats.keys()), + look_behind_lines=12000, + ) for format_name, format_opts in list(all_formats.items()): - logging.debug(('Checking {}'.format(format_name))) - topic_name = f'format_tests_{format_name}' + logging.debug(("Checking {}".format(format_name))) + topic_name = f"format_tests_{format_name}" # shift offsets by 1 if format supports empty value - offsets = [1, 2, 3] if format_opts.get('supports_empty_value', False) else [0, 1, 2] - result = instance.query('SELECT * FROM test.kafka_{format_name}_mv;'.format(format_name=format_name)) - expected = '''\ + offsets = ( + [1, 2, 3] if format_opts.get("supports_empty_value", False) else [0, 1, 2] + ) + result = instance.query( + "SELECT * FROM test.kafka_{format_name}_mv;".format(format_name=format_name) + ) + expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} 1 0 AM 0.5 1 {topic_name} 0 {offset_1} 2 0 AM 0.5 1 {topic_name} 0 {offset_1} @@ -681,13 +807,21 @@ def test_kafka_formats(kafka_cluster): 14 0 AM 0.5 1 {topic_name} 0 {offset_1} 15 0 AM 0.5 1 {topic_name} 0 {offset_1} 0 0 AM 0.5 1 {topic_name} 0 {offset_2} -'''.format(topic_name=topic_name, offset_0=offsets[0], offset_1=offsets[1], offset_2=offsets[2]) - assert TSV(result) == TSV(expected), 'Proper result for format: {}'.format(format_name) +""".format( + topic_name=topic_name, + offset_0=offsets[0], + offset_1=offsets[1], + offset_2=offsets[2], + ) + assert TSV(result) == TSV(expected), "Proper result for format: {}".format( + format_name + ) kafka_delete_topic(admin_client, topic_name) + # Since everything is async and shaky when receiving messages from Kafka, # we may want to try and check results multiple times in a loop. -def kafka_check_result(result, check=False, ref_file='test_kafka_json.reference'): +def kafka_check_result(result, check=False, ref_file="test_kafka_json.reference"): fpath = p.join(p.dirname(__file__), ref_file) with open(fpath) as reference: if check: @@ -708,7 +842,7 @@ def decode_avro(message): # https://stackoverflow.com/a/57692111/1555175 def describe_consumer_group(kafka_cluster, name): - client = BrokerConnection('localhost', kafka_cluster.kafka_port, socket.AF_INET) + client = BrokerConnection("localhost", kafka_cluster.kafka_port, socket.AF_INET) client.connect_blocking() list_members_in_groups = DescribeGroupsRequest_v1(groups=[name]) @@ -717,25 +851,35 @@ def describe_consumer_group(kafka_cluster, name): for resp, f in client.recv(): f.success(resp) - (error_code, group_id, state, protocol_type, protocol, members) = future.value.groups[0] + ( + error_code, + group_id, + state, + protocol_type, + protocol, + members, + ) = future.value.groups[0] res = [] for member in members: (member_id, client_id, client_host, member_metadata, member_assignment) = member member_info = {} - member_info['member_id'] = member_id - member_info['client_id'] = client_id - member_info['client_host'] = client_host + member_info["member_id"] = member_id + member_info["client_id"] = client_id + member_info["client_host"] = client_host member_topics_assignment = [] - for (topic, partitions) in MemberAssignment.decode(member_assignment).assignment: - member_topics_assignment.append({'topic': topic, 'partitions': partitions}) - member_info['assignment'] = member_topics_assignment + for (topic, partitions) in MemberAssignment.decode( + member_assignment + ).assignment: + member_topics_assignment.append({"topic": topic, "partitions": partitions}) + member_info["assignment"] = member_topics_assignment res.append(member_info) return res # Fixtures + @pytest.fixture(scope="module") def kafka_cluster(): try: @@ -749,7 +893,7 @@ def kafka_cluster(): @pytest.fixture(autouse=True) def kafka_setup_teardown(): - instance.query('DROP DATABASE IF EXISTS test; CREATE DATABASE test;') + instance.query("DROP DATABASE IF EXISTS test; CREATE DATABASE test;") # logging.debug("kafka is available - running test") yield # run test @@ -757,10 +901,18 @@ def kafka_setup_teardown(): # Tests def test_kafka_issue11308(kafka_cluster): # Check that matview does respect Kafka SETTINGS - kafka_produce(kafka_cluster, 'issue11308', ['{"t": 123, "e": {"x": "woof"} }', '{"t": 123, "e": {"x": "woof"} }', - '{"t": 124, "e": {"x": "test"} }']) + kafka_produce( + kafka_cluster, + "issue11308", + [ + '{"t": 123, "e": {"x": "woof"} }', + '{"t": 123, "e": {"x": "woof"} }', + '{"t": 124, "e": {"x": "test"} }', + ], + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.persistent_kafka ( time UInt64, some_string String @@ -783,31 +935,39 @@ def test_kafka_issue11308(kafka_cluster): `t` AS `time`, `e.x` AS `some_string` FROM test.kafka; - ''') + """ + ) - while int(instance.query('SELECT count() FROM test.persistent_kafka')) < 3: + while int(instance.query("SELECT count() FROM test.persistent_kafka")) < 3: time.sleep(1) - result = instance.query('SELECT * FROM test.persistent_kafka ORDER BY time;') + result = instance.query("SELECT * FROM test.persistent_kafka ORDER BY time;") - instance.query(''' + instance.query( + """ DROP TABLE test.persistent_kafka; DROP TABLE test.persistent_kafka_mv; - ''') + """ + ) - expected = '''\ + expected = """\ 123 woof 123 woof 124 test -''' +""" assert TSV(result) == TSV(expected) def test_kafka_issue4116(kafka_cluster): # Check that format_csv_delimiter parameter works now - as part of all available format settings. - kafka_produce(kafka_cluster, 'issue4116', ['1|foo', '2|bar', '42|answer', '100|multi\n101|row\n103|message']) + kafka_produce( + kafka_cluster, + "issue4116", + ["1|foo", "2|bar", "42|answer", "100|multi\n101|row\n103|message"], + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (a UInt64, b String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -817,28 +977,32 @@ def test_kafka_issue4116(kafka_cluster): kafka_format = 'CSV', kafka_row_delimiter = '\\n', format_csv_delimiter = '|'; - ''') + """ + ) - result = instance.query('SELECT * FROM test.kafka ORDER BY a;') + result = instance.query("SELECT * FROM test.kafka ORDER BY a;") - expected = '''\ + expected = """\ 1 foo 2 bar 42 answer 100 multi 101 row 103 message -''' +""" assert TSV(result) == TSV(expected) def test_kafka_consumer_hang(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "consumer_hang" kafka_create_topic(admin_client, topic_name, num_partitions=8) - instance.query(f''' + instance.query( + f""" DROP TABLE IF EXISTS test.kafka; DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; @@ -852,26 +1016,29 @@ def test_kafka_consumer_hang(kafka_cluster): kafka_num_consumers = 8; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = Memory(); CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) - instance.wait_for_log_line('kafka.*Stalled', repetitions=20) + instance.wait_for_log_line("kafka.*Stalled", repetitions=20) # This should trigger heartbeat fail, # which will trigger REBALANCE_IN_PROGRESS, # and which can lead to consumer hang. - kafka_cluster.pause_container('kafka1') - instance.wait_for_log_line('heartbeat error') - kafka_cluster.unpause_container('kafka1') + kafka_cluster.pause_container("kafka1") + instance.wait_for_log_line("heartbeat error") + kafka_cluster.unpause_container("kafka1") # logging.debug("Attempt to drop") - instance.query('DROP TABLE test.kafka') + instance.query("DROP TABLE test.kafka") # kafka_cluster.open_bash_shell('instance') - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) # original problem appearance was a sequence of the following messages in librdkafka logs: # BROKERFAIL -> |ASSIGN| -> REBALANCE_IN_PROGRESS -> "waiting for rebalance_cb" (repeated forever) @@ -879,10 +1046,18 @@ def test_kafka_consumer_hang(kafka_cluster): # from a user perspective: we expect no hanging 'drop' queries # 'dr'||'op' to avoid self matching - assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + assert ( + int( + instance.query( + "select count() from system.processes where position(lower(query),'dr'||'op')>0" + ) + ) + == 0 + ) # cleanup unread messages so kafka will not wait reading consumers to delete topic - instance.query(f''' + instance.query( + f""" CREATE TABLE test.kafka (key UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -891,21 +1066,25 @@ def test_kafka_consumer_hang(kafka_cluster): kafka_group_name = '{topic_name}', kafka_format = 'JSONEachRow', kafka_num_consumers = 8; - ''') + """ + ) - num_read = int(instance.query('SELECT count() FROM test.kafka')) + num_read = int(instance.query("SELECT count() FROM test.kafka")) logging.debug(f"read {num_read} from {topic_name} before delete") - instance.query('DROP TABLE test.kafka') + instance.query("DROP TABLE test.kafka") kafka_delete_topic(admin_client, topic_name) def test_kafka_consumer_hang2(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "consumer_hang2" kafka_create_topic(admin_client, topic_name) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -923,14 +1102,15 @@ def test_kafka_consumer_hang2(kafka_cluster): kafka_commit_on_select = 1, kafka_group_name = 'consumer_hang2', kafka_format = 'JSONEachRow'; - ''') + """ + ) # first consumer subscribe the topic, try to poll some data, and go to rest - instance.query('SELECT * FROM test.kafka') + instance.query("SELECT * FROM test.kafka") # second consumer do the same leading to rebalance in the first # consumer, try to poll some data - instance.query('SELECT * FROM test.kafka2') + instance.query("SELECT * FROM test.kafka2") # echo 'SELECT * FROM test.kafka; SELECT * FROM test.kafka2; DROP TABLE test.kafka;' | clickhouse client -mn & # kafka_cluster.open_bash_shell('instance') @@ -939,22 +1119,30 @@ def test_kafka_consumer_hang2(kafka_cluster): # one of those queries was failing because of # https://github.com/edenhill/librdkafka/issues/2077 # https://github.com/edenhill/librdkafka/issues/2898 - instance.query('DROP TABLE test.kafka') - instance.query('DROP TABLE test.kafka2') + instance.query("DROP TABLE test.kafka") + instance.query("DROP TABLE test.kafka2") # from a user perspective: we expect no hanging 'drop' queries # 'dr'||'op' to avoid self matching - assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 + assert ( + int( + instance.query( + "select count() from system.processes where position(lower(query),'dr'||'op')>0" + ) + ) + == 0 + ) kafka_delete_topic(admin_client, topic_name) def test_kafka_csv_with_delimiter(kafka_cluster): messages = [] for i in range(50): - messages.append('{i}, {i}'.format(i=i)) - kafka_produce(kafka_cluster, 'csv', messages) + messages.append("{i}, {i}".format(i=i)) + kafka_produce(kafka_cluster, "csv", messages) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -962,11 +1150,12 @@ def test_kafka_csv_with_delimiter(kafka_cluster): kafka_commit_on_select = 1, kafka_group_name = 'csv', kafka_format = 'CSV'; - ''') + """ + ) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break @@ -976,10 +1165,11 @@ def test_kafka_csv_with_delimiter(kafka_cluster): def test_kafka_tsv_with_delimiter(kafka_cluster): messages = [] for i in range(50): - messages.append('{i}\t{i}'.format(i=i)) - kafka_produce(kafka_cluster, 'tsv', messages) + messages.append("{i}\t{i}".format(i=i)) + kafka_produce(kafka_cluster, "tsv", messages) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -987,11 +1177,12 @@ def test_kafka_tsv_with_delimiter(kafka_cluster): kafka_commit_on_select = 1, kafka_group_name = 'tsv', kafka_format = 'TSV'; - ''') + """ + ) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break @@ -999,11 +1190,14 @@ def test_kafka_tsv_with_delimiter(kafka_cluster): def test_kafka_select_empty(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "empty" kafka_create_topic(admin_client, topic_name) - instance.query(f''' + instance.query( + f""" CREATE TABLE test.kafka (key UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1012,24 +1206,26 @@ def test_kafka_select_empty(kafka_cluster): kafka_group_name = '{topic_name}', kafka_format = 'TSV', kafka_row_delimiter = '\\n'; - ''') + """ + ) - assert int(instance.query('SELECT count() FROM test.kafka')) == 0 + assert int(instance.query("SELECT count() FROM test.kafka")) == 0 kafka_delete_topic(admin_client, topic_name) def test_kafka_json_without_delimiter(kafka_cluster): - messages = '' + messages = "" for i in range(25): - messages += json.dumps({'key': i, 'value': i}) + '\n' - kafka_produce(kafka_cluster, 'json', [messages]) + messages += json.dumps({"key": i, "value": i}) + "\n" + kafka_produce(kafka_cluster, "json", [messages]) - messages = '' + messages = "" for i in range(25, 50): - messages += json.dumps({'key': i, 'value': i}) + '\n' - kafka_produce(kafka_cluster, 'json', [messages]) + messages += json.dumps({"key": i, "value": i}) + "\n" + kafka_produce(kafka_cluster, "json", [messages]) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1037,11 +1233,12 @@ def test_kafka_json_without_delimiter(kafka_cluster): kafka_group_name = 'json', kafka_commit_on_select = 1, kafka_format = 'JSONEachRow'; - ''') + """ + ) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break @@ -1049,11 +1246,12 @@ def test_kafka_json_without_delimiter(kafka_cluster): def test_kafka_protobuf(kafka_cluster): - kafka_produce_protobuf_messages(kafka_cluster, 'pb', 0, 20) - kafka_produce_protobuf_messages(kafka_cluster, 'pb', 20, 1) - kafka_produce_protobuf_messages(kafka_cluster, 'pb', 21, 29) + kafka_produce_protobuf_messages(kafka_cluster, "pb", 0, 20) + kafka_produce_protobuf_messages(kafka_cluster, "pb", 20, 1) + kafka_produce_protobuf_messages(kafka_cluster, "pb", 21, 29) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1062,11 +1260,12 @@ def test_kafka_protobuf(kafka_cluster): kafka_format = 'Protobuf', kafka_commit_on_select = 1, kafka_schema = 'kafka.proto:KeyValuePair'; - ''') + """ + ) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break @@ -1074,12 +1273,19 @@ def test_kafka_protobuf(kafka_cluster): def test_kafka_string_field_on_first_position_in_protobuf(kafka_cluster): -# https://github.com/ClickHouse/ClickHouse/issues/12615 - kafka_produce_protobuf_social(kafka_cluster, 'string_field_on_first_position_in_protobuf', 0, 20) - kafka_produce_protobuf_social(kafka_cluster, 'string_field_on_first_position_in_protobuf', 20, 1) - kafka_produce_protobuf_social(kafka_cluster, 'string_field_on_first_position_in_protobuf', 21, 29) + # https://github.com/ClickHouse/ClickHouse/issues/12615 + kafka_produce_protobuf_social( + kafka_cluster, "string_field_on_first_position_in_protobuf", 0, 20 + ) + kafka_produce_protobuf_social( + kafka_cluster, "string_field_on_first_position_in_protobuf", 20, 1 + ) + kafka_produce_protobuf_social( + kafka_cluster, "string_field_on_first_position_in_protobuf", 21, 29 + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka ( username String, timestamp Int32 @@ -1091,10 +1297,11 @@ SETTINGS kafka_format = 'Protobuf', kafka_commit_on_select = 1, kafka_schema = 'social:User'; - ''') + """ + ) - result = instance.query('SELECT * FROM test.kafka', ignore_error=True) - expected = '''\ + result = instance.query("SELECT * FROM test.kafka", ignore_error=True) + expected = """\ John Doe 0 1000000 John Doe 1 1000001 John Doe 2 1000002 @@ -1145,11 +1352,13 @@ John Doe 46 1000046 John Doe 47 1000047 John Doe 48 1000048 John Doe 49 1000049 -''' +""" assert TSV(result) == TSV(expected) + def test_kafka_protobuf_no_delimiter(kafka_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1158,21 +1367,29 @@ def test_kafka_protobuf_no_delimiter(kafka_cluster): kafka_format = 'ProtobufSingle', kafka_commit_on_select = 1, kafka_schema = 'kafka.proto:KeyValuePair'; - ''') + """ + ) - kafka_produce_protobuf_messages_no_delimeters(kafka_cluster, 'pb_no_delimiter', 0, 20) - kafka_produce_protobuf_messages_no_delimeters(kafka_cluster, 'pb_no_delimiter', 20, 1) - kafka_produce_protobuf_messages_no_delimeters(kafka_cluster, 'pb_no_delimiter', 21, 29) + kafka_produce_protobuf_messages_no_delimeters( + kafka_cluster, "pb_no_delimiter", 0, 20 + ) + kafka_produce_protobuf_messages_no_delimeters( + kafka_cluster, "pb_no_delimiter", 20, 1 + ) + kafka_produce_protobuf_messages_no_delimeters( + kafka_cluster, "pb_no_delimiter", 21, 29 + ) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break kafka_check_result(result, True) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka_writer (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1181,26 +1398,29 @@ def test_kafka_protobuf_no_delimiter(kafka_cluster): kafka_format = 'ProtobufSingle', kafka_commit_on_select = 1, kafka_schema = 'kafka.proto:KeyValuePair'; - ''') + """ + ) - instance.query("INSERT INTO test.kafka_writer VALUES (13,'Friday'),(42,'Answer to the Ultimate Question of Life, the Universe, and Everything'), (110, 'just a number')") + instance.query( + "INSERT INTO test.kafka_writer VALUES (13,'Friday'),(42,'Answer to the Ultimate Question of Life, the Universe, and Everything'), (110, 'just a number')" + ) time.sleep(1) result = instance.query("SELECT * FROM test.kafka ORDER BY key", ignore_error=True) - expected = '''\ + expected = """\ 13 Friday 42 Answer to the Ultimate Question of Life, the Universe, and Everything 110 just a number -''' +""" assert TSV(result) == TSV(expected) - def test_kafka_materialized_view(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -1215,38 +1435,44 @@ def test_kafka_materialized_view(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'mv', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "mv", messages) while True: - result = instance.query('SELECT * FROM test.view') + result = instance.query("SELECT * FROM test.view") if kafka_check_result(result): break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) kafka_check_result(result, True) def test_kafka_recreate_kafka_table(kafka_cluster): - ''' - Checks that materialized view work properly after dropping and recreating the Kafka table. - ''' + """ + Checks that materialized view work properly after dropping and recreating the Kafka table. + """ # line for backporting: # admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "recreate_kafka_table" kafka_create_topic(admin_client, topic_name, num_partitions=6) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -1264,22 +1490,30 @@ def test_kafka_recreate_kafka_table(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) messages = [] for i in range(120): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster,'recreate_kafka_table', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "recreate_kafka_table", messages) - instance.wait_for_log_line('kafka.*Committed offset [0-9]+.*recreate_kafka_table', repetitions=6, look_behind_lines=100) + instance.wait_for_log_line( + "kafka.*Committed offset [0-9]+.*recreate_kafka_table", + repetitions=6, + look_behind_lines=100, + ) - instance.query(''' + instance.query( + """ DROP TABLE test.kafka; - ''') + """ + ) - kafka_produce(kafka_cluster,'recreate_kafka_table', messages) + kafka_produce(kafka_cluster, "recreate_kafka_table", messages) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1289,17 +1523,24 @@ def test_kafka_recreate_kafka_table(kafka_cluster): kafka_num_consumers = 6, kafka_flush_interval_ms = 1000, kafka_skip_broken_messages = 1048577; - ''') + """ + ) - instance.wait_for_log_line('kafka.*Committed offset [0-9]+.*recreate_kafka_table', repetitions=6, look_behind_lines=100) + instance.wait_for_log_line( + "kafka.*Committed offset [0-9]+.*recreate_kafka_table", + repetitions=6, + look_behind_lines=100, + ) # data was not flushed yet (it will be flushed 7.5 sec after creating MV) assert int(instance.query("SELECT count() FROM test.view")) == 240 - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) kafka_delete_topic(admin_client, topic_name) @@ -1326,28 +1567,33 @@ def test_librdkafka_compression(kafka_cluster): """ - supported_compression_types = ['gzip', 'snappy', 'lz4', 'zstd', 'uncompressed'] + supported_compression_types = ["gzip", "snappy", "lz4", "zstd", "uncompressed"] messages = [] expected = [] - value = 'foobarbaz'*10 + value = "foobarbaz" * 10 number_of_messages = 50 for i in range(number_of_messages): - messages.append(json.dumps({'key': i, 'value': value})) - expected.append(f'{i}\t{value}') + messages.append(json.dumps({"key": i, "value": value})) + expected.append(f"{i}\t{value}") - expected = '\n'.join(expected) + expected = "\n".join(expected) for compression_type in supported_compression_types: - logging.debug(('Check compression {}'.format(compression_type))) + logging.debug(("Check compression {}".format(compression_type))) - topic_name = 'test_librdkafka_compression_{}'.format(compression_type) - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + topic_name = "test_librdkafka_compression_{}".format(compression_type) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) - kafka_create_topic(admin_client, topic_name, config={'compression.type': compression_type}) + kafka_create_topic( + admin_client, topic_name, config={"compression.type": compression_type} + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1357,22 +1603,26 @@ def test_librdkafka_compression(kafka_cluster): kafka_flush_interval_ms = 1000; CREATE MATERIALIZED VIEW test.consumer Engine=Log AS SELECT * FROM test.kafka; - '''.format(topic_name=topic_name) ) + """.format( + topic_name=topic_name + ) + ) kafka_produce(kafka_cluster, topic_name, messages) instance.wait_for_log_line("Committed offset {}".format(number_of_messages)) - result = instance.query('SELECT * FROM test.consumer') + result = instance.query("SELECT * FROM test.consumer") assert TSV(result) == TSV(expected) - instance.query('DROP TABLE test.kafka SYNC') - instance.query('DROP TABLE test.consumer SYNC') + instance.query("DROP TABLE test.kafka SYNC") + instance.query("DROP TABLE test.consumer SYNC") kafka_delete_topic(admin_client, topic_name) def test_kafka_materialized_view_with_subquery(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -1387,28 +1637,32 @@ def test_kafka_materialized_view_with_subquery(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM (SELECT * FROM test.kafka); - ''') + """ + ) messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'mvsq', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "mvsq", messages) while True: - result = instance.query('SELECT * FROM test.view') + result = instance.query("SELECT * FROM test.view") if kafka_check_result(result): break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) kafka_check_result(result, True) def test_kafka_many_materialized_views(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view1; DROP TABLE IF EXISTS test.view2; DROP TABLE IF EXISTS test.consumer1; @@ -1430,25 +1684,28 @@ def test_kafka_many_materialized_views(kafka_cluster): SELECT * FROM test.kafka; CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS SELECT * FROM test.kafka; - ''') + """ + ) messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'mmv', messages) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "mmv", messages) while True: - result1 = instance.query('SELECT * FROM test.view1') - result2 = instance.query('SELECT * FROM test.view2') + result1 = instance.query("SELECT * FROM test.view1") + result2 = instance.query("SELECT * FROM test.view2") if kafka_check_result(result1) and kafka_check_result(result2): break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer1; DROP TABLE test.consumer2; DROP TABLE test.view1; DROP TABLE test.view2; - ''') + """ + ) kafka_check_result(result1, True) kafka_check_result(result2, True) @@ -1458,10 +1715,14 @@ def test_kafka_flush_on_big_message(kafka_cluster): # Create batchs of messages of size ~100Kb kafka_messages = 1000 batch_messages = 1000 - messages = [json.dumps({'key': i, 'value': 'x' * 100}) * batch_messages for i in range(kafka_messages)] - kafka_produce(kafka_cluster, 'flush', messages) + messages = [ + json.dumps({"key": i, "value": "x" * 100}) * batch_messages + for i in range(kafka_messages) + ] + kafka_produce(kafka_cluster, "flush", messages) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value String) @@ -1476,42 +1737,52 @@ def test_kafka_flush_on_big_message(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) - client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) received = False while not received: try: - offsets = client.list_consumer_group_offsets('flush') + offsets = client.list_consumer_group_offsets("flush") for topic, offset in list(offsets.items()): - if topic.topic == 'flush' and offset.offset == kafka_messages: + if topic.topic == "flush" and offset.offset == kafka_messages: received = True break except kafka.errors.GroupCoordinatorNotAvailableError: continue while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") if int(result) == kafka_messages * batch_messages: break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) - assert int(result) == kafka_messages * batch_messages, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == kafka_messages * batch_messages + ), "ClickHouse lost some messages: {}".format(result) def test_kafka_virtual_columns(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_config = { # default retention, since predefined timestamp_ms is used. - 'retention.ms': '-1', + "retention.ms": "-1", } kafka_create_topic(admin_client, "virt1", config=topic_config) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1519,38 +1790,43 @@ def test_kafka_virtual_columns(kafka_cluster): kafka_group_name = 'virt1', kafka_commit_on_select = 1, kafka_format = 'JSONEachRow'; - ''') + """ + ) - messages = '' + messages = "" for i in range(25): - messages += json.dumps({'key': i, 'value': i}) + '\n' - kafka_produce(kafka_cluster, 'virt1', [messages], 0) + messages += json.dumps({"key": i, "value": i}) + "\n" + kafka_produce(kafka_cluster, "virt1", [messages], 0) - messages = '' + messages = "" for i in range(25, 50): - messages += json.dumps({'key': i, 'value': i}) + '\n' - kafka_produce(kafka_cluster, 'virt1', [messages], 0) + messages += json.dumps({"key": i, "value": i}) + "\n" + kafka_produce(kafka_cluster, "virt1", [messages], 0) - result = '' + result = "" while True: result += instance.query( - '''SELECT _key, key, _topic, value, _offset, _partition, _timestamp = 0 ? '0000-00-00 00:00:00' : toString(_timestamp) AS _timestamp FROM test.kafka''', - ignore_error=True) - if kafka_check_result(result, False, 'test_kafka_virtual1.reference'): + """SELECT _key, key, _topic, value, _offset, _partition, _timestamp = 0 ? '0000-00-00 00:00:00' : toString(_timestamp) AS _timestamp FROM test.kafka""", + ignore_error=True, + ) + if kafka_check_result(result, False, "test_kafka_virtual1.reference"): break - kafka_check_result(result, True, 'test_kafka_virtual1.reference') + kafka_check_result(result, True, "test_kafka_virtual1.reference") def test_kafka_virtual_columns_with_materialized_view(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_config = { # default retention, since predefined timestamp_ms is used. - 'retention.ms': '-1', + "retention.ms": "-1", } kafka_create_topic(admin_client, "virt2", config=topic_config) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -1565,31 +1841,38 @@ def test_kafka_virtual_columns_with_materialized_view(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT *, _key as kafka_key, _topic as topic, _offset as offset, _partition as partition, _timestamp = 0 ? '0000-00-00 00:00:00' : toString(_timestamp) as timestamp FROM test.kafka; - ''') + """ + ) messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) - kafka_produce(kafka_cluster, 'virt2', messages, 0) + messages.append(json.dumps({"key": i, "value": i})) + kafka_produce(kafka_cluster, "virt2", messages, 0) - sql = 'SELECT kafka_key, key, topic, value, offset, partition, timestamp FROM test.view ORDER BY kafka_key, key' + sql = "SELECT kafka_key, key, topic, value, offset, partition, timestamp FROM test.view ORDER BY kafka_key, key" result = instance.query(sql) iterations = 0 - while not kafka_check_result(result, False, 'test_kafka_virtual2.reference') and iterations < 10: + while ( + not kafka_check_result(result, False, "test_kafka_virtual2.reference") + and iterations < 10 + ): time.sleep(3) iterations += 1 result = instance.query(sql) - kafka_check_result(result, True, 'test_kafka_virtual2.reference') + kafka_check_result(result, True, "test_kafka_virtual2.reference") - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) def test_kafka_insert(kafka_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1598,35 +1881,37 @@ def test_kafka_insert(kafka_cluster): kafka_format = 'TSV', kafka_commit_on_select = 1, kafka_row_delimiter = '\\n'; - ''') + """ + ) values = [] for i in range(50): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) while True: try: instance.query("INSERT INTO test.kafka VALUES {}".format(values)) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise messages = [] while True: - messages.extend(kafka_consume(kafka_cluster, 'insert1')) + messages.extend(kafka_consume(kafka_cluster, "insert1")) if len(messages) == 50: break - result = '\n'.join(messages) + result = "\n".join(messages) kafka_check_result(result, True) def test_kafka_produce_consume(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -1641,7 +1926,8 @@ def test_kafka_produce_consume(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) messages_num = 10000 @@ -1649,14 +1935,14 @@ def test_kafka_produce_consume(kafka_cluster): values = [] for i in range(messages_num): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) while True: try: instance.query("INSERT INTO test.kafka VALUES {}".format(values)) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise @@ -1670,24 +1956,29 @@ def test_kafka_produce_consume(kafka_cluster): thread.start() while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") time.sleep(1) if int(result) == messages_num * threads_num: break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) for thread in threads: thread.join() - assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result) def test_kafka_commit_on_block_write(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -1703,7 +1994,8 @@ def test_kafka_commit_on_block_write(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) cancel = threading.Event() @@ -1713,23 +2005,26 @@ def test_kafka_commit_on_block_write(kafka_cluster): while not cancel.is_set(): messages = [] for _ in range(101): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 - kafka_produce(kafka_cluster, 'block', messages) + kafka_produce(kafka_cluster, "block", messages) kafka_thread = threading.Thread(target=produce) kafka_thread.start() - while int(instance.query('SELECT count() FROM test.view')) == 0: + while int(instance.query("SELECT count() FROM test.view")) == 0: time.sleep(1) cancel.set() - instance.query(''' + instance.query( + """ DROP TABLE test.kafka; - ''') + """ + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1738,34 +2033,40 @@ def test_kafka_commit_on_block_write(kafka_cluster): kafka_format = 'JSONEachRow', kafka_max_block_size = 100, kafka_row_delimiter = '\\n'; - ''') + """ + ) - while int(instance.query('SELECT uniqExact(key) FROM test.view')) < i[0]: + while int(instance.query("SELECT uniqExact(key) FROM test.view")) < i[0]: time.sleep(1) - result = int(instance.query('SELECT count() == uniqExact(key) FROM test.view')) + result = int(instance.query("SELECT count() == uniqExact(key) FROM test.view")) - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) kafka_thread.join() - assert result == 1, 'Messages from kafka get duplicated!' + assert result == 1, "Messages from kafka get duplicated!" def test_kafka_virtual_columns2(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_config = { # default retention, since predefined timestamp_ms is used. - 'retention.ms': '-1', + "retention.ms": "-1", } kafka_create_topic(admin_client, "virt2_0", num_partitions=2, config=topic_config) kafka_create_topic(admin_client, "virt2_1", num_partitions=2, config=topic_config) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -1776,40 +2077,101 @@ def test_kafka_virtual_columns2(kafka_cluster): CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp), toUnixTimestamp64Milli(_timestamp_ms), _headers.name, _headers.value FROM test.kafka; - ''') + """ + ) - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(cluster.kafka_port), value_serializer=producer_serializer, key_serializer=producer_serializer) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(cluster.kafka_port), + value_serializer=producer_serializer, + key_serializer=producer_serializer, + ) - producer.send(topic='virt2_0', value=json.dumps({'value': 1}), partition=0, key='k1', timestamp_ms=1577836801001, - headers=[('content-encoding', b'base64')]) - producer.send(topic='virt2_0', value=json.dumps({'value': 2}), partition=0, key='k2', timestamp_ms=1577836802002, - headers=[('empty_value', b''), ('', b'empty name'), ('', b''), ('repetition', b'1'), ('repetition', b'2')]) + producer.send( + topic="virt2_0", + value=json.dumps({"value": 1}), + partition=0, + key="k1", + timestamp_ms=1577836801001, + headers=[("content-encoding", b"base64")], + ) + producer.send( + topic="virt2_0", + value=json.dumps({"value": 2}), + partition=0, + key="k2", + timestamp_ms=1577836802002, + headers=[ + ("empty_value", b""), + ("", b"empty name"), + ("", b""), + ("repetition", b"1"), + ("repetition", b"2"), + ], + ) producer.flush() - producer.send(topic='virt2_0', value=json.dumps({'value': 3}), partition=1, key='k3', timestamp_ms=1577836803003, - headers=[('b', b'b'), ('a', b'a')]) - producer.send(topic='virt2_0', value=json.dumps({'value': 4}), partition=1, key='k4', timestamp_ms=1577836804004, - headers=[('a', b'a'), ('b', b'b')]) + producer.send( + topic="virt2_0", + value=json.dumps({"value": 3}), + partition=1, + key="k3", + timestamp_ms=1577836803003, + headers=[("b", b"b"), ("a", b"a")], + ) + producer.send( + topic="virt2_0", + value=json.dumps({"value": 4}), + partition=1, + key="k4", + timestamp_ms=1577836804004, + headers=[("a", b"a"), ("b", b"b")], + ) producer.flush() - producer.send(topic='virt2_1', value=json.dumps({'value': 5}), partition=0, key='k5', timestamp_ms=1577836805005) - producer.send(topic='virt2_1', value=json.dumps({'value': 6}), partition=0, key='k6', timestamp_ms=1577836806006) + producer.send( + topic="virt2_1", + value=json.dumps({"value": 5}), + partition=0, + key="k5", + timestamp_ms=1577836805005, + ) + producer.send( + topic="virt2_1", + value=json.dumps({"value": 6}), + partition=0, + key="k6", + timestamp_ms=1577836806006, + ) producer.flush() - producer.send(topic='virt2_1', value=json.dumps({'value': 7}), partition=1, key='k7', timestamp_ms=1577836807007) - producer.send(topic='virt2_1', value=json.dumps({'value': 8}), partition=1, key='k8', timestamp_ms=1577836808008) + producer.send( + topic="virt2_1", + value=json.dumps({"value": 7}), + partition=1, + key="k7", + timestamp_ms=1577836807007, + ) + producer.send( + topic="virt2_1", + value=json.dumps({"value": 8}), + partition=1, + key="k8", + timestamp_ms=1577836808008, + ) producer.flush() - instance.wait_for_log_line('kafka.*Committed offset 2.*virt2_[01]', repetitions=4, look_behind_lines=6000) + instance.wait_for_log_line( + "kafka.*Committed offset 2.*virt2_[01]", repetitions=4, look_behind_lines=6000 + ) - members = describe_consumer_group(kafka_cluster, 'virt2') + members = describe_consumer_group(kafka_cluster, "virt2") # pprint.pprint(members) # members[0]['client_id'] = 'ClickHouse-instance-test-kafka-0' # members[1]['client_id'] = 'ClickHouse-instance-test-kafka-1' result = instance.query("SELECT * FROM test.view ORDER BY value", ignore_error=True) - expected = '''\ + expected = """\ 1 k1 virt2_0 0 0 1577836801 1577836801001 ['content-encoding'] ['base64'] 2 k2 virt2_0 0 1 1577836802 1577836802002 ['empty_value','','','repetition','repetition'] ['','empty name','','1','2'] 3 k3 virt2_0 1 0 1577836803 1577836803003 ['b','a'] ['b','a'] @@ -1818,26 +2180,32 @@ def test_kafka_virtual_columns2(kafka_cluster): 6 k6 virt2_1 0 1 1577836806 1577836806006 [] [] 7 k7 virt2_1 1 0 1577836807 1577836807007 [] [] 8 k8 virt2_1 1 1 1577836808 1577836808008 [] [] -''' +""" assert TSV(result) == TSV(expected) - instance.query(''' + instance.query( + """ DROP TABLE test.kafka; DROP TABLE test.view; - ''') + """ + ) kafka_delete_topic(admin_client, "virt2_0") kafka_delete_topic(admin_client, "virt2_1") instance.rotate_logs() + def test_kafka_produce_key_timestamp(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "insert3" kafka_create_topic(admin_client, topic_name) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka_writer (key UInt64, value UInt64, _key String, _timestamp DateTime('UTC')) @@ -1858,18 +2226,29 @@ def test_kafka_produce_key_timestamp(kafka_cluster): CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT key, value, inserted_key, toUnixTimestamp(inserted_timestamp), _key, _topic, _partition, _offset, toUnixTimestamp(_timestamp) FROM test.kafka; - ''') + """ + ) - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(1, 1, 'k1', 1577836801)) - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(2, 2, 'k2', 1577836802)) instance.query( - "INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({})),({},{},'{}',toDateTime({}))".format(3, 3, - 'k3', - 1577836803, - 4, 4, - 'k4', - 1577836804)) - instance.query("INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format(5, 5, 'k5', 1577836805)) + "INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format( + 1, 1, "k1", 1577836801 + ) + ) + instance.query( + "INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format( + 2, 2, "k2", 1577836802 + ) + ) + instance.query( + "INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({})),({},{},'{}',toDateTime({}))".format( + 3, 3, "k3", 1577836803, 4, 4, "k4", 1577836804 + ) + ) + instance.query( + "INSERT INTO test.kafka_writer VALUES ({},{},'{}',toDateTime({}))".format( + 5, 5, "k5", 1577836805 + ) + ) instance.wait_for_log_line("Committed offset 5") @@ -1877,13 +2256,13 @@ def test_kafka_produce_key_timestamp(kafka_cluster): # logging.debug(result) - expected = '''\ + expected = """\ 1 1 k1 1577836801 k1 insert3 0 0 1577836801 2 2 k2 1577836802 k2 insert3 0 1 1577836802 3 3 k3 1577836803 k3 insert3 0 2 1577836803 4 4 k4 1577836804 k4 insert3 0 3 1577836804 5 5 k5 1577836805 k5 insert3 0 4 1577836805 -''' +""" assert TSV(result) == TSV(expected) @@ -1891,14 +2270,17 @@ def test_kafka_produce_key_timestamp(kafka_cluster): def test_kafka_insert_avro(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_config = { # default retention, since predefined timestamp_ms is used. - 'retention.ms': '-1', + "retention.ms": "-1", } kafka_create_topic(admin_client, "avro1", config=topic_config) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka; CREATE TABLE test.kafka (key UInt64, value UInt64, _timestamp DateTime('UTC')) ENGINE = Kafka @@ -1907,20 +2289,26 @@ def test_kafka_insert_avro(kafka_cluster): kafka_group_name = 'avro1', kafka_commit_on_select = 1, kafka_format = 'Avro'; - ''') + """ + ) - - instance.query("INSERT INTO test.kafka select number*10 as key, number*100 as value, 1636505534 as _timestamp from numbers(4) SETTINGS output_format_avro_rows_in_file = 2, output_format_avro_codec = 'deflate'") + instance.query( + "INSERT INTO test.kafka select number*10 as key, number*100 as value, 1636505534 as _timestamp from numbers(4) SETTINGS output_format_avro_rows_in_file = 2, output_format_avro_codec = 'deflate'" + ) messages = [] while True: - messages.extend(kafka_consume(kafka_cluster, 'avro1', needDecode = False, timestamp = 1636505534)) + messages.extend( + kafka_consume( + kafka_cluster, "avro1", needDecode=False, timestamp=1636505534 + ) + ) if len(messages) == 2: break - result = '' + result = "" for a_message in messages: - result += decode_avro(a_message) + '\n' + result += decode_avro(a_message) + "\n" expected_result = """{'key': 0, 'value': 0, '_timestamp': 1636505534} {'key': 10, 'value': 100, '_timestamp': 1636505534} @@ -1929,19 +2317,22 @@ def test_kafka_insert_avro(kafka_cluster): {'key': 30, 'value': 300, '_timestamp': 1636505534} """ - assert (result == expected_result) + assert result == expected_result def test_kafka_produce_consume_avro(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "insert_avro" kafka_create_topic(admin_client, topic_name) num_rows = 75 - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.kafka; DROP TABLE IF EXISTS test.kafka_writer; @@ -1963,27 +2354,41 @@ def test_kafka_produce_consume_avro(kafka_cluster): CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT key, value FROM test.kafka; - ''') + """ + ) - instance.query("INSERT INTO test.kafka_writer select number*10 as key, number*100 as value from numbers({num_rows}) SETTINGS output_format_avro_rows_in_file = 7".format(num_rows=num_rows)) + instance.query( + "INSERT INTO test.kafka_writer select number*10 as key, number*100 as value from numbers({num_rows}) SETTINGS output_format_avro_rows_in_file = 7".format( + num_rows=num_rows + ) + ) - instance.wait_for_log_line("Committed offset {offset}".format(offset=math.ceil(num_rows/7))) + instance.wait_for_log_line( + "Committed offset {offset}".format(offset=math.ceil(num_rows / 7)) + ) - expected_num_rows = instance.query("SELECT COUNT(1) FROM test.view", ignore_error=True) - assert (int(expected_num_rows) == num_rows) + expected_num_rows = instance.query( + "SELECT COUNT(1) FROM test.view", ignore_error=True + ) + assert int(expected_num_rows) == num_rows - expected_max_key = instance.query("SELECT max(key) FROM test.view", ignore_error=True) - assert (int(expected_max_key) == (num_rows - 1) * 10) + expected_max_key = instance.query( + "SELECT max(key) FROM test.view", ignore_error=True + ) + assert int(expected_max_key) == (num_rows - 1) * 10 kafka_delete_topic(admin_client, topic_name) def test_kafka_flush_by_time(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "flush_by_time" kafka_create_topic(admin_client, topic_name) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; @@ -2001,40 +2406,45 @@ def test_kafka_flush_by_time(kafka_cluster): CREATE TABLE test.view (key UInt64, value UInt64, ts DateTime64(3) MATERIALIZED now64(3)) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) cancel = threading.Event() def produce(): while not cancel.is_set(): messages = [] - messages.append(json.dumps({'key': 0, 'value': 0})) - kafka_produce(kafka_cluster, 'flush_by_time', messages) + messages.append(json.dumps({"key": 0, "value": 0})) + kafka_produce(kafka_cluster, "flush_by_time", messages) time.sleep(0.8) kafka_thread = threading.Thread(target=produce) kafka_thread.start() - instance.query(''' + instance.query( + """ CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) time.sleep(18) - result = instance.query('SELECT uniqExact(ts) = 2, count() >= 15 FROM test.view') + result = instance.query("SELECT uniqExact(ts) = 2, count() >= 15 FROM test.view") cancel.set() kafka_thread.join() # kafka_cluster.open_bash_shell('instance') - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) - assert TSV(result) == TSV('1 1') + assert TSV(result) == TSV("1 1") kafka_delete_topic(admin_client, topic_name) @@ -2044,13 +2454,14 @@ def test_kafka_flush_by_block_size(kafka_cluster): def produce(): while not cancel.is_set(): messages = [] - messages.append(json.dumps({'key': 0, 'value': 0})) - kafka_produce(kafka_cluster, 'flush_by_block_size', messages) + messages.append(json.dumps({"key": 0, "value": 0})) + kafka_produce(kafka_cluster, "flush_by_block_size", messages) kafka_thread = threading.Thread(target=produce) kafka_thread.start() - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; @@ -2071,11 +2482,15 @@ def test_kafka_flush_by_block_size(kafka_cluster): CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) # Wait for Kafka engine to consume this data - while 1 != int(instance.query( - "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'")): + while 1 != int( + instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" + ) + ): time.sleep(0.5) cancel.set() @@ -2085,23 +2500,30 @@ def test_kafka_flush_by_block_size(kafka_cluster): result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") # logging.debug(result) - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) # 100 = first poll should return 100 messages (and rows) # not waiting for stream_flush_interval_ms - assert int( - result) == 100, 'Messages from kafka should be flushed when block of size kafka_max_block_size is formed!' + assert ( + int(result) == 100 + ), "Messages from kafka should be flushed when block of size kafka_max_block_size is formed!" + def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "topic_with_multiple_partitions2" kafka_create_topic(admin_client, topic_name, num_partitions=10) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; CREATE TABLE test.kafka (key UInt64, value UInt64) @@ -2117,7 +2539,8 @@ def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; - ''') + """ + ) messages = [] count = 0 @@ -2125,27 +2548,30 @@ def test_kafka_lot_of_partitions_partial_commit_of_bulk(kafka_cluster): rows = [] for dummy_row in range(random.randrange(3, 10)): count = count + 1 - rows.append(json.dumps({'key': count, 'value': count})) + rows.append(json.dumps({"key": count, "value": count})) messages.append("\n".join(rows)) - kafka_produce(kafka_cluster, 'topic_with_multiple_partitions2', messages) + kafka_produce(kafka_cluster, "topic_with_multiple_partitions2", messages) - instance.wait_for_log_line('kafka.*Stalled', repetitions=5) + instance.wait_for_log_line("kafka.*Stalled", repetitions=5) - result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.view') + result = instance.query("SELECT count(), uniqExact(key), max(key) FROM test.view") logging.debug(result) - assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(count)) + assert TSV(result) == TSV("{0}\t{0}\t{0}".format(count)) - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) kafka_delete_topic(admin_client, topic_name) def test_kafka_rebalance(kafka_cluster): NUMBER_OF_CONSURRENT_CONSUMERS = 11 - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination ( key UInt64, @@ -2159,13 +2585,16 @@ def test_kafka_rebalance(kafka_cluster): ) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) # kafka_cluster.open_bash_shell('instance') # time.sleep(2) - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "topic_with_multiple_partitions" kafka_create_topic(admin_client, topic_name, num_partitions=11) @@ -2177,18 +2606,21 @@ def test_kafka_rebalance(kafka_cluster): while not cancel.is_set(): messages = [] for _ in range(59): - messages.append(json.dumps({'key': msg_index[0], 'value': msg_index[0]})) + messages.append( + json.dumps({"key": msg_index[0], "value": msg_index[0]}) + ) msg_index[0] += 1 - kafka_produce(kafka_cluster, 'topic_with_multiple_partitions', messages) + kafka_produce(kafka_cluster, "topic_with_multiple_partitions", messages) kafka_thread = threading.Thread(target=produce) kafka_thread.start() for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS): - table_name = 'kafka_consumer{}'.format(consumer_index) + table_name = "kafka_consumer{}".format(consumer_index) logging.debug(("Setting up {}".format(table_name))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.{0}; DROP TABLE IF EXISTS test.{0}_mv; CREATE TABLE test.{0} (key UInt64, value UInt64) @@ -2210,29 +2642,50 @@ def test_kafka_rebalance(kafka_cluster): _timestamp, '{0}' as _consumed_by FROM test.{0}; - '''.format(table_name)) + """.format( + table_name + ) + ) # kafka_cluster.open_bash_shell('instance') # Waiting for test.kafka_consumerX to start consume ... - instance.wait_for_log_line('kafka_consumer{}.*Polled offset [0-9]+'.format(consumer_index)) + instance.wait_for_log_line( + "kafka_consumer{}.*Polled offset [0-9]+".format(consumer_index) + ) cancel.set() # I leave last one working by intent (to finish consuming after all rebalances) for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS - 1): logging.debug(("Dropping test.kafka_consumer{}".format(consumer_index))) - instance.query('DROP TABLE IF EXISTS test.kafka_consumer{} SYNC'.format(consumer_index)) + instance.query( + "DROP TABLE IF EXISTS test.kafka_consumer{} SYNC".format(consumer_index) + ) # logging.debug(instance.query('SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination')) # kafka_cluster.open_bash_shell('instance') while 1: - messages_consumed = int(instance.query('SELECT uniqExact(key) FROM test.destination')) + messages_consumed = int( + instance.query("SELECT uniqExact(key) FROM test.destination") + ) if messages_consumed >= msg_index[0]: break time.sleep(1) - logging.debug(("Waiting for finishing consuming (have {}, should be {})".format(messages_consumed, msg_index[0]))) + logging.debug( + ( + "Waiting for finishing consuming (have {}, should be {})".format( + messages_consumed, msg_index[0] + ) + ) + ) - logging.debug((instance.query('SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination'))) + logging.debug( + ( + instance.query( + "SELECT count(), uniqExact(key), max(key) + 1 FROM test.destination" + ) + ) + ) # Some queries to debug... # SELECT * FROM test.destination where key in (SELECT key FROM test.destination group by key having count() <> 1) @@ -2254,31 +2707,40 @@ def test_kafka_rebalance(kafka_cluster): # # select * from test.reference_mv left join test.destination using (key,_topic,_offset,_partition) where test.destination._consumed_by = ''; - result = int(instance.query('SELECT count() == uniqExact(key) FROM test.destination')) + result = int( + instance.query("SELECT count() == uniqExact(key) FROM test.destination") + ) for consumer_index in range(NUMBER_OF_CONSURRENT_CONSUMERS): logging.debug(("kafka_consumer{}".format(consumer_index))) - table_name = 'kafka_consumer{}'.format(consumer_index) - instance.query(''' + table_name = "kafka_consumer{}".format(consumer_index) + instance.query( + """ DROP TABLE IF EXISTS test.{0}; DROP TABLE IF EXISTS test.{0}_mv; - '''.format(table_name)) + """.format( + table_name + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; - ''') + """ + ) kafka_thread.join() - assert result == 1, 'Messages from kafka get duplicated!' + assert result == 1, "Messages from kafka get duplicated!" kafka_delete_topic(admin_client, topic_name) def test_kafka_no_holes_when_write_suffix_failed(kafka_cluster): - messages = [json.dumps({'key': j + 1, 'value': 'x' * 300}) for j in range(22)] - kafka_produce(kafka_cluster, 'no_holes_when_write_suffix_failed', messages) + messages = [json.dumps({"key": j + 1, "value": "x" * 300}) for j in range(22)] + kafka_produce(kafka_cluster, "no_holes_when_write_suffix_failed", messages) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; @@ -2294,41 +2756,49 @@ def test_kafka_no_holes_when_write_suffix_failed(kafka_cluster): CREATE TABLE test.view (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/kafkatest/tables/no_holes_when_write_suffix_failed', 'node1') ORDER BY key; - ''') + """ + ) # init PartitionManager (it starts container) earlier pm = PartitionManager() - instance.query(''' + instance.query( + """ CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka WHERE NOT sleepEachRow(0.25); - ''') + """ + ) instance.wait_for_log_line("Polled batch of 20 messages") # the tricky part here is that disconnect should happen after write prefix, but before write suffix # we have 0.25 (sleepEachRow) * 20 ( Rows ) = 5 sec window after "Polled batch of 20 messages" # while materialized view is working to inject zookeeper failure pm.drop_instance_zk_connections(instance) - instance.wait_for_log_line("Error.*(session has been expired|Connection loss).*while pushing to view") + instance.wait_for_log_line( + "Error.*(session has been expired|Connection loss).*while pushing to view" + ) pm.heal_all() instance.wait_for_log_line("Committed offset 22") - result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.view') + result = instance.query("SELECT count(), uniqExact(key), max(key) FROM test.view") logging.debug(result) # kafka_cluster.open_bash_shell('instance') - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) - assert TSV(result) == TSV('22\t22\t22') + assert TSV(result) == TSV("22\t22\t22") def test_exception_from_destructor(kafka_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -2336,35 +2806,45 @@ def test_exception_from_destructor(kafka_cluster): kafka_group_name = '', kafka_commit_on_select = 1, kafka_format = 'JSONEachRow'; - ''') - instance.query_and_get_error(''' + """ + ) + instance.query_and_get_error( + """ SELECT * FROM test.kafka; - ''') - instance.query(''' + """ + ) + instance.query( + """ DROP TABLE test.kafka; - ''') + """ + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', kafka_topic_list = 'xyz', kafka_group_name = '', kafka_format = 'JSONEachRow'; - ''') - instance.query(''' + """ + ) + instance.query( + """ DROP TABLE test.kafka; - ''') + """ + ) # kafka_cluster.open_bash_shell('instance') - assert TSV(instance.query('SELECT 1')) == TSV('1') + assert TSV(instance.query("SELECT 1")) == TSV("1") def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): - messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(1)] - kafka_produce(kafka_cluster, 'commits_of_unprocessed_messages_on_drop', messages) + messages = [json.dumps({"key": j + 1, "value": j + 1}) for j in range(1)] + kafka_produce(kafka_cluster, "commits_of_unprocessed_messages_on_drop", messages) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination SYNC; CREATE TABLE test.destination ( key UInt64, @@ -2398,10 +2878,11 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): _partition, _timestamp FROM test.kafka; - ''') + """ + ) # Waiting for test.kafka_consumer to start consume - instance.wait_for_log_line('Committed offset [0-9]+') + instance.wait_for_log_line("Committed offset [0-9]+") cancel = threading.Event() @@ -2411,20 +2892,25 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): while not cancel.is_set(): messages = [] for _ in range(113): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 - kafka_produce(kafka_cluster, 'commits_of_unprocessed_messages_on_drop', messages) + kafka_produce( + kafka_cluster, "commits_of_unprocessed_messages_on_drop", messages + ) time.sleep(0.5) kafka_thread = threading.Thread(target=produce) kafka_thread.start() time.sleep(4) - instance.query(''' + instance.query( + """ DROP TABLE test.kafka SYNC; - ''') + """ + ) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -2433,31 +2919,37 @@ def test_commits_of_unprocessed_messages_on_drop(kafka_cluster): kafka_format = 'JSONEachRow', kafka_max_block_size = 10000, kafka_flush_interval_ms = 1000; - ''') + """ + ) cancel.set() - instance.wait_for_log_line('kafka.*Stalled', repetitions=5) + instance.wait_for_log_line("kafka.*Stalled", repetitions=5) # kafka_cluster.open_bash_shell('instance') # SELECT key, _timestamp, _offset FROM test.destination where runningDifference(key) <> 1 ORDER BY key; - result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.destination') + result = instance.query( + "SELECT count(), uniqExact(key), max(key) FROM test.destination" + ) logging.debug(result) - instance.query(''' + instance.query( + """ DROP TABLE test.kafka_consumer SYNC; DROP TABLE test.destination SYNC; - ''') + """ + ) kafka_thread.join() - assert TSV(result) == TSV('{0}\t{0}\t{0}'.format(i[0] - 1)), 'Missing data!' + assert TSV(result) == TSV("{0}\t{0}\t{0}".format(i[0] - 1)), "Missing data!" def test_bad_reschedule(kafka_cluster): - messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(20000)] - kafka_produce(kafka_cluster, 'test_bad_reschedule', messages) + messages = [json.dumps({"key": j + 1, "value": j + 1}) for j in range(20000)] + kafka_produce(kafka_cluster, "test_bad_reschedule", messages) - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -2478,18 +2970,27 @@ def test_bad_reschedule(kafka_cluster): _partition, _timestamp FROM test.kafka; - ''') + """ + ) instance.wait_for_log_line("Committed offset 20000") - assert int(instance.query("SELECT max(consume_ts) - min(consume_ts) FROM test.destination")) < 8 + assert ( + int( + instance.query( + "SELECT max(consume_ts) - min(consume_ts) FROM test.destination" + ) + ) + < 8 + ) def test_kafka_duplicates_when_commit_failed(kafka_cluster): - messages = [json.dumps({'key': j + 1, 'value': 'x' * 300}) for j in range(22)] - kafka_produce(kafka_cluster, 'duplicates_when_commit_failed', messages) + messages = [json.dumps({"key": j + 1, "value": "x" * 300}) for j in range(22)] + kafka_produce(kafka_cluster, "duplicates_when_commit_failed", messages) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view SYNC; DROP TABLE IF EXISTS test.consumer SYNC; @@ -2505,45 +3006,52 @@ def test_kafka_duplicates_when_commit_failed(kafka_cluster): CREATE TABLE test.view (key UInt64, value String) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) - instance.query(''' + instance.query( + """ CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka WHERE NOT sleepEachRow(0.25); - ''') + """ + ) instance.wait_for_log_line("Polled batch of 20 messages") # the tricky part here is that disconnect should happen after write prefix, but before we do commit # we have 0.25 (sleepEachRow) * 20 ( Rows ) = 5 sec window after "Polled batch of 20 messages" # while materialized view is working to inject zookeeper failure - kafka_cluster.pause_container('kafka1') + kafka_cluster.pause_container("kafka1") # if we restore the connection too fast (<30sec) librdkafka will not report any timeout # (alternative is to decrease the default session timeouts for librdkafka) # # when the delay is too long (>50sec) broker will decide to remove us from the consumer group, # and will start answering "Broker: Unknown member" - instance.wait_for_log_line("Exception during commit attempt: Local: Waiting for coordinator", timeout=45) + instance.wait_for_log_line( + "Exception during commit attempt: Local: Waiting for coordinator", timeout=45 + ) instance.wait_for_log_line("All commit attempts failed", look_behind_lines=500) - kafka_cluster.unpause_container('kafka1') + kafka_cluster.unpause_container("kafka1") # kafka_cluster.open_bash_shell('instance') instance.wait_for_log_line("Committed offset 22") - result = instance.query('SELECT count(), uniqExact(key), max(key) FROM test.view') + result = instance.query("SELECT count(), uniqExact(key), max(key) FROM test.view") logging.debug(result) - instance.query(''' + instance.query( + """ DROP TABLE test.consumer SYNC; DROP TABLE test.view SYNC; - ''') + """ + ) # After https://github.com/edenhill/librdkafka/issues/2631 # timeout triggers rebalance, making further commits to the topic after getting back online # impossible. So we have a duplicate in that scenario, but we report that situation properly. - assert TSV(result) == TSV('42\t22\t22') + assert TSV(result) == TSV("42\t22\t22") # if we came to partition end we will repeat polling until reaching kafka_max_block_size or flush_interval @@ -2553,7 +3061,8 @@ def test_kafka_duplicates_when_commit_failed(kafka_cluster): # easier to understand, so let's keep it as is for now. # also we can came to eof because we drained librdkafka internal queue too fast def test_premature_flush_on_eof(kafka_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -2573,17 +3082,19 @@ def test_premature_flush_on_eof(kafka_cluster): ) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) # messages created here will be consumed immedeately after MV creation # reaching topic EOF. # But we should not do flush immedeately after reaching EOF, because # next poll can return more data, and we should respect kafka_flush_interval_ms # and try to form bigger block - messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(1)] - kafka_produce(kafka_cluster, 'premature_flush_on_eof', messages) + messages = [json.dumps({"key": j + 1, "value": j + 1}) for j in range(1)] + kafka_produce(kafka_cluster, "premature_flush_on_eof", messages) - instance.query(''' + instance.query( + """ CREATE MATERIALIZED VIEW test.kafka_consumer TO test.destination AS SELECT key, @@ -2594,7 +3105,8 @@ def test_premature_flush_on_eof(kafka_cluster): _partition, _timestamp FROM test.kafka; - ''') + """ + ) # all subscriptions/assignments done during select, so it start sending data to test.destination # immediately after creation of MV @@ -2603,7 +3115,7 @@ def test_premature_flush_on_eof(kafka_cluster): instance.wait_for_log_line("Stalled") # produce more messages after delay - kafka_produce(kafka_cluster, 'premature_flush_on_eof', messages) + kafka_produce(kafka_cluster, "premature_flush_on_eof", messages) # data was not flushed yet (it will be flushed 7.5 sec after creating MV) assert int(instance.query("SELECT count() FROM test.destination")) == 0 @@ -2611,22 +3123,27 @@ def test_premature_flush_on_eof(kafka_cluster): instance.wait_for_log_line("Committed offset 2") # it should be single part, i.e. single insert - result = instance.query('SELECT _part, count() FROM test.destination group by _part') - assert TSV(result) == TSV('all_1_1_0\t2') + result = instance.query( + "SELECT _part, count() FROM test.destination group by _part" + ) + assert TSV(result) == TSV("all_1_1_0\t2") - instance.query(''' + instance.query( + """ DROP TABLE test.kafka_consumer; DROP TABLE test.destination; - ''') + """ + ) def test_kafka_unavailable(kafka_cluster): - messages = [json.dumps({'key': j + 1, 'value': j + 1}) for j in range(20000)] - kafka_produce(kafka_cluster, 'test_bad_reschedule', messages) + messages = [json.dumps({"key": j + 1, "value": j + 1}) for j in range(20000)] + kafka_produce(kafka_cluster, "test_bad_reschedule", messages) - kafka_cluster.pause_container('kafka1') + kafka_cluster.pause_container("kafka1") - instance.query(''' + instance.query( + """ CREATE TABLE test.test_bad_reschedule (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -2647,16 +3164,19 @@ def test_kafka_unavailable(kafka_cluster): _partition, _timestamp FROM test.test_bad_reschedule; - ''') + """ + ) instance.query("SELECT * FROM test.test_bad_reschedule") instance.query("SELECT count() FROM test.destination_unavailable") # enough to trigger issue time.sleep(30) - kafka_cluster.unpause_container('kafka1') + kafka_cluster.unpause_container("kafka1") - while int(instance.query("SELECT count() FROM test.destination_unavailable")) < 20000: + while ( + int(instance.query("SELECT count() FROM test.destination_unavailable")) < 20000 + ): print("Waiting for consume") time.sleep(1) @@ -2666,7 +3186,8 @@ def test_kafka_issue14202(kafka_cluster): INSERT INTO Kafka Engine from an empty SELECT sub query was leading to failure """ - instance.query(''' + instance.query( + """ CREATE TABLE test.empty_table ( dt Date, some_string String @@ -2681,20 +3202,25 @@ def test_kafka_issue14202(kafka_cluster): kafka_topic_list = 'issue14202', kafka_group_name = 'issue14202', kafka_format = 'JSONEachRow'; - ''') + """ + ) instance.query( - 'INSERT INTO test.kafka_q SELECT t, some_string FROM ( SELECT dt AS t, some_string FROM test.empty_table )') + "INSERT INTO test.kafka_q SELECT t, some_string FROM ( SELECT dt AS t, some_string FROM test.empty_table )" + ) # check instance is alive - assert TSV(instance.query('SELECT 1')) == TSV('1') - instance.query(''' + assert TSV(instance.query("SELECT 1")) == TSV("1") + instance.query( + """ DROP TABLE test.empty_table; DROP TABLE test.kafka_q; - ''') + """ + ) def test_kafka_csv_with_thread_per_consumer(kafka_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', @@ -2705,26 +3231,30 @@ def test_kafka_csv_with_thread_per_consumer(kafka_cluster): kafka_num_consumers = 4, kafka_commit_on_select = 1, kafka_thread_per_consumer = 1; - ''') + """ + ) messages = [] for i in range(50): - messages.append('{i}, {i}'.format(i=i)) - kafka_produce(kafka_cluster, 'csv_with_thread_per_consumer', messages) + messages.append("{i}, {i}".format(i=i)) + kafka_produce(kafka_cluster, "csv_with_thread_per_consumer", messages) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break kafka_check_result(result, True) + def random_string(size=8): - return ''.join(random.choices(string.ascii_uppercase + string.digits, k=size)) + return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) + def test_kafka_engine_put_errors_to_stream(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka; DROP TABLE IF EXISTS test.kafka_data; DROP TABLE IF EXISTS test.kafka_errors; @@ -2750,54 +3280,63 @@ def test_kafka_engine_put_errors_to_stream(kafka_cluster): _raw_message AS raw, _error AS error FROM test.kafka WHERE length(_error) > 0; - ''') + """ + ) messages = [] for i in range(128): if i % 2 == 0: - messages.append(json.dumps({'i': i, 's': random_string(8)})) + messages.append(json.dumps({"i": i, "s": random_string(8)})) else: # Unexpected json content for table test.kafka. - messages.append(json.dumps({'i': 'n_' + random_string(4), 's': random_string(8)})) + messages.append( + json.dumps({"i": "n_" + random_string(4), "s": random_string(8)}) + ) - kafka_produce(kafka_cluster, 'kafka_engine_put_errors_to_stream', messages) + kafka_produce(kafka_cluster, "kafka_engine_put_errors_to_stream", messages) instance.wait_for_log_line("Committed offset 128") - assert TSV(instance.query('SELECT count() FROM test.kafka_data')) == TSV('64') - assert TSV(instance.query('SELECT count() FROM test.kafka_errors')) == TSV('64') + assert TSV(instance.query("SELECT count() FROM test.kafka_data")) == TSV("64") + assert TSV(instance.query("SELECT count() FROM test.kafka_errors")) == TSV("64") - instance.query(''' + instance.query( + """ DROP TABLE test.kafka; DROP TABLE test.kafka_data; DROP TABLE test.kafka_errors; - ''') + """ + ) + def gen_normal_json(): return '{"i":1000, "s":"ABC123abc"}' + def gen_malformed_json(): return '{"i":"n1000", "s":"1000"}' -def gen_message_with_jsons(jsons = 10, malformed = 0): + +def gen_message_with_jsons(jsons=10, malformed=0): s = io.StringIO() # we don't care on which position error will be added # (we skip whole broken message), but we need to be # sure that at least one error will be added, # otherwise test will fail. - error_pos = random.randint(0,jsons-1) + error_pos = random.randint(0, jsons - 1) - for i in range (jsons): + for i in range(jsons): if malformed and i == error_pos: s.write(gen_malformed_json()) else: s.write(gen_normal_json()) - s.write(' ') + s.write(" ") return s.getvalue() def test_kafka_engine_put_errors_to_stream_with_random_malformed_json(kafka_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka; DROP TABLE IF EXISTS test.kafka_data; DROP TABLE IF EXISTS test.kafka_errors; @@ -2824,7 +3363,8 @@ def test_kafka_engine_put_errors_to_stream_with_random_malformed_json(kafka_clus _raw_message AS raw, _error AS error FROM test.kafka WHERE length(_error) > 0; - ''') + """ + ) messages = [] for i in range(128): @@ -2833,205 +3373,215 @@ def test_kafka_engine_put_errors_to_stream_with_random_malformed_json(kafka_clus else: messages.append(gen_message_with_jsons(10, 0)) - kafka_produce(kafka_cluster, 'kafka_engine_put_errors_to_stream_with_random_malformed_json', messages) + kafka_produce( + kafka_cluster, + "kafka_engine_put_errors_to_stream_with_random_malformed_json", + messages, + ) instance.wait_for_log_line("Committed offset 128") # 64 good messages, each containing 10 rows - assert TSV(instance.query('SELECT count() FROM test.kafka_data')) == TSV('640') + assert TSV(instance.query("SELECT count() FROM test.kafka_data")) == TSV("640") # 64 bad messages, each containing some broken row - assert TSV(instance.query('SELECT count() FROM test.kafka_errors')) == TSV('64') + assert TSV(instance.query("SELECT count() FROM test.kafka_errors")) == TSV("64") - instance.query(''' + instance.query( + """ DROP TABLE test.kafka; DROP TABLE test.kafka_data; DROP TABLE test.kafka_errors; - ''') + """ + ) + def test_kafka_formats_with_broken_message(kafka_cluster): # data was dumped from clickhouse itself in a following manner # clickhouse-client --format=Native --query='SELECT toInt64(number) as id, toUInt16( intDiv( id, 65536 ) ) as blockNo, reinterpretAsString(19777) as val1, toFloat32(0.5) as val2, toUInt8(1) as val3 from numbers(100) ORDER BY id' | xxd -ps | tr -d '\n' | sed 's/\(..\)/\\x\1/g' - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) all_formats = { ## Text formats ## # dumped with clickhouse-client ... | perl -pe 's/\n/\\n/; s/\t/\\t/g;' - 'JSONEachRow': { - 'data_sample': [ + "JSONEachRow": { + "data_sample": [ '{"id":"0","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', '{"id":"1","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"2","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"3","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"4","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"5","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"6","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"7","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"8","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"9","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"10","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"11","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"12","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"13","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"14","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n{"id":"15","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', '{"id":"0","blockNo":0,"val1":"AM","val2":0.5,"val3":1}\n', # broken message '{"id":"0","blockNo":"BAD","val1":"AM","val2":0.5,"val3":1}', ], - 'expected':'''{"raw_message":"{\\"id\\":\\"0\\",\\"blockNo\\":\\"BAD\\",\\"val1\\":\\"AM\\",\\"val2\\":0.5,\\"val3\\":1}","error":"Cannot parse input: expected '\\"' before: 'BAD\\",\\"val1\\":\\"AM\\",\\"val2\\":0.5,\\"val3\\":1}': (while reading the value of key blockNo)"}''', - 'supports_empty_value': True, - 'printable': True, + "expected": """{"raw_message":"{\\"id\\":\\"0\\",\\"blockNo\\":\\"BAD\\",\\"val1\\":\\"AM\\",\\"val2\\":0.5,\\"val3\\":1}","error":"Cannot parse input: expected '\\"' before: 'BAD\\",\\"val1\\":\\"AM\\",\\"val2\\":0.5,\\"val3\\":1}': (while reading the value of key blockNo)"}""", + "supports_empty_value": True, + "printable": True, }, # JSONAsString doesn't fit to that test, and tested separately - 'JSONCompactEachRow': { - 'data_sample': [ + "JSONCompactEachRow": { + "data_sample": [ '["0", 0, "AM", 0.5, 1]\n', '["1", 0, "AM", 0.5, 1]\n["2", 0, "AM", 0.5, 1]\n["3", 0, "AM", 0.5, 1]\n["4", 0, "AM", 0.5, 1]\n["5", 0, "AM", 0.5, 1]\n["6", 0, "AM", 0.5, 1]\n["7", 0, "AM", 0.5, 1]\n["8", 0, "AM", 0.5, 1]\n["9", 0, "AM", 0.5, 1]\n["10", 0, "AM", 0.5, 1]\n["11", 0, "AM", 0.5, 1]\n["12", 0, "AM", 0.5, 1]\n["13", 0, "AM", 0.5, 1]\n["14", 0, "AM", 0.5, 1]\n["15", 0, "AM", 0.5, 1]\n', '["0", 0, "AM", 0.5, 1]\n', # broken message '["0", "BAD", "AM", 0.5, 1]', ], - 'expected':'''{"raw_message":"[\\"0\\", \\"BAD\\", \\"AM\\", 0.5, 1]","error":"Cannot parse input: expected '\\"' before: 'BAD\\", \\"AM\\", 0.5, 1]': (while reading the value of key blockNo)"}''', - 'supports_empty_value': True, - 'printable':True, + "expected": """{"raw_message":"[\\"0\\", \\"BAD\\", \\"AM\\", 0.5, 1]","error":"Cannot parse input: expected '\\"' before: 'BAD\\", \\"AM\\", 0.5, 1]': (while reading the value of key blockNo)"}""", + "supports_empty_value": True, + "printable": True, }, - 'JSONCompactEachRowWithNamesAndTypes': { - 'data_sample': [ + "JSONCompactEachRowWithNamesAndTypes": { + "data_sample": [ '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["0", 0, "AM", 0.5, 1]\n', '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["1", 0, "AM", 0.5, 1]\n["2", 0, "AM", 0.5, 1]\n["3", 0, "AM", 0.5, 1]\n["4", 0, "AM", 0.5, 1]\n["5", 0, "AM", 0.5, 1]\n["6", 0, "AM", 0.5, 1]\n["7", 0, "AM", 0.5, 1]\n["8", 0, "AM", 0.5, 1]\n["9", 0, "AM", 0.5, 1]\n["10", 0, "AM", 0.5, 1]\n["11", 0, "AM", 0.5, 1]\n["12", 0, "AM", 0.5, 1]\n["13", 0, "AM", 0.5, 1]\n["14", 0, "AM", 0.5, 1]\n["15", 0, "AM", 0.5, 1]\n', '["id", "blockNo", "val1", "val2", "val3"]\n["Int64", "UInt16", "String", "Float32", "UInt8"]\n["0", 0, "AM", 0.5, 1]\n', # broken message '["0", "BAD", "AM", 0.5, 1]', ], - 'expected':'''{"raw_message":"[\\"0\\", \\"BAD\\", \\"AM\\", 0.5, 1]","error":"Cannot parse JSON string: expected opening quote"}''', - 'printable':True, + "expected": """{"raw_message":"[\\"0\\", \\"BAD\\", \\"AM\\", 0.5, 1]","error":"Cannot parse JSON string: expected opening quote"}""", + "printable": True, }, - 'TSKV': { - 'data_sample': [ - 'id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', - 'id=1\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=2\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=3\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=4\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=5\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=6\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=7\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=8\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=9\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=10\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=11\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=12\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=13\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=14\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=15\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', - 'id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n', + "TSKV": { + "data_sample": [ + "id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n", + "id=1\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=2\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=3\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=4\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=5\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=6\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=7\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=8\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=9\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=10\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=11\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=12\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=13\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=14\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\nid=15\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n", + "id=0\tblockNo=0\tval1=AM\tval2=0.5\tval3=1\n", # broken message - 'id=0\tblockNo=BAD\tval1=AM\tval2=0.5\tval3=1\n', + "id=0\tblockNo=BAD\tval1=AM\tval2=0.5\tval3=1\n", ], - 'expected':'{"raw_message":"id=0\\tblockNo=BAD\\tval1=AM\\tval2=0.5\\tval3=1\\n","error":"Found garbage after field in TSKV format: blockNo: (at row 1)\\n"}', - 'printable':True, + "expected": '{"raw_message":"id=0\\tblockNo=BAD\\tval1=AM\\tval2=0.5\\tval3=1\\n","error":"Found garbage after field in TSKV format: blockNo: (at row 1)\\n"}', + "printable": True, }, - 'CSV': { - 'data_sample': [ + "CSV": { + "data_sample": [ '0,0,"AM",0.5,1\n', '1,0,"AM",0.5,1\n2,0,"AM",0.5,1\n3,0,"AM",0.5,1\n4,0,"AM",0.5,1\n5,0,"AM",0.5,1\n6,0,"AM",0.5,1\n7,0,"AM",0.5,1\n8,0,"AM",0.5,1\n9,0,"AM",0.5,1\n10,0,"AM",0.5,1\n11,0,"AM",0.5,1\n12,0,"AM",0.5,1\n13,0,"AM",0.5,1\n14,0,"AM",0.5,1\n15,0,"AM",0.5,1\n', '0,0,"AM",0.5,1\n', # broken message '0,"BAD","AM",0.5,1\n', ], - 'expected':'''{"raw_message":"0,\\"BAD\\",\\"AM\\",0.5,1\\n","error":"Cannot parse input: expected '\\"' before: 'BAD\\",\\"AM\\",0.5,1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}''', - 'printable':True, - 'supports_empty_value': True, + "expected": """{"raw_message":"0,\\"BAD\\",\\"AM\\",0.5,1\\n","error":"Cannot parse input: expected '\\"' before: 'BAD\\",\\"AM\\",0.5,1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}""", + "printable": True, + "supports_empty_value": True, }, - 'TSV': { - 'data_sample': [ - '0\t0\tAM\t0.5\t1\n', - '1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - '0\t0\tAM\t0.5\t1\n', + "TSV": { + "data_sample": [ + "0\t0\tAM\t0.5\t1\n", + "1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "0\t0\tAM\t0.5\t1\n", # broken message - '0\tBAD\tAM\t0.5\t1\n', + "0\tBAD\tAM\t0.5\t1\n", ], - 'expected':'''{"raw_message":"0\\tBAD\\tAM\\t0.5\\t1\\n","error":"Cannot parse input: expected '\\\\t' before: 'BAD\\\\tAM\\\\t0.5\\\\t1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}''', - 'supports_empty_value': True, - 'printable':True, + "expected": """{"raw_message":"0\\tBAD\\tAM\\t0.5\\t1\\n","error":"Cannot parse input: expected '\\\\t' before: 'BAD\\\\tAM\\\\t0.5\\\\t1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}""", + "supports_empty_value": True, + "printable": True, }, - 'CSVWithNames': { - 'data_sample': [ + "CSVWithNames": { + "data_sample": [ '"id","blockNo","val1","val2","val3"\n0,0,"AM",0.5,1\n', '"id","blockNo","val1","val2","val3"\n1,0,"AM",0.5,1\n2,0,"AM",0.5,1\n3,0,"AM",0.5,1\n4,0,"AM",0.5,1\n5,0,"AM",0.5,1\n6,0,"AM",0.5,1\n7,0,"AM",0.5,1\n8,0,"AM",0.5,1\n9,0,"AM",0.5,1\n10,0,"AM",0.5,1\n11,0,"AM",0.5,1\n12,0,"AM",0.5,1\n13,0,"AM",0.5,1\n14,0,"AM",0.5,1\n15,0,"AM",0.5,1\n', '"id","blockNo","val1","val2","val3"\n0,0,"AM",0.5,1\n', # broken message '"id","blockNo","val1","val2","val3"\n0,"BAD","AM",0.5,1\n', ], - 'expected':'''{"raw_message":"\\"id\\",\\"blockNo\\",\\"val1\\",\\"val2\\",\\"val3\\"\\n0,\\"BAD\\",\\"AM\\",0.5,1\\n","error":"Cannot parse input: expected '\\"' before: 'BAD\\",\\"AM\\",0.5,1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}''', - 'printable':True, + "expected": """{"raw_message":"\\"id\\",\\"blockNo\\",\\"val1\\",\\"val2\\",\\"val3\\"\\n0,\\"BAD\\",\\"AM\\",0.5,1\\n","error":"Cannot parse input: expected '\\"' before: 'BAD\\",\\"AM\\",0.5,1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}""", + "printable": True, }, - 'Values': { - 'data_sample': [ + "Values": { + "data_sample": [ "(0,0,'AM',0.5,1)", "(1,0,'AM',0.5,1),(2,0,'AM',0.5,1),(3,0,'AM',0.5,1),(4,0,'AM',0.5,1),(5,0,'AM',0.5,1),(6,0,'AM',0.5,1),(7,0,'AM',0.5,1),(8,0,'AM',0.5,1),(9,0,'AM',0.5,1),(10,0,'AM',0.5,1),(11,0,'AM',0.5,1),(12,0,'AM',0.5,1),(13,0,'AM',0.5,1),(14,0,'AM',0.5,1),(15,0,'AM',0.5,1)", "(0,0,'AM',0.5,1)", # broken message "(0,'BAD','AM',0.5,1)", ], - 'expected':r'''{"raw_message":"(0,'BAD','AM',0.5,1)","error":"Cannot parse string 'BAD' as UInt16: syntax error at begin of string. Note: there are toUInt16OrZero and toUInt16OrNull functions, which returns zero\/NULL instead of throwing exception"}''', - 'supports_empty_value': True, - 'printable':True, + "expected": r"""{"raw_message":"(0,'BAD','AM',0.5,1)","error":"Cannot parse string 'BAD' as UInt16: syntax error at begin of string. Note: there are toUInt16OrZero and toUInt16OrNull functions, which returns zero\/NULL instead of throwing exception"}""", + "supports_empty_value": True, + "printable": True, }, - 'TSVWithNames': { - 'data_sample': [ - 'id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n', + "TSVWithNames": { + "data_sample": [ + "id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\n0\t0\tAM\t0.5\t1\n", # broken message - 'id\tblockNo\tval1\tval2\tval3\n0\tBAD\tAM\t0.5\t1\n', + "id\tblockNo\tval1\tval2\tval3\n0\tBAD\tAM\t0.5\t1\n", ], - 'expected':'''{"raw_message":"id\\tblockNo\\tval1\\tval2\\tval3\\n0\\tBAD\\tAM\\t0.5\\t1\\n","error":"Cannot parse input: expected '\\\\t' before: 'BAD\\\\tAM\\\\t0.5\\\\t1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}''', - 'supports_empty_value': True, - 'printable':True, + "expected": """{"raw_message":"id\\tblockNo\\tval1\\tval2\\tval3\\n0\\tBAD\\tAM\\t0.5\\t1\\n","error":"Cannot parse input: expected '\\\\t' before: 'BAD\\\\tAM\\\\t0.5\\\\t1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}""", + "supports_empty_value": True, + "printable": True, }, - 'TSVWithNamesAndTypes': { - 'data_sample': [ - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n', - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n', + "TSVWithNamesAndTypes": { + "data_sample": [ + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n1\t0\tAM\t0.5\t1\n2\t0\tAM\t0.5\t1\n3\t0\tAM\t0.5\t1\n4\t0\tAM\t0.5\t1\n5\t0\tAM\t0.5\t1\n6\t0\tAM\t0.5\t1\n7\t0\tAM\t0.5\t1\n8\t0\tAM\t0.5\t1\n9\t0\tAM\t0.5\t1\n10\t0\tAM\t0.5\t1\n11\t0\tAM\t0.5\t1\n12\t0\tAM\t0.5\t1\n13\t0\tAM\t0.5\t1\n14\t0\tAM\t0.5\t1\n15\t0\tAM\t0.5\t1\n", + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\t0\tAM\t0.5\t1\n", # broken message - 'id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\tBAD\tAM\t0.5\t1\n', + "id\tblockNo\tval1\tval2\tval3\nInt64\tUInt16\tString\tFloat32\tUInt8\n0\tBAD\tAM\t0.5\t1\n", ], - 'expected':'''{"raw_message":"id\\tblockNo\\tval1\\tval2\\tval3\\nInt64\\tUInt16\\tString\\tFloat32\\tUInt8\\n0\\tBAD\\tAM\\t0.5\\t1\\n","error":"Cannot parse input: expected '\\\\t' before: 'BAD\\\\tAM\\\\t0.5\\\\t1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}''', - 'printable':True, + "expected": """{"raw_message":"id\\tblockNo\\tval1\\tval2\\tval3\\nInt64\\tUInt16\\tString\\tFloat32\\tUInt8\\n0\\tBAD\\tAM\\t0.5\\t1\\n","error":"Cannot parse input: expected '\\\\t' before: 'BAD\\\\tAM\\\\t0.5\\\\t1\\\\n': Could not print diagnostic info because two last rows aren't in buffer (rare case)\\n"}""", + "printable": True, }, - 'Native': { - 'data_sample': [ - b'\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01', - b'\x05\x0f\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01', - b'\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01', + "Native": { + "data_sample": [ + b"\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01", + b"\x05\x0f\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x09\x00\x00\x00\x00\x00\x00\x00\x0a\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01", + b"\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x55\x49\x6e\x74\x31\x36\x00\x00\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01", # broken message - b'\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x53\x74\x72\x69\x6e\x67\x03\x42\x41\x44\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01', + b"\x05\x01\x02\x69\x64\x05\x49\x6e\x74\x36\x34\x00\x00\x00\x00\x00\x00\x00\x00\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x06\x53\x74\x72\x69\x6e\x67\x03\x42\x41\x44\x04\x76\x61\x6c\x31\x06\x53\x74\x72\x69\x6e\x67\x02\x41\x4d\x04\x76\x61\x6c\x32\x07\x46\x6c\x6f\x61\x74\x33\x32\x00\x00\x00\x3f\x04\x76\x61\x6c\x33\x05\x55\x49\x6e\x74\x38\x01", ], - 'expected':'''{"raw_message":"050102696405496E743634000000000000000007626C6F636B4E6F06537472696E67034241440476616C3106537472696E6702414D0476616C3207466C6F617433320000003F0476616C330555496E743801","error":"Cannot convert: String to UInt16"}''', - 'printable':False, + "expected": """{"raw_message":"050102696405496E743634000000000000000007626C6F636B4E6F06537472696E67034241440476616C3106537472696E6702414D0476616C3207466C6F617433320000003F0476616C330555496E743801","error":"Cannot convert: String to UInt16"}""", + "printable": False, }, - 'RowBinary': { - 'data_sample': [ - b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', + "RowBinary": { + "data_sample": [ + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", # broken message - b'\x00\x00\x00\x00\x00\x00\x00\x00\x03\x42\x41\x44\x02\x41\x4d\x00\x00\x00\x3f\x01', + b"\x00\x00\x00\x00\x00\x00\x00\x00\x03\x42\x41\x44\x02\x41\x4d\x00\x00\x00\x3f\x01", ], - 'expected':'{"raw_message":"00000000000000000342414402414D0000003F01","error":"Cannot read all data. Bytes read: 9. Bytes expected: 65.: (at row 1)\\n"}', - 'printable':False, + "expected": '{"raw_message":"00000000000000000342414402414D0000003F01","error":"Cannot read all data. Bytes read: 9. Bytes expected: 65.: (at row 1)\\n"}', + "printable": False, }, - 'RowBinaryWithNamesAndTypes': { - 'data_sample': [ - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01', + "RowBinaryWithNamesAndTypes": { + "data_sample": [ + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x55\x49\x6e\x74\x31\x36\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x41\x4d\x00\x00\x00\x3f\x01", # broken message - b'\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x53\x74\x72\x69\x6e\x67\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x03\x42\x41\x44\x02\x41\x4d\x00\x00\x00\x3f\x01', + b"\x05\x02\x69\x64\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x04\x76\x61\x6c\x31\x04\x76\x61\x6c\x32\x04\x76\x61\x6c\x33\x05\x49\x6e\x74\x36\x34\x06\x53\x74\x72\x69\x6e\x67\x06\x53\x74\x72\x69\x6e\x67\x07\x46\x6c\x6f\x61\x74\x33\x32\x05\x55\x49\x6e\x74\x38\x00\x00\x00\x00\x00\x00\x00\x00\x03\x42\x41\x44\x02\x41\x4d\x00\x00\x00\x3f\x01", ], - 'expected':'{"raw_message":"0502696407626C6F636B4E6F0476616C310476616C320476616C3305496E74363406537472696E6706537472696E6707466C6F617433320555496E743800000000000000000342414402414D0000003F01","error":"Type of \'blockNo\' must be UInt16, not String"}', - 'printable':False, + "expected": '{"raw_message":"0502696407626C6F636B4E6F0476616C310476616C320476616C3305496E74363406537472696E6706537472696E6707466C6F617433320555496E743800000000000000000342414402414D0000003F01","error":"Type of \'blockNo\' must be UInt16, not String"}', + "printable": False, }, - 'ORC': { - 'data_sample': [ - b'\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', - b'\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x0f\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x7e\x25\x0e\x2e\x46\x43\x21\x46\x4b\x09\xad\x00\x06\x00\x33\x00\x00\x0a\x17\x0a\x03\x00\x00\x00\x12\x10\x08\x0f\x22\x0a\x0a\x02\x41\x4d\x12\x02\x41\x4d\x18\x3c\x50\x00\x3a\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x7e\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x66\x73\x3d\xd3\x00\x06\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x02\x10\x02\x18\x1e\x50\x00\x05\x00\x00\x0c\x00\x2b\x00\x00\x31\x32\x33\x34\x35\x36\x37\x38\x39\x31\x30\x31\x31\x31\x32\x31\x33\x31\x34\x31\x35\x09\x00\x00\x06\x01\x03\x02\x09\x00\x00\xc0\x0e\x00\x00\x07\x00\x00\x42\x00\x80\x05\x00\x00\x41\x4d\x0a\x00\x00\xe3\xe2\x42\x01\x00\x09\x00\x00\xc0\x0e\x02\x00\x05\x00\x00\x0c\x01\x94\x00\x00\x2d\xca\xc1\x0e\x80\x30\x08\x03\xd0\xc1\x60\x2e\xf3\x62\x76\x6a\xe2\x0e\xfe\xff\x57\x5a\x3b\x0f\xe4\x51\xe8\x68\xbd\x5d\x05\xe7\xf8\x34\x40\x3a\x6e\x59\xb1\x64\xe0\x91\xa9\xbf\xb1\x97\xd2\x95\x9d\x1e\xca\x55\x3a\x6d\xb4\xd2\xdd\x0b\x74\x9a\x74\xf7\x12\x39\xbd\x97\x7f\x7c\x06\xbb\xa6\x8d\x97\x17\xb4\x00\x00\xe3\x4a\xe6\x62\xe1\xe0\x0f\x60\xe0\xe2\xe3\xe0\x17\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\xe0\x57\xe2\xe0\x62\x34\x14\x62\xb4\x94\xd0\x02\x8a\xc8\x73\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\xc2\x06\x28\x26\xc4\x25\xca\xc1\x6f\xc4\xcb\xc5\x68\x20\xc4\x6c\xa0\x67\x2a\xc5\x6c\xae\x67\x0a\x14\xe6\x87\x1a\xc6\x24\xc0\x24\x21\x07\x32\x0c\x00\x4a\x01\x00\xe3\x60\x16\x58\xc3\x24\xc5\xcd\xc1\x2c\x30\x89\x51\xc2\x4b\xc1\x57\x83\x5f\x49\x83\x83\x47\x88\x95\x91\x89\x99\x85\x55\x8a\x3d\x29\x27\x3f\x39\xdb\x2f\x5f\x8a\x29\x33\x45\x8a\xa5\x2c\x31\xc7\x10\x4c\x1a\x81\x49\x63\x25\x26\x0e\x46\x20\x66\x07\x63\x36\x0e\x3e\x0d\x26\x03\x10\x9f\xd1\x80\xdf\x8a\x85\x83\x3f\x80\xc1\x8a\x8f\x83\x5f\x88\x8d\x83\x41\x80\x41\x82\x21\x80\x21\x82\xd5\x4a\x80\x83\x5f\x89\x83\x8b\xd1\x50\x88\xd1\x52\x42\x0b\x28\x22\x6f\x25\x04\x14\xe1\xe2\x62\x72\xf4\x15\x02\x62\x09\x1b\xa0\x98\x90\x95\x28\x07\xbf\x11\x2f\x17\xa3\x81\x10\xb3\x81\x9e\xa9\x14\xb3\xb9\x9e\x29\x50\x98\x1f\x6a\x18\x93\x00\x93\x84\x1c\xc8\x30\x87\x09\x7e\x1e\x0c\x00\x08\xa8\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x5d\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', - b'\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', + "ORC": { + "data_sample": [ + b"\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", + b"\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x0f\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x7e\x25\x0e\x2e\x46\x43\x21\x46\x4b\x09\xad\x00\x06\x00\x33\x00\x00\x0a\x17\x0a\x03\x00\x00\x00\x12\x10\x08\x0f\x22\x0a\x0a\x02\x41\x4d\x12\x02\x41\x4d\x18\x3c\x50\x00\x3a\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x7e\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x66\x73\x3d\xd3\x00\x06\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x0f\x12\x06\x08\x02\x10\x02\x18\x1e\x50\x00\x05\x00\x00\x0c\x00\x2b\x00\x00\x31\x32\x33\x34\x35\x36\x37\x38\x39\x31\x30\x31\x31\x31\x32\x31\x33\x31\x34\x31\x35\x09\x00\x00\x06\x01\x03\x02\x09\x00\x00\xc0\x0e\x00\x00\x07\x00\x00\x42\x00\x80\x05\x00\x00\x41\x4d\x0a\x00\x00\xe3\xe2\x42\x01\x00\x09\x00\x00\xc0\x0e\x02\x00\x05\x00\x00\x0c\x01\x94\x00\x00\x2d\xca\xc1\x0e\x80\x30\x08\x03\xd0\xc1\x60\x2e\xf3\x62\x76\x6a\xe2\x0e\xfe\xff\x57\x5a\x3b\x0f\xe4\x51\xe8\x68\xbd\x5d\x05\xe7\xf8\x34\x40\x3a\x6e\x59\xb1\x64\xe0\x91\xa9\xbf\xb1\x97\xd2\x95\x9d\x1e\xca\x55\x3a\x6d\xb4\xd2\xdd\x0b\x74\x9a\x74\xf7\x12\x39\xbd\x97\x7f\x7c\x06\xbb\xa6\x8d\x97\x17\xb4\x00\x00\xe3\x4a\xe6\x62\xe1\xe0\x0f\x60\xe0\xe2\xe3\xe0\x17\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\xe0\x57\xe2\xe0\x62\x34\x14\x62\xb4\x94\xd0\x02\x8a\xc8\x73\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\xc2\x06\x28\x26\xc4\x25\xca\xc1\x6f\xc4\xcb\xc5\x68\x20\xc4\x6c\xa0\x67\x2a\xc5\x6c\xae\x67\x0a\x14\xe6\x87\x1a\xc6\x24\xc0\x24\x21\x07\x32\x0c\x00\x4a\x01\x00\xe3\x60\x16\x58\xc3\x24\xc5\xcd\xc1\x2c\x30\x89\x51\xc2\x4b\xc1\x57\x83\x5f\x49\x83\x83\x47\x88\x95\x91\x89\x99\x85\x55\x8a\x3d\x29\x27\x3f\x39\xdb\x2f\x5f\x8a\x29\x33\x45\x8a\xa5\x2c\x31\xc7\x10\x4c\x1a\x81\x49\x63\x25\x26\x0e\x46\x20\x66\x07\x63\x36\x0e\x3e\x0d\x26\x03\x10\x9f\xd1\x80\xdf\x8a\x85\x83\x3f\x80\xc1\x8a\x8f\x83\x5f\x88\x8d\x83\x41\x80\x41\x82\x21\x80\x21\x82\xd5\x4a\x80\x83\x5f\x89\x83\x8b\xd1\x50\x88\xd1\x52\x42\x0b\x28\x22\x6f\x25\x04\x14\xe1\xe2\x62\x72\xf4\x15\x02\x62\x09\x1b\xa0\x98\x90\x95\x28\x07\xbf\x11\x2f\x17\xa3\x81\x10\xb3\x81\x9e\xa9\x14\xb3\xb9\x9e\x29\x50\x98\x1f\x6a\x18\x93\x00\x93\x84\x1c\xc8\x30\x87\x09\x7e\x1e\x0c\x00\x08\xa8\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x5d\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", + b"\x4f\x52\x43\x11\x00\x00\x0a\x06\x12\x04\x08\x01\x50\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x30\x00\x00\xe3\x12\xe7\x62\x65\x00\x01\x21\x3e\x0e\x46\x25\x0e\x2e\x46\x03\x21\x46\x03\x09\xa6\x00\x06\x00\x32\x00\x00\xe3\x92\xe4\x62\x65\x00\x01\x21\x01\x0e\x46\x25\x2e\x2e\x26\x47\x5f\x21\x20\x96\x60\x09\x60\x00\x00\x36\x00\x00\xe3\x92\xe1\x62\x65\x00\x01\x21\x61\x0e\x46\x23\x5e\x2e\x46\x03\x21\x66\x03\x3d\x53\x29\x10\x11\xc0\x00\x00\x2b\x00\x00\x0a\x13\x0a\x03\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x05\x00\x00\xff\x00\x03\x00\x00\x30\x07\x00\x00\x40\x00\x80\x05\x00\x00\x41\x4d\x07\x00\x00\x42\x00\x80\x03\x00\x00\x0a\x07\x00\x00\x42\x00\x80\x05\x00\x00\xff\x01\x88\x00\x00\x4d\xca\xc1\x0a\x80\x30\x0c\x03\xd0\x2e\x6b\xcb\x98\x17\xf1\x14\x50\xfc\xff\xcf\xb4\x66\x1e\x3c\x84\x47\x9a\xce\x1c\xb9\x1b\xb7\xf9\xda\x48\x09\x9e\xb2\xf3\x92\xce\x5b\x86\xf6\x56\x7f\x21\x41\x2f\x51\xa6\x7a\xd7\x1d\xe5\xea\xae\x3d\xca\xd5\x83\x71\x60\xd8\x17\xfc\x62\x0f\xa8\x00\x00\xe3\x4a\xe6\x62\xe1\x60\x0c\x60\xe0\xe2\xe3\x60\x14\x62\xe3\x60\x10\x60\x90\x60\x08\x60\x88\x60\xe5\x12\xe0\x60\x54\xe2\xe0\x62\x34\x10\x62\x34\x90\x60\x02\x8a\x70\x71\x09\x01\x45\xb8\xb8\x98\x1c\x7d\x85\x80\x58\x82\x05\x28\xc6\xcd\x25\xca\xc1\x68\xc4\x0b\x52\xc5\x6c\xa0\x67\x2a\x05\x22\xc0\x4a\x21\x86\x31\x09\x30\x81\xb5\xb2\x02\x00\x36\x01\x00\x25\x8c\xbd\x0a\xc2\x30\x14\x85\x73\x6f\x92\xf6\x92\x6a\x09\x01\x21\x64\x92\x4e\x75\x91\x58\x71\xc9\x64\x27\x5d\x2c\x1d\x5d\xfd\x59\xc4\x42\x37\x5f\xc0\x17\xe8\x23\x9b\xc6\xe1\x3b\x70\x0f\xdf\xb9\xc4\xf5\x17\x5d\x41\x5c\x4f\x60\x37\xeb\x53\x0d\x55\x4d\x0b\x23\x01\xb9\x90\x2e\xbf\x0f\xe3\xe3\xdd\x8d\x0e\x5f\x4f\x27\x3e\xb7\x61\x97\xb2\x49\xb9\xaf\x90\x20\x92\x27\x32\x2a\x6b\xf4\xf3\x0d\x1e\x82\x20\xe8\x59\x28\x09\x4c\x46\x4c\x33\xcb\x7a\x76\x95\x41\x47\x9f\x14\x78\x03\xde\x62\x6c\x54\x30\xb1\x51\x0a\xdb\x8b\x89\x58\x11\xbb\x22\xac\x08\x9a\xe5\x6c\x71\xbf\x3d\xb8\x39\x92\xfa\x7f\x86\x1a\xd3\x54\x1e\xa7\xee\xcc\x7e\x08\x9e\x01\x10\x01\x18\x80\x80\x10\x22\x02\x00\x0c\x28\x57\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", # broken message - b'\x4f\x52\x43\x0a\x0b\x0a\x03\x00\x00\x00\x12\x04\x08\x01\x50\x00\x0a\x15\x0a\x05\x00\x00\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x0a\x12\x0a\x06\x00\x00\x00\x00\x00\x00\x12\x08\x08\x01\x42\x02\x08\x06\x50\x00\x0a\x12\x0a\x06\x00\x00\x00\x00\x00\x00\x12\x08\x08\x01\x42\x02\x08\x04\x50\x00\x0a\x29\x0a\x04\x00\x00\x00\x00\x12\x21\x08\x01\x1a\x1b\x09\x00\x00\x00\x00\x00\x00\xe0\x3f\x11\x00\x00\x00\x00\x00\x00\xe0\x3f\x19\x00\x00\x00\x00\x00\x00\xe0\x3f\x50\x00\x0a\x15\x0a\x05\x00\x00\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\xff\x80\xff\x80\xff\x00\xff\x80\xff\x03\x42\x41\x44\xff\x80\xff\x02\x41\x4d\xff\x80\x00\x00\x00\x3f\xff\x80\xff\x01\x0a\x06\x08\x06\x10\x00\x18\x0d\x0a\x06\x08\x06\x10\x01\x18\x17\x0a\x06\x08\x06\x10\x02\x18\x14\x0a\x06\x08\x06\x10\x03\x18\x14\x0a\x06\x08\x06\x10\x04\x18\x2b\x0a\x06\x08\x06\x10\x05\x18\x17\x0a\x06\x08\x00\x10\x00\x18\x02\x0a\x06\x08\x00\x10\x01\x18\x02\x0a\x06\x08\x01\x10\x01\x18\x02\x0a\x06\x08\x00\x10\x02\x18\x02\x0a\x06\x08\x02\x10\x02\x18\x02\x0a\x06\x08\x01\x10\x02\x18\x03\x0a\x06\x08\x00\x10\x03\x18\x02\x0a\x06\x08\x02\x10\x03\x18\x02\x0a\x06\x08\x01\x10\x03\x18\x02\x0a\x06\x08\x00\x10\x04\x18\x02\x0a\x06\x08\x01\x10\x04\x18\x04\x0a\x06\x08\x00\x10\x05\x18\x02\x0a\x06\x08\x01\x10\x05\x18\x02\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x1a\x03\x47\x4d\x54\x0a\x59\x0a\x04\x08\x01\x50\x00\x0a\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x0a\x08\x08\x01\x42\x02\x08\x06\x50\x00\x0a\x08\x08\x01\x42\x02\x08\x04\x50\x00\x0a\x21\x08\x01\x1a\x1b\x09\x00\x00\x00\x00\x00\x00\xe0\x3f\x11\x00\x00\x00\x00\x00\x00\xe0\x3f\x19\x00\x00\x00\x00\x00\x00\xe0\x3f\x50\x00\x0a\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x08\x03\x10\xec\x02\x1a\x0c\x08\x03\x10\x8e\x01\x18\x1d\x20\xc1\x01\x28\x01\x22\x2e\x08\x0c\x12\x05\x01\x02\x03\x04\x05\x1a\x02\x69\x64\x1a\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x1a\x04\x76\x61\x6c\x31\x1a\x04\x76\x61\x6c\x32\x1a\x04\x76\x61\x6c\x33\x20\x00\x28\x00\x30\x00\x22\x08\x08\x04\x20\x00\x28\x00\x30\x00\x22\x08\x08\x08\x20\x00\x28\x00\x30\x00\x22\x08\x08\x08\x20\x00\x28\x00\x30\x00\x22\x08\x08\x05\x20\x00\x28\x00\x30\x00\x22\x08\x08\x01\x20\x00\x28\x00\x30\x00\x30\x01\x3a\x04\x08\x01\x50\x00\x3a\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x3a\x08\x08\x01\x42\x02\x08\x06\x50\x00\x3a\x08\x08\x01\x42\x02\x08\x04\x50\x00\x3a\x21\x08\x01\x1a\x1b\x09\x00\x00\x00\x00\x00\x00\xe0\x3f\x11\x00\x00\x00\x00\x00\x00\xe0\x3f\x19\x00\x00\x00\x00\x00\x00\xe0\x3f\x50\x00\x3a\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x40\x90\x4e\x48\x01\x08\xd5\x01\x10\x00\x18\x80\x80\x04\x22\x02\x00\x0b\x28\x5b\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18', + b"\x4f\x52\x43\x0a\x0b\x0a\x03\x00\x00\x00\x12\x04\x08\x01\x50\x00\x0a\x15\x0a\x05\x00\x00\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x0a\x12\x0a\x06\x00\x00\x00\x00\x00\x00\x12\x08\x08\x01\x42\x02\x08\x06\x50\x00\x0a\x12\x0a\x06\x00\x00\x00\x00\x00\x00\x12\x08\x08\x01\x42\x02\x08\x04\x50\x00\x0a\x29\x0a\x04\x00\x00\x00\x00\x12\x21\x08\x01\x1a\x1b\x09\x00\x00\x00\x00\x00\x00\xe0\x3f\x11\x00\x00\x00\x00\x00\x00\xe0\x3f\x19\x00\x00\x00\x00\x00\x00\xe0\x3f\x50\x00\x0a\x15\x0a\x05\x00\x00\x00\x00\x00\x12\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\xff\x80\xff\x80\xff\x00\xff\x80\xff\x03\x42\x41\x44\xff\x80\xff\x02\x41\x4d\xff\x80\x00\x00\x00\x3f\xff\x80\xff\x01\x0a\x06\x08\x06\x10\x00\x18\x0d\x0a\x06\x08\x06\x10\x01\x18\x17\x0a\x06\x08\x06\x10\x02\x18\x14\x0a\x06\x08\x06\x10\x03\x18\x14\x0a\x06\x08\x06\x10\x04\x18\x2b\x0a\x06\x08\x06\x10\x05\x18\x17\x0a\x06\x08\x00\x10\x00\x18\x02\x0a\x06\x08\x00\x10\x01\x18\x02\x0a\x06\x08\x01\x10\x01\x18\x02\x0a\x06\x08\x00\x10\x02\x18\x02\x0a\x06\x08\x02\x10\x02\x18\x02\x0a\x06\x08\x01\x10\x02\x18\x03\x0a\x06\x08\x00\x10\x03\x18\x02\x0a\x06\x08\x02\x10\x03\x18\x02\x0a\x06\x08\x01\x10\x03\x18\x02\x0a\x06\x08\x00\x10\x04\x18\x02\x0a\x06\x08\x01\x10\x04\x18\x04\x0a\x06\x08\x00\x10\x05\x18\x02\x0a\x06\x08\x01\x10\x05\x18\x02\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x12\x04\x08\x00\x10\x00\x1a\x03\x47\x4d\x54\x0a\x59\x0a\x04\x08\x01\x50\x00\x0a\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x0a\x08\x08\x01\x42\x02\x08\x06\x50\x00\x0a\x08\x08\x01\x42\x02\x08\x04\x50\x00\x0a\x21\x08\x01\x1a\x1b\x09\x00\x00\x00\x00\x00\x00\xe0\x3f\x11\x00\x00\x00\x00\x00\x00\xe0\x3f\x19\x00\x00\x00\x00\x00\x00\xe0\x3f\x50\x00\x0a\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x08\x03\x10\xec\x02\x1a\x0c\x08\x03\x10\x8e\x01\x18\x1d\x20\xc1\x01\x28\x01\x22\x2e\x08\x0c\x12\x05\x01\x02\x03\x04\x05\x1a\x02\x69\x64\x1a\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x1a\x04\x76\x61\x6c\x31\x1a\x04\x76\x61\x6c\x32\x1a\x04\x76\x61\x6c\x33\x20\x00\x28\x00\x30\x00\x22\x08\x08\x04\x20\x00\x28\x00\x30\x00\x22\x08\x08\x08\x20\x00\x28\x00\x30\x00\x22\x08\x08\x08\x20\x00\x28\x00\x30\x00\x22\x08\x08\x05\x20\x00\x28\x00\x30\x00\x22\x08\x08\x01\x20\x00\x28\x00\x30\x00\x30\x01\x3a\x04\x08\x01\x50\x00\x3a\x0c\x08\x01\x12\x06\x08\x00\x10\x00\x18\x00\x50\x00\x3a\x08\x08\x01\x42\x02\x08\x06\x50\x00\x3a\x08\x08\x01\x42\x02\x08\x04\x50\x00\x3a\x21\x08\x01\x1a\x1b\x09\x00\x00\x00\x00\x00\x00\xe0\x3f\x11\x00\x00\x00\x00\x00\x00\xe0\x3f\x19\x00\x00\x00\x00\x00\x00\xe0\x3f\x50\x00\x3a\x0c\x08\x01\x12\x06\x08\x02\x10\x02\x18\x02\x50\x00\x40\x90\x4e\x48\x01\x08\xd5\x01\x10\x00\x18\x80\x80\x04\x22\x02\x00\x0b\x28\x5b\x30\x06\x82\xf4\x03\x03\x4f\x52\x43\x18", ], - 'expected':r'''{"raw_message":"4F52430A0B0A030000001204080150000A150A050000000000120C0801120608001000180050000A120A06000000000000120808014202080650000A120A06000000000000120808014202080450000A290A0400000000122108011A1B09000000000000E03F11000000000000E03F19000000000000E03F50000A150A050000000000120C080112060802100218025000FF80FF80FF00FF80FF03424144FF80FF02414DFF800000003FFF80FF010A0608061000180D0A060806100118170A060806100218140A060806100318140A0608061004182B0A060806100518170A060800100018020A060800100118020A060801100118020A060800100218020A060802100218020A060801100218030A060800100318020A060802100318020A060801100318020A060800100418020A060801100418040A060800100518020A060801100518021204080010001204080010001204080010001204080010001204080010001204080010001A03474D540A590A04080150000A0C0801120608001000180050000A0808014202080650000A0808014202080450000A2108011A1B09000000000000E03F11000000000000E03F19000000000000E03F50000A0C080112060802100218025000080310EC021A0C0803108E01181D20C1012801222E080C120501020304051A0269641A07626C6F636B4E6F1A0476616C311A0476616C321A0476616C33200028003000220808042000280030002208080820002800300022080808200028003000220808052000280030002208080120002800300030013A04080150003A0C0801120608001000180050003A0808014202080650003A0808014202080450003A2108011A1B09000000000000E03F11000000000000E03F19000000000000E03F50003A0C08011206080210021802500040904E480108D5011000188080042202000B285B300682F403034F524318","error":"Cannot parse string 'BAD' as UInt16: syntax error at begin of string. Note: there are toUInt16OrZero and toUInt16OrNull functions, which returns zero\/NULL instead of throwing exception."}''', - 'printable':False, - } + "expected": r"""{"raw_message":"4F52430A0B0A030000001204080150000A150A050000000000120C0801120608001000180050000A120A06000000000000120808014202080650000A120A06000000000000120808014202080450000A290A0400000000122108011A1B09000000000000E03F11000000000000E03F19000000000000E03F50000A150A050000000000120C080112060802100218025000FF80FF80FF00FF80FF03424144FF80FF02414DFF800000003FFF80FF010A0608061000180D0A060806100118170A060806100218140A060806100318140A0608061004182B0A060806100518170A060800100018020A060800100118020A060801100118020A060800100218020A060802100218020A060801100218030A060800100318020A060802100318020A060801100318020A060800100418020A060801100418040A060800100518020A060801100518021204080010001204080010001204080010001204080010001204080010001204080010001A03474D540A590A04080150000A0C0801120608001000180050000A0808014202080650000A0808014202080450000A2108011A1B09000000000000E03F11000000000000E03F19000000000000E03F50000A0C080112060802100218025000080310EC021A0C0803108E01181D20C1012801222E080C120501020304051A0269641A07626C6F636B4E6F1A0476616C311A0476616C321A0476616C33200028003000220808042000280030002208080820002800300022080808200028003000220808052000280030002208080120002800300030013A04080150003A0C0801120608001000180050003A0808014202080650003A0808014202080450003A2108011A1B09000000000000E03F11000000000000E03F19000000000000E03F50003A0C08011206080210021802500040904E480108D5011000188080042202000B285B300682F403034F524318","error":"Cannot parse string 'BAD' as UInt16: syntax error at begin of string. Note: there are toUInt16OrZero and toUInt16OrNull functions, which returns zero\/NULL instead of throwing exception."}""", + "printable": False, + }, } - topic_name_prefix = 'format_tests_4_stream_' + topic_name_prefix = "format_tests_4_stream_" for format_name, format_opts in list(all_formats.items()): - logging.debug(f'Set up {format_name}') + logging.debug(f"Set up {format_name}") topic_name = f"{topic_name_prefix}{format_name}" - data_sample = format_opts['data_sample'] + data_sample = format_opts["data_sample"] data_prefix = [] - raw_message = '_raw_message' + raw_message = "_raw_message" # prepend empty value when supported - if format_opts.get('supports_empty_value', False): - data_prefix = data_prefix + [''] - if format_opts.get('printable', False) == False: - raw_message = 'hex(_raw_message)' + if format_opts.get("supports_empty_value", False): + data_prefix = data_prefix + [""] + if format_opts.get("printable", False) == False: + raw_message = "hex(_raw_message)" kafka_produce(kafka_cluster, topic_name, data_prefix + data_sample) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka_{format_name}; CREATE TABLE test.kafka_{format_name} ( @@ -3057,16 +3607,27 @@ def test_kafka_formats_with_broken_message(kafka_cluster): CREATE MATERIALIZED VIEW test.kafka_errors_{format_name}_mv Engine=Log AS SELECT {raw_message} as raw_message, _error as error, _topic as topic, _partition as partition, _offset as offset FROM test.kafka_{format_name} WHERE length(_error) > 0; - '''.format(topic_name=topic_name, format_name=format_name, raw_message=raw_message, - extra_settings=format_opts.get('extra_settings') or '')) + """.format( + topic_name=topic_name, + format_name=format_name, + raw_message=raw_message, + extra_settings=format_opts.get("extra_settings") or "", + ) + ) for format_name, format_opts in list(all_formats.items()): - logging.debug('Checking {format_name}') + logging.debug("Checking {format_name}") topic_name = f"{topic_name_prefix}{format_name}" # shift offsets by 1 if format supports empty value - offsets = [1, 2, 3] if format_opts.get('supports_empty_value', False) else [0, 1, 2] - result = instance.query('SELECT * FROM test.kafka_data_{format_name}_mv;'.format(format_name=format_name)) - expected = '''\ + offsets = ( + [1, 2, 3] if format_opts.get("supports_empty_value", False) else [0, 1, 2] + ) + result = instance.query( + "SELECT * FROM test.kafka_data_{format_name}_mv;".format( + format_name=format_name + ) + ) + expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} 1 0 AM 0.5 1 {topic_name} 0 {offset_1} 2 0 AM 0.5 1 {topic_name} 0 {offset_1} @@ -3084,19 +3645,37 @@ def test_kafka_formats_with_broken_message(kafka_cluster): 14 0 AM 0.5 1 {topic_name} 0 {offset_1} 15 0 AM 0.5 1 {topic_name} 0 {offset_1} 0 0 AM 0.5 1 {topic_name} 0 {offset_2} -'''.format(topic_name=topic_name, offset_0=offsets[0], offset_1=offsets[1], offset_2=offsets[2]) +""".format( + topic_name=topic_name, + offset_0=offsets[0], + offset_1=offsets[1], + offset_2=offsets[2], + ) # print(('Checking result\n {result} \n expected \n {expected}\n'.format(result=str(result), expected=str(expected)))) - assert TSV(result) == TSV(expected), 'Proper result for format: {}'.format(format_name) - errors_result = ast.literal_eval(instance.query('SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv format JSONEachRow'.format(format_name=format_name))) - errors_expected = ast.literal_eval(format_opts['expected']) + assert TSV(result) == TSV(expected), "Proper result for format: {}".format( + format_name + ) + errors_result = ast.literal_eval( + instance.query( + "SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv format JSONEachRow".format( + format_name=format_name + ) + ) + ) + errors_expected = ast.literal_eval(format_opts["expected"]) # print(errors_result.strip()) # print(errors_expected.strip()) - assert errors_result['raw_message'] == errors_expected['raw_message'], 'Proper raw_message for format: {}'.format(format_name) + assert ( + errors_result["raw_message"] == errors_expected["raw_message"] + ), "Proper raw_message for format: {}".format(format_name) # Errors text can change, just checking prefixes - assert errors_expected['error'] in errors_result['error'], 'Proper error for format: {}'.format(format_name) + assert ( + errors_expected["error"] in errors_result["error"] + ), "Proper error for format: {}".format(format_name) kafka_delete_topic(admin_client, topic_name) -def wait_for_new_data(table_name, prev_count = 0, max_retries = 120): + +def wait_for_new_data(table_name, prev_count=0, max_retries=120): retries = 0 while True: new_count = int(instance.query("SELECT count() FROM {}".format(table_name))) @@ -3109,15 +3688,19 @@ def wait_for_new_data(table_name, prev_count = 0, max_retries = 120): if retries > max_retries: raise Exception("No new data :(") + def test_kafka_consumer_failover(kafka_cluster): # for backporting: # admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) topic_name = "kafka_consumer_failover" kafka_create_topic(admin_client, topic_name, num_partitions=2) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.kafka; DROP TABLE IF EXISTS test.kafka2; @@ -3167,85 +3750,157 @@ def test_kafka_consumer_failover(kafka_cluster): CREATE MATERIALIZED VIEW test.kafka3_mv TO test.destination AS SELECT key, value, 'kafka3' as _consumed_by FROM test.kafka3; - ''') + """ + ) - - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(cluster.kafka_port), value_serializer=producer_serializer, key_serializer=producer_serializer) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(cluster.kafka_port), + value_serializer=producer_serializer, + key_serializer=producer_serializer, + ) ## all 3 attached, 2 working - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':1,'value': 1}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':1,'value': 1}), partition=1) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 1, "value": 1}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 1, "value": 1}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination') + prev_count = wait_for_new_data("test.destination") ## 2 attached, 2 working - instance.query('DETACH TABLE test.kafka') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':2,'value': 2}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':2,'value': 2}), partition=1) + instance.query("DETACH TABLE test.kafka") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 2, "value": 2}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 2, "value": 2}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) ## 1 attached, 1 working - instance.query('DETACH TABLE test.kafka2') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':3,'value': 3}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':3,'value': 3}), partition=1) + instance.query("DETACH TABLE test.kafka2") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 3, "value": 3}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 3, "value": 3}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) ## 2 attached, 2 working - instance.query('ATTACH TABLE test.kafka') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':4,'value': 4}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':4,'value': 4}), partition=1) + instance.query("ATTACH TABLE test.kafka") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 4, "value": 4}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 4, "value": 4}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) ## 1 attached, 1 working - instance.query('DETACH TABLE test.kafka3') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':5,'value': 5}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':5,'value': 5}), partition=1) + instance.query("DETACH TABLE test.kafka3") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 5, "value": 5}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 5, "value": 5}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) ## 2 attached, 2 working - instance.query('ATTACH TABLE test.kafka2') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':6,'value': 6}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':6,'value': 6}), partition=1) + instance.query("ATTACH TABLE test.kafka2") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 6, "value": 6}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 6, "value": 6}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) ## 3 attached, 2 working - instance.query('ATTACH TABLE test.kafka3') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':7,'value': 7}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':7,'value': 7}), partition=1) + instance.query("ATTACH TABLE test.kafka3") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 7, "value": 7}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 7, "value": 7}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) ## 2 attached, same 2 working - instance.query('DETACH TABLE test.kafka3') - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':8,'value': 8}), partition=0) - producer.send(topic='kafka_consumer_failover', value=json.dumps({'key':8,'value': 8}), partition=1) + instance.query("DETACH TABLE test.kafka3") + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 8, "value": 8}), + partition=0, + ) + producer.send( + topic="kafka_consumer_failover", + value=json.dumps({"key": 8, "value": 8}), + partition=1, + ) producer.flush() - prev_count = wait_for_new_data('test.destination', prev_count) + prev_count = wait_for_new_data("test.destination", prev_count) kafka_delete_topic(admin_client, topic_name) def test_kafka_predefined_configuration(kafka_cluster): - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) - topic_name = 'conf' + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) + topic_name = "conf" kafka_create_topic(admin_client, topic_name) messages = [] for i in range(50): - messages.append('{i}, {i}'.format(i=i)) + messages.append("{i}, {i}".format(i=i)) kafka_produce(kafka_cluster, topic_name, messages) - instance.query(f''' + instance.query( + f""" CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka(kafka1, kafka_format='CSV'); - ''') + """ + ) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.kafka', ignore_error=True) + result += instance.query("SELECT * FROM test.kafka", ignore_error=True) if kafka_check_result(result): break kafka_check_result(result, True) @@ -3256,33 +3911,40 @@ def test_issue26643(kafka_cluster): # for backporting: # admin_client = KafkaAdminClient(bootstrap_servers="localhost:9092") - admin_client = KafkaAdminClient(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port)) - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), value_serializer=producer_serializer) + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port), + value_serializer=producer_serializer, + ) topic_list = [] - topic_list.append(NewTopic(name="test_issue26643", num_partitions=4, replication_factor=1)) + topic_list.append( + NewTopic(name="test_issue26643", num_partitions=4, replication_factor=1) + ) admin_client.create_topics(new_topics=topic_list, validate_only=False) msg = message_with_repeated_pb2.Message( tnow=1629000000, - server='server1', - clien='host1', + server="server1", + clien="host1", sPort=443, cPort=50000, r=[ - message_with_repeated_pb2.dd(name='1', type=444, ttl=123123, data=b'adsfasd'), - message_with_repeated_pb2.dd(name='2') + message_with_repeated_pb2.dd( + name="1", type=444, ttl=123123, data=b"adsfasd" + ), + message_with_repeated_pb2.dd(name="2"), ], - method='GET' + method="GET", ) - data = b'' + data = b"" serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg - msg = message_with_repeated_pb2.Message( - tnow=1629000002 - ) + msg = message_with_repeated_pb2.Message(tnow=1629000002) serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg @@ -3293,7 +3955,8 @@ def test_issue26643(kafka_cluster): producer.send(topic="test_issue26643", value=data) producer.flush() - instance.query(''' + instance.query( + """ CREATE TABLE IF NOT EXISTS test.test_queue ( `tnow` UInt32, @@ -3322,7 +3985,7 @@ def test_issue26643(kafka_cluster): CREATE TABLE test.log ( - `tnow` DateTime CODEC(DoubleDelta, LZ4), + `tnow` DateTime('Asia/Istanbul') CODEC(DoubleDelta, LZ4), `server` LowCardinality(String), `client` LowCardinality(String), `sPort` LowCardinality(UInt16), @@ -3354,22 +4017,23 @@ def test_issue26643(kafka_cluster): a.`r.data` AS `r.data`, a.method AS method FROM test.test_queue AS a; - ''') + """ + ) instance.wait_for_log_line("Committed offset") - result = instance.query('SELECT * FROM test.log') + result = instance.query("SELECT * FROM test.log") - expected = '''\ + expected = """\ 2021-08-15 07:00:00 server1 443 50000 ['1','2'] [0,0] [444,0] [123123,0] ['adsfasd',''] GET 2021-08-15 07:00:02 0 0 [] [] [] [] [] 2021-08-15 07:00:02 0 0 [] [] [] [] [] -''' +""" assert TSV(result) == TSV(expected) # kafka_cluster.open_bash_shell('instance') -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh index 0a746eb1a67..45fb93792e0 100644 --- a/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh @@ -90,6 +90,7 @@ create_admin_user() { } create_keytabs() { + rm /tmp/keytab/*.keytab # kadmin.local -q "addprinc -randkey hdfs/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}" diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index d06f971557b..fb00403b952 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -7,7 +7,13 @@ from helpers.cluster import ClickHouseCluster import subprocess cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_kerberized_hdfs=True, user_configs=[], main_configs=['configs/hdfs.xml']) +node1 = cluster.add_instance( + "node1", + with_kerberized_hdfs=True, + user_configs=[], + main_configs=["configs/hdfs.xml"], +) + @pytest.fixture(scope="module") def started_cluster(): @@ -32,13 +38,18 @@ def test_read_table(started_cluster): api_read = hdfs_api.read_data("/simple_table_function") assert api_read == data - select_read = node1.query("select * from hdfs('hdfs://kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')") + select_read = node1.query( + "select * from hdfs('hdfs://kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')" + ) assert select_read == data + def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query("create table SimpleHDFSStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage1', 'TSV')") + node1.query( + "create table SimpleHDFSStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage1', 'TSV')" + ) node1.query("insert into SimpleHDFSStorage2 values (1, 'Mark', 72.53)") api_read = hdfs_api.read_data("/simple_storage1") @@ -47,12 +58,15 @@ def test_read_write_storage(started_cluster): select_read = node1.query("select * from SimpleHDFSStorage2") assert select_read == "1\tMark\t72.53\n" + def test_write_storage_not_expired(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query("create table SimpleHDFSStorageNotExpired (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage_not_expired', 'TSV')") + node1.query( + "create table SimpleHDFSStorageNotExpired (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage_not_expired', 'TSV')" + ) - time.sleep(15) # wait for ticket expiration + time.sleep(15) # wait for ticket expiration node1.query("insert into SimpleHDFSStorageNotExpired values (1, 'Mark', 72.53)") api_read = hdfs_api.read_data("/simple_storage_not_expired") @@ -61,18 +75,28 @@ def test_write_storage_not_expired(started_cluster): select_read = node1.query("select * from SimpleHDFSStorageNotExpired") assert select_read == "1\tMark\t72.53\n" + def test_two_users(started_cluster): hdfs_api = started_cluster.hdfs_api - node1.query("create table HDFSStorOne (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/storage_user_one', 'TSV')") + node1.query( + "create table HDFSStorOne (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/storage_user_one', 'TSV')" + ) node1.query("insert into HDFSStorOne values (1, 'Real', 86.00)") - node1.query("create table HDFSStorTwo (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV')") + node1.query( + "create table HDFSStorTwo (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV')" + ) node1.query("insert into HDFSStorTwo values (1, 'Ideal', 74.00)") - select_read_1 = node1.query("select * from hdfs('hdfs://kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV', 'id UInt64, text String, number Float64')") + select_read_1 = node1.query( + "select * from hdfs('hdfs://kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV', 'id UInt64, text String, number Float64')" + ) + + select_read_2 = node1.query( + "select * from hdfs('hdfs://suser@kerberizedhdfs1:9010/storage_user_one', 'TSV', 'id UInt64, text String, number Float64')" + ) - select_read_2 = node1.query("select * from hdfs('hdfs://suser@kerberizedhdfs1:9010/storage_user_one', 'TSV', 'id UInt64, text String, number Float64')") def test_read_table_expired(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -80,35 +104,49 @@ def test_read_table_expired(started_cluster): data = "1\tSerialize\t555.222\n2\tData\t777.333\n" hdfs_api.write_data("/simple_table_function_relogin", data) - started_cluster.pause_container('hdfskerberos') + started_cluster.pause_container("hdfskerberos") time.sleep(15) try: - select_read = node1.query("select * from hdfs('hdfs://reloginuser&kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')") + select_read = node1.query( + "select * from hdfs('hdfs://reloginuser&kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')" + ) assert False, "Exception have to be thrown" except Exception as ex: assert "DB::Exception: kinit failure:" in str(ex) - started_cluster.unpause_container('hdfskerberos') + started_cluster.unpause_container("hdfskerberos") + def test_prohibited(started_cluster): - node1.query("create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')") + node1.query( + "create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')" + ) try: node1.query("insert into HDFSStorTwoProhibited values (1, 'SomeOne', 74.00)") assert False, "Exception have to be thrown" except Exception as ex: - assert "Unable to open HDFS file: /storage_user_two_prohibited error: Permission denied: user=specuser, access=WRITE" in str(ex) + assert ( + "Unable to open HDFS file: /storage_user_two_prohibited error: Permission denied: user=specuser, access=WRITE" + in str(ex) + ) + def test_cache_path(started_cluster): - node1.query("create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')") + node1.query( + "create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')" + ) try: node1.query("insert into HDFSStorCachePath values (1, 'FatMark', 92.53)") assert False, "Exception have to be thrown" except Exception as ex: - assert "DB::Exception: hadoop.security.kerberos.ticket.cache.path cannot be set per user" in str(ex) + assert ( + "DB::Exception: hadoop.security.kerberos.ticket.cache.path cannot be set per user" + in str(ex) + ) -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh index 723868ec68a..07437c42359 100644 --- a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh @@ -90,6 +90,7 @@ create_admin_user() { } create_keytabs() { + rm /tmp/keytab/*.keytab kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@${REALM}" kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@${REALM}" diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 567a9b7184d..f4aea059c05 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -20,20 +20,27 @@ from kafka.protocol.group import MemberAssignment import socket cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs=['configs/kafka.xml'], - user_configs=['configs/users.xml'], - with_kerberized_kafka=True, - clickhouse_path_dir="clickhouse_path") +instance = cluster.add_instance( + "instance", + main_configs=["configs/kafka.xml"], + user_configs=["configs/users.xml"], + with_kerberized_kafka=True, + clickhouse_path_dir="clickhouse_path", +) + def producer_serializer(x): return x.encode() if isinstance(x, str) else x + def get_kafka_producer(port, serializer): errors = [] for _ in range(15): try: - producer = KafkaProducer(bootstrap_servers="localhost:{}".format(port), value_serializer=serializer) + producer = KafkaProducer( + bootstrap_servers="localhost:{}".format(port), + value_serializer=serializer, + ) logging.debug("Kafka Connection establised: localhost:{}".format(port)) return producer except Exception as e: @@ -42,9 +49,16 @@ def get_kafka_producer(port, serializer): raise Exception("Connection not establised, {}".format(errors)) + def kafka_produce(kafka_cluster, topic, messages, timestamp=None): - logging.debug("kafka_produce server:{}:{} topic:{}".format("localhost", kafka_cluster.kerberized_kafka_port, topic)) - producer = get_kafka_producer(kafka_cluster.kerberized_kafka_port, producer_serializer) + logging.debug( + "kafka_produce server:{}:{} topic:{}".format( + "localhost", kafka_cluster.kerberized_kafka_port, topic + ) + ) + producer = get_kafka_producer( + kafka_cluster.kerberized_kafka_port, producer_serializer + ) for message in messages: producer.send(topic=topic, value=message, timestamp_ms=timestamp) producer.flush() @@ -52,13 +66,16 @@ def kafka_produce(kafka_cluster, topic, messages, timestamp=None): # Fixtures + @pytest.fixture(scope="module") def kafka_cluster(): try: cluster.start() if instance.is_debug_build(): # https://github.com/ClickHouse/ClickHouse/issues/27651 - pytest.skip("librdkafka calls system function for kinit which does not pass harmful check in debug build") + pytest.skip( + "librdkafka calls system function for kinit which does not pass harmful check in debug build" + ) yield cluster finally: cluster.shutdown() @@ -66,15 +83,27 @@ def kafka_cluster(): @pytest.fixture(autouse=True) def kafka_setup_teardown(): - instance.query('DROP DATABASE IF EXISTS test; CREATE DATABASE test;') + instance.query("DROP DATABASE IF EXISTS test; CREATE DATABASE test;") yield # run test + # Tests -def test_kafka_json_as_string(kafka_cluster): - kafka_produce(kafka_cluster, 'kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) - instance.query(''' +def test_kafka_json_as_string(kafka_cluster): + kafka_produce( + kafka_cluster, + "kafka_json_as_string", + [ + '{"t": 123, "e": {"x": "woof"} }', + "", + '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}', + ], + ) + + instance.query( + """ CREATE TABLE test.kafka (field String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', @@ -83,24 +112,29 @@ def test_kafka_json_as_string(kafka_cluster): kafka_group_name = 'kafka_json_as_string', kafka_format = 'JSONAsString', kafka_flush_interval_ms=1000; - ''') + """ + ) time.sleep(3) - result = instance.query('SELECT * FROM test.kafka;') - expected = '''\ + result = instance.query("SELECT * FROM test.kafka;") + expected = """\ {"t": 123, "e": {"x": "woof"} } {"t": 124, "e": {"x": "test"} } {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} -''' +""" assert TSV(result) == TSV(expected) - assert instance.contains_in_log("Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows") + assert instance.contains_in_log( + "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" + ) + def test_kafka_json_as_string_no_kdc(kafka_cluster): # When the test is run alone (not preceded by any other kerberized kafka test), # we need a ticket to # assert instance.contains_in_log("Ticket expired") - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka_no_kdc_warm_up (field String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', @@ -109,16 +143,27 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): kafka_commit_on_select = 1, kafka_format = 'JSONAsString', kafka_flush_interval_ms=1000; - ''') + """ + ) - instance.query('SELECT * FROM test.kafka_no_kdc_warm_up;') + instance.query("SELECT * FROM test.kafka_no_kdc_warm_up;") - kafka_produce(kafka_cluster, 'kafka_json_as_string_no_kdc', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + kafka_produce( + kafka_cluster, + "kafka_json_as_string_no_kdc", + [ + '{"t": 123, "e": {"x": "woof"} }', + "", + '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}', + ], + ) - kafka_cluster.pause_container('kafka_kerberos') - time.sleep(45) # wait for ticket expiration + kafka_cluster.pause_container("kafka_kerberos") + time.sleep(45) # wait for ticket expiration - instance.query(''' + instance.query( + """ CREATE TABLE test.kafka_no_kdc (field String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', @@ -127,13 +172,13 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): kafka_commit_on_select = 1, kafka_format = 'JSONAsString', kafka_flush_interval_ms=1000; - ''') + """ + ) - result = instance.query('SELECT * FROM test.kafka_no_kdc;') - expected = '' - - kafka_cluster.unpause_container('kafka_kerberos') + result = instance.query("SELECT * FROM test.kafka_no_kdc;") + expected = "" + kafka_cluster.unpause_container("kafka_kerberos") assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") @@ -141,7 +186,7 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): assert instance.contains_in_log("Kerberos ticket refresh failed") -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index 67b5b42b1ec..76713ea7f3b 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -10,10 +10,15 @@ from helpers.cluster import ClickHouseCluster def started_cluster(request): try: cluster = ClickHouseCluster(__file__) - node = cluster.add_instance('node', - main_configs=["configs_secure/config.d/ssl_conf.xml", "configs/named_collections.xml"], - with_mongo=True, - with_mongo_secure=request.param) + node = cluster.add_instance( + "node", + main_configs=[ + "configs_secure/config.d/ssl_conf.xml", + "configs/named_collections.xml", + ], + with_mongo=True, + with_mongo_secure=request.param, + ) cluster.start() yield cluster finally: @@ -21,76 +26,95 @@ def started_cluster(request): def get_mongo_connection(started_cluster, secure=False, with_credentials=True): - connection_str = '' + connection_str = "" if with_credentials: - connection_str = 'mongodb://root:clickhouse@localhost:{}'.format(started_cluster.mongo_port) + connection_str = "mongodb://root:clickhouse@localhost:{}".format( + started_cluster.mongo_port + ) else: - connection_str = 'mongodb://localhost:{}'.format(started_cluster.mongo_no_cred_port) + connection_str = "mongodb://localhost:{}".format( + started_cluster.mongo_no_cred_port + ) if secure: - connection_str += '/?tls=true&tlsAllowInvalidCertificates=true' + connection_str += "/?tls=true&tlsAllowInvalidCertificates=true" return pymongo.MongoClient(connection_str) -@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_simple_select(started_cluster): mongo_connection = get_mongo_connection(started_cluster) - db = mongo_connection['test'] - db.add_user('root', 'clickhouse') - simple_mongo_table = db['simple_table'] + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i)}) + data.append({"key": i, "data": hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instances['node'] + node = started_cluster.instances["node"] node.query( - "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')") + "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse')" + ) - assert node.query("SELECT COUNT() FROM simple_mongo_table") == '100\n' - assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + '\n' + assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n" + assert ( + node.query("SELECT sum(key) FROM simple_mongo_table") + == str(sum(range(0, 100))) + "\n" + ) - assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + '\n' + assert ( + node.query("SELECT data from simple_mongo_table where key = 42") + == hex(42 * 42) + "\n" + ) node.query("DROP TABLE simple_mongo_table") simple_mongo_table.drop() -@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_complex_data_type(started_cluster): mongo_connection = get_mongo_connection(started_cluster) - db = mongo_connection['test'] - db.add_user('root', 'clickhouse') - incomplete_mongo_table = db['complex_table'] + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + incomplete_mongo_table = db["complex_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i), 'dict': {'a': i, 'b': str(i)}}) + data.append({"key": i, "data": hex(i * i), "dict": {"a": i, "b": str(i)}}) incomplete_mongo_table.insert_many(data) - node = started_cluster.instances['node'] + node = started_cluster.instances["node"] node.query( - "CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')") + "CREATE TABLE incomplete_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'complex_table', 'root', 'clickhouse')" + ) - assert node.query("SELECT COUNT() FROM incomplete_mongo_table") == '100\n' - assert node.query("SELECT sum(key) FROM incomplete_mongo_table") == str(sum(range(0, 100))) + '\n' + assert node.query("SELECT COUNT() FROM incomplete_mongo_table") == "100\n" + assert ( + node.query("SELECT sum(key) FROM incomplete_mongo_table") + == str(sum(range(0, 100))) + "\n" + ) - assert node.query("SELECT data from incomplete_mongo_table where key = 42") == hex(42 * 42) + '\n' + assert ( + node.query("SELECT data from incomplete_mongo_table where key = 42") + == hex(42 * 42) + "\n" + ) node.query("DROP TABLE incomplete_mongo_table") incomplete_mongo_table.drop() -@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_incorrect_data_type(started_cluster): mongo_connection = get_mongo_connection(started_cluster) - db = mongo_connection['test'] - db.add_user('root', 'clickhouse') - strange_mongo_table = db['strange_table'] + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + strange_mongo_table = db["strange_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i), 'aaaa': 'Hello'}) + data.append({"key": i, "data": hex(i * i), "aaaa": "Hello"}) strange_mongo_table.insert_many(data) - node = started_cluster.instances['node'] + node = started_cluster.instances["node"] node.query( - "CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") + "CREATE TABLE strange_mongo_table(key String, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')" + ) with pytest.raises(QueryRuntimeException): node.query("SELECT COUNT() FROM strange_mongo_table") @@ -99,7 +123,8 @@ def test_incorrect_data_type(started_cluster): node.query("SELECT uniq(key) FROM strange_mongo_table") node.query( - "CREATE TABLE strange_mongo_table2(key UInt64, data String, bbbb String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')") + "CREATE TABLE strange_mongo_table2(key UInt64, data String, bbbb String) ENGINE = MongoDB('mongo1:27017', 'test', 'strange_table', 'root', 'clickhouse')" + ) with pytest.raises(QueryRuntimeException): node.query("SELECT bbbb FROM strange_mongo_table2") @@ -107,79 +132,102 @@ def test_incorrect_data_type(started_cluster): node.query("DROP TABLE strange_mongo_table2") strange_mongo_table.drop() -@pytest.mark.parametrize('started_cluster', [True], indirect=['started_cluster']) + +@pytest.mark.parametrize("started_cluster", [True], indirect=["started_cluster"]) def test_secure_connection(started_cluster): mongo_connection = get_mongo_connection(started_cluster, secure=True) - db = mongo_connection['test'] - db.add_user('root', 'clickhouse') - simple_mongo_table = db['simple_table'] + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i)}) + data.append({"key": i, "data": hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instances['node'] + node = started_cluster.instances["node"] node.query( - "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', 'ssl=true')") + "CREATE TABLE simple_mongo_table(key UInt64, data String) ENGINE = MongoDB('mongo1:27017', 'test', 'simple_table', 'root', 'clickhouse', 'ssl=true')" + ) - assert node.query("SELECT COUNT() FROM simple_mongo_table") == '100\n' - assert node.query("SELECT sum(key) FROM simple_mongo_table") == str(sum(range(0, 100))) + '\n' + assert node.query("SELECT COUNT() FROM simple_mongo_table") == "100\n" + assert ( + node.query("SELECT sum(key) FROM simple_mongo_table") + == str(sum(range(0, 100))) + "\n" + ) - assert node.query("SELECT data from simple_mongo_table where key = 42") == hex(42 * 42) + '\n' + assert ( + node.query("SELECT data from simple_mongo_table where key = 42") + == hex(42 * 42) + "\n" + ) node.query("DROP TABLE simple_mongo_table") simple_mongo_table.drop() -@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_predefined_connection_configuration(started_cluster): mongo_connection = get_mongo_connection(started_cluster) - db = mongo_connection['test'] - db.add_user('root', 'clickhouse') - simple_mongo_table = db['simple_table'] + db = mongo_connection["test"] + db.add_user("root", "clickhouse") + simple_mongo_table = db["simple_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i)}) + data.append({"key": i, "data": hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instances['node'] - node.query("create table simple_mongo_table(key UInt64, data String) engine = MongoDB(mongo1)") - assert node.query("SELECT count() FROM simple_mongo_table") == '100\n' + node = started_cluster.instances["node"] + node.query( + "create table simple_mongo_table(key UInt64, data String) engine = MongoDB(mongo1)" + ) + assert node.query("SELECT count() FROM simple_mongo_table") == "100\n" simple_mongo_table.drop() -@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_no_credentials(started_cluster): mongo_connection = get_mongo_connection(started_cluster, with_credentials=False) - db = mongo_connection['test'] - simple_mongo_table = db['simple_table'] + db = mongo_connection["test"] + simple_mongo_table = db["simple_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i)}) + data.append({"key": i, "data": hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instances['node'] - node.query("create table simple_mongo_table_2(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', '', '')") - assert node.query("SELECT count() FROM simple_mongo_table_2") == '100\n' + node = started_cluster.instances["node"] + node.query( + "create table simple_mongo_table_2(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', '', '')" + ) + assert node.query("SELECT count() FROM simple_mongo_table_2") == "100\n" simple_mongo_table.drop() -@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster']) + +@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"]) def test_auth_source(started_cluster): mongo_connection = get_mongo_connection(started_cluster, with_credentials=False) - admin_db = mongo_connection['admin'] - admin_db.add_user('root', 'clickhouse', roles=[{ 'role': "userAdminAnyDatabase", 'db': "admin" }, "readWriteAnyDatabase"]) - simple_mongo_table = admin_db['simple_table'] + admin_db = mongo_connection["admin"] + admin_db.add_user( + "root", + "clickhouse", + roles=[{"role": "userAdminAnyDatabase", "db": "admin"}, "readWriteAnyDatabase"], + ) + simple_mongo_table = admin_db["simple_table"] data = [] for i in range(0, 50): - data.append({'key': i, 'data': hex(i * i)}) + data.append({"key": i, "data": hex(i * i)}) simple_mongo_table.insert_many(data) - db = mongo_connection['test'] - simple_mongo_table = db['simple_table'] + db = mongo_connection["test"] + simple_mongo_table = db["simple_table"] data = [] for i in range(0, 100): - data.append({'key': i, 'data': hex(i * i)}) + data.append({"key": i, "data": hex(i * i)}) simple_mongo_table.insert_many(data) - node = started_cluster.instances['node'] - node.query("create table simple_mongo_table_fail(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse')") + node = started_cluster.instances["node"] + node.query( + "create table simple_mongo_table_fail(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse')" + ) node.query_and_get_error("SELECT count() FROM simple_mongo_table_fail") - node.query("create table simple_mongo_table_ok(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse', 'authSource=admin')") - assert node.query("SELECT count() FROM simple_mongo_table_ok") == '100\n' + node.query( + "create table simple_mongo_table_ok(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse', 'authSource=admin')" + ) + assert node.query("SELECT count() FROM simple_mongo_table_ok") == "100\n" simple_mongo_table.drop() diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index 713a8793f48..34ef17327f9 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -10,9 +10,20 @@ from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml', 'configs/named_collections.xml'], with_mysql=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_mysql_cluster=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], user_configs=['configs/users.xml'], with_mysql=True) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/named_collections.xml"], + with_mysql=True, +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_mysql_cluster=True +) +node3 = cluster.add_instance( + "node3", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.xml"], + with_mysql=True, +) create_table_sql_template = """ CREATE TABLE `clickhouse`.`{}` ( @@ -28,18 +39,24 @@ drop_table_sql_template = """ DROP TABLE IF EXISTS `clickhouse`.`{}`; """ + def get_mysql_conn(started_cluster, host): - conn = pymysql.connect(user='root', password='clickhouse', host=host, port=started_cluster.mysql_port) + conn = pymysql.connect( + user="root", password="clickhouse", host=host, port=started_cluster.mysql_port + ) return conn + def create_mysql_table(conn, tableName): with conn.cursor() as cursor: cursor.execute(create_table_sql_template.format(tableName)) + def drop_mysql_table(conn, tableName): with conn.cursor() as cursor: cursor.execute(drop_table_sql_template.format(tableName)) + def create_mysql_db(conn, name): with conn.cursor() as cursor: cursor.execute("DROP DATABASE IF EXISTS {}".format(name)) @@ -52,11 +69,11 @@ def started_cluster(): cluster.start() conn = get_mysql_conn(cluster, cluster.mysql_ip) - create_mysql_db(conn, 'clickhouse') + create_mysql_db(conn, "clickhouse") ## create mysql db and table conn1 = get_mysql_conn(cluster, cluster.mysql2_ip) - create_mysql_db(conn1, 'clickhouse') + create_mysql_db(conn1, "clickhouse") yield cluster finally: @@ -64,274 +81,424 @@ def started_cluster(): def test_many_connections(started_cluster): - table_name = 'test_many_connections' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_many_connections" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) - node1.query("INSERT INTO {} (id, name) SELECT number, concat('name_', toString(number)) from numbers(10) ".format(table_name)) + node1.query( + "INSERT INTO {} (id, name) SELECT number, concat('name_', toString(number)) from numbers(10) ".format( + table_name + ) + ) query = "SELECT count() FROM (" - for i in range (24): + for i in range(24): query += "SELECT id FROM {t} UNION ALL " query += "SELECT id FROM {t})" - assert node1.query(query.format(t=table_name)) == '250\n' + assert node1.query(query.format(t=table_name)) == "250\n" drop_mysql_table(conn, table_name) conn.close() def test_insert_select(started_cluster): - table_name = 'test_insert_select' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_insert_select" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( - table_name)) - assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '30000' + table_name + ) + ) + assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == "10000" + assert ( + node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == "30000" + ) conn.close() def test_replace_select(started_cluster): - table_name = 'test_replace_select' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_replace_select" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse', 1); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( - table_name)) + table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( - table_name)) - assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '30000' + table_name + ) + ) + assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == "10000" + assert ( + node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == "30000" + ) conn.close() def test_insert_on_duplicate_select(started_cluster): - table_name = 'test_insert_on_duplicate_select' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_insert_on_duplicate_select" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse', 0, 'update money = money + values(money)'); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( - table_name)) + table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( - table_name)) - assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '60000' + table_name + ) + ) + assert node1.query("SELECT count() FROM {}".format(table_name)).rstrip() == "10000" + assert ( + node1.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == "60000" + ) conn.close() def test_where(started_cluster): - table_name = 'test_where' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_where" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) node1.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000) ".format( - table_name)) - assert node1.query("SELECT count() FROM {} WHERE name LIKE '%name_%'".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT count() FROM {} WHERE name NOT LIKE '%tmp_%'".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT count() FROM {} WHERE money IN (1, 2, 3)".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT count() FROM {} WHERE money IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '0' - assert node1.query( - "SELECT count() FROM {} WHERE money NOT IN (1, 2, 4, 5, 6)".format(table_name)).rstrip() == '10000' - assert node1.query( - "SELECT count() FROM {} WHERE name LIKE concat('name_', toString(1))".format(table_name)).rstrip() == '1' + table_name + ) + ) + assert ( + node1.query( + "SELECT count() FROM {} WHERE name LIKE '%name_%'".format(table_name) + ).rstrip() + == "10000" + ) + assert ( + node1.query( + "SELECT count() FROM {} WHERE name NOT LIKE '%tmp_%'".format(table_name) + ).rstrip() + == "10000" + ) + assert ( + node1.query( + "SELECT count() FROM {} WHERE money IN (1, 2, 3)".format(table_name) + ).rstrip() + == "10000" + ) + assert ( + node1.query( + "SELECT count() FROM {} WHERE money IN (1, 2, 4, 5, 6)".format(table_name) + ).rstrip() + == "0" + ) + assert ( + node1.query( + "SELECT count() FROM {} WHERE money NOT IN (1, 2, 4, 5, 6)".format( + table_name + ) + ).rstrip() + == "10000" + ) + assert ( + node1.query( + "SELECT count() FROM {} WHERE name LIKE concat('name_', toString(1))".format( + table_name + ) + ).rstrip() + == "1" + ) conn.close() def test_table_function(started_cluster): conn = get_mysql_conn(started_cluster, cluster.mysql_ip) - drop_mysql_table(conn, 'table_function') - create_mysql_table(conn, 'table_function') - table_function = "mysql('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('table_function') - assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '0' + drop_mysql_table(conn, "table_function") + create_mysql_table(conn, "table_function") + table_function = ( + "mysql('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format( + "table_function" + ) + ) + assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == "0" node1.query( "INSERT INTO {} (id, name, money) select number, concat('name_', toString(number)), 3 from numbers(10000)".format( - 'TABLE FUNCTION ' + table_function)) - assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '10000' - assert node1.query("SELECT sum(c) FROM (" - "SELECT count() as c FROM {} WHERE id % 3 == 0" - " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 1" - " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 2)".format(table_function, - table_function, - table_function)).rstrip() == '10000' - assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '30000' - node1.query("INSERT INTO {} (id, name, age, money) SELECT id + 100000, name, age, money FROM {}".format( - 'TABLE FUNCTION ' + table_function, table_function)) - assert node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() == '60000' + "TABLE FUNCTION " + table_function + ) + ) + assert ( + node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == "10000" + ) + assert ( + node1.query( + "SELECT sum(c) FROM (" + "SELECT count() as c FROM {} WHERE id % 3 == 0" + " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 1" + " UNION ALL SELECT count() as c FROM {} WHERE id % 3 == 2)".format( + table_function, table_function, table_function + ) + ).rstrip() + == "10000" + ) + assert ( + node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() + == "30000" + ) + node1.query( + "INSERT INTO {} (id, name, age, money) SELECT id + 100000, name, age, money FROM {}".format( + "TABLE FUNCTION " + table_function, table_function + ) + ) + assert ( + node1.query("SELECT sum(`money`) FROM {}".format(table_function)).rstrip() + == "60000" + ) conn.close() def test_binary_type(started_cluster): conn = get_mysql_conn(started_cluster, cluster.mysql_ip) - drop_mysql_table(conn, 'binary_type') + drop_mysql_table(conn, "binary_type") with conn.cursor() as cursor: - cursor.execute("CREATE TABLE clickhouse.binary_type (id INT PRIMARY KEY, data BINARY(16) NOT NULL)") - table_function = "mysql('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('binary_type') - node1.query("INSERT INTO {} VALUES (42, 'clickhouse')".format('TABLE FUNCTION ' + table_function)) - assert node1.query("SELECT * FROM {}".format(table_function)) == '42\tclickhouse\\0\\0\\0\\0\\0\\0\n' + cursor.execute( + "CREATE TABLE clickhouse.binary_type (id INT PRIMARY KEY, data BINARY(16) NOT NULL)" + ) + table_function = ( + "mysql('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format( + "binary_type" + ) + ) + node1.query( + "INSERT INTO {} VALUES (42, 'clickhouse')".format( + "TABLE FUNCTION " + table_function + ) + ) + assert ( + node1.query("SELECT * FROM {}".format(table_function)) + == "42\tclickhouse\\0\\0\\0\\0\\0\\0\n" + ) def test_enum_type(started_cluster): - table_name = 'test_enum_type' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_enum_type" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, source Enum8('IP' = 1, 'URL' = 2)) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse', 1); -'''.format(table_name, table_name)) - node1.query("INSERT INTO {} (id, name, age, money, source) VALUES (1, 'name', 0, 0, 'URL')".format(table_name)) - assert node1.query("SELECT source FROM {} LIMIT 1".format(table_name)).rstrip() == 'URL' +""".format( + table_name, table_name + ) + ) + node1.query( + "INSERT INTO {} (id, name, age, money, source) VALUES (1, 'name', 0, 0, 'URL')".format( + table_name + ) + ) + assert ( + node1.query("SELECT source FROM {} LIMIT 1".format(table_name)).rstrip() + == "URL" + ) conn.close() - def test_mysql_distributed(started_cluster): - table_name = 'test_replicas' + table_name = "test_replicas" conn1 = get_mysql_conn(started_cluster, started_cluster.mysql_ip) conn2 = get_mysql_conn(started_cluster, started_cluster.mysql2_ip) conn3 = get_mysql_conn(started_cluster, started_cluster.mysql3_ip) conn4 = get_mysql_conn(started_cluster, started_cluster.mysql4_ip) - create_mysql_db(conn1, 'clickhouse') - create_mysql_db(conn2, 'clickhouse') - create_mysql_db(conn3, 'clickhouse') - create_mysql_db(conn4, 'clickhouse') + create_mysql_db(conn1, "clickhouse") + create_mysql_db(conn2, "clickhouse") + create_mysql_db(conn3, "clickhouse") + create_mysql_db(conn4, "clickhouse") create_mysql_table(conn1, table_name) create_mysql_table(conn2, table_name) create_mysql_table(conn3, table_name) create_mysql_table(conn4, table_name) - node2.query('DROP TABLE IF EXISTS test_replicas') + node2.query("DROP TABLE IF EXISTS test_replicas") # Storage with with 3 replicas - node2.query(''' + node2.query( + """ CREATE TABLE test_replicas (id UInt32, name String, age UInt32, money UInt32) - ENGINE = MySQL('mysql{2|3|4}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') + ENGINE = MySQL('mysql{2|3|4}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ + ) # Fill remote tables with different data to be able to check nodes = [node1, node2, node2, node2] for i in range(1, 5): - nodes[i-1].query('DROP TABLE IF EXISTS test_replica{}'.format(i)) - nodes[i-1].query(''' + nodes[i - 1].query("DROP TABLE IF EXISTS test_replica{}".format(i)) + nodes[i - 1].query( + """ CREATE TABLE test_replica{} (id UInt32, name String, age UInt32, money UInt32) - ENGINE = MySQL('mysql{}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse');'''.format(i, 57 if i==1 else i)) - nodes[i-1].query("INSERT INTO test_replica{} (id, name) SELECT number, 'host{}' from numbers(10) ".format(i, i)) + ENGINE = MySQL('mysql{}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse');""".format( + i, 57 if i == 1 else i + ) + ) + nodes[i - 1].query( + "INSERT INTO test_replica{} (id, name) SELECT number, 'host{}' from numbers(10) ".format( + i, i + ) + ) # test multiple ports parsing - result = node2.query('''SELECT DISTINCT(name) FROM mysql('mysql{57|2|3}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') - assert(result == 'host1\n' or result == 'host2\n' or result == 'host3\n') - result = node2.query('''SELECT DISTINCT(name) FROM mysql('mysql57:3306|mysql2:3306|mysql3:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') - assert(result == 'host1\n' or result == 'host2\n' or result == 'host3\n') + result = node2.query( + """SELECT DISTINCT(name) FROM mysql('mysql{57|2|3}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ + ) + assert result == "host1\n" or result == "host2\n" or result == "host3\n" + result = node2.query( + """SELECT DISTINCT(name) FROM mysql('mysql57:3306|mysql2:3306|mysql3:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ + ) + assert result == "host1\n" or result == "host2\n" or result == "host3\n" # check all replicas are traversed query = "SELECT * FROM (" - for i in range (3): + for i in range(3): query += "SELECT name FROM test_replicas UNION DISTINCT " query += "SELECT name FROM test_replicas)" result = node2.query(query) - assert(result == 'host2\nhost3\nhost4\n') + assert result == "host2\nhost3\nhost4\n" # Storage with with two shards, each has 2 replicas - node2.query('DROP TABLE IF EXISTS test_shards') + node2.query("DROP TABLE IF EXISTS test_shards") - node2.query(''' + node2.query( + """ CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('MySQL', 'mysql{57|2}:3306,mysql{3|4}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') + ENGINE = ExternalDistributed('MySQL', 'mysql{57|2}:3306,mysql{3|4}:3306', 'clickhouse', 'test_replicas', 'root', 'clickhouse'); """ + ) # Check only one replica in each shard is used result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - assert(result == 'host1\nhost3\n') + assert result == "host1\nhost3\n" # check all replicas are traversed query = "SELECT name FROM (" - for i in range (3): + for i in range(3): query += "SELECT name FROM test_shards UNION DISTINCT " query += "SELECT name FROM test_shards) ORDER BY name" result = node2.query(query) - assert(result == 'host1\nhost2\nhost3\nhost4\n') + assert result == "host1\nhost2\nhost3\nhost4\n" # disconnect mysql57 - started_cluster.pause_container('mysql57') + started_cluster.pause_container("mysql57") result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - started_cluster.unpause_container('mysql57') - assert(result == 'host2\nhost4\n' or result == 'host3\nhost4\n') + started_cluster.unpause_container("mysql57") + assert result == "host2\nhost4\n" or result == "host3\nhost4\n" def test_external_settings(started_cluster): - table_name = 'test_external_settings' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_external_settings" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, started_cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node3.query(f'DROP TABLE IF EXISTS {table_name}') - node3.query(''' + node3.query(f"DROP TABLE IF EXISTS {table_name}") + node3.query( + """ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); -'''.format(table_name, table_name)) +""".format( + table_name, table_name + ) + ) node3.query( "INSERT INTO {}(id, name, money) select number, concat('name_', toString(number)), 3 from numbers(100) ".format( - table_name)) - assert node3.query("SELECT count() FROM {}".format(table_name)).rstrip() == '100' - assert node3.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == '300' - node3.query("select value from system.settings where name = 'max_block_size' FORMAT TSV") == "2\n" - node3.query("select value from system.settings where name = 'external_storage_max_read_rows' FORMAT TSV") == "0\n" - assert node3.query("SELECT COUNT(DISTINCT blockNumber()) FROM {} FORMAT TSV".format(table_name)) == '50\n' + table_name + ) + ) + assert node3.query("SELECT count() FROM {}".format(table_name)).rstrip() == "100" + assert node3.query("SELECT sum(money) FROM {}".format(table_name)).rstrip() == "300" + node3.query( + "select value from system.settings where name = 'max_block_size' FORMAT TSV" + ) == "2\n" + node3.query( + "select value from system.settings where name = 'external_storage_max_read_rows' FORMAT TSV" + ) == "0\n" + assert ( + node3.query( + "SELECT COUNT(DISTINCT blockNumber()) FROM {} FORMAT TSV".format(table_name) + ) + == "50\n" + ) conn.close() def test_settings_connection_wait_timeout(started_cluster): - table_name = 'test_settings_connection_wait_timeout' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_settings_connection_wait_timeout" + node1.query(f"DROP TABLE IF EXISTS {table_name}") wait_timeout = 2 conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {} ( id UInt32, @@ -341,10 +508,16 @@ def test_settings_connection_wait_timeout(started_cluster): ) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse') SETTINGS connection_wait_timeout={}, connection_pool_size=1 - '''.format(table_name, table_name, wait_timeout) + """.format( + table_name, table_name, wait_timeout + ) ) - node1.query("INSERT INTO {} (id, name) SELECT number, concat('name_', toString(number)) from numbers(10) ".format(table_name)) + node1.query( + "INSERT INTO {} (id, name) SELECT number, concat('name_', toString(number)) from numbers(10) ".format( + table_name + ) + ) def worker(): node1.query("SELECT sleepEachRow(1) FROM {}".format(table_name)) @@ -356,7 +529,10 @@ def test_settings_connection_wait_timeout(started_cluster): time.sleep(1) started = time.time() - with pytest.raises(QueryRuntimeException, match=r"Exception: mysqlxx::Pool is full \(connection_wait_timeout is exceeded\)"): + with pytest.raises( + QueryRuntimeException, + match=r"Exception: mysqlxx::Pool is full \(connection_wait_timeout is exceeded\)", + ): node1.query("SELECT sleepEachRow(1) FROM {}".format(table_name)) ended = time.time() assert (ended - started) >= wait_timeout @@ -369,70 +545,98 @@ def test_settings_connection_wait_timeout(started_cluster): def test_predefined_connection_configuration(started_cluster): conn = get_mysql_conn(started_cluster, started_cluster.mysql_ip) - table_name = 'test_table' + table_name = "test_table" drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(mysql1); - ''') - node1.query("INSERT INTO test_table (id, name, money) select number, toString(number), number from numbers(100)") - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + """ + ) + node1.query( + "INSERT INTO test_table (id, name, money) select number, toString(number), number from numbers(100)" + ) + assert node1.query(f"SELECT count() FROM test_table").rstrip() == "100" - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(mysql1, replace_query=1); - ''') - node1.query("INSERT INTO test_table (id, name, money) select number, toString(number), number from numbers(100)") - node1.query("INSERT INTO test_table (id, name, money) select number, toString(number), number from numbers(100)") - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + """ + ) + node1.query( + "INSERT INTO test_table (id, name, money) select number, toString(number), number from numbers(100)" + ) + node1.query( + "INSERT INTO test_table (id, name, money) select number, toString(number), number from numbers(100)" + ) + assert node1.query(f"SELECT count() FROM test_table").rstrip() == "100" - node1.query_and_get_error(''' + node1.query_and_get_error( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(mysql1, query=1); - ''') - node1.query_and_get_error(''' + """ + ) + node1.query_and_get_error( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(mysql1, replace_query=1, on_duplicate_clause='kek'); - ''') - node1.query_and_get_error(''' + """ + ) + node1.query_and_get_error( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(fff); - ''') - node1.query_and_get_error(''' + """ + ) + node1.query_and_get_error( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(mysql2); - ''') + """ + ) - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (id UInt32, name String, age UInt32, money UInt32) ENGINE MySQL(mysql3, port=3306); - ''') - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + """ + ) + assert node1.query(f"SELECT count() FROM test_table").rstrip() == "100" - assert 'Connection pool cannot have zero size' in node1.query_and_get_error("SELECT count() FROM mysql(mysql1, table='test_table', connection_pool_size=0)") - assert 'Connection pool cannot have zero size' in node1.query_and_get_error("SELECT count() FROM mysql(mysql4)") - assert int(node1.query("SELECT count() FROM mysql(mysql4, connection_pool_size=1)")) == 100 + assert "Connection pool cannot have zero size" in node1.query_and_get_error( + "SELECT count() FROM mysql(mysql1, table='test_table', connection_pool_size=0)" + ) + assert "Connection pool cannot have zero size" in node1.query_and_get_error( + "SELECT count() FROM mysql(mysql4)" + ) + assert ( + int(node1.query("SELECT count() FROM mysql(mysql4, connection_pool_size=1)")) + == 100 + ) # Regression for (k, v) IN ((k, v)) def test_mysql_in(started_cluster): - table_name = 'test_mysql_in' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_mysql_in" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) - node1.query(''' + node1.query( + """ CREATE TABLE {} ( id UInt32, @@ -441,52 +645,94 @@ def test_mysql_in(started_cluster): money UInt32 ) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse') - '''.format(table_name, table_name) + """.format( + table_name, table_name + ) ) - node1.query("INSERT INTO {} (id, name) SELECT number, concat('name_', toString(number)) from numbers(10) ".format(table_name)) + node1.query( + "INSERT INTO {} (id, name) SELECT number, concat('name_', toString(number)) from numbers(10) ".format( + table_name + ) + ) node1.query("SELECT * FROM {} WHERE (id) IN (1)".format(table_name)) node1.query("SELECT * FROM {} WHERE (id) IN (1, 2)".format(table_name)) - node1.query("SELECT * FROM {} WHERE (id, name) IN ((1, 'name_1'))".format(table_name)) - node1.query("SELECT * FROM {} WHERE (id, name) IN ((1, 'name_1'),(1, 'name_1'))".format(table_name)) + node1.query( + "SELECT * FROM {} WHERE (id, name) IN ((1, 'name_1'))".format(table_name) + ) + node1.query( + "SELECT * FROM {} WHERE (id, name) IN ((1, 'name_1'),(1, 'name_1'))".format( + table_name + ) + ) drop_mysql_table(conn, table_name) conn.close() + def test_mysql_null(started_cluster): - table_name = 'test_mysql_in' - node1.query(f'DROP TABLE IF EXISTS {table_name}') + table_name = "test_mysql_in" + node1.query(f"DROP TABLE IF EXISTS {table_name}") conn = get_mysql_conn(started_cluster, cluster.mysql_ip) drop_mysql_table(conn, table_name) with conn.cursor() as cursor: - cursor.execute(""" + cursor.execute( + """ CREATE TABLE `clickhouse`.`{}` ( `id` int(11) NOT NULL, `money` int NULL default NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB; - """.format(table_name)) + """.format( + table_name + ) + ) - node1.query(''' + node1.query( + """ CREATE TABLE {} ( id UInt32, money Nullable(UInt32) ) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse') - '''.format(table_name, table_name) + """.format( + table_name, table_name + ) ) - node1.query("INSERT INTO {} (id, money) SELECT number, if(number%2, NULL, 1) from numbers(10) ".format(table_name)) + node1.query( + "INSERT INTO {} (id, money) SELECT number, if(number%2, NULL, 1) from numbers(10) ".format( + table_name + ) + ) - assert int(node1.query("SELECT count() FROM {} WHERE money IS NULL SETTINGS external_table_strict_query=1".format(table_name))) == 5 - assert int(node1.query("SELECT count() FROM {} WHERE money IS NOT NULL SETTINGS external_table_strict_query=1".format(table_name))) == 5 + assert ( + int( + node1.query( + "SELECT count() FROM {} WHERE money IS NULL SETTINGS external_table_strict_query=1".format( + table_name + ) + ) + ) + == 5 + ) + assert ( + int( + node1.query( + "SELECT count() FROM {} WHERE money IS NOT NULL SETTINGS external_table_strict_query=1".format( + table_name + ) + ) + ) + == 5 + ) drop_mysql_table(conn, table_name) conn.close() -if __name__ == '__main__': +if __name__ == "__main__": with contextmanager(started_cluster)() as cluster: for name, instance in list(cluster.instances.items()): print(name, instance.ip_address) diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 87337a6b459..8366ca5dc25 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -3,10 +3,15 @@ import pytest from multiprocessing.dummy import Pool from helpers.cluster import ClickHouseCluster +from helpers.postgres_utility import get_postgres_conn cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/named_collections.xml'], with_postgres=True) -node2 = cluster.add_instance('node2', main_configs=['configs/named_collections.xml'], with_postgres_cluster=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/named_collections.xml"], with_postgres=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/named_collections.xml"], with_postgres_cluster=True +) @pytest.fixture(scope="module") @@ -19,6 +24,7 @@ def started_cluster(): finally: cluster.shutdown() + @pytest.fixture(autouse=True) def setup_teardown(): print("PostgreSQL is available - running test") @@ -26,56 +32,73 @@ def setup_teardown(): node1.query("DROP DATABASE test") node1.query("CREATE DATABASE test") + def test_postgres_select_insert(started_cluster): cursor = started_cluster.postgres_conn.cursor() - table_name = 'test_many' - table = f'''postgresql('{started_cluster.postgres_ip}:{started_cluster.postgres_port}', 'postgres', '{table_name}', 'postgres', 'mysecretpassword')''' - cursor.execute(f'DROP TABLE IF EXISTS {table_name}') - cursor.execute(f'CREATE TABLE {table_name} (a integer, b text, c integer)') + table_name = "test_many" + table = f"""postgresql('{started_cluster.postgres_ip}:{started_cluster.postgres_port}', 'postgres', '{table_name}', 'postgres', 'mysecretpassword')""" + cursor.execute(f"DROP TABLE IF EXISTS {table_name}") + cursor.execute(f"CREATE TABLE {table_name} (a integer, b text, c integer)") - result = node1.query(f''' + result = node1.query( + f""" INSERT INTO TABLE FUNCTION {table} - SELECT number, concat('name_', toString(number)), 3 from numbers(10000)''') + SELECT number, concat('name_', toString(number)), 3 from numbers(10000)""" + ) check1 = f"SELECT count() FROM {table}" check2 = f"SELECT Sum(c) FROM {table}" check3 = f"SELECT count(c) FROM {table} WHERE a % 2 == 0" check4 = f"SELECT count() FROM {table} WHERE b LIKE concat('name_', toString(1))" - assert (node1.query(check1)).rstrip() == '10000' - assert (node1.query(check2)).rstrip() == '30000' - assert (node1.query(check3)).rstrip() == '5000' - assert (node1.query(check4)).rstrip() == '1' + assert (node1.query(check1)).rstrip() == "10000" + assert (node1.query(check2)).rstrip() == "30000" + assert (node1.query(check3)).rstrip() == "5000" + assert (node1.query(check4)).rstrip() == "1" # Triggers issue https://github.com/ClickHouse/ClickHouse/issues/26088 # for i in range(1, 1000): # assert (node1.query(check1)).rstrip() == '10000', f"Failed on {i}" - cursor.execute(f'DROP TABLE {table_name} ') + cursor.execute(f"DROP TABLE {table_name} ") def test_postgres_conversions(started_cluster): cursor = started_cluster.postgres_conn.cursor() - cursor.execute(f'DROP TABLE IF EXISTS test_types') - cursor.execute(f'DROP TABLE IF EXISTS test_array_dimensions') + cursor.execute(f"DROP TABLE IF EXISTS test_types") + cursor.execute(f"DROP TABLE IF EXISTS test_array_dimensions") cursor.execute( - '''CREATE TABLE test_types ( + """CREATE TABLE test_types ( a smallint, b integer, c bigint, d real, e double precision, f serial, g bigserial, - h timestamp, i date, j decimal(5, 3), k numeric, l boolean)''') - node1.query(''' + h timestamp, i date, j decimal(5, 3), k numeric, l boolean)""" + ) + node1.query( + """ INSERT INTO TABLE FUNCTION postgresql('postgres1:5432', 'postgres', 'test_types', 'postgres', 'mysecretpassword') VALUES - (-32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12.012345', '2000-05-12', 22.222, 22.222, 1)''') - result = node1.query(''' - SELECT a, b, c, d, e, f, g, h, i, j, toDecimal128(k, 3), l FROM postgresql('postgres1:5432', 'postgres', 'test_types', 'postgres', 'mysecretpassword')''') - assert(result == '-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t22.222\t22.222\t1\n') - - cursor.execute("INSERT INTO test_types (l) VALUES (TRUE), (true), ('yes'), ('y'), ('1');") - cursor.execute("INSERT INTO test_types (l) VALUES (FALSE), (false), ('no'), ('off'), ('0');") - expected = "1\n1\n1\n1\n1\n1\n0\n0\n0\n0\n0\n" - result = node1.query('''SELECT l FROM postgresql('postgres1:5432', 'postgres', 'test_types', 'postgres', 'mysecretpassword')''') - assert(result == expected) + (-32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12.012345', '2000-05-12', 22.222, 22.222, 1)""" + ) + result = node1.query( + """ + SELECT a, b, c, d, e, f, g, h, i, j, toDecimal128(k, 3), l FROM postgresql('postgres1:5432', 'postgres', 'test_types', 'postgres', 'mysecretpassword')""" + ) + assert ( + result + == "-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t22.222\t22.222\t1\n" + ) cursor.execute( - '''CREATE TABLE IF NOT EXISTS test_array_dimensions + "INSERT INTO test_types (l) VALUES (TRUE), (true), ('yes'), ('y'), ('1');" + ) + cursor.execute( + "INSERT INTO test_types (l) VALUES (FALSE), (false), ('no'), ('off'), ('0');" + ) + expected = "1\n1\n1\n1\n1\n1\n0\n0\n0\n0\n0\n" + result = node1.query( + """SELECT l FROM postgresql('postgres1:5432', 'postgres', 'test_types', 'postgres', 'mysecretpassword')""" + ) + assert result == expected + + cursor.execute( + """CREATE TABLE IF NOT EXISTS test_array_dimensions ( a Date[] NOT NULL, -- Date b Timestamp[] NOT NULL, -- DateTime64(6) @@ -87,24 +110,29 @@ def test_postgres_conversions(started_cluster): h Integer[][][], -- Nullable(Int32) i Char(2)[][][][], -- Nullable(String) k Char(2)[] -- Nullable(String) - )''') + )""" + ) - result = node1.query(''' - DESCRIBE TABLE postgresql('postgres1:5432', 'postgres', 'test_array_dimensions', 'postgres', 'mysecretpassword')''') - expected = ('a\tArray(Date)\t\t\t\t\t\n' + - 'b\tArray(DateTime64(6))\t\t\t\t\t\n' + - 'c\tArray(Array(Float32))\t\t\t\t\t\n' + - 'd\tArray(Array(Float64))\t\t\t\t\t\n' + - 'e\tArray(Array(Array(Decimal(5, 5))))\t\t\t\t\t\n' + - 'f\tArray(Array(Array(Int32)))\t\t\t\t\t\n' + - 'g\tArray(Array(Array(Array(Array(String)))))\t\t\t\t\t\n' + - 'h\tArray(Array(Array(Nullable(Int32))))\t\t\t\t\t\n' + - 'i\tArray(Array(Array(Array(Nullable(String)))))\t\t\t\t\t\n' + - 'k\tArray(Nullable(String))' - ) - assert(result.rstrip() == expected) + result = node1.query( + """ + DESCRIBE TABLE postgresql('postgres1:5432', 'postgres', 'test_array_dimensions', 'postgres', 'mysecretpassword')""" + ) + expected = ( + "a\tArray(Date)\t\t\t\t\t\n" + + "b\tArray(DateTime64(6))\t\t\t\t\t\n" + + "c\tArray(Array(Float32))\t\t\t\t\t\n" + + "d\tArray(Array(Float64))\t\t\t\t\t\n" + + "e\tArray(Array(Array(Decimal(5, 5))))\t\t\t\t\t\n" + + "f\tArray(Array(Array(Int32)))\t\t\t\t\t\n" + + "g\tArray(Array(Array(Array(Array(String)))))\t\t\t\t\t\n" + + "h\tArray(Array(Array(Nullable(Int32))))\t\t\t\t\t\n" + + "i\tArray(Array(Array(Array(Nullable(String)))))\t\t\t\t\t\n" + + "k\tArray(Nullable(String))" + ) + assert result.rstrip() == expected - node1.query("INSERT INTO TABLE FUNCTION postgresql('postgres1:5432', 'postgres', 'test_array_dimensions', 'postgres', 'mysecretpassword') " + node1.query( + "INSERT INTO TABLE FUNCTION postgresql('postgres1:5432', 'postgres', 'test_array_dimensions', 'postgres', 'mysecretpassword') " "VALUES (" "['2000-05-12', '2000-05-12'], " "['2000-05-12 12:12:12.012345', '2000-05-12 12:12:12.012345'], " @@ -116,125 +144,179 @@ def test_postgres_conversions(started_cluster): "[[[1, NULL], [NULL, 1]], [[NULL, NULL], [NULL, NULL]], [[4, 4], [5, 5]]], " "[[[[NULL]]]], " "[]" - ")") + ")" + ) - result = node1.query(''' - SELECT * FROM postgresql('postgres1:5432', 'postgres', 'test_array_dimensions', 'postgres', 'mysecretpassword')''') + result = node1.query( + """ + SELECT * FROM postgresql('postgres1:5432', 'postgres', 'test_array_dimensions', 'postgres', 'mysecretpassword')""" + ) expected = ( - "['2000-05-12','2000-05-12']\t" + - "['2000-05-12 12:12:12.012345','2000-05-12 12:12:12.012345']\t" + - "[[1.12345],[1.12345],[1.12345]]\t" + - "[[1.1234567891],[1.1234567891],[1.1234567891]]\t" + - "[[[0.11111,0.11111]],[[0.22222,0.22222]],[[0.33333,0.33333]]]\t" + "['2000-05-12','2000-05-12']\t" + + "['2000-05-12 12:12:12.012345','2000-05-12 12:12:12.012345']\t" + + "[[1.12345],[1.12345],[1.12345]]\t" + + "[[1.1234567891],[1.1234567891],[1.1234567891]]\t" + + "[[[0.11111,0.11111]],[[0.22222,0.22222]],[[0.33333,0.33333]]]\t" "[[[1,1],[1,1]],[[3,3],[3,3]],[[4,4],[5,5]]]\t" "[[[[['winx','winx','winx']]]]]\t" "[[[1,NULL],[NULL,1]],[[NULL,NULL],[NULL,NULL]],[[4,4],[5,5]]]\t" "[[[[NULL]]]]\t" "[]\n" - ) - assert(result == expected) + ) + assert result == expected - cursor.execute(f'DROP TABLE test_types') - cursor.execute(f'DROP TABLE test_array_dimensions') + cursor.execute(f"DROP TABLE test_types") + cursor.execute(f"DROP TABLE test_array_dimensions") def test_non_default_scema(started_cluster): - node1.query('DROP TABLE IF EXISTS test_pg_table_schema') - node1.query('DROP TABLE IF EXISTS test_pg_table_schema_with_dots') + node1.query("DROP TABLE IF EXISTS test_pg_table_schema") + node1.query("DROP TABLE IF EXISTS test_pg_table_schema_with_dots") cursor = started_cluster.postgres_conn.cursor() - cursor.execute('DROP SCHEMA IF EXISTS test_schema CASCADE') + cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") cursor.execute('DROP SCHEMA IF EXISTS "test.nice.schema" CASCADE') - cursor.execute('CREATE SCHEMA test_schema') - cursor.execute('CREATE TABLE test_schema.test_table (a integer)') - cursor.execute('INSERT INTO test_schema.test_table SELECT i FROM generate_series(0, 99) as t(i)') + cursor.execute("CREATE SCHEMA test_schema") + cursor.execute("CREATE TABLE test_schema.test_table (a integer)") + cursor.execute( + "INSERT INTO test_schema.test_table SELECT i FROM generate_series(0, 99) as t(i)" + ) - node1.query(''' + node1.query( + """ CREATE TABLE test.test_pg_table_schema (a UInt32) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_table', 'postgres', 'mysecretpassword', 'test_schema'); - ''') + """ + ) - result = node1.query('SELECT * FROM test.test_pg_table_schema') - expected = node1.query('SELECT number FROM numbers(100)') - assert(result == expected) + result = node1.query("SELECT * FROM test.test_pg_table_schema") + expected = node1.query("SELECT number FROM numbers(100)") + assert result == expected - table_function = '''postgresql('postgres1:5432', 'postgres', 'test_table', 'postgres', 'mysecretpassword', 'test_schema')''' - result = node1.query(f'SELECT * FROM {table_function}') - assert(result == expected) + table_function = """postgresql('postgres1:5432', 'postgres', 'test_table', 'postgres', 'mysecretpassword', 'test_schema')""" + result = node1.query(f"SELECT * FROM {table_function}") + assert result == expected cursor.execute('''CREATE SCHEMA "test.nice.schema"''') - cursor.execute('''CREATE TABLE "test.nice.schema"."test.nice.table" (a integer)''') - cursor.execute('INSERT INTO "test.nice.schema"."test.nice.table" SELECT i FROM generate_series(0, 99) as t(i)') + cursor.execute("""CREATE TABLE "test.nice.schema"."test.nice.table" (a integer)""") + cursor.execute( + 'INSERT INTO "test.nice.schema"."test.nice.table" SELECT i FROM generate_series(0, 99) as t(i)' + ) - node1.query(''' + node1.query( + """ CREATE TABLE test.test_pg_table_schema_with_dots (a UInt32) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test.nice.table', 'postgres', 'mysecretpassword', 'test.nice.schema'); - ''') - result = node1.query('SELECT * FROM test.test_pg_table_schema_with_dots') - assert(result == expected) + """ + ) + result = node1.query("SELECT * FROM test.test_pg_table_schema_with_dots") + assert result == expected - cursor.execute('INSERT INTO "test_schema"."test_table" SELECT i FROM generate_series(100, 199) as t(i)') - result = node1.query(f'SELECT * FROM {table_function}') - expected = node1.query('SELECT number FROM numbers(200)') - assert(result == expected) + cursor.execute( + 'INSERT INTO "test_schema"."test_table" SELECT i FROM generate_series(100, 199) as t(i)' + ) + result = node1.query(f"SELECT * FROM {table_function}") + expected = node1.query("SELECT number FROM numbers(200)") + assert result == expected - cursor.execute('DROP SCHEMA test_schema CASCADE') + cursor.execute("DROP SCHEMA test_schema CASCADE") cursor.execute('DROP SCHEMA "test.nice.schema" CASCADE') - node1.query('DROP TABLE test.test_pg_table_schema') - node1.query('DROP TABLE test.test_pg_table_schema_with_dots') + node1.query("DROP TABLE test.test_pg_table_schema") + node1.query("DROP TABLE test.test_pg_table_schema_with_dots") def test_concurrent_queries(started_cluster): - cursor = started_cluster.postgres_conn.cursor() + conn = get_postgres_conn( + started_cluster.postgres_ip, started_cluster.postgres_port, database=False + ) + cursor = conn.cursor() + database_name = "concurrent_test" - node1.query(''' - CREATE TABLE test_table (key UInt32, value UInt32) - ENGINE = PostgreSQL('postgres1:5432', 'postgres', 'test_table', 'postgres', 'mysecretpassword')''') + cursor.execute(f"DROP DATABASE IF EXISTS {database_name}") + cursor.execute(f"CREATE DATABASE {database_name}") + conn = get_postgres_conn( + started_cluster.postgres_ip, + started_cluster.postgres_port, + database=True, + database_name=database_name, + ) + cursor = conn.cursor() + cursor.execute("CREATE TABLE test_table (key integer, value integer)") - cursor.execute('CREATE TABLE test_table (key integer, value integer)') + node1.query( + f""" + CREATE TABLE test.test_table (key UInt32, value UInt32) + ENGINE = PostgreSQL(postgres1, database='{database_name}', table='test_table') + """ + ) + + node1.query( + f""" + CREATE TABLE test.stat (numbackends UInt32, datname String) + ENGINE = PostgreSQL(postgres1, database='{database_name}', table='pg_stat_database') + """ + ) - prev_count = node1.count_in_log('New connection to postgres1:5432') def node_select(_): for i in range(20): - result = node1.query("SELECT * FROM test_table", user='default') - busy_pool = Pool(20) - p = busy_pool.map_async(node_select, range(20)) - p.wait() - count = node1.count_in_log('New connection to postgres1:5432') - logging.debug(f'count {count}, prev_count {prev_count}') - # 16 is default size for connection pool - assert(int(count) <= int(prev_count) + 16) + result = node1.query("SELECT * FROM test.test_table", user="default") def node_insert(_): - for i in range(5): - result = node1.query("INSERT INTO test_table SELECT number, number FROM numbers(1000)", user='default') - - busy_pool = Pool(5) - p = busy_pool.map_async(node_insert, range(5)) - p.wait() - result = node1.query("SELECT count() FROM test_table", user='default') - logging.debug(result) - assert(int(result) == 5 * 5 * 1000) + for i in range(20): + result = node1.query( + "INSERT INTO test.test_table SELECT number, number FROM numbers(1000)", + user="default", + ) def node_insert_select(_): - for i in range(5): - result = node1.query("INSERT INTO test_table SELECT number, number FROM numbers(1000)", user='default') - result = node1.query("SELECT * FROM test_table LIMIT 100", user='default') + for i in range(20): + result = node1.query( + "INSERT INTO test.test_table SELECT number, number FROM numbers(1000)", + user="default", + ) + result = node1.query( + "SELECT * FROM test.test_table LIMIT 100", user="default" + ) - busy_pool = Pool(5) - p = busy_pool.map_async(node_insert_select, range(5)) + busy_pool = Pool(30) + p = busy_pool.map_async(node_select, range(30)) p.wait() - result = node1.query("SELECT count() FROM test_table", user='default') - logging.debug(result) - assert(int(result) == 5 * 5 * 1000 * 2) - node1.query('DROP TABLE test_table;') - cursor.execute('DROP TABLE test_table;') + count = int( + node1.query( + f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'" + ) + ) + print(count) + assert count <= 18 - count = node1.count_in_log('New connection to postgres1:5432') - logging.debug(f'count {count}, prev_count {prev_count}') - assert(int(count) <= int(prev_count) + 16) + busy_pool = Pool(30) + p = busy_pool.map_async(node_insert, range(30)) + p.wait() + + count = int( + node1.query( + f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'" + ) + ) + print(count) + assert count <= 18 + + busy_pool = Pool(30) + p = busy_pool.map_async(node_insert_select, range(30)) + p.wait() + + count = int( + node1.query( + f"SELECT numbackends FROM test.stat WHERE datname = '{database_name}'" + ) + ) + print(count) + assert count <= 18 + + node1.query("DROP TABLE test.test_table;") + node1.query("DROP TABLE test.stat;") def test_postgres_distributed(started_cluster): @@ -245,82 +327,106 @@ def test_postgres_distributed(started_cluster): cursors = [cursor0, cursor1, cursor2, cursor3] for i in range(4): - cursors[i].execute('DROP TABLE IF EXISTS test_replicas') - cursors[i].execute('CREATE TABLE test_replicas (id Integer, name Text)') - cursors[i].execute(f"""INSERT INTO test_replicas select i, 'host{i+1}' from generate_series(0, 99) as t(i);"""); + cursors[i].execute("DROP TABLE IF EXISTS test_replicas") + cursors[i].execute("CREATE TABLE test_replicas (id Integer, name Text)") + cursors[i].execute( + f"""INSERT INTO test_replicas select i, 'host{i+1}' from generate_series(0, 99) as t(i);""" + ) # test multiple ports parsing - result = node2.query('''SELECT DISTINCT(name) FROM postgresql('postgres{1|2|3}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); ''') - assert(result == 'host1\n' or result == 'host2\n' or result == 'host3\n') - result = node2.query('''SELECT DISTINCT(name) FROM postgresql('postgres2:5431|postgres3:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); ''') - assert(result == 'host3\n' or result == 'host2\n') + result = node2.query( + """SELECT DISTINCT(name) FROM postgresql('postgres{1|2|3}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ + ) + assert result == "host1\n" or result == "host2\n" or result == "host3\n" + result = node2.query( + """SELECT DISTINCT(name) FROM postgresql('postgres2:5431|postgres3:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ + ) + assert result == "host3\n" or result == "host2\n" # Create storage with with 3 replicas - node2.query('DROP TABLE IF EXISTS test_replicas') - node2.query(''' + node2.query("DROP TABLE IF EXISTS test_replicas") + node2.query( + """ CREATE TABLE test_replicas (id UInt32, name String) - ENGINE = PostgreSQL('postgres{2|3|4}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); ''') + ENGINE = PostgreSQL('postgres{2|3|4}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ + ) # Check all replicas are traversed query = "SELECT name FROM (" - for i in range (3): + for i in range(3): query += "SELECT name FROM test_replicas UNION DISTINCT " query += "SELECT name FROM test_replicas) ORDER BY name" result = node2.query(query) - assert(result == 'host2\nhost3\nhost4\n') + assert result == "host2\nhost3\nhost4\n" # Create storage with with two two shards, each has 2 replicas - node2.query('DROP TABLE IF EXISTS test_shards') + node2.query("DROP TABLE IF EXISTS test_shards") - node2.query(''' + node2.query( + """ CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('PostgreSQL', 'postgres{1|2}:5432,postgres{3|4}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); ''') + ENGINE = ExternalDistributed('PostgreSQL', 'postgres{1|2}:5432,postgres{3|4}:5432', 'postgres', 'test_replicas', 'postgres', 'mysecretpassword'); """ + ) # Check only one replica in each shard is used result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - assert(result == 'host1\nhost3\n') + assert result == "host1\nhost3\n" - node2.query(''' + node2.query( + """ CREATE TABLE test_shards2 (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('PostgreSQL', postgres4, description='postgres{1|2}:5432,postgres{3|4}:5432'); ''') + ENGINE = ExternalDistributed('PostgreSQL', postgres4, description='postgres{1|2}:5432,postgres{3|4}:5432'); """ + ) result = node2.query("SELECT DISTINCT(name) FROM test_shards2 ORDER BY name") - assert(result == 'host1\nhost3\n') + assert result == "host1\nhost3\n" # Check all replicas are traversed query = "SELECT name FROM (" - for i in range (3): + for i in range(3): query += "SELECT name FROM test_shards UNION DISTINCT " query += "SELECT name FROM test_shards) ORDER BY name" result = node2.query(query) - assert(result == 'host1\nhost2\nhost3\nhost4\n') + assert result == "host1\nhost2\nhost3\nhost4\n" # Disconnect postgres1 - started_cluster.pause_container('postgres1') + started_cluster.pause_container("postgres1") result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - started_cluster.unpause_container('postgres1') - assert(result == 'host2\nhost4\n' or result == 'host3\nhost4\n') - node2.query('DROP TABLE test_shards') - node2.query('DROP TABLE test_replicas') + started_cluster.unpause_container("postgres1") + assert result == "host2\nhost4\n" or result == "host3\nhost4\n" + node2.query("DROP TABLE test_shards") + node2.query("DROP TABLE test_replicas") def test_datetime_with_timezone(started_cluster): cursor = started_cluster.postgres_conn.cursor() cursor.execute("DROP TABLE IF EXISTS test_timezone") node1.query("DROP TABLE IF EXISTS test.test_timezone") - cursor.execute("CREATE TABLE test_timezone (ts timestamp without time zone, ts_z timestamp with time zone)") - cursor.execute("insert into test_timezone select '2014-04-04 20:00:00', '2014-04-04 20:00:00'::timestamptz at time zone 'America/New_York';") + cursor.execute( + "CREATE TABLE test_timezone (ts timestamp without time zone, ts_z timestamp with time zone)" + ) + cursor.execute( + "insert into test_timezone select '2014-04-04 20:00:00', '2014-04-04 20:00:00'::timestamptz at time zone 'America/New_York';" + ) cursor.execute("select * from test_timezone") result = cursor.fetchall()[0] - logging.debug(f'{result[0]}, {str(result[1])[:-6]}') - node1.query("create table test.test_timezone ( ts DateTime, ts_z DateTime('America/New_York')) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_timezone', 'postgres', 'mysecretpassword');") - assert(node1.query("select ts from test.test_timezone").strip() == str(result[0])) + logging.debug(f"{result[0]}, {str(result[1])[:-6]}") + node1.query( + "create table test.test_timezone ( ts DateTime, ts_z DateTime('America/New_York')) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_timezone', 'postgres', 'mysecretpassword');" + ) + assert node1.query("select ts from test.test_timezone").strip() == str(result[0]) # [:-6] because 2014-04-04 16:00:00+00:00 -> 2014-04-04 16:00:00 - assert(node1.query("select ts_z from test.test_timezone").strip() == str(result[1])[:-6]) - assert(node1.query("select * from test.test_timezone") == "2014-04-04 20:00:00\t2014-04-04 16:00:00\n") + assert ( + node1.query("select ts_z from test.test_timezone").strip() + == str(result[1])[:-6] + ) + assert ( + node1.query("select * from test.test_timezone") + == "2014-04-04 20:00:00\t2014-04-04 16:00:00\n" + ) cursor.execute("DROP TABLE test_timezone") node1.query("DROP TABLE test.test_timezone") @@ -329,125 +435,197 @@ def test_postgres_ndim(started_cluster): cursor = started_cluster.postgres_conn.cursor() cursor.execute("DROP TABLE IF EXISTS arr1, arr2") - cursor.execute('CREATE TABLE arr1 (a Integer[])') + cursor.execute("CREATE TABLE arr1 (a Integer[])") cursor.execute("INSERT INTO arr1 SELECT '{{1}, {2}}'") # The point is in creating a table via 'as select *', in postgres att_ndim will not be correct in this case. - cursor.execute('CREATE TABLE arr2 AS SELECT * FROM arr1') - cursor.execute("SELECT attndims AS dims FROM pg_attribute WHERE attrelid = 'arr2'::regclass; ") + cursor.execute("CREATE TABLE arr2 AS SELECT * FROM arr1") + cursor.execute( + "SELECT attndims AS dims FROM pg_attribute WHERE attrelid = 'arr2'::regclass; " + ) result = cursor.fetchall()[0] - assert(int(result[0]) == 0) + assert int(result[0]) == 0 - result = node1.query('''SELECT toTypeName(a) FROM postgresql('postgres1:5432', 'postgres', 'arr2', 'postgres', 'mysecretpassword')''') - assert(result.strip() == "Array(Array(Nullable(Int32)))") + result = node1.query( + """SELECT toTypeName(a) FROM postgresql('postgres1:5432', 'postgres', 'arr2', 'postgres', 'mysecretpassword')""" + ) + assert result.strip() == "Array(Array(Nullable(Int32)))" cursor.execute("DROP TABLE arr1, arr2") def test_postgres_on_conflict(started_cluster): cursor = started_cluster.postgres_conn.cursor() - table = 'test_conflict' - cursor.execute(f'DROP TABLE IF EXISTS {table}') - cursor.execute(f'CREATE TABLE {table} (a integer PRIMARY KEY, b text, c integer)') + table = "test_conflict" + cursor.execute(f"DROP TABLE IF EXISTS {table}") + cursor.execute(f"CREATE TABLE {table} (a integer PRIMARY KEY, b text, c integer)") - node1.query(''' + node1.query( + """ CREATE TABLE test.test_conflict (a UInt32, b String, c Int32) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_conflict', 'postgres', 'mysecretpassword', '', 'ON CONFLICT DO NOTHING'); - ''') - node1.query(f''' INSERT INTO test.{table} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') - node1.query(f''' INSERT INTO test.{table} SELECT number, concat('name_', toString(number)), 4 from numbers(100)''') + """ + ) + node1.query( + f""" INSERT INTO test.{table} SELECT number, concat('name_', toString(number)), 3 from numbers(100)""" + ) + node1.query( + f""" INSERT INTO test.{table} SELECT number, concat('name_', toString(number)), 4 from numbers(100)""" + ) check1 = f"SELECT count() FROM test.{table}" - assert (node1.query(check1)).rstrip() == '100' + assert (node1.query(check1)).rstrip() == "100" - table_func = f'''postgresql('{started_cluster.postgres_ip}:{started_cluster.postgres_port}', 'postgres', '{table}', 'postgres', 'mysecretpassword', '', 'ON CONFLICT DO NOTHING')''' - node1.query(f'''INSERT INTO TABLE FUNCTION {table_func} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') - node1.query(f'''INSERT INTO TABLE FUNCTION {table_func} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') + table_func = f"""postgresql('{started_cluster.postgres_ip}:{started_cluster.postgres_port}', 'postgres', '{table}', 'postgres', 'mysecretpassword', '', 'ON CONFLICT DO NOTHING')""" + node1.query( + f"""INSERT INTO TABLE FUNCTION {table_func} SELECT number, concat('name_', toString(number)), 3 from numbers(100)""" + ) + node1.query( + f"""INSERT INTO TABLE FUNCTION {table_func} SELECT number, concat('name_', toString(number)), 3 from numbers(100)""" + ) check1 = f"SELECT count() FROM test.{table}" - assert (node1.query(check1)).rstrip() == '100' + assert (node1.query(check1)).rstrip() == "100" - cursor.execute(f'DROP TABLE {table} ') + cursor.execute(f"DROP TABLE {table} ") def test_predefined_connection_configuration(started_cluster): cursor = started_cluster.postgres_conn.cursor() - cursor.execute(f'DROP TABLE IF EXISTS test_table') - cursor.execute(f'CREATE TABLE test_table (a integer PRIMARY KEY, b integer)') + cursor.execute(f"DROP TABLE IF EXISTS test_table") + cursor.execute(f"CREATE TABLE test_table (a integer PRIMARY KEY, b integer)") - node1.query(''' + node1.query( + """ DROP TABLE IF EXISTS test.test_table; CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1); - ''') - node1.query(f''' INSERT INTO test.test_table SELECT number, number from numbers(100)''') - assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') + """ + ) + node1.query( + f""" INSERT INTO test.test_table SELECT number, number from numbers(100)""" + ) + assert node1.query(f"SELECT count() FROM test.test_table").rstrip() == "100" - node1.query(''' + node1.query( + """ DROP TABLE test.test_table; CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1, on_conflict='ON CONFLICT DO NOTHING'); - ''') - node1.query(f''' INSERT INTO test.test_table SELECT number, number from numbers(100)''') - node1.query(f''' INSERT INTO test.test_table SELECT number, number from numbers(100)''') - assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') + """ + ) + node1.query( + f""" INSERT INTO test.test_table SELECT number, number from numbers(100)""" + ) + node1.query( + f""" INSERT INTO test.test_table SELECT number, number from numbers(100)""" + ) + assert node1.query(f"SELECT count() FROM test.test_table").rstrip() == "100" - node1.query('DROP TABLE test.test_table;') - node1.query_and_get_error(''' + node1.query("DROP TABLE test.test_table;") + node1.query_and_get_error( + """ CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1, 'ON CONFLICT DO NOTHING'); - ''') - node1.query_and_get_error(''' + """ + ) + node1.query_and_get_error( + """ CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres2); - ''') - node1.query_and_get_error(''' + """ + ) + node1.query_and_get_error( + """ CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(unknown_collection); - ''') + """ + ) - node1.query(''' + node1.query( + """ CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1, port=5432, database='postgres', table='test_table'); - ''') - assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') + """ + ) + assert node1.query(f"SELECT count() FROM test.test_table").rstrip() == "100" - node1.query(''' + node1.query( + """ DROP TABLE test.test_table; CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres3, port=5432); - ''') - assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') + """ + ) + assert node1.query(f"SELECT count() FROM test.test_table").rstrip() == "100" - assert (node1.query(f"SELECT count() FROM postgresql(postgres1)").rstrip() == '100') - node1.query("INSERT INTO TABLE FUNCTION postgresql(postgres1, on_conflict='ON CONFLICT DO NOTHING') SELECT number, number from numbers(100)") - assert (node1.query(f"SELECT count() FROM postgresql(postgres1)").rstrip() == '100') + assert node1.query(f"SELECT count() FROM postgresql(postgres1)").rstrip() == "100" + node1.query( + "INSERT INTO TABLE FUNCTION postgresql(postgres1, on_conflict='ON CONFLICT DO NOTHING') SELECT number, number from numbers(100)" + ) + assert node1.query(f"SELECT count() FROM postgresql(postgres1)").rstrip() == "100" - cursor.execute('DROP SCHEMA IF EXISTS test_schema CASCADE') - cursor.execute('CREATE SCHEMA test_schema') - cursor.execute('CREATE TABLE test_schema.test_table (a integer)') - node1.query("INSERT INTO TABLE FUNCTION postgresql(postgres1, schema='test_schema', on_conflict='ON CONFLICT DO NOTHING') SELECT number from numbers(200)") - assert (node1.query(f"SELECT count() FROM postgresql(postgres1, schema='test_schema')").rstrip() == '200') + cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") + cursor.execute("CREATE SCHEMA test_schema") + cursor.execute("CREATE TABLE test_schema.test_table (a integer)") + node1.query( + "INSERT INTO TABLE FUNCTION postgresql(postgres1, schema='test_schema', on_conflict='ON CONFLICT DO NOTHING') SELECT number from numbers(200)" + ) + assert ( + node1.query( + f"SELECT count() FROM postgresql(postgres1, schema='test_schema')" + ).rstrip() + == "200" + ) - cursor.execute('DROP SCHEMA test_schema CASCADE') - cursor.execute(f'DROP TABLE test_table ') + cursor.execute("DROP SCHEMA test_schema CASCADE") + cursor.execute(f"DROP TABLE test_table ") def test_where_false(started_cluster): cursor = started_cluster.postgres_conn.cursor() cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute('CREATE TABLE test (a Integer)') + cursor.execute("CREATE TABLE test (a Integer)") cursor.execute("INSERT INTO test SELECT 1") - result = node1.query("SELECT count() FROM postgresql('postgres1:5432', 'postgres', 'test', 'postgres', 'mysecretpassword') WHERE 1=0") - assert(int(result) == 0) - result = node1.query("SELECT count() FROM postgresql('postgres1:5432', 'postgres', 'test', 'postgres', 'mysecretpassword') WHERE 0") - assert(int(result) == 0) - result = node1.query("SELECT count() FROM postgresql('postgres1:5432', 'postgres', 'test', 'postgres', 'mysecretpassword') WHERE 1=1") - assert(int(result) == 1) + result = node1.query( + "SELECT count() FROM postgresql('postgres1:5432', 'postgres', 'test', 'postgres', 'mysecretpassword') WHERE 1=0" + ) + assert int(result) == 0 + result = node1.query( + "SELECT count() FROM postgresql('postgres1:5432', 'postgres', 'test', 'postgres', 'mysecretpassword') WHERE 0" + ) + assert int(result) == 0 + result = node1.query( + "SELECT count() FROM postgresql('postgres1:5432', 'postgres', 'test', 'postgres', 'mysecretpassword') WHERE 1=1" + ) + assert int(result) == 1 cursor.execute("DROP TABLE test") -if __name__ == '__main__': +def test_datetime64(started_cluster): + cursor = started_cluster.postgres_conn.cursor() + cursor.execute("drop table if exists test") + cursor.execute("create table test (ts timestamp)") + cursor.execute("insert into test select '1960-01-01 20:00:00';") + + result = node1.query("select * from postgresql(postgres1, table='test')") + assert result.strip() == "1960-01-01 20:00:00.000000" + + +def test_uuid(started_cluster): + cursor = started_cluster.postgres_conn.cursor() + cursor.execute("drop table if exists test") + cursor.execute("create table test (u uuid)") + cursor.execute("""CREATE EXTENSION IF NOT EXISTS "uuid-ossp";""") + cursor.execute("insert into test select uuid_generate_v1();") + + result = node1.query( + "select toTypeName(u) from postgresql(postgres1, table='test')" + ) + assert result.strip() == "Nullable(UUID)" + + +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_postgresql_replica/test.py b/tests/integration/test_storage_postgresql_replica/test.py index 4602d567b46..e51a9335a65 100644 --- a/tests/integration/test_storage_postgresql_replica/test.py +++ b/tests/integration/test_storage_postgresql_replica/test.py @@ -11,7 +11,12 @@ from helpers.test_tools import TSV import threading cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', main_configs=['configs/log_conf.xml'], with_postgres=True, stay_alive=True) +instance = cluster.add_instance( + "instance", + main_configs=["configs/log_conf.xml"], + with_postgres=True, + stay_alive=True, +) postgres_table_template = """ CREATE TABLE IF NOT EXISTS {} ( @@ -19,44 +24,63 @@ postgres_table_template = """ """ queries = [ - 'INSERT INTO postgresql_replica select i, i from generate_series(0, 10000) as t(i);', - 'DELETE FROM postgresql_replica WHERE (value*value) % 3 = 0;', - 'UPDATE postgresql_replica SET value = value + 125 WHERE key % 2 = 0;', + "INSERT INTO postgresql_replica select i, i from generate_series(0, 10000) as t(i);", + "DELETE FROM postgresql_replica WHERE (value*value) % 3 = 0;", + "UPDATE postgresql_replica SET value = value + 125 WHERE key % 2 = 0;", "UPDATE postgresql_replica SET key=key+20000 WHERE key%2=0", - 'INSERT INTO postgresql_replica select i, i from generate_series(40000, 50000) as t(i);', - 'DELETE FROM postgresql_replica WHERE key % 10 = 0;', - 'UPDATE postgresql_replica SET value = value + 101 WHERE key % 2 = 1;', + "INSERT INTO postgresql_replica select i, i from generate_series(40000, 50000) as t(i);", + "DELETE FROM postgresql_replica WHERE key % 10 = 0;", + "UPDATE postgresql_replica SET value = value + 101 WHERE key % 2 = 1;", "UPDATE postgresql_replica SET key=key+80000 WHERE key%2=1", - 'DELETE FROM postgresql_replica WHERE value % 2 = 0;', - 'UPDATE postgresql_replica SET value = value + 2000 WHERE key % 5 = 0;', - 'INSERT INTO postgresql_replica select i, i from generate_series(200000, 250000) as t(i);', - 'DELETE FROM postgresql_replica WHERE value % 3 = 0;', - 'UPDATE postgresql_replica SET value = value * 2 WHERE key % 3 = 0;', + "DELETE FROM postgresql_replica WHERE value % 2 = 0;", + "UPDATE postgresql_replica SET value = value + 2000 WHERE key % 5 = 0;", + "INSERT INTO postgresql_replica select i, i from generate_series(200000, 250000) as t(i);", + "DELETE FROM postgresql_replica WHERE value % 3 = 0;", + "UPDATE postgresql_replica SET value = value * 2 WHERE key % 3 = 0;", "UPDATE postgresql_replica SET key=key+500000 WHERE key%2=1", - 'INSERT INTO postgresql_replica select i, i from generate_series(1000000, 1050000) as t(i);', - 'DELETE FROM postgresql_replica WHERE value % 9 = 2;', + "INSERT INTO postgresql_replica select i, i from generate_series(1000000, 1050000) as t(i);", + "DELETE FROM postgresql_replica WHERE value % 9 = 2;", "UPDATE postgresql_replica SET key=key+10000000", - 'UPDATE postgresql_replica SET value = value + 2 WHERE key % 3 = 1;', - 'DELETE FROM postgresql_replica WHERE value%5 = 0;' - ] + "UPDATE postgresql_replica SET value = value + 2 WHERE key % 3 = 1;", + "DELETE FROM postgresql_replica WHERE value%5 = 0;", +] @pytest.mark.timeout(30) -def check_tables_are_synchronized(table_name, order_by='key', postgres_database='postgres_database'): - expected = instance.query('select * from {}.{} order by {};'.format(postgres_database, table_name, order_by)) - result = instance.query('select * from test.{} order by {};'.format(table_name, order_by)) +def check_tables_are_synchronized( + table_name, order_by="key", postgres_database="postgres_database" +): + expected = instance.query( + "select * from {}.{} order by {};".format( + postgres_database, table_name, order_by + ) + ) + result = instance.query( + "select * from test.{} order by {};".format(table_name, order_by) + ) while result != expected: time.sleep(0.5) - result = instance.query('select * from test.{} order by {};'.format(table_name, order_by)) + result = instance.query( + "select * from test.{} order by {};".format(table_name, order_by) + ) - assert(result == expected) + assert result == expected -def get_postgres_conn(ip, port, database=False, auto_commit=True, database_name='postgres_database'): + +def get_postgres_conn( + ip, port, database=False, auto_commit=True, database_name="postgres_database" +): if database == True: - conn_string = "host={} port={} dbname='{}' user='postgres' password='mysecretpassword'".format(ip, port, database_name) + conn_string = "host={} port={} dbname='{}' user='postgres' password='mysecretpassword'".format( + ip, port, database_name + ) else: - conn_string = "host={} port={} user='postgres' password='mysecretpassword'".format(ip, port) + conn_string = ( + "host={} port={} user='postgres' password='mysecretpassword'".format( + ip, port + ) + ) conn = psycopg2.connect(conn_string) if auto_commit: @@ -64,29 +88,43 @@ def get_postgres_conn(ip, port, database=False, auto_commit=True, database_name= conn.autocommit = True return conn + def create_postgres_db(cursor, name): cursor.execute("CREATE DATABASE {}".format(name)) -def create_clickhouse_postgres_db(ip, port, name='postgres_database'): - instance.query(''' + +def create_clickhouse_postgres_db(ip, port, name="postgres_database"): + instance.query( + """ CREATE DATABASE {} - ENGINE = PostgreSQL('{}:{}', '{}', 'postgres', 'mysecretpassword')'''.format(name, ip, port, name)) + ENGINE = PostgreSQL('{}:{}', '{}', 'postgres', 'mysecretpassword')""".format( + name, ip, port, name + ) + ) + def create_materialized_table(ip, port): - instance.query(''' + instance.query( + """ CREATE TABLE test.postgresql_replica (key UInt64, value UInt64) ENGINE = MaterializedPostgreSQL( '{}:{}', 'postgres_database', 'postgresql_replica', 'postgres', 'mysecretpassword') - PRIMARY KEY key; '''.format(ip, port)) + PRIMARY KEY key; """.format( + ip, port + ) + ) + def create_postgres_table(cursor, table_name, replica_identity_full=False): cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) cursor.execute(postgres_table_template.format(table_name)) if replica_identity_full: - cursor.execute('ALTER TABLE {} REPLICA IDENTITY FULL;'.format(table_name)) + cursor.execute("ALTER TABLE {} REPLICA IDENTITY FULL;".format(table_name)) -def postgresql_replica_check_result(result, check=False, ref_file='test_postgresql_replica.reference'): +def postgresql_replica_check_result( + result, check=False, ref_file="test_postgresql_replica.reference" +): fpath = p.join(p.dirname(__file__), ref_file) with open(fpath) as reference: if check: @@ -99,14 +137,14 @@ def postgresql_replica_check_result(result, check=False, ref_file='test_postgres def started_cluster(): try: cluster.start() - conn = get_postgres_conn(ip=cluster.postgres_ip, - port=cluster.postgres_port) + conn = get_postgres_conn(ip=cluster.postgres_ip, port=cluster.postgres_port) cursor = conn.cursor() - create_postgres_db(cursor, 'postgres_database') - create_clickhouse_postgres_db(ip=cluster.postgres_ip, - port=cluster.postgres_port) + create_postgres_db(cursor, "postgres_database") + create_clickhouse_postgres_db( + ip=cluster.postgres_ip, port=cluster.postgres_port + ) - instance.query('CREATE DATABASE test') + instance.query("CREATE DATABASE test") yield cluster finally: @@ -115,486 +153,601 @@ def started_cluster(): @pytest.mark.timeout(320) def test_initial_load_from_snapshot(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") while postgresql_replica_check_result(result) == False: time.sleep(0.2) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_no_connection_at_startup(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) time.sleep(3) - instance.query('DETACH TABLE test.postgresql_replica') - started_cluster.pause_container('postgres1') + instance.query("DETACH TABLE test.postgresql_replica") + started_cluster.pause_container("postgres1") - instance.query('ATTACH TABLE test.postgresql_replica') + instance.query("ATTACH TABLE test.postgresql_replica") time.sleep(3) - started_cluster.unpause_container('postgres1') + started_cluster.unpause_container("postgres1") - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") while int(result) == 0: - time.sleep(0.5); - result = instance.query('SELECT count() FROM test.postgresql_replica;') + time.sleep(0.5) + result = instance.query("SELECT count() FROM test.postgresql_replica;") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') - cursor.execute('DROP TABLE postgresql_replica;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_detach_attach_is_ok(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) == 0): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) == 0: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") postgresql_replica_check_result(result, True) - instance.query('DETACH TABLE test.postgresql_replica') - instance.query('ATTACH TABLE test.postgresql_replica') + instance.query("DETACH TABLE test.postgresql_replica") + instance.query("ATTACH TABLE test.postgresql_replica") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") while postgresql_replica_check_result(result) == False: time.sleep(0.5) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_replicating_insert_queries(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(10)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(10)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 10): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 10: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 10 + number, 10 + number from numbers(10)") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 20 + number, 20 + number from numbers(10)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 10 + number, 10 + number from numbers(10)" + ) + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 20 + number, 20 + number from numbers(10)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 30): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 30: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 30 + number, 30 + number from numbers(10)") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 40 + number, 40 + number from numbers(10)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 30 + number, 30 + number from numbers(10)" + ) + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 40 + number, 40 + number from numbers(10)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 50): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 50: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') - cursor.execute('DROP TABLE postgresql_replica;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_replicating_delete_queries(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") while postgresql_replica_check_result(result) == False: time.sleep(0.2) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, 50 + number from numbers(50)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, 50 + number from numbers(50)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") while int(result) != 100: time.sleep(0.5) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - cursor.execute('DELETE FROM postgresql_replica WHERE key > 49;') + cursor.execute("DELETE FROM postgresql_replica WHERE key > 49;") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") while postgresql_replica_check_result(result) == False: time.sleep(0.5) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_replicating_update_queries(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number + 10 from numbers(50)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number + 10 from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 50): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 50: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - cursor.execute('UPDATE postgresql_replica SET value = value - 10;') + cursor.execute("UPDATE postgresql_replica SET value = value - 10;") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") while postgresql_replica_check_result(result) == False: time.sleep(0.5) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_resume_from_written_version(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number + 10 from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number + 10 from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 50): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 50: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, 50 + number from numbers(50)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT 50 + number, 50 + number from numbers(50)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 100): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 100: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - instance.query('DETACH TABLE test.postgresql_replica') + instance.query("DETACH TABLE test.postgresql_replica") - cursor.execute('DELETE FROM postgresql_replica WHERE key > 49;') - cursor.execute('UPDATE postgresql_replica SET value = value - 10;') + cursor.execute("DELETE FROM postgresql_replica WHERE key > 49;") + cursor.execute("UPDATE postgresql_replica SET value = value - 10;") - instance.query('ATTACH TABLE test.postgresql_replica') + instance.query("ATTACH TABLE test.postgresql_replica") - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") while postgresql_replica_check_result(result) == False: time.sleep(0.5) - result = instance.query('SELECT * FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) @pytest.mark.timeout(320) def test_many_replication_messages(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 100000): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 100000: time.sleep(0.2) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") print("SYNC OK") - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000, 100000)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000, 100000)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') - while (int(result) != 200000): + result = instance.query("SELECT count() FROM test.postgresql_replica;") + while int(result) != 200000: time.sleep(1) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") print("INSERT OK") - result = instance.query('SELECT key FROM test.postgresql_replica ORDER BY key;') + result = instance.query("SELECT key FROM test.postgresql_replica ORDER BY key;") expected = instance.query("SELECT number from numbers(200000)") - assert(result == expected) + assert result == expected - cursor.execute('UPDATE postgresql_replica SET value = key + 1 WHERE key < 100000;') + cursor.execute("UPDATE postgresql_replica SET value = key + 1 WHERE key < 100000;") - result = instance.query('SELECT key FROM test.postgresql_replica WHERE value = key + 1 ORDER BY key;') + result = instance.query( + "SELECT key FROM test.postgresql_replica WHERE value = key + 1 ORDER BY key;" + ) expected = instance.query("SELECT number from numbers(100000)") - while (result != expected): + while result != expected: time.sleep(1) - result = instance.query('SELECT key FROM test.postgresql_replica WHERE value = key + 1 ORDER BY key;') + result = instance.query( + "SELECT key FROM test.postgresql_replica WHERE value = key + 1 ORDER BY key;" + ) print("UPDATE OK") - cursor.execute('DELETE FROM postgresql_replica WHERE key % 2 = 1;') - cursor.execute('DELETE FROM postgresql_replica WHERE key != value;') + cursor.execute("DELETE FROM postgresql_replica WHERE key % 2 = 1;") + cursor.execute("DELETE FROM postgresql_replica WHERE key != value;") - result = instance.query('SELECT count() FROM (SELECT * FROM test.postgresql_replica);') - while (int(result) != 50000): + result = instance.query( + "SELECT count() FROM (SELECT * FROM test.postgresql_replica);" + ) + while int(result) != 50000: time.sleep(1) - result = instance.query('SELECT count() FROM (SELECT * FROM test.postgresql_replica);') + result = instance.query( + "SELECT count() FROM (SELECT * FROM test.postgresql_replica);" + ) print("DELETE OK") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") @pytest.mark.timeout(320) def test_connection_loss(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) i = 50 while i < 100000: - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT {} + number, number from numbers(10000)".format(i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT {} + number, number from numbers(10000)".format( + i + ) + ) i += 10000 - started_cluster.pause_container('postgres1') + started_cluster.pause_container("postgres1") - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") print(int(result)) time.sleep(6) - started_cluster.unpause_container('postgres1') + started_cluster.unpause_container("postgres1") - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") while int(result) < 100050: time.sleep(1) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - cursor.execute('DROP TABLE postgresql_replica;') - assert(int(result) == 100050) + cursor.execute("DROP TABLE postgresql_replica;") + assert int(result) == 100050 @pytest.mark.timeout(320) def test_clickhouse_restart(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(50)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) i = 50 while i < 100000: - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT {} + number, number from numbers(10000)".format(i)) + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT {} + number, number from numbers(10000)".format( + i + ) + ) i += 10000 instance.restart_clickhouse() - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") while int(result) < 100050: time.sleep(1) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") print(result) - assert(int(result) == 100050) + assert int(result) == 100050 def test_rename_table(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(25)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(25)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") while int(result) != 25: time.sleep(0.5) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") - instance.query('RENAME TABLE test.postgresql_replica TO test.postgresql_replica_renamed') - assert(int(instance.query('SELECT count() FROM test.postgresql_replica_renamed;')) == 25) + instance.query( + "RENAME TABLE test.postgresql_replica TO test.postgresql_replica_renamed" + ) + assert ( + int(instance.query("SELECT count() FROM test.postgresql_replica_renamed;")) + == 25 + ) - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(25, 25)") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(25, 25)" + ) - result = instance.query('SELECT count() FROM test.postgresql_replica_renamed;') + result = instance.query("SELECT count() FROM test.postgresql_replica_renamed;") while int(result) != 50: time.sleep(0.5) - result = instance.query('SELECT count() FROM test.postgresql_replica_renamed;') + result = instance.query("SELECT count() FROM test.postgresql_replica_renamed;") - result = instance.query('SELECT * FROM test.postgresql_replica_renamed ORDER BY key;') + result = instance.query( + "SELECT * FROM test.postgresql_replica_renamed ORDER BY key;" + ) postgresql_replica_check_result(result, True) - cursor.execute('DROP TABLE postgresql_replica;') - instance.query('DROP TABLE IF EXISTS test.postgresql_replica_renamed') + cursor.execute("DROP TABLE postgresql_replica;") + instance.query("DROP TABLE IF EXISTS test.postgresql_replica_renamed") def test_virtual_columns(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(10)") - result = instance.query('SELECT count() FROM test.postgresql_replica;') + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(10)" + ) + result = instance.query("SELECT count() FROM test.postgresql_replica;") while int(result) != 10: time.sleep(0.5) - result = instance.query('SELECT count() FROM test.postgresql_replica;') + result = instance.query("SELECT count() FROM test.postgresql_replica;") # just check that it works, no check with `expected` becuase _version is taken as LSN, which will be different each time. - result = instance.query('SELECT key, value, _sign, _version FROM test.postgresql_replica;') + result = instance.query( + "SELECT key, value, _sign, _version FROM test.postgresql_replica;" + ) print(result) - cursor.execute('DROP TABLE postgresql_replica;') + cursor.execute("DROP TABLE postgresql_replica;") def test_abrupt_connection_loss_while_heavy_replication(started_cluster): instance.query("DROP DATABASE IF EXISTS test_database") - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for i in range(len(queries)): query = queries[i] cursor.execute(query) - print('query {}'.format(query)) + print("query {}".format(query)) - started_cluster.pause_container('postgres1') + started_cluster.pause_container("postgres1") result = instance.query("SELECT count() FROM test.postgresql_replica") - print(result) # Just debug + print(result) # Just debug - started_cluster.unpause_container('postgres1') + started_cluster.unpause_container("postgres1") - check_tables_are_synchronized('postgresql_replica'); + check_tables_are_synchronized("postgresql_replica") result = instance.query("SELECT count() FROM test.postgresql_replica") - print(result) # Just debug + print(result) # Just debug def test_abrupt_server_restart_while_heavy_replication(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); + create_postgres_table(cursor, "postgresql_replica") - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port) + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) for query in queries: cursor.execute(query) - print('query {}'.format(query)) + print("query {}".format(query)) instance.restart_clickhouse() result = instance.query("SELECT count() FROM test.postgresql_replica") - print(result) # Just debug + print(result) # Just debug - check_tables_are_synchronized('postgresql_replica'); + check_tables_are_synchronized("postgresql_replica") result = instance.query("SELECT count() FROM test.postgresql_replica") - print(result) # Just debug + print(result) # Just debug def test_drop_table_immediately(started_cluster): - conn = get_postgres_conn(ip=started_cluster.postgres_ip, - port=started_cluster.postgres_port, - database=True) + conn = get_postgres_conn( + ip=started_cluster.postgres_ip, + port=started_cluster.postgres_port, + database=True, + ) cursor = conn.cursor() - create_postgres_table(cursor, 'postgresql_replica'); - instance.query("INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000)") + create_postgres_table(cursor, "postgresql_replica") + instance.query( + "INSERT INTO postgres_database.postgresql_replica SELECT number, number from numbers(100000)" + ) - instance.query('DROP TABLE IF EXISTS test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) - instance.query('DROP TABLE test.postgresql_replica') - create_materialized_table(ip=started_cluster.postgres_ip, port=started_cluster.postgres_port) - check_tables_are_synchronized('postgresql_replica'); - instance.query('DROP TABLE test.postgresql_replica') + instance.query("DROP TABLE IF EXISTS test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) + instance.query("DROP TABLE test.postgresql_replica") + create_materialized_table( + ip=started_cluster.postgres_ip, port=started_cluster.postgres_port + ) + check_tables_are_synchronized("postgresql_replica") + instance.query("DROP TABLE test.postgresql_replica") -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py index 6abc087dc75..df5c29adc6d 100644 --- a/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py +++ b/tests/integration/test_storage_rabbitmq/rabbitmq_pb2.py @@ -12,60 +12,85 @@ from google.protobuf import symbol_database as _symbol_database _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( - name='clickhouse_path/format_schemas/rabbitmq.proto', - package='', - syntax='proto3', + name="clickhouse_path/format_schemas/rabbitmq.proto", + package="", + syntax="proto3", serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n-clickhouse_path/format_schemas/rabbitmq.proto\"+\n\rKeyValueProto\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3' + serialized_pb=b'\n-clickhouse_path/format_schemas/rabbitmq.proto"+\n\rKeyValueProto\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\tb\x06proto3', ) _KEYVALUEPROTO = _descriptor.Descriptor( - name='KeyValueProto', - full_name='KeyValueProto', + name="KeyValueProto", + full_name="KeyValueProto", filename=None, file=DESCRIPTOR, containing_type=None, create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='KeyValueProto.key', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + name="key", + full_name="KeyValueProto.key", + index=0, + number=1, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), _descriptor.FieldDescriptor( - name='value', full_name='KeyValueProto.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ + name="value", + full_name="KeyValueProto.value", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), ], + extensions=[], nested_types=[], - enum_types=[ - ], + enum_types=[], serialized_options=None, is_extendable=False, - syntax='proto3', + syntax="proto3", extension_ranges=[], - oneofs=[ - ], + oneofs=[], serialized_start=49, serialized_end=92, ) -DESCRIPTOR.message_types_by_name['KeyValueProto'] = _KEYVALUEPROTO +DESCRIPTOR.message_types_by_name["KeyValueProto"] = _KEYVALUEPROTO _sym_db.RegisterFileDescriptor(DESCRIPTOR) -KeyValueProto = _reflection.GeneratedProtocolMessageType('KeyValueProto', (_message.Message,), { - 'DESCRIPTOR': _KEYVALUEPROTO, - '__module__': 'clickhouse_path.format_schemas.rabbitmq_pb2' - # @@protoc_insertion_point(class_scope:KeyValueProto) -}) +KeyValueProto = _reflection.GeneratedProtocolMessageType( + "KeyValueProto", + (_message.Message,), + { + "DESCRIPTOR": _KEYVALUEPROTO, + "__module__": "clickhouse_path.format_schemas.rabbitmq_pb2" + # @@protoc_insertion_point(class_scope:KeyValueProto) + }, +) _sym_db.RegisterMessage(KeyValueProto) # @@protoc_insertion_point(module_scope) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index a3d99159cb2..d5011607556 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -18,16 +18,22 @@ from helpers.test_tools import TSV from . import rabbitmq_pb2 cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', - main_configs=['configs/rabbitmq.xml', 'configs/macros.xml', 'configs/named_collection.xml'], - user_configs=['configs/users.xml'], - with_rabbitmq=True) +instance = cluster.add_instance( + "instance", + main_configs=[ + "configs/rabbitmq.xml", + "configs/macros.xml", + "configs/named_collection.xml", + ], + user_configs=["configs/users.xml"], + with_rabbitmq=True, +) # Helpers -def rabbitmq_check_result(result, check=False, ref_file='test_rabbitmq_json.reference'): +def rabbitmq_check_result(result, check=False, ref_file="test_rabbitmq_json.reference"): fpath = p.join(p.dirname(__file__), ref_file) with open(fpath) as reference: if check: @@ -35,6 +41,7 @@ def rabbitmq_check_result(result, check=False, ref_file='test_rabbitmq_json.refe else: return TSV(result) == TSV(reference) + def wait_rabbitmq_to_start(rabbitmq_docker_id, timeout=180): start = time.time() while time.time() - start < timeout: @@ -47,26 +54,28 @@ def wait_rabbitmq_to_start(rabbitmq_docker_id, timeout=180): logging.debug("Can't connect to RabbitMQ " + str(ex)) time.sleep(0.5) + def kill_rabbitmq(rabbitmq_id): - p = subprocess.Popen(('docker', 'stop', rabbitmq_id), stdout=subprocess.PIPE) + p = subprocess.Popen(("docker", "stop", rabbitmq_id), stdout=subprocess.PIPE) p.communicate() return p.returncode == 0 def revive_rabbitmq(rabbitmq_id): - p = subprocess.Popen(('docker', 'start', rabbitmq_id), stdout=subprocess.PIPE) + p = subprocess.Popen(("docker", "start", rabbitmq_id), stdout=subprocess.PIPE) p.communicate() wait_rabbitmq_to_start(rabbitmq_id) # Fixtures + @pytest.fixture(scope="module") def rabbitmq_cluster(): try: cluster.start() logging.debug("rabbitmq_id is {}".format(instance.cluster.rabbitmq_docker_id)) - instance.query('CREATE DATABASE test') + instance.query("CREATE DATABASE test") yield cluster @@ -78,14 +87,16 @@ def rabbitmq_cluster(): def rabbitmq_setup_teardown(): print("RabbitMQ is available - running test") yield # run test - instance.query('DROP DATABASE test NO DELAY') - instance.query('CREATE DATABASE test') + instance.query("DROP DATABASE test NO DELAY") + instance.query("CREATE DATABASE test") # Tests + def test_rabbitmq_select(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = '{}:5672', @@ -93,27 +104,34 @@ def test_rabbitmq_select(rabbitmq_cluster): rabbitmq_commit_on_select = 1, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; - '''.format(rabbitmq_cluster.rabbitmq_host)) + """.format( + rabbitmq_cluster.rabbitmq_host + ) + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) for message in messages: - channel.basic_publish(exchange='select', routing_key='', body=message) + channel.basic_publish(exchange="select", routing_key="", body=message) connection.close() # The order of messages in select * from test.rabbitmq is not guaranteed, so sleep to collect everything in one select time.sleep(1) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.rabbitmq ORDER BY key', ignore_error=True) + result += instance.query( + "SELECT * FROM test.rabbitmq ORDER BY key", ignore_error=True + ) if rabbitmq_check_result(result): break @@ -121,7 +139,8 @@ def test_rabbitmq_select(rabbitmq_cluster): def test_rabbitmq_select_empty(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = '{}:5672', @@ -129,47 +148,58 @@ def test_rabbitmq_select_empty(rabbitmq_cluster): rabbitmq_commit_on_select = 1, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; - '''.format(rabbitmq_cluster.rabbitmq_host)) + """.format( + rabbitmq_cluster.rabbitmq_host + ) + ) - assert int(instance.query('SELECT count() FROM test.rabbitmq')) == 0 + assert int(instance.query("SELECT count() FROM test.rabbitmq")) == 0 def test_rabbitmq_json_without_delimiter(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = '{}:5672', rabbitmq_commit_on_select = 1, rabbitmq_exchange_name = 'json', rabbitmq_format = 'JSONEachRow' - '''.format(rabbitmq_cluster.rabbitmq_host)) + """.format( + rabbitmq_cluster.rabbitmq_host + ) + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - messages = '' + messages = "" for i in range(25): - messages += json.dumps({'key': i, 'value': i}) + '\n' + messages += json.dumps({"key": i, "value": i}) + "\n" all_messages = [messages] for message in all_messages: - channel.basic_publish(exchange='json', routing_key='', body=message) + channel.basic_publish(exchange="json", routing_key="", body=message) - messages = '' + messages = "" for i in range(25, 50): - messages += json.dumps({'key': i, 'value': i}) + '\n' + messages += json.dumps({"key": i, "value": i}) + "\n" all_messages = [messages] for message in all_messages: - channel.basic_publish(exchange='json', routing_key='', body=message) + channel.basic_publish(exchange="json", routing_key="", body=message) connection.close() time.sleep(1) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.rabbitmq ORDER BY key', ignore_error=True) + result += instance.query( + "SELECT * FROM test.rabbitmq ORDER BY key", ignore_error=True + ) if rabbitmq_check_result(result): break @@ -177,7 +207,8 @@ def test_rabbitmq_json_without_delimiter(rabbitmq_cluster): def test_rabbitmq_csv_with_delimiter(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -185,26 +216,31 @@ def test_rabbitmq_csv_with_delimiter(rabbitmq_cluster): rabbitmq_commit_on_select = 1, rabbitmq_format = 'CSV', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(50): - messages.append('{i}, {i}'.format(i=i)) + messages.append("{i}, {i}".format(i=i)) for message in messages: - channel.basic_publish(exchange='csv', routing_key='', body=message) + channel.basic_publish(exchange="csv", routing_key="", body=message) connection.close() time.sleep(1) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.rabbitmq ORDER BY key', ignore_error=True) + result += instance.query( + "SELECT * FROM test.rabbitmq ORDER BY key", ignore_error=True + ) if rabbitmq_check_result(result): break @@ -212,7 +248,8 @@ def test_rabbitmq_csv_with_delimiter(rabbitmq_cluster): def test_rabbitmq_tsv_with_delimiter(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -226,24 +263,27 @@ def test_rabbitmq_tsv_with_delimiter(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(50): - messages.append('{i}\t{i}'.format(i=i)) + messages.append("{i}\t{i}".format(i=i)) for message in messages: - channel.basic_publish(exchange='tsv', routing_key='', body=message) + channel.basic_publish(exchange="tsv", routing_key="", body=message) connection.close() - result = '' + result = "" while True: - result = instance.query('SELECT * FROM test.view ORDER BY key') + result = instance.query("SELECT * FROM test.view ORDER BY key") if rabbitmq_check_result(result): break @@ -251,31 +291,37 @@ def test_rabbitmq_tsv_with_delimiter(rabbitmq_cluster): def test_rabbitmq_macros(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = '{rabbitmq_host}:{rabbitmq_port}', rabbitmq_commit_on_select = 1, rabbitmq_exchange_name = '{rabbitmq_exchange_name}', rabbitmq_format = '{rabbitmq_format}' - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - message = '' + message = "" for i in range(50): - message += json.dumps({'key': i, 'value': i}) + '\n' - channel.basic_publish(exchange='macro', routing_key='', body=message) + message += json.dumps({"key": i, "value": i}) + "\n" + channel.basic_publish(exchange="macro", routing_key="", body=message) connection.close() time.sleep(1) - result = '' + result = "" while True: - result += instance.query('SELECT * FROM test.rabbitmq ORDER BY key', ignore_error=True) + result += instance.query( + "SELECT * FROM test.rabbitmq ORDER BY key", ignore_error=True + ) if rabbitmq_check_result(result): break @@ -283,7 +329,8 @@ def test_rabbitmq_macros(rabbitmq_cluster): def test_rabbitmq_materialized_view(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -301,25 +348,28 @@ def test_rabbitmq_materialized_view(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS SELECT * FROM test.rabbitmq group by (key, value); - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) for message in messages: - channel.basic_publish(exchange='mv', routing_key='', body=message) + channel.basic_publish(exchange="mv", routing_key="", body=message) time_limit_sec = 60 deadline = time.monotonic() + time_limit_sec while time.monotonic() < deadline: - result = instance.query('SELECT * FROM test.view ORDER BY key') - if (rabbitmq_check_result(result)): + result = instance.query("SELECT * FROM test.view ORDER BY key") + if rabbitmq_check_result(result): break rabbitmq_check_result(result, True) @@ -327,8 +377,8 @@ def test_rabbitmq_materialized_view(rabbitmq_cluster): deadline = time.monotonic() + time_limit_sec while time.monotonic() < deadline: - result = instance.query('SELECT * FROM test.view2 ORDER BY key') - if (rabbitmq_check_result(result)): + result = instance.query("SELECT * FROM test.view2 ORDER BY key") + if rabbitmq_check_result(result): break rabbitmq_check_result(result, True) @@ -336,7 +386,8 @@ def test_rabbitmq_materialized_view(rabbitmq_cluster): def test_rabbitmq_materialized_view_with_subquery(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -348,21 +399,24 @@ def test_rabbitmq_materialized_view_with_subquery(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM (SELECT * FROM test.rabbitmq); - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) for message in messages: - channel.basic_publish(exchange='mvsq', routing_key='', body=message) + channel.basic_publish(exchange="mvsq", routing_key="", body=message) while True: - result = instance.query('SELECT * FROM test.view ORDER BY key') + result = instance.query("SELECT * FROM test.view ORDER BY key") if rabbitmq_check_result(result): break @@ -371,7 +425,8 @@ def test_rabbitmq_materialized_view_with_subquery(rabbitmq_cluster): def test_rabbitmq_many_materialized_views(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view1; DROP TABLE IF EXISTS test.view2; DROP TABLE IF EXISTS test.consumer1; @@ -392,31 +447,36 @@ def test_rabbitmq_many_materialized_views(rabbitmq_cluster): SELECT * FROM test.rabbitmq; CREATE MATERIALIZED VIEW test.consumer2 TO test.view2 AS SELECT * FROM test.rabbitmq; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(50): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) for message in messages: - channel.basic_publish(exchange='mmv', routing_key='', body=message) + channel.basic_publish(exchange="mmv", routing_key="", body=message) while True: - result1 = instance.query('SELECT * FROM test.view1 ORDER BY key') - result2 = instance.query('SELECT * FROM test.view2 ORDER BY key') + result1 = instance.query("SELECT * FROM test.view1 ORDER BY key") + result2 = instance.query("SELECT * FROM test.view2 ORDER BY key") if rabbitmq_check_result(result1) and rabbitmq_check_result(result2): break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer1; DROP TABLE test.consumer2; DROP TABLE test.view1; DROP TABLE test.view2; - ''') + """ + ) connection.close() rabbitmq_check_result(result1, True) @@ -425,7 +485,8 @@ def test_rabbitmq_many_materialized_views(rabbitmq_cluster): @pytest.mark.skip(reason="clichouse_path with rabbitmq.proto fails to be exported") def test_rabbitmq_protobuf(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value String) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -437,43 +498,46 @@ def test_rabbitmq_protobuf(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - data = '' + data = "" for i in range(0, 20): msg = rabbitmq_pb2.KeyValueProto() msg.key = i msg.value = str(i) serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg - channel.basic_publish(exchange='pb', routing_key='', body=data) - data = '' + channel.basic_publish(exchange="pb", routing_key="", body=data) + data = "" for i in range(20, 21): msg = rabbitmq_pb2.KeyValueProto() msg.key = i msg.value = str(i) serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg - channel.basic_publish(exchange='pb', routing_key='', body=data) - data = '' + channel.basic_publish(exchange="pb", routing_key="", body=data) + data = "" for i in range(21, 50): msg = rabbitmq_pb2.KeyValueProto() msg.key = i msg.value = str(i) serialized_msg = msg.SerializeToString() data = data + _VarintBytes(len(serialized_msg)) + serialized_msg - channel.basic_publish(exchange='pb', routing_key='', body=data) + channel.basic_publish(exchange="pb", routing_key="", body=data) connection.close() - result = '' + result = "" while True: - result = instance.query('SELECT * FROM test.view ORDER BY key') + result = instance.query("SELECT * FROM test.view ORDER BY key") if rabbitmq_check_result(result): break @@ -484,14 +548,20 @@ def test_rabbitmq_big_message(rabbitmq_cluster): # Create batchs of messages of size ~100Kb rabbitmq_messages = 1000 batch_messages = 1000 - messages = [json.dumps({'key': i, 'value': 'x' * 100}) * batch_messages for i in range(rabbitmq_messages)] + messages = [ + json.dumps({"key": i, "value": "x" * 100}) * batch_messages + for i in range(rabbitmq_messages) + ] - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value String) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -502,26 +572,30 @@ def test_rabbitmq_big_message(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; - ''') + """ + ) for message in messages: - channel.basic_publish(exchange='big', routing_key='', body=message) + channel.basic_publish(exchange="big", routing_key="", body=message) while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") if int(result) == batch_messages * rabbitmq_messages: break connection.close() - assert int(result) == rabbitmq_messages * batch_messages, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == rabbitmq_messages * batch_messages + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): NUM_CONSUMERS = 10 NUM_QUEUES = 10 - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -536,13 +610,16 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): SETTINGS old_parts_lifetime=5, cleanup_delay_period=2, cleanup_delay_period_random_add=3; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT *, _channel_id AS channel_id FROM test.rabbitmq; - ''') + """ + ) i = [0] messages_num = 10000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) def produce(): connection = pika.BlockingConnection(parameters) @@ -550,14 +627,18 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 current = 0 for message in messages: current += 1 mes_id = str(current) - channel.basic_publish(exchange='test_sharding', routing_key='', - properties=pika.BasicProperties(message_id=mes_id), body=message) + channel.basic_publish( + exchange="test_sharding", + routing_key="", + properties=pika.BasicProperties(message_id=mes_id), + body=message, + ) connection.close() threads = [] @@ -569,9 +650,9 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() - result1 = '' + result1 = "" while True: - result1 = instance.query('SELECT count() FROM test.view') + result1 = instance.query("SELECT count() FROM test.view") time.sleep(1) if int(result1) == messages_num * threads_num: break @@ -581,7 +662,9 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): for thread in threads: thread.join() - assert int(result1) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result1) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result) assert int(result2) == 10 @@ -589,7 +672,8 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): NUM_MV = 5 NUM_CONSUMERS = 4 - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -599,10 +683,12 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): rabbitmq_num_queues = 5, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) for mv_id in range(NUM_MV): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.combo_{0}; DROP TABLE IF EXISTS test.combo_{0}_mv; CREATE TABLE test.combo_{0} (key UInt64, value UInt64) @@ -610,15 +696,20 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.combo_{0}_mv TO test.combo_{0} AS SELECT * FROM test.rabbitmq; - '''.format(mv_id)) + """.format( + mv_id + ) + ) time.sleep(2) i = [0] messages_num = 10000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) def produce(): connection = pika.BlockingConnection(parameters) @@ -626,11 +717,15 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 for msg_id in range(messages_num): - channel.basic_publish(exchange='combo', routing_key='', - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + channel.basic_publish( + exchange="combo", + routing_key="", + properties=pika.BasicProperties(message_id=str(msg_id)), + body=messages[msg_id], + ) connection.close() threads = [] @@ -645,7 +740,9 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): while True: result = 0 for mv_id in range(NUM_MV): - result += int(instance.query('SELECT count() FROM test.combo_{0}'.format(mv_id))) + result += int( + instance.query("SELECT count() FROM test.combo_{0}".format(mv_id)) + ) if int(result) == messages_num * threads_num * NUM_MV: break time.sleep(1) @@ -654,16 +751,23 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): thread.join() for mv_id in range(NUM_MV): - instance.query(''' + instance.query( + """ DROP TABLE test.combo_{0}_mv; DROP TABLE test.combo_{0}; - '''.format(mv_id)) + """.format( + mv_id + ) + ) - assert int(result) == messages_num * threads_num * NUM_MV, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * threads_num * NUM_MV + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_insert(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -672,28 +776,31 @@ def test_rabbitmq_insert(rabbitmq_cluster): rabbitmq_routing_key_list = 'insert1', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) consumer_connection = pika.BlockingConnection(parameters) consumer = consumer_connection.channel() - result = consumer.queue_declare(queue='') + result = consumer.queue_declare(queue="") queue_name = result.method.queue - consumer.queue_bind(exchange='insert', queue=queue_name, routing_key='insert1') + consumer.queue_bind(exchange="insert", queue=queue_name, routing_key="insert1") values = [] for i in range(50): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) while True: try: instance.query("INSERT INTO test.rabbitmq VALUES {}".format(values)) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise @@ -703,19 +810,20 @@ def test_rabbitmq_insert(rabbitmq_cluster): def onReceived(channel, method, properties, body): i = 0 insert_messages.append(body.decode()) - if (len(insert_messages) == 50): + if len(insert_messages) == 50: channel.stop_consuming() consumer.basic_consume(onReceived, queue_name) consumer.start_consuming() consumer_connection.close() - result = '\n'.join(insert_messages) + result = "\n".join(insert_messages) rabbitmq_check_result(result, True) def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -724,29 +832,36 @@ def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): rabbitmq_routing_key_list = 'test=insert,topic=headers', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) consumer_connection = pika.BlockingConnection(parameters) consumer = consumer_connection.channel() - result = consumer.queue_declare(queue='') + result = consumer.queue_declare(queue="") queue_name = result.method.queue - consumer.queue_bind(exchange='insert_headers', queue=queue_name, routing_key="", - arguments={'x-match': 'all', 'test': 'insert', 'topic': 'headers'}) + consumer.queue_bind( + exchange="insert_headers", + queue=queue_name, + routing_key="", + arguments={"x-match": "all", "test": "insert", "topic": "headers"}, + ) values = [] for i in range(50): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) while True: try: instance.query("INSERT INTO test.rabbitmq VALUES {}".format(values)) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise @@ -756,19 +871,20 @@ def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): def onReceived(channel, method, properties, body): i = 0 insert_messages.append(body.decode()) - if (len(insert_messages) == 50): + if len(insert_messages) == 50: channel.stop_consuming() consumer.basic_consume(onReceived, queue_name) consumer.start_consuming() consumer_connection.close() - result = '\n'.join(insert_messages) + result = "\n".join(insert_messages) rabbitmq_check_result(result, True) def test_rabbitmq_many_inserts(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.rabbitmq_many; DROP TABLE IF EXISTS test.rabbitmq_consume; DROP TABLE IF EXISTS test.view_many; @@ -789,21 +905,24 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): rabbitmq_routing_key_list = 'insert2', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) messages_num = 10000 values = [] for i in range(messages_num): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) def insert(): while True: try: - instance.query("INSERT INTO test.rabbitmq_many VALUES {}".format(values)) + instance.query( + "INSERT INTO test.rabbitmq_many VALUES {}".format(values) + ) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise @@ -816,36 +935,43 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() - instance.query(''' + instance.query( + """ CREATE TABLE test.view_many (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; CREATE MATERIALIZED VIEW test.consumer_many TO test.view_many AS SELECT * FROM test.rabbitmq_consume; - ''') + """ + ) for thread in threads: thread.join() while True: - result = instance.query('SELECT count() FROM test.view_many') + result = instance.query("SELECT count() FROM test.view_many") print(result, messages_num * threads_num) if int(result) == messages_num * threads_num: break time.sleep(1) - instance.query(''' + instance.query( + """ DROP TABLE test.rabbitmq_consume; DROP TABLE test.rabbitmq_many; DROP TABLE test.consumer_many; DROP TABLE test.view_many; - ''') + """ + ) - assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_overloaded_insert(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view_overload; DROP TABLE IF EXISTS test.consumer_overload; DROP TABLE IF EXISTS test.rabbitmq_consume; @@ -875,7 +1001,8 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): SETTINGS old_parts_lifetime=5, cleanup_delay_period=2, cleanup_delay_period_random_add=3; CREATE MATERIALIZED VIEW test.consumer_overload TO test.view_overload AS SELECT * FROM test.rabbitmq_consume; - ''') + """ + ) messages_num = 100000 @@ -883,14 +1010,16 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): values = [] for i in range(messages_num): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) while True: try: - instance.query("INSERT INTO test.rabbitmq_overload VALUES {}".format(values)) + instance.query( + "INSERT INTO test.rabbitmq_overload VALUES {}".format(values) + ) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise @@ -904,37 +1033,44 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): thread.start() while True: - result = instance.query('SELECT count() FROM test.view_overload') + result = instance.query("SELECT count() FROM test.view_overload") time.sleep(1) if int(result) == messages_num * threads_num: break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer_overload; DROP TABLE test.view_overload; DROP TABLE test.rabbitmq_consume; DROP TABLE test.rabbitmq_overload; - ''') + """ + ) for thread in threads: thread.join() - assert int(result) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_direct_exchange(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key SETTINGS old_parts_lifetime=5, cleanup_delay_period=2, cleanup_delay_period_random_add=3; - ''') + """ + ) num_tables = 5 for consumer_id in range(num_tables): print(("Setting up table {}".format(consumer_id))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.direct_exchange_{0}; DROP TABLE IF EXISTS test.direct_exchange_{0}_mv; CREATE TABLE test.direct_exchange_{0} (key UInt64, value UInt64) @@ -949,19 +1085,24 @@ def test_rabbitmq_direct_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.direct_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.direct_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) i = [0] messages_num = 1000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 key_num = 0 @@ -971,42 +1112,56 @@ def test_rabbitmq_direct_exchange(rabbitmq_cluster): for message in messages: mes_id = str(randrange(10)) channel.basic_publish( - exchange='direct_exchange_testing', routing_key=key, - properties=pika.BasicProperties(message_id=mes_id), body=message) + exchange="direct_exchange_testing", + routing_key=key, + properties=pika.BasicProperties(message_id=mes_id), + body=message, + ) connection.close() while True: - result = instance.query('SELECT count() FROM test.destination') + result = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result) == messages_num * num_tables: break for consumer_id in range(num_tables): - instance.query(''' + instance.query( + """ DROP TABLE test.direct_exchange_{0}_mv; DROP TABLE test.direct_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; - ''') + """ + ) - assert int(result) == messages_num * num_tables, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * num_tables + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_fanout_exchange(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) num_tables = 5 for consumer_id in range(num_tables): print(("Setting up table {}".format(consumer_id))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.fanout_exchange_{0}; DROP TABLE IF EXISTS test.fanout_exchange_{0}_mv; CREATE TABLE test.fanout_exchange_{0} (key UInt64, value UInt64) @@ -1021,58 +1176,78 @@ def test_rabbitmq_fanout_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.fanout_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.fanout_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) i = [0] messages_num = 1000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 for msg_id in range(messages_num): - channel.basic_publish(exchange='fanout_exchange_testing', routing_key='', - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + channel.basic_publish( + exchange="fanout_exchange_testing", + routing_key="", + properties=pika.BasicProperties(message_id=str(msg_id)), + body=messages[msg_id], + ) connection.close() while True: - result = instance.query('SELECT count() FROM test.destination') + result = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result) == messages_num * num_tables: break for consumer_id in range(num_tables): - instance.query(''' + instance.query( + """ DROP TABLE test.fanout_exchange_{0}_mv; DROP TABLE test.fanout_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE test.destination; - ''') + """ + ) - assert int(result) == messages_num * num_tables, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * num_tables + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_topic_exchange(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) num_tables = 5 for consumer_id in range(num_tables): print(("Setting up table {}".format(consumer_id))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.topic_exchange_{0}; DROP TABLE IF EXISTS test.topic_exchange_{0}_mv; CREATE TABLE test.topic_exchange_{0} (key UInt64, value UInt64) @@ -1087,11 +1262,15 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.topic_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.topic_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) for consumer_id in range(num_tables): print(("Setting up table {}".format(num_tables + consumer_id))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.topic_exchange_{0}; DROP TABLE IF EXISTS test.topic_exchange_{0}_mv; CREATE TABLE test.topic_exchange_{0} (key UInt64, value UInt64) @@ -1106,19 +1285,24 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.topic_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.topic_exchange_{0}; - '''.format(num_tables + consumer_id)) + """.format( + num_tables + consumer_id + ) + ) i = [0] messages_num = 1000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 key_num = 0 @@ -1126,50 +1310,65 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): key = "topic." + str(key_num) key_num += 1 for message in messages: - channel.basic_publish(exchange='topic_exchange_testing', routing_key=key, body=message) + channel.basic_publish( + exchange="topic_exchange_testing", routing_key=key, body=message + ) key = "random.logs" current = 0 for msg_id in range(messages_num): - channel.basic_publish(exchange='topic_exchange_testing', routing_key=key, - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + channel.basic_publish( + exchange="topic_exchange_testing", + routing_key=key, + properties=pika.BasicProperties(message_id=str(msg_id)), + body=messages[msg_id], + ) connection.close() while True: - result = instance.query('SELECT count() FROM test.destination') + result = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result) == messages_num * num_tables + messages_num * num_tables: break for consumer_id in range(num_tables * 2): - instance.query(''' + instance.query( + """ DROP TABLE test.topic_exchange_{0}_mv; DROP TABLE test.topic_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE test.destination; - ''') + """ + ) - assert int( - result) == messages_num * num_tables + messages_num * num_tables, 'ClickHouse lost some messages: {}'.format( - result) + assert ( + int(result) == messages_num * num_tables + messages_num * num_tables + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_hash_exchange(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64, channel_id String) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) num_tables = 4 for consumer_id in range(num_tables): - table_name = 'rabbitmq_consumer{}'.format(consumer_id) + table_name = "rabbitmq_consumer{}".format(consumer_id) print(("Setting up {}".format(table_name))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.{0}; DROP TABLE IF EXISTS test.{0}_mv; CREATE TABLE test.{0} (key UInt64, value UInt64) @@ -1183,13 +1382,18 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.{0}_mv TO test.destination AS SELECT key, value, _channel_id AS channel_id FROM test.{0}; - '''.format(table_name)) + """.format( + table_name + ) + ) i = [0] messages_num = 500 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) def produce(): # init connection here because otherwise python rabbitmq client might fail @@ -1197,11 +1401,15 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): channel = connection.channel() messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 for msg_id in range(messages_num): - channel.basic_publish(exchange='hash_exchange_testing', routing_key=str(msg_id), - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + channel.basic_publish( + exchange="hash_exchange_testing", + routing_key=str(msg_id), + properties=pika.BasicProperties(message_id=str(msg_id)), + body=messages[msg_id], + ) connection.close() threads = [] @@ -1213,9 +1421,9 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() - result1 = '' + result1 = "" while True: - result1 = instance.query('SELECT count() FROM test.destination') + result1 = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result1) == messages_num * threads_num: break @@ -1223,32 +1431,43 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): result2 = instance.query("SELECT count(DISTINCT channel_id) FROM test.destination") for consumer_id in range(num_tables): - table_name = 'rabbitmq_consumer{}'.format(consumer_id) - instance.query(''' + table_name = "rabbitmq_consumer{}".format(consumer_id) + instance.query( + """ DROP TABLE test.{0}_mv; DROP TABLE test.{0}; - '''.format(table_name)) + """.format( + table_name + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE test.destination; - ''') + """ + ) for thread in threads: thread.join() - assert int(result1) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result1) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result) assert int(result2) == 4 * num_tables def test_rabbitmq_multiple_bindings(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.bindings; DROP TABLE IF EXISTS test.bindings_mv; CREATE TABLE test.bindings (key UInt64, value UInt64) @@ -1261,13 +1480,16 @@ def test_rabbitmq_multiple_bindings(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.bindings_mv TO test.destination AS SELECT * FROM test.bindings; - ''') + """ + ) i = [0] messages_num = 500 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) def produce(): # init connection here because otherwise python rabbitmq client might fail @@ -1276,14 +1498,16 @@ def test_rabbitmq_multiple_bindings(rabbitmq_cluster): messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 - keys = ['key1', 'key2', 'key3', 'key4', 'key5'] + keys = ["key1", "key2", "key3", "key4", "key5"] for key in keys: for message in messages: - channel.basic_publish(exchange='multiple_bindings_testing', routing_key=key, body=message) + channel.basic_publish( + exchange="multiple_bindings_testing", routing_key=key, body=message + ) connection.close() @@ -1297,7 +1521,7 @@ def test_rabbitmq_multiple_bindings(rabbitmq_cluster): thread.start() while True: - result = instance.query('SELECT count() FROM test.destination') + result = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result) == messages_num * threads_num * 5: break @@ -1305,27 +1529,34 @@ def test_rabbitmq_multiple_bindings(rabbitmq_cluster): for thread in threads: thread.join() - instance.query(''' + instance.query( + """ DROP TABLE test.bindings; DROP TABLE test.bindings_mv; DROP TABLE test.destination; - ''') + """ + ) - assert int(result) == messages_num * threads_num * 5, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * threads_num * 5 + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_headers_exchange(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) num_tables_to_receive = 2 for consumer_id in range(num_tables_to_receive): print(("Setting up table {}".format(consumer_id))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.headers_exchange_{0}; DROP TABLE IF EXISTS test.headers_exchange_{0}_mv; CREATE TABLE test.headers_exchange_{0} (key UInt64, value UInt64) @@ -1339,12 +1570,16 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.headers_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.headers_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) num_tables_to_ignore = 2 for consumer_id in range(num_tables_to_ignore): print(("Setting up table {}".format(consumer_id + num_tables_to_receive))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.headers_exchange_{0}; DROP TABLE IF EXISTS test.headers_exchange_{0}_mv; CREATE TABLE test.headers_exchange_{0} (key UInt64, value UInt64) @@ -1357,54 +1592,71 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.headers_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.headers_exchange_{0}; - '''.format(consumer_id + num_tables_to_receive)) + """.format( + consumer_id + num_tables_to_receive + ) + ) i = [0] messages_num = 1000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 fields = {} - fields['format'] = 'logs' - fields['type'] = 'report' - fields['year'] = '2020' + fields["format"] = "logs" + fields["type"] = "report" + fields["year"] = "2020" for msg_id in range(messages_num): - channel.basic_publish(exchange='headers_exchange_testing', routing_key='', - properties=pika.BasicProperties(headers=fields, message_id=str(msg_id)), - body=messages[msg_id]) + channel.basic_publish( + exchange="headers_exchange_testing", + routing_key="", + properties=pika.BasicProperties(headers=fields, message_id=str(msg_id)), + body=messages[msg_id], + ) connection.close() while True: - result = instance.query('SELECT count() FROM test.destination') + result = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result) == messages_num * num_tables_to_receive: break for consumer_id in range(num_tables_to_receive + num_tables_to_ignore): - instance.query(''' + instance.query( + """ DROP TABLE test.headers_exchange_{0}_mv; DROP TABLE test.headers_exchange_{0}; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE test.destination; - ''') + """ + ) - assert int(result) == messages_num * num_tables_to_receive, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result) == messages_num * num_tables_to_receive + ), "ClickHouse lost some messages: {}".format(result) def test_rabbitmq_virtual_columns(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_virtuals (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1412,10 +1664,13 @@ def test_rabbitmq_virtual_columns(rabbitmq_cluster): rabbitmq_format = 'JSONEachRow'; CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, key, _exchange_name, _channel_id, _delivery_tag, _redelivered FROM test.rabbitmq_virtuals; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -1423,26 +1678,28 @@ def test_rabbitmq_virtual_columns(rabbitmq_cluster): i = 0 messages = [] for _ in range(message_num): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) i += 1 for message in messages: - channel.basic_publish(exchange='virtuals', routing_key='', body=message) + channel.basic_publish(exchange="virtuals", routing_key="", body=message) while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") time.sleep(1) if int(result) == message_num: break connection.close() - result = instance.query(''' + result = instance.query( + """ SELECT key, value, _exchange_name, SUBSTRING(_channel_id, 1, 3), _delivery_tag, _redelivered FROM test.view ORDER BY key - ''') + """ + ) - expected = '''\ + expected = """\ 0 0 virtuals 1_0 1 0 1 1 virtuals 1_0 2 0 2 2 virtuals 1_0 3 0 @@ -1453,18 +1710,21 @@ def test_rabbitmq_virtual_columns(rabbitmq_cluster): 7 7 virtuals 1_0 8 0 8 8 virtuals 1_0 9 0 9 9 virtuals 1_0 10 0 -''' +""" - instance.query(''' + instance.query( + """ DROP TABLE test.rabbitmq_virtuals; DROP TABLE test.view; - ''') + """ + ) assert TSV(result) == TSV(expected) def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_virtuals_mv (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1476,10 +1736,13 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT *, _exchange_name as exchange_name, _channel_id as channel_id, _delivery_tag as delivery_tag, _redelivered as redelivered FROM test.rabbitmq_virtuals_mv; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -1487,14 +1750,14 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): i = 0 messages = [] for _ in range(message_num): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) i += 1 for message in messages: - channel.basic_publish(exchange='virtuals_mv', routing_key='', body=message) + channel.basic_publish(exchange="virtuals_mv", routing_key="", body=message) while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") time.sleep(1) if int(result) == message_num: break @@ -1502,8 +1765,9 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): connection.close() result = instance.query( - "SELECT key, value, exchange_name, SUBSTRING(channel_id, 1, 3), delivery_tag, redelivered FROM test.view ORDER BY delivery_tag") - expected = '''\ + "SELECT key, value, exchange_name, SUBSTRING(channel_id, 1, 3), delivery_tag, redelivered FROM test.view ORDER BY delivery_tag" + ) + expected = """\ 0 0 virtuals_mv 1_0 1 0 1 1 virtuals_mv 1_0 2 0 2 2 virtuals_mv 1_0 3 0 @@ -1514,29 +1778,34 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): 7 7 virtuals_mv 1_0 8 0 8 8 virtuals_mv 1_0 9 0 9 9 virtuals_mv 1_0 10 0 -''' +""" - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; DROP TABLE test.rabbitmq_virtuals_mv - ''') + """ + ) assert TSV(result) == TSV(expected) def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.destination; CREATE TABLE test.destination(key UInt64, value UInt64, channel_id String) ENGINE = MergeTree() ORDER BY key; - ''') + """ + ) num_tables = 4 for table_id in range(num_tables): print(("Setting up table {}".format(table_id))) - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.many_consumers_{0}; DROP TABLE IF EXISTS test.many_consumers_{0}_mv; CREATE TABLE test.many_consumers_{0} (key UInt64, value UInt64) @@ -1550,13 +1819,18 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.many_consumers_{0}_mv TO test.destination AS SELECT key, value, _channel_id as channel_id FROM test.many_consumers_{0}; - '''.format(table_id)) + """.format( + table_id + ) + ) i = [0] messages_num = 1000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) def produce(): connection = pika.BlockingConnection(parameters) @@ -1564,11 +1838,15 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 for msg_id in range(messages_num): - channel.basic_publish(exchange='many_consumers', routing_key='', - properties=pika.BasicProperties(message_id=str(msg_id)), body=messages[msg_id]) + channel.basic_publish( + exchange="many_consumers", + routing_key="", + properties=pika.BasicProperties(message_id=str(msg_id)), + body=messages[msg_id], + ) connection.close() threads = [] @@ -1580,9 +1858,9 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() - result1 = '' + result1 = "" while True: - result1 = instance.query('SELECT count() FROM test.destination') + result1 = instance.query("SELECT count() FROM test.destination") time.sleep(1) if int(result1) == messages_num * threads_num: break @@ -1593,22 +1871,31 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): thread.join() for consumer_id in range(num_tables): - instance.query(''' + instance.query( + """ DROP TABLE test.many_consumers_{0}; DROP TABLE test.many_consumers_{0}_mv; - '''.format(consumer_id)) + """.format( + consumer_id + ) + ) - instance.query(''' + instance.query( + """ DROP TABLE test.destination; - ''') + """ + ) - assert int(result1) == messages_num * threads_num, 'ClickHouse lost some messages: {}'.format(result) + assert ( + int(result1) == messages_num * threads_num + ), "ClickHouse lost some messages: {}".format(result) # 4 tables, 2 consumers for each table => 8 consumer tags assert int(result2) == 8 def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.consume; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree @@ -1630,10 +1917,13 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): rabbitmq_persistent = '1', rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -1641,19 +1931,21 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): values = [] for i in range(messages_num): values.append("({i}, {i})".format(i=i)) - values = ','.join(values) + values = ",".join(values) while True: try: - instance.query("INSERT INTO test.producer_reconnect VALUES {}".format(values)) + instance.query( + "INSERT INTO test.producer_reconnect VALUES {}".format(values) + ) break except QueryRuntimeException as e: - if 'Local: Timed out.' in str(e): + if "Local: Timed out." in str(e): continue else: raise - while int(instance.query('SELECT count() FROM test.view')) == 0: + while int(instance.query("SELECT count() FROM test.view")) == 0: time.sleep(0.1) kill_rabbitmq(rabbitmq_cluster.rabbitmq_docker_id) @@ -1661,21 +1953,26 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): revive_rabbitmq(rabbitmq_cluster.rabbitmq_docker_id) while True: - result = instance.query('SELECT count(DISTINCT key) FROM test.view') + result = instance.query("SELECT count(DISTINCT key) FROM test.view") time.sleep(1) if int(result) == messages_num: break - instance.query(''' + instance.query( + """ DROP TABLE test.consume; DROP TABLE test.producer_reconnect; - ''') + """ + ) - assert int(result) == messages_num, 'ClickHouse lost some messages: {}'.format(result) + assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( + result + ) def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.consumer_reconnect (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1684,33 +1981,42 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): rabbitmq_num_queues = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; - ''') + """ + ) i = 0 messages_num = 150000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for _ in range(messages_num): - messages.append(json.dumps({'key': i, 'value': i})) + messages.append(json.dumps({"key": i, "value": i})) i += 1 for msg_id in range(messages_num): - channel.basic_publish(exchange='consumer_reconnect', routing_key='', body=messages[msg_id], - properties=pika.BasicProperties(delivery_mode=2, message_id=str(msg_id))) + channel.basic_publish( + exchange="consumer_reconnect", + routing_key="", + body=messages[msg_id], + properties=pika.BasicProperties(delivery_mode=2, message_id=str(msg_id)), + ) connection.close() - instance.query(''' + instance.query( + """ CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.consumer_reconnect; - ''') + """ + ) - while int(instance.query('SELECT count() FROM test.view')) == 0: + while int(instance.query("SELECT count() FROM test.view")) == 0: print(3) time.sleep(0.1) @@ -1726,21 +2032,26 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): # revive_rabbitmq() while True: - result = instance.query('SELECT count(DISTINCT key) FROM test.view') + result = instance.query("SELECT count(DISTINCT key) FROM test.view") time.sleep(1) if int(result) == messages_num: break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.consumer_reconnect; - ''') + """ + ) - assert int(result) == messages_num, 'ClickHouse lost some messages: {}'.format(result) + assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( + result + ) def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1754,10 +2065,13 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() @@ -1769,46 +2083,56 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): while not cancel.is_set(): messages = [] for _ in range(101): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 for message in messages: - channel.basic_publish(exchange='block', routing_key='', body=message) + channel.basic_publish(exchange="block", routing_key="", body=message) rabbitmq_thread = threading.Thread(target=produce) rabbitmq_thread.start() - while int(instance.query('SELECT count() FROM test.view')) == 0: + while int(instance.query("SELECT count() FROM test.view")) == 0: time.sleep(1) cancel.set() - instance.query('DETACH TABLE test.rabbitmq;') + instance.query("DETACH TABLE test.rabbitmq;") - while int(instance.query("SELECT count() FROM system.tables WHERE database='test' AND name='rabbitmq'")) == 1: + while ( + int( + instance.query( + "SELECT count() FROM system.tables WHERE database='test' AND name='rabbitmq'" + ) + ) + == 1 + ): time.sleep(1) - instance.query('ATTACH TABLE test.rabbitmq;') + instance.query("ATTACH TABLE test.rabbitmq;") - while int(instance.query('SELECT uniqExact(key) FROM test.view')) < i[0]: + while int(instance.query("SELECT uniqExact(key) FROM test.view")) < i[0]: time.sleep(1) - result = int(instance.query('SELECT count() == uniqExact(key) FROM test.view')) + result = int(instance.query("SELECT count() == uniqExact(key) FROM test.view")) - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; - ''') + """ + ) rabbitmq_thread.join() connection.close() - assert result == 1, 'Messages from RabbitMQ get duplicated!' + assert result == 1, "Messages from RabbitMQ get duplicated!" def test_rabbitmq_no_connection_at_startup_1(rabbitmq_cluster): # no connection when table is initialized - rabbitmq_cluster.pause_container('rabbitmq1') - instance.query_and_get_error(''' + rabbitmq_cluster.pause_container("rabbitmq1") + instance.query_and_get_error( + """ CREATE TABLE test.cs (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1816,12 +2140,14 @@ def test_rabbitmq_no_connection_at_startup_1(rabbitmq_cluster): rabbitmq_format = 'JSONEachRow', rabbitmq_num_consumers = '5', rabbitmq_row_delimiter = '\\n'; - ''') - rabbitmq_cluster.unpause_container('rabbitmq1') + """ + ) + rabbitmq_cluster.unpause_container("rabbitmq1") def test_rabbitmq_no_connection_at_startup_2(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.cs (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1834,39 +2160,51 @@ def test_rabbitmq_no_connection_at_startup_2(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.cs; - ''') + """ + ) instance.query("DETACH TABLE test.cs") - rabbitmq_cluster.pause_container('rabbitmq1') + rabbitmq_cluster.pause_container("rabbitmq1") instance.query("ATTACH TABLE test.cs") - rabbitmq_cluster.unpause_container('rabbitmq1') + rabbitmq_cluster.unpause_container("rabbitmq1") messages_num = 1000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() for i in range(messages_num): - message = json.dumps({'key': i, 'value': i}) - channel.basic_publish(exchange='cs', routing_key='', body=message, - properties=pika.BasicProperties(delivery_mode=2, message_id=str(i))) + message = json.dumps({"key": i, "value": i}) + channel.basic_publish( + exchange="cs", + routing_key="", + body=message, + properties=pika.BasicProperties(delivery_mode=2, message_id=str(i)), + ) connection.close() while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") time.sleep(1) if int(result) == messages_num: break - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.cs; - ''') + """ + ) - assert int(result) == messages_num, 'ClickHouse lost some messages: {}'.format(result) + assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( + result + ) def test_rabbitmq_format_factory_settings(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.format_settings ( id String, date DateTime ) ENGINE = RabbitMQ @@ -1874,106 +2212,136 @@ def test_rabbitmq_format_factory_settings(rabbitmq_cluster): rabbitmq_exchange_name = 'format_settings', rabbitmq_format = 'JSONEachRow', date_time_input_format = 'best_effort'; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - message = json.dumps({"id":"format_settings_test","date":"2021-01-19T14:42:33.1829214Z"}) - expected = instance.query('''SELECT parseDateTimeBestEffort(CAST('2021-01-19T14:42:33.1829214Z', 'String'))''') + message = json.dumps( + {"id": "format_settings_test", "date": "2021-01-19T14:42:33.1829214Z"} + ) + expected = instance.query( + """SELECT parseDateTimeBestEffort(CAST('2021-01-19T14:42:33.1829214Z', 'String'))""" + ) - channel.basic_publish(exchange='format_settings', routing_key='', body=message) - result = '' + channel.basic_publish(exchange="format_settings", routing_key="", body=message) + result = "" while True: - result = instance.query('SELECT date FROM test.format_settings') + result = instance.query("SELECT date FROM test.format_settings") if result == expected: - break; + break - instance.query(''' + instance.query( + """ CREATE TABLE test.view ( id String, date DateTime ) ENGINE = MergeTree ORDER BY id; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.format_settings; - ''') + """ + ) - channel.basic_publish(exchange='format_settings', routing_key='', body=message) - result = '' + channel.basic_publish(exchange="format_settings", routing_key="", body=message) + result = "" while True: - result = instance.query('SELECT date FROM test.view') + result = instance.query("SELECT date FROM test.view") if result == expected: - break; + break connection.close() - instance.query(''' + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.format_settings; - ''') + """ + ) - assert(result == expected) + assert result == expected def test_rabbitmq_vhost(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_vhost (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'vhost', rabbitmq_format = 'JSONEachRow', rabbitmq_vhost = '/' - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - channel.basic_publish(exchange='vhost', routing_key='', body=json.dumps({'key': 1, 'value': 2})) + channel.basic_publish( + exchange="vhost", routing_key="", body=json.dumps({"key": 1, "value": 2}) + ) connection.close() while True: - result = instance.query('SELECT * FROM test.rabbitmq_vhost ORDER BY key', ignore_error=True) + result = instance.query( + "SELECT * FROM test.rabbitmq_vhost ORDER BY key", ignore_error=True + ) if result == "1\t2\n": break def test_rabbitmq_drop_table_properly(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_drop (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'drop', rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'rabbit_queue_drop' - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - channel.basic_publish(exchange='drop', routing_key='', body=json.dumps({'key': 1, 'value': 2})) + channel.basic_publish( + exchange="drop", routing_key="", body=json.dumps({"key": 1, "value": 2}) + ) while True: - result = instance.query('SELECT * FROM test.rabbitmq_drop ORDER BY key', ignore_error=True) + result = instance.query( + "SELECT * FROM test.rabbitmq_drop ORDER BY key", ignore_error=True + ) if result == "1\t2\n": break - exists = channel.queue_declare(queue='rabbit_queue_drop', passive=True) - assert(exists) + exists = channel.queue_declare(queue="rabbit_queue_drop", passive=True) + assert exists instance.query("DROP TABLE test.rabbitmq_drop") time.sleep(30) try: - exists = channel.queue_declare(callback, queue='rabbit_queue_drop', passive=True) + exists = channel.queue_declare( + callback, queue="rabbit_queue_drop", passive=True + ) except Exception as e: exists = False - assert(not exists) + assert not exists def test_rabbitmq_queue_settings(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_settings (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -1981,53 +2349,67 @@ def test_rabbitmq_queue_settings(rabbitmq_cluster): rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'rabbit_queue_settings', rabbitmq_queue_settings_list = 'x-max-length=10,x-overflow=reject-publish' - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() for i in range(50): - channel.basic_publish(exchange='rabbit_exchange', routing_key='', body=json.dumps({'key': 1, 'value': 2})) + channel.basic_publish( + exchange="rabbit_exchange", + routing_key="", + body=json.dumps({"key": 1, "value": 2}), + ) connection.close() - instance.query(''' + instance.query( + """ CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq_settings; - ''') + """ + ) time.sleep(5) - result = instance.query('SELECT count() FROM test.rabbitmq_settings', ignore_error=True) + result = instance.query( + "SELECT count() FROM test.rabbitmq_settings", ignore_error=True + ) while int(result) != 10: time.sleep(0.5) - result = instance.query('SELECT count() FROM test.view', ignore_error=True) + result = instance.query("SELECT count() FROM test.view", ignore_error=True) - instance.query('DROP TABLE test.rabbitmq_settings') + instance.query("DROP TABLE test.rabbitmq_settings") # queue size is 10, but 50 messages were sent, they will be dropped (setting x-overflow = reject-publish) and only 10 will remain. - assert(int(result) == 10) + assert int(result) == 10 def test_rabbitmq_queue_consume(rabbitmq_cluster): - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - channel.queue_declare(queue='rabbit_queue', durable=True) + channel.queue_declare(queue="rabbit_queue", durable=True) i = [0] messages_num = 1000 + def produce(): connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for _ in range(messages_num): - message = json.dumps({'key': i[0], 'value': i[0]}) - channel.basic_publish(exchange='', routing_key='rabbit_queue', body=message) + message = json.dumps({"key": i[0], "value": i[0]}) + channel.basic_publish(exchange="", routing_key="rabbit_queue", body=message) i[0] += 1 threads = [] @@ -2038,7 +2420,8 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_queue (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -2049,11 +2432,12 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): ENGINE = MergeTree ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq_queue; - ''') + """ + ) - result = '' + result = "" while True: - result = instance.query('SELECT count() FROM test.view') + result = instance.query("SELECT count() FROM test.view") if int(result) == messages_num * threads_num: break time.sleep(1) @@ -2061,13 +2445,14 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): for thread in threads: thread.join() - instance.query('DROP TABLE test.rabbitmq_queue') + instance.query("DROP TABLE test.rabbitmq_queue") def test_rabbitmq_produce_consume_avro(rabbitmq_cluster): num_rows = 75 - instance.query(''' + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.rabbit; DROP TABLE IF EXISTS test.rabbit_writer; @@ -2090,38 +2475,51 @@ def test_rabbitmq_produce_consume_avro(rabbitmq_cluster): CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT key, value FROM test.rabbit; - ''') - - instance.query("INSERT INTO test.rabbit_writer select number*10 as key, number*100 as value from numbers({num_rows}) SETTINGS output_format_avro_rows_in_file = 7".format(num_rows=num_rows)) + """ + ) + instance.query( + "INSERT INTO test.rabbit_writer select number*10 as key, number*100 as value from numbers({num_rows}) SETTINGS output_format_avro_rows_in_file = 7".format( + num_rows=num_rows + ) + ) # Ideally we should wait for an event time.sleep(3) - expected_num_rows = instance.query("SELECT COUNT(1) FROM test.view", ignore_error=True) - assert (int(expected_num_rows) == num_rows) + expected_num_rows = instance.query( + "SELECT COUNT(1) FROM test.view", ignore_error=True + ) + assert int(expected_num_rows) == num_rows - expected_max_key = instance.query("SELECT max(key) FROM test.view", ignore_error=True) - assert (int(expected_max_key) == (num_rows - 1) * 10) + expected_max_key = instance.query( + "SELECT max(key) FROM test.view", ignore_error=True + ) + assert int(expected_max_key) == (num_rows - 1) * 10 def test_rabbitmq_bad_args(rabbitmq_cluster): - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - channel.exchange_declare(exchange='f', exchange_type='fanout') - instance.query_and_get_error(''' + channel.exchange_declare(exchange="f", exchange_type="fanout") + instance.query_and_get_error( + """ CREATE TABLE test.drop (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'f', rabbitmq_format = 'JSONEachRow'; - ''') + """ + ) def test_rabbitmq_issue_30691(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq_drop (json String) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -2129,30 +2527,57 @@ def test_rabbitmq_issue_30691(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n', -- Works only if adding this setting rabbitmq_format = 'LineAsString', rabbitmq_queue_base = '30691'; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - channel.basic_publish(exchange='30691', routing_key='', body=json.dumps({"event_type": "purge", "as_src": 1234, "as_dst": 0, "as_path": "", - "local_pref": 100, "med": 0, "peer_as_dst": 0, - "ip_src": "", "ip_dst": "", - "port_src": 443, "port_dst": 41930, "ip_proto": "tcp", - "tos": 0, "stamp_inserted": "2021-10-26 15:20:00", - "stamp_updated": "2021-10-26 15:23:14", "packets": 2, "bytes": 1216, "writer_id": "default_amqp/449206"})) - result = '' + channel.basic_publish( + exchange="30691", + routing_key="", + body=json.dumps( + { + "event_type": "purge", + "as_src": 1234, + "as_dst": 0, + "as_path": "", + "local_pref": 100, + "med": 0, + "peer_as_dst": 0, + "ip_src": "", + "ip_dst": "", + "port_src": 443, + "port_dst": 41930, + "ip_proto": "tcp", + "tos": 0, + "stamp_inserted": "2021-10-26 15:20:00", + "stamp_updated": "2021-10-26 15:23:14", + "packets": 2, + "bytes": 1216, + "writer_id": "default_amqp/449206", + } + ), + ) + result = "" while True: - result = instance.query('SELECT * FROM test.rabbitmq_drop', ignore_error=True) + result = instance.query("SELECT * FROM test.rabbitmq_drop", ignore_error=True) print(result) if result != "": break - assert(result.strip() =="""{"event_type": "purge", "as_src": 1234, "as_dst": 0, "as_path": "", "local_pref": 100, "med": 0, "peer_as_dst": 0, "ip_src": "", "ip_dst": "", "port_src": 443, "port_dst": 41930, "ip_proto": "tcp", "tos": 0, "stamp_inserted": "2021-10-26 15:20:00", "stamp_updated": "2021-10-26 15:23:14", "packets": 2, "bytes": 1216, "writer_id": "default_amqp/449206"}""") + assert ( + result.strip() + == """{"event_type": "purge", "as_src": 1234, "as_dst": 0, "as_path": "", "local_pref": 100, "med": 0, "peer_as_dst": 0, "ip_src": "", "ip_dst": "", "port_src": 443, "port_dst": 41930, "ip_proto": "tcp", "tos": 0, "stamp_inserted": "2021-10-26 15:20:00", "stamp_updated": "2021-10-26 15:23:14", "packets": 2, "bytes": 1216, "writer_id": "default_amqp/449206"}""" + ) def test_rabbitmq_drop_mv(rabbitmq_cluster): - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -2164,53 +2589,67 @@ def test_rabbitmq_drop_mv(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; - ''') + """ + ) - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() messages = [] for i in range(20): - channel.basic_publish(exchange='mv', routing_key='', body=json.dumps({'key': i, 'value': i})) + channel.basic_publish( + exchange="mv", routing_key="", body=json.dumps({"key": i, "value": i}) + ) - instance.query('DROP VIEW test.consumer') + instance.query("DROP VIEW test.consumer") for i in range(20, 40): - channel.basic_publish(exchange='mv', routing_key='', body=json.dumps({'key': i, 'value': i})) + channel.basic_publish( + exchange="mv", routing_key="", body=json.dumps({"key": i, "value": i}) + ) - instance.query(''' + instance.query( + """ CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; - ''') + """ + ) for i in range(40, 50): - channel.basic_publish(exchange='mv', routing_key='', body=json.dumps({'key': i, 'value': i})) + channel.basic_publish( + exchange="mv", routing_key="", body=json.dumps({"key": i, "value": i}) + ) while True: - result = instance.query('SELECT * FROM test.view ORDER BY key') - if (rabbitmq_check_result(result)): + result = instance.query("SELECT * FROM test.view ORDER BY key") + if rabbitmq_check_result(result): break rabbitmq_check_result(result, True) - instance.query('DROP VIEW test.consumer') + instance.query("DROP VIEW test.consumer") for i in range(50, 60): - channel.basic_publish(exchange='mv', routing_key='', body=json.dumps({'key': i, 'value': i})) + channel.basic_publish( + exchange="mv", routing_key="", body=json.dumps({"key": i, "value": i}) + ) connection.close() count = 0 while True: - count = int(instance.query('SELECT count() FROM test.rabbitmq')) - if (count): + count = int(instance.query("SELECT count() FROM test.rabbitmq")) + if count: break - assert(count > 0) + assert count > 0 def test_rabbitmq_random_detach(rabbitmq_cluster): NUM_CONSUMERS = 2 NUM_QUEUES = 2 - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', @@ -2224,13 +2663,16 @@ def test_rabbitmq_random_detach(rabbitmq_cluster): ORDER BY key; CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT *, _channel_id AS channel_id FROM test.rabbitmq; - ''') + """ + ) i = [0] messages_num = 10000 - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) def produce(): connection = pika.BlockingConnection(parameters) @@ -2238,10 +2680,15 @@ def test_rabbitmq_random_detach(rabbitmq_cluster): messages = [] for i in range(messages_num): - messages.append(json.dumps({'key': i[0], 'value': i[0]})) + messages.append(json.dumps({"key": i[0], "value": i[0]})) i[0] += 1 mes_id = str(i) - channel.basic_publish(exchange='test_sharding', routing_key='', properties=pika.BasicProperties(message_id=mes_id), body=message) + channel.basic_publish( + exchange="test_sharding", + routing_key="", + properties=pika.BasicProperties(message_id=mes_id), + body=message, + ) connection.close() threads = [] @@ -2253,33 +2700,41 @@ def test_rabbitmq_random_detach(rabbitmq_cluster): time.sleep(random.uniform(0, 1)) thread.start() - #time.sleep(5) - #kill_rabbitmq(rabbitmq_cluster.rabbitmq_docker_id) - #instance.query("detach table test.rabbitmq") - #revive_rabbitmq(rabbitmq_cluster.rabbitmq_docker_id) + # time.sleep(5) + # kill_rabbitmq(rabbitmq_cluster.rabbitmq_docker_id) + # instance.query("detach table test.rabbitmq") + # revive_rabbitmq(rabbitmq_cluster.rabbitmq_docker_id) for thread in threads: thread.join() def test_rabbitmq_predefined_configuration(rabbitmq_cluster): - credentials = pika.PlainCredentials('root', 'clickhouse') - parameters = pika.ConnectionParameters(rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, '/', credentials) + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, rabbitmq_cluster.rabbitmq_port, "/", credentials + ) connection = pika.BlockingConnection(parameters) channel = connection.channel() - instance.query(''' + instance.query( + """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) - ENGINE = RabbitMQ(rabbit1, rabbitmq_vhost = '/') ''') + ENGINE = RabbitMQ(rabbit1, rabbitmq_vhost = '/') """ + ) - channel.basic_publish(exchange='named', routing_key='', body=json.dumps({'key': 1, 'value': 2})) + channel.basic_publish( + exchange="named", routing_key="", body=json.dumps({"key": 1, "value": 2}) + ) while True: - result = instance.query('SELECT * FROM test.rabbitmq ORDER BY key', ignore_error=True) + result = instance.query( + "SELECT * FROM test.rabbitmq ORDER BY key", ignore_error=True + ) if result == "1\t2\n": break -if __name__ == '__main__': +if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") cluster.shutdown() diff --git a/tests/integration/test_storage_s3/s3_mocks/echo.py b/tests/integration/test_storage_s3/s3_mocks/echo.py index ced84e54d62..5103d7ebc15 100644 --- a/tests/integration/test_storage_s3/s3_mocks/echo.py +++ b/tests/integration/test_storage_s3/s3_mocks/echo.py @@ -19,11 +19,10 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): self.send_header("Content-Type", "text/plain") self.end_headers() - def do_GET(self): self.do_HEAD() if self.path.startswith("/get-my-path/"): - self.wfile.write(b'/' + self.path.split('/', maxsplit=2)[2].encode()) + self.wfile.write(b"/" + self.path.split("/", maxsplit=2)[2].encode()) elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/s3_mocks/mock_s3.py b/tests/integration/test_storage_s3/s3_mocks/mock_s3.py index 9009d345f49..870353ebaa8 100644 --- a/tests/integration/test_storage_s3/s3_mocks/mock_s3.py +++ b/tests/integration/test_storage_s3/s3_mocks/mock_s3.py @@ -3,26 +3,26 @@ import sys from bottle import abort, route, run, request, response -@route('/redirected/<_path:path>') +@route("/redirected/<_path:path>") def infinite_redirect(_path): response.set_header("Location", request.url) response.status = 307 - return 'Redirected' + return "Redirected" -@route('/<_bucket>/<_path:path>') +@route("/<_bucket>/<_path:path>") def server(_bucket, _path): for name in request.headers: - if name == 'Authorization' and request.headers[name] == 'Bearer TOKEN': - return '1, 2, 3' + if name == "Authorization" and request.headers[name] == "Bearer TOKEN": + return "1, 2, 3" response.status = 403 - response.content_type = 'text/xml' + response.content_type = "text/xml" return 'ForbiddenErrorForbidden Errortxfbd566d03042474888193-00608d7537' -@route('/') +@route("/") def ping(): - return 'OK' + return "OK" -run(host='0.0.0.0', port=int(sys.argv[1])) +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index ca2d6103cf6..103dd30340c 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -8,7 +8,7 @@ import sys def gen_n_digit_number(n): assert 0 < n < 19 - return random.randint(10**(n-1), 10**n-1) + return random.randint(10 ** (n - 1), 10**n - 1) sum_in_4_column = 0 @@ -19,6 +19,7 @@ def gen_line(): columns = 4 row = [] + def add_number(): digits = random.randint(1, 18) row.append(gen_n_digit_number(digits)) @@ -37,7 +38,10 @@ def gen_line(): random.seed("Unstable server/1.0") # Generating some "random" data and append a line which contains sum of numbers in column 4. -lines = b"".join((gen_line() for _ in range(500000))) + f"0,0,0,{-sum_in_4_column}\n".encode() +lines = ( + b"".join((gen_line() for _ in range(500000))) + + f"0,0,0,{-sum_in_4_column}\n".encode() +) class RequestHandler(http.server.BaseHTTPRequestHandler): @@ -47,7 +51,9 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): self.end_bytes = len(lines) self.size = self.end_bytes self.send_block_size = 256 - self.stop_at = random.randint(900000, 1300000) // self.send_block_size # Block size is 1024**2. + self.stop_at = ( + random.randint(900000, 1300000) // self.send_block_size + ) # Block size is 1024**2. if "Range" in self.headers: cr = self.headers["Range"] @@ -55,9 +61,12 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): assert parts[0] == "bytes" self.from_bytes = int(parts[1]) if parts[2]: - self.end_bytes = int(parts[2])+1 + self.end_bytes = int(parts[2]) + 1 self.send_response(206) - self.send_header("Content-Range", f"bytes {self.from_bytes}-{self.end_bytes-1}/{self.size}") + self.send_header( + "Content-Range", + f"bytes {self.from_bytes}-{self.end_bytes-1}/{self.size}", + ) else: self.send_response(200) @@ -76,17 +85,20 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): self.send_header("Content-Type", "text/plain") self.end_headers() - def do_GET(self): self.do_HEAD() if self.path == "/root/test.csv": - for c, i in enumerate(range(self.from_bytes, self.end_bytes, self.send_block_size)): - self.wfile.write(lines[i:min(i+self.send_block_size, self.end_bytes)]) + for c, i in enumerate( + range(self.from_bytes, self.end_bytes, self.send_block_size) + ): + self.wfile.write( + lines[i : min(i + self.send_block_size, self.end_bytes)] + ) if (c + 1) % self.stop_at == 0: - #self.wfile._sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 0, 0)) - #self.wfile._sock.shutdown(socket.SHUT_RDWR) - #self.wfile._sock.close() - print('Dropping connection') + # self.wfile._sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 0, 0)) + # self.wfile._sock.shutdown(socket.SHUT_RDWR) + # self.wfile._sock.close() + print("Dropping connection") break elif self.path == "/": diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 4366a1f034e..dd29d0a5d6a 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -17,48 +17,56 @@ MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/dummy/configs/config.d/defaultS3.xml'.format(get_instances_dir())) +CONFIG_PATH = os.path.join( + SCRIPT_DIR, "./{}/dummy/configs/config.d/defaultS3.xml".format(get_instances_dir()) +) # Creates S3 bucket for tests and allows anonymous read-write access to it. def prepare_s3_bucket(started_cluster): # Allows read-write access for bucket without authorization. - bucket_read_write_policy = {"Version": "2012-10-17", - "Statement": [ - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:GetBucketLocation", - "Resource": "arn:aws:s3:::root" - }, - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::root" - }, - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::root/*" - }, - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:PutObject", - "Resource": "arn:aws:s3:::root/*" - } - ]} + bucket_read_write_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:GetBucketLocation", + "Resource": "arn:aws:s3:::root", + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::root", + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::root/*", + }, + { + "Sid": "", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::root/*", + }, + ], + } minio_client = started_cluster.minio_client - minio_client.set_bucket_policy(started_cluster.minio_bucket, json.dumps(bucket_read_write_policy)) + minio_client.set_bucket_policy( + started_cluster.minio_bucket, json.dumps(bucket_read_write_policy) + ) - started_cluster.minio_restricted_bucket = "{}-with-auth".format(started_cluster.minio_bucket) + started_cluster.minio_restricted_bucket = "{}-with-auth".format( + started_cluster.minio_bucket + ) if minio_client.bucket_exists(started_cluster.minio_restricted_bucket): minio_client.remove_bucket(started_cluster.minio_restricted_bucket) @@ -87,11 +95,22 @@ def get_s3_file_content(started_cluster, bucket, filename, decode=True): def started_cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("restricted_dummy", main_configs=["configs/config_for_test_remote_host_filter.xml"], - with_minio=True) - cluster.add_instance("dummy", with_minio=True, main_configs=["configs/defaultS3.xml", "configs/named_collections.xml"]) - cluster.add_instance("s3_max_redirects", with_minio=True, main_configs=["configs/defaultS3.xml"], - user_configs=["configs/s3_max_redirects.xml"]) + cluster.add_instance( + "restricted_dummy", + main_configs=["configs/config_for_test_remote_host_filter.xml"], + with_minio=True, + ) + cluster.add_instance( + "dummy", + with_minio=True, + main_configs=["configs/defaultS3.xml", "configs/named_collections.xml"], + ) + cluster.add_instance( + "s3_max_redirects", + with_minio=True, + main_configs=["configs/defaultS3.xml"], + user_configs=["configs/s3_max_redirects.xml"], + ) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -116,20 +135,27 @@ def run_query(instance, query, stdin=None, settings=None): # Test simple put. Also checks that wrong credentials produce an error with every compression method. -@pytest.mark.parametrize("maybe_auth,positive,compression", [ - pytest.param("", True, 'auto', id="positive"), - pytest.param("'minio','minio123',", True, 'auto', id="auth_positive"), - pytest.param("'wrongid','wrongkey',", False, 'auto', id="auto"), - pytest.param("'wrongid','wrongkey',", False, 'gzip', id="gzip"), - pytest.param("'wrongid','wrongkey',", False, 'deflate', id="deflate"), - pytest.param("'wrongid','wrongkey',", False, 'brotli', id="brotli"), - pytest.param("'wrongid','wrongkey',", False, 'xz', id="xz"), - pytest.param("'wrongid','wrongkey',", False, 'zstd', id="zstd") -]) +@pytest.mark.parametrize( + "maybe_auth,positive,compression", + [ + pytest.param("", True, "auto", id="positive"), + pytest.param("'minio','minio123',", True, "auto", id="auth_positive"), + pytest.param("'wrongid','wrongkey',", False, "auto", id="auto"), + pytest.param("'wrongid','wrongkey',", False, "gzip", id="gzip"), + pytest.param("'wrongid','wrongkey',", False, "deflate", id="deflate"), + pytest.param("'wrongid','wrongkey',", False, "brotli", id="brotli"), + pytest.param("'wrongid','wrongkey',", False, "xz", id="xz"), + pytest.param("'wrongid','wrongkey',", False, "zstd", id="zstd"), + ], +) def test_put(started_cluster, maybe_auth, positive, compression): # type: (ClickHouseCluster) -> None - bucket = started_cluster.minio_bucket if not maybe_auth else started_cluster.minio_restricted_bucket + bucket = ( + started_cluster.minio_bucket + if not maybe_auth + else started_cluster.minio_restricted_bucket + ) instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_format = "column1 UInt32, column2 UInt32, column3 UInt32" values = "(1, 2, 3), (3, 2, 1), (78, 43, 45)" @@ -166,7 +192,9 @@ def test_partition_by(started_cluster): assert "78,43,45\n" == get_s3_file_content(started_cluster, bucket, "test_45.csv") filename = "test2_{_partition_id}.csv" - instance.query(f"create table p ({table_format}) engine=S3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'CSV') partition by column3") + instance.query( + f"create table p ({table_format}) engine=S3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'CSV') partition by column3" + ) instance.query(f"insert into p values {values}") assert "1,2,3\n" == get_s3_file_content(started_cluster, bucket, "test2_3.csv") assert "3,2,1\n" == get_s3_file_content(started_cluster, bucket, "test2_1.csv") @@ -186,7 +214,9 @@ def test_partition_by_string_column(started_cluster): run_query(instance, put_query) - assert '1,"foo/bar"\n' == get_s3_file_content(started_cluster, bucket, "test_foo/bar.csv") + assert '1,"foo/bar"\n' == get_s3_file_content( + started_cluster, bucket, "test_foo/bar.csv" + ) assert '3,"йцук"\n' == get_s3_file_content(started_cluster, bucket, "test_йцук.csv") assert '78,"你好"\n' == get_s3_file_content(started_cluster, bucket, "test_你好.csv") @@ -208,10 +238,7 @@ def test_partition_by_const_column(started_cluster): assert values_csv == get_s3_file_content(started_cluster, bucket, "test_88.csv") -@pytest.mark.parametrize("special", [ - "space", - "plus" -]) +@pytest.mark.parametrize("special", ["space", "plus"]) def test_get_file_with_special(started_cluster, special): symbol = {"space": " ", "plus": "+"}[special] urlsafe_symbol = {"space": "%20", "plus": "%2B"}[special] @@ -219,26 +246,41 @@ def test_get_file_with_special(started_cluster, special): bucket = started_cluster.minio_restricted_bucket instance = started_cluster.instances["dummy"] table_format = "column1 UInt32, column2 UInt32, column3 UInt32" - values = [[12549, 2463, 19893], [64021, 38652, 66703], [81611, 39650, 83516], [11079, 59507, 61546], [51764, 69952, 6876], [41165, 90293, 29095], [40167, 78432, 48309], [81629, 81327, 11855], [55852, 21643, 98507], [6738, 54643, 41155]] - values_csv = ('\n'.join((','.join(map(str, row)) for row in values)) + '\n').encode() + values = [ + [12549, 2463, 19893], + [64021, 38652, 66703], + [81611, 39650, 83516], + [11079, 59507, 61546], + [51764, 69952, 6876], + [41165, 90293, 29095], + [40167, 78432, 48309], + [81629, 81327, 11855], + [55852, 21643, 98507], + [6738, 54643, 41155], + ] + values_csv = ( + "\n".join((",".join(map(str, row)) for row in values)) + "\n" + ).encode() filename = f"get_file_with_{special}_{symbol}two.csv" put_s3_file_content(started_cluster, bucket, filename, values_csv) get_query = f"SELECT * FROM s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/get_file_with_{special}_{urlsafe_symbol}two.csv', {auth}'CSV', '{table_format}') FORMAT TSV" - assert [list(map(int, l.split())) for l in run_query(instance, get_query).splitlines()] == values + assert [ + list(map(int, l.split())) for l in run_query(instance, get_query).splitlines() + ] == values get_query = f"SELECT * FROM s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/get_file_with_{special}*.csv', {auth}'CSV', '{table_format}') FORMAT TSV" - assert [list(map(int, l.split())) for l in run_query(instance, get_query).splitlines()] == values + assert [ + list(map(int, l.split())) for l in run_query(instance, get_query).splitlines() + ] == values get_query = f"SELECT * FROM s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/get_file_with_{special}_{urlsafe_symbol}*.csv', {auth}'CSV', '{table_format}') FORMAT TSV" - assert [list(map(int, l.split())) for l in run_query(instance, get_query).splitlines()] == values + assert [ + list(map(int, l.split())) for l in run_query(instance, get_query).splitlines() + ] == values -@pytest.mark.parametrize("special", [ - "space", - "plus", - "plus2" -]) +@pytest.mark.parametrize("special", ["space", "plus", "plus2"]) def test_get_path_with_special(started_cluster, special): symbol = {"space": "%20", "plus": "%2B", "plus2": "%2B"}[special] safe_symbol = {"space": "%20", "plus": "+", "plus2": "%2B"}[special] @@ -250,9 +292,7 @@ def test_get_path_with_special(started_cluster, special): # Test put no data to S3. -@pytest.mark.parametrize("auth", [ - pytest.param("'minio','minio123',", id="minio") -]) +@pytest.mark.parametrize("auth", [pytest.param("'minio','minio123',", id="minio")]) def test_empty_put(started_cluster, auth): # type: (ClickHouseCluster, str) -> None @@ -265,20 +305,37 @@ def test_empty_put(started_cluster, auth): CREATE TABLE empty_table ( {} ) ENGINE = Null() - """.format(table_format) + """.format( + table_format + ) run_query(instance, drop_empty_table_query) run_query(instance, create_empty_table_query) filename = "empty_put_test.csv" put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') select * from empty_table".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, auth, table_format) + started_cluster.minio_ip, + MINIO_INTERNAL_PORT, + bucket, + filename, + auth, + table_format, + ) run_query(instance, put_query) try: - run_query(instance, "select count(*) from s3('http://{}:{}/{}/{}', {}'CSV', '{}')".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, auth, table_format)) + run_query( + instance, + "select count(*) from s3('http://{}:{}/{}/{}', {}'CSV', '{}')".format( + started_cluster.minio_ip, + MINIO_INTERNAL_PORT, + bucket, + filename, + auth, + table_format, + ), + ) assert False, "Query should be failed." except helpers.client.QueryRuntimeException as e: @@ -286,20 +343,33 @@ def test_empty_put(started_cluster, auth): # Test put values in CSV format. -@pytest.mark.parametrize("maybe_auth,positive", [ - pytest.param("", True, id="positive"), - pytest.param("'minio','minio123',", True, id="auth_positive"), - pytest.param("'wrongid','wrongkey',", False, id="negative"), -]) +@pytest.mark.parametrize( + "maybe_auth,positive", + [ + pytest.param("", True, id="positive"), + pytest.param("'minio','minio123',", True, id="auth_positive"), + pytest.param("'wrongid','wrongkey',", False, id="negative"), + ], +) def test_put_csv(started_cluster, maybe_auth, positive): # type: (ClickHouseCluster, bool, str) -> None - bucket = started_cluster.minio_bucket if not maybe_auth else started_cluster.minio_restricted_bucket + bucket = ( + started_cluster.minio_bucket + if not maybe_auth + else started_cluster.minio_restricted_bucket + ) instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_format = "column1 UInt32, column2 UInt32, column3 UInt32" filename = "test.csv" put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV settings s3_truncate_on_insert=1".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, maybe_auth, table_format) + started_cluster.minio_ip, + MINIO_INTERNAL_PORT, + bucket, + filename, + maybe_auth, + table_format, + ) csv_data = "8,9,16\n11,18,13\n22,14,2\n" try: @@ -323,13 +393,24 @@ def test_put_get_with_redirect(started_cluster): values_csv = "1,1,1\n1,1,1\n11,11,11\n" filename = "test.csv" query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values settings s3_truncate_on_insert=1 {}".format( - started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format, values) + started_cluster.minio_redirect_host, + started_cluster.minio_redirect_port, + bucket, + filename, + table_format, + values, + ) run_query(instance, query) assert values_csv == get_s3_file_content(started_cluster, bucket, filename) query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/{}', 'CSV', '{}')".format( - started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format) + started_cluster.minio_redirect_host, + started_cluster.minio_redirect_port, + bucket, + filename, + table_format, + ) stdout = run_query(instance, query) assert list(map(str.split, stdout.splitlines())) == [ @@ -351,12 +432,24 @@ def test_put_with_zero_redirect(started_cluster): # Should work without redirect query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values settings s3_truncate_on_insert=1 {}".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, table_format, values) + started_cluster.minio_ip, + MINIO_INTERNAL_PORT, + bucket, + filename, + table_format, + values, + ) run_query(instance, query) # Should not work with redirect query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values settings s3_truncate_on_insert=1 {}".format( - started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format, values) + started_cluster.minio_redirect_host, + started_cluster.minio_redirect_port, + bucket, + filename, + table_format, + values, + ) exception_raised = False try: run_query(instance, query) @@ -369,40 +462,69 @@ def test_put_with_zero_redirect(started_cluster): def test_put_get_with_globs(started_cluster): # type: (ClickHouseCluster) -> None - unique_prefix = random.randint(1,10000) + unique_prefix = random.randint(1, 10000) bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_format = "column1 UInt32, column2 UInt32, column3 UInt32" max_path = "" for i in range(10): for j in range(10): - path = "{}/{}_{}/{}.csv".format(unique_prefix, i, random.choice(['a', 'b', 'c', 'd']), j) + path = "{}/{}_{}/{}.csv".format( + unique_prefix, i, random.choice(["a", "b", "c", "d"]), j + ) max_path = max(path, max_path) values = "({},{},{})".format(i, j, i + j) query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, path, table_format, values) + started_cluster.minio_ip, + MINIO_INTERNAL_PORT, + bucket, + path, + table_format, + values, + ) run_query(instance, query) query = "select sum(column1), sum(column2), sum(column3), min(_file), max(_path) from s3('http://{}:{}/{}/{}/*_{{a,b,c,d}}/%3f.csv', 'CSV', '{}')".format( - started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, unique_prefix, table_format) + started_cluster.minio_redirect_host, + started_cluster.minio_redirect_port, + bucket, + unique_prefix, + table_format, + ) assert run_query(instance, query).splitlines() == [ - "450\t450\t900\t0.csv\t{bucket}/{max_path}".format(bucket=bucket, max_path=max_path)] + "450\t450\t900\t0.csv\t{bucket}/{max_path}".format( + bucket=bucket, max_path=max_path + ) + ] minio = started_cluster.minio_client - for obj in list(minio.list_objects(started_cluster.minio_bucket, prefix='{}/'.format(unique_prefix), recursive=True)): + for obj in list( + minio.list_objects( + started_cluster.minio_bucket, + prefix="{}/".format(unique_prefix), + recursive=True, + ) + ): minio.remove_object(started_cluster.minio_bucket, obj.object_name) # Test multipart put. -@pytest.mark.parametrize("maybe_auth,positive", [ - pytest.param("", True, id="positive"), - pytest.param("'wrongid','wrongkey'", False, id="negative"), - # ("'minio','minio123',",True), Redirect with credentials not working with nginx. -]) +@pytest.mark.parametrize( + "maybe_auth,positive", + [ + pytest.param("", True, id="positive"), + pytest.param("'wrongid','wrongkey'", False, id="negative"), + # ("'minio','minio123',",True), Redirect with credentials not working with nginx. + ], +) def test_multipart_put(started_cluster, maybe_auth, positive): # type: (ClickHouseCluster) -> None - bucket = started_cluster.minio_bucket if not maybe_auth else started_cluster.minio_restricted_bucket + bucket = ( + started_cluster.minio_bucket + if not maybe_auth + else started_cluster.minio_restricted_bucket + ) instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_format = "column1 UInt32, column2 UInt32, column3 UInt32" @@ -421,11 +543,24 @@ def test_multipart_put(started_cluster, maybe_auth, positive): filename = "test_multipart.csv" put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV".format( - started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, maybe_auth, table_format) + started_cluster.minio_redirect_host, + started_cluster.minio_redirect_port, + bucket, + filename, + maybe_auth, + table_format, + ) try: - run_query(instance, put_query, stdin=csv_data, settings={'s3_min_upload_part_size': min_part_size_bytes, - 's3_max_single_part_upload_size': 0}) + run_query( + instance, + put_query, + stdin=csv_data, + settings={ + "s3_min_upload_part_size": min_part_size_bytes, + "s3_max_single_part_upload_size": 0, + }, + ) except helpers.client.QueryRuntimeException: if positive: raise @@ -444,12 +579,18 @@ def test_remote_host_filter(started_cluster): format = "column1 UInt32, column2 UInt32, column3 UInt32" query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/test.csv', 'CSV', '{}')".format( - "invalid_host", MINIO_INTERNAL_PORT, started_cluster.minio_bucket, format) + "invalid_host", MINIO_INTERNAL_PORT, started_cluster.minio_bucket, format + ) assert "not allowed in configuration file" in instance.query_and_get_error(query) other_values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format( - "invalid_host", MINIO_INTERNAL_PORT, started_cluster.minio_bucket, format, other_values) + "invalid_host", + MINIO_INTERNAL_PORT, + started_cluster.minio_bucket, + format, + other_values, + ) assert "not allowed in configuration file" in instance.query_and_get_error(query) @@ -476,25 +617,39 @@ def test_s3_glob_scheherazade(started_cluster): nights_per_job = 1001 // 30 jobs = [] for night in range(0, 1001, nights_per_job): + def add_tales(start, end): for i in range(start, end): path = "night_{}/tale.csv".format(i) query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, path, table_format, values) + started_cluster.minio_ip, + MINIO_INTERNAL_PORT, + bucket, + path, + table_format, + values, + ) run_query(instance, query) - jobs.append(threading.Thread(target=add_tales, args=(night, min(night + nights_per_job, 1001)))) + jobs.append( + threading.Thread( + target=add_tales, args=(night, min(night + nights_per_job, 1001)) + ) + ) jobs[-1].start() for job in jobs: job.join() query = "select count(), sum(column1), sum(column2), sum(column3) from s3('http://{}:{}/{}/night_*/tale.csv', 'CSV', '{}')".format( - started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, table_format) + started_cluster.minio_redirect_host, + started_cluster.minio_redirect_port, + bucket, + table_format, + ) assert run_query(instance, query).splitlines() == ["1001\t1001\t1001\t1001"] - def run_s3_mocks(started_cluster): logging.info("Starting s3 mocks") mocks = ( @@ -505,33 +660,46 @@ def run_s3_mocks(started_cluster): for mock_filename, container, port in mocks: container_id = started_cluster.get_container_id(container) current_dir = os.path.dirname(__file__) - started_cluster.copy_file_to_container(container_id, os.path.join(current_dir, "s3_mocks", mock_filename), mock_filename) - started_cluster.exec_in_container(container_id, ["python", mock_filename, port], detach=True) + started_cluster.copy_file_to_container( + container_id, + os.path.join(current_dir, "s3_mocks", mock_filename), + mock_filename, + ) + started_cluster.exec_in_container( + container_id, ["python", mock_filename, port], detach=True + ) # Wait for S3 mocks to start for mock_filename, container, port in mocks: num_attempts = 100 for attempt in range(num_attempts): - ping_response = started_cluster.exec_in_container(started_cluster.get_container_id(container), - ["curl", "-s", f"http://localhost:{port}/"], nothrow=True) - if ping_response != 'OK': + ping_response = started_cluster.exec_in_container( + started_cluster.get_container_id(container), + ["curl", "-s", f"http://localhost:{port}/"], + nothrow=True, + ) + if ping_response != "OK": if attempt == num_attempts - 1: - assert ping_response == 'OK', 'Expected "OK", but got "{}"'.format(ping_response) + assert ping_response == "OK", 'Expected "OK", but got "{}"'.format( + ping_response + ) else: time.sleep(1) else: - logging.debug(f"mock {mock_filename} ({port}) answered {ping_response} on attempt {attempt}") + logging.debug( + f"mock {mock_filename} ({port}) answered {ping_response} on attempt {attempt}" + ) break logging.info("S3 mocks started") def replace_config(old, new): - config = open(CONFIG_PATH, 'r') + config = open(CONFIG_PATH, "r") config_lines = config.readlines() config.close() config_lines = [line.replace(old, new) for line in config_lines] - config = open(CONFIG_PATH, 'w') + config = open(CONFIG_PATH, "w") config.writelines(config_lines) config.close() @@ -542,28 +710,36 @@ def test_custom_auth_headers(started_cluster): get_query = "select * from s3('http://resolver:8080/{bucket}/{file}', 'CSV', '{table_format}')".format( bucket=started_cluster.minio_restricted_bucket, file=filename, - table_format=table_format) + table_format=table_format, + ) instance = started_cluster.instances["dummy"] # type: ClickHouseInstance result = run_query(instance, get_query) - assert result == '1\t2\t3\n' + assert result == "1\t2\t3\n" instance.query("DROP TABLE IF EXISTS test") instance.query( "CREATE TABLE test ({table_format}) ENGINE = S3('http://resolver:8080/{bucket}/{file}', 'CSV')".format( bucket=started_cluster.minio_restricted_bucket, file=filename, - table_format=table_format - )) - assert run_query(instance, "SELECT * FROM test") == '1\t2\t3\n' + table_format=table_format, + ) + ) + assert run_query(instance, "SELECT * FROM test") == "1\t2\t3\n" - replace_config("
Authorization: Bearer TOKEN", "
Authorization: Bearer INVALID_TOKEN") + replace_config( + "
Authorization: Bearer TOKEN", + "
Authorization: Bearer INVALID_TOKEN", + ) instance.query("SYSTEM RELOAD CONFIG") ret, err = instance.query_and_get_answer_with_error("SELECT * FROM test") assert ret == "" and err != "" - replace_config("
Authorization: Bearer INVALID_TOKEN", "
Authorization: Bearer TOKEN") + replace_config( + "
Authorization: Bearer INVALID_TOKEN", + "
Authorization: Bearer TOKEN", + ) instance.query("SYSTEM RELOAD CONFIG") - assert run_query(instance, "SELECT * FROM test") == '1\t2\t3\n' + assert run_query(instance, "SELECT * FROM test") == "1\t2\t3\n" instance.query("DROP TABLE test") @@ -578,7 +754,7 @@ def test_custom_auth_headers_exclusion(started_cluster): print(result) assert ei.value.returncode == 243 - assert 'Forbidden Error' in ei.value.stderr + assert "Forbidden Error" in ei.value.stderr def test_infinite_redirect(started_cluster): @@ -595,10 +771,15 @@ def test_infinite_redirect(started_cluster): exception_raised = True finally: assert exception_raised -@pytest.mark.parametrize("extension,method", [ - pytest.param("bin", "gzip", id="bin"), - pytest.param("gz", "auto", id="gz"), -]) + + +@pytest.mark.parametrize( + "extension,method", + [ + pytest.param("bin", "gzip", id="bin"), + pytest.param("gz", "auto", id="gz"), + ], +) def test_storage_s3_get_gzip(started_cluster, extension, method): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] @@ -620,7 +801,7 @@ def test_storage_s3_get_gzip(started_cluster, extension, method): "Jerry Gonzalez,15", "Angela James,10", "Norman Ortega,33", - "" + "", ] run_query(instance, f"DROP TABLE IF EXISTS {name}") @@ -630,10 +811,13 @@ def test_storage_s3_get_gzip(started_cluster, extension, method): compressed.close() put_s3_file_content(started_cluster, bucket, filename, buf.getvalue()) - run_query(instance, f"""CREATE TABLE {name} (name String, id UInt32) ENGINE = S3( + run_query( + instance, + f"""CREATE TABLE {name} (name String, id UInt32) ENGINE = S3( 'http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{filename}', 'CSV', - '{method}')""") + '{method}')""", + ) run_query(instance, f"SELECT sum(id) FROM {name}").splitlines() == ["565"] run_query(instance, f"DROP TABLE {name}") @@ -670,21 +854,25 @@ def test_storage_s3_put_uncompressed(started_cluster): "'Kathie Dawson',100", "'Gregg Mcquistion',11", ] - run_query(instance, "CREATE TABLE {} (name String, id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format( - name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename)) + run_query( + instance, + "CREATE TABLE {} (name String, id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format( + name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename + ), + ) run_query(instance, "INSERT INTO {} VALUES ({})".format(name, "),(".join(data))) run_query(instance, "SELECT sum(id) FROM {}".format(name)).splitlines() == ["753"] uncompressed_content = get_s3_file_content(started_cluster, bucket, filename) - assert sum([ int(i.split(',')[1]) for i in uncompressed_content.splitlines() ]) == 753 + assert sum([int(i.split(",")[1]) for i in uncompressed_content.splitlines()]) == 753 -@pytest.mark.parametrize("extension,method", [ - pytest.param("bin", "gzip", id="bin"), - pytest.param("gz", "auto", id="gz") -]) +@pytest.mark.parametrize( + "extension,method", + [pytest.param("bin", "gzip", id="bin"), pytest.param("gz", "auto", id="gz")], +) def test_storage_s3_put_gzip(started_cluster, extension, method): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] @@ -705,21 +893,26 @@ def test_storage_s3_put_gzip(started_cluster, extension, method): "'Myrtle Pelt',93", "'Sylvia Naffziger',18", "'Amanda Cave',83", - "'Yolanda Joseph',89" + "'Yolanda Joseph',89", ] - run_query(instance, f"""CREATE TABLE {name} (name String, id UInt32) ENGINE = S3( + run_query( + instance, + f"""CREATE TABLE {name} (name String, id UInt32) ENGINE = S3( 'http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{filename}', 'CSV', - '{method}')""") + '{method}')""", + ) run_query(instance, f"INSERT INTO {name} VALUES ({'),('.join(data)})") run_query(instance, f"SELECT sum(id) FROM {name}").splitlines() == ["708"] - buf = io.BytesIO(get_s3_file_content(started_cluster, bucket, filename, decode=False)) + buf = io.BytesIO( + get_s3_file_content(started_cluster, bucket, filename, decode=False) + ) f = gzip.GzipFile(fileobj=buf, mode="rb") uncompressed_content = f.read().decode() - assert sum([ int(i.split(',')[1]) for i in uncompressed_content.splitlines() ]) == 708 + assert sum([int(i.split(",")[1]) for i in uncompressed_content.splitlines()]) == 708 def test_truncate_table(started_cluster): @@ -727,8 +920,11 @@ def test_truncate_table(started_cluster): instance = started_cluster.instances["dummy"] # type: ClickHouseInstance name = "truncate" - instance.query("CREATE TABLE {} (id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format( - name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, name)) + instance.query( + "CREATE TABLE {} (id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format( + name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, name + ) + ) instance.query("INSERT INTO {} SELECT number FROM numbers(10)".format(name)) result = instance.query("SELECT * FROM {}".format(name)) @@ -738,11 +934,14 @@ def test_truncate_table(started_cluster): minio = started_cluster.minio_client timeout = 30 while timeout > 0: - if len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0: + if ( + len(list(minio.list_objects(started_cluster.minio_bucket, "truncate/"))) + == 0 + ): return timeout -= 1 time.sleep(1) - assert(len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0) + assert len(list(minio.list_objects(started_cluster.minio_bucket, "truncate/"))) == 0 assert instance.query("SELECT * FROM {}".format(name)) == "" @@ -752,30 +951,47 @@ def test_predefined_connection_configuration(started_cluster): name = "test_table" instance.query("drop table if exists {}".format(name)) - instance.query("CREATE TABLE {} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')".format(name)) + instance.query( + "CREATE TABLE {} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')".format(name) + ) instance.query("INSERT INTO {} SELECT number FROM numbers(10)".format(name)) result = instance.query("SELECT * FROM {}".format(name)) assert result == instance.query("SELECT number FROM numbers(10)") - result = instance.query("SELECT * FROM s3(s3_conf1, format='CSV', structure='id UInt32')") + result = instance.query( + "SELECT * FROM s3(s3_conf1, format='CSV', structure='id UInt32')" + ) assert result == instance.query("SELECT number FROM numbers(10)") result = "" + + def test_url_reconnect_in_the_middle(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] table_format = "id String, data String" filename = "test_url_reconnect_{}.tsv".format(random.randint(0, 1000)) - instance.query(f"""insert into table function + instance.query( + f"""insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'TSV', '{table_format}') - select number, randomPrintableASCII(number % 1000) from numbers(1000000)""") + select number, randomPrintableASCII(number % 1000) from numbers(1000000)""" + ) with PartitionManager() as pm: - pm_rule_reject = {'probability': 0.02, 'destination': instance.ip_address, 'source_port': started_cluster.minio_port, 'action': 'REJECT --reject-with tcp-reset'} - pm_rule_drop_all = {'destination': instance.ip_address, 'source_port': started_cluster.minio_port, 'action': 'DROP'} + pm_rule_reject = { + "probability": 0.02, + "destination": instance.ip_address, + "source_port": started_cluster.minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm_rule_drop_all = { + "destination": instance.ip_address, + "source_port": started_cluster.minio_port, + "action": "DROP", + } pm._add_rule(pm_rule_reject) def select(): @@ -783,8 +999,9 @@ def test_url_reconnect_in_the_middle(started_cluster): result = instance.query( f"""select sum(cityHash64(x)) from (select toUInt64(id) + sleep(0.1) as x from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'TSV', '{table_format}') - settings http_max_tries = 10, http_retry_max_backoff_ms=2000, http_send_timeout=1, http_receive_timeout=1)""") - assert(int(result) == 3914219105369203805) + settings http_max_tries = 10, http_retry_max_backoff_ms=2000, http_send_timeout=1, http_receive_timeout=1)""" + ) + assert int(result) == 3914219105369203805 thread = threading.Thread(target=select) thread.start() @@ -797,7 +1014,7 @@ def test_url_reconnect_in_the_middle(started_cluster): thread.join() - assert(int(result) == 3914219105369203805) + assert int(result) == 3914219105369203805 def test_seekable_formats(started_cluster): @@ -805,21 +1022,29 @@ def test_seekable_formats(started_cluster): instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" - instance.query(f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1") + instance.query( + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1" + ) result = instance.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" - exec_query_with_retry(instance, f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1") + exec_query_with_retry( + instance, + f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1", + ) result = instance.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 instance.query("SYSTEM FLUSH LOGS") - result = instance.query(f"SELECT formatReadableSize(memory_usage) FROM system.query_log WHERE startsWith(query, 'SELECT count() FROM s3') AND memory_usage > 0 ORDER BY event_time desc") - print(result[:3]) - assert(int(result[:3]) < 200) + result = instance.query( + f"SELECT formatReadableSize(memory_usage) FROM system.query_log WHERE startsWith(query, 'SELECT count() FROM s3') AND memory_usage > 0 ORDER BY event_time desc" + ) + + result = result[: result.index(".")] + assert int(result) < 200 def test_seekable_formats_url(started_cluster): @@ -827,23 +1052,31 @@ def test_seekable_formats_url(started_cluster): instance = started_cluster.instances["dummy"] table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" - instance.query(f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1") + instance.query( + f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1" + ) table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_parquet', 'Parquet', 'a Int32, b String')" result = instance.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" - exec_query_with_retry(instance, f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1") + exec_query_with_retry( + instance, + f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1", + ) table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_orc', 'ORC', 'a Int32, b String')" result = instance.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 instance.query("SYSTEM FLUSH LOGS") - result = instance.query(f"SELECT formatReadableSize(memory_usage) FROM system.query_log WHERE startsWith(query, 'SELECT count() FROM url') AND memory_usage > 0 ORDER BY event_time desc") - print(result[:3]) - assert(int(result[:3]) < 200) + result = instance.query( + f"SELECT formatReadableSize(memory_usage) FROM system.query_log WHERE startsWith(query, 'SELECT count() FROM url') AND memory_usage > 0 ORDER BY event_time desc" + ) + + result = result[: result.index(".")] + assert int(result) < 200 def test_empty_file(started_cluster): @@ -851,62 +1084,69 @@ def test_empty_file(started_cluster): instance = started_cluster.instances["dummy"] name = "empty" - url = f'http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{name}' + url = f"http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{name}" minio = started_cluster.minio_client minio.put_object(bucket, name, io.BytesIO(b""), 0) table_function = f"s3('{url}', 'CSV', 'id Int32')" result = instance.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 0) + assert int(result) == 0 def test_insert_with_path_with_globs(started_cluster): instance = started_cluster.instances["dummy"] table_function_3 = f"s3('http://minio1:9001/root/test_parquet*', 'minio', 'minio123', 'Parquet', 'a Int32, b String')" - instance.query_and_get_error(f"insert into table function {table_function_3} SELECT number, randomString(100) FROM numbers(500)") + instance.query_and_get_error( + f"insert into table function {table_function_3} SELECT number, randomString(100) FROM numbers(500)" + ) def test_s3_schema_inference(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] - instance.query(f"insert into table function s3(s3_native, structure='a Int32, b String', format='Native') select number, randomString(100) from numbers(5000000)") + instance.query( + f"insert into table function s3(s3_native, structure='a Int32, b String', format='Native') select number, randomString(100) from numbers(5000000)" + ) result = instance.query(f"desc s3(s3_native, format='Native')") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" result = instance.query(f"select count(*) from s3(s3_native, format='Native')") - assert(int(result) == 5000000) + assert int(result) == 5000000 - instance.query(f"create table schema_inference engine=S3(s3_native, format='Native')") + instance.query( + f"create table schema_inference engine=S3(s3_native, format='Native')" + ) result = instance.query(f"desc schema_inference") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" result = instance.query(f"select count(*) from schema_inference") - assert(int(result) == 5000000) + assert int(result) == 5000000 - table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_native', 'Native')" result = instance.query(f"desc {table_function}") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" result = instance.query(f"select count(*) from {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 - instance.query(f"create table schema_inference_2 engine=URL('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_native', 'Native')") + instance.query( + f"create table schema_inference_2 engine=URL('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_native', 'Native')" + ) result = instance.query(f"desc schema_inference_2") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" result = instance.query(f"select count(*) from schema_inference_2") - assert(int(result) == 5000000) + assert int(result) == 5000000 table_function = f"s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_native', 'Native')" result = instance.query(f"desc {table_function}") assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" result = instance.query(f"select count(*) from {table_function}") - assert(int(result) == 5000000) + assert int(result) == 5000000 def test_empty_file(started_cluster): @@ -914,14 +1154,14 @@ def test_empty_file(started_cluster): instance = started_cluster.instances["dummy"] name = "empty" - url = f'http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{name}' + url = f"http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{name}" minio = started_cluster.minio_client minio.put_object(bucket, name, io.BytesIO(b""), 0) table_function = f"s3('{url}', 'CSV', 'id Int32')" result = instance.query(f"SELECT count() FROM {table_function}") - assert(int(result) == 0) + assert int(result) == 0 def test_overwrite(started_cluster): @@ -931,12 +1171,18 @@ def test_overwrite(started_cluster): table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" instance.query(f"create table test_overwrite as {table_function}") instance.query(f"truncate table test_overwrite") - instance.query(f"insert into test_overwrite select number, randomString(100) from numbers(50) settings s3_truncate_on_insert=1") - instance.query_and_get_error(f"insert into test_overwrite select number, randomString(100) from numbers(100)") - instance.query(f"insert into test_overwrite select number, randomString(100) from numbers(200) settings s3_truncate_on_insert=1") + instance.query( + f"insert into test_overwrite select number, randomString(100) from numbers(50) settings s3_truncate_on_insert=1" + ) + instance.query_and_get_error( + f"insert into test_overwrite select number, randomString(100) from numbers(100)" + ) + instance.query( + f"insert into test_overwrite select number, randomString(100) from numbers(200) settings s3_truncate_on_insert=1" + ) result = instance.query(f"select count() from test_overwrite") - assert(int(result) == 200) + assert int(result) == 200 def test_create_new_files_on_insert(started_cluster): @@ -946,26 +1192,40 @@ def test_create_new_files_on_insert(started_cluster): table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" instance.query(f"create table test_multiple_inserts as {table_function}") instance.query(f"truncate table test_multiple_inserts") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10) settings s3_truncate_on_insert=1") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1") - + instance.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(10) settings s3_truncate_on_insert=1" + ) + instance.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1" + ) + instance.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1" + ) + result = instance.query(f"select count() from test_multiple_inserts") - assert(int(result) == 60) + assert int(result) == 60 instance.query(f"drop table test_multiple_inserts") - table_function = f"s3(s3_parquet_gz, structure='a Int32, b String', format='Parquet')" + table_function = ( + f"s3(s3_parquet_gz, structure='a Int32, b String', format='Parquet')" + ) instance.query(f"create table test_multiple_inserts as {table_function}") instance.query(f"truncate table test_multiple_inserts") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10) settings s3_truncate_on_insert=1") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1") - - result = instance.query(f"select count() from test_multiple_inserts") - assert(int(result) == 60) + instance.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(10) settings s3_truncate_on_insert=1" + ) + instance.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1" + ) + instance.query( + f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1" + ) + + result = instance.query(f"select count() from test_multiple_inserts") + assert int(result) == 60 + - def test_format_detection(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] @@ -973,24 +1233,145 @@ def test_format_detection(started_cluster): instance.query(f"create table arrow_table_s3 (x UInt64) engine=S3(s3_arrow)") instance.query(f"insert into arrow_table_s3 select 1") result = instance.query(f"select * from s3(s3_arrow)") - assert(int(result) == 1) + assert int(result) == 1 - result = instance.query(f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')") - assert(int(result) == 1) - - result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')") - assert(int(result) == 1) + result = instance.query( + f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')" + ) + assert int(result) == 1 + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')" + ) + assert int(result) == 1 instance.query(f"create table parquet_table_s3 (x UInt64) engine=S3(s3_parquet2)") instance.query(f"insert into parquet_table_s3 select 1") result = instance.query(f"select * from s3(s3_parquet2)") - assert(int(result) == 1) + assert int(result) == 1 - result = instance.query(f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')") - assert(int(result) == 1) + result = instance.query( + f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')" + ) + assert int(result) == 1 - result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')") - assert(int(result) == 1) + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')" + ) + assert int(result) == 1 +def test_schema_inference_from_globs(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + instance.query( + f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test1.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL" + ) + instance.query( + f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test2.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select 0" + ) + + url_filename = "test{1,2}.jsoncompacteachrow" + result = instance.query( + f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')" + ) + assert result.strip() == "c1\tNullable(Float64)" + + result = instance.query( + f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}')" + ) + assert sorted(result.split()) == ["0", "\\N"] + + result = instance.query( + f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow')" + ) + assert result.strip() == "c1\tNullable(Float64)" + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow')" + ) + assert sorted(result.split()) == ["0", "\\N"] + + +def test_signatures(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + instance.query(f"create table test_signatures (x UInt64) engine=S3(s3_arrow)") + instance.query(f"truncate table test_signatures") + instance.query(f"insert into test_signatures select 1") + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')" + ) + assert int(result) == 1 + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'Arrow', 'x UInt64')" + ) + assert int(result) == 1 + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'minio', 'minio123')" + ) + assert int(result) == 1 + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'Arrow', 'x UInt64', 'auto')" + ) + assert int(result) == 1 + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'minio', 'minio123', 'Arrow')" + ) + assert int(result) == 1 + + +def test_select_columns(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + name = "test_table2" + structure = "id UInt32, value1 Int32, value2 Int32" + + instance.query(f"drop table if exists {name}") + instance.query( + f"CREATE TABLE {name} ({structure}) ENGINE = S3(s3_conf1, format='Parquet')" + ) + + limit = 10000000 + instance.query( + f"INSERT INTO {name} SELECT * FROM generateRandom('{structure}') LIMIT {limit} SETTINGS s3_truncate_on_insert=1" + ) + instance.query(f"SELECT value2 FROM {name}") + + instance.query("SYSTEM FLUSH LOGS") + result1 = instance.query( + f"SELECT read_bytes FROM system.query_log WHERE type='QueryFinish' and query LIKE 'SELECT value2 FROM {name}'" + ) + + instance.query(f"SELECT * FROM {name}") + instance.query("SYSTEM FLUSH LOGS") + result2 = instance.query( + f"SELECT read_bytes FROM system.query_log WHERE type='QueryFinish' and query LIKE 'SELECT * FROM {name}'" + ) + + assert int(result1) * 3 <= int(result2) + + +def test_insert_select_schema_inference(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + instance.query( + f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_insert_select.native') select toUInt64(1) as x" + ) + result = instance.query( + f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_insert_select.native')" + ) + assert result.strip() == "x\tUInt64" + + result = instance.query( + f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_insert_select.native')" + ) + assert int(result) == 1 diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py index 1ced71bc849..6ffb38bd8d7 100644 --- a/tests/integration/test_storage_url/test.py +++ b/tests/integration/test_storage_url/test.py @@ -4,11 +4,14 @@ from helpers.cluster import ClickHouseCluster uuids = [] + @pytest.fixture(scope="module") def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("node1", main_configs=["configs/conf.xml"], with_nginx=True) + cluster.add_instance( + "node1", main_configs=["configs/conf.xml"], with_nginx=True + ) cluster.start() yield cluster @@ -20,10 +23,18 @@ def cluster(): def test_partition_by(cluster): node1 = cluster.instances["node1"] - node1.query(f"insert into table function url(url1) partition by column3 values (1, 2, 3), (3, 2, 1), (1, 3, 2)") - result = node1.query(f"select * from url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')") - assert(result.strip() == "3\t2\t1") - result = node1.query(f"select * from url('http://nginx:80/test_2', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')") - assert(result.strip() == "1\t3\t2") - result = node1.query(f"select * from url('http://nginx:80/test_3', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')") - assert(result.strip() == "1\t2\t3") + node1.query( + f"insert into table function url(url1) partition by column3 values (1, 2, 3), (3, 2, 1), (1, 3, 2)" + ) + result = node1.query( + f"select * from url('http://nginx:80/test_1', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')" + ) + assert result.strip() == "3\t2\t1" + result = node1.query( + f"select * from url('http://nginx:80/test_2', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')" + ) + assert result.strip() == "1\t3\t2" + result = node1.query( + f"select * from url('http://nginx:80/test_3', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32')" + ) + assert result.strip() == "1\t2\t3" diff --git a/tests/integration/test_system_clusters_actual_information/test.py b/tests/integration/test_system_clusters_actual_information/test.py index 48f654dc30a..865c80db1c9 100644 --- a/tests/integration/test_system_clusters_actual_information/test.py +++ b/tests/integration/test_system_clusters_actual_information/test.py @@ -10,19 +10,26 @@ from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) -node_1 = cluster.add_instance('node_1', with_zookeeper=True) +node = cluster.add_instance( + "node", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) +node_1 = cluster.add_instance("node_1", with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node_1.query_with_retry('DROP TABLE IF EXISTS replicated') + node_1.query_with_retry("DROP TABLE IF EXISTS replicated") - node_1.query_with_retry('''CREATE TABLE replicated (id UInt32, date Date) ENGINE = - ReplicatedMergeTree('/clickhouse/tables/replicated', 'node_1') ORDER BY id PARTITION BY toYYYYMM(date)''') + node_1.query_with_retry( + """CREATE TABLE replicated (id UInt32, date Date) ENGINE = + ReplicatedMergeTree('/clickhouse/tables/replicated', 'node_1') ORDER BY id PARTITION BY toYYYYMM(date)""" + ) - node.query_with_retry("CREATE TABLE distributed (id UInt32, date Date) ENGINE = Distributed('test_cluster', 'default', 'replicated')") + node.query_with_retry( + "CREATE TABLE distributed (id UInt32, date Date) ENGINE = Distributed('test_cluster', 'default', 'replicated')" + ) yield cluster @@ -30,21 +37,26 @@ def started_cluster(): cluster.shutdown() - def test(started_cluster): cluster.pause_container("node_1") node.query("SYSTEM RELOAD CONFIG") - node.query_and_get_error("SELECT count() FROM distributed SETTINGS receive_timeout=1") + node.query_and_get_error( + "SELECT count() FROM distributed SETTINGS receive_timeout=1" + ) - result = node.query("SELECT errors_count, estimated_recovery_time FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + result = node.query( + "SELECT errors_count, estimated_recovery_time FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'" + ) errors_count, recovery_time = map(int, result.split()) assert errors_count == 3 - while True: + while True: time.sleep(1) - result = node.query("SELECT errors_count, estimated_recovery_time FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + result = node.query( + "SELECT errors_count, estimated_recovery_time FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'" + ) prev_time = recovery_time errors_count, recovery_time = map(int, result.split()) @@ -58,4 +70,3 @@ def test(started_cluster): assert errors_count == 0 cluster.unpause_container("node_1") - diff --git a/tests/integration/test_system_ddl_worker_queue/test.py b/tests/integration/test_system_ddl_worker_queue/test.py index c5037fc400e..4659e5b92e8 100644 --- a/tests/integration/test_system_ddl_worker_queue/test.py +++ b/tests/integration/test_system_ddl_worker_queue/test.py @@ -4,10 +4,18 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node3 = cluster.add_instance( + "node3", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node4 = cluster.add_instance( + "node4", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) nodes = [node1, node2, node3, node4] @@ -20,13 +28,17 @@ def started_cluster(): for i, node in enumerate([node1, node2]): node.query("CREATE DATABASE testdb") node.query( - '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format( - i)) + """CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;""".format( + i + ) + ) for i, node in enumerate([node3, node4]): node.query("CREATE DATABASE testdb") node.query( - '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format( - i)) + """CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;""".format( + i + ) + ) yield cluster finally: @@ -34,15 +46,25 @@ def started_cluster(): def test_distributed_ddl_queue(started_cluster): - node1.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") - node3.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node1.query( + "INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)" + ) + node3.query( + "INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)" + ) node2.query("SYSTEM SYNC REPLICA testdb.test_table") node4.query("SYSTEM SYNC REPLICA testdb.test_table") - node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", - settings={"replication_alter_partitions_sync": "2"}) + node1.query( + "ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", + settings={"replication_alter_partitions_sync": "2"}, + ) for node in nodes: node.query("SYSTEM SYNC REPLICA testdb.test_table") assert node.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" - assert node.query( - "SELECT If((SELECT count(*) FROM system.distributed_ddl_queue WHERE cluster='test_cluster' AND entry='query-0000000000') > 0, 'ok', 'fail')") == "ok\n" + assert ( + node.query( + "SELECT If((SELECT count(*) FROM system.distributed_ddl_queue WHERE cluster='test_cluster' AND entry='query-0000000000') > 0, 'ok', 'fail')" + ) + == "ok\n" + ) diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py index 407e66d56a7..d9ab76d2d61 100644 --- a/tests/integration/test_system_flush_logs/test.py +++ b/tests/integration/test_system_flush_logs/test.py @@ -6,22 +6,21 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node_default') +node = cluster.add_instance("node_default") system_logs = [ # disabled by default - ('system.text_log', 0), - + ("system.text_log", 0), # enabled by default - ('system.query_log', 1), - ('system.query_thread_log', 1), - ('system.part_log', 1), - ('system.trace_log', 1), - ('system.metric_log', 1), + ("system.query_log", 1), + ("system.query_thread_log", 1), + ("system.part_log", 1), + ("system.trace_log", 1), + ("system.metric_log", 1), ] -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -30,14 +29,14 @@ def start_cluster(): cluster.shutdown() -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def flush_logs(): - node.query('SYSTEM FLUSH LOGS') + node.query("SYSTEM FLUSH LOGS") -@pytest.mark.parametrize('table,exists', system_logs) +@pytest.mark.parametrize("table,exists", system_logs) def test_system_logs(flush_logs, table, exists): - q = 'SELECT * FROM {}'.format(table) + q = "SELECT * FROM {}".format(table) if exists: node.query(q) else: @@ -47,13 +46,16 @@ def test_system_logs(flush_logs, table, exists): # Logic is tricky, let's check that there is no hang in case of message queue # is not empty (this is another code path in the code). def test_system_logs_non_empty_queue(): - node.query('SELECT 1', settings={ - # right now defaults are the same, - # this set explicitly to avoid depends from defaults. - 'log_queries': 1, - 'log_queries_min_type': 'QUERY_START', - }) - node.query('SYSTEM FLUSH LOGS') + node.query( + "SELECT 1", + settings={ + # right now defaults are the same, + # this set explicitly to avoid depends from defaults. + "log_queries": 1, + "log_queries_min_type": "QUERY_START", + }, + ) + node.query("SYSTEM FLUSH LOGS") def test_system_suspend(): diff --git a/tests/integration/test_system_logs_comment/__init__.py b/tests/integration/test_system_logs_comment/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_system_logs_comment/test.py b/tests/integration/test_system_logs_comment/test.py new file mode 100644 index 00000000000..0659a2689a0 --- /dev/null +++ b/tests/integration/test_system_logs_comment/test.py @@ -0,0 +1,49 @@ +# pylint: disable=line-too-long +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name + +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node_default", stay_alive=True) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_system_logs_comment(): + node.exec_in_container( + [ + "bash", + "-c", + f"""echo " + + + ENGINE = MergeTree + PARTITION BY (event_date) + ORDER BY (event_time) + TTL event_date + INTERVAL 14 DAY DELETE + SETTINGS ttl_only_drop_parts=1 + COMMENT 'test_comment' + + + + + " > /etc/clickhouse-server/config.d/yyy-override-query_log.xml + """, + ] + ) + node.restart_clickhouse() + + node.query("select 1") + node.query("system flush logs") + + comment = node.query("SELECT comment FROM system.tables WHERE name = 'query_log'") + assert comment == "test_comment\n" diff --git a/tests/integration/test_system_logs_recreate/test.py b/tests/integration/test_system_logs_recreate/test.py index c0afa8cd555..387ad35dda2 100644 --- a/tests/integration/test_system_logs_recreate/test.py +++ b/tests/integration/test_system_logs_recreate/test.py @@ -6,9 +6,10 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node_default', stay_alive=True) +node = cluster.add_instance("node_default", stay_alive=True) -@pytest.fixture(scope='module', autouse=True) + +@pytest.fixture(scope="module", autouse=True) def start_cluster(): try: cluster.start() @@ -20,23 +21,34 @@ def start_cluster(): def test_system_logs_recreate(): system_logs = [ # enabled by default - 'query_log', - 'query_thread_log', - 'part_log', - 'trace_log', - 'metric_log', + "query_log", + "query_thread_log", + "part_log", + "trace_log", + "metric_log", ] - node.query('SYSTEM FLUSH LOGS') + node.query("SYSTEM FLUSH LOGS") for table in system_logs: - assert 'ENGINE = MergeTree' in node.query(f'SHOW CREATE TABLE system.{table}') - assert 'ENGINE = Null' not in node.query(f'SHOW CREATE TABLE system.{table}') - assert len(node.query(f"SHOW TABLES FROM system LIKE '{table}%'").strip().split('\n')) == 1 + assert "ENGINE = MergeTree" in node.query(f"SHOW CREATE TABLE system.{table}") + assert "ENGINE = Null" not in node.query(f"SHOW CREATE TABLE system.{table}") + assert ( + len( + node.query(f"SHOW TABLES FROM system LIKE '{table}%'") + .strip() + .split("\n") + ) + == 1 + ) # NOTE: we use zzz- prefix to make it the last file, # so that it will be applied last. for table in system_logs: - node.exec_in_container(['bash', '-c', f"""echo " + node.exec_in_container( + [ + "bash", + "-c", + f"""echo " <{table}> ENGINE = Null @@ -44,41 +56,74 @@ def test_system_logs_recreate(): " > /etc/clickhouse-server/config.d/zzz-override-{table}.xml - """]) + """, + ] + ) node.restart_clickhouse() - node.query('SYSTEM FLUSH LOGS') + node.query("SYSTEM FLUSH LOGS") for table in system_logs: - assert 'ENGINE = MergeTree' not in node.query(f'SHOW CREATE TABLE system.{table}') - assert 'ENGINE = Null' in node.query(f'SHOW CREATE TABLE system.{table}') - assert len(node.query(f"SHOW TABLES FROM system LIKE '{table}%'").strip().split('\n')) == 2 + assert "ENGINE = MergeTree" not in node.query( + f"SHOW CREATE TABLE system.{table}" + ) + assert "ENGINE = Null" in node.query(f"SHOW CREATE TABLE system.{table}") + assert ( + len( + node.query(f"SHOW TABLES FROM system LIKE '{table}%'") + .strip() + .split("\n") + ) + == 2 + ) for table in system_logs: - node.exec_in_container(['rm', f'/etc/clickhouse-server/config.d/zzz-override-{table}.xml']) + node.exec_in_container( + ["rm", f"/etc/clickhouse-server/config.d/zzz-override-{table}.xml"] + ) node.restart_clickhouse() - node.query('SYSTEM FLUSH LOGS') + node.query("SYSTEM FLUSH LOGS") for table in system_logs: - assert 'ENGINE = MergeTree' in node.query(f'SHOW CREATE TABLE system.{table}') - assert 'ENGINE = Null' not in node.query(f'SHOW CREATE TABLE system.{table}') - assert len(node.query(f"SHOW TABLES FROM system LIKE '{table}%'").strip().split('\n')) == 3 + assert "ENGINE = MergeTree" in node.query(f"SHOW CREATE TABLE system.{table}") + assert "ENGINE = Null" not in node.query(f"SHOW CREATE TABLE system.{table}") + assert ( + len( + node.query(f"SHOW TABLES FROM system LIKE '{table}%'") + .strip() + .split("\n") + ) + == 3 + ) - node.query('SYSTEM FLUSH LOGS') + node.query("SYSTEM FLUSH LOGS") # Ensure that there was no superfluous RENAME's # IOW that the table created only when the structure is indeed different. for table in system_logs: - assert len(node.query(f"SHOW TABLES FROM system LIKE '{table}%'").strip().split('\n')) == 3 + assert ( + len( + node.query(f"SHOW TABLES FROM system LIKE '{table}%'") + .strip() + .split("\n") + ) + == 3 + ) def test_drop_system_log(): - node.exec_in_container(['bash', '-c', f"""echo " + node.exec_in_container( + [ + "bash", + "-c", + f"""echo " 1000000 " > /etc/clickhouse-server/config.d/yyy-override-query_log.xml - """]) + """, + ] + ) node.restart_clickhouse() node.query("select 1") node.query("system flush logs") @@ -89,5 +134,7 @@ def test_drop_system_log(): node.query("select 3") node.query("system flush logs") assert node.query("select count() > 0 from system.query_log") == "1\n" - node.exec_in_container(['rm', f'/etc/clickhouse-server/config.d/yyy-override-query_log.xml']) + node.exec_in_container( + ["rm", f"/etc/clickhouse-server/config.d/yyy-override-query_log.xml"] + ) node.restart_clickhouse() diff --git a/tests/integration/test_system_merges/test.py b/tests/integration/test_system_merges/test.py index 672b637f783..9239cb11065 100644 --- a/tests/integration/test_system_merges/test.py +++ b/tests/integration/test_system_merges/test.py @@ -6,23 +6,29 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - macros={"shard": 0, "replica": 1}) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/logs_config.xml"], + with_zookeeper=True, + macros={"shard": 0, "replica": 1}, +) -node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml'], - with_zookeeper=True, - macros={"shard": 0, "replica": 2}) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/logs_config.xml"], + with_zookeeper=True, + macros={"shard": 0, "replica": 2}, +) @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() - node1.query('CREATE DATABASE test ENGINE=Ordinary') # Different paths with Atomic - node2.query('CREATE DATABASE test ENGINE=Ordinary') + node1.query( + "CREATE DATABASE test ENGINE=Ordinary" + ) # Different paths with Atomic + node2.query("CREATE DATABASE test ENGINE=Ordinary") yield cluster finally: @@ -33,10 +39,7 @@ def split_tsv(data): return [x.split("\t") for x in data.splitlines()] -@pytest.mark.parametrize("replicated", [ - "", - "replicated" -]) +@pytest.mark.parametrize("replicated", ["", "replicated"]) def test_merge_simple(started_cluster, replicated): try: clickhouse_path = "/var/lib/clickhouse" @@ -45,25 +48,36 @@ def test_merge_simple(started_cluster, replicated): name = db_name + "." + table_name table_path = "data/" + db_name + "/" + table_name nodes = [node1, node2] if replicated else [node1] - engine = "ReplicatedMergeTree('/clickhouse/test_merge_simple', '{replica}')" if replicated else "MergeTree()" + engine = ( + "ReplicatedMergeTree('/clickhouse/test_merge_simple', '{replica}')" + if replicated + else "MergeTree()" + ) node_check = nodes[-1] starting_block = 0 if replicated else 1 for node in nodes: - node.query(""" + node.query( + """ CREATE TABLE {name} ( `a` Int64 ) ENGINE = {engine} ORDER BY sleep(2) - """.format(engine=engine, name=name)) + """.format( + engine=engine, name=name + ) + ) node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) node1.query("INSERT INTO {name} VALUES (2)".format(name=name)) node1.query("INSERT INTO {name} VALUES (3)".format(name=name)) - parts = ["all_{}_{}_0".format(x, x) for x in range(starting_block, starting_block + 3)] + parts = [ + "all_{}_{}_0".format(x, x) + for x in range(starting_block, starting_block + 3) + ] result_part = "all_{}_{}_1".format(starting_block, starting_block + 2) def optimize(): @@ -75,38 +89,54 @@ def test_merge_simple(started_cluster, replicated): t.start() time.sleep(1) - assert split_tsv(node_check.query(""" + assert ( + split_tsv( + node_check.query( + """ SELECT database, table, num_parts, source_part_names, source_part_paths, result_part_name, result_part_path, partition_id, is_mutation FROM system.merges WHERE table = '{name}' - """.format(name=table_name))) == [ - [ - db_name, - table_name, - "3", - "['{}','{}','{}']".format(*parts), - "['{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/']".format( - *parts, clickhouse=clickhouse_path, table_path=table_path), - result_part, - "{clickhouse}/{table_path}/{}/".format(result_part, clickhouse=clickhouse_path, table_path=table_path), - "all", - "0" + """.format( + name=table_name + ) + ) + ) + == [ + [ + db_name, + table_name, + "3", + "['{}','{}','{}']".format(*parts), + "['{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/','{clickhouse}/{table_path}/{}/']".format( + *parts, clickhouse=clickhouse_path, table_path=table_path + ), + result_part, + "{clickhouse}/{table_path}/{}/".format( + result_part, clickhouse=clickhouse_path, table_path=table_path + ), + "all", + "0", + ] ] - ] + ) t.join() wait.join() - assert node_check.query("SELECT * FROM system.merges WHERE table = '{name}'".format(name=table_name)) == "" + assert ( + node_check.query( + "SELECT * FROM system.merges WHERE table = '{name}'".format( + name=table_name + ) + ) + == "" + ) finally: for node in nodes: node.query("DROP TABLE {name}".format(name=name)) -@pytest.mark.parametrize("replicated", [ - "", - "replicated" -]) +@pytest.mark.parametrize("replicated", ["", "replicated"]) def test_mutation_simple(started_cluster, replicated): try: clickhouse_path = "/var/lib/clickhouse" @@ -115,53 +145,88 @@ def test_mutation_simple(started_cluster, replicated): name = db_name + "." + table_name table_path = "data/" + db_name + "/" + table_name nodes = [node1, node2] if replicated else [node1] - engine = "ReplicatedMergeTree('/clickhouse/test_mutation_simple', '{replica}')" if replicated else "MergeTree()" + engine = ( + "ReplicatedMergeTree('/clickhouse/test_mutation_simple', '{replica}')" + if replicated + else "MergeTree()" + ) node_check = nodes[-1] starting_block = 0 if replicated else 1 for node in nodes: - node.query(""" + node.query( + """ CREATE TABLE {name} ( `a` Int64 ) ENGINE = {engine} ORDER BY tuple() - """.format(engine=engine, name=name)) + """.format( + engine=engine, name=name + ) + ) node1.query("INSERT INTO {name} VALUES (1)".format(name=name)) part = "all_{}_{}_0".format(starting_block, starting_block) - result_part = "all_{}_{}_0_{}".format(starting_block, starting_block, starting_block + 1) + result_part = "all_{}_{}_0_{}".format( + starting_block, starting_block, starting_block + 1 + ) def alter(): - node1.query("ALTER TABLE {name} UPDATE a = 42 WHERE sleep(2) OR 1".format(name=name), settings={ - 'mutations_sync': 1, - }) + node1.query( + "ALTER TABLE {name} UPDATE a = 42 WHERE sleep(2) OR 1".format( + name=name + ), + settings={ + "mutations_sync": 1, + }, + ) t = threading.Thread(target=alter) t.start() time.sleep(1) - assert split_tsv(node_check.query(""" + assert ( + split_tsv( + node_check.query( + """ SELECT database, table, num_parts, source_part_names, source_part_paths, result_part_name, result_part_path, partition_id, is_mutation FROM system.merges WHERE table = '{name}' - """.format(name=table_name))) == [ - [ - db_name, - table_name, - "1", - "['{}']".format(part), - "['{clickhouse}/{table_path}/{}/']".format(part, clickhouse=clickhouse_path, table_path=table_path), - result_part, - "{clickhouse}/{table_path}/{}/".format(result_part, clickhouse=clickhouse_path, table_path=table_path), - "all", - "1" - ], - ] + """.format( + name=table_name + ) + ) + ) + == [ + [ + db_name, + table_name, + "1", + "['{}']".format(part), + "['{clickhouse}/{table_path}/{}/']".format( + part, clickhouse=clickhouse_path, table_path=table_path + ), + result_part, + "{clickhouse}/{table_path}/{}/".format( + result_part, clickhouse=clickhouse_path, table_path=table_path + ), + "all", + "1", + ], + ] + ) t.join() - assert node_check.query("SELECT * FROM system.merges WHERE table = '{name}'".format(name=table_name)) == "" + assert ( + node_check.query( + "SELECT * FROM system.merges WHERE table = '{name}'".format( + name=table_name + ) + ) + == "" + ) finally: for node in nodes: diff --git a/tests/integration/test_system_metrics/test.py b/tests/integration/test_system_metrics/test.py index efcc6f88a24..439e8b66db1 100644 --- a/tests/integration/test_system_metrics/test.py +++ b/tests/integration/test_system_metrics/test.py @@ -9,17 +9,24 @@ from helpers.network import PartitionManager def fill_nodes(nodes, shard): for node in nodes: node.query( - ''' + """ CREATE DATABASE test; CREATE TABLE test.test_table(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}') ORDER BY id PARTITION BY toYYYYMM(date) SETTINGS min_replicated_logs_to_keep=3, max_replicated_logs_to_keep=5, cleanup_delay_period=0, cleanup_delay_period_random_add=0; - '''.format(shard=shard, replica=node.name)) + """.format( + shard=shard, replica=node.name + ) + ) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -37,31 +44,62 @@ def start_cluster(): finally: cluster.shutdown() + def test_readonly_metrics(start_cluster): - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'") == "0\n" + assert ( + node1.query("SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'") + == "0\n" + ) with PartitionManager() as pm: ## make node1 readonly -> heal -> readonly -> heal -> detach table -> heal -> attach table pm.drop_instance_zk_connections(node1) - assert_eq_with_retry(node1, "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", "1\n", retry_count=300, sleep_time=1) + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "1\n", + retry_count=300, + sleep_time=1, + ) pm.heal_all() - assert_eq_with_retry(node1, "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", "0\n", retry_count=300, sleep_time=1) + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "0\n", + retry_count=300, + sleep_time=1, + ) pm.drop_instance_zk_connections(node1) - assert_eq_with_retry(node1, "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", "1\n", retry_count=300, sleep_time=1) - + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "1\n", + retry_count=300, + sleep_time=1, + ) node1.query("DETACH TABLE test.test_table") - assert "0\n" == node1.query("SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'") + assert "0\n" == node1.query( + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'" + ) pm.heal_all() node1.query("ATTACH TABLE test.test_table") - assert_eq_with_retry(node1, "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", "0\n", retry_count=300, sleep_time=1) + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "0\n", + retry_count=300, + sleep_time=1, + ) -#For LowCardinality-columns, the bytes for N rows is not N*size of 1 row. + +# For LowCardinality-columns, the bytes for N rows is not N*size of 1 row. def test_metrics_storage_buffer_size(start_cluster): - node1.query(''' + node1.query( + """ CREATE TABLE test.test_mem_table ( `str` LowCardinality(String) @@ -73,18 +111,49 @@ def test_metrics_storage_buffer_size(start_cluster): `str` LowCardinality(String) ) ENGINE = Buffer('test', 'test_mem_table', 1, 600, 600, 1000, 100000, 100000, 10000000); - ''') + """ + ) - #before flush + # before flush node1.query("INSERT INTO test.buffer_table VALUES('hello');") - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'StorageBufferRows'") == "1\n" - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'") == "24\n" + assert ( + node1.query( + "SELECT value FROM system.metrics WHERE metric = 'StorageBufferRows'" + ) + == "1\n" + ) + assert ( + node1.query( + "SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'" + ) + == "24\n" + ) node1.query("INSERT INTO test.buffer_table VALUES('hello');") - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'StorageBufferRows'") == "2\n" - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'") == "25\n" + assert ( + node1.query( + "SELECT value FROM system.metrics WHERE metric = 'StorageBufferRows'" + ) + == "2\n" + ) + assert ( + node1.query( + "SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'" + ) + == "25\n" + ) - #flush + # flush node1.query("OPTIMIZE TABLE test.buffer_table") - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'StorageBufferRows'") == "0\n" - assert node1.query("SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'") == "0\n" + assert ( + node1.query( + "SELECT value FROM system.metrics WHERE metric = 'StorageBufferRows'" + ) + == "0\n" + ) + assert ( + node1.query( + "SELECT value FROM system.metrics WHERE metric = 'StorageBufferBytes'" + ) + == "0\n" + ) diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py index 50f4afd1abe..9138a934554 100644 --- a/tests/integration/test_system_queries/test.py +++ b/tests/integration/test_system_queries/test.py @@ -17,15 +17,24 @@ def started_cluster(): global instance try: cluster = ClickHouseCluster(__file__) - cluster.add_instance('ch1', - main_configs=["configs/config.d/clusters_config.xml", "configs/config.d/query_log.xml"], - dictionaries=["configs/dictionaries/dictionary_clickhouse_cache.xml", - "configs/dictionaries/dictionary_clickhouse_flat.xml"]) + cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters_config.xml", + "configs/config.d/query_log.xml", + ], + dictionaries=[ + "configs/dictionaries/dictionary_clickhouse_cache.xml", + "configs/dictionaries/dictionary_clickhouse_flat.xml", + ], + ) cluster.start() - instance = cluster.instances['ch1'] - instance.query('CREATE DATABASE dictionaries ENGINE = Dictionary') - instance.query('CREATE TABLE dictionary_source (id UInt64, value UInt8) ENGINE = Memory') + instance = cluster.instances["ch1"] + instance.query("CREATE DATABASE dictionaries ENGINE = Dictionary") + instance.query( + "CREATE TABLE dictionary_source (id UInt64, value UInt8) ENGINE = Memory" + ) yield cluster finally: @@ -34,104 +43,154 @@ def started_cluster(): def test_SYSTEM_RELOAD_DICTIONARY(started_cluster): - instance = cluster.instances['ch1'] + instance = cluster.instances["ch1"] instance.query("SYSTEM RELOAD DICTIONARIES") - assert TSV(instance.query( - "SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))")) == TSV("0\t0\n") + assert TSV( + instance.query( + "SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))" + ) + ) == TSV("0\t0\n") instance.query("INSERT INTO dictionary_source VALUES (0, 0)") - assert TSV(instance.query( - "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))")) == TSV( - "0\t0\n") + assert TSV( + instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))" + ) + ) == TSV("0\t0\n") instance.query("INSERT INTO dictionary_source VALUES (1, 1)") - assert TSV(instance.query( - "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))")) == TSV( - "0\t0\n") + assert TSV( + instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictHas('clickhouse_cache', toUInt64(1))" + ) + ) == TSV("0\t0\n") instance.query("SYSTEM RELOAD DICTIONARY clickhouse_cache") - assert TSV(instance.query( - "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))")) == TSV( - "0\t1\n") - assert TSV(instance.query( - "SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))")) == TSV("0\t0\n") + assert TSV( + instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))" + ) + ) == TSV("0\t1\n") + assert TSV( + instance.query( + "SELECT dictHas('clickhouse_flat', toUInt64(0)), dictHas('clickhouse_flat', toUInt64(1))" + ) + ) == TSV("0\t0\n") instance.query("SYSTEM RELOAD DICTIONARIES") - assert TSV(instance.query( - "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))")) == TSV( - "0\t1\n") - assert TSV(instance.query( - "SELECT dictGetUInt8('clickhouse_flat', 'value', toUInt64(0)), dictGetUInt8('clickhouse_flat', 'value', toUInt64(1))")) == TSV( - "0\t1\n") + assert TSV( + instance.query( + "SELECT dictGetUInt8('clickhouse_cache', 'value', toUInt64(0)), dictGetUInt8('clickhouse_cache', 'value', toUInt64(1))" + ) + ) == TSV("0\t1\n") + assert TSV( + instance.query( + "SELECT dictGetUInt8('clickhouse_flat', 'value', toUInt64(0)), dictGetUInt8('clickhouse_flat', 'value', toUInt64(1))" + ) + ) == TSV("0\t1\n") def test_DROP_DNS_CACHE(started_cluster): - instance = cluster.instances['ch1'] + instance = cluster.instances["ch1"] - instance.exec_in_container(['bash', '-c', 'echo 127.0.0.1 localhost > /etc/hosts'], privileged=True, user='root') - instance.exec_in_container(['bash', '-c', 'echo ::1 localhost >> /etc/hosts'], privileged=True, user='root') + instance.exec_in_container( + ["bash", "-c", "echo 127.0.0.1 localhost > /etc/hosts"], + privileged=True, + user="root", + ) + instance.exec_in_container( + ["bash", "-c", "echo ::1 localhost >> /etc/hosts"], privileged=True, user="root" + ) - instance.exec_in_container(['bash', '-c', 'echo 127.255.255.255 lost_host >> /etc/hosts'], privileged=True, - user='root') + instance.exec_in_container( + ["bash", "-c", "echo 127.255.255.255 lost_host >> /etc/hosts"], + privileged=True, + user="root", + ) instance.query("SYSTEM DROP DNS CACHE") with pytest.raises(QueryRuntimeException): instance.query("SELECT * FROM remote('lost_host', 'system', 'one')") instance.query( - "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')") + "CREATE TABLE distributed_lost_host (dummy UInt8) ENGINE = Distributed(lost_host_cluster, 'system', 'one')" + ) with pytest.raises(QueryRuntimeException): instance.query("SELECT * FROM distributed_lost_host") - instance.exec_in_container(['bash', '-c', 'echo 127.0.0.1 localhost > /etc/hosts'], privileged=True, user='root') - instance.exec_in_container(['bash', '-c', 'echo ::1 localhost >> /etc/hosts'], privileged=True, user='root') + instance.exec_in_container( + ["bash", "-c", "echo 127.0.0.1 localhost > /etc/hosts"], + privileged=True, + user="root", + ) + instance.exec_in_container( + ["bash", "-c", "echo ::1 localhost >> /etc/hosts"], privileged=True, user="root" + ) - instance.exec_in_container(['bash', '-c', 'echo 127.0.0.1 lost_host >> /etc/hosts'], privileged=True, user='root') + instance.exec_in_container( + ["bash", "-c", "echo 127.0.0.1 lost_host >> /etc/hosts"], + privileged=True, + user="root", + ) instance.query("SYSTEM DROP DNS CACHE") instance.query("SELECT * FROM remote('lost_host', 'system', 'one')") instance.query("SELECT * FROM distributed_lost_host") - assert TSV(instance.query( - "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'")) == TSV( - "lost_host\t127.0.0.1\n") + assert TSV( + instance.query( + "SELECT DISTINCT host_name, host_address FROM system.clusters WHERE cluster='lost_host_cluster'" + ) + ) == TSV("lost_host\t127.0.0.1\n") def test_RELOAD_CONFIG_AND_MACROS(started_cluster): macros = "ro" - create_macros = 'echo "{}" > /etc/clickhouse-server/config.d/macros.xml'.format(macros) + create_macros = 'echo "{}" > /etc/clickhouse-server/config.d/macros.xml'.format( + macros + ) - instance = cluster.instances['ch1'] + instance = cluster.instances["ch1"] - instance.exec_in_container(['bash', '-c', create_macros], privileged=True, user='root') + instance.exec_in_container( + ["bash", "-c", create_macros], privileged=True, user="root" + ) instance.query("SYSTEM RELOAD CONFIG") - assert TSV(instance.query("select * from system.macros")) == TSV("instance\tch1\nmac\tro\n") + assert TSV(instance.query("select * from system.macros")) == TSV( + "instance\tch1\nmac\tro\n" + ) def test_system_flush_logs(started_cluster): - instance = cluster.instances['ch1'] - instance.query(''' + instance = cluster.instances["ch1"] + instance.query( + """ SET log_queries = 0; SYSTEM FLUSH LOGS; TRUNCATE TABLE system.query_log; - ''') + """ + ) for i in range(4): # Sleep to execute flushing from background thread at first query # by expiration of flush_interval_millisecond and test probable race condition. time.sleep(0.5) - result = instance.query(''' + result = instance.query( + """ SELECT 1 FORMAT Null; SET log_queries = 0; SYSTEM FLUSH LOGS; - SELECT count() FROM system.query_log;''') - instance.query(''' + SELECT count() FROM system.query_log;""" + ) + instance.query( + """ SET log_queries = 0; SYSTEM FLUSH LOGS; TRUNCATE TABLE system.query_log; - ''') - assert TSV(result) == TSV('4') + """ + ) + assert TSV(result) == TSV("4") -if __name__ == '__main__': +if __name__ == "__main__": with contextmanager(started_cluster)() as cluster: for name, instance in list(cluster.instances.items()): print(name, instance.ip_address) diff --git a/tests/integration/test_system_replicated_fetches/test.py b/tests/integration/test_system_replicated_fetches/test.py index fcbdd4addd9..2b516ebf69b 100644 --- a/tests/integration/test_system_replicated_fetches/test.py +++ b/tests/integration/test_system_replicated_fetches/test.py @@ -11,8 +11,9 @@ import string import json cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) + @pytest.fixture(scope="module") def started_cluster(): @@ -24,21 +25,35 @@ def started_cluster(): finally: cluster.shutdown() + def get_random_string(length): - return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + return "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(length) + ) + def test_system_replicated_fetches(started_cluster): - node1.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple()") - node2.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple()") + node1.query( + "CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple()" + ) + node2.query( + "CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple()" + ) with PartitionManager() as pm: node2.query("SYSTEM STOP FETCHES t") - node1.query("INSERT INTO t SELECT number, '{}' FROM numbers(10000)".format(get_random_string(104857))) + node1.query( + "INSERT INTO t SELECT number, '{}' FROM numbers(10000)".format( + get_random_string(104857) + ) + ) pm.add_network_delay(node1, 80) node2.query("SYSTEM START FETCHES t") fetches_result = [] for _ in range(1000): - result = json.loads(node2.query("SELECT * FROM system.replicated_fetches FORMAT JSON")) + result = json.loads( + node2.query("SELECT * FROM system.replicated_fetches FORMAT JSON") + ) if not result["data"]: if fetches_result: break @@ -52,45 +67,69 @@ def test_system_replicated_fetches(started_cluster): assert node2.query("SELECT COUNT() FROM t") == "10000\n" for elem in fetches_result: - elem['bytes_read_compressed'] = float(elem['bytes_read_compressed']) - elem['total_size_bytes_compressed'] = float(elem['total_size_bytes_compressed']) - elem['progress'] = float(elem['progress']) - elem['elapsed'] = float(elem['elapsed']) + elem["bytes_read_compressed"] = float(elem["bytes_read_compressed"]) + elem["total_size_bytes_compressed"] = float(elem["total_size_bytes_compressed"]) + elem["progress"] = float(elem["progress"]) + elem["elapsed"] = float(elem["elapsed"]) assert len(fetches_result) > 0 first_non_empty = fetches_result[0] - assert first_non_empty['database'] == "default" - assert first_non_empty['table'] == "t" - assert first_non_empty['source_replica_hostname'] == 'node1' - assert first_non_empty['source_replica_port'] == 9009 - assert first_non_empty['source_replica_path'] == '/clickhouse/test/t/replicas/1' - assert first_non_empty['interserver_scheme'] == 'http' - assert first_non_empty['result_part_name'] == 'all_0_0_0' - assert first_non_empty['result_part_path'].startswith('/var/lib/clickhouse/') - assert first_non_empty['result_part_path'].endswith('all_0_0_0/') - assert first_non_empty['partition_id'] == 'all' - assert first_non_empty['URI'].startswith('http://node1:9009/?endpoint=DataPartsExchange') + assert first_non_empty["database"] == "default" + assert first_non_empty["table"] == "t" + assert first_non_empty["source_replica_hostname"] == "node1" + assert first_non_empty["source_replica_port"] == 9009 + assert first_non_empty["source_replica_path"] == "/clickhouse/test/t/replicas/1" + assert first_non_empty["interserver_scheme"] == "http" + assert first_non_empty["result_part_name"] == "all_0_0_0" + assert first_non_empty["result_part_path"].startswith("/var/lib/clickhouse/") + assert first_non_empty["result_part_path"].endswith("all_0_0_0/") + assert first_non_empty["partition_id"] == "all" + assert first_non_empty["URI"].startswith( + "http://node1:9009/?endpoint=DataPartsExchange" + ) for elem in fetches_result: - assert elem['bytes_read_compressed'] <= elem['total_size_bytes_compressed'], "Bytes read ({}) more than total bytes ({}). It's a bug".format(elem['bytes_read_compressed'], elem['total_size_bytes_compressed']) - assert 0.0 <= elem['progress'] <= 1.0, "Progress shouldn't less than 0 and bigger than 1, got {}".format(elem['progress']) - assert 0.0 <= elem['elapsed'], "Elapsed time must be greater than 0, got {}".format(elem['elapsed']) + assert ( + elem["bytes_read_compressed"] <= elem["total_size_bytes_compressed"] + ), "Bytes read ({}) more than total bytes ({}). It's a bug".format( + elem["bytes_read_compressed"], elem["total_size_bytes_compressed"] + ) + assert ( + 0.0 <= elem["progress"] <= 1.0 + ), "Progress shouldn't less than 0 and bigger than 1, got {}".format( + elem["progress"] + ) + assert ( + 0.0 <= elem["elapsed"] + ), "Elapsed time must be greater than 0, got {}".format(elem["elapsed"]) - prev_progress = first_non_empty['progress'] + prev_progress = first_non_empty["progress"] for elem in fetches_result: - assert elem['progress'] >= prev_progress, "Progress decreasing prev{}, next {}? It's a bug".format(prev_progress, elem['progress']) - prev_progress = elem['progress'] + assert ( + elem["progress"] >= prev_progress + ), "Progress decreasing prev{}, next {}? It's a bug".format( + prev_progress, elem["progress"] + ) + prev_progress = elem["progress"] - prev_bytes = first_non_empty['bytes_read_compressed'] + prev_bytes = first_non_empty["bytes_read_compressed"] for elem in fetches_result: - assert elem['bytes_read_compressed'] >= prev_bytes, "Bytes read decreasing prev {}, next {}? It's a bug".format(prev_bytes, elem['bytes_read_compressed']) - prev_bytes = elem['bytes_read_compressed'] + assert ( + elem["bytes_read_compressed"] >= prev_bytes + ), "Bytes read decreasing prev {}, next {}? It's a bug".format( + prev_bytes, elem["bytes_read_compressed"] + ) + prev_bytes = elem["bytes_read_compressed"] - prev_elapsed = first_non_empty['elapsed'] + prev_elapsed = first_non_empty["elapsed"] for elem in fetches_result: - assert elem['elapsed'] >= prev_elapsed, "Elapsed time decreasing prev {}, next {}? It's a bug".format(prev_elapsed, elem['elapsed']) - prev_elapsed = elem['elapsed'] + assert ( + elem["elapsed"] >= prev_elapsed + ), "Elapsed time decreasing prev {}, next {}? It's a bug".format( + prev_elapsed, elem["elapsed"] + ) + prev_elapsed = elem["elapsed"] node1.query("DROP TABLE IF EXISTS t SYNC") node2.query("DROP TABLE IF EXISTS t SYNC") diff --git a/tests/integration/test_table_functions_access_rights/test.py b/tests/integration/test_table_functions_access_rights/test.py index 90106303315..705150c8bdd 100644 --- a/tests/integration/test_table_functions_access_rights/test.py +++ b/tests/integration/test_table_functions_access_rights/test.py @@ -3,7 +3,7 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance') +instance = cluster.add_instance("instance") @pytest.fixture(scope="module", autouse=True) @@ -11,8 +11,12 @@ def started_cluster(): try: cluster.start() - instance.query("CREATE TABLE table1(x UInt32) ENGINE = MergeTree ORDER BY tuple()") - instance.query("CREATE TABLE table2(x UInt32) ENGINE = MergeTree ORDER BY tuple()") + instance.query( + "CREATE TABLE table1(x UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) + instance.query( + "CREATE TABLE table2(x UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) instance.query("INSERT INTO table1 VALUES (1)") instance.query("INSERT INTO table2 VALUES (2)") @@ -35,21 +39,29 @@ def test_merge(): assert instance.query(select_query) == "1\n2\n" instance.query("CREATE USER A") - assert "it's necessary to have grant CREATE TEMPORARY TABLE ON *.*" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant CREATE TEMPORARY TABLE ON *.*" + in instance.query_and_get_error(select_query, user="A") + ) instance.query("GRANT CREATE TEMPORARY TABLE ON *.* TO A") - assert "no tables in database matches" in instance.query_and_get_error(select_query, user = 'A') + assert "no tables in database matches" in instance.query_and_get_error( + select_query, user="A" + ) instance.query("GRANT SELECT ON default.table1 TO A") - assert instance.query(select_query, user = 'A') == "1\n" + assert instance.query(select_query, user="A") == "1\n" instance.query("GRANT SELECT ON default.* TO A") - assert instance.query(select_query, user = 'A') == "1\n2\n" + assert instance.query(select_query, user="A") == "1\n2\n" instance.query("REVOKE SELECT ON default.table1 FROM A") - assert instance.query(select_query, user = 'A') == "2\n" + assert instance.query(select_query, user="A") == "2\n" instance.query("REVOKE ALL ON default.* FROM A") instance.query("GRANT SELECT ON default.table1 TO A") instance.query("GRANT INSERT ON default.table2 TO A") - assert "it's necessary to have grant SELECT ON default.table2" in instance.query_and_get_error(select_query, user = 'A') + assert ( + "it's necessary to have grant SELECT ON default.table2" + in instance.query_and_get_error(select_query, user="A") + ) diff --git a/tests/integration/test_tcp_handler_http_responses/test_case.py b/tests/integration/test_tcp_handler_http_responses/test_case.py index 38b5ba909a7..2fc53674ca4 100644 --- a/tests/integration/test_tcp_handler_http_responses/test_case.py +++ b/tests/integration/test_tcp_handler_http_responses/test_case.py @@ -7,16 +7,15 @@ import requests cluster = ClickHouseCluster(__file__) node_with_http = cluster.add_instance( - 'node_with_http', - main_configs=["configs/config.d/http-port-31337.xml"] + "node_with_http", main_configs=["configs/config.d/http-port-31337.xml"] ) HTTP_PORT = 31337 node_without_http = cluster.add_instance( - 'node_without_http', - main_configs=["configs/config.d/no-http-port.xml"] + "node_without_http", main_configs=["configs/config.d/no-http-port.xml"] ) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -26,17 +25,15 @@ def start_cluster(): finally: cluster.shutdown() + def test_request_to_http_full_instance(start_cluster): - response = requests.get( - f'http://{node_with_http.ip_address}:9000' - ) + response = requests.get(f"http://{node_with_http.ip_address}:9000") assert response.status_code == 400 assert str(HTTP_PORT) in response.text + def test_request_to_http_less_instance(start_cluster): - response = requests.post( - f'http://{node_without_http.ip_address}:9000' - ) + response = requests.post(f"http://{node_without_http.ip_address}:9000") assert response.status_code == 400 assert str(HTTP_PORT) not in response.text - assert "8123" not in response.text + assert "8123" not in response.text diff --git a/tests/integration/test_text_log_level/test.py b/tests/integration/test_text_log_level/test.py index 44679481266..dc0ae6333d6 100644 --- a/tests/integration/test_text_log_level/test.py +++ b/tests/integration/test_text_log_level/test.py @@ -7,10 +7,10 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=["configs/config.d/text_log.xml"]) +node = cluster.add_instance("node", main_configs=["configs/config.d/text_log.xml"]) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def start_cluster(): try: cluster.start() @@ -23,11 +23,27 @@ def start_cluster(): def test_basic(start_cluster): with pytest.raises(QueryRuntimeException): # generates log with "Error" level - node.query('SELECT * FROM no_such_table') + node.query("SELECT * FROM no_such_table") - node.query('SYSTEM FLUSH LOGS') + node.query("SYSTEM FLUSH LOGS") - assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Trace'")) == 0 - assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Debug'")) == 0 - assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Information'")) >= 1 - assert int(node.query("SELECT count() FROM system.text_log WHERE level = 'Error'")) >= 1 + assert ( + int(node.query("SELECT count() FROM system.text_log WHERE level = 'Trace'")) + == 0 + ) + assert ( + int(node.query("SELECT count() FROM system.text_log WHERE level = 'Debug'")) + == 0 + ) + assert ( + int( + node.query( + "SELECT count() FROM system.text_log WHERE level = 'Information'" + ) + ) + >= 1 + ) + assert ( + int(node.query("SELECT count() FROM system.text_log WHERE level = 'Error'")) + >= 1 + ) diff --git a/tests/integration/test_timezone_config/test.py b/tests/integration/test_timezone_config/test.py index ac12eddc709..e4a9f75abab 100644 --- a/tests/integration/test_timezone_config/test.py +++ b/tests/integration/test_timezone_config/test.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/config.xml']) +node = cluster.add_instance("node", main_configs=["configs/config.xml"]) @pytest.fixture(scope="module") @@ -17,3 +17,53 @@ def start_cluster(): def test_check_timezone_config(start_cluster): assert node.query("SELECT toDateTime(1111111111)") == "2005-03-17 17:58:31\n" + + +def test_overflow_toDate(start_cluster): + assert node.query("SELECT toDate('2999-12-31','UTC')") == "2149-06-06\n" + assert node.query("SELECT toDate('2021-12-21','UTC')") == "2021-12-21\n" + assert node.query("SELECT toDate('1000-12-31','UTC')") == "1970-01-01\n" + + +def test_overflow_toDate32(start_cluster): + assert node.query("SELECT toDate32('2999-12-31','UTC')") == "2283-11-11\n" + assert node.query("SELECT toDate32('2021-12-21','UTC')") == "2021-12-21\n" + assert node.query("SELECT toDate32('1000-12-31','UTC')") == "1925-01-01\n" + + +def test_overflow_toDateTime(start_cluster): + assert ( + node.query("SELECT toDateTime('2999-12-31 00:00:00','UTC')") + == "2106-02-07 06:28:15\n" + ) + assert ( + node.query("SELECT toDateTime('2106-02-07 06:28:15','UTC')") + == "2106-02-07 06:28:15\n" + ) + assert ( + node.query("SELECT toDateTime('1970-01-01 00:00:00','UTC')") + == "1970-01-01 00:00:00\n" + ) + assert ( + node.query("SELECT toDateTime('1000-01-01 00:00:00','UTC')") + == "1970-01-01 00:00:00\n" + ) + + +def test_overflow_parseDateTimeBestEffort(start_cluster): + assert ( + node.query("SELECT parseDateTimeBestEffort('2999-12-31 00:00:00','UTC')") + == "2106-02-07 06:28:15\n" + ) + assert ( + node.query("SELECT parseDateTimeBestEffort('2106-02-07 06:28:15','UTC')") + == "2106-02-07 06:28:15\n" + ) + assert ( + node.query("SELECT parseDateTimeBestEffort('1970-01-01 00:00:00','UTC')") + == "1970-01-01 00:00:00\n" + ) + assert ( + node.query("SELECT parseDateTimeBestEffort('1000-01-01 00:00:00','UTC')") + == "1970-01-01 00:00:00\n" + ) diff --git a/tests/integration/test_tmp_policy/test.py b/tests/integration/test_tmp_policy/test.py index f7174c3b695..c919d9a0c3d 100644 --- a/tests/integration/test_tmp_policy/test.py +++ b/tests/integration/test_tmp_policy/test.py @@ -7,12 +7,14 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', - main_configs=["configs/config.d/storage_configuration.xml"], - tmpfs=['/disk1:size=100M', '/disk2:size=100M']) +node = cluster.add_instance( + "node", + main_configs=["configs/config.d/storage_configuration.xml"], + tmpfs=["/disk1:size=100M", "/disk2:size=100M"], +) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def start_cluster(): try: cluster.start() @@ -22,15 +24,19 @@ def start_cluster(): def test_different_versions(start_cluster): - query = 'SELECT count(ignore(*)) FROM (SELECT * FROM system.numbers LIMIT 1e7) GROUP BY number' + query = "SELECT count(ignore(*)) FROM (SELECT * FROM system.numbers LIMIT 1e7) GROUP BY number" settings = { - 'max_bytes_before_external_group_by': 1 << 20, - 'max_bytes_before_external_sort': 1 << 20, + "max_bytes_before_external_group_by": 1 << 20, + "max_bytes_before_external_sort": 1 << 20, } - assert node.contains_in_log('Setting up /disk1/ to store temporary data in it') - assert node.contains_in_log('Setting up /disk2/ to store temporary data in it') + assert node.contains_in_log("Setting up /disk1/ to store temporary data in it") + assert node.contains_in_log("Setting up /disk2/ to store temporary data in it") node.query(query, settings=settings) - assert node.contains_in_log('Writing part of aggregation data into temporary file /disk1/') - assert node.contains_in_log('Writing part of aggregation data into temporary file /disk2/') + assert node.contains_in_log( + "Writing part of aggregation data into temporary file /disk1/" + ) + assert node.contains_in_log( + "Writing part of aggregation data into temporary file /disk2/" + ) diff --git a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml index a76e984e4e6..ae1dc9dd038 100644 --- a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml @@ -76,6 +76,14 @@ + + +
+ jbod1 +
+ + +
diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 254447478f9..49d7ab4f2fc 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -15,21 +15,31 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', - main_configs=['configs/logs_config.xml', "configs/config.d/instant_moves.xml", - "configs/config.d/storage_configuration.xml", - "configs/config.d/cluster.xml", ], - with_zookeeper=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 1}) +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs/logs_config.xml", + "configs/config.d/instant_moves.xml", + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", + ], + with_zookeeper=True, + tmpfs=["/jbod1:size=40M", "/jbod2:size=40M", "/external:size=200M"], + macros={"shard": 0, "replica": 1}, +) -node2 = cluster.add_instance('node2', - main_configs=['configs/logs_config.xml', "configs/config.d/instant_moves.xml", - "configs/config.d/storage_configuration.xml", - "configs/config.d/cluster.xml", ], - with_zookeeper=True, - tmpfs=['/jbod1:size=40M', '/jbod2:size=40M', '/external:size=200M'], - macros={"shard": 0, "replica": 2}) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs/logs_config.xml", + "configs/config.d/instant_moves.xml", + "configs/config.d/storage_configuration.xml", + "configs/config.d/cluster.xml", + ], + with_zookeeper=True, + tmpfs=["/jbod1:size=40M", "/jbod2:size=40M", "/external:size=200M"], + macros={"shard": 0, "replica": 2}, +) @pytest.fixture(scope="module") @@ -47,12 +57,20 @@ def get_used_disks_for_table(node, table_name, partition=None): suffix = "" else: suffix = "and partition='{}'".format(partition) - return node.query(""" + return ( + node.query( + """ SELECT disk_name FROM system.parts WHERE table == '{name}' AND active=1 {suffix} ORDER BY modification_time - """.format(name=table_name, suffix=suffix)).strip().split('\n') + """.format( + name=table_name, suffix=suffix + ) + ) + .strip() + .split("\n") + ) def check_used_disks_with_retry(node, table_name, expected_disks, retries): @@ -63,33 +81,55 @@ def check_used_disks_with_retry(node, table_name, expected_disks, retries): time.sleep(0.5) return False + # Use unique table name for flaky checker, that run tests multiple times def unique_table_name(base_name): - return f'{base_name}_{int(time.time())}' + return f"{base_name}_{int(time.time())}" + def wait_parts_mover(node, table, *args, **kwargs): # wait for MergeTreePartsMover - assert_logs_contain_with_retry(node, f'default.{table}.*Removed part from old location', *args, **kwargs) + assert_logs_contain_with_retry( + node, f"default.{table}.*Removed part from old location", *args, **kwargs + ) -@pytest.mark.parametrize("name,engine,alter", [ - pytest.param("mt_test_rule_with_invalid_destination", "MergeTree()", 0, id="case0"), - pytest.param("replicated_mt_test_rule_with_invalid_destination", - "ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')", 0, id="case1"), - pytest.param("mt_test_rule_with_invalid_destination", "MergeTree()", 1, id="case2"), - pytest.param("replicated_mt_test_rule_with_invalid_destination", - "ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')", 1, id="case3"), -]) +@pytest.mark.parametrize( + "name,engine,alter", + [ + pytest.param( + "mt_test_rule_with_invalid_destination", "MergeTree()", 0, id="case0" + ), + pytest.param( + "replicated_mt_test_rule_with_invalid_destination", + "ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')", + 0, + id="case1", + ), + pytest.param( + "mt_test_rule_with_invalid_destination", "MergeTree()", 1, id="case2" + ), + pytest.param( + "replicated_mt_test_rule_with_invalid_destination", + "ReplicatedMergeTree('/clickhouse/replicated_test_rule_with_invalid_destination', '1')", + 1, + id="case3", + ), + ], +) def test_rule_with_invalid_destination(started_cluster, name, engine, alter): name = unique_table_name(name) try: + def get_command(x, policy): x = x or "" if alter and x: return """ ALTER TABLE {name} MODIFY TTL {expression} - """.format(expression=x, name=name) + """.format( + expression=x, name=name + ) else: return """ CREATE TABLE {name} ( @@ -99,13 +139,17 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): ORDER BY tuple() {expression} SETTINGS storage_policy='{policy}' - """.format(expression=x, name=name, engine=engine, policy=policy) + """.format( + expression=x, name=name, engine=engine, policy=policy + ) if alter: node1.query(get_command(None, "small_jbod_with_external")) with pytest.raises(QueryRuntimeException): - node1.query(get_command("TTL d1 TO DISK 'unknown'", "small_jbod_with_external")) + node1.query( + get_command("TTL d1 TO DISK 'unknown'", "small_jbod_with_external") + ) node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) @@ -113,7 +157,9 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): node1.query(get_command(None, "small_jbod_with_external")) with pytest.raises(QueryRuntimeException): - node1.query(get_command("TTL d1 TO VOLUME 'unknown'", "small_jbod_with_external")) + node1.query( + get_command("TTL d1 TO VOLUME 'unknown'", "small_jbod_with_external") + ) node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) @@ -135,19 +181,41 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_inserts_to_disk_do_not_work", "MergeTree()", 0, id="mt_test_inserts_to_disk_do_not_work"), - pytest.param("replicated_mt_test_inserts_to_disk_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')", 0, id="replicated_mt_test_inserts_to_disk_do_not_work"), - pytest.param("mt_test_inserts_to_disk_work", "MergeTree()", 1, id="mt_test_inserts_to_disk_work_1"), - pytest.param("replicated_mt_test_inserts_to_disk_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_work', '1')", 1, id="replicated_mt_test_inserts_to_disk_work_1"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param( + "mt_test_inserts_to_disk_do_not_work", + "MergeTree()", + 0, + id="mt_test_inserts_to_disk_do_not_work", + ), + pytest.param( + "replicated_mt_test_inserts_to_disk_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_do_not_work', '1')", + 0, + id="replicated_mt_test_inserts_to_disk_do_not_work", + ), + pytest.param( + "mt_test_inserts_to_disk_work", + "MergeTree()", + 1, + id="mt_test_inserts_to_disk_work_1", + ), + pytest.param( + "replicated_mt_test_inserts_to_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_disk_work', '1')", + 1, + id="replicated_mt_test_inserts_to_disk_work_1", + ), + ], +) def test_inserts_to_disk_work(started_cluster, name, engine, positive): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -155,18 +223,33 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): ORDER BY tuple() TTL d1 TO DISK 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) data = [] # 10MB in total for i in range(10): - data.append(("randomPrintableASCII(1024*1024)", "toDateTime({})".format( - time.time() - 1 if i > 0 or positive else time.time() + 300))) + data.append( + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format( + time.time() - 1 if i > 0 or positive else time.time() + 300 + ), + ) + ) - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: try: @@ -175,30 +258,49 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): pass -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_test_moves_work_after_storage_policy_change", "MergeTree()", id="mt_test_moves_work_after_storage_policy_change"), - pytest.param("replicated_mt_test_moves_work_after_storage_policy_change", - "ReplicatedMergeTree('/clickhouse/test_moves_work_after_storage_policy_change', '1')", id="replicated_mt_test_moves_work_after_storage_policy_change"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param( + "mt_test_moves_work_after_storage_policy_change", + "MergeTree()", + id="mt_test_moves_work_after_storage_policy_change", + ), + pytest.param( + "replicated_mt_test_moves_work_after_storage_policy_change", + "ReplicatedMergeTree('/clickhouse/test_moves_work_after_storage_policy_change', '1')", + id="replicated_mt_test_moves_work_after_storage_policy_change", + ), + ], +) def test_moves_work_after_storage_policy_change(started_cluster, name, engine): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime ) ENGINE = {engine} ORDER BY tuple() - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query( """ALTER TABLE {name} MODIFY SETTING storage_policy='default_with_small_jbod_with_external'""".format( - name=name)) + name=name + ) + ) # Second expression is preferred because d1 > now()-3600. node1.query( - """ALTER TABLE {name} MODIFY TTL now()-3600 TO DISK 'jbod1', d1 TO DISK 'external'""".format(name=name)) + """ALTER TABLE {name} MODIFY TTL now()-3600 TO DISK 'jbod1', d1 TO DISK 'external'""".format( + name=name + ) + ) wait_expire_1 = 12 wait_expire_2 = 4 @@ -206,9 +308,15 @@ def test_moves_work_after_storage_policy_change(started_cluster, name, engine): data = [] # 10MB in total for i in range(10): - data.append(("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time_1))) + data.append( + ("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time_1)) + ) - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} @@ -217,25 +325,49 @@ def test_moves_work_after_storage_policy_change(started_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_moves_to_disk_do_not_work", "MergeTree()", 0, id="mt_test_moves_to_disk_do_not_work"), - pytest.param("replicated_mt_test_moves_to_disk_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')", 0, id="replicated_mt_test_moves_to_disk_do_not_work"), - pytest.param("mt_test_moves_to_disk_work", "MergeTree()", 1, id="mt_test_moves_to_disk_work"), - pytest.param("replicated_mt_test_moves_to_disk_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_work', '1')", 1, id="replicated_mt_test_moves_to_disk_work"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param( + "mt_test_moves_to_disk_do_not_work", + "MergeTree()", + 0, + id="mt_test_moves_to_disk_do_not_work", + ), + pytest.param( + "replicated_mt_test_moves_to_disk_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_do_not_work', '1')", + 0, + id="replicated_mt_test_moves_to_disk_do_not_work", + ), + pytest.param( + "mt_test_moves_to_disk_work", + "MergeTree()", + 1, + id="mt_test_moves_to_disk_work", + ), + pytest.param( + "replicated_mt_test_moves_to_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_work', '1')", + 1, + id="replicated_mt_test_moves_to_disk_work", + ), + ], +) def test_moves_to_disk_work(started_cluster, name, engine, positive): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -243,22 +375,35 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): ORDER BY tuple() TTL d1 TO DISK 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) wait_expire_1 = 12 wait_expire_2 = 20 time_1 = time.time() + wait_expire_1 time_2 = time.time() + wait_expire_1 + wait_expire_2 - wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread = threading.Thread( + target=time.sleep, args=(wait_expire_1,) + ) wait_expire_1_thread.start() data = [] # 10MB in total for i in range(10): - data.append(("randomPrintableASCII(1024*1024)", - "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) + data.append( + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time_1 if i > 0 or positive else time_2), + ) + ) - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} @@ -268,22 +413,35 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_test_moves_to_volume_work", "MergeTree()", id="mt_test_moves_to_volume_work"), - pytest.param("replicated_mt_test_moves_to_volume_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')", id="replicated_mt_test_moves_to_volume_work"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param( + "mt_test_moves_to_volume_work", + "MergeTree()", + id="mt_test_moves_to_volume_work", + ), + pytest.param( + "replicated_mt_test_moves_to_volume_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_volume_work', '1')", + id="replicated_mt_test_moves_to_volume_work", + ), + ], +) def test_moves_to_volume_work(started_cluster, name, engine): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( p1 Int64, s1 String, @@ -293,7 +451,10 @@ def test_moves_to_volume_work(started_cluster, name, engine): PARTITION BY p1 TTL d1 TO VOLUME 'external' SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) wait_expire_1 = 10 time_1 = time.time() + wait_expire_1 @@ -302,38 +463,70 @@ def test_moves_to_volume_work(started_cluster, name, engine): data = [] # 10MB in total for i in range(5): data.append( - (str(p), "randomPrintableASCII(1024*1024)", "toDateTime({})".format(time_1))) + ( + str(p), + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time_1), + ) + ) node1.query( - "INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + "INSERT INTO {} (p1, s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) - assert set(used_disks) == {'jbod1', 'jbod2'} + assert set(used_disks) == {"jbod1", "jbod2"} wait_parts_mover(node1, name, retry_count=40) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_inserts_to_volume_do_not_work", "MergeTree()", 0, id="mt_test_inserts_to_volume_do_not_work"), - pytest.param("replicated_mt_test_inserts_to_volume_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')", 0, id="replicated_mt_test_inserts_to_volume_do_not_work"), - pytest.param("mt_test_inserts_to_volume_work", "MergeTree()", 1, id="mt_test_inserts_to_volume_work"), - pytest.param("replicated_mt_test_inserts_to_volume_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_work', '1')", 1, id="replicated_mt_test_inserts_to_volume_work"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param( + "mt_test_inserts_to_volume_do_not_work", + "MergeTree()", + 0, + id="mt_test_inserts_to_volume_do_not_work", + ), + pytest.param( + "replicated_mt_test_inserts_to_volume_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_do_not_work', '1')", + 0, + id="replicated_mt_test_inserts_to_volume_do_not_work", + ), + pytest.param( + "mt_test_inserts_to_volume_work", + "MergeTree()", + 1, + id="mt_test_inserts_to_volume_work", + ), + pytest.param( + "replicated_mt_test_inserts_to_volume_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_inserts_to_volume_work', '1')", + 1, + id="replicated_mt_test_inserts_to_volume_work", + ), + ], +) def test_inserts_to_volume_work(started_cluster, name, engine, positive): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( p1 Int64, s1 String, @@ -343,56 +536,90 @@ def test_inserts_to_volume_work(started_cluster, name, engine, positive): PARTITION BY p1 TTL d1 TO VOLUME 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query("SYSTEM STOP MOVES {name}".format(name=name)) for p in range(2): data = [] # 20MB in total for i in range(10): - data.append((str(p), "randomPrintableASCII(1024*1024)", "toDateTime({})".format( - time.time() - 1 if i > 0 or positive else time.time() + 300))) + data.append( + ( + str(p), + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format( + time.time() - 1 if i > 0 or positive else time.time() + 300 + ), + ) + ) node1.query( - "INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + "INSERT INTO {} (p1, s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "20" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "20" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_test_moves_to_disk_eventually_work", "MergeTree()", id="mt_test_moves_to_disk_eventually_work"), - pytest.param("replicated_mt_test_moves_to_disk_eventually_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')", id="replicated_mt_test_moves_to_disk_eventually_work"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param( + "mt_test_moves_to_disk_eventually_work", + "MergeTree()", + id="mt_test_moves_to_disk_eventually_work", + ), + pytest.param( + "replicated_mt_test_moves_to_disk_eventually_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_to_disk_eventually_work', '1')", + id="replicated_mt_test_moves_to_disk_eventually_work", + ), + ], +) def test_moves_to_disk_eventually_work(started_cluster, name, engine): name = unique_table_name(name) try: name_temp = name + "_temp" - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String ) ENGINE = MergeTree() ORDER BY tuple() SETTINGS storage_policy='only_jbod2' - """.format(name=name_temp)) + """.format( + name=name_temp + ) + ) data = [] # 35MB in total for i in range(35): data.append("randomPrintableASCII(1024*1024)") - node1.query("INSERT INTO {} VALUES {}".format(name_temp, ",".join(["(" + x + ")" for x in data]))) + node1.query( + "INSERT INTO {} VALUES {}".format( + name_temp, ",".join(["(" + x + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name_temp) assert set(used_disks) == {"jbod2"} - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -400,14 +627,25 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): ORDER BY tuple() TTL d1 TO DISK 'jbod2' SETTINGS storage_policy='jbod1_with_jbod2' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) data = [] # 10MB in total for i in range(10): data.append( - ("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time.time() - 1))) + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time.time() - 1), + ) + ) - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} @@ -418,7 +656,9 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod2"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name_temp)) @@ -430,7 +670,8 @@ def test_replicated_download_ttl_info(started_cluster): engine = "ReplicatedMergeTree('/clickhouse/test_replicated_download_ttl_info', '{replica}')" try: for i, node in enumerate((node1, node2), start=1): - node.query(""" + node.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -438,11 +679,18 @@ def test_replicated_download_ttl_info(started_cluster): ORDER BY tuple() TTL d1 TO DISK 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query("SYSTEM STOP MOVES {}".format(name)) - node2.query("INSERT INTO {} (s1, d1) VALUES (randomPrintableASCII(1024*1024), toDateTime({}))".format(name, time.time() - 100)) + node2.query( + "INSERT INTO {} (s1, d1) VALUES (randomPrintableASCII(1024*1024), toDateTime({}))".format( + name, time.time() - 100 + ) + ) assert set(get_used_disks_for_table(node2, name)) == {"external"} @@ -459,19 +707,41 @@ def test_replicated_download_ttl_info(started_cluster): continue -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_merges_to_disk_do_not_work", "MergeTree()", 0, id="mt_test_merges_to_disk_do_not_work"), - pytest.param("replicated_mt_test_merges_to_disk_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')", 0, id="mt_test_merges_to_disk_do_not_work"), - pytest.param("mt_test_merges_to_disk_work", "MergeTree()", 1, id="mt_test_merges_to_disk_work"), - pytest.param("replicated_mt_test_merges_to_disk_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_work', '1')", 1, id="replicated_mt_test_merges_to_disk_work"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param( + "mt_test_merges_to_disk_do_not_work", + "MergeTree()", + 0, + id="mt_test_merges_to_disk_do_not_work", + ), + pytest.param( + "replicated_mt_test_merges_to_disk_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_do_not_work', '1')", + 0, + id="mt_test_merges_to_disk_do_not_work", + ), + pytest.param( + "mt_test_merges_to_disk_work", + "MergeTree()", + 1, + id="mt_test_merges_to_disk_work", + ), + pytest.param( + "replicated_mt_test_merges_to_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_merges_to_disk_work', '1')", + 1, + id="replicated_mt_test_merges_to_disk_work", + ), + ], +) def test_merges_to_disk_work(started_cluster, name, engine, positive): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -479,7 +749,10 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): ORDER BY tuple() TTL d1 TO DISK 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) node1.query("SYSTEM STOP MERGES {}".format(name)) node1.query("SYSTEM STOP MOVES {}".format(name)) @@ -489,22 +762,39 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): time_1 = time.time() + wait_expire_1 time_2 = time.time() + wait_expire_1 + wait_expire_2 - wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread = threading.Thread( + target=time.sleep, args=(wait_expire_1,) + ) wait_expire_1_thread.start() for _ in range(2): data = [] # 16MB in total for i in range(8): - data.append(("randomPrintableASCII(1024*1024)", - "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) + data.append( + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format( + time_1 if i > 0 or positive else time_2 + ), + ) + ) node1.query( - "INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert "2" == node1.query( - "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert ( + "2" + == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format( + name + ) + ).strip() + ) wait_expire_1_thread.join() time.sleep(wait_expire_2 / 2) @@ -514,43 +804,70 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} - assert "1" == node1.query( - "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert ( + "1" + == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format( + name + ) + ).strip() + ) - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "16" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "16" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_test_merges_with_full_disk_work", "MergeTree()", id="mt_test_merges_with_full_disk_work"), - pytest.param("replicated_mt_test_merges_with_full_disk_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')", id="replicated_mt_test_merges_with_full_disk_work"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param( + "mt_test_merges_with_full_disk_work", + "MergeTree()", + id="mt_test_merges_with_full_disk_work", + ), + pytest.param( + "replicated_mt_test_merges_with_full_disk_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_merges_with_full_disk_work', '1')", + id="replicated_mt_test_merges_with_full_disk_work", + ), + ], +) def test_merges_with_full_disk_work(started_cluster, name, engine): name = unique_table_name(name) try: name_temp = name + "_temp" - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String ) ENGINE = MergeTree() ORDER BY tuple() SETTINGS storage_policy='only_jbod2' - """.format(name=name_temp)) + """.format( + name=name_temp + ) + ) data = [] # 35MB in total for i in range(35): data.append("randomPrintableASCII(1024*1024)") - node1.query("INSERT INTO {} VALUES {}".format(name_temp, ",".join(["(" + x + ")" for x in data]))) + node1.query( + "INSERT INTO {} VALUES {}".format( + name_temp, ",".join(["(" + x + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name_temp) assert set(used_disks) == {"jbod2"} - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -558,25 +875,41 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): ORDER BY tuple() TTL d1 TO DISK 'jbod2' SETTINGS storage_policy='jbod1_with_jbod2' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) wait_expire_1 = 10 time_1 = time.time() + wait_expire_1 - wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread = threading.Thread( + target=time.sleep, args=(wait_expire_1,) + ) wait_expire_1_thread.start() for _ in range(2): data = [] # 12MB in total for i in range(6): - data.append(("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time_1))) # 1MB row + data.append( + ("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time_1)) + ) # 1MB row node1.query( - "INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert "2" == node1.query( - "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert ( + "2" + == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format( + name + ) + ).strip() + ) wait_expire_1_thread.join() @@ -585,29 +918,59 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} # Merged to the same disk against the rule. - assert "1" == node1.query( - "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert ( + "1" + == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format( + name + ) + ).strip() + ) - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "12" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "12" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name_temp)) node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_moves_after_merges_do_not_work", "MergeTree()", 0, id="mt_test_moves_after_merges_do_not_work"), - pytest.param("replicated_mt_test_moves_after_merges_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')", 0, id="replicated_mt_test_moves_after_merges_do_not_work"), - pytest.param("mt_test_moves_after_merges_work", "MergeTree()", 1, id="mt_test_moves_after_merges_work"), - pytest.param("replicated_mt_test_moves_after_merges_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_work', '1')", 1, id="replicated_mt_test_moves_after_merges_work"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param( + "mt_test_moves_after_merges_do_not_work", + "MergeTree()", + 0, + id="mt_test_moves_after_merges_do_not_work", + ), + pytest.param( + "replicated_mt_test_moves_after_merges_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_do_not_work', '1')", + 0, + id="replicated_mt_test_moves_after_merges_do_not_work", + ), + pytest.param( + "mt_test_moves_after_merges_work", + "MergeTree()", + 1, + id="mt_test_moves_after_merges_work", + ), + pytest.param( + "replicated_mt_test_moves_after_merges_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_merges_work', '1')", + 1, + id="replicated_mt_test_moves_after_merges_work", + ), + ], +) def test_moves_after_merges_work(started_cluster, name, engine, positive): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -615,31 +978,51 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): ORDER BY tuple() TTL d1 TO DISK 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) wait_expire_1 = 16 wait_expire_2 = 20 time_1 = time.time() + wait_expire_1 time_2 = time.time() + wait_expire_1 + wait_expire_2 - wait_expire_1_thread = threading.Thread(target=time.sleep, args=(wait_expire_1,)) + wait_expire_1_thread = threading.Thread( + target=time.sleep, args=(wait_expire_1,) + ) wait_expire_1_thread.start() for _ in range(2): data = [] # 14MB in total for i in range(7): - data.append(("randomPrintableASCII(1024*1024)", - "toDateTime({})".format(time_1 if i > 0 or positive else time_2))) # 1MB row + data.append( + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format( + time_1 if i > 0 or positive else time_2 + ), + ) + ) # 1MB row node1.query( - "INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) node1.query("OPTIMIZE TABLE {}".format(name)) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert "1" == node1.query( - "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format(name)).strip() + assert ( + "1" + == node1.query( + "SELECT count() FROM system.parts WHERE table = '{}' AND active = 1".format( + name + ) + ).strip() + ) wait_expire_1_thread.join() time.sleep(wait_expire_2 / 2) @@ -647,31 +1030,81 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external" if positive else "jbod1"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "14" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "14" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine,positive,bar", [ - pytest.param("mt_test_moves_after_alter_do_not_work", "MergeTree()", 0, "DELETE", id="mt_negative"), - pytest.param("replicated_mt_test_moves_after_alter_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')", 0, "DELETE", id="repicated_negative"), - pytest.param("mt_test_moves_after_alter_work", "MergeTree()", 1, "DELETE", id="mt_positive"), - pytest.param("replicated_mt_test_moves_after_alter_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')", 1, "DELETE", id="repicated_positive"), - pytest.param("mt_test_moves_after_alter_do_not_work", "MergeTree()", 0, "TO DISK 'external'", id="mt_external_negative"), - pytest.param("replicated_mt_test_moves_after_alter_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')", 0, "TO DISK 'external'", id="replicated_external_negative"), - pytest.param("mt_test_moves_after_alter_work", "MergeTree()", 1, "TO DISK 'external'", id="mt_external_positive"), - pytest.param("replicated_mt_test_moves_after_alter_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')", 1, "TO DISK 'external'", id="replicated_external_positive"), -]) +@pytest.mark.parametrize( + "name,engine,positive,bar", + [ + pytest.param( + "mt_test_moves_after_alter_do_not_work", + "MergeTree()", + 0, + "DELETE", + id="mt_negative", + ), + pytest.param( + "replicated_mt_test_moves_after_alter_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')", + 0, + "DELETE", + id="repicated_negative", + ), + pytest.param( + "mt_test_moves_after_alter_work", + "MergeTree()", + 1, + "DELETE", + id="mt_positive", + ), + pytest.param( + "replicated_mt_test_moves_after_alter_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')", + 1, + "DELETE", + id="repicated_positive", + ), + pytest.param( + "mt_test_moves_after_alter_do_not_work", + "MergeTree()", + 0, + "TO DISK 'external'", + id="mt_external_negative", + ), + pytest.param( + "replicated_mt_test_moves_after_alter_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_do_not_work', '1')", + 0, + "TO DISK 'external'", + id="replicated_external_negative", + ), + pytest.param( + "mt_test_moves_after_alter_work", + "MergeTree()", + 1, + "TO DISK 'external'", + id="mt_external_positive", + ), + pytest.param( + "replicated_mt_test_moves_after_alter_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_moves_after_alter_work', '1')", + 1, + "TO DISK 'external'", + id="replicated_external_positive", + ), + ], +) def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, bar): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -679,40 +1112,64 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, b ORDER BY tuple() TTL d1 TO DISK 'external' SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) if positive: - node1.query(""" + node1.query( + """ ALTER TABLE {name} MODIFY TTL d1 + INTERVAL 15 MINUTE {bar} - """.format(name=name, bar=bar)) # That shall disable TTL. + """.format( + name=name, bar=bar + ) + ) # That shall disable TTL. data = [] # 10MB in total for i in range(10): data.append( - ("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time.time() - 1))) # 1MB row - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time.time() - 1), + ) + ) # 1MB row + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1" if positive else "external"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine", [ - pytest.param("mt_test_materialize_ttl_in_partition", "MergeTree()", id="mt"), - pytest.param("replicated_mt_test_materialize_ttl_in_partition", - "ReplicatedMergeTree('/clickhouse/test_materialize_ttl_in_partition', '1')", id="replicated"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("mt_test_materialize_ttl_in_partition", "MergeTree()", id="mt"), + pytest.param( + "replicated_mt_test_materialize_ttl_in_partition", + "ReplicatedMergeTree('/clickhouse/test_materialize_ttl_in_partition', '1')", + id="replicated", + ), + ], +) def test_materialize_ttl_in_partition(started_cluster, name, engine): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( p1 Int8, s1 String, @@ -721,61 +1178,109 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): ORDER BY p1 PARTITION BY p1 SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) data = [] # 5MB in total for i in range(5): - data.append((str(i), "randomPrintableASCII(1024*1024)", - "toDateTime({})".format(time.time() - 1))) # 1MB row + data.append( + ( + str(i), + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time.time() - 1), + ) + ) # 1MB row node1.query( - "INSERT INTO {} (p1, s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + "INSERT INTO {} (p1, s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - node1.query(""" + node1.query( + """ ALTER TABLE {name} MODIFY TTL d1 TO DISK 'external' SETTINGS materialize_ttl_after_modify = 0 - """.format(name=name)) + """.format( + name=name + ) + ) time.sleep(3) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - node1.query(""" + node1.query( + """ ALTER TABLE {name} MATERIALIZE TTL IN PARTITION 2 - """.format(name=name)) + """.format( + name=name + ) + ) - node1.query(""" + node1.query( + """ ALTER TABLE {name} MATERIALIZE TTL IN PARTITION 4 - """.format(name=name)) + """.format( + name=name + ) + ) time.sleep(3) used_disks_sets = [] for i in range(len(data)): - used_disks_sets.append(set(get_used_disks_for_table(node1, name, partition=i))) + used_disks_sets.append( + set(get_used_disks_for_table(node1, name, partition=i)) + ) - assert used_disks_sets == [{"jbod1"}, {"jbod1"}, {"external"}, {"jbod1"}, {"external"}] + assert used_disks_sets == [ + {"jbod1"}, + {"jbod1"}, + {"external"}, + {"jbod1"}, + {"external"}, + ] - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == str(len(data)) + assert node1.query( + "SELECT count() FROM {name}".format(name=name) + ).strip() == str(len(data)) finally: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_alter_multiple_ttls_positive", "MergeTree()", True, id="positive"), - pytest.param("mt_replicated_test_alter_multiple_ttls_positive", - "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True, id="replicated_positive"), - pytest.param("mt_test_alter_multiple_ttls_negative", "MergeTree()", False, id="negative"), - pytest.param("mt_replicated_test_alter_multiple_ttls_negative", - "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_negative', '1')", False, id="replicated_negative"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param( + "mt_test_alter_multiple_ttls_positive", "MergeTree()", True, id="positive" + ), + pytest.param( + "mt_replicated_test_alter_multiple_ttls_positive", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", + True, + id="replicated_positive", + ), + pytest.param( + "mt_test_alter_multiple_ttls_negative", "MergeTree()", False, id="negative" + ), + pytest.param( + "mt_replicated_test_alter_multiple_ttls_negative", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_negative', '1')", + False, + id="replicated_negative", + ), + ], +) def test_alter_multiple_ttls(started_cluster, name, engine, positive): name = unique_table_name(name) @@ -802,7 +1307,8 @@ limitations under the License.""" """ now = time.time() try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( p1 Int64, s1 String, @@ -813,14 +1319,21 @@ limitations under the License.""" TTL d1 + INTERVAL 34 SECOND TO DISK 'jbod2', d1 + INTERVAL 64 SECOND TO VOLUME 'external' SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0 - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - node1.query(""" + node1.query( + """ ALTER TABLE {name} MODIFY TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', d1 + INTERVAL 14 SECOND TO VOLUME 'external', d1 + INTERVAL 19 SECOND DELETE - """.format(name=name)) + """.format( + name=name + ) + ) for p in range(3): data = [] # 6MB in total @@ -828,13 +1341,23 @@ limitations under the License.""" for i in range(2): p1 = p d1 = now - 1 if i > 0 or positive else now + 300 - data.append("({}, randomPrintableASCII(1024*1024), toDateTime({}))".format(p1, d1)) - node1.query("INSERT INTO {name} (p1, s1, d1) VALUES {values}".format(name=name, values=",".join(data))) + data.append( + "({}, randomPrintableASCII(1024*1024), toDateTime({}))".format( + p1, d1 + ) + ) + node1.query( + "INSERT INTO {name} (p1, s1, d1) VALUES {values}".format( + name=name, values=",".join(data) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod2"} if positive else {"jbod1", "jbod2"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] + assert node1.query( + "SELECT count() FROM {name}".format(name=name) + ).splitlines() == ["6"] if positive: expected_disks = {"external"} @@ -843,12 +1366,16 @@ limitations under the License.""" check_used_disks_with_retry(node1, name, expected_disks, 50) - assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] + assert node1.query( + "SELECT count() FROM {name}".format(name=name) + ).splitlines() == ["6"] time.sleep(5) for i in range(50): - rows_count = int(node1.query("SELECT count() FROM {name}".format(name=name)).strip()) + rows_count = int( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() + ) if positive: if rows_count == 0: break @@ -867,16 +1394,23 @@ limitations under the License.""" node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) -@pytest.mark.parametrize("name,engine", [ - pytest.param("concurrently_altering_ttl_mt", "MergeTree()", id="mt"), - pytest.param("concurrently_altering_ttl_replicated_mt", - "ReplicatedMergeTree('/clickhouse/concurrently_altering_ttl_replicated_mt', '1')", id="replicated_mt"), -]) +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param("concurrently_altering_ttl_mt", "MergeTree()", id="mt"), + pytest.param( + "concurrently_altering_ttl_replicated_mt", + "ReplicatedMergeTree('/clickhouse/concurrently_altering_ttl_replicated_mt', '1')", + id="replicated_mt", + ), + ], +) def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( EventDate Date, number UInt64 @@ -884,7 +1418,10 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): ORDER BY tuple() PARTITION BY toYYYYMM(EventDate) SETTINGS storage_policy='jbods_with_external' - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) values = list({random.randint(1, 1000000) for _ in range(0, 1000)}) @@ -892,8 +1429,12 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): for i in range(num): day = random.randint(11, 30) value = values.pop() - month = '0' + str(random.choice([3, 4])) - node1.query("INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format(name, m=month, d=day, v=value)) + month = "0" + str(random.choice([3, 4])) + node1.query( + "INSERT INTO {} VALUES(toDate('2019-{m}-{d}'), {v})".format( + name, m=month, d=day, v=value + ) + ) def alter_move(num): def produce_alter_move(node, name): @@ -901,9 +1442,15 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): if move_type == "PART": for _ in range(10): try: - parts = node1.query( - "SELECT name from system.parts where table = '{}' and active = 1".format( - name)).strip().split('\n') + parts = ( + node1.query( + "SELECT name from system.parts where table = '{}' and active = 1".format( + name + ) + ) + .strip() + .split("\n") + ) break except QueryRuntimeException: pass @@ -920,8 +1467,15 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): else: move_volume = random.choice(["'main'", "'external'"]) try: - node1.query("ALTER TABLE {} MOVE {mt} {mp} TO {md} {mv}".format( - name, mt=move_type, mp=move_part, md=move_disk, mv=move_volume)) + node1.query( + "ALTER TABLE {} MOVE {mt} {mp} TO {md} {mv}".format( + name, + mt=move_type, + mp=move_part, + md=move_disk, + mv=move_volume, + ) + ) except QueryRuntimeException: pass @@ -931,7 +1485,9 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): def alter_update(num): for i in range(num): try: - node1.query("ALTER TABLE {} UPDATE number = number + 1 WHERE 1".format(name)) + node1.query( + "ALTER TABLE {} UPDATE number = number + 1 WHERE 1".format(name) + ) except: pass @@ -940,19 +1496,30 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): ttls = [] for j in range(random.randint(1, 10)): what = random.choice( - ["TO VOLUME 'main'", "TO VOLUME 'external'", "TO DISK 'jbod1'", "TO DISK 'jbod2'", - "TO DISK 'external'"]) + [ + "TO VOLUME 'main'", + "TO VOLUME 'external'", + "TO DISK 'jbod1'", + "TO DISK 'jbod2'", + "TO DISK 'external'", + ] + ) when = "now()+{}".format(random.randint(-1, 5)) ttls.append("{} {}".format(when, what)) try: - node1.query("ALTER TABLE {} MODIFY TTL {}".format(name, ", ".join(ttls))) + node1.query( + "ALTER TABLE {} MODIFY TTL {}".format(name, ", ".join(ttls)) + ) except QueryRuntimeException: pass def optimize_table(num): for i in range(num): try: # optimize may throw after concurrent alter - node1.query("OPTIMIZE TABLE {} FINAL".format(name), settings={'optimize_throw_if_noop': '1'}) + node1.query( + "OPTIMIZE TABLE {} FINAL".format(name), + settings={"optimize_throw_if_noop": "1"}, + ) break except: pass @@ -976,15 +1543,19 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): @pytest.mark.skip(reason="Flacky test") -@pytest.mark.parametrize("name,positive", [ - pytest.param("test_double_move_while_select_negative", 0, id="negative"), - pytest.param("test_double_move_while_select_positive", 1, id="positive"), -]) +@pytest.mark.parametrize( + "name,positive", + [ + pytest.param("test_double_move_while_select_negative", 0, id="negative"), + pytest.param("test_double_move_while_select_positive", 1, id="positive"), + ], +) def test_double_move_while_select(started_cluster, name, positive): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( n Int64, s String @@ -992,59 +1563,104 @@ def test_double_move_while_select(started_cluster, name, positive): ORDER BY tuple() PARTITION BY n SETTINGS storage_policy='small_jbod_with_external' - """.format(name=name)) + """.format( + name=name + ) + ) node1.query( - "INSERT INTO {name} VALUES (1, randomPrintableASCII(10*1024*1024))".format(name=name)) + "INSERT INTO {name} VALUES (1, randomPrintableASCII(10*1024*1024))".format( + name=name + ) + ) parts = node1.query( - "SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)).splitlines() + "SELECT name FROM system.parts WHERE table = '{name}' AND active = 1".format( + name=name + ) + ).splitlines() assert len(parts) == 1 - node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format(name=name, part=parts[0])) + node1.query( + "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'external'".format( + name=name, part=parts[0] + ) + ) def long_select(): if positive: - node1.query("SELECT sleep(3), sleep(2), sleep(1), n FROM {name}".format(name=name)) + node1.query( + "SELECT sleep(3), sleep(2), sleep(1), n FROM {name}".format( + name=name + ) + ) thread = threading.Thread(target=long_select) thread.start() time.sleep(1) - node1.query("ALTER TABLE {name} MOVE PART '{part}' TO DISK 'jbod1'".format(name=name, part=parts[0])) + node1.query( + "ALTER TABLE {name} MOVE PART '{part}' TO DISK 'jbod1'".format( + name=name, part=parts[0] + ) + ) # Fill jbod1 to force ClickHouse to make move of partition 1 to external. node1.query( - "INSERT INTO {name} VALUES (2, randomPrintableASCII(9*1024*1024))".format(name=name)) + "INSERT INTO {name} VALUES (2, randomPrintableASCII(9*1024*1024))".format( + name=name + ) + ) node1.query( - "INSERT INTO {name} VALUES (3, randomPrintableASCII(9*1024*1024))".format(name=name)) + "INSERT INTO {name} VALUES (3, randomPrintableASCII(9*1024*1024))".format( + name=name + ) + ) node1.query( - "INSERT INTO {name} VALUES (4, randomPrintableASCII(9*1024*1024))".format(name=name)) + "INSERT INTO {name} VALUES (4, randomPrintableASCII(9*1024*1024))".format( + name=name + ) + ) wait_parts_mover(node1, name, retry_count=40) # If SELECT locked old part on external, move shall fail. assert node1.query( - "SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'" - .format(name=name, part=parts[0])).splitlines() == ["jbod1" if positive else "external"] + "SELECT disk_name FROM system.parts WHERE table = '{name}' AND active = 1 AND name = '{part}'".format( + name=name, part=parts[0] + ) + ).splitlines() == ["jbod1" if positive else "external"] thread.join() - assert node1.query("SELECT n FROM {name} ORDER BY n".format(name=name)).splitlines() == ["1", "2", "3", "4"] + assert node1.query( + "SELECT n FROM {name} ORDER BY n".format(name=name) + ).splitlines() == ["1", "2", "3", "4"] finally: node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) -@pytest.mark.parametrize("name,engine,positive", [ - pytest.param("mt_test_alter_with_merge_do_not_work", "MergeTree()", 0, id="mt"), - pytest.param("replicated_mt_test_alter_with_merge_do_not_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_do_not_work', '1')", 0, id="replicated"), - pytest.param("mt_test_alter_with_merge_work", "MergeTree()", 1, id="mt_work"), - pytest.param("replicated_mt_test_alter_with_merge_work", - "ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_work', '1')", 1, id="replicated_work"), -]) +@pytest.mark.parametrize( + "name,engine,positive", + [ + pytest.param("mt_test_alter_with_merge_do_not_work", "MergeTree()", 0, id="mt"), + pytest.param( + "replicated_mt_test_alter_with_merge_do_not_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_do_not_work', '1')", + 0, + id="replicated", + ), + pytest.param("mt_test_alter_with_merge_work", "MergeTree()", 1, id="mt_work"), + pytest.param( + "replicated_mt_test_alter_with_merge_work", + "ReplicatedMergeTree('/clickhouse/replicated_test_alter_with_merge_work', '1')", + 1, + id="replicated_work", + ), + ], +) def test_alter_with_merge_work(started_cluster, name, engine, positive): name = unique_table_name(name) @@ -1063,7 +1679,8 @@ limitations under the License.""" and parts are merged. """ try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -1072,12 +1689,18 @@ limitations under the License.""" TTL d1 + INTERVAL 3000 SECOND TO DISK 'jbod2', d1 + INTERVAL 6000 SECOND TO VOLUME 'external' SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0 - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) def optimize_table(num): for i in range(num): try: # optimize may throw after concurrent alter - node1.query("OPTIMIZE TABLE {} FINAL".format(name), settings={'optimize_throw_if_noop': '1'}) + node1.query( + "OPTIMIZE TABLE {} FINAL".format(name), + settings={"optimize_throw_if_noop": "1"}, + ) break except: pass @@ -1087,26 +1710,44 @@ limitations under the License.""" now = time.time() for i in range(2): d1 = now - 1 if positive else now + 300 - data.append("(randomPrintableASCII(1024*1024), toDateTime({}))".format(d1)) + data.append( + "(randomPrintableASCII(1024*1024), toDateTime({}))".format(d1) + ) values = ",".join(data) - node1.query("INSERT INTO {name} (s1, d1) VALUES {values}".format(name=name, values=values)) + node1.query( + "INSERT INTO {name} (s1, d1) VALUES {values}".format( + name=name, values=values + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1", "jbod2"} - node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"] + node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == [ + "6" + ] - node1.query(""" + node1.query( + """ ALTER TABLE {name} MODIFY TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', d1 + INTERVAL 5 SECOND TO VOLUME 'external', d1 + INTERVAL 10 SECOND DELETE - """.format(name=name)) + """.format( + name=name + ) + ) optimize_table(20) - assert node1.query( - "SELECT count() FROM system.parts WHERE table = '{name}' AND active = 1".format(name=name)) == "1\n" + assert ( + node1.query( + "SELECT count() FROM system.parts WHERE table = '{name}' AND active = 1".format( + name=name + ) + ) + == "1\n" + ) time.sleep(5) @@ -1130,17 +1771,38 @@ limitations under the License.""" node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) -@pytest.mark.parametrize("name,dest_type,engine", [ - pytest.param("mt_test_disabled_ttl_move_on_insert_work", "DISK", "MergeTree()", id="disk"), - pytest.param("mt_test_disabled_ttl_move_on_insert_work", "VOLUME", "MergeTree()", id="volume"), - pytest.param("replicated_mt_test_disabled_ttl_move_on_insert_work", "DISK", "ReplicatedMergeTree('/clickhouse/replicated_test_disabled_ttl_move_on_insert_work', '1')", id="replicated_disk"), - pytest.param("replicated_mt_test_disabled_ttl_move_on_insert_work", "VOLUME", "ReplicatedMergeTree('/clickhouse/replicated_test_disabled_ttl_move_on_insert_work', '1')", id="replicated_volume"), -]) +@pytest.mark.parametrize( + "name,dest_type,engine", + [ + pytest.param( + "mt_test_disabled_ttl_move_on_insert_work", "DISK", "MergeTree()", id="disk" + ), + pytest.param( + "mt_test_disabled_ttl_move_on_insert_work", + "VOLUME", + "MergeTree()", + id="volume", + ), + pytest.param( + "replicated_mt_test_disabled_ttl_move_on_insert_work", + "DISK", + "ReplicatedMergeTree('/clickhouse/replicated_test_disabled_ttl_move_on_insert_work', '1')", + id="replicated_disk", + ), + pytest.param( + "replicated_mt_test_disabled_ttl_move_on_insert_work", + "VOLUME", + "ReplicatedMergeTree('/clickhouse/replicated_test_disabled_ttl_move_on_insert_work', '1')", + id="replicated_volume", + ), + ], +) def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): name = unique_table_name(name) try: - node1.query(""" + node1.query( + """ CREATE TABLE {name} ( s1 String, d1 DateTime @@ -1148,29 +1810,129 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): ORDER BY tuple() TTL d1 TO {dest_type} 'external' SETTINGS storage_policy='jbod_without_instant_ttl_move' - """.format(name=name, dest_type=dest_type, engine=engine)) + """.format( + name=name, dest_type=dest_type, engine=engine + ) + ) node1.query("SYSTEM STOP MOVES {}".format(name)) data = [] # 10MB in total for i in range(10): - data.append(("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time.time() - 1))) + data.append( + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time.time() - 1), + ) + ) - node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) node1.query("SYSTEM START MOVES {}".format(name)) time.sleep(3) used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"external"} - assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) finally: try: node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) except: pass + + +@pytest.mark.parametrize( + "name,dest_type", + [ + pytest.param("replicated_mt_move_if_exists", "DISK", id="replicated_disk"), + pytest.param("replicated_mt_move_if_exists", "VOLUME", id="replicated_volume"), + ], +) +def test_ttl_move_if_exists(started_cluster, name, dest_type): + name = unique_table_name(name) + + try: + query_template = """ + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_mt_move_if_exists', '{node_name}') + ORDER BY tuple() + TTL d1 TO {dest_type} {if_exists} 'external' + SETTINGS storage_policy='{policy}' + """ + + with pytest.raises(QueryRuntimeException): + node1.query( + query_template.format( + name=name, + node_name=node1.name, + dest_type=dest_type, + if_exists="", + policy="only_jbod_1", + ) + ) + + for (node, policy) in zip( + [node1, node2], ["only_jbod_1", "small_jbod_with_external"] + ): + node.query( + query_template.format( + name=name, + node_name=node.name, + dest_type=dest_type, + if_exists="IF EXISTS", + policy=policy, + ) + ) + + data = [] # 10MB in total + for i in range(10): + data.append( + ( + "randomPrintableASCII(1024*1024)", + "toDateTime({})".format(time.time() - 1), + ) + ) + + node1.query( + "INSERT INTO {} (s1, d1) VALUES {}".format( + name, ",".join(["(" + ",".join(x) + ")" for x in data]) + ) + ) + node2.query("SYSTEM SYNC REPLICA {}".format(name)) + + time.sleep(5) + + used_disks1 = get_used_disks_for_table(node1, name) + assert set(used_disks1) == {"jbod1"} + + used_disks2 = get_used_disks_for_table(node2, name) + assert set(used_disks2) == {"external"} + + assert ( + node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) + assert ( + node2.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + ) + + finally: + try: + node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node2.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + except: + pass diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index f37c28b2a80..bcdb2d25912 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -6,14 +6,36 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV, exec_query_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True) -node2 = cluster.add_instance('node2', with_zookeeper=True) +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) -node3 = cluster.add_instance('node3', with_zookeeper=True) -node4 = cluster.add_instance('node4', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.12.4.5', stay_alive=True, with_installed_binary=True) +node3 = cluster.add_instance("node3", with_zookeeper=True) +node4 = cluster.add_instance( + "node4", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.12.4.5", + stay_alive=True, + with_installed_binary=True, +) + +node5 = cluster.add_instance( + "node5", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.12.4.5", + stay_alive=True, + with_installed_binary=True, +) +node6 = cluster.add_instance( + "node6", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.12.4.5", + stay_alive=True, + with_installed_binary=True, +) -node5 = cluster.add_instance('node5', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.12.4.5', stay_alive=True, with_installed_binary=True) -node6 = cluster.add_instance('node6', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.12.4.5', stay_alive=True, with_installed_binary=True) @pytest.fixture(scope="module") def started_cluster(): @@ -33,25 +55,37 @@ def drop_table(nodes, table_name): for node in nodes: node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + # Column TTL works only with wide parts, because it's very expensive to apply it for compact parts def test_ttl_columns(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_ttl(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_columns', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0, min_bytes_for_wide_part=0; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) - node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)") - node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)") + node1.query( + "INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)" + ) + node1.query( + "INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)" + ) time.sleep(1) # sleep to allow use ttl merge selector for second time node1.query("OPTIMIZE TABLE test_ttl FINAL") expected = "1\t0\t0\n2\t0\t0\n" - assert TSV(node1.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV(expected) - assert TSV(node2.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV(expected) + assert TSV(node1.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV( + expected + ) + assert TSV(node2.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV( + expected + ) def test_merge_with_ttl_timeout(started_cluster): @@ -59,22 +93,32 @@ def test_merge_with_ttl_timeout(started_cluster): drop_table([node1, node2], table) for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS min_bytes_for_wide_part=0; - '''.format(replica=node.name, table=table)) + """.format( + replica=node.name, table=table + ) + ) node1.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) node2.query("SYSTEM STOP TTL MERGES {table}".format(table=table)) for i in range(1, 4): node1.query( - "INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format(day=i, table=table)) + "INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format( + day=i, table=table + ) + ) - assert node1.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "0\n" - assert node2.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "0\n" + assert ( + node1.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "0\n" + ) + assert ( + node2.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "0\n" + ) node1.query("SYSTEM START TTL MERGES {table}".format(table=table)) node2.query("SYSTEM START TTL MERGES {table}".format(table=table)) @@ -83,19 +127,26 @@ def test_merge_with_ttl_timeout(started_cluster): for i in range(1, 4): node1.query( - "INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format(day=i, table=table)) + "INSERT INTO {table} VALUES (toDateTime('2000-10-{day:02d} 10:00:00'), 1, 2, 3)".format( + day=i, table=table + ) + ) time.sleep(15) # TTL merges shall not happen. - assert node1.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "3\n" - assert node2.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "3\n" + assert ( + node1.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "3\n" + ) + assert ( + node2.query("SELECT countIf(a = 0) FROM {table}".format(table=table)) == "3\n" + ) def test_ttl_many_columns(started_cluster): drop_table([node1, node2], "test_ttl_2") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_ttl_2(date DateTime, id UInt32, a Int32 TTL date, _idx Int32 TTL date, @@ -103,13 +154,20 @@ def test_ttl_many_columns(started_cluster): _partition Int32 TTL date) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_2', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0; - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) node1.query("SYSTEM STOP TTL MERGES test_ttl_2") node2.query("SYSTEM STOP TTL MERGES test_ttl_2") - node1.query("INSERT INTO test_ttl_2 VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)") - node1.query("INSERT INTO test_ttl_2 VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)") + node1.query( + "INSERT INTO test_ttl_2 VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)" + ) + node1.query( + "INSERT INTO test_ttl_2 VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)" + ) node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) @@ -126,24 +184,38 @@ def test_ttl_many_columns(started_cluster): node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) expected = "1\t0\t0\t0\t0\n6\t7\t8\t9\t10\n" - assert TSV(node1.query("SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id")) == TSV(expected) - assert TSV(node2.query("SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id")) == TSV(expected) + assert TSV( + node1.query( + "SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id" + ) + ) == TSV(expected) + assert TSV( + node2.query( + "SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id" + ) + ) == TSV(expected) -@pytest.mark.parametrize("delete_suffix", [ - "", - "DELETE", -]) +@pytest.mark.parametrize( + "delete_suffix", + [ + "", + "DELETE", + ], +) def test_ttl_table(started_cluster, delete_suffix): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_ttl(date DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 1 DAY {delete_suffix} SETTINGS merge_with_ttl_timeout=0; - '''.format(replica=node.name, delete_suffix=delete_suffix)) + """.format( + replica=node.name, delete_suffix=delete_suffix + ) + ) node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1)") node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2)") @@ -158,23 +230,33 @@ def test_modify_ttl(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_ttl(d DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_modify', '{replica}') ORDER BY id - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) node1.query( - "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") + "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" + ) node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) - node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2") + node1.query( + "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2" + ) assert node2.query("SELECT id FROM test_ttl") == "2\n3\n" - node2.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2") + node2.query( + "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2" + ) assert node1.query("SELECT id FROM test_ttl") == "3\n" - node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2") + node1.query( + "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2" + ) assert node2.query("SELECT id FROM test_ttl") == "" @@ -182,35 +264,49 @@ def test_modify_column_ttl(started_cluster): drop_table([node1, node2], "test_ttl") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_column', '{replica}') ORDER BY d - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) node1.query( - "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)") + "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" + ) node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) - node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2") + node1.query( + "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2" + ) assert node2.query("SELECT id FROM test_ttl") == "42\n2\n3\n" - node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2") + node1.query( + "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2" + ) assert node1.query("SELECT id FROM test_ttl") == "42\n42\n3\n" - node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2") + node1.query( + "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2" + ) assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n" def test_ttl_double_delete_rule_returns_error(started_cluster): drop_table([node1, node2], "test_ttl") try: - node1.query(''' + node1.query( + """ CREATE TABLE test_ttl(date DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_double_delete', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 1 DAY, date + INTERVAL 2 DAY SETTINGS merge_with_ttl_timeout=0 - '''.format(replica=node1.name)) + """.format( + replica=node1.name + ) + ) assert False except client.QueryRuntimeException: pass @@ -221,26 +317,41 @@ def test_ttl_double_delete_rule_returns_error(started_cluster): def optimize_with_retry(node, table_name, retry=20): for i in range(retry): try: - node.query("OPTIMIZE TABLE {name} FINAL SETTINGS optimize_throw_if_noop = 1".format(name=table_name), settings={"optimize_throw_if_noop": "1"}) + node.query( + "OPTIMIZE TABLE {name} FINAL SETTINGS optimize_throw_if_noop = 1".format( + name=table_name + ), + settings={"optimize_throw_if_noop": "1"}, + ) break except e: time.sleep(0.5) -@pytest.mark.parametrize("name,engine", [ - pytest.param("test_ttl_alter_delete", "MergeTree()", id="test_ttl_alter_delete"), - pytest.param("test_replicated_ttl_alter_delete", "ReplicatedMergeTree('/clickhouse/test_replicated_ttl_alter_delete', '1')", id="test_ttl_alter_delete_replicated"), -]) + +@pytest.mark.parametrize( + "name,engine", + [ + pytest.param( + "test_ttl_alter_delete", "MergeTree()", id="test_ttl_alter_delete" + ), + pytest.param( + "test_replicated_ttl_alter_delete", + "ReplicatedMergeTree('/clickhouse/test_replicated_ttl_alter_delete', '1')", + id="test_ttl_alter_delete_replicated", + ), + ], +) def test_ttl_alter_delete(started_cluster, name, engine): """Copyright 2019, Altinity LTD -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License.""" + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.""" """Check compatibility with old TTL delete expressions to make sure that: * alter modify of column's TTL delete expression works @@ -260,84 +371,143 @@ limitations under the License.""" ORDER BY tuple() TTL d1 + INTERVAL 1 DAY DELETE SETTINGS min_bytes_for_wide_part=0 - """.format(name=name, engine=engine)) + """.format( + name=name, engine=engine + ) + ) - node1.query("""ALTER TABLE {name} MODIFY COLUMN s1 String TTL d1 + INTERVAL 1 SECOND""".format(name=name)) + node1.query( + """ALTER TABLE {name} MODIFY COLUMN s1 String TTL d1 + INTERVAL 1 SECOND""".format( + name=name + ) + ) node1.query("""ALTER TABLE {name} ADD COLUMN b1 Int32""".format(name=name)) - node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello1', 1, toDateTime({time}))""".format(name=name, - time=time.time())) - node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello2', 2, toDateTime({time}))""".format(name=name, - time=time.time() + 360)) + node1.query( + """INSERT INTO {name} (s1, b1, d1) VALUES ('hello1', 1, toDateTime({time}))""".format( + name=name, time=time.time() + ) + ) + node1.query( + """INSERT INTO {name} (s1, b1, d1) VALUES ('hello2', 2, toDateTime({time}))""".format( + name=name, time=time.time() + 360 + ) + ) time.sleep(1) optimize_with_retry(node1, name) - r = node1.query("SELECT s1, b1 FROM {name} ORDER BY b1, s1".format(name=name)).splitlines() + r = node1.query( + "SELECT s1, b1 FROM {name} ORDER BY b1, s1".format(name=name) + ).splitlines() assert r == ["\t1", "hello2\t2"] - node1.query("""ALTER TABLE {name} MODIFY COLUMN b1 Int32 TTL d1""".format(name=name)) - node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello3', 3, toDateTime({time}))""".format(name=name, - time=time.time())) + node1.query( + """ALTER TABLE {name} MODIFY COLUMN b1 Int32 TTL d1""".format(name=name) + ) + node1.query( + """INSERT INTO {name} (s1, b1, d1) VALUES ('hello3', 3, toDateTime({time}))""".format( + name=name, time=time.time() + ) + ) time.sleep(1) optimize_with_retry(node1, name) - r = node1.query("SELECT s1, b1 FROM {name} ORDER BY b1, s1".format(name=name)).splitlines() + r = node1.query( + "SELECT s1, b1 FROM {name} ORDER BY b1, s1".format(name=name) + ).splitlines() assert r == ["\t0", "\t0", "hello2\t2"] + def test_ttl_empty_parts(started_cluster): drop_table([node1, node2], "test_ttl_empty_parts") for node in [node1, node2]: node.query( - ''' + """ CREATE TABLE test_ttl_empty_parts(date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_empty_parts', '{replica}') ORDER BY id SETTINGS max_bytes_to_merge_at_min_space_in_pool = 1, max_bytes_to_merge_at_max_space_in_pool = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) - for i in range (1, 7): - node1.query("INSERT INTO test_ttl_empty_parts SELECT '2{}00-01-0{}', number FROM numbers(1000)".format(i % 2, i)) + for i in range(1, 7): + node1.query( + "INSERT INTO test_ttl_empty_parts SELECT '2{}00-01-0{}', number FROM numbers(1000)".format( + i % 2, i + ) + ) assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "6000\n" - assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == \ - "all_0_0_0\nall_1_1_0\nall_2_2_0\nall_3_3_0\nall_4_4_0\nall_5_5_0\n" + assert ( + node1.query( + "SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name" + ) + == "all_0_0_0\nall_1_1_0\nall_2_2_0\nall_3_3_0\nall_4_4_0\nall_5_5_0\n" + ) node1.query("ALTER TABLE test_ttl_empty_parts MODIFY TTL date") assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "3000\n" - time.sleep(3) # Wait for cleanup thread - assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == \ - "all_0_0_0_6\nall_2_2_0_6\nall_4_4_0_6\n" + time.sleep(3) # Wait for cleanup thread + assert ( + node1.query( + "SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name" + ) + == "all_0_0_0_6\nall_2_2_0_6\nall_4_4_0_6\n" + ) for node in [node1, node2]: - node.query("ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_min_space_in_pool = 1000000000") - node.query("ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 1000000000") + node.query( + "ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_min_space_in_pool = 1000000000" + ) + node.query( + "ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 1000000000" + ) - optimize_with_retry(node1, 'test_ttl_empty_parts') - assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == "all_0_4_1_6\n" + optimize_with_retry(node1, "test_ttl_empty_parts") + assert ( + node1.query( + "SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name" + ) + == "all_0_4_1_6\n" + ) # Check that after removing empty parts mutations and merges works - node1.query("INSERT INTO test_ttl_empty_parts SELECT '2100-01-20', number FROM numbers(1000)") - node1.query("ALTER TABLE test_ttl_empty_parts DELETE WHERE id % 2 = 0 SETTINGS mutations_sync = 2") + node1.query( + "INSERT INTO test_ttl_empty_parts SELECT '2100-01-20', number FROM numbers(1000)" + ) + node1.query( + "ALTER TABLE test_ttl_empty_parts DELETE WHERE id % 2 = 0 SETTINGS mutations_sync = 2" + ) assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "2000\n" - optimize_with_retry(node1, 'test_ttl_empty_parts') - assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == "all_0_7_2_8\n" + optimize_with_retry(node1, "test_ttl_empty_parts") + assert ( + node1.query( + "SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name" + ) + == "all_0_7_2_8\n" + ) - node2.query('SYSTEM SYNC REPLICA test_ttl_empty_parts', timeout=20) + node2.query("SYSTEM SYNC REPLICA test_ttl_empty_parts", timeout=20) - error_msg = ' default.test_ttl_empty_parts (ReplicatedMergeTreeCleanupThread)' + error_msg = ( + " default.test_ttl_empty_parts (ReplicatedMergeTreeCleanupThread)" + ) assert not node1.contains_in_log(error_msg) assert not node2.contains_in_log(error_msg) + @pytest.mark.parametrize( - ('node_left', 'node_right', 'num_run'), - [(node1, node2, 0), (node3, node4, 1), (node5, node6, 2)] + ("node_left", "node_right", "num_run"), + [(node1, node2, 0), (node3, node4, 1), (node5, node6, 2)], ) def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): drop_table([node_left, node_right], "test_ttl_delete") @@ -346,36 +516,49 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): for node in [node_left, node_right]: node.query( - ''' + """ CREATE TABLE test_ttl_delete(date DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_delete_{suff}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 - '''.format(suff=num_run, replica=node.name)) + """.format( + suff=num_run, replica=node.name + ) + ) node.query( - ''' + """ CREATE TABLE test_ttl_group_by(date DateTime, id UInt32, val UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_group_by_{suff}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND GROUP BY id SET val = sum(val) SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 - '''.format(suff=num_run, replica=node.name)) + """.format( + suff=num_run, replica=node.name + ) + ) node.query( - ''' + """ CREATE TABLE test_ttl_where(date DateTime, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_where_{suff}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND DELETE WHERE id % 2 = 1 SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 - '''.format(suff=num_run, replica=node.name)) + """.format( + suff=num_run, replica=node.name + ) + ) node_left.query("INSERT INTO test_ttl_delete VALUES (now(), 1)") - node_left.query("INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 2)") + node_left.query( + "INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 2)" + ) node_right.query("INSERT INTO test_ttl_delete VALUES (now(), 3)") - node_right.query("INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 4)") + node_right.query( + "INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 4)" + ) node_left.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 1)") node_left.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 2)") @@ -392,8 +575,8 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): if node_right.with_installed_binary: node_right.restart_with_latest_version() - - time.sleep(5) # Wait for TTL + + time.sleep(5) # Wait for TTL # after restart table can be in readonly mode exec_query_with_retry(node_right, "OPTIMIZE TABLE test_ttl_delete FINAL") diff --git a/tests/integration/test_union_header/test.py b/tests/integration/test_union_header/test.py index edbf4dddecf..f883057c1d8 100644 --- a/tests/integration/test_union_header/test.py +++ b/tests/integration/test_union_header/test.py @@ -4,8 +4,12 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node1 = cluster.add_instance( + "node1", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) +node2 = cluster.add_instance( + "node2", main_configs=["configs/remote_servers.xml"], with_zookeeper=True +) @pytest.fixture(scope="module") @@ -14,7 +18,8 @@ def started_cluster(): cluster.start() for node in (node1, node2): - node.query(''' + node.query( + """ CREATE TABLE default.t1_local ( event_date Date DEFAULT toDate(event_time), @@ -23,12 +28,15 @@ def started_cluster(): account_id String ) ENGINE = MergeTree(event_date, (event_time, account_id), 8192); - ''') + """ + ) - node.query(''' + node.query( + """ CREATE TABLE default.t1 AS default.t1_local ENGINE = Distributed('two_shards', 'default', 't1_local', rand()); - ''') + """ + ) yield cluster @@ -37,7 +45,12 @@ def started_cluster(): def test_read(started_cluster): - assert node1.query('''SELECT event_date, event_time, log_type + assert ( + node1.query( + """SELECT event_date, event_time, log_type FROM default.t1 WHERE (log_type = 30305) AND (account_id = '111111') - LIMIT 1''').strip() == '' + LIMIT 1""" + ).strip() + == "" + ) diff --git a/tests/integration/test_user_defined_object_persistence/test.py b/tests/integration/test_user_defined_object_persistence/test.py index 6993bc13615..8d775411b61 100644 --- a/tests/integration/test_user_defined_object_persistence/test.py +++ b/tests/integration/test_user_defined_object_persistence/test.py @@ -2,7 +2,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance', stay_alive=True) +instance = cluster.add_instance("instance", stay_alive=True) @pytest.fixture(scope="module", autouse=True) @@ -21,7 +21,7 @@ def test_persistence(): instance.query(create_function_query1) instance.query(create_function_query2) - + assert instance.query("SELECT MySum1(1,2)") == "3\n" assert instance.query("SELECT MySum2(1,2)") == "5\n" @@ -35,5 +35,9 @@ def test_persistence(): instance.restart_clickhouse() - assert "Unknown function MySum1" in instance.query_and_get_error("SELECT MySum1(1, 2)") - assert "Unknown function MySum2" in instance.query_and_get_error("SELECT MySum2(1, 2)") + assert "Unknown function MySum1" in instance.query_and_get_error( + "SELECT MySum1(1, 2)" + ) + assert "Unknown function MySum2" in instance.query_and_get_error( + "SELECT MySum2(1, 2)" + ) diff --git a/tests/integration/test_user_directories/test.py b/tests/integration/test_user_directories/test.py index 1ce4e377f2b..45afb86f464 100644 --- a/tests/integration/test_user_directories/test.py +++ b/tests/integration/test_user_directories/test.py @@ -6,7 +6,7 @@ from helpers.test_tools import TSV SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', stay_alive=True) +node = cluster.add_instance("node", stay_alive=True) @pytest.fixture(scope="module", autouse=True) @@ -15,7 +15,13 @@ def started_cluster(): cluster.start() for i in range(2, 8): - node.exec_in_container(["cp", "/etc/clickhouse-server/users.xml", "/etc/clickhouse-server/users{}.xml".format(i)]) + node.exec_in_container( + [ + "cp", + "/etc/clickhouse-server/users.xml", + "/etc/clickhouse-server/users{}.xml".format(i), + ] + ) yield cluster @@ -24,56 +30,146 @@ def started_cluster(): def test_old_style(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/old_style.xml"), - '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/old_style.xml"), + "/etc/clickhouse-server/config.d/z.xml", + ) node.restart_clickhouse() assert node.query("SELECT * FROM system.user_directories") == TSV( - [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users2.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access2\\\\/"}', 2]]) + [ + [ + "users.xml", + "users.xml", + '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users2.xml"}', + 1, + ], + [ + "local directory", + "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access2\\\\/"}', + 2, + ], + ] + ) def test_local_directories(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/local_directories.xml"), - '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/local_directories.xml"), + "/etc/clickhouse-server/config.d/z.xml", + ) node.restart_clickhouse() assert node.query("SELECT * FROM system.user_directories") == TSV( - [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users3.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3\\\\/"}', 2], - ["local directory (ro)", "local directory", - '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3-ro\\\\/","readonly":true}', 3]]) + [ + [ + "users.xml", + "users.xml", + '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users3.xml"}', + 1, + ], + [ + "local directory", + "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3\\\\/"}', + 2, + ], + [ + "local directory (ro)", + "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3-ro\\\\/","readonly":true}', + 3, + ], + ] + ) def test_relative_path(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/relative_path.xml"), - '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/relative_path.xml"), + "/etc/clickhouse-server/config.d/z.xml", + ) node.restart_clickhouse() assert node.query("SELECT * FROM system.user_directories") == TSV( - [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users4.xml"}', 1]]) + [ + [ + "users.xml", + "users.xml", + '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users4.xml"}', + 1, + ] + ] + ) def test_memory(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/memory.xml"), '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/memory.xml"), + "/etc/clickhouse-server/config.d/z.xml", + ) node.restart_clickhouse() assert node.query("SELECT * FROM system.user_directories") == TSV( - [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users5.xml"}', 1], - ["memory", "memory", '{}', 2]]) + [ + [ + "users.xml", + "users.xml", + '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users5.xml"}', + 1, + ], + ["memory", "memory", "{}", 2], + ] + ) def test_mixed_style(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/mixed_style.xml"), - '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/mixed_style.xml"), + "/etc/clickhouse-server/config.d/z.xml", + ) node.restart_clickhouse() assert node.query("SELECT * FROM system.user_directories") == TSV( - [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users6.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6\\\\/"}', 2], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6a\\\\/"}', 3], - ["memory", "memory", '{}', 4]]) + [ + [ + "users.xml", + "users.xml", + '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users6.xml"}', + 1, + ], + [ + "local directory", + "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6\\\\/"}', + 2, + ], + [ + "local directory", + "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6a\\\\/"}', + 3, + ], + ["memory", "memory", "{}", 4], + ] + ) def test_duplicates(): - node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/duplicates.xml"), - '/etc/clickhouse-server/config.d/z.xml') + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "configs/duplicates.xml"), + "/etc/clickhouse-server/config.d/z.xml", + ) node.restart_clickhouse() assert node.query("SELECT * FROM system.user_directories") == TSV( - [["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users7.xml"}', 1], - ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access7\\\\/"}', 2]]) + [ + [ + "users.xml", + "users.xml", + '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users7.xml"}', + 1, + ], + [ + "local directory", + "local directory", + '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access7\\\\/"}', + 2, + ], + ] + ) diff --git a/tests/integration/test_user_ip_restrictions/test.py b/tests/integration/test_user_ip_restrictions/test.py index a7344fd1a45..e41febfa2f5 100644 --- a/tests/integration/test_user_ip_restrictions/test.py +++ b/tests/integration/test_user_ip_restrictions/test.py @@ -4,23 +4,52 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node_ipv4 = cluster.add_instance('node_ipv4', main_configs=[], user_configs=['configs/users_ipv4.xml'], - ipv4_address='10.5.172.77') -client_ipv4_ok = cluster.add_instance('client_ipv4_ok', main_configs=[], user_configs=[], ipv4_address='10.5.172.10') -client_ipv4_ok_direct = cluster.add_instance('client_ipv4_ok_direct', main_configs=[], user_configs=[], - ipv4_address='10.5.173.1') -client_ipv4_ok_full_mask = cluster.add_instance('client_ipv4_ok_full_mask', main_configs=[], user_configs=[], - ipv4_address='10.5.175.77') -client_ipv4_bad = cluster.add_instance('client_ipv4_bad', main_configs=[], user_configs=[], ipv4_address='10.5.173.10') +node_ipv4 = cluster.add_instance( + "node_ipv4", + main_configs=[], + user_configs=["configs/users_ipv4.xml"], + ipv4_address="10.5.172.77", +) +client_ipv4_ok = cluster.add_instance( + "client_ipv4_ok", main_configs=[], user_configs=[], ipv4_address="10.5.172.10" +) +client_ipv4_ok_direct = cluster.add_instance( + "client_ipv4_ok_direct", main_configs=[], user_configs=[], ipv4_address="10.5.173.1" +) +client_ipv4_ok_full_mask = cluster.add_instance( + "client_ipv4_ok_full_mask", + main_configs=[], + user_configs=[], + ipv4_address="10.5.175.77", +) +client_ipv4_bad = cluster.add_instance( + "client_ipv4_bad", main_configs=[], user_configs=[], ipv4_address="10.5.173.10" +) -node_ipv6 = cluster.add_instance('node_ipv6', main_configs=["configs/config_ipv6.xml"], - user_configs=['configs/users_ipv6.xml'], ipv6_address='2001:3984:3989::1:1000') -client_ipv6_ok = cluster.add_instance('client_ipv6_ok', main_configs=[], user_configs=[], - ipv6_address='2001:3984:3989::5555') -client_ipv6_ok_direct = cluster.add_instance('client_ipv6_ok_direct', main_configs=[], user_configs=[], - ipv6_address='2001:3984:3989::1:1111') -client_ipv6_bad = cluster.add_instance('client_ipv6_bad', main_configs=[], user_configs=[], - ipv6_address='2001:3984:3989::1:1112') +node_ipv6 = cluster.add_instance( + "node_ipv6", + main_configs=["configs/config_ipv6.xml"], + user_configs=["configs/users_ipv6.xml"], + ipv6_address="2001:3984:3989::1:1000", +) +client_ipv6_ok = cluster.add_instance( + "client_ipv6_ok", + main_configs=[], + user_configs=[], + ipv6_address="2001:3984:3989::5555", +) +client_ipv6_ok_direct = cluster.add_instance( + "client_ipv6_ok_direct", + main_configs=[], + user_configs=[], + ipv6_address="2001:3984:3989::1:1111", +) +client_ipv6_bad = cluster.add_instance( + "client_ipv6_bad", + main_configs=[], + user_configs=[], + ipv6_address="2001:3984:3989::1:1112", +) @pytest.fixture(scope="module") @@ -36,30 +65,62 @@ def setup_cluster(): def test_ipv4(setup_cluster): try: client_ipv4_ok.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, - user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'", + ], + privileged=True, + user="root", + ) except Exception as ex: - assert False, "allowed client with 10.5.172.10 cannot connect to server with allowed mask '10.5.172.0/24'" + assert ( + False + ), "allowed client with 10.5.172.10 cannot connect to server with allowed mask '10.5.172.0/24'" try: client_ipv4_ok_direct.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, - user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'", + ], + privileged=True, + user="root", + ) except Exception as ex: - assert False, "allowed client with 10.5.173.1 cannot connect to server with allowed ip '10.5.173.1'" + assert ( + False + ), "allowed client with 10.5.173.1 cannot connect to server with allowed ip '10.5.173.1'" try: client_ipv4_ok_full_mask.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, - user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'", + ], + privileged=True, + user="root", + ) except Exception as ex: - assert False, "allowed client with 10.5.175.77 cannot connect to server with allowed ip '10.5.175.0/255.255.255.0'" + assert ( + False + ), "allowed client with 10.5.175.77 cannot connect to server with allowed ip '10.5.175.0/255.255.255.0'" try: client_ipv4_bad.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, - user='root') - assert False, "restricted client with 10.5.173.10 can connect to server with allowed mask '10.5.172.0/24'" + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'", + ], + privileged=True, + user="root", + ) + assert ( + False + ), "restricted client with 10.5.173.10 can connect to server with allowed mask '10.5.172.0/24'" except AssertionError: raise except Exception as ex: @@ -69,24 +130,48 @@ def test_ipv4(setup_cluster): def test_ipv6(setup_cluster): try: client_ipv6_ok.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989::1:1000 --query 'select 1'"], - privileged=True, user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 2001:3984:3989::1:1000 --query 'select 1'", + ], + privileged=True, + user="root", + ) except Exception as ex: print(ex) - assert False, "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" + assert ( + False + ), "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" try: client_ipv6_ok_direct.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], - privileged=True, user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'", + ], + privileged=True, + user="root", + ) except Exception as ex: - assert False, "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed ip '2001:3984:3989:0:0:0:1:1111'" + assert ( + False + ), "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed ip '2001:3984:3989:0:0:0:1:1111'" try: client_ipv6_bad.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], - privileged=True, user='root') - assert False, "restricted client with 2001:3984:3989:0:0:0:1:1112 can connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" + [ + "bash", + "-c", + "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'", + ], + privileged=True, + user="root", + ) + assert ( + False + ), "restricted client with 2001:3984:3989:0:0:0:1:1112 can connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" except AssertionError: raise except Exception as ex: diff --git a/tests/integration/test_user_zero_database_access/configs/users.xml b/tests/integration/test_user_zero_database_access/configs/users.xml index 8c8dfbb5b7e..25c598aa560 100644 --- a/tests/integration/test_user_zero_database_access/configs/users.xml +++ b/tests/integration/test_user_zero_database_access/configs/users.xml @@ -37,6 +37,24 @@ db1 + + + clickhouse + + ::/0 + + default + default + + + + + + ::/0 + + default + default + diff --git a/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py b/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py index dd3789cde57..747c022a3b0 100644 --- a/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py +++ b/tests/integration/test_user_zero_database_access/test_user_zero_database_access.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', user_configs=["configs/users.xml"]) +node = cluster.add_instance("node", user_configs=["configs/users.xml"]) @pytest.fixture(scope="module") @@ -19,7 +19,13 @@ def start_cluster(): def test_user_zero_database_access(start_cluster): try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'DROP DATABASE test'"], user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'no_access' --query 'DROP DATABASE test'", + ], + user="root", + ) assert False, "user with no access rights dropped database test" except AssertionError: raise @@ -28,21 +34,37 @@ def test_user_zero_database_access(start_cluster): try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'DROP DATABASE test'"], user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'has_access' --query 'DROP DATABASE test'", + ], + user="root", + ) except Exception as ex: assert False, "user with access rights can't drop database test" try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test'"], - user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test'", + ], + user="root", + ) except Exception as ex: assert False, "user with access rights can't create database test" try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'no_access' --query 'CREATE DATABASE test2'"], - user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'no_access' --query 'CREATE DATABASE test2'", + ], + user="root", + ) assert False, "user with no access rights created database test2" except AssertionError: raise @@ -51,9 +73,16 @@ def test_user_zero_database_access(start_cluster): try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test2'"], - user='root') - assert False, "user with limited access rights created database test2 which is outside of his scope of rights" + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'has_access' --query 'CREATE DATABASE test2'", + ], + user="root", + ) + assert ( + False + ), "user with limited access rights created database test2 which is outside of his scope of rights" except AssertionError: raise except Exception as ex: @@ -61,12 +90,52 @@ def test_user_zero_database_access(start_cluster): try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'CREATE DATABASE test2'"], user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'default' --query 'CREATE DATABASE test2'", + ], + user="root", + ) except Exception as ex: assert False, "user with full access rights can't create database test2" try: node.exec_in_container( - ["bash", "-c", "/usr/bin/clickhouse client --user 'default' --query 'DROP DATABASE test2'"], user='root') + [ + "bash", + "-c", + "/usr/bin/clickhouse client --user 'default' --query 'DROP DATABASE test2'", + ], + user="root", + ) except Exception as ex: assert False, "user with full access rights can't drop database test2" + + try: + name = node.exec_in_container( + [ + "bash", + "-c", + "export CLICKHOUSE_USER=env_user_not_with_password && /usr/bin/clickhouse client --query 'SELECT currentUser()'", + ], + user="root", + ) + assert name.strip() == "env_user_not_with_password" + except Exception as ex: + assert False, "set env CLICKHOUSE_USER can not connect server" + + try: + name = node.exec_in_container( + [ + "bash", + "-c", + "export CLICKHOUSE_USER=env_user_with_password && export CLICKHOUSE_PASSWORD=clickhouse && /usr/bin/clickhouse client --query 'SELECT currentUser()'", + ], + user="root", + ) + assert name.strip() == "env_user_with_password" + except Exception as ex: + assert ( + False + ), "set env CLICKHOUSE_USER CLICKHOUSE_PASSWORD can not connect server" diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index 4e5d925852c..3332fe69e86 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -2,49 +2,92 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry + cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', stay_alive=True) +node1 = cluster.add_instance("node1", stay_alive=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='21.2', with_installed_binary=True, stay_alive=True) +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="21.2", + with_installed_binary=True, + stay_alive=True, +) # Use differents nodes because if there is node.restart_from_latest_version(), then in later tests # it will be with latest version, but shouldn't, order of tests in CI is shuffled. -node3 = cluster.add_instance('node3', image='yandex/clickhouse-server', tag='21.5', with_installed_binary=True, stay_alive=True) -node4 = cluster.add_instance('node4', image='yandex/clickhouse-server', tag='21.5', with_installed_binary=True, stay_alive=True) -node5 = cluster.add_instance('node5', image='yandex/clickhouse-server', tag='21.5', with_installed_binary=True, stay_alive=True) -node6 = cluster.add_instance('node6', image='yandex/clickhouse-server', tag='21.5', with_installed_binary=True, stay_alive=True) +node3 = cluster.add_instance( + "node3", + image="yandex/clickhouse-server", + tag="21.5", + with_installed_binary=True, + stay_alive=True, +) +node4 = cluster.add_instance( + "node4", + image="yandex/clickhouse-server", + tag="21.5", + with_installed_binary=True, + stay_alive=True, +) +node5 = cluster.add_instance( + "node5", + image="yandex/clickhouse-server", + tag="21.5", + with_installed_binary=True, + stay_alive=True, +) +node6 = cluster.add_instance( + "node6", + image="yandex/clickhouse-server", + tag="21.5", + with_installed_binary=True, + stay_alive=True, +) - -def insert_data(node, table_name='test_table', n=1, col2=1): - node.query(""" INSERT INTO {} +def insert_data(node, table_name="test_table", n=1, col2=1): + node.query( + """ INSERT INTO {} SELECT toDateTime(NOW()), {}, sumMapState(arrayMap(i -> 1, range(300)), arrayMap(i -> 1, range(300))) - FROM numbers({});""".format(table_name, col2, n)) + FROM numbers({});""".format( + table_name, col2, n + ) + ) -def create_table(node, name='test_table', version=None): +def create_table(node, name="test_table", version=None): node.query("DROP TABLE IF EXISTS {};".format(name)) if version is None: - node.query(""" + node.query( + """ CREATE TABLE {} ( `col1` DateTime, `col2` Int64, `col3` AggregateFunction(sumMap, Array(UInt8), Array(UInt8)) ) - ENGINE = AggregatingMergeTree() ORDER BY (col1, col2) """.format(name)) + ENGINE = AggregatingMergeTree() ORDER BY (col1, col2) """.format( + name + ) + ) else: - node.query(""" + node.query( + """ CREATE TABLE {} ( `col1` DateTime, `col2` Int64, `col3` AggregateFunction({}, sumMap, Array(UInt8), Array(UInt8)) ) - ENGINE = AggregatingMergeTree() ORDER BY (col1, col2) """.format(name, version)) + ENGINE = AggregatingMergeTree() ORDER BY (col1, col2) """.format( + name, version + ) + ) @pytest.fixture(scope="module") @@ -57,35 +100,51 @@ def start_cluster(): def test_modulo_partition_key_issue_23508(start_cluster): - node2.query("CREATE TABLE test (id Int64, v UInt64, value String) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/table1', '1', v) PARTITION BY id % 20 ORDER BY (id, v)") - node2.query("INSERT INTO test SELECT number, number, toString(number) FROM numbers(10)") + node2.query( + "CREATE TABLE test (id Int64, v UInt64, value String) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/table1', '1', v) PARTITION BY id % 20 ORDER BY (id, v)" + ) + node2.query( + "INSERT INTO test SELECT number, number, toString(number) FROM numbers(10)" + ) expected = node2.query("SELECT number, number, toString(number) FROM numbers(10)") - partition_data = node2.query("SELECT partition, name FROM system.parts WHERE table='test' ORDER BY partition") - assert(expected == node2.query("SELECT * FROM test ORDER BY id")) + partition_data = node2.query( + "SELECT partition, name FROM system.parts WHERE table='test' ORDER BY partition" + ) + assert expected == node2.query("SELECT * FROM test ORDER BY id") node2.restart_with_latest_version() - assert(expected == node2.query("SELECT * FROM test ORDER BY id")) - assert(partition_data == node2.query("SELECT partition, name FROM system.parts WHERE table='test' ORDER BY partition")) + assert expected == node2.query("SELECT * FROM test ORDER BY id") + assert partition_data == node2.query( + "SELECT partition, name FROM system.parts WHERE table='test' ORDER BY partition" + ) # Test from issue 16587 def test_aggregate_function_versioning_issue_16587(start_cluster): for node in [node1, node3]: node.query("DROP TABLE IF EXISTS test_table;") - node.query(""" + node.query( + """ CREATE TABLE test_table (`col1` DateTime, `col2` Int64) - ENGINE = MergeTree() ORDER BY col1""") - node.query("insert into test_table select '2020-10-26 00:00:00', 1929292 from numbers(300)") + ENGINE = MergeTree() ORDER BY col1""" + ) + node.query( + "insert into test_table select '2020-10-26 00:00:00', 1929292 from numbers(300)" + ) expected = "([1],[600])" - result_on_old_version = node3.query("select sumMap(sm) from (select sumMap([1],[1]) as sm from remote('127.0.0.{1,2}', default.test_table) group by col1, col2);").strip() - assert(result_on_old_version != expected) + result_on_old_version = node3.query( + "select sumMap(sm) from (select sumMap([1],[1]) as sm from remote('127.0.0.{1,2}', default.test_table) group by col1, col2);" + ).strip() + assert result_on_old_version != expected - result_on_new_version = node1.query("select sumMap(sm) from (select sumMap([1],[1]) as sm from remote('127.0.0.{1,2}', default.test_table) group by col1, col2);").strip() - assert(result_on_new_version == expected) + result_on_new_version = node1.query( + "select sumMap(sm) from (select sumMap([1],[1]) as sm from remote('127.0.0.{1,2}', default.test_table) group by col1, col2);" + ).strip() + assert result_on_new_version == expected def test_aggregate_function_versioning_fetch_data_from_old_to_new_server(start_cluster): @@ -95,14 +154,20 @@ def test_aggregate_function_versioning_fetch_data_from_old_to_new_server(start_c expected = "([1],[300])" - new_server_data = node1.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(new_server_data == expected) + new_server_data = node1.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert new_server_data == expected - old_server_data = node4.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(old_server_data != expected) + old_server_data = node4.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert old_server_data != expected - data_from_old_to_new_server = node1.query("select finalizeAggregation(col3) from remote('node4', default.test_table);").strip() - assert(data_from_old_to_new_server == old_server_data) + data_from_old_to_new_server = node1.query( + "select finalizeAggregation(col3) from remote('node4', default.test_table);" + ).strip() + assert data_from_old_to_new_server == old_server_data def test_aggregate_function_versioning_server_upgrade(start_cluster): @@ -112,83 +177,117 @@ def test_aggregate_function_versioning_server_upgrade(start_cluster): insert_data(node5, col2=1) # Serialization with version 0, server does not support versioning of aggregate function states. - old_server_data = node5.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(old_server_data == "([1],[44])") + old_server_data = node5.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert old_server_data == "([1],[44])" create = node5.query("describe table default.test_table;").strip() - assert(create.strip().endswith("col3\tAggregateFunction(sumMap, Array(UInt8), Array(UInt8))")) - print('Ok 1') + assert create.strip().endswith( + "col3\tAggregateFunction(sumMap, Array(UInt8), Array(UInt8))" + ) + print("Ok 1") # Upgrade server. node5.restart_with_latest_version() # Deserialized with version 0, server supports versioning. - upgraded_server_data = node5.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(upgraded_server_data == "([1],[44])") + upgraded_server_data = node5.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert upgraded_server_data == "([1],[44])" create = node5.query("describe table default.test_table;").strip() - assert(create.strip().endswith("col3\tAggregateFunction(sumMap, Array(UInt8), Array(UInt8))")) - print('Ok 2') + assert create.strip().endswith( + "col3\tAggregateFunction(sumMap, Array(UInt8), Array(UInt8))" + ) + print("Ok 2") create = node1.query("describe table default.test_table;").strip() print(create) - assert(create.strip().endswith("col3\tAggregateFunction(1, sumMap, Array(UInt8), Array(UInt8))")) + assert create.strip().endswith( + "col3\tAggregateFunction(1, sumMap, Array(UInt8), Array(UInt8))" + ) # Data from upgraded server to new server. Deserialize with version 0. - data_from_upgraded_to_new_server = node1.query("select finalizeAggregation(col3) from remote('node5', default.test_table);").strip() - assert(data_from_upgraded_to_new_server == upgraded_server_data == "([1],[44])") - print('Ok 3') + data_from_upgraded_to_new_server = node1.query( + "select finalizeAggregation(col3) from remote('node5', default.test_table);" + ).strip() + assert data_from_upgraded_to_new_server == upgraded_server_data == "([1],[44])" + print("Ok 3") # Data is serialized according to version 0 (though one of the states is version 1, but result is version 0). - upgraded_server_data = node5.query("select finalizeAggregation(col3) from remote('127.0.0.{1,2}', default.test_table);").strip() - assert(upgraded_server_data == "([1],[44])\n([1],[44])") - print('Ok 4') + upgraded_server_data = node5.query( + "select finalizeAggregation(col3) from remote('127.0.0.{1,2}', default.test_table);" + ).strip() + assert upgraded_server_data == "([1],[44])\n([1],[44])" + print("Ok 4") # Check insertion after server upgarde. insert_data(node5, col2=2) # Check newly inserted data is still serialized with 0 version. - upgraded_server_data = node5.query("select finalizeAggregation(col3) from default.test_table order by col2;").strip() - assert(upgraded_server_data == "([1],[44])\n([1],[44])") - print('Ok 5') + upgraded_server_data = node5.query( + "select finalizeAggregation(col3) from default.test_table order by col2;" + ).strip() + assert upgraded_server_data == "([1],[44])\n([1],[44])" + print("Ok 5") # New table has latest version. - new_server_data = node1.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(new_server_data == "([1],[300])") - print('Ok 6') + new_server_data = node1.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert new_server_data == "([1],[300])" + print("Ok 6") # Insert from new server (with version 1) to upgraded server (where version will be 0), result version 0. - node1.query("insert into table function remote('node5', default.test_table) select * from default.test_table;").strip() - upgraded_server_data = node5.query("select finalizeAggregation(col3) from default.test_table order by col2;").strip() - assert(upgraded_server_data == "([1],[44])\n([1],[44])\n([1],[44])") - print('Ok 7') + node1.query( + "insert into table function remote('node5', default.test_table) select * from default.test_table;" + ).strip() + upgraded_server_data = node5.query( + "select finalizeAggregation(col3) from default.test_table order by col2;" + ).strip() + assert upgraded_server_data == "([1],[44])\n([1],[44])\n([1],[44])" + print("Ok 7") # But new table gets data with latest version. insert_data(node1) - new_server_data = node1.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(new_server_data == "([1],[300])\n([1],[300])") - print('Ok 8') + new_server_data = node1.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert new_server_data == "([1],[300])\n([1],[300])" + print("Ok 8") # Create table with column implicitly with older version (version 0). - create_table(node1, name='test_table_0', version=0) - insert_data(node1, table_name='test_table_0', col2=3) - data = node1.query("select finalizeAggregation(col3) from default.test_table_0;").strip() - assert(data == "([1],[44])") - print('Ok') + create_table(node1, name="test_table_0", version=0) + insert_data(node1, table_name="test_table_0", col2=3) + data = node1.query( + "select finalizeAggregation(col3) from default.test_table_0;" + ).strip() + assert data == "([1],[44])" + print("Ok") # Insert from new server to upgraded server to a new table but the version was set implicitly to 0, so data version 0. - node1.query("insert into table function remote('node5', default.test_table) select * from default.test_table_0;").strip() - upgraded_server_data = node5.query("select finalizeAggregation(col3) from default.test_table order by col2;").strip() - assert(upgraded_server_data == "([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])") - print('Ok') + node1.query( + "insert into table function remote('node5', default.test_table) select * from default.test_table_0;" + ).strip() + upgraded_server_data = node5.query( + "select finalizeAggregation(col3) from default.test_table order by col2;" + ).strip() + assert upgraded_server_data == "([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])" + print("Ok") def test_aggregate_function_versioning_persisting_metadata(start_cluster): for node in [node1, node6]: create_table(node) insert_data(node) - data = node1.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(data == "([1],[300])") - data = node6.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(data == "([1],[44])") + data = node1.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert data == "([1],[300])" + data = node6.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert data == "([1],[44])" node6.restart_with_latest_version() @@ -199,18 +298,32 @@ def test_aggregate_function_versioning_persisting_metadata(start_cluster): for node in [node1, node6]: insert_data(node) - new_server_data = node1.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(new_server_data == "([1],[300])\n([1],[300])") + new_server_data = node1.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert new_server_data == "([1],[300])\n([1],[300])" - upgraded_server_data = node6.query("select finalizeAggregation(col3) from default.test_table;").strip() - assert(upgraded_server_data == "([1],[44])\n([1],[44])") + upgraded_server_data = node6.query( + "select finalizeAggregation(col3) from default.test_table;" + ).strip() + assert upgraded_server_data == "([1],[44])\n([1],[44])" for node in [node1, node6]: node.restart_clickhouse() insert_data(node) - result = node1.query("select finalizeAggregation(col3) from remote('127.0.0.{1,2}', default.test_table);").strip() - assert(result == "([1],[300])\n([1],[300])\n([1],[300])\n([1],[300])\n([1],[300])\n([1],[300])") + result = node1.query( + "select finalizeAggregation(col3) from remote('127.0.0.{1,2}', default.test_table);" + ).strip() + assert ( + result + == "([1],[300])\n([1],[300])\n([1],[300])\n([1],[300])\n([1],[300])\n([1],[300])" + ) - result = node6.query("select finalizeAggregation(col3) from remote('127.0.0.{1,2}', default.test_table);").strip() - assert(result == "([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])") + result = node6.query( + "select finalizeAggregation(col3) from remote('127.0.0.{1,2}', default.test_table);" + ).strip() + assert ( + result + == "([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])\n([1],[44])" + ) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 3c22f2ed380..2971cbc9792 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -6,12 +6,30 @@ from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', - with_installed_binary=True, stay_alive=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', - with_installed_binary=True, stay_alive=True) -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server', tag='20.1.10.70', - with_installed_binary=True, stay_alive=True) +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.1.10.70", + with_installed_binary=True, + stay_alive=True, +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.1.10.70", + with_installed_binary=True, + stay_alive=True, +) +node3 = cluster.add_instance( + "node3", + with_zookeeper=True, + image="yandex/clickhouse-server", + tag="20.1.10.70", + with_installed_binary=True, + stay_alive=True, +) @pytest.fixture(scope="module") @@ -29,7 +47,9 @@ def test_mutate_and_upgrade(start_cluster): node.query("DROP TABLE IF EXISTS mt") node.query( "CREATE TABLE mt (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple()".format( - node.name)) + node.name + ) + ) node1.query("INSERT INTO mt VALUES ('2020-02-13', 1), ('2020-02-13', 2);") @@ -42,7 +62,9 @@ def test_mutate_and_upgrade(start_cluster): node2.restart_with_latest_version(signal=9) # After hard restart table can be in readonly mode - exec_query_with_retry(node2, "INSERT INTO mt VALUES ('2020-02-13', 3)", retry_count=60) + exec_query_with_retry( + node2, "INSERT INTO mt VALUES ('2020-02-13', 3)", retry_count=60 + ) exec_query_with_retry(node1, "SYSTEM SYNC REPLICA mt", retry_count=60) assert node1.query("SELECT COUNT() FROM mt") == "2\n" @@ -62,8 +84,10 @@ def test_mutate_and_upgrade(start_cluster): assert node1.query("SELECT COUNT() FROM mt") == "2\n" assert node2.query("SELECT COUNT() FROM mt") == "2\n" - node1.query("ALTER TABLE mt MODIFY COLUMN id Int32 DEFAULT 0", - settings={"replication_alter_partitions_sync": "2"}) + node1.query( + "ALTER TABLE mt MODIFY COLUMN id Int32 DEFAULT 0", + settings={"replication_alter_partitions_sync": "2"}, + ) node2.query("OPTIMIZE TABLE mt FINAL") @@ -78,7 +102,8 @@ def test_upgrade_while_mutation(start_cluster): node3.query("DROP TABLE IF EXISTS mt1") node3.query( - "CREATE TABLE mt1 (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', 'node3') ORDER BY tuple()") + "CREATE TABLE mt1 (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', 'node3') ORDER BY tuple()" + ) node3.query("INSERT INTO mt1 select '2020-02-13', number from numbers(100000)") @@ -91,7 +116,9 @@ def test_upgrade_while_mutation(start_cluster): # checks for readonly exec_query_with_retry(node3, "OPTIMIZE TABLE mt1", sleep_time=5, retry_count=60) - node3.query("ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) + node3.query( + "ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"} + ) # will delete nothing, but previous async mutation will finish with this query assert_eq_with_retry(node3, "SELECT COUNT() from mt1", "50000\n") diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 95d9db27a7d..d3d90ca0d4f 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -3,20 +3,33 @@ import pytest import logging from helpers.cluster import ClickHouseCluster -cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_root_a.xml') +cluster = ClickHouseCluster( + __file__, zookeeper_config_path="configs/zookeeper_config_root_a.xml" +) + +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], +) +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"], +) +node3 = cluster.add_instance( + "node3", + with_zookeeper=True, + main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_b.xml"], +) -node1 = cluster.add_instance('node1', with_zookeeper=True, - main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"]) -node2 = cluster.add_instance('node2', with_zookeeper=True, - main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_a.xml"]) -node3 = cluster.add_instance('node3', with_zookeeper=True, - main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_root_b.xml"]) def create_zk_roots(zk): - zk.ensure_path('/root_a') - zk.ensure_path('/root_b') + zk.ensure_path("/root_a") + zk.ensure_path("/root_b") logging.debug(f"Create ZK roots:{zk.get_children('/')}") + @pytest.fixture(scope="module", autouse=True) def started_cluster(): try: @@ -28,30 +41,40 @@ def started_cluster(): finally: cluster.shutdown() + def test_chroot_with_same_root(started_cluster): for i, node in enumerate([node1, node2]): - node.query('DROP TABLE IF EXISTS simple SYNC') - node.query(''' + node.query("DROP TABLE IF EXISTS simple SYNC") + node.query( + """ CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) for j in range(2): # Second insert to test deduplication node.query("INSERT INTO simple VALUES ({0}, {0})".format(i)) time.sleep(1) - assert node1.query('select count() from simple').strip() == '2' - assert node2.query('select count() from simple').strip() == '2' + assert node1.query("select count() from simple").strip() == "2" + assert node2.query("select count() from simple").strip() == "2" + def test_chroot_with_different_root(started_cluster): for i, node in [(1, node1), (3, node3)]: - node.query('DROP TABLE IF EXISTS simple_different SYNC') - node.query(''' + node.query("DROP TABLE IF EXISTS simple_different SYNC") + node.query( + """ CREATE TABLE simple_different (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}', date, id, 8192); - '''.format(replica=node.name)) + """.format( + replica=node.name + ) + ) for j in range(2): # Second insert to test deduplication node.query("INSERT INTO simple_different VALUES ({0}, {0})".format(i)) - assert node1.query('select count() from simple_different').strip() == '1' - assert node3.query('select count() from simple_different').strip() == '1' + assert node1.query("select count() from simple_different").strip() == "1" + assert node3.query("select count() from simple_different").strip() == "1" diff --git a/tests/integration/test_zookeeper_config/test_password.py b/tests/integration/test_zookeeper_config/test_password.py index 09c15cfd0cf..580b426db6f 100644 --- a/tests/integration/test_zookeeper_config/test_password.py +++ b/tests/integration/test_zookeeper_config/test_password.py @@ -1,5 +1,3 @@ - - import time import pytest from helpers.cluster import ClickHouseCluster @@ -7,10 +5,19 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__, name="password") # TODO ACL not implemented in Keeper. -node1 = cluster.add_instance('node1', with_zookeeper=True, - main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_with_password.xml"]) - -node2 = cluster.add_instance('node2', with_zookeeper=True, main_configs=["configs/remote_servers.xml"]) +node1 = cluster.add_instance( + "node1", + with_zookeeper=True, + main_configs=[ + "configs/remote_servers.xml", + "configs/zookeeper_config_with_password.xml", + ], +) + +node2 = cluster.add_instance( + "node2", with_zookeeper=True, main_configs=["configs/remote_servers.xml"] +) + @pytest.fixture(scope="module", autouse=True) def started_cluster(): @@ -21,16 +28,23 @@ def started_cluster(): finally: cluster.shutdown() -def test_identity(started_cluster): - node1.query('DROP TABLE IF EXISTS simple SYNC') - node1.query(''' +def test_identity(started_cluster): + node1.query("DROP TABLE IF EXISTS simple SYNC") + + node1.query( + """ CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); - '''.format(replica=node1.name)) + """.format( + replica=node1.name + ) + ) with pytest.raises(Exception): - node2.query(''' + node2.query( + """ CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1', date, id, 8192); - ''') + """ + ) diff --git a/tests/integration/test_zookeeper_config/test_secure.py b/tests/integration/test_zookeeper_config/test_secure.py index c0b2216d677..f540a504024 100644 --- a/tests/integration/test_zookeeper_config/test_secure.py +++ b/tests/integration/test_zookeeper_config/test_secure.py @@ -1,5 +1,5 @@ import threading -import os +import os from tempfile import NamedTemporaryFile import pytest @@ -7,20 +7,36 @@ from helpers.cluster import ClickHouseCluster TEST_DIR = os.path.dirname(__file__) -cluster = ClickHouseCluster(__file__, name="secure", - zookeeper_certfile=os.path.join(TEST_DIR, "configs_secure", "client.crt"), - zookeeper_keyfile=os.path.join(TEST_DIR, "configs_secure", "client.key")) +cluster = ClickHouseCluster( + __file__, + name="secure", + zookeeper_certfile=os.path.join(TEST_DIR, "configs_secure", "client.crt"), + zookeeper_keyfile=os.path.join(TEST_DIR, "configs_secure", "client.key"), +) + +node1 = cluster.add_instance( + "node1", + main_configs=[ + "configs_secure/client.crt", + "configs_secure/client.key", + "configs_secure/conf.d/remote_servers.xml", + "configs_secure/conf.d/ssl_conf.xml", + "configs/zookeeper_config_with_ssl.xml", + ], + with_zookeeper_secure=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=[ + "configs_secure/client.crt", + "configs_secure/client.key", + "configs_secure/conf.d/remote_servers.xml", + "configs_secure/conf.d/ssl_conf.xml", + "configs/zookeeper_config_with_ssl.xml", + ], + with_zookeeper_secure=True, +) -node1 = cluster.add_instance('node1', main_configs=["configs_secure/client.crt", "configs_secure/client.key", - "configs_secure/conf.d/remote_servers.xml", - "configs_secure/conf.d/ssl_conf.xml", - "configs/zookeeper_config_with_ssl.xml"], - with_zookeeper_secure=True) -node2 = cluster.add_instance('node2', main_configs=["configs_secure/client.crt", "configs_secure/client.key", - "configs_secure/conf.d/remote_servers.xml", - "configs_secure/conf.d/ssl_conf.xml", - "configs/zookeeper_config_with_ssl.xml"], - with_zookeeper_secure=True) @pytest.fixture(scope="module", autouse=True) def started_cluster(): @@ -31,6 +47,7 @@ def started_cluster(): finally: cluster.shutdown() + # NOTE this test have to be ported to Keeper def test_secure_connection(started_cluster): # no asserts, connection works @@ -43,8 +60,18 @@ def test_secure_connection(started_cluster): # just checking for race conditions for _ in range(threads_number): - threads.append(threading.Thread(target=(lambda: - [node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'") for _ in range(iterations)]))) + threads.append( + threading.Thread( + target=( + lambda: [ + node1.query( + "SELECT count() FROM system.zookeeper WHERE path = '/'" + ) + for _ in range(iterations) + ] + ) + ) + ) for thread in threads: thread.start() diff --git a/tests/performance/README.md b/tests/performance/README.md index 69d97e5c3d2..2391a1a2ece 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -21,6 +21,6 @@ TODO @akuzm ### How to validate single test ``` -pip3 install clickhouse_driver +pip3 install clickhouse_driver scipy ../../docker/test/performance-comparison/perf.py --runs 1 insert_parallel.xml ``` diff --git a/tests/performance/asof.xml b/tests/performance/asof.xml new file mode 100644 index 00000000000..d1c5ead0e65 --- /dev/null +++ b/tests/performance/asof.xml @@ -0,0 +1,50 @@ + + + + test.hits + + + + SELECT * + FROM + ( + SELECT + RegionID, + toDateTime(EventDate) AS date, + count() AS quantity + FROM test.hits + GROUP BY + RegionID, + date + ) AS regions + ASOF LEFT JOIN + ( + SELECT + RegionID, + EventTime + FROM test.hits + ) AS ids ON (regions.RegionID = ids.RegionID) AND (regions.date < ids.EventTime) + + + + SELECT * FROM + ( + SELECT + toStartOfDay(now()) + INTERVAL (seconds_since_start_of_day % 86000) SECOND AS date, + fingerprint % 50 as fingerprint, + multiIf(browserId % 3 == 0, 'firefox', browserId % 3 == 1, 'edge', 'chrome') as browser + FROM generateRandom('seconds_since_start_of_day UInt32, fingerprint UInt8, browserId UInt8') LIMIT 100000 + ) AS origins + ASOF LEFT JOIN + ( + SELECT + toStartOfDay(now()) + INTERVAL (seconds_since_start_of_day % 86000) SECOND AS date, + fingerprint % 50 as fingerprint, + multiIf(language % 2 == 0, 'es', 'en') as lang + FROM generateRandom('seconds_since_start_of_day UInt32, fingerprint UInt8, language UInt8') LIMIT 5000000 + ) AS visits + ON (visits.fingerprint = origins.fingerprint AND visits.date >= origins.date) + FORMAT Null + + + diff --git a/tests/performance/classification.xml b/tests/performance/classification.xml index 370e2c49d29..9c55a6c7f29 100644 --- a/tests/performance/classification.xml +++ b/tests/performance/classification.xml @@ -7,14 +7,14 @@ hits_100m_single - SELECT detectLanguage(SearchPhrase) FROM hits_100m_single FORMAT Null - SELECT detectLanguageMixed(SearchPhrase) FROM hits_100m_single FORMAT Null + SELECT detectLanguage(SearchPhrase) FROM hits_100m_single LIMIT 10000000 FORMAT Null + SELECT detectLanguageMixed(SearchPhrase) FROM hits_100m_single LIMIT 10000000 FORMAT Null SELECT detectTonality(SearchPhrase) FROM hits_100m_single FORMAT Null - SELECT detectProgrammingLanguage(SearchPhrase) FROM hits_100m_single FORMAT Null - SELECT detectLanguageUnknown(SearchPhrase) FROM hits_100m_single FORMAT Null - SELECT detectCharset(SearchPhrase) FROM hits_100m_single FORMAT Null + SELECT detectProgrammingLanguage(SearchPhrase) FROM hits_100m_single LIMIT 10000000 FORMAT Null + SELECT detectLanguageUnknown(SearchPhrase) FROM hits_100m_single LIMIT 500000 FORMAT Null + SELECT detectCharset(SearchPhrase) FROM hits_100m_single LIMIT 500000 FORMAT Null diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index 0c3d85f9659..f210c807b12 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -83,7 +83,7 @@ time_zone UTC - Europe/Moscow + Asia/Istanbul Asia/Kolkata diff --git a/tests/performance/date_time_short.xml b/tests/performance/date_time_short.xml index 826e1619ab7..de859710670 100644 --- a/tests/performance/date_time_short.xml +++ b/tests/performance/date_time_short.xml @@ -18,7 +18,7 @@ time_zone - Europe/Moscow + Asia/Istanbul diff --git a/tests/performance/generate_table_function.xml b/tests/performance/generate_table_function.xml index bc49a7de1bd..c219d73b6cf 100644 --- a/tests/performance/generate_table_function.xml +++ b/tests/performance/generate_table_function.xml @@ -4,8 +4,8 @@ SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5)))', 0, 10, 10) LIMIT 100000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200))', 0, 10, 10) LIMIT 1000000000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 0, 10, 10) LIMIT 1000000000); - SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 0, 10, 10) LIMIT 100000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Asia/Istanbul\')', 0, 10, 10) LIMIT 1000000000); + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Asia/Istanbul\')', 0, 10, 10) LIMIT 100000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('f32 Float32, f64 Float64', 0, 10, 10) LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 0, 10, 10) LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Int64)', 0, 10, 10) LIMIT 1000000000); diff --git a/tests/performance/h3.xml b/tests/performance/h3.xml index ce00ebbc9ec..5b697884c42 100644 --- a/tests/performance/h3.xml +++ b/tests/performance/h3.xml @@ -1,6 +1,4 @@ - - - SELECT count() FROM zeros(100000) WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, 15)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(geoToH3(37.62 + rand(1) / 0x100000000, 55.75 + rand(2) / 0x100000000, toUInt8(15))) diff --git a/tests/performance/merge_tree_insert.xml b/tests/performance/merge_tree_insert.xml new file mode 100644 index 00000000000..1e987d27d50 --- /dev/null +++ b/tests/performance/merge_tree_insert.xml @@ -0,0 +1,41 @@ + + + + + integer_primary_key_table_name + + merge_tree_insert_1 + merge_tree_insert_2 + merge_tree_insert_3 + + + + + string_primary_key_table_name + + merge_tree_insert_4 + merge_tree_insert_5 + merge_tree_insert_6 + + + + + CREATE TABLE merge_tree_insert_1 (value_1 UInt64, value_2 UInt64, value_3 UInt64) ENGINE = MergeTree ORDER BY (value_1) + CREATE TABLE merge_tree_insert_2 (value_1 UInt64, value_2 UInt64, value_3 UInt64) ENGINE = MergeTree ORDER BY (value_1, value_2) + CREATE TABLE merge_tree_insert_3 (value_1 UInt64, value_2 UInt64, value_3 UInt64) ENGINE = MergeTree ORDER BY (value_1, value_2, value_3) + CREATE TABLE merge_tree_insert_4 (value_1 String, value_2 String, value_3 String) ENGINE = MergeTree ORDER BY (value_1) + CREATE TABLE merge_tree_insert_5 (value_1 String, value_2 String, value_3 String) ENGINE = MergeTree ORDER BY (value_1, value_2) + CREATE TABLE merge_tree_insert_6 (value_1 String, value_2 String, value_3 String) ENGINE = MergeTree ORDER BY (value_1, value_2, value_3) + + INSERT INTO {integer_primary_key_table_name} SELECT rand64(0), rand64(1), rand64(2) FROM system.numbers LIMIT 500000 + INSERT INTO {integer_primary_key_table_name} SELECT rand64(0), rand64(1), rand64(2) FROM system.numbers LIMIT 1000000 + INSERT INTO {integer_primary_key_table_name} SELECT rand64(0), rand64(1), rand64(2) FROM system.numbers LIMIT 1500000 + + INSERT INTO {string_primary_key_table_name} SELECT toString(rand64(0)), toString(rand64(1)), toString(rand64(2)) FROM system.numbers LIMIT 500000 + INSERT INTO {string_primary_key_table_name} SELECT toString(rand64(0)), toString(rand64(1)), toString(rand64(2)) FROM system.numbers LIMIT 1000000 + INSERT INTO {string_primary_key_table_name} SELECT toString(rand64(0)), toString(rand64(1)), toString(rand64(2)) FROM system.numbers LIMIT 1500000 + + DROP TABLE IF EXISTS {integer_primary_key_table_name} + DROP TABLE IF EXISTS {string_primary_key_table_name} + + diff --git a/tests/performance/merge_tree_many_partitions.xml b/tests/performance/merge_tree_many_partitions.xml index 2a8a52943a3..5de6061abf3 100644 --- a/tests/performance/merge_tree_many_partitions.xml +++ b/tests/performance/merge_tree_many_partitions.xml @@ -1,11 +1,13 @@ - CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x - INSERT INTO bad_partitions SELECT * FROM numbers(10000) - 0 + 1 + 20G + CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x + INSERT INTO bad_partitions SELECT * FROM numbers(10000) + SELECT count() FROM bad_partitions DROP TABLE IF EXISTS bad_partitions diff --git a/tests/performance/merge_tree_many_partitions_2.xml b/tests/performance/merge_tree_many_partitions_2.xml index 6799153ed65..a265713269f 100644 --- a/tests/performance/merge_tree_many_partitions_2.xml +++ b/tests/performance/merge_tree_many_partitions_2.xml @@ -1,14 +1,14 @@ - CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x - INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) - - - 0 + 1 + 20G + CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x + INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) + SELECT sum(ignore(*)) FROM bad_partitions DROP TABLE IF EXISTS bad_partitions diff --git a/tests/performance/parallel_final.xml b/tests/performance/parallel_final.xml index bd6a921fc68..775926d1ee8 100644 --- a/tests/performance/parallel_final.xml +++ b/tests/performance/parallel_final.xml @@ -2,7 +2,8 @@ 1024 - 16 + 1 + 20G diff --git a/tests/performance/read_in_order_many_parts.xml b/tests/performance/read_in_order_many_parts.xml index 065d12fadd2..d0b2c0e87c8 100644 --- a/tests/performance/read_in_order_many_parts.xml +++ b/tests/performance/read_in_order_many_parts.xml @@ -5,6 +5,7 @@ 2000 10000000 8 + 15G diff --git a/tests/performance/sparse_column.xml b/tests/performance/sparse_column.xml index 6523d37df44..1d270165c68 100644 --- a/tests/performance/sparse_column.xml +++ b/tests/performance/sparse_column.xml @@ -25,7 +25,7 @@ CREATE TABLE test_sparse_{ratio} (id UInt64, u8 UInt8, u64 UInt64, str String) ENGINE = MergeTree ORDER BY id - SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9 + SETTINGS ratio_of_defaults_for_sparse_serialization = 0.8 SYSTEM STOP MERGES test_{serialization}_{ratio} @@ -54,5 +54,8 @@ SELECT sum(u64) FROM test_{serialization}_{ratio} GROUP BY id % 11 FORMAT Null SELECT uniq(str) FROM test_{serialization}_{ratio} GROUP BY id % 11 FORMAT Null - + SELECT count() FROM test_{serialization}_{ratio} WHERE u64 > 0 + SELECT count() FROM test_{serialization}_{ratio} WHERE notEmpty(str) + + DROP TABLE IF EXISTS test_{serialization}_{ratio} diff --git a/tests/queries/0_stateless/00076_ip_coding_functions.sql b/tests/queries/0_stateless/00076_ip_coding_functions.sql index 659267c61ed..f693b336e57 100644 --- a/tests/queries/0_stateless/00076_ip_coding_functions.sql +++ b/tests/queries/0_stateless/00076_ip_coding_functions.sql @@ -1,3 +1,5 @@ +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + select IPv4StringToNum('') == 0; select IPv4StringToNum(materialize('')) == 0; select IPv4StringToNum('not an ip string') == 0; diff --git a/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql b/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql index f9cbf92db41..e29a166c1ee 100644 --- a/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql +++ b/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql @@ -1,5 +1,6 @@ -- Tags: replica, distributed +SET allow_experimental_parallel_reading_from_replicas = 0; SET max_parallel_replicas = 2; DROP TABLE IF EXISTS report; diff --git a/tests/queries/0_stateless/00135_duplicate_group_by_keys_segfault.sql b/tests/queries/0_stateless/00135_duplicate_group_by_keys_segfault.sql index 16356046a36..c54593056cf 100644 --- a/tests/queries/0_stateless/00135_duplicate_group_by_keys_segfault.sql +++ b/tests/queries/0_stateless/00135_duplicate_group_by_keys_segfault.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-settings + SET max_rows_to_read = 1000000; SET read_overflow_mode = 'break'; SELECT concat(toString(number % 256 AS n), '') AS s, n, max(s) FROM system.numbers_mt GROUP BY s, n, n, n, n, n, n, n, n, n ORDER BY s, n; diff --git a/tests/queries/0_stateless/00161_rounding_functions.reference b/tests/queries/0_stateless/00161_rounding_functions.reference index f0b1bc3f8ab..cca2ed3e0f7 100644 --- a/tests/queries/0_stateless/00161_rounding_functions.reference +++ b/tests/queries/0_stateless/00161_rounding_functions.reference @@ -682,3 +682,5 @@ 12345.6789 12340 12300 12000 10000 0 12345.6 12345.67 12345.678 12345.6789 12345.6789 64 64 2 0 0 0.5 0 -0.5 -0.5 -0.125 +2 20 200 5 50 500 5 50 500 +2 20 200 5 50 500 5 50 500 diff --git a/tests/queries/0_stateless/00161_rounding_functions.sql b/tests/queries/0_stateless/00161_rounding_functions.sql index cc3542338bb..abdc1e7317b 100644 --- a/tests/queries/0_stateless/00161_rounding_functions.sql +++ b/tests/queries/0_stateless/00161_rounding_functions.sql @@ -44,4 +44,7 @@ SELECT 12345.6789 AS x, floor(x, -1), floor(x, -2), floor(x, -3), floor(x, -4), SELECT roundToExp2(100), roundToExp2(64), roundToExp2(3), roundToExp2(0), roundToExp2(-1); SELECT roundToExp2(0.9), roundToExp2(0), roundToExp2(-0.5), roundToExp2(-0.6), roundToExp2(-0.2); +select round(2, 4) round2, round(20, 4) round20, round(200, 4) round200, round(5, 4) round5, round(50, 4) round50, round(500, 4) round500, round(toInt32(5), 4) roundInt5, round(toInt32(50), 4) roundInt50, round(toInt32(500), 4) roundInt500; +select roundBankers(2, 4) round2, roundBankers(20, 4) round20, roundBankers(200, 4) round200, roundBankers(5, 4) round5, roundBankers(50, 4) round50, roundBankers(500, 4) round500, roundBankers(toInt32(5), 4) roundInt5, roundBankers(toInt32(50), 4) roundInt50, roundBankers(toInt32(500), 4) roundInt500; + SELECT ceil(29375422, -54212) --{serverError 69} diff --git a/tests/queries/0_stateless/00183_skip_unavailable_shards.sql b/tests/queries/0_stateless/00183_skip_unavailable_shards.sql index 0f05618e746..71187d75fe6 100644 --- a/tests/queries/0_stateless/00183_skip_unavailable_shards.sql +++ b/tests/queries/0_stateless/00183_skip_unavailable_shards.sql @@ -1,4 +1,4 @@ --- Tags: shard +-- Tags: shard, no-fasttest SET send_logs_level = 'fatal'; SELECT count() FROM remote('{127,1}.0.0.{2,3}', system.one) SETTINGS skip_unavailable_shards = 1; diff --git a/tests/queries/0_stateless/00284_external_aggregation.sql b/tests/queries/0_stateless/00284_external_aggregation.sql index cd9abec59a8..057cb749521 100644 --- a/tests/queries/0_stateless/00284_external_aggregation.sql +++ b/tests/queries/0_stateless/00284_external_aggregation.sql @@ -1,5 +1,9 @@ +-- Tags: long + SET max_bytes_before_external_group_by = 100000000; -SET max_memory_usage = 351000000; +SET max_memory_usage = 410000000; +SET group_by_two_level_threshold = 100000; +SET group_by_two_level_threshold_bytes = 50000000; SELECT sum(k), sum(c) FROM (SELECT number AS k, count() AS c FROM (SELECT * FROM system.numbers LIMIT 10000000) GROUP BY k); SELECT sum(k), sum(c), max(u) FROM (SELECT number AS k, count() AS c, uniqArray(range(number % 16)) AS u FROM (SELECT * FROM system.numbers LIMIT 1000000) GROUP BY k); diff --git a/tests/queries/0_stateless/00398_url_functions.reference b/tests/queries/0_stateless/00398_url_functions.reference index e5f89fbea78..feba95fb1b3 100644 --- a/tests/queries/0_stateless/00398_url_functions.reference +++ b/tests/queries/0_stateless/00398_url_functions.reference @@ -59,6 +59,14 @@ query=hello world+foo+bar query=hello world+foo+bar query=hello world+foo+bar query=hello world foo+bar + + +\N +\N +hello%20world%20foo%2Bbar +hello+world+foo%2Bbar +http://paul@127.0.0.1/?query=hello world foo+bar +http://paul@127.0.0.1/?query=hello world foo+bar ====FRAGMENT==== @@ -74,6 +82,10 @@ query=hello world+foo+bar#a=b query=hello world+foo+bar#a=b #a=b query=hello world foo+bar#a=b +hello%20world%20foo%2Bbar%23a%3Db +hello+world+foo%2Bbar%23a%3Db +http://paul@127.0.0.1/?query=hello world foo+bar#a=b +http://paul@127.0.0.1/?query=hello world foo+bar#a=b ====CUT TO FIRST SIGNIFICANT SUBDOMAIN==== example.com example.com diff --git a/tests/queries/0_stateless/00398_url_functions.sql b/tests/queries/0_stateless/00398_url_functions.sql index ea71ed226d7..66fe591bb58 100644 --- a/tests/queries/0_stateless/00398_url_functions.sql +++ b/tests/queries/0_stateless/00398_url_functions.sql @@ -64,6 +64,14 @@ SELECT decodeURLComponent(queryString('http://127.0.0.1:443/?query=hello%20world SELECT decodeURLComponent(queryString('http://paul@127.0.0.1:443/?query=hello%20world+foo%2Bbar')); SELECT decodeURLComponent(queryString('//paul@127.0.0.1:443/?query=hello%20world+foo%2Bbar')); SELECT decodeURLFormComponent(queryString('//paul@127.0.0.1:443/?query=hello%20world+foo%2Bbar')); +SELECT encodeURLComponent(''); +SELECT encodeURLFormComponent(''); +SELECT encodeURLComponent(NULL); +SELECT encodeURLFormComponent(NULL); +SELECT encodeURLComponent('hello world foo+bar'); +SELECT encodeURLFormComponent('hello world foo+bar'); +SELECT decodeURLComponent(encodeURLComponent('http://paul@127.0.0.1/?query=hello world foo+bar')); +SELECT decodeURLFormComponent(encodeURLFormComponent('http://paul@127.0.0.1/?query=hello world foo+bar')); SELECT '====FRAGMENT===='; SELECT decodeURLComponent(fragment('http://127.0.0.1/?query=hello%20world+foo%2Bbar')); @@ -81,6 +89,10 @@ SELECT decodeURLComponent(queryStringAndFragment('http://paul@127.0.0.1/?query=h SELECT decodeURLComponent(queryStringAndFragment('//paul@127.0.0.1/?query=hello%20world+foo%2Bbar#a=b')); SELECT decodeURLComponent(queryStringAndFragment('//paul@127.0.0.1/#a=b')); SELECT decodeURLFormComponent(queryStringAndFragment('//paul@127.0.0.1/?query=hello%20world+foo%2Bbar#a=b')); +SELECT encodeURLComponent('hello world foo+bar#a=b'); +SELECT encodeURLFormComponent('hello world foo+bar#a=b'); +SELECT decodeURLComponent(encodeURLComponent('http://paul@127.0.0.1/?query=hello world foo+bar#a=b')); +SELECT decodeURLFormComponent(encodeURLFormComponent('http://paul@127.0.0.1/?query=hello world foo+bar#a=b')); SELECT '====CUT TO FIRST SIGNIFICANT SUBDOMAIN===='; SELECT cutToFirstSignificantSubdomain('http://www.example.com'); diff --git a/tests/queries/0_stateless/00474_readonly_settings.reference b/tests/queries/0_stateless/00474_readonly_settings.reference index b1da40ce414..e2b45931965 100644 --- a/tests/queries/0_stateless/00474_readonly_settings.reference +++ b/tests/queries/0_stateless/00474_readonly_settings.reference @@ -2,13 +2,11 @@ "value": 4611686018427387904 "name": "value", "value": "4611686018427387904" -value -value -Cannot modify 'output_format_json_quote_64bit_integers' setting in readonly mode +OK +OK "name": "value", "value": "9223372036854775808" "name": "value", "value": 9223372036854775808 -value -value -Cannot modify 'output_format_json_quote_64bit_integers' setting in readonly mode +OK +OK diff --git a/tests/queries/0_stateless/00474_readonly_settings.sh b/tests/queries/0_stateless/00474_readonly_settings.sh index 0887ecfa14e..07b78c64a7e 100755 --- a/tests/queries/0_stateless/00474_readonly_settings.sh +++ b/tests/queries/0_stateless/00474_readonly_settings.sh @@ -9,13 +9,15 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=0 | grep value $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=1 | grep value -$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode' -$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode' +$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL" +$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" | grep value ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" | grep value -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=readonly&session_timeout=3600" -d 'SET readonly = 1' +#${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=readonly&session_timeout=3600" -d 'SET readonly = 1' + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode.' && echo "OK" || echo "FAIL" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL" -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode.' -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=readonly&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" 2>&1 | grep -o 'value\|Cannot modify .* setting in readonly mode' diff --git a/tests/queries/0_stateless/00502_custom_partitioning_local.sql b/tests/queries/0_stateless/00502_custom_partitioning_local.sql index eda267c0445..b7eb08c919e 100644 --- a/tests/queries/0_stateless/00502_custom_partitioning_local.sql +++ b/tests/queries/0_stateless/00502_custom_partitioning_local.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage SELECT '*** Not partitioned ***'; DROP TABLE IF EXISTS not_partitioned; @@ -18,7 +19,7 @@ ALTER TABLE not_partitioned DETACH PARTITION ID 'all'; SELECT 'Sum after DETACH PARTITION:'; SELECT sum(x) FROM not_partitioned; SELECT 'system.detached_parts after DETACH PARTITION:'; -SELECT * FROM system.detached_parts WHERE table = 'not_partitioned'; +SELECT * FROM system.detached_parts WHERE database = currentDatabase() AND table = 'not_partitioned'; DROP TABLE not_partitioned; diff --git a/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.reference b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.reference new file mode 100644 index 00000000000..8a1218a1024 --- /dev/null +++ b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.reference @@ -0,0 +1,5 @@ +1 +2 +3 +4 +5 diff --git a/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.sql b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.sql new file mode 100644 index 00000000000..1802fadc57b --- /dev/null +++ b/tests/queries/0_stateless/00612_http_max_query_size_for_distributed.sql @@ -0,0 +1,19 @@ +-- Tags: no-parallel + +DROP TABLE IF EXISTS data_00612; +DROP TABLE IF EXISTS dist_00612; + +CREATE TABLE data_00612 (key UInt64, val UInt64) ENGINE = MergeTree ORDER BY key; +CREATE TABLE dist_00612 AS data_00612 ENGINE = Distributed(test_shard_localhost, currentDatabase(), data_00612, rand()); + +SET insert_distributed_sync=1; +SET prefer_localhost_replica=0; +SET max_query_size=29; +INSERT INTO dist_00612 VALUES(1, 1), (2, 2), (3, 3), (4, 4), (5, 5); +SELECT key FROM dist_00612; + +SET max_query_size=262144; +SET insert_distributed_sync=0; +SET prefer_localhost_replica=1; +DROP TABLE dist_00612; +DROP TABLE data_00612; diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index d78f93d6bb3..1aa02864815 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-parallel +# Tags: zookeeper, no-parallel, no-s3-storage, no-backward-compatibility-check # Because REPLACE PARTITION does not forces immediate removal of replaced data parts from local filesystem # (it tries to do it as quick as possible, but it still performed in separate thread asynchronously) diff --git a/tests/queries/0_stateless/00632_get_sample_block_cache.sql b/tests/queries/0_stateless/00632_get_sample_block_cache.sql index 3c1b7ed70e4..f9b241bbf1e 100644 --- a/tests/queries/0_stateless/00632_get_sample_block_cache.sql +++ b/tests/queries/0_stateless/00632_get_sample_block_cache.sql @@ -1,3 +1,5 @@ +-- Tags: long + SET joined_subquery_requires_alias = 0; -- This test (SELECT) without cache can take tens minutes diff --git a/tests/queries/0_stateless/00646_url_engine.python b/tests/queries/0_stateless/00646_url_engine.python index 4f47e819328..d1836817867 100644 --- a/tests/queries/0_stateless/00646_url_engine.python +++ b/tests/queries/0_stateless/00646_url_engine.python @@ -120,18 +120,14 @@ class CSVHTTPServer(BaseHTTPRequestHandler): class HTTPServerV6(HTTPServer): address_family = socket.AF_INET6 -def start_server(requests_amount): +def start_server(): if IS_IPV6: httpd = HTTPServerV6(HTTP_SERVER_ADDRESS, CSVHTTPServer) else: httpd = HTTPServer(HTTP_SERVER_ADDRESS, CSVHTTPServer) - def real_func(): - for i in range(requests_amount): - httpd.handle_request() - - t = threading.Thread(target=real_func) - return t + t = threading.Thread(target=httpd.serve_forever) + return t, httpd # test section @@ -201,7 +197,7 @@ def main(): 'select double, count(*) from {tbl} group by double': "7.7\t2\n9.9\t10" } - t = start_server(len(select_only_requests) * 2 + (len(insert_requests) + len(select_requests)) * 2) + t, httpd = start_server() t.start() # test table with url engine test_select(table_name="test_table_select", requests=list(select_only_requests.keys()), answers=list(select_only_requests.values()), test_data=test_data) @@ -211,6 +207,8 @@ def main(): test_insert(table_name="test_table_insert", requests_insert=insert_requests, requests_select=list(select_requests.keys()), answers=list(select_requests.values())) #test insert into table function url test_insert(requests_insert=insert_requests, requests_select=list(select_requests.keys()), answers=list(select_requests.values())) + + httpd.shutdown() t.join() print("PASSED") diff --git a/tests/queries/0_stateless/00702_join_on_dups.sql b/tests/queries/0_stateless/00702_join_on_dups.sql index 48c80446f83..869d39308cc 100644 --- a/tests/queries/0_stateless/00702_join_on_dups.sql +++ b/tests/queries/0_stateless/00702_join_on_dups.sql @@ -33,7 +33,7 @@ select s.*, j.* from (select * from X) as s right join (select * from Y) as j on select 'full'; select X.*, Y.* from X full join Y on X.id = Y.id order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'full subs'; -select s.*, j.* from (select * from X) as s full join (select * from Y) as j on s.id = j.id order by s.id, s.x_a; +select s.*, j.* from (select * from X) as s full join (select * from Y) as j on s.id = j.id order by s.id, s.x_a, s.x_b, j.id, j.y_a, j.y_b; --select 'full expr'; --select X.*, Y.* from X full join Y on (X.id + 1) = (Y.id + 1) order by id; @@ -48,14 +48,14 @@ select 'self inner nullable vs not nullable 2'; select Y.*, s.* from Y inner join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a, Y.y_b, s.id, s.y_a, s.y_b; select 'self left'; -select X.*, s.* from X left join (select * from X) as s on X.id = s.id order by X.id, X.x_a, s.x_a; +select X.*, s.* from X left join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self left nullable'; -select X.*, s.* from X left join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a; +select X.*, s.* from X left join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self left nullable vs not nullable'; -select X.*, s.* from X left join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a; +select X.*, s.* from X left join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; -- TODO: s.y_b == '' instead of NULL select 'self left nullable vs not nullable 2'; -select Y.*, s.* from Y left join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a; +select Y.*, s.* from Y left join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a, Y.y_b, s.id, s.y_a, s.y_b; select 'self right'; select X.*, s.* from X right join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; diff --git a/tests/queries/0_stateless/00702_join_with_using_dups.sql b/tests/queries/0_stateless/00702_join_with_using_dups.sql index cf0c053a144..1499473ed11 100644 --- a/tests/queries/0_stateless/00702_join_with_using_dups.sql +++ b/tests/queries/0_stateless/00702_join_with_using_dups.sql @@ -23,7 +23,7 @@ select 'right subs'; select s.*, j.* from (select * from X) as s right join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name; select 'full'; -select X.*, Y.* from X full join Y using id order by X.id, Y.id; +select X.*, Y.* from X full join Y using id order by X.id, Y.id, X.x_name, Y.y_name; select 'full subs'; select s.*, j.* from (select * from X) as s full join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name; diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index f9e33645527..d9a3631a7dd 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-s3-storage set -e @@ -30,7 +30,6 @@ $CLICKHOUSE_CLIENT $settings -q "$touching_many_parts_query" &> /dev/null $CLICKHOUSE_CLIENT $settings -q "SYSTEM FLUSH LOGS" - -$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase() ORDER BY event_time DESC LIMIT 1;" +$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] as opened_files FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase() ORDER BY event_time DESC, opened_files DESC LIMIT 1;" $CLICKHOUSE_CLIENT $settings -q "DROP TABLE IF EXISTS merge_tree_table;" diff --git a/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql b/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql index a1859220c6c..e8d923389e5 100644 --- a/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql +++ b/tests/queries/0_stateless/00732_quorum_insert_lost_part_and_alive_part_zookeeper_long.sql @@ -1,4 +1,4 @@ --- Tags: long, zookeeper, no-replicated-database +-- Tags: long, zookeeper, no-replicated-database, no-backward-compatibility-check -- Tag no-replicated-database: Fails due to additional replicas or shards SET send_logs_level = 'fatal'; diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index b534b1820ba..72e80956f59 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: long CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -14,7 +15,7 @@ $CLICKHOUSE_CLIENT -q "select name from system.table_functions format TSV;" > "$ # if you want long run use: env SQL_FUZZY_RUNS=100000 clickhouse-test sql_fuzzy for SQL_FUZZY_RUN in $(seq "${SQL_FUZZY_RUNS:=5}"); do - env SQL_FUZZY_RUN="$SQL_FUZZY_RUN" perl "$CURDIR"/00746_sql_fuzzy.pl | timeout 60 $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 + env SQL_FUZZY_RUN="$SQL_FUZZY_RUN" perl "$CURDIR"/00746_sql_fuzzy.pl | clickhouse_client_timeout 60 $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 if [[ $($CLICKHOUSE_CLIENT -q "SELECT 'Still alive'") != 'Still alive' ]]; then break fi diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index dc54e4630d6..ca3df7c4cd4 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/00808_not_optimize_predicate.sql b/tests/queries/0_stateless/00808_not_optimize_predicate.sql index 7c1e57706e2..ba8f5eb5753 100644 --- a/tests/queries/0_stateless/00808_not_optimize_predicate.sql +++ b/tests/queries/0_stateless/00808_not_optimize_predicate.sql @@ -1,4 +1,5 @@ SET send_logs_level = 'fatal'; +SET convert_query_to_cnf = 0; DROP TABLE IF EXISTS test_00808; CREATE TABLE test_00808(date Date, id Int8, name String, value Int64, sign Int8) ENGINE = CollapsingMergeTree(sign) ORDER BY (id, date); diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 19d9b006cd7..8e04ce7ff1e 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -7,64 +7,51 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -echo "DROP TABLE IF EXISTS concurrent_alter_column" | ${CLICKHOUSE_CLIENT} -echo "CREATE TABLE concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} - +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE IF EXISTS concurrent_alter_column; + CREATE TABLE concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple(); +" function thread1() { - while true; do - for i in {1..500}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n --query_id=alter_00816_1 - done + for i in {1..500}; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;" + done | ${CLICKHOUSE_CLIENT} -n } function thread2() { - while true; do - echo "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_2; - sleep "$(echo 0.0$RANDOM)"; - echo "ALTER TABLE concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_2; - done + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column DROP COLUMN d" } function thread3() { - while true; do - echo "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_3; - sleep "$(echo 0.0$RANDOM)"; - echo "ALTER TABLE concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_3; - done + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column DROP COLUMN e" } function thread4() { - while true; do - echo "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_4; - sleep "$(echo 0.0$RANDOM)"; - echo "ALTER TABLE concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_4; - done + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column DROP COLUMN f" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 TIMEOUT=30 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & wait -echo "DROP TABLE concurrent_alter_column NO DELAY" | ${CLICKHOUSE_CLIENT} # NO DELAY has effect only for Atomic database - -# Wait for alters and check for deadlocks (in case of deadlock this loop will not finish) -while true; do - echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter\\_00816\\_%'" | ${CLICKHOUSE_CLIENT} | grep -q -F 'alter' || break - sleep 1; -done - +$CLICKHOUSE_CLIENT -q "DROP TABLE concurrent_alter_column NO DELAY" echo 'did not crash' diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.sh b/tests/queries/0_stateless/00825_protobuf_format_persons.sh index bb376e6ed70..465b27aa683 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_persons.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_persons.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash # Tags: no-fasttest +# End-to-end test of serialization/deserialization of a table with different +# data types to/from Protobuf format. +# Cf. 02240_protobuflist_format_persons.sh + # To generate reference file for this test use the following commands: # ninja ProtobufDelimitedMessagesSerializer # build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer diff --git a/tests/queries/0_stateless/00826_cross_to_inner_join.sql b/tests/queries/0_stateless/00826_cross_to_inner_join.sql index 392ade02ab7..ce0c8ea2bfc 100644 --- a/tests/queries/0_stateless/00826_cross_to_inner_join.sql +++ b/tests/queries/0_stateless/00826_cross_to_inner_join.sql @@ -1,4 +1,6 @@ SET enable_optimize_predicate_expression = 0; +SET optimize_move_to_prewhere = 1; +SET convert_query_to_cnf = 0; select * from system.one l cross join system.one r; diff --git a/tests/queries/0_stateless/00849_multiple_comma_join_2.sql b/tests/queries/0_stateless/00849_multiple_comma_join_2.sql index 58535f556d9..eabede3ff00 100644 --- a/tests/queries/0_stateless/00849_multiple_comma_join_2.sql +++ b/tests/queries/0_stateless/00849_multiple_comma_join_2.sql @@ -1,4 +1,5 @@ SET enable_optimize_predicate_expression = 0; +SET convert_query_to_cnf = 0; DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/00900_long_parquet.reference b/tests/queries/0_stateless/00900_long_parquet.reference index 9ee4fc11a55..bbdad7243bd 100644 --- a/tests/queries/0_stateless/00900_long_parquet.reference +++ b/tests/queries/0_stateless/00900_long_parquet.reference @@ -34,32 +34,32 @@ ContextLock Number of times the lock of Context was acquired or tried to acquire. This is global lock. Query Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries. original: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 converted: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 diff: dest: -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 -80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 2004-05-06 07:08:09.012000000 +80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 2006-08-09 10:11:12.345000000 min: --128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 --108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 -127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 +-128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 2003-02-03 04:05:06.789000000 +-108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 2002-02-03 04:05:06.789000000 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 2004-05-06 07:08:09.012000000 +127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 2004-02-03 04:05:06.789000000 max: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 -80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 2006-08-09 10:11:12.345000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 dest from null: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 -\N \N \N \N \N \N \N \N \N \N \N \N \N \N +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789000000 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789000000 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789000000 +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N 1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] 1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] 2 [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] diff --git a/tests/queries/0_stateless/00900_long_parquet.sh b/tests/queries/0_stateless/00900_long_parquet.sh index 6d2d57b5a6e..5d6317b2787 100755 --- a/tests/queries/0_stateless/00900_long_parquet.sh +++ b/tests/queries/0_stateless/00900_long_parquet.sh @@ -46,20 +46,20 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types2" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types3" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types4" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(9)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(9)) ENGINE = Memory" # convert min type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date, datetime64 DateTime64(9)) ENGINE = Memory" # convert max type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime, datetime64 DateTime64(9)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06', toDateTime64('2002-02-03 04:05:06.789', 9))" # min -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06', toDateTime64('2003-02-03 04:05:06.789', 9))" # max -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06', toDateTime64('2004-02-03 04:05:06.789', 9))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" @@ -73,11 +73,11 @@ echo diff: diff "${CLICKHOUSE_TMP}"/parquet_all_types_1.dump "${CLICKHOUSE_TMP}"/parquet_all_types_2.dump ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE parquet_types2" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06', toDateTime64('2004-05-06 07:08:09.012', 9))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types3 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types3 FORMAT Parquet" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12', toDateTime64('2006-08-09 10:11:12.345', 9))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types4 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types2 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types4 FORMAT Parquet" @@ -92,9 +92,9 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types4 ORDER BY int8" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types5" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_types6" ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE parquet_types2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64(9))) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64(9))) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types5 ORDER BY int8 FORMAT Parquet" > "${CLICKHOUSE_TMP}"/parquet_all_types_5.parquet ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types5 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types6 FORMAT Parquet" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_types1 ORDER BY int8 FORMAT Parquet" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_types6 FORMAT Parquet" diff --git a/tests/queries/0_stateless/00900_long_parquet_load.reference b/tests/queries/0_stateless/00900_long_parquet_load.reference index 89da3c6fa43..6ecff505b2e 100644 --- a/tests/queries/0_stateless/00900_long_parquet_load.reference +++ b/tests/queries/0_stateless/00900_long_parquet_load.reference @@ -339,6 +339,9 @@ Code: 33. DB::ParsingEx---tion: Error while reading Parquet data: IOError: Unkno (NULL) === Try load data from single_nan.parquet \N +=== Try load data from test_setting_input_format_use_lowercase_column_name.parquet +123 1 +456 2 === Try load data from userdata1.parquet 1454486129 1 Amanda Jordan ajordan0@com.com Female 1.197.201.2 6759521864920116 Indonesia 3/8/1971 49756.53 Internal Auditor 1E+02 1454519043 2 Albert Freeman afreeman1@is.gd Male 218.111.175.34 Canada 1/16/1968 150280.17 Accountant IV diff --git a/tests/queries/0_stateless/00900_long_parquet_load.sh b/tests/queries/0_stateless/00900_long_parquet_load.sh index e47185261fb..8e6ea24edb4 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load.sh @@ -40,11 +40,13 @@ DATA_DIR=$CUR_DIR/data_parquet # Code: 349. DB::Ex---tion: Can not insert NULL data into non-nullable column "phoneNumbers": data for INSERT was parsed from stdin for NAME in $(find "$DATA_DIR"/*.parquet -print0 | xargs -0 -n 1 basename | LC_ALL=C sort); do - echo === Try load data from "$NAME" - JSON=$DATA_DIR/$NAME.json COLUMNS_FILE=$DATA_DIR/$NAME.columns + ([ -z "$PARQUET_READER" ] || [ ! -s "$PARQUET_READER" ]) && [ ! -s "$COLUMNS_FILE" ] && continue + + echo === Try load data from "$NAME" + # If you want change or add .parquet file - rm data_parquet/*.json data_parquet/*.columns [ -n "$PARQUET_READER" ] && [ ! -s "$COLUMNS_FILE" ] && [ ! -s "$JSON" ] && "$PARQUET_READER" --json "$DATA_DIR"/"$NAME" > "$JSON" [ ! -s "$COLUMNS_FILE" ] && "$CUR_DIR"/helpers/00900_parquet_create_table_columns.py "$JSON" > "$COLUMNS_FILE" diff --git a/tests/queries/0_stateless/00900_orc_load.reference b/tests/queries/0_stateless/00900_orc_load.reference index 04c72eccb17..d4590fe5790 100644 --- a/tests/queries/0_stateless/00900_orc_load.reference +++ b/tests/queries/0_stateless/00900_orc_load.reference @@ -1,4 +1,4 @@ -0 0 0 0 0 2019-01-01 test1 -2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 -0 0 0 0 0 2019-01-01 test1 -2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 +0 0 0 0 0 2019-01-01 test1 2019-01-01 02:03:04.567 +2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 2019-01-01 02:03:04.567 +0 0 0 0 0 2019-01-01 test1 2019-01-01 02:03:04.567 +2147483647 -1 9223372036854775806 123.345345 345345.3453451212 2019-01-01 test2 2019-01-01 02:03:04.567 diff --git a/tests/queries/0_stateless/00900_orc_load.sh b/tests/queries/0_stateless/00900_orc_load.sh index 4503c8eb8a5..3779092c511 100755 --- a/tests/queries/0_stateless/00900_orc_load.sh +++ b/tests/queries/0_stateless/00900_orc_load.sh @@ -5,12 +5,16 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -DATA_FILE=$CUR_DIR/data_orc/test.orc +DATA_FILE=$CUR_DIR/data_orc/test_$CLICKHOUSE_TEST_UNIQUE_NAME.orc ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String) ENGINE = Memory" -cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" -timeout 3 ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < $DATA_FILE -${CLICKHOUSE_CLIENT} --query="select * from orc_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String, datetime64 DateTime64(3)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="insert into orc_load values (0, 0, 0, 0, 0, '2019-01-01', 'test1', toDateTime64('2019-01-01 02:03:04.567', 3)), (2147483647, -1, 9223372036854775806, 123.345345, 345345.3453451212, '2019-01-01', 'test2', toDateTime64('2019-01-01 02:03:04.567', 3))" +${CLICKHOUSE_CLIENT} --query="select * from orc_load FORMAT ORC" > $DATA_FILE +${CLICKHOUSE_CLIENT} --query="truncate table orc_load" +cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" +clickhouse_client_timeout 3 ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < $DATA_FILE +${CLICKHOUSE_CLIENT} --query="select * from orc_load" ${CLICKHOUSE_CLIENT} --query="drop table orc_load" +rm -rf "$DATA_FILE" diff --git a/tests/queries/0_stateless/00909_kill_not_initialized_query.sh b/tests/queries/0_stateless/00909_kill_not_initialized_query.sh index 531652a33e7..816bab4c491 100755 --- a/tests/queries/0_stateless/00909_kill_not_initialized_query.sh +++ b/tests/queries/0_stateless/00909_kill_not_initialized_query.sh @@ -35,7 +35,7 @@ $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query='$query_to_kill' ASYNC" &>/dev/nul sleep 1 # Kill $query_for_pending SYNC. This query is not blocker, so it should be killed fast. -timeout 20 ${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE query='$query_for_pending' SYNC" &>/dev/null +clickhouse_client_timeout 20 ${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE query='$query_for_pending' SYNC" &>/dev/null # Both queries have to be killed, doesn't matter with SYNC or ASYNC kill for _ in {1..15} diff --git a/tests/queries/0_stateless/00926_geo_to_h3.reference b/tests/queries/0_stateless/00926_geo_to_h3.reference index ad594f0e81f..074584ead16 100644 --- a/tests/queries/0_stateless/00926_geo_to_h3.reference +++ b/tests/queries/0_stateless/00926_geo_to_h3.reference @@ -4,12 +4,12 @@ 644325528491955313 644325528627451570 644325529094369568 -644325528491955313 +639821928864584823 644325528491955313 644325528491955313 644325528627451570 644325529094369568 -55.720762 37.598135 644325528491955313 +55.720762 37.598135 639821928864584823 55.720762 37.598135 644325528491955313 55.72076201 37.598135 644325528491955313 55.763241 37.660183 644325528627451570 diff --git a/tests/queries/0_stateless/00926_geo_to_h3.sql b/tests/queries/0_stateless/00926_geo_to_h3.sql index ed8e154fd9e..a86548d3555 100644 --- a/tests/queries/0_stateless/00926_geo_to_h3.sql +++ b/tests/queries/0_stateless/00926_geo_to_h3.sql @@ -11,9 +11,10 @@ INSERT INTO table1 VALUES(55.72076201, 37.59813500, 15); INSERT INTO table1 VALUES(55.72076200, 37.59813500, 14); select geoToH3(37.63098076, 55.77922738, 15); +select geoToH3(37.63098076, 55.77922738, 24); -- { serverError 69 } select geoToH3(lon, lat, resolution) from table1 order by lat, lon, resolution; -select geoToH3(lon, lat, 15) AS k from table1 order by lat, lon, k; -select lat, lon, geoToH3(lon, lat, 15) AS k from table1 order by lat, lon, k; +select geoToH3(lon, lat, resolution) AS k from table1 order by lat, lon, k; +select lat, lon, geoToH3(lon, lat, resolution) AS k from table1 order by lat, lon, k; select geoToH3(lon, lat, resolution) AS k, count(*) from table1 group by k order by k; DROP TABLE table1 diff --git a/tests/queries/0_stateless/00926_multimatch.sql b/tests/queries/0_stateless/00926_multimatch.sql index 6a90bfec914..90cc289b5a5 100644 --- a/tests/queries/0_stateless/00926_multimatch.sql +++ b/tests/queries/0_stateless/00926_multimatch.sql @@ -1,5 +1,4 @@ --- Tags: no-fasttest --- Tag no-fasttest: Hyperscan +-- Tags: no-fasttest, use-hyperscan select 0 = multiMatchAny(materialize('mpnsguhwsitzvuleiwebwjfitmsg'), ['wbirxqoabpblrnvvmjizj', 'cfcxhuvrexyzyjsh', 'oldhtubemyuqlqbwvwwkwin', 'bumoozxdkjglzu', 'intxlfohlxmajjomw', 'dxkeghohv', 'arsvmwwkjeopnlwnan', 'ouugllgowpqtaxslcopkytbfhifaxbgt', 'hkedmjlbcrzvryaopjqdjjc', 'tbqkljywstuahzh', 'o', 'wowoclosyfcuwotmvjygzuzhrery', 'vpefjiffkhlggntcu', 'ytdixvasrorhripzfhjdmlhqksmctyycwp']) from system.numbers limit 10; select 0 = multiMatchAny(materialize('qjjzqexjpgkglgxpzrbqbnskq'), ['vaiatcjacmlffdzsejpdareqzy', 'xspcfzdufkmecud', 'bcvtbuqtctq', 'nkcopwbfytgemkqcfnnno', 'dylxnzuyhq', 'tno', 'scukuhufly', 'cdyquzuqlptv', 'ohluyfeksyxepezdhqmtfmgkvzsyph', 'ualzwtahvqvtijwp', 'jg', 'gwbawqlngzcknzgtmlj', 'qimvjcgbkkp', 'eaedbcgyrdvv', 'qcwrncjoewwedyyewcdkh', 'uqcvhngoqngmitjfxpznqomertqnqcveoqk', 'ydrgjiankgygpm', 'axepgap']) from system.numbers limit 10; diff --git a/tests/queries/0_stateless/00929_multi_match_edit_distance.sql b/tests/queries/0_stateless/00929_multi_match_edit_distance.sql index 176e2a24b28..5f13477aa28 100644 --- a/tests/queries/0_stateless/00929_multi_match_edit_distance.sql +++ b/tests/queries/0_stateless/00929_multi_match_edit_distance.sql @@ -1,5 +1,4 @@ --- Tags: no-fasttest --- Tag no-fasttest: Hyperscan +-- Tags: no-fasttest, use-hyperscan SET send_logs_level = 'fatal'; diff --git a/tests/queries/0_stateless/00938_ipv6_cidr_range.sql b/tests/queries/0_stateless/00938_ipv6_cidr_range.sql index 3fa4c7c5d3f..1ceefa8cfb3 100644 --- a/tests/queries/0_stateless/00938_ipv6_cidr_range.sql +++ b/tests/queries/0_stateless/00938_ipv6_cidr_range.sql @@ -9,7 +9,7 @@ SELECT 'tests'; DROP TABLE IF EXISTS ipv6_range; CREATE TABLE ipv6_range(ip IPv6, cidr UInt8) ENGINE = Memory; -INSERT INTO ipv6_range (ip, cidr) VALUES (IPv6StringToNum('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 0), (IPv6StringToNum('2001:0db8:0000:85a3:ffff:ffff:ffff:ffff'), 32), (IPv6StringToNum('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'), 16), (IPv6StringToNum('2001:df8:0:85a3::ac1f:8001'), 32), (IPv6StringToNum('2001:0db8:85a3:85a3:0000:0000:ac1f:8001'), 16), (IPv6StringToNum('0000:0000:0000:0000:0000:0000:0000:0000'), 8), (IPv6StringToNum('ffff:0000:0000:0000:0000:0000:0000:0000'), 4); +INSERT INTO ipv6_range (ip, cidr) VALUES ('2001:0db8:0000:85a3:0000:0000:ac1f:8001', 0), ('2001:0db8:0000:85a3:ffff:ffff:ffff:ffff', 32), ('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', 16), ('2001:df8:0:85a3::ac1f:8001', 32), ('2001:0db8:85a3:85a3:0000:0000:ac1f:8001', 16), ('0000:0000:0000:0000:0000:0000:0000:0000', 8), ('ffff:0000:0000:0000:0000:0000:0000:0000', 4); WITH IPv6CIDRToRange(IPv6StringToNum('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32) as ip_range SELECT COUNT(*) FROM ipv6_range WHERE ip BETWEEN tupleElement(ip_range, 1) AND tupleElement(ip_range, 2); diff --git a/tests/queries/0_stateless/00941_system_columns_race_condition.sh b/tests/queries/0_stateless/00941_system_columns_race_condition.sh index 69dfb30cd2c..24f159c7d24 100755 --- a/tests/queries/0_stateless/00941_system_columns_race_condition.sh +++ b/tests/queries/0_stateless/00941_system_columns_race_condition.sh @@ -9,40 +9,45 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table" -$CLICKHOUSE_CLIENT -q "CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a" +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE IF EXISTS alter_table; + CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a; +" function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - while true; do $CLICKHOUSE_CLIENT --query "SELECT name FROM system.columns UNION ALL SELECT name FROM system.columns FORMAT Null"; done + $CLICKHOUSE_CLIENT --query "SELECT name FROM system.columns UNION ALL SELECT name FROM system.columns FORMAT Null" } function thread2() { - while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;"; done + $CLICKHOUSE_CLIENT -n --query " + ALTER TABLE alter_table ADD COLUMN h String; + ALTER TABLE alter_table MODIFY COLUMN h UInt64; + ALTER TABLE alter_table DROP COLUMN h; + " } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; +export -f thread1 +export -f thread2 -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread1 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & -timeout 15 bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread1 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +clickhouse_client_loop_timeout 15 thread2 2> /dev/null & wait diff --git a/tests/queries/0_stateless/00942_dataparts_500.sh b/tests/queries/0_stateless/00942_dataparts_500.sh index 19cb1138aa8..7e1a7f15810 100755 --- a/tests/queries/0_stateless/00942_dataparts_500.sh +++ b/tests/queries/0_stateless/00942_dataparts_500.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# Tags: no-backward-compatibility-check # Test fix for issue #5066 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/00960_live_view_watch_events_live.py b/tests/queries/0_stateless/00960_live_view_watch_events_live.py index 9327bc59f80..46c561516ba 100755 --- a/tests/queries/0_stateless/00960_live_view_watch_events_live.py +++ b/tests/queries/0_stateless/00960_live_view_watch_events_live.py @@ -6,45 +6,47 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - client1.send('WATCH test.lv EVENTS') - client1.expect('version') - client1.expect('1.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect('2.*' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect('3.*' + end_of_block) + client1.send("WATCH test.lv EVENTS") + client1.expect("version") + client1.expect("1.*" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (1),(2),(3)") + client1.expect("2.*" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (4),(5),(6)") + client1.expect("3.*" + end_of_block) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh.py b/tests/queries/0_stateless/00962_live_view_periodic_refresh.py index 5dd357a314a..ac399d3c4c8 100755 --- a/tests/queries/0_stateless/00962_live_view_periodic_refresh.py +++ b/tests/queries/0_stateless/00962_live_view_periodic_refresh.py @@ -6,38 +6,41 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send("CREATE LIVE VIEW test.lv WITH REFRESH 1" - " AS SELECT value FROM system.events WHERE event = 'OSCPUVirtualTimeMicroseconds'") + client1.send( + "CREATE LIVE VIEW test.lv WITH REFRESH 1" + " AS SELECT value FROM system.events WHERE event = 'OSCPUVirtualTimeMicroseconds'" + ) client1.expect(prompt) - client1.send('WATCH test.lv FORMAT JSONEachRow') + client1.send("WATCH test.lv FORMAT JSONEachRow") client1.expect(r'"_version":' + end_of_block) client1.expect(r'"_version":' + end_of_block) client1.expect(r'"_version":' + end_of_block) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py b/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py index 95b5530436d..3bc649e92dc 100755 --- a/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py +++ b/tests/queries/0_stateless/00962_live_view_periodic_refresh_and_timeout.py @@ -7,48 +7,52 @@ import time import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send("CREATE LIVE VIEW test.lv WITH TIMEOUT 60 AND REFRESH 1" - " AS SELECT value FROM system.events WHERE event = 'OSCPUVirtualTimeMicroseconds'") + client1.send( + "CREATE LIVE VIEW test.lv WITH TIMEOUT 60 AND REFRESH 1" + " AS SELECT value FROM system.events WHERE event = 'OSCPUVirtualTimeMicroseconds'" + ) client1.expect(prompt) - client1.send('WATCH test.lv FORMAT JSONEachRow') + client1.send("WATCH test.lv FORMAT JSONEachRow") client1.expect(r'"_version":' + end_of_block) client1.expect(r'"_version":' + end_of_block) client1.expect(r'"_version":' + end_of_block) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) # poll until live view table is dropped start_time = time.time() while True: - client1.send('SELECT * FROM test.lv FORMAT JSONEachRow') + client1.send("SELECT * FROM test.lv FORMAT JSONEachRow") client1.expect(prompt) - if 'Table test.lv doesn\'t exist' in client1.before: + if "Table test.lv doesn't exist" in client1.before: break if time.time() - start_time > 90: break # check table is dropped - client1.send('DROP TABLE test.lv') - client1.expect('Table test.lv doesn\'t exist') + client1.send("DROP TABLE test.lv") + client1.expect("Table test.lv doesn't exist") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py b/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py index 57cb57d03c5..9d2a26c83c0 100755 --- a/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py +++ b/tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py @@ -6,65 +6,68 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('DROP DICTIONARY IF EXITS test.dict') + client1.send("DROP DICTIONARY IF EXITS test.dict") client1.expect(prompt) - - client1.send("CREATE TABLE test.mt (a Int32, b Int32) Engine=MergeTree order by tuple()") + + client1.send( + "CREATE TABLE test.mt (a Int32, b Int32) Engine=MergeTree order by tuple()" + ) + client1.expect(prompt) + client1.send( + "CREATE DICTIONARY test.dict(a Int32, b Int32) PRIMARY KEY a LAYOUT(FLAT()) " + + "SOURCE(CLICKHOUSE(db 'test' table 'mt')) LIFETIME(1)" + ) client1.expect(prompt) - client1.send("CREATE DICTIONARY test.dict(a Int32, b Int32) PRIMARY KEY a LAYOUT(FLAT()) " + \ - "SOURCE(CLICKHOUSE(db 'test' table 'mt')) LIFETIME(1)") - client1.expect(prompt) client1.send("CREATE LIVE VIEW test.lv WITH REFRESH 1 AS SELECT * FROM test.dict") client1.expect(prompt) client2.send("INSERT INTO test.mt VALUES (1,2)") - client2.expect(prompt) + client2.expect(prompt) - client1.send('WATCH test.lv FORMAT JSONEachRow') + client1.send("WATCH test.lv FORMAT JSONEachRow") client1.expect(r'"_version":"1"') - + client2.send("INSERT INTO test.mt VALUES (2,2)") - client2.expect(prompt) + client2.expect(prompt) client1.expect(r'"_version":"2"') - + client2.send("INSERT INTO test.mt VALUES (3,2)") - client2.expect(prompt) + client2.expect(prompt) client1.expect(r'"_version":"3"') - + # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP DICTIONARY IF EXISTS test.dict') + client1.send("DROP DICTIONARY IF EXISTS test.dict") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - - - diff --git a/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py b/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py index 98d65b47d39..0358c28bf91 100755 --- a/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py +++ b/tests/queries/0_stateless/00962_temporary_live_view_watch_live.py @@ -6,45 +6,47 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect('_version') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) + client1.send("WATCH test.lv") + client1.expect("_version") + client1.expect(r"0.*1" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (1),(2),(3)") + client1.expect(r"6.*2" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (4),(5),(6)") + client1.expect(r"21.*3" + end_of_block) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py b/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py index c352004078b..bafb283e487 100755 --- a/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py +++ b/tests/queries/0_stateless/00964_live_view_watch_events_heartbeat.py @@ -6,49 +6,51 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') + client1.send("SET live_view_heartbeat_interval=1") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - client1.send('WATCH test.lv EVENTS FORMAT CSV') - client1.expect('Progress: 1.00 rows.*\)') - client2.send('INSERT INTO test.mt VALUES (1)') + client1.send("WATCH test.lv EVENTS FORMAT CSV") + client1.expect("Progress: 1.00 rows.*\)") + client2.send("INSERT INTO test.mt VALUES (1)") client2.expect(prompt) - client1.expect('Progress: 2.00 rows.*\)') - client2.send('INSERT INTO test.mt VALUES (2),(3)') + client1.expect("Progress: 2.00 rows.*\)") + client2.send("INSERT INTO test.mt VALUES (2),(3)") client2.expect(prompt) # wait for heartbeat - client1.expect('Progress: 3.00 rows.*\)') + client1.expect("Progress: 3.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py b/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py index c1c14e8615b..3cb1220bb49 100755 --- a/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py +++ b/tests/queries/0_stateless/00965_live_view_watch_heartbeat.py @@ -6,47 +6,49 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('SET live_view_heartbeat_interval=1') + client1.send("SET live_view_heartbeat_interval=1") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv WITH TIMEOUT AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect('_version') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) + client1.send("WATCH test.lv") + client1.expect("_version") + client1.expect(r"0.*1" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (1),(2),(3)") + client1.expect(r"6.*2" + end_of_block) # wait for heartbeat - client1.expect('Progress: 2.00 rows.*\)') + client1.expect("Progress: 2.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql index 0d82519e4d3..555e7a98380 100644 --- a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql +++ b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql @@ -1,5 +1,7 @@ -- Tags: shard +SET prefer_localhost_replica = 1; + SELECT count() FROM remote('127.0.0.1,localhos', system.one); -- { serverError 198 } SELECT count() FROM remote('127.0.0.1|localhos', system.one); diff --git a/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/tests/queries/0_stateless/00966_live_view_watch_events_http.py index dcbae5498bf..1f2ddae23d6 100755 --- a/tests/queries/0_stateless/00966_live_view_watch_events_http.py +++ b/tests/queries/0_stateless/00966_live_view_watch_events_http.py @@ -5,39 +5,45 @@ import os import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block from httpclient import client as http_client log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1: +with client(name="client1>", log=log) as client1: client1.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - try: - with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: - client2.expect('.*1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + with http_client( + { + "method": "GET", + "url": "/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS", + }, + name="client2>", + log=log, + ) as client2: + client2.expect(".*1\n") + client1.send("INSERT INTO test.mt VALUES (1),(2),(3)") client1.expect(prompt) - client2.expect('.*2\n') + client2.expect(".*2\n") finally: - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00967_live_view_watch_http.py b/tests/queries/0_stateless/00967_live_view_watch_http.py index 05d7905e8ed..92e192cc7f2 100755 --- a/tests/queries/0_stateless/00967_live_view_watch_http.py +++ b/tests/queries/0_stateless/00967_live_view_watch_http.py @@ -5,38 +5,45 @@ import os import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block from httpclient import client as http_client log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1: +with client(name="client1>", log=log) as client1: client1.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt") client1.expect(prompt) try: - with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&query=WATCH%20test.lv'}, name='client2>', log=log) as client2: - client2.expect('.*0\t1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + with http_client( + { + "method": "GET", + "url": "/?allow_experimental_live_view=1&query=WATCH%20test.lv", + }, + name="client2>", + log=log, + ) as client2: + client2.expect(".*0\t1\n") + client1.send("INSERT INTO test.mt VALUES (1),(2),(3)") client1.expect(prompt) - client2.expect('.*6\t2\n') + client2.expect(".*6\t2\n") finally: - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py b/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py index fbd7660bafe..8c5126bbaf3 100755 --- a/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py +++ b/tests/queries/0_stateless/00970_live_view_watch_events_http_heartbeat.py @@ -5,43 +5,59 @@ import os import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block from httpclient import client as http_client log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1: +with client(name="client1>", log=log) as client1: client1.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', escape=True) + with http_client( + { + "method": "GET", + "url": "/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20EVENTS%20FORMAT%20JSONEachRowWithProgress", + }, + name="client2>", + log=log, + ) as client2: + client2.expect( + '{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}\n', + escape=True, + ) client2.expect('{"row":{"version":"1"}', escape=True) - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) + client2.expect( + '{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', + escape=True, + ) # heartbeat is provided by progress message - client2.expect('{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', escape=True) + client2.expect( + '{"progress":{"read_rows":"1","read_bytes":"8","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}', + escape=True, + ) - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.send("INSERT INTO test.mt VALUES (1),(2),(3)") client1.expect(prompt) client2.expect('{"row":{"version":"2"}}\n', escape=True) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py b/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py index db5e1698a10..117f7b7c786 100755 --- a/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py +++ b/tests/queries/0_stateless/00971_live_view_watch_http_heartbeat.py @@ -5,44 +5,53 @@ import os import sys CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block from httpclient import client as http_client log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1: +with client(name="client1>", log=log) as client1: client1.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress'}, name='client2>', log=log) as client2: - client2.expect('"progress".*',) + with http_client( + { + "method": "GET", + "url": "/?allow_experimental_live_view=1&live_view_heartbeat_interval=1&query=WATCH%20test.lv%20FORMAT%20JSONEachRowWithProgress", + }, + name="client2>", + log=log, + ) as client2: + client2.expect( + '"progress".*', + ) client2.expect('{"row":{"sum(a)":"0","_version":"1"}}\n', escape=True) client2.expect('"progress".*\n') # heartbeat is provided by progress message client2.expect('"progress".*\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.send("INSERT INTO test.mt VALUES (1),(2),(3)") client1.expect(prompt) client2.expect('"progress".*"read_rows":"2".*\n') client2.expect('{"row":{"sum(a)":"6","_version":"2"}}\n', escape=True) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00974_query_profiler.sql b/tests/queries/0_stateless/00974_query_profiler.sql index 24e4241b813..b697bd56800 100644 --- a/tests/queries/0_stateless/00974_query_profiler.sql +++ b/tests/queries/0_stateless/00974_query_profiler.sql @@ -1,4 +1,4 @@ --- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest +-- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest, no-cpu-aarch64 -- Tag no-fasttest: Not sure why fail even in sequential mode. Disabled for now to make some progress. SET allow_introspection_functions = 1; diff --git a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py b/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py index 81e2764d64f..ef144d044c2 100755 --- a/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py +++ b/tests/queries/0_stateless/00979_live_view_watch_continuous_aggregates.py @@ -6,63 +6,79 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (time DateTime, location String, temperature UInt32) Engine=MergeTree order by tuple()') + client1.send( + "CREATE TABLE test.mt (time DateTime, location String, temperature UInt32) Engine=MergeTree order by tuple()" + ) client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT toStartOfDay(time) AS day, location, avg(temperature) FROM test.mt GROUP BY day, location ORDER BY day, location') + client1.send( + "CREATE LIVE VIEW test.lv AS SELECT toStartOfDay(time) AS day, location, avg(temperature) FROM test.mt GROUP BY day, location ORDER BY day, location" + ) client1.expect(prompt) - client1.send('WATCH test.lv FORMAT CSVWithNames') - client2.send("INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','New York',60),('2019-01-01 00:10:00','New York',70)") + client1.send("WATCH test.lv FORMAT CSVWithNames") + client2.send( + "INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','New York',60),('2019-01-01 00:10:00','New York',70)" + ) client2.expect(prompt) client1.expect(r'"2019-01-01 00:00:00","New York",65') - client2.send("INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','Moscow',30),('2019-01-01 00:10:00', 'Moscow', 40)") + client2.send( + "INSERT INTO test.mt VALUES ('2019-01-01 00:00:00','Moscow',30),('2019-01-01 00:10:00', 'Moscow', 40)" + ) client2.expect(prompt) client1.expect(r'"2019-01-01 00:00:00","Moscow",35') client1.expect(r'"2019-01-01 00:00:00","New York",65') - client2.send("INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','New York',50),('2019-01-02 00:10:00','New York',60)") + client2.send( + "INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','New York',50),('2019-01-02 00:10:00','New York',60)" + ) client2.expect(prompt) client1.expect(r'"2019-01-01 00:00:00","Moscow",35') client1.expect(r'"2019-01-01 00:00:00","New York",65') client1.expect(r'"2019-01-02 00:00:00","New York",55') - client2.send("INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','Moscow',20),('2019-01-02 00:10:00', 'Moscow', 30)") + client2.send( + "INSERT INTO test.mt VALUES ('2019-01-02 00:00:00','Moscow',20),('2019-01-02 00:10:00', 'Moscow', 30)" + ) client2.expect(prompt) client1.expect(r'"2019-01-01 00:00:00","Moscow",35') client1.expect(r'"2019-01-01 00:00:00","New York",65') client1.expect(r'"2019-01-02 00:00:00","Moscow",25') client1.expect(r'"2019-01-02 00:00:00","New York",55') - client2.send("INSERT INTO test.mt VALUES ('2019-01-02 00:03:00','New York',40),('2019-01-02 00:06:00','New York',30)") + client2.send( + "INSERT INTO test.mt VALUES ('2019-01-02 00:03:00','New York',40),('2019-01-02 00:06:00','New York',30)" + ) client2.expect(prompt) client1.expect(r'"2019-01-01 00:00:00","Moscow",35') client1.expect(r'"2019-01-01 00:00:00","New York",65') client1.expect(r'"2019-01-02 00:00:00","Moscow",25') client1.expect(r'"2019-01-02 00:00:00","New York",45') # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00979_live_view_watch_live.py b/tests/queries/0_stateless/00979_live_view_watch_live.py index 7bbae932da7..b099b56ae48 100755 --- a/tests/queries/0_stateless/00979_live_view_watch_live.py +++ b/tests/queries/0_stateless/00979_live_view_watch_live.py @@ -6,51 +6,53 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt") client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect('_version') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) + client1.send("WATCH test.lv") + client1.expect("_version") + client1.expect(r"0.*1" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (1),(2),(3)") + client1.expect(r"6.*2" + end_of_block) client2.expect(prompt) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) + client2.send("INSERT INTO test.mt VALUES (4),(5),(6)") + client1.expect(r"21.*3" + end_of_block) client2.expect(prompt) for i in range(1, 129): - client2.send('INSERT INTO test.mt VALUES (1)') - client1.expect(r'%d.*%d' % (21 + i, 3 + i) + end_of_block) - client2.expect(prompt) + client2.send("INSERT INTO test.mt VALUES (1)") + client1.expect(r"%d.*%d" % (21 + i, 3 + i) + end_of_block) + client2.expect(prompt) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py b/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py index ed2fe61f4b7..a7c1adac214 100755 --- a/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py +++ b/tests/queries/0_stateless/00979_live_view_watch_live_with_subquery.py @@ -6,51 +6,55 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client2.send('SET allow_experimental_live_view = 1') + client2.send("SET allow_experimental_live_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send(' DROP TABLE IF EXISTS test.mt') + client1.send(" DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT * FROM ( SELECT sum(A.a) FROM (SELECT * FROM test.mt) AS A )') + client1.send( + "CREATE LIVE VIEW test.lv AS SELECT * FROM ( SELECT sum(A.a) FROM (SELECT * FROM test.mt) AS A )" + ) client1.expect(prompt) - client1.send('WATCH test.lv') - client1.expect('_version') - client1.expect(r'0.*1' + end_of_block) - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') - client1.expect(r'6.*2' + end_of_block) + client1.send("WATCH test.lv") + client1.expect("_version") + client1.expect(r"0.*1" + end_of_block) + client2.send("INSERT INTO test.mt VALUES (1),(2),(3)") + client1.expect(r"6.*2" + end_of_block) client2.expect(prompt) - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') - client1.expect(r'21.*3' + end_of_block) + client2.send("INSERT INTO test.mt VALUES (4),(5),(6)") + client1.expect(r"21.*3" + end_of_block) client2.expect(prompt) - for i in range(1,129): - client2.send('INSERT INTO test.mt VALUES (1)') - client1.expect(r'%d.*%d' % (21+i, 3+i) + end_of_block) - client2.expect(prompt) + for i in range(1, 129): + client2.send("INSERT INTO test.mt VALUES (1)") + client1.expect(r"%d.*%d" % (21 + i, 3 + i) + end_of_block) + client2.expect(prompt) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) - client1.expect(prompt) - client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send("DROP TABLE test.lv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh b/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh index 8243c6bde62..731e96b055e 100755 --- a/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh +++ b/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race +# Tags: race, long # This test is disabled because it triggers internal assert in Thread Sanitizer. # Thread Sanitizer does not support for more than 64 mutexes to be locked in a single thread. @@ -11,67 +11,68 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table" -$CLICKHOUSE_CLIENT -q "CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1" +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE IF EXISTS alter_table; + CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1; +" function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null" } function thread2() { - while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String '0'; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;"; done + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String '0'; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;" } function thread3() { - while true; do $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; done + $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)" } function thread4() { - while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table FINAL"; done + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table FINAL" } function thread5() { - while true; do $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table DELETE WHERE rand() % 2 = 1"; done + $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table DELETE WHERE rand() % 2 = 1" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; -export -f thread5; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 +export -f thread5 TIMEOUT=30 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & wait diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index 8dbd10fc27b..3cd61c9972a 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -9,76 +9,95 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -mn -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; - CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; - CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 + CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') + ORDER BY a + PARTITION BY b % 10 + SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; + + CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') + ORDER BY a + PARTITION BY b % 10 + SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; " function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null" } function thread2() { - while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; ALTER TABLE alter_table0 DROP COLUMN h;"; done + $CLICKHOUSE_CLIENT -nm --query " + ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; + ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; + ALTER TABLE alter_table0 DROP COLUMN h; + " } function thread3() { - while true; do $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table0 SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; done + $CLICKHOUSE_CLIENT -q " + INSERT INTO alter_table0 + SELECT + rand(1), rand(2), 1 / rand(3), toString(rand(4)), + [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), + (rand(8), rand(9)) + FROM numbers(100000)" } function thread4() { - while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL"; done + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL" } function thread5() { - while true; do $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table0 DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288"; done + $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table0 DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; -export -f thread5; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 +export -f thread5 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & wait check_replication_consistency "alter_table" "count(), sum(a), sum(b), round(sum(c))" $CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & $CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & + wait diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 6b1df1d45a0..3fe7b0443eb 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, zookeeper, no-parallel +# Tags: race, zookeeper, no-parallel, no-backward-compatibility-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -12,93 +12,87 @@ set -e function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; - done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null" } function thread2() { - while true; do - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table_$REPLICA ADD COLUMN h String '0'; ALTER TABLE alter_table_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE alter_table_$REPLICA DROP COLUMN h;"; - done + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table_$REPLICA ADD COLUMN h String '0'; ALTER TABLE alter_table_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE alter_table_$REPLICA DROP COLUMN h;" } function thread3() { - while true; do - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table_$REPLICA SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; - done + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table_$REPLICA SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)" } function thread4() { - while true; do - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table_$REPLICA FINAL"; - sleep 0.$RANDOM; - done + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table_$REPLICA FINAL" + sleep 0.$RANDOM } function thread5() { - while true; do - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table_$REPLICA DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288"; - sleep 0.$RANDOM; - done + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table_$REPLICA DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288" + sleep 0.$RANDOM } function thread6() { - while true; do - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS alter_table_$REPLICA; - CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$REPLICA') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; - sleep 0.$RANDOM; - done + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -mn -q " + DROP TABLE IF EXISTS alter_table_$REPLICA; + + CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$REPLICA') + ORDER BY a + PARTITION BY b % 10 + SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0; + " + sleep 0.$RANDOM } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; -export -f thread5; -export -f thread6; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 +export -f thread5 +export -f thread6 TIMEOUT=30 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & wait diff --git a/tests/queries/0_stateless/01001_rename_merge_race_condition.sh b/tests/queries/0_stateless/01001_rename_merge_race_condition.sh index 253d06c038c..9b5f92628f7 100755 --- a/tests/queries/0_stateless/01001_rename_merge_race_condition.sh +++ b/tests/queries/0_stateless/01001_rename_merge_race_condition.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race +# Tags: race, long CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -7,35 +7,32 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; -$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2" +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory" function thread1() { - while true; do - seq 1 1000 | sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' | $CLICKHOUSE_CLIENT -n - done + seq 1 1000 | { + sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' + } | $CLICKHOUSE_CLIENT -n } function thread2() { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM merge('$CLICKHOUSE_DATABASE', '^test[12]$')" - done + $CLICKHOUSE_CLIENT --query "SELECT * FROM merge('$CLICKHOUSE_DATABASE', '^test[12]$')" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; +export -f thread1 +export -f thread2 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & wait -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2" diff --git a/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh b/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh index b29a42a7356..ee38cf4eb7e 100755 --- a/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh +++ b/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh @@ -12,48 +12,39 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test (x UInt8, s String MATERIALIZED to function thread1() { - while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT rand() FROM numbers(1000)"; - done + $CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT rand() FROM numbers(1000)" } function thread2() { - while true; do - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x Nullable(UInt8);"; - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x UInt8;"; - sleep 0.0$RANDOM - done + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x Nullable(UInt8);" + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x UInt8;" + sleep 0.0$RANDOM } function thread3() { - while true; do - $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM test FORMAT Null"; - done + $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM test FORMAT Null" } function thread4() { - while true; do - $CLICKHOUSE_CLIENT -n --query "OPTIMIZE TABLE test FINAL"; - sleep 0.1$RANDOM - done + $CLICKHOUSE_CLIENT -n --query "OPTIMIZE TABLE test FINAL" + sleep 0.1$RANDOM } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01003_kill_query_race_condition.sh b/tests/queries/0_stateless/01003_kill_query_race_condition.sh index 934d18d2670..64caf0f88d1 100755 --- a/tests/queries/0_stateless/01003_kill_query_race_condition.sh +++ b/tests/queries/0_stateless/01003_kill_query_race_condition.sh @@ -9,44 +9,37 @@ set -e function thread1() { - while true; do - $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM numbers(1000000000)" --format Null; - done + $CLICKHOUSE_CLIENT --query_id=hello --query "SELECT count() FROM numbers(1000000000)" --format Null; } function thread2() { - while true; do - $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello'" --format Null; - sleep 0.$RANDOM - done + $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello'" --format Null + sleep 0.$RANDOM } function thread3() { - while true; do - $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null; - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null; - done + $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; +export -f thread1 +export -f thread2 +export -f thread3 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01004_rename_deadlock.sh b/tests/queries/0_stateless/01004_rename_deadlock.sh index f0adf136e94..4a3f23883c0 100755 --- a/tests/queries/0_stateless/01004_rename_deadlock.sh +++ b/tests/queries/0_stateless/01004_rename_deadlock.sh @@ -14,48 +14,41 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test2 (x UInt8) ENGINE = MergeTree ORDE function thread1() { - while true; do - $CLICKHOUSE_CLIENT --query "RENAME TABLE test1 TO test_tmp, test2 TO test1, test_tmp TO test2" - done + $CLICKHOUSE_CLIENT --query "RENAME TABLE test1 TO test_tmp, test2 TO test1, test_tmp TO test2" } function thread2() { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM test1 UNION ALL SELECT * FROM test2" --format Null - done + $CLICKHOUSE_CLIENT --query "SELECT * FROM test1 UNION ALL SELECT * FROM test2" --format Null } function thread3() { - while true; do - # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables" --format Null - done + # NOTE: database = $CLICKHOUSE_DATABASE is unwanted + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables" --format Null } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; +export -f thread1 +export -f thread2 +export -f thread3 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & wait sleep 1 diff --git a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh index ef352606b69..948032f19e7 100755 --- a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh +++ b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh @@ -12,36 +12,31 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt8) ENGINE = MergeTree ORDE function thread1() { - while true; do - $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x Nullable(UInt8)" - $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x UInt8" - done + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x UInt8" } function thread2() { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT x FROM test1 WHERE x IN (SELECT x FROM remote('127.0.0.2', '$CLICKHOUSE_DATABASE', test1))" --format Null - done + $CLICKHOUSE_CLIENT --query "SELECT x FROM test1 WHERE x IN (SELECT x FROM remote('127.0.0.2', '$CLICKHOUSE_DATABASE', test1))" --format Null } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; +export -f thread1 +export -f thread2 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh index 9f4b2241732..7fa5f5456a2 100755 --- a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh +++ b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: deadlock +# Tags: deadlock, long CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -16,48 +16,39 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE b (x UInt8) ENGINE = MergeTree ORDER BY function thread1() { - while true; do - # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - seq 1 100 | awk '{ print "SELECT x FROM a WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n - done + # NOTE: database = $CLICKHOUSE_DATABASE is unwanted + seq 1 100 | awk '{ print "SELECT x FROM a WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n } function thread2() { - while true; do - # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - seq 1 100 | awk '{ print "SELECT x FROM b WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n - done + # NOTE: database = $CLICKHOUSE_DATABASE is unwanted + seq 1 100 | awk '{ print "SELECT x FROM b WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n } function thread3() { - while true; do - $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x Nullable(UInt8)" - $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x UInt8" - done + $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x UInt8" } function thread4() { - while true; do - $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x Nullable(UInt8)" - $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x UInt8" - done + $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x UInt8" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql b/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql index af747c93678..7804ce32a5a 100644 --- a/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql +++ b/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel, no-fasttest +-- Tags: no-parallel, no-fasttest, no-random-settings SET max_memory_usage = 32000000; SET join_on_disk_max_files_to_merge = 4; diff --git a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh index 55bbfb3ff11..290708ccde4 100755 --- a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh +++ b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh @@ -19,7 +19,7 @@ ${CLICKHOUSE_CLIENT} -n -q " INSERT INTO $R1 VALUES (1) " -timeout 10s ${CLICKHOUSE_CLIENT} -n -q " +clickhouse_client_timeout 10s ${CLICKHOUSE_CLIENT} --receive_timeout 1 -n -q " SET receive_timeout=1; SYSTEM SYNC REPLICA $R2 " 2>&1 | grep -F -q "Code: 159. DB::Exception" && echo 'OK' || echo 'Failed!' diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh index 44a27ea08f4..d1953dbfd0f 100755 --- a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh +++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -7,95 +7,67 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) export CURR_DATABASE="test_lazy_01014_concurrent_${CLICKHOUSE_DATABASE}" - function recreate_lazy_func1() { - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; - "; - - while true; do - $CLICKHOUSE_CLIENT -q " - DETACH TABLE $CURR_DATABASE.log; - "; - - $CLICKHOUSE_CLIENT -q " - ATTACH TABLE $CURR_DATABASE.log; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + DETACH TABLE $CURR_DATABASE.log; + ATTACH TABLE $CURR_DATABASE.log; + " } function recreate_lazy_func2() { - while true; do - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; - "; - - $CLICKHOUSE_CLIENT -q " - DROP TABLE $CURR_DATABASE.tlog; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; + DROP TABLE $CURR_DATABASE.tlog; + " } function recreate_lazy_func3() { - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; - "; - - while true; do - $CLICKHOUSE_CLIENT -q " - ATTACH TABLE $CURR_DATABASE.slog; - "; - - $CLICKHOUSE_CLIENT -q " - DETACH TABLE $CURR_DATABASE.slog; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + ATTACH TABLE $CURR_DATABASE.slog; + DETACH TABLE $CURR_DATABASE.slog; + " } function recreate_lazy_func4() { - while true; do - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; - "; - - $CLICKHOUSE_CLIENT -q " - DROP TABLE $CURR_DATABASE.tlog2; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; + DROP TABLE $CURR_DATABASE.tlog2; + " } function show_tables_func() { - while true; do - $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE database = '$CURR_DATABASE' FORMAT Null"; - done + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE database = '$CURR_DATABASE' FORMAT Null" } -export -f recreate_lazy_func1; -export -f recreate_lazy_func2; -export -f recreate_lazy_func3; -export -f recreate_lazy_func4; -export -f show_tables_func; +export -f recreate_lazy_func1 +export -f recreate_lazy_func2 +export -f recreate_lazy_func3 +export -f recreate_lazy_func4 +export -f show_tables_func ${CLICKHOUSE_CLIENT} -n -q " DROP DATABASE IF EXISTS $CURR_DATABASE; CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); + + CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; + CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; " TIMEOUT=30 -timeout $TIMEOUT bash -c recreate_lazy_func1 2> /dev/null & -timeout $TIMEOUT bash -c recreate_lazy_func2 2> /dev/null & -timeout $TIMEOUT bash -c recreate_lazy_func3 2> /dev/null & -timeout $TIMEOUT bash -c recreate_lazy_func4 2> /dev/null & -timeout $TIMEOUT bash -c show_tables_func 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT show_tables_func 2> /dev/null & wait sleep 1 @@ -108,4 +80,3 @@ ${CLICKHOUSE_CLIENT} -q "ATTACH TABLE $CURR_DATABASE.tlog2;" 2>/dev/null ${CLICKHOUSE_CLIENT} -q "DROP DATABASE $CURR_DATABASE" echo "Test OK" - diff --git a/tests/queries/0_stateless/01016_input_null_as_default.reference b/tests/queries/0_stateless/01016_input_null_as_default.reference index d7010f42d4e..7a84006caf7 100644 --- a/tests/queries/0_stateless/01016_input_null_as_default.reference +++ b/tests/queries/0_stateless/01016_input_null_as_default.reference @@ -33,3 +33,9 @@ Values 1 world 3 2019-07-23 [1,2,3] ('tuple',3.14) 2 Hello 123 2019-06-19 [] ('test',2.71828) 3 Hello 42 2019-06-19 [1,2,3] ('default',0.75) +default_by_other_column +1 2 ('tuple',3) +10 10 ('default',2.5) +100 100 ('default',25) +100 100 ('default',25) +100 100 ('default',25) diff --git a/tests/queries/0_stateless/01016_input_null_as_default.sh b/tests/queries/0_stateless/01016_input_null_as_default.sh index bfeaca0fcac..24d93b2703c 100755 --- a/tests/queries/0_stateless/01016_input_null_as_default.sh +++ b/tests/queries/0_stateless/01016_input_null_as_default.sh @@ -5,7 +5,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS null_as_default"; +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS default_by_other_column"; $CLICKHOUSE_CLIENT --query="CREATE TABLE null_as_default (i Int8, s String DEFAULT 'Hello', n UInt64 DEFAULT 42, d Date DEFAULT '2019-06-19', a Array(UInt8) DEFAULT [1, 2, 3], t Tuple(String, Float64) DEFAULT ('default', i / 4)) ENGINE = Memory"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE default_by_other_column (a Float32 DEFAULT 100, b Float64 DEFAULT a, c Tuple(String, Float64) DEFAULT ('default', b / 4)) ENGINE = Memory"; echo 'CSV' echo '\N, 1, \N, "2019-07-22", "[10, 20, 30]", \N @@ -62,3 +64,12 @@ echo '(NULL, '\''1'\'', (null), '\''2019-07-22'\'', ([10, 20, 30]), (NuLl)), (3, null, (null), null, (null), (null))' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default VALUES"; $CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i"; $CLICKHOUSE_CLIENT --query="DROP TABLE null_as_default"; + +echo 'default_by_other_column' +$CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO default_by_other_column(c) VALUES(null)"; +$CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO default_by_other_column(b, c) VALUES(null, null)"; +$CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO default_by_other_column(a, b, c) VALUES(null, null, null)"; +$CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO default_by_other_column(a) VALUES(10)"; +$CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO default_by_other_column(a, b, c) VALUES(1, 2, ('tuple', 3))"; +$CLICKHOUSE_CLIENT --query="SELECT * FROM default_by_other_column ORDER BY a"; +$CLICKHOUSE_CLIENT --query="DROP TABLE default_by_other_column"; diff --git a/tests/queries/0_stateless/01016_macros.reference b/tests/queries/0_stateless/01016_macros.reference index 6ca8ba9557b..8847c5f6288 100644 --- a/tests/queries/0_stateless/01016_macros.reference +++ b/tests/queries/0_stateless/01016_macros.reference @@ -1,2 +1,3 @@ test Hello, world! Hello, world! +1 diff --git a/tests/queries/0_stateless/01016_macros.sql b/tests/queries/0_stateless/01016_macros.sql index be00b7094cb..75b32239cba 100644 --- a/tests/queries/0_stateless/01016_macros.sql +++ b/tests/queries/0_stateless/01016_macros.sql @@ -1,2 +1,3 @@ SELECT * FROM system.macros WHERE macro = 'test'; SELECT getMacro('test'); +select isConstant(getMacro('test')); diff --git a/tests/queries/0_stateless/01016_simhash_minhash.reference b/tests/queries/0_stateless/01016_simhash_minhash.reference index 3a668e6dcdb..d4fdcfea6a5 100644 --- a/tests/queries/0_stateless/01016_simhash_minhash.reference +++ b/tests/queries/0_stateless/01016_simhash_minhash.reference @@ -50,92 +50,92 @@ (14260447771268573594,3863279269132177973) uniqExact 6 ngramSimHash -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 3906262823 -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 2857686823 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 2 676648743 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 1012193063 +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 2857686823 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 3092567843 +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 3906262823 ngramSimHashCaseInsensitive +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 2824132391 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 2891240999 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 3092567591 -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 2824132391 ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 3908359975 ngramSimHashUTF8 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 3159676711 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 2 676648743 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 1012193063 ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 2924795687 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 3159676711 ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 3897874215 ngramSimHashCaseInsensitiveUTF8 -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 3906262823 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 3092567591 ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 2824132391 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 2891241255 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 3092567591 +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 3906262823 wordShingleSimHash -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 857724390 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 404215270 -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 991679910 -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 425963587 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 404215014 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 404215270 +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 425963587 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 563598566 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 857724390 +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 991679910 wordShingleSimHashCaseInsensitive -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 959182215 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 429118950 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 420713958 ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 421737795 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 429118950 +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 959182215 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 964941252 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 965465540 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 420713958 wordShingleSimHashUTF8 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 857724390 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 404215270 -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 991679910 -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 425963587 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 404215014 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 404215270 +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 425963587 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 563598566 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 857724390 +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 991679910 wordShingleSimHashCaseInsensitiveUTF8 -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 959182215 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 429118950 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 420713958 ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 421737795 +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 429118950 +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 959182215 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 964941252 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 965465540 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 420713958 ngramMinHash -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,17443426065825246292) -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,17443426065825246292) ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (4388091710993602029,17613327300639166679) +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,17443426065825246292) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (7962672159337006560,17443426065825246292) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,17443426065825246292) ngramMinHashCaseInsensitive -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,8535005350590298790) -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,8535005350590298790) ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (4388091710993602029,17613327300639166679) +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,8535005350590298790) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (7962672159337006560,8535005350590298790) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,8535005350590298790) ngramMinHashUTF8 +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (4388091710993602029,17613327300639166679) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,17443426065825246292) -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,17443426065825246292) -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (4388091710993602029,17613327300639166679) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (7962672159337006560,17443426065825246292) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,17443426065825246292) ngramMinHashCaseInsensitiveUTF8 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,8535005350590298790) -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,8535005350590298790) ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (4388091710993602029,17613327300639166679) +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (6021986790841777095,8535005350590298790) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (7962672159337006560,8535005350590298790) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (13225377334870249827,8535005350590298790) wordShingleMinHash -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (18148981179837829400,14581416672396321264) -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (5044918525503962090,12338022931991160906) +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (18148981179837829400,6048943706095721476) +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (18148981179837829400,14581416672396321264) wordShingleMinHashCaseInsensitive -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (15504011608613565061,6048943706095721476) -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (5044918525503962090,3381836163833256482) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (15504011608613565061,6048943706095721476) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (15504011608613565061,14581416672396321264) +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) wordShingleMinHashUTF8 -ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (18148981179837829400,14581416672396321264) -ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (5044918525503962090,12338022931991160906) -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (18148981179837829400,6048943706095721476) -wordShingleMinHashCaseInsensitiveUTF8 -ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (15504011608613565061,6048943706095721476) ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (18148981179837829400,6048943706095721476) +ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (18148981179837829400,14581416672396321264) +wordShingleMinHashCaseInsensitiveUTF8 ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all of your structured data into the system, and it is immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all your structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems.\n:::::::\nClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (using columns after decompression only). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the read / write availability of the system.\nClickHouse is simple and works out of the box. It simplifies all processing of your data: it loads all structured data into the system and immediately becomes available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 3 (5044918525503962090,3381836163833256482) +ClickHouse makes full use of all available hardware to process every request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (only used columns after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid single points of failure. Downtime for one site or the entire data center will not affect the system\'s read and write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they immediately become available for building reports. The SQL dialect allows you to express the desired result without resorting to any non-standard APIs that can be found in some alternative systems. 1 (15504011608613565061,6048943706095721476) ClickHouse makes full use of all available hardware to process each request as quickly as possible. Peak performance for a single query is over 2 terabytes per second (used columns only after unpacking). In a distributed setup, reads are automatically balanced across healthy replicas to avoid increased latency.\nClickHouse supports asynchronous multi-master replication and can be deployed across multiple data centers. All nodes are equal to avoid a single point of failure. Downtime for one site or the entire data center will not affect the system\'s read / write availability.\nClickHouse is simple and works out of the box. It simplifies all the processing of your data: it loads all your structured data into the system, and they are immediately available for building reports. The SQL dialect allows you to express the desired result without resorting to any of the non-standard APIs found in some alternative systems. 1 (15504011608613565061,14581416672396321264) +ClickHouse uses all available hardware to its full potential to process each query as fast as possible. Peak processing performance for a single query stands at more than 2 terabytes per second (after decompression, only used columns). In distributed setup reads are automatically balanced among healthy replicas to avoid increasing latency.\nClickHouse supports multi-master asynchronous replication and can be deployed across multiple datacenters. All nodes are equal, which allows avoiding having single points of failure. Downtime of a single node or the whole datacenter wont affect the systems availability for both reads and writes.\nClickHouse is simple and works out-of-the-box. It streamlines all your data processing: ingest all your structured data into the system and it becomes instantly available for building reports. SQL dialect allows expressing the desired result without involving any custom non-standard API that could be found in some alternative systems. 1 (16224204290372720939,13975393268888698430) diff --git a/tests/queries/0_stateless/01016_simhash_minhash.sql b/tests/queries/0_stateless/01016_simhash_minhash.sql index 01af9451381..1e77b487851 100644 --- a/tests/queries/0_stateless/01016_simhash_minhash.sql +++ b/tests/queries/0_stateless/01016_simhash_minhash.sql @@ -75,38 +75,38 @@ SELECT 'uniqExact', uniqExact(s) FROM defaults; SELECT 'ngramSimHash'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHash(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHash(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramSimHashCaseInsensitive'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHashCaseInsensitive(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHashCaseInsensitive(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramSimHashUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHashUTF8(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHashUTF8(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramSimHashCaseInsensitiveUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHashCaseInsensitiveUTF8(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramSimHashCaseInsensitiveUTF8(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleSimHash'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHash(s, 2) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHash(s, 2) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleSimHashCaseInsensitive'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHashCaseInsensitive(s, 2) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHashCaseInsensitive(s, 2) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleSimHashUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHashUTF8(s, 2) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHashUTF8(s, 2) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleSimHashCaseInsensitiveUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHashCaseInsensitiveUTF8(s, 2) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleSimHashCaseInsensitiveUTF8(s, 2) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramMinHash'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHash(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHash(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramMinHashCaseInsensitive'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHashCaseInsensitive(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHashCaseInsensitive(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramMinHashUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHashUTF8(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHashUTF8(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'ngramMinHashCaseInsensitiveUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHashCaseInsensitiveUTF8(s) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), ngramMinHashCaseInsensitiveUTF8(s) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleMinHash'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHash(s, 2, 3) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHash(s, 2, 3) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleMinHashCaseInsensitive'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashCaseInsensitive(s, 2, 3) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashCaseInsensitive(s, 2, 3) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleMinHashUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashUTF8(s, 2, 3) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashUTF8(s, 2, 3) as h FROM defaults GROUP BY h ORDER BY h; SELECT 'wordShingleMinHashCaseInsensitiveUTF8'; -SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashCaseInsensitiveUTF8(s, 2, 3) as h FROM defaults GROUP BY h; +SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashCaseInsensitiveUTF8(s, 2, 3) as h FROM defaults GROUP BY h ORDER BY h; SELECT wordShingleSimHash('foobar', 9223372036854775807); -- { serverError 69 } SELECT wordShingleSimHash('foobar', 1001); -- { serverError 69 } diff --git a/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql b/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql index d47dc6b8d5f..69bd15e3f54 100644 --- a/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql +++ b/tests/queries/0_stateless/01017_uniqCombined_memory_usage.sql @@ -1,4 +1,4 @@ --- Tags: no-tsan, no-asan, no-msan, no-replicated-database +-- Tags: no-tsan, no-asan, no-msan, no-replicated-database, no-random-settings -- Tag no-tsan: Fine thresholds on memory usage -- Tag no-asan: Fine thresholds on memory usage -- Tag no-msan: Fine thresholds on memory usage @@ -7,6 +7,8 @@ -- sizeof(HLL) is (2^K * 6 / 8) -- hence max_memory_usage for 100 rows = (96<<10)*100 = 9830400 +SET use_uncompressed_cache = 0; + -- HashTable for UInt32 (used until (1<<13) elements), hence 8192 elements SELECT 'UInt32'; SET max_memory_usage = 4000000; @@ -19,6 +21,8 @@ SELECT 'UInt64'; SET max_memory_usage = 4000000; SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(reinterpretAsString(number % 4096)) u FROM numbers(4096 * 100) GROUP BY k); -- { serverError 241 } SET max_memory_usage = 9830400; + + SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(reinterpretAsString(number % 4096)) u FROM numbers(4096 * 100) GROUP BY k); SELECT 'K=16'; diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh index 872b0a7c1a1..e98fb57c2a4 100755 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -mn -q " DROP DATABASE IF EXISTS database_for_dict; DROP TABLE IF EXISTS table_for_dict1; DROP TABLE IF EXISTS table_for_dict2; @@ -20,96 +20,104 @@ $CLICKHOUSE_CLIENT -n -q " CREATE DATABASE database_for_dict; - CREATE DICTIONARY database_for_dict.dict1 (key_column UInt64, value_column String) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict1' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) LIFETIME(MIN 1 MAX 5) LAYOUT(FLAT()); + CREATE DICTIONARY database_for_dict.dict1 (key_column UInt64, value_column String) + PRIMARY KEY key_column + SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict1' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) + LIFETIME(MIN 1 MAX 5) + LAYOUT(FLAT()); - CREATE DICTIONARY database_for_dict.dict2 (key_column UInt64, value_column String) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict2' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) LIFETIME(MIN 1 MAX 5) LAYOUT(CACHE(SIZE_IN_CELLS 150)); + CREATE DICTIONARY database_for_dict.dict2 (key_column UInt64, value_column String) + PRIMARY KEY key_column + SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict2' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) + LIFETIME(MIN 1 MAX 5) + LAYOUT(CACHE(SIZE_IN_CELLS 150)); " function thread1() { - while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.dictionaries FORMAT Null"; done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.dictionaries FORMAT Null" } function thread2() { - while true; do CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict1" ||: ; done + $CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict1" ||: } function thread3() { - while true; do CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict2" ||:; done + $CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict2" ||: } function thread4() { - while true; do $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -n -q " SELECT * FROM database_for_dict.dict1 FORMAT Null; SELECT * FROM database_for_dict.dict2 FORMAT Null; - " ||: ; done + " ||: } function thread5() { - while true; do $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -n -q " SELECT dictGetString('database_for_dict.dict1', 'value_column', toUInt64(number)) from numbers(1000) FROM FORMAT Null; SELECT dictGetString('database_for_dict.dict2', 'value_column', toUInt64(number)) from numbers(1000) FROM FORMAT Null; - " ||: ; done + " ||: } function thread6() { - while true; do $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict1"; done + $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict1" } function thread7() { - while true; do $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict2"; done + $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict2" } -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; -export -f thread5; -export -f thread6; -export -f thread7; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 +export -f thread5 +export -f thread6 +export -f thread7 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & -timeout $TIMEOUT bash -c thread7 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & -timeout $TIMEOUT bash -c thread7 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & -timeout $TIMEOUT bash -c thread7 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & -timeout $TIMEOUT bash -c thread6 2> /dev/null & -timeout $TIMEOUT bash -c thread7 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & wait $CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" diff --git a/tests/queries/0_stateless/01018_ip_dictionary_long.sql b/tests/queries/0_stateless/01018_ip_dictionary_long.sql index 7d9dfeb1bae..647c36429cc 100644 --- a/tests/queries/0_stateless/01018_ip_dictionary_long.sql +++ b/tests/queries/0_stateless/01018_ip_dictionary_long.sql @@ -44,7 +44,7 @@ LAYOUT(IP_TRIE()) LIFETIME(MIN 10 MAX 100); -- fuzzer -SELECT '127.0.0.0/24' = dictGetString('database_for_dict.dict_ipv4_trie', 'prefixprefixprefixprefix', tuple(IPv4StringToNum('127.0.0.0127.0.0.0'))); -- { serverError 36 } +SELECT '127.0.0.0/24' = dictGetString('database_for_dict.dict_ipv4_trie', 'prefixprefixprefixprefix', tuple(IPv4StringToNumOrDefault('127.0.0.0127.0.0.0'))); -- { serverError 36 } SELECT 0 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'asn', tuple(IPv4StringToNum('0.0.0.0'))); SELECT 1 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'asn', tuple(IPv4StringToNum('128.0.0.0'))); diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index 54a7e940377..aa3e1b612dc 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: long set -e @@ -29,19 +30,15 @@ EOF function alter_thread() { - trap 'exit' INT - ALTERS[0]="ALTER TABLE mv MODIFY QUERY SELECT v FROM src;" ALTERS[1]="ALTER TABLE mv MODIFY QUERY SELECT v * 2 as v FROM src;" - while true; do - $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 -q "${ALTERS[$RANDOM % 2]}" - sleep "$(echo 0.$RANDOM)"; - done + $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 -q "${ALTERS[$RANDOM % 2]}" + sleep 0.$RANDOM } -export -f alter_thread; -timeout 10 bash -c alter_thread & +export -f alter_thread +clickhouse_client_loop_timeout 10 alter_thread & for _ in {1..100}; do # Retry (hopefully retriable (deadlock avoided)) errors. diff --git a/tests/queries/0_stateless/01020_function_array_compact.sql b/tests/queries/0_stateless/01020_function_array_compact.sql index d4aaa4d3fca..29adb007dc4 100644 --- a/tests/queries/0_stateless/01020_function_array_compact.sql +++ b/tests/queries/0_stateless/01020_function_array_compact.sql @@ -7,5 +7,5 @@ select arrayCompact([1,1,2]); select arrayCompact([1,2,1]); select arrayCompact([2,1,1]); select arrayCompact([1,2,2,3,3,3,4,4,4,4,5,5,5,5,5]); -SELECT arrayCompact(x->0, [NULL]); -SELECT toString(arrayCompact(x->0, [NULL])); +SELECT arrayCompact(arrayMap(x->0, [NULL])); +SELECT toString(arrayCompact(arrayMap(x->0, [NULL]))); diff --git a/tests/queries/0_stateless/01025_array_compact_generic.reference b/tests/queries/0_stateless/01025_array_compact_generic.reference index d95e269cd3f..572c7ee140c 100644 --- a/tests/queries/0_stateless/01025_array_compact_generic.reference +++ b/tests/queries/0_stateless/01025_array_compact_generic.reference @@ -15,3 +15,6 @@ ['0','1','2'] ['0','1','2'] ['0','1','2'] +[(0,0),(3,1),(6,2),(9,0)] +[('0','0'),('3','1'),('6','2'),('9','0')] +[('0',0),('3',1),('6',2),('9',0)] diff --git a/tests/queries/0_stateless/01025_array_compact_generic.sql b/tests/queries/0_stateless/01025_array_compact_generic.sql index bea39bfbd44..4446d10e9d4 100644 --- a/tests/queries/0_stateless/01025_array_compact_generic.sql +++ b/tests/queries/0_stateless/01025_array_compact_generic.sql @@ -5,4 +5,7 @@ SELECT arrayCompact([1, 1, NULL, NULL, 2, 2, 2]); SELECT arrayCompact([1, 1, NULL, NULL, nan, nan, 2, 2, 2]); SELECT arrayCompact(['hello', '', '', '', 'world', 'world']); SELECT arrayCompact([[[]], [[], []], [[], []], [[]]]); -SELECT arrayCompact(x -> toString(intDiv(x, 3)), range(number)) FROM numbers(10); +SELECT arrayCompact(arrayMap(x -> toString(intDiv(x, 3)), range(number))) FROM numbers(10); +SELECT arrayCompact(x -> x.2, groupArray((number, intDiv(number, 3) % 3))) FROM numbers(10); +SELECT arrayCompact(x -> x.2, groupArray((toString(number), toString(intDiv(number, 3) % 3)))) FROM numbers(10); +SELECT arrayCompact(x -> x.2, groupArray((toString(number), intDiv(number, 3) % 3))) FROM numbers(10); diff --git a/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh b/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh index 938eeff5b67..711a015f044 100755 --- a/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh +++ b/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh @@ -37,5 +37,5 @@ $CLICKHOUSE_CLIENT --query=""" ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), ('bar', 'd')) ORDER BY Payload LIMIT 1 BY Phrase - ) ORDER BY Payload, Payload + ) ORDER BY Payload, Phrase """ diff --git a/tests/queries/0_stateless/01031_semi_anti_join.sql b/tests/queries/0_stateless/01031_semi_anti_join.sql index 19ea219563a..388b3d2fe4c 100644 --- a/tests/queries/0_stateless/01031_semi_anti_join.sql +++ b/tests/queries/0_stateless/01031_semi_anti_join.sql @@ -10,16 +10,16 @@ INSERT INTO t2 (x, s) VALUES (2, 'b1'), (2, 'b2'), (4, 'b3'), (4, 'b4'), (4, 'b5 SET join_use_nulls = 0; SELECT 'semi left'; -SELECT t1.*, t2.* FROM t1 SEMI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 SEMI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; SELECT 'semi right'; -SELECT t1.*, t2.* FROM t1 SEMI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 SEMI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; SELECT 'anti left'; -SELECT t1.*, t2.* FROM t1 ANTI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 ANTI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; SELECT 'anti right'; -SELECT t1.*, t2.* FROM t1 ANTI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 ANTI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; DROP TABLE t1; DROP TABLE t2; diff --git a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh index b8b261aefad..74b7494f041 100755 --- a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper +# Tags: zookeeper, no-fasttest CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none diff --git a/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql b/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql index 4eea4fd47c7..6d1c7fd5ef6 100644 --- a/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql +++ b/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql @@ -1,5 +1,7 @@ -- Tags: replica, distributed +set allow_experimental_parallel_reading_from_replicas=0; + drop table if exists test_max_parallel_replicas_lr; -- If you wonder why the table is named with "_lr" suffix in this test. diff --git a/tests/queries/0_stateless/01034_sample_final_distributed.sql b/tests/queries/0_stateless/01034_sample_final_distributed.sql index b784b35cbb3..a81fef645db 100644 --- a/tests/queries/0_stateless/01034_sample_final_distributed.sql +++ b/tests/queries/0_stateless/01034_sample_final_distributed.sql @@ -1,5 +1,7 @@ -- Tags: distributed +set allow_experimental_parallel_reading_from_replicas = 0; + drop table if exists sample_final; create table sample_final (CounterID UInt32, EventDate Date, EventTime DateTime, UserID UInt64, Sign Int8) engine = CollapsingMergeTree(Sign) order by (CounterID, EventDate, intHash32(UserID), EventTime) sample by intHash32(UserID); insert into sample_final select number / (8192 * 4), toDate('2019-01-01'), toDateTime('2019-01-01 00:00:01') + number, number / (8192 * 2), number % 3 = 1 ? -1 : 1 from numbers(1000000); diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh index 16331a49bc4..19c97ae96ec 100755 --- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-parallel +# Tags: zookeeper, no-parallel, no-fasttest set -e @@ -15,66 +15,46 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = Repli function thread1() { - while true; - do - $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" --query_id=query1 - done + $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst" } function thread2() { - while true; - do - $CLICKHOUSE_CLIENT --query="INSERT INTO src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" --query_id=query2 - done + $CLICKHOUSE_CLIENT --query="INSERT INTO src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" } function thread3() { - while true; - do - $CLICKHOUSE_CLIENT --query="SELECT * FROM src" --query_id=query3 1> /dev/null - done + $CLICKHOUSE_CLIENT --query="SELECT * FROM src" > /dev/null } function thread4() { - while true; - do - $CLICKHOUSE_CLIENT --query="SELECT * FROM dst" --query_id=query4 1> /dev/null - done + $CLICKHOUSE_CLIENT --query="SELECT * FROM dst" > /dev/null } function thread5() { - while true; - do - $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" --query_id=query5 - done + $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; -export -f thread5; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 +export -f thread5 TIMEOUT=30 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread4 2> /dev/null & -timeout $TIMEOUT bash -c thread5 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & wait echo "DROP TABLE src NO DELAY" | ${CLICKHOUSE_CLIENT} echo "DROP TABLE dst NO DELAY" | ${CLICKHOUSE_CLIENT} -sleep 5 - -# Check for deadlocks -echo "SELECT * FROM system.processes WHERE query_id LIKE 'query%'" | ${CLICKHOUSE_CLIENT} echo 'did not crash' diff --git a/tests/queries/0_stateless/01048_window_view_parser.reference b/tests/queries/0_stateless/01048_window_view_parser.reference index c055971bef3..947b68c3a89 100644 --- a/tests/queries/0_stateless/01048_window_view_parser.reference +++ b/tests/queries/0_stateless/01048_window_view_parser.reference @@ -1,34 +1,34 @@ ---TUMBLE--- ||---WINDOW COLUMN NAME--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1))`\nORDER BY `windowID(timestamp, toIntervalSecond(1))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(1))`\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN ALIAS--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ||---IDENTIFIER--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (b, `windowID(timestamp, toIntervalSecond(\'1\'))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, b)\nSETTINGS index_granularity = 8192 ||---FUNCTION--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (`plus(a, b)`, `windowID(timestamp, toIntervalSecond(\'1\'))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 ||---TimeZone--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`\nSETTINGS index_granularity = 8192 ||---DATA COLUMN ALIAS--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (b, `windowID(timestamp, toIntervalSecond(\'1\'))`)\nSETTINGS index_granularity = 8192 ||---JOIN--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ---HOP--- ||---WINDOW COLUMN NAME--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`\nORDER BY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN ALIAS--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 ||---IDENTIFIER--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (b, `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, b)\nSETTINGS index_granularity = 8192 ||---FUNCTION--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (`plus(a, b)`, `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 ||---TimeZone--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`\nORDER BY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`\nSETTINGS index_granularity = 8192 ||---DATA COLUMN ALIAS--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY (b, `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`)\nSETTINGS index_granularity = 8192 ||---JOIN--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh b/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh index 04b1f8b65ce..6df9cc3e258 100755 --- a/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh +++ b/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh @@ -18,56 +18,44 @@ $CLICKHOUSE_CLIENT --query="insert into test_01054.ints values (3, 3, 3, 3, 3, 3 function thread1() { - for _ in {1..100} - do RAND_NUMBER_THREAD1=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD1));" - done } function thread2() { - for _ in {1..100} - do RAND_NUMBER_THREAD2=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD2));" - done } function thread3() { - for _ in {1..100} - do RAND_NUMBER_THREAD3=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD3));" - done } function thread4() { - for _ in {1..100} - do RAND_NUMBER_THREAD4=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD4));" - done } -export -f thread1; -export -f thread2; -export -f thread3; -export -f thread4; +export -f thread1 +export -f thread2 +export -f thread3 +export -f thread4 TIMEOUT=10 # shellcheck disable=SC2188 -timeout $TIMEOUT bash -c thread1 > /dev/null 2>&1 & -timeout $TIMEOUT bash -c thread2 > /dev/null 2>&1 & -timeout $TIMEOUT bash -c thread3 > /dev/null 2>&1 & -timeout $TIMEOUT bash -c thread4 > /dev/null 2>&1 & +clickhouse_client_loop_timeout $TIMEOUT thread1 > /dev/null 2>&1 & +clickhouse_client_loop_timeout $TIMEOUT thread2 > /dev/null 2>&1 & +clickhouse_client_loop_timeout $TIMEOUT thread3 > /dev/null 2>&1 & +clickhouse_client_loop_timeout $TIMEOUT thread4 > /dev/null 2>&1 & wait diff --git a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql index d59b8fc30ac..6d2bb2964d6 100644 --- a/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql +++ b/tests/queries/0_stateless/01056_predicate_optimizer_bugs.sql @@ -1,5 +1,6 @@ SET enable_optimize_predicate_expression = 1; SET joined_subquery_requires_alias = 0; +SET convert_query_to_cnf = 0; -- https://github.com/ClickHouse/ClickHouse/issues/3885 -- https://github.com/ClickHouse/ClickHouse/issues/5485 diff --git a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py index 02e97ee7a17..be139c153aa 100755 --- a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py +++ b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py @@ -4,7 +4,7 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block @@ -12,46 +12,54 @@ log = None # uncomment the line below for debugging # log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_window_view = 1') + client1.send("SET allow_experimental_window_view = 1") client1.expect(prompt) - client1.send('SET window_view_heartbeat_interval = 1') + client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) - client2.send('SET allow_experimental_window_view = 1') + client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) - client1.send('CREATE DATABASE 01056_window_view_proc_hop_watch') + client1.send("CREATE DATABASE 01056_window_view_proc_hop_watch") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01056_window_view_proc_hop_watch.mt') + client1.send("DROP TABLE IF EXISTS 01056_window_view_proc_hop_watch.mt") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01056_window_view_proc_hop_watch.wv') + client1.send("DROP TABLE IF EXISTS 01056_window_view_proc_hop_watch.wv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS `.inner.wv`') + client1.send("DROP TABLE IF EXISTS `.inner.wv`") client1.expect(prompt) - client1.send('CREATE TABLE 01056_window_view_proc_hop_watch.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()') + client1.send( + "CREATE TABLE 01056_window_view_proc_hop_watch.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client1.send("CREATE WINDOW VIEW 01056_window_view_proc_hop_watch.wv AS SELECT count(a) AS count FROM 01056_window_view_proc_hop_watch.mt GROUP BY hop(timestamp, INTERVAL '1' SECOND, INTERVAL '1' SECOND, 'US/Samoa') AS wid;") + client1.send( + "CREATE WINDOW VIEW 01056_window_view_proc_hop_watch.wv AS SELECT count(a) AS count FROM 01056_window_view_proc_hop_watch.mt GROUP BY hop(timestamp, INTERVAL '1' SECOND, INTERVAL '1' SECOND, 'US/Samoa') AS wid;" + ) client1.expect(prompt) - - client1.send('WATCH 01056_window_view_proc_hop_watch.wv') - client1.expect('Query id' + end_of_block) - client2.send("INSERT INTO 01056_window_view_proc_hop_watch.mt VALUES (1, now('US/Samoa') + 1)") - client1.expect('1' + end_of_block) - client1.expect('Progress: 1.00 rows.*\)') + + client1.send("WATCH 01056_window_view_proc_hop_watch.wv") + client1.expect("Query id" + end_of_block) + client2.send( + "INSERT INTO 01056_window_view_proc_hop_watch.mt VALUES (1, now('US/Samoa') + 1)" + ) + client1.expect("1" + end_of_block) + client1.expect("Progress: 1.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE 01056_window_view_proc_hop_watch.wv') + client1.send("DROP TABLE 01056_window_view_proc_hop_watch.wv") client1.expect(prompt) - client1.send('DROP TABLE 01056_window_view_proc_hop_watch.mt') + client1.send("DROP TABLE 01056_window_view_proc_hop_watch.mt") client1.expect(prompt) - client1.send('DROP DATABASE IF EXISTS 01056_window_view_proc_hop_watch') + client1.send("DROP DATABASE IF EXISTS 01056_window_view_proc_hop_watch") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py index 638182ac216..f5024cb11ab 100755 --- a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py +++ b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py @@ -4,59 +4,71 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_window_view = 1') + client1.send("SET allow_experimental_window_view = 1") client1.expect(prompt) - client1.send('SET window_view_heartbeat_interval = 1') + client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) - client1.send('CREATE DATABASE db_01059_event_hop_watch_strict_asc') + client1.send("CREATE DATABASE db_01059_event_hop_watch_strict_asc") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.mt') + client1.send("DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.mt") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.wv NO DELAY') + client1.send("DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.wv NO DELAY") client1.expect(prompt) - client1.send("CREATE TABLE db_01059_event_hop_watch_strict_asc.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()") + client1.send( + "CREATE TABLE db_01059_event_hop_watch_strict_asc.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client1.send("CREATE WINDOW VIEW db_01059_event_hop_watch_strict_asc.wv WATERMARK=STRICTLY_ASCENDING AS SELECT count(a) AS count, hopEnd(wid) as w_end FROM db_01059_event_hop_watch_strict_asc.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid;") + client1.send( + "CREATE WINDOW VIEW db_01059_event_hop_watch_strict_asc.wv WATERMARK=STRICTLY_ASCENDING AS SELECT count(a) AS count, hopEnd(wid) as w_end FROM db_01059_event_hop_watch_strict_asc.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid;" + ) client1.expect(prompt) - client1.send('WATCH db_01059_event_hop_watch_strict_asc.wv') - client1.expect('Query id' + end_of_block) - client2.send("INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));") + client1.send("WATCH db_01059_event_hop_watch_strict_asc.wv") + client1.expect("Query id" + end_of_block) + client2.send( + "INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));" + ) client2.expect("Ok.") - client2.send("INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));") + client2.send( + "INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));" + ) client2.expect("Ok.") - client1.expect('1*1990-01-01 12:00:02' + end_of_block) - client1.expect('Progress: 1.00 rows.*\)') + client1.expect("1*1990-01-01 12:00:02" + end_of_block) + client1.expect("Progress: 1.00 rows.*\)") - client2.send("INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:10', 'US/Samoa'));") + client2.send( + "INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:10', 'US/Samoa'));" + ) client2.expect("Ok.") - client1.expect('1*1990-01-01 12:00:06' + end_of_block) - client1.expect('1*1990-01-01 12:00:08' + end_of_block) - client1.expect('Progress: 3.00 rows.*\)') + client1.expect("1*1990-01-01 12:00:06" + end_of_block) + client1.expect("1*1990-01-01 12:00:08" + end_of_block) + client1.expect("Progress: 3.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE db_01059_event_hop_watch_strict_asc.wv NO DELAY') + client1.send("DROP TABLE db_01059_event_hop_watch_strict_asc.wv NO DELAY") client1.expect(prompt) - client1.send('DROP TABLE db_01059_event_hop_watch_strict_asc.mt') + client1.send("DROP TABLE db_01059_event_hop_watch_strict_asc.mt") client1.expect(prompt) - client1.send('DROP DATABASE IF EXISTS db_01059_event_hop_watch_strict_asc') + client1.send("DROP DATABASE IF EXISTS db_01059_event_hop_watch_strict_asc") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01060_avro.reference b/tests/queries/0_stateless/01060_avro.reference index 224a369d993..a375ae280a9 100644 --- a/tests/queries/0_stateless/01060_avro.reference +++ b/tests/queries/0_stateless/01060_avro.reference @@ -42,6 +42,7 @@ not compatible = compression 1000 1000 +1000 = other 0 1000 diff --git a/tests/queries/0_stateless/01060_avro.sh b/tests/queries/0_stateless/01060_avro.sh index 1cfe5582d0a..3c70927db25 100755 --- a/tests/queries/0_stateless/01060_avro.sh +++ b/tests/queries/0_stateless/01060_avro.sh @@ -50,8 +50,12 @@ echo '=' compression cat "$DATA_DIR"/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' cat "$DATA_DIR"/simple.deflate.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' -#snappy is optional -#cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +# snappy is optional +if [ "$( ${CLICKHOUSE_LOCAL} -q "SELECT value FROM system.build_options where name = 'USE_SNAPPY' LIMIT 1")" == "1" ]; then +cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +else +echo 1000 +fi echo '=' other #no data diff --git a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.reference b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.reference index 55c1ee45827..de722f47f08 100644 --- a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.reference +++ b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.reference @@ -1,3 +1,7 @@ -3 1990-01-01 12:00:05 -2 1990-01-01 12:00:10 -2 1990-01-01 12:00:15 +1 1 1990-01-01 12:00:05 +1 2 1990-01-01 12:00:05 +1 3 1990-01-01 12:00:05 +1 4 1990-01-01 12:00:10 +1 5 1990-01-01 12:00:10 +1 6 1990-01-01 12:00:15 +1 7 1990-01-01 12:00:15 diff --git a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh index e570f405f62..9163fe8af27 100755 --- a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh +++ b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh @@ -10,25 +10,25 @@ DROP TABLE IF EXISTS mt; DROP TABLE IF EXISTS dst; DROP TABLE IF EXISTS wv; -CREATE TABLE dst(count UInt64, w_end DateTime) Engine=MergeTree ORDER BY tuple(); -CREATE TABLE mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); -CREATE WINDOW VIEW wv TO dst WATERMARK=ASCENDING AS SELECT count(a) AS count, tumbleEnd(wid) AS w_end FROM mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid; +CREATE TABLE dst(count UInt64, market Int32, w_end DateTime) Engine=MergeTree ORDER BY tuple(); +CREATE TABLE mt(a Int32, market Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); +CREATE WINDOW VIEW wv TO dst WATERMARK=ASCENDING AS SELECT count(a) AS count, market, tumbleEnd(wid) AS w_end FROM mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, market; -INSERT INTO mt VALUES (1, '1990/01/01 12:00:00'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:01'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:02'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:05'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:06'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:10'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:11'); -INSERT INTO mt VALUES (1, '1990/01/01 12:00:30'); +INSERT INTO mt VALUES (1, 1, '1990/01/01 12:00:00'); +INSERT INTO mt VALUES (1, 2, '1990/01/01 12:00:01'); +INSERT INTO mt VALUES (1, 3, '1990/01/01 12:00:02'); +INSERT INTO mt VALUES (1, 4, '1990/01/01 12:00:05'); +INSERT INTO mt VALUES (1, 5, '1990/01/01 12:00:06'); +INSERT INTO mt VALUES (1, 6, '1990/01/01 12:00:10'); +INSERT INTO mt VALUES (1, 7, '1990/01/01 12:00:11'); +INSERT INTO mt VALUES (1, 8, '1990/01/01 12:00:30'); EOF while true; do - $CLICKHOUSE_CLIENT --query="SELECT count(*) FROM dst" | grep -q "3" && break || sleep .5 ||: + $CLICKHOUSE_CLIENT --query="SELECT count(*) FROM dst" | grep -q "7" && break || sleep .5 ||: done -$CLICKHOUSE_CLIENT --query="SELECT * FROM dst ORDER BY w_end;" +$CLICKHOUSE_CLIENT --query="SELECT * FROM dst ORDER BY market, w_end;" $CLICKHOUSE_CLIENT --query="DROP TABLE wv" $CLICKHOUSE_CLIENT --query="DROP TABLE mt" $CLICKHOUSE_CLIENT --query="DROP TABLE dst" diff --git a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py index 6be3e08665c..7f23e983ba2 100755 --- a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py +++ b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py @@ -4,61 +4,77 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_window_view = 1') + client1.send("SET allow_experimental_window_view = 1") client1.expect(prompt) - client1.send('SET window_view_heartbeat_interval = 1') + client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) - client2.send('SET allow_experimental_window_view = 1') + client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) - client1.send('CREATE DATABASE 01062_window_view_event_hop_watch_asc') + client1.send("CREATE DATABASE 01062_window_view_event_hop_watch_asc") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.mt') + client1.send("DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.mt") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.wv NO DELAY') + client1.send( + "DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.wv NO DELAY" + ) client1.expect(prompt) - client1.send('CREATE TABLE 01062_window_view_event_hop_watch_asc.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()') + client1.send( + "CREATE TABLE 01062_window_view_event_hop_watch_asc.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client1.send("CREATE WINDOW VIEW 01062_window_view_event_hop_watch_asc.wv WATERMARK=ASCENDING AS SELECT count(a) AS count, hopEnd(wid) AS w_end FROM 01062_window_view_event_hop_watch_asc.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid") + client1.send( + "CREATE WINDOW VIEW 01062_window_view_event_hop_watch_asc.wv WATERMARK=ASCENDING AS SELECT count(a) AS count, hopEnd(wid) AS w_end FROM 01062_window_view_event_hop_watch_asc.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid" + ) client1.expect(prompt) - - client1.send('WATCH 01062_window_view_event_hop_watch_asc.wv') - client1.expect('Query id' + end_of_block) - client2.send("INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:00');") + + client1.send("WATCH 01062_window_view_event_hop_watch_asc.wv") + client1.expect("Query id" + end_of_block) + client2.send( + "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:00');" + ) client2.expect(prompt) - client2.send("INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:05');") + client2.send( + "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:05');" + ) client2.expect(prompt) - client1.expect('1*' + end_of_block) - client2.send("INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:06');") + client1.expect("1*" + end_of_block) + client2.send( + "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:06');" + ) client2.expect(prompt) - client2.send("INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:10');") + client2.send( + "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, '1990/01/01 12:00:10');" + ) client2.expect(prompt) - client1.expect('1' + end_of_block) - client1.expect('2' + end_of_block) - client1.expect('Progress: 3.00 rows.*\)') + client1.expect("1" + end_of_block) + client1.expect("2" + end_of_block) + client1.expect("Progress: 3.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE 01062_window_view_event_hop_watch_asc.wv NO DELAY') + client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.wv NO DELAY") client1.expect(prompt) - client1.send('DROP TABLE 01062_window_view_event_hop_watch_asc.mt') + client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.mt") client1.expect(prompt) - client1.send('DROP DATABASE IF EXISTS 01062_window_view_event_hop_watch_asc') + client1.send("DROP DATABASE IF EXISTS 01062_window_view_event_hop_watch_asc") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py index b828c5116da..92d2b56ed34 100755 --- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py +++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py @@ -4,7 +4,7 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block @@ -12,48 +12,54 @@ log = None # uncomment the line below for debugging # log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_window_view = 1') + client1.send("SET allow_experimental_window_view = 1") client1.expect(prompt) - client1.send('SET window_view_heartbeat_interval = 1') + client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) - client2.send('SET allow_experimental_window_view = 1') + client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.wv') + client1.send("DROP TABLE IF EXISTS test.wv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS `.inner.wv`') + client1.send("DROP TABLE IF EXISTS `.inner.wv`") client1.expect(prompt) - client1.send('CREATE TABLE test.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()') + client1.send( + "CREATE TABLE test.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client1.send("CREATE WINDOW VIEW test.wv WATERMARK=INTERVAL '2' SECOND AS SELECT count(a) AS count, hopEnd(wid) AS w_end FROM test.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid") + client1.send( + "CREATE WINDOW VIEW test.wv WATERMARK=INTERVAL '2' SECOND AS SELECT count(a) AS count, hopEnd(wid) AS w_end FROM test.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid" + ) client1.expect(prompt) - - client1.send('WATCH test.wv') + + client1.send("WATCH test.wv") client2.send("INSERT INTO test.mt VALUES (1, '1990/01/01 12:00:00');") client2.expect(prompt) client2.send("INSERT INTO test.mt VALUES (1, '1990/01/01 12:00:05');") client2.expect(prompt) - client1.expect('1*' + end_of_block) + client1.expect("1*" + end_of_block) client2.send("INSERT INTO test.mt VALUES (1, '1990/01/01 12:00:06');") client2.send("INSERT INTO test.mt VALUES (1, '1990/01/01 12:00:10');") client2.expect(prompt) - client1.expect('1*' + end_of_block) - client1.expect('2*' + end_of_block) + client1.expect("1*" + end_of_block) + client1.expect("2*" + end_of_block) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE test.wv') + client1.send("DROP TABLE test.wv") client1.expect(prompt) - client1.send('DROP TABLE test.mt') + client1.send("DROP TABLE test.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py index eb2b7835483..4c675fcabfb 100755 --- a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py +++ b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py @@ -1,62 +1,72 @@ #!/usr/bin/env python3 -#Tags: no-parallel +# Tags: no-parallel import os import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_window_view = 1') + client1.send("SET allow_experimental_window_view = 1") client1.expect(prompt) - client1.send('SET window_view_heartbeat_interval = 1') + client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) - client2.send('SET allow_experimental_window_view = 1') + client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) - client1.send('CREATE DATABASE 01069_window_view_proc_tumble_watch') + client1.send("CREATE DATABASE 01069_window_view_proc_tumble_watch") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.mt') + client1.send("DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.mt") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.wv NO DELAY') + client1.send("DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.wv NO DELAY") client1.expect(prompt) - client1.send('CREATE TABLE 01069_window_view_proc_tumble_watch.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()') + client1.send( + "CREATE TABLE 01069_window_view_proc_tumble_watch.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client1.send("CREATE WINDOW VIEW 01069_window_view_proc_tumble_watch.wv AS SELECT count(a) AS count FROM 01069_window_view_proc_tumble_watch.mt GROUP BY tumble(timestamp, INTERVAL '1' SECOND, 'US/Samoa') AS wid;") + client1.send( + "CREATE WINDOW VIEW 01069_window_view_proc_tumble_watch.wv AS SELECT count(a) AS count FROM 01069_window_view_proc_tumble_watch.mt GROUP BY tumble(timestamp, INTERVAL '1' SECOND, 'US/Samoa') AS wid;" + ) client1.expect(prompt) - client1.send('WATCH 01069_window_view_proc_tumble_watch.wv') - client1.expect('Query id' + end_of_block) - client2.send("INSERT INTO 01069_window_view_proc_tumble_watch.mt VALUES (1, now('US/Samoa') + 3)") + client1.send("WATCH 01069_window_view_proc_tumble_watch.wv") + client1.expect("Query id" + end_of_block) + client2.send( + "INSERT INTO 01069_window_view_proc_tumble_watch.mt VALUES (1, now('US/Samoa') + 3)" + ) client2.expect("Ok.") - client1.expect('1' + end_of_block) - client1.expect('Progress: 1.00 rows.*\)') - client2.send("INSERT INTO 01069_window_view_proc_tumble_watch.mt VALUES (1, now('US/Samoa') + 3)") + client1.expect("1" + end_of_block) + client1.expect("Progress: 1.00 rows.*\)") + client2.send( + "INSERT INTO 01069_window_view_proc_tumble_watch.mt VALUES (1, now('US/Samoa') + 3)" + ) client2.expect("Ok.") - client1.expect('1' + end_of_block) - client1.expect('Progress: 2.00 rows.*\)') + client1.expect("1" + end_of_block) + client1.expect("Progress: 2.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE 01069_window_view_proc_tumble_watch.wv NO DELAY') + client1.send("DROP TABLE 01069_window_view_proc_tumble_watch.wv NO DELAY") client1.expect(prompt) - client1.send('DROP TABLE 01069_window_view_proc_tumble_watch.mt') + client1.send("DROP TABLE 01069_window_view_proc_tumble_watch.mt") client1.expect(prompt) - client1.send('DROP DATABASE IF EXISTS 01069_window_view_proc_tumble_watch') + client1.send("DROP DATABASE IF EXISTS 01069_window_view_proc_tumble_watch") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01070_mutations_with_dependencies.sql b/tests/queries/0_stateless/01070_mutations_with_dependencies.sql index 6f297e25b95..506fd23904f 100644 --- a/tests/queries/0_stateless/01070_mutations_with_dependencies.sql +++ b/tests/queries/0_stateless/01070_mutations_with_dependencies.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-s3-storage drop table if exists ttl; set mutations_sync = 2; diff --git a/tests/queries/0_stateless/01070_window_view_watch_events.py b/tests/queries/0_stateless/01070_window_view_watch_events.py index 51330ce1c01..2bf732d68e5 100755 --- a/tests/queries/0_stateless/01070_window_view_watch_events.py +++ b/tests/queries/0_stateless/01070_window_view_watch_events.py @@ -6,7 +6,7 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block @@ -14,47 +14,57 @@ log = None # uncomment the line below for debugging # log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2: client1.expect(prompt) client2.expect(prompt) - client1.send('SET allow_experimental_window_view = 1') + client1.send("SET allow_experimental_window_view = 1") client1.expect(prompt) - client1.send('SET window_view_heartbeat_interval = 1') + client1.send("SET window_view_heartbeat_interval = 1") client1.expect(prompt) - client2.send('SET allow_experimental_window_view = 1') + client2.send("SET allow_experimental_window_view = 1") client2.expect(prompt) - client1.send('CREATE DATABASE IF NOT EXISTS 01070_window_view_watch_events') + client1.send("CREATE DATABASE IF NOT EXISTS 01070_window_view_watch_events") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01070_window_view_watch_events.mt NO DELAY') + client1.send("DROP TABLE IF EXISTS 01070_window_view_watch_events.mt NO DELAY") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS 01070_window_view_watch_events.wv NO DELAY') + client1.send("DROP TABLE IF EXISTS 01070_window_view_watch_events.wv NO DELAY") client1.expect(prompt) - client1.send("CREATE TABLE 01070_window_view_watch_events.mt(a Int32, timestamp DateTime('US/Samoa')) ENGINE=MergeTree ORDER BY tuple()") + client1.send( + "CREATE TABLE 01070_window_view_watch_events.mt(a Int32, timestamp DateTime('US/Samoa')) ENGINE=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client1.send("CREATE WINDOW VIEW 01070_window_view_watch_events.wv WATERMARK=ASCENDING AS SELECT count(a) AS count, tumbleEnd(wid) AS w_end FROM 01070_window_view_watch_events.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid") + client1.send( + "CREATE WINDOW VIEW 01070_window_view_watch_events.wv WATERMARK=ASCENDING AS SELECT count(a) AS count, tumbleEnd(wid) AS w_end FROM 01070_window_view_watch_events.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid" + ) client1.expect(prompt) - client1.send('WATCH 01070_window_view_watch_events.wv EVENTS') - client1.expect('Query id' + end_of_block) - client2.send("INSERT INTO 01070_window_view_watch_events.mt VALUES (1, '1990/01/01 12:00:00');") + client1.send("WATCH 01070_window_view_watch_events.wv EVENTS") + client1.expect("Query id" + end_of_block) + client2.send( + "INSERT INTO 01070_window_view_watch_events.mt VALUES (1, '1990/01/01 12:00:00');" + ) client2.expect("Ok.") - client2.send("INSERT INTO 01070_window_view_watch_events.mt VALUES (1, '1990/01/01 12:00:06');") + client2.send( + "INSERT INTO 01070_window_view_watch_events.mt VALUES (1, '1990/01/01 12:00:06');" + ) client2.expect("Ok.") - client1.expect('1990-01-01 12:00:05' + end_of_block) - client1.expect('Progress: 1.00 rows.*\)') + client1.expect("1990-01-01 12:00:05" + end_of_block) + client1.expect("Progress: 1.00 rows.*\)") # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send('DROP TABLE 01070_window_view_watch_events.wv NO DELAY;') + client1.send("DROP TABLE 01070_window_view_watch_events.wv NO DELAY;") client1.expect(prompt) - client1.send('DROP TABLE 01070_window_view_watch_events.mt;') + client1.send("DROP TABLE 01070_window_view_watch_events.mt;") client1.expect(prompt) - client1.send('DROP DATABASE IF EXISTS 01070_window_view_watch_events;') + client1.send("DROP DATABASE IF EXISTS 01070_window_view_watch_events;") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh index 17068dcbdf9..a1a38370554 100755 --- a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh +++ b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh @@ -38,30 +38,24 @@ LAYOUT(CACHE(SIZE_IN_CELLS 10)); function thread1() { - for _ in {1..50} - do - # This query will be ended with exception, because source dictionary has UUID as a key type. - $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(1));" - done + # This query will be ended with exception, because source dictionary has UUID as a key type. + $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(1));" } function thread2() { - for _ in {1..50} - do - # This query will be ended with exception, because source dictionary has UUID as a key type. - $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(2));" - done + # This query will be ended with exception, because source dictionary has UUID as a key type. + $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(2));" } -export -f thread1; -export -f thread2; +export -f thread1 +export -f thread2 TIMEOUT=5 -timeout $TIMEOUT bash -c thread1 > /dev/null 2>&1 & -timeout $TIMEOUT bash -c thread2 > /dev/null 2>&1 & +clickhouse_client_loop_timeout $TIMEOUT thread1 > /dev/null 2>&1 & +clickhouse_client_loop_timeout $TIMEOUT thread2 > /dev/null 2>&1 & wait diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index bbc16121cb6..6c9acd6d0cd 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -21,7 +21,12 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" + $CLICKHOUSE_CLIENT -nm --query " + CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') + ORDER BY key + SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0; + " done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" @@ -40,59 +45,52 @@ INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mut # Run mutation on random replica function correct_alter_thread() { - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1"; - sleep 1 - done + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1" + sleep 1 } # This thread add some data to table. function insert_thread() { - VALUES=(7 8 9) - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 5 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" + sleep 0.$RANDOM } function detach_attach_thread() { - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" - sleep 0.$RANDOM - sleep 0.$RANDOM - sleep 0.$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$REPLICA" - done + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" + sleep 0.$RANDOM + sleep 0.$RANDOM + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$REPLICA" } echo "Starting alters" -export -f correct_alter_thread; -export -f insert_thread; -export -f detach_attach_thread; +export -f correct_alter_thread +export -f insert_thread +export -f detach_attach_thread # We assign a lot of mutations so timeout shouldn't be too big TIMEOUT=15 -timeout $TIMEOUT bash -c detach_attach_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT detach_attach_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql b/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql index 2d6007c678c..3a05e4507a2 100644 --- a/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql +++ b/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage +-- Output slightly different plan drop table if exists t; create table t (a Int, b Int) engine = MergeTree order by (a, b) settings index_granularity = 400; diff --git a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index 06d6ef6a94b..77278ad0bf9 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -15,7 +15,12 @@ done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT -nm --query " + CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_column', '$i') + ORDER BY key + SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000; + " done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_1 SELECT number, number + 10 from numbers(100000)" @@ -27,58 +32,54 @@ done function alter_thread() { - while true; do - REPLICA=$(($RANDOM % 3 + 1)) - ADD=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency - DROP=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 3 + 1)) + ADD=$(($RANDOM % 5 + 1)) + # additionaly we don't wait anything for more heavy concurrency + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0" + DROP=$(($RANDOM % 5 + 1)) + # additionaly we don't wait anything for more heavy concurrency + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0" + sleep 0.$RANDOM } function optimize_thread() { - while true; do - REPLICA=$(($RANDOM % 3 + 1)) - $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_alter_add_drop_$REPLICA FINAL SETTINGS replication_alter_partitions_sync=0"; - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_alter_add_drop_$REPLICA FINAL SETTINGS replication_alter_partitions_sync=0" + sleep 0.$RANDOM } function insert_thread() { - while true; do - REPLICA=$(($RANDOM % 3 + 1)) - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_$REPLICA VALUES($RANDOM, 7)" - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_$REPLICA VALUES($RANDOM, 7)" + sleep 0.$RANDOM } echo "Starting alters" -export -f alter_thread; -export -f optimize_thread; -export -f insert_thread; +export -f alter_thread +export -f optimize_thread +export -f insert_thread TIMEOUT=30 # Sometimes we detach and attach tables -timeout $TIMEOUT bash -c alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index c95554d26bb..20a5f9af108 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-parallel, no-fasttest +# Tags: zookeeper, no-parallel, no-fasttest, no-backward-compatibility-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -12,11 +12,24 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" + $CLICKHOUSE_CLIENT -nm --query " + CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_detach', '$i') + ORDER BY key + SETTINGS + max_replicated_mutations_in_queue=1000, + number_of_free_entries_in_pool_to_execute_mutation=0, + max_replicated_merges_in_queue=1000, + temporary_directories_lifetime=10, + cleanup_delay_period=3, + cleanup_delay_period_random_add=0; + " done -$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10)" -$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10, 40)" +$CLICKHOUSE_CLIENT -nm --query " + INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10); + INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10, 40); +" for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i" @@ -31,12 +44,11 @@ INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alt function correct_alter_thread() { TYPES=(Float64 String UInt8 UInt32) - while true; do - REPLICA=$(($RANDOM % 3 + 1)) - TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 3 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + # additionaly we don't wait anything for more heavy concurrency + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0" + sleep 0.$RANDOM } # This thread add some data to table. After we finish we can check, that @@ -44,43 +56,38 @@ function correct_alter_thread() # insert queries will fail sometime because of wrong types. function insert_thread() { - VALUES=(7.0 7 '7') - while true; do - REPLICA=$(($RANDOM % 3 + 1)) - VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 3 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" + sleep 0.$RANDOM } function detach_attach_thread() { - while true; do - REPLICA=$(($RANDOM % 3 + 1)) - $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_alter_detach_$REPLICA" - sleep 0.$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$REPLICA" - done + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_alter_detach_$REPLICA" + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$REPLICA" } echo "Starting alters" -export -f correct_alter_thread; -export -f insert_thread; -export -f detach_attach_thread; +export -f correct_alter_thread +export -f insert_thread +export -f detach_attach_thread TIMEOUT=15 # Sometimes we detach and attach tables -timeout $TIMEOUT bash -c detach_attach_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT detach_attach_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh index ba8d89aad3c..aef23c460d8 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh @@ -14,7 +14,11 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT -nm --query " + CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_mt', '$i') + ORDER BY key + SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" @@ -36,12 +40,10 @@ INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alt function correct_alter_thread() { TYPES=(Float64 String UInt8 UInt32) - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 5 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM } # This thread add some data to table. After we finish we can check, that @@ -49,56 +51,49 @@ function correct_alter_thread() # insert queries will fail sometime because of wrong types. function insert_thread() { - VALUES=(7.0 7 '7') - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 5 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" + sleep 0.$RANDOM } # Some select load, to be sure, that our selects work in concurrent execution with alters function select_thread() { - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) FROM concurrent_alter_mt_$REPLICA" 1>/dev/null - sleep 0.$RANDOM - done + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) FROM concurrent_alter_mt_$REPLICA" 1>/dev/null + sleep 0.$RANDOM } echo "Starting alters" -export -f correct_alter_thread; -export -f insert_thread; -export -f select_thread; +export -f correct_alter_thread +export -f insert_thread +export -f select_thread TIMEOUT=30 # Selects should run successfully -timeout $TIMEOUT bash -c select_thread & -timeout $TIMEOUT bash -c select_thread & -timeout $TIMEOUT bash -c select_thread & +clickhouse_client_loop_timeout $TIMEOUT select_thread & +clickhouse_client_loop_timeout $TIMEOUT select_thread & +clickhouse_client_loop_timeout $TIMEOUT select_thread & +clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & - - -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01083_cross_to_inner_with_like.sql b/tests/queries/0_stateless/01083_cross_to_inner_with_like.sql index 644190cbddf..6ec6e80692c 100644 --- a/tests/queries/0_stateless/01083_cross_to_inner_with_like.sql +++ b/tests/queries/0_stateless/01083_cross_to_inner_with_like.sql @@ -1,3 +1,5 @@ +SET convert_query_to_cnf = 0; + DROP TABLE IF EXISTS n; DROP TABLE IF EXISTS r; diff --git a/tests/queries/0_stateless/01085_max_distributed_connections.sh b/tests/queries/0_stateless/01085_max_distributed_connections.sh index 34862289d1e..8a74935c05d 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections.sh @@ -18,6 +18,6 @@ while [[ $i -lt $retries ]]; do # 10 less then 20 seconds (20 streams), but long enough to cover possible load peaks # "$@" left to pass manual options (like --experimental_use_processors 0) during manual testing - timeout 10s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break + clickhouse_client_timeout 10s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break ((++i)) done diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sh b/tests/queries/0_stateless/01086_odbc_roundtrip.sh index 8e59bfd7f4d..20066c6b34c 100755 --- a/tests/queries/0_stateless/01086_odbc_roundtrip.sh +++ b/tests/queries/0_stateless/01086_odbc_roundtrip.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Tags: no-asan, no-msan, no-fasttest +# Tags: no-asan, no-msan, no-fasttest, no-cpu-aarch64 # Tag no-msan: can't pass because odbc libraries are not instrumented +# Tag no-cpu-aarch64: clickhouse-odbc is not setup for arm CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -17,7 +18,7 @@ ${CLICKHOUSE_CLIENT} --query "select count() > 1 as ok from (select * from odbc( ${CLICKHOUSE_CLIENT} --query "CREATE TABLE t (x UInt8, y Float32, z String) ENGINE = Memory" ${CLICKHOUSE_CLIENT} --query "INSERT INTO t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d')" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (ANSI)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (Unicode)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" +${CLICKHOUSE_CLIENT} --query "SELECT x, y, z FROM odbc('DSN={ClickHouse DSN (ANSI)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" +${CLICKHOUSE_CLIENT} --query "SELECT x, y, z FROM odbc('DSN={ClickHouse DSN (Unicode)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" ${CLICKHOUSE_CLIENT} --query "DROP TABLE t" diff --git a/tests/queries/0_stateless/01092_memory_profiler.sql b/tests/queries/0_stateless/01092_memory_profiler.sql index 9d042860ac0..3869bf941c0 100644 --- a/tests/queries/0_stateless/01092_memory_profiler.sql +++ b/tests/queries/0_stateless/01092_memory_profiler.sql @@ -1,4 +1,4 @@ --- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest +-- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest, no-cpu-aarch64 SET allow_introspection_functions = 1; diff --git a/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql b/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql index 4e011bf6b31..de93166d891 100644 --- a/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql +++ b/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql @@ -2,6 +2,8 @@ -- set insert_distributed_sync = 1; -- see https://github.com/ClickHouse/ClickHouse/issues/18971 +SET allow_experimental_parallel_reading_from_replicas = 0; -- see https://github.com/ClickHouse/ClickHouse/issues/34525 + DROP TABLE IF EXISTS local_01099_a; DROP TABLE IF EXISTS local_01099_b; DROP TABLE IF EXISTS distributed_01099_a; diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh index da99a13e97f..9b6e1e05f2d 100755 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest, no-cpu-aarch64 # Tag no-fasttest: avoid dependency on qemu -- invonvenient when running locally CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh index 95f8dfc0377..d73ce1aeef4 100755 --- a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh +++ b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh @@ -9,54 +9,53 @@ set -e function thread1() { - while true; do - $CLICKHOUSE_CLIENT -q "INSERT INTO concurrent_optimize_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(10000)"; - done + $CLICKHOUSE_CLIENT -q "INSERT INTO concurrent_optimize_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(10000)" } function thread2() { - while true; do - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE concurrent_optimize_table FINAL"; - sleep 0.$RANDOM; - done + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE concurrent_optimize_table FINAL" + sleep 0.$RANDOM } function thread3() { - while true; do - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS concurrent_optimize_table; - CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_optimize_table', '1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; - sleep 0.$RANDOM; - sleep 0.$RANDOM; - sleep 0.$RANDOM; - done + $CLICKHOUSE_CLIENT -mn -q " + DROP TABLE IF EXISTS concurrent_optimize_table; + CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_optimize_table', '1') + ORDER BY a + PARTITION BY b % 10 + SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + sleep 0.$RANDOM + sleep 0.$RANDOM + sleep 0.$RANDOM } -export -f thread1; -export -f thread2; -export -f thread3; +export -f thread1 +export -f thread2 +export -f thread3 TIMEOUT=15 -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -timeout $TIMEOUT bash -c thread1 2> /dev/null & -timeout $TIMEOUT bash -c thread2 2> /dev/null & -timeout $TIMEOUT bash -c thread3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh index 14ba90d5ec0..abd5c0d6a4f 100755 --- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh +++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh @@ -14,35 +14,29 @@ done function rename_thread_1() { - while true; do - $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1 TO replica_01108_1_tmp, - replica_01108_2 TO replica_01108_2_tmp, - replica_01108_3 TO replica_01108_3_tmp, - replica_01108_4 TO replica_01108_4_tmp"; - sleep 0.$RANDOM; - done + $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1 TO replica_01108_1_tmp, + replica_01108_2 TO replica_01108_2_tmp, + replica_01108_3 TO replica_01108_3_tmp, + replica_01108_4 TO replica_01108_4_tmp" + sleep 0.$RANDOM } function rename_thread_2() { - while true; do - $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1_tmp TO replica_01108_2, - replica_01108_2_tmp TO replica_01108_3, - replica_01108_3_tmp TO replica_01108_4, - replica_01108_4_tmp TO replica_01108_1"; - sleep 0.$RANDOM; - done + $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1_tmp TO replica_01108_2, + replica_01108_2_tmp TO replica_01108_3, + replica_01108_3_tmp TO replica_01108_4, + replica_01108_4_tmp TO replica_01108_1" + sleep 0.$RANDOM } function restart_replicas_loop() { - while true; do - for i in $(seq 4); do - $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}"; - $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}_tmp"; - done - sleep 0.$RANDOM; + for i in $(seq 4); do + $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}" + $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}_tmp" done + sleep 0.$RANDOM } function restart_thread_1() { @@ -54,17 +48,17 @@ function restart_thread_2() restart_replicas_loop } -export -f rename_thread_1; -export -f rename_thread_2; -export -f restart_thread_1; -export -f restart_thread_2; +export -f rename_thread_1 +export -f rename_thread_2 +export -f restart_thread_1 +export -f restart_thread_2 TIMEOUT=10 -timeout $TIMEOUT bash -c rename_thread_1 2> /dev/null & -timeout $TIMEOUT bash -c rename_thread_2 2> /dev/null & -timeout $TIMEOUT bash -c restart_thread_1 2> /dev/null & -timeout $TIMEOUT bash -c restart_thread_2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT rename_thread_1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT rename_thread_2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT restart_thread_1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT restart_thread_2 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index ae592740551..b21f88ea215 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-fasttest +# Tag no-fasttest: 45 seconds running CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference b/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference index 0bf24c68939..dc0207a7b9f 100644 --- a/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference +++ b/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference @@ -1,3 +1,3 @@ [1] test query -[0,0] [0,0] [2,1] test query +[1,2] [3,4] [2,1] test query [] [] [] [] diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh index 50e6f91b49b..175df7fd9b9 100755 --- a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -13,31 +13,30 @@ $CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01150" $CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t1 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" $CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t2 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" -function thread_detach_attach { - while true; do - $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 219' -e '(query: ' - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 82' -e '(query: ' - sleep 0.0$RANDOM - done +function thread_detach_attach() +{ + $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 219' -e '(query: ' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 82' -e '(query: ' + sleep 0.0$RANDOM } -function thread_rename { - while true; do - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' - sleep 0.0$RANDOM - done +function thread_rename() +{ + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM } export -f thread_detach_attach export -f thread_rename -timeout 20 bash -c "thread_detach_attach" & -timeout 20 bash -c 'thread_rename' & +clickhouse_client_loop_timeout 20 thread_detach_attach & +clickhouse_client_loop_timeout 20 thread_rename & + wait sleep 1 diff --git a/tests/queries/0_stateless/01154_move_partition_long.sh b/tests/queries/0_stateless/01154_move_partition_long.sh index 6b0b0773cb6..4a0ed7808da 100755 --- a/tests/queries/0_stateless/01154_move_partition_long.sh +++ b/tests/queries/0_stateless/01154_move_partition_long.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash -# Tags: long, no-parallel +# Tags: long, no-parallel, no-s3-storage +# FIXME: s3 storage should work OK, it +# reproduces bug which exists not only in S3 version. CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -24,99 +26,85 @@ wait #function create_drop_thread() #{ -# while true; do -# REPLICA=$(($RANDOM % 16)) -# $CLICKHOUSE_CLIENT -q "DROP TABLE src_$REPLICA;" -# arr=("$@") -# engine=${arr[$RANDOM % ${#arr[@]}]} -# $CLICKHOUSE_CLIENT -q "CREATE TABLE src_$REPLICA (p UInt64, k UInt64, v UInt64) ENGINE=$engine PARTITION BY p % 10 ORDER BY k" -# sleep 0.$RANDOM; -# done +# REPLICA=$(($RANDOM % 16)) +# $CLICKHOUSE_CLIENT -q "DROP TABLE src_$REPLICA;" +# arr=("$@") +# engine=${arr[$RANDOM % ${#arr[@]}]} +# $CLICKHOUSE_CLIENT -q "CREATE TABLE src_$REPLICA (p UInt64, k UInt64, v UInt64) ENGINE=$engine PARTITION BY p % 10 ORDER BY k" +# sleep 0.$RANDOM #} function insert_thread() { - while true; do - REPLICA=$(($RANDOM % 16)) - LIMIT=$(($RANDOM % 100)) - $CLICKHOUSE_CLIENT -q "INSERT INTO $1_$REPLICA SELECT * FROM generateRandom('p UInt64, k UInt64, v UInt64') LIMIT $LIMIT" 2>/dev/null - done + REPLICA=$(($RANDOM % 16)) + LIMIT=$(($RANDOM % 100)) + $CLICKHOUSE_CLIENT -q "INSERT INTO $1_$REPLICA SELECT * FROM generateRandom('p UInt64, k UInt64, v UInt64') LIMIT $LIMIT" 2>/dev/null } function move_partition_src_dst_thread() { - while true; do - FROM_REPLICA=$(($RANDOM % 16)) - TO_REPLICA=$(($RANDOM % 16)) - PARTITION=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$FROM_REPLICA MOVE PARTITION $PARTITION TO TABLE dst_$TO_REPLICA" 2>/dev/null - sleep 0.$RANDOM; - done + FROM_REPLICA=$(($RANDOM % 16)) + TO_REPLICA=$(($RANDOM % 16)) + PARTITION=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$FROM_REPLICA MOVE PARTITION $PARTITION TO TABLE dst_$TO_REPLICA" 2>/dev/null + sleep 0.$RANDOM } function replace_partition_src_src_thread() { - while true; do - FROM_REPLICA=$(($RANDOM % 16)) - TO_REPLICA=$(($RANDOM % 16)) - PARTITION=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$TO_REPLICA REPLACE PARTITION $PARTITION FROM src_$FROM_REPLICA" 2>/dev/null - sleep 0.$RANDOM; - done + FROM_REPLICA=$(($RANDOM % 16)) + TO_REPLICA=$(($RANDOM % 16)) + PARTITION=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$TO_REPLICA REPLACE PARTITION $PARTITION FROM src_$FROM_REPLICA" 2>/dev/null + sleep 0.$RANDOM } function drop_partition_thread() { - while true; do - REPLICA=$(($RANDOM % 16)) - PARTITION=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PARTITION $PARTITION" 2>/dev/null - sleep 0.$RANDOM; - done + REPLICA=$(($RANDOM % 16)) + PARTITION=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PARTITION $PARTITION" 2>/dev/null + sleep 0.$RANDOM } function optimize_thread() { - while true; do - REPLICA=$(($RANDOM % 16)) - TABLE="src" - if (( RANDOM % 2 )); then - TABLE="dst" - fi - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE ${TABLE}_$REPLICA" 2>/dev/null - sleep 0.$RANDOM; - done + REPLICA=$(($RANDOM % 16)) + TABLE="src" + if (( RANDOM % 2 )); then + TABLE="dst" + fi + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE ${TABLE}_$REPLICA" 2>/dev/null + sleep 0.$RANDOM } function drop_part_thread() { - while true; do - REPLICA=$(($RANDOM % 16)) - part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1") - $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null - sleep 0.$RANDOM; - done + REPLICA=$(($RANDOM % 16)) + part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1") + $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null + sleep 0.$RANDOM } #export -f create_drop_thread; -export -f insert_thread; -export -f move_partition_src_dst_thread; -export -f replace_partition_src_src_thread; -export -f drop_partition_thread; -export -f optimize_thread; -export -f drop_part_thread; +export -f insert_thread +export -f move_partition_src_dst_thread +export -f replace_partition_src_src_thread +export -f drop_partition_thread +export -f optimize_thread +export -f drop_part_thread TIMEOUT=60 -#timeout $TIMEOUT bash -c "create_drop_thread ${engines[@]}" & -timeout $TIMEOUT bash -c 'insert_thread src' & -timeout $TIMEOUT bash -c 'insert_thread src' & -timeout $TIMEOUT bash -c 'insert_thread dst' & -timeout $TIMEOUT bash -c move_partition_src_dst_thread & -timeout $TIMEOUT bash -c replace_partition_src_src_thread & -timeout $TIMEOUT bash -c drop_partition_thread & -timeout $TIMEOUT bash -c optimize_thread & -timeout $TIMEOUT bash -c drop_part_thread & +#clickhouse_client_loop_timeout $TIMEOUT "create_drop_thread ${engines[@]}" & +clickhouse_client_loop_timeout $TIMEOUT insert_thread src & +clickhouse_client_loop_timeout $TIMEOUT insert_thread src & +clickhouse_client_loop_timeout $TIMEOUT insert_thread dst & +clickhouse_client_loop_timeout $TIMEOUT move_partition_src_dst_thread & +clickhouse_client_loop_timeout $TIMEOUT replace_partition_src_src_thread & +clickhouse_client_loop_timeout $TIMEOUT drop_partition_thread & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread & +clickhouse_client_loop_timeout $TIMEOUT drop_part_thread & wait check_replication_consistency "dst_" "count(), sum(p), sum(k), sum(v)" diff --git a/tests/queries/0_stateless/01155_rename_move_materialized_view.sql b/tests/queries/0_stateless/01155_rename_move_materialized_view.sql index 0b672cbddbf..e0546ec8117 100644 --- a/tests/queries/0_stateless/01155_rename_move_materialized_view.sql +++ b/tests/queries/0_stateless/01155_rename_move_materialized_view.sql @@ -1,5 +1,7 @@ -- Tags: no-parallel +SET prefer_localhost_replica = 1; + DROP DATABASE IF EXISTS test_01155_ordinary; DROP DATABASE IF EXISTS test_01155_atomic; diff --git a/tests/queries/0_stateless/01160_table_dependencies.sh b/tests/queries/0_stateless/01160_table_dependencies.sh index a0a3f05c6a9..4cfad526f24 100755 --- a/tests/queries/0_stateless/01160_table_dependencies.sh +++ b/tests/queries/0_stateless/01160_table_dependencies.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-backward-compatibility-check:21.12.1.8761 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01162_strange_mutations.sh b/tests/queries/0_stateless/01162_strange_mutations.sh index fecb1b8d8c0..c759d113f84 100755 --- a/tests/queries/0_stateless/01162_strange_mutations.sh +++ b/tests/queries/0_stateless/01162_strange_mutations.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT -q "CREATE OR REPLACE VIEW t1 AS SELECT number * 10 AS id, nu for engine in "${engines[@]}" do $CLICKHOUSE_CLIENT -q "drop table if exists t" - $CLICKHOUSE_CLIENT -q "create table t (n int) engine=$engine" + $CLICKHOUSE_CLIENT -q "create table t (n int) engine=$engine" 2>&1| grep -Ev "Removing leftovers from table|removed by another replica" $CLICKHOUSE_CLIENT -q "select engine from system.tables where database=currentDatabase() and name='t'" $CLICKHOUSE_CLIENT -q "insert into t values (1)" $CLICKHOUSE_CLIENT -q "insert into t values (2)" @@ -25,7 +25,7 @@ do $CLICKHOUSE_CLIENT -q "drop table t" $CLICKHOUSE_CLIENT -q "drop table if exists test" - $CLICKHOUSE_CLIENT -q "CREATE TABLE test ENGINE=$engine AS SELECT number + 100 AS n, 0 AS test FROM numbers(50)" + $CLICKHOUSE_CLIENT -q "CREATE TABLE test ENGINE=$engine AS SELECT number + 100 AS n, 0 AS test FROM numbers(50)" 2>&1| grep -Ev "Removing leftovers from table|removed by another replica" $CLICKHOUSE_CLIENT -q "select count(), sum(n), sum(test) from test" if [[ $engine == *"ReplicatedMergeTree"* ]]; then $CLICKHOUSE_CLIENT -q "ALTER TABLE test diff --git a/tests/queries/0_stateless/01164_detach_attach_partition_race.sh b/tests/queries/0_stateless/01164_detach_attach_partition_race.sh index a7c87351ff4..27658650b91 100755 --- a/tests/queries/0_stateless/01164_detach_attach_partition_race.sh +++ b/tests/queries/0_stateless/01164_detach_attach_partition_race.sh @@ -12,36 +12,30 @@ $CLICKHOUSE_CLIENT -q "insert into mt values (3)" function thread_insert() { - while true; do - $CLICKHOUSE_CLIENT -q "insert into mt values (rand())"; - done + $CLICKHOUSE_CLIENT -q "insert into mt values (rand())"; } function thread_detach_attach() { - while true; do - $CLICKHOUSE_CLIENT -q "alter table mt detach partition id 'all'"; - $CLICKHOUSE_CLIENT -q "alter table mt attach partition id 'all'"; - done + $CLICKHOUSE_CLIENT -q "alter table mt detach partition id 'all'"; + $CLICKHOUSE_CLIENT -q "alter table mt attach partition id 'all'"; } function thread_drop_detached() { - while true; do - $CLICKHOUSE_CLIENT --allow_drop_detached -q "alter table mt drop detached partition id 'all'"; - done + $CLICKHOUSE_CLIENT --allow_drop_detached -q "alter table mt drop detached partition id 'all'"; } -export -f thread_insert; -export -f thread_detach_attach; -export -f thread_drop_detached; +export -f thread_insert +export -f thread_detach_attach +export -f thread_drop_detached TIMEOUT=10 -timeout $TIMEOUT bash -c thread_insert & -timeout $TIMEOUT bash -c thread_detach_attach 2> /dev/null & -timeout $TIMEOUT bash -c thread_detach_attach 2> /dev/null & -timeout $TIMEOUT bash -c thread_drop_detached 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread_insert & +clickhouse_client_loop_timeout $TIMEOUT thread_detach_attach 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread_detach_attach 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT thread_drop_detached 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh index e4a23055ae6..e632841bd01 100755 --- a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh @@ -36,7 +36,7 @@ function run_until_out_contains() RAND_COMMENT="01175_DDL_$RANDOM" LOG_COMMENT="${CLICKHOUSE_LOG_COMMENT}_$RAND_COMMENT" -CLICKHOUSE_CLIENT_WITH_SETTINGS=${CLICKHOUSE_CLIENT/--log_comment=\'${CLICKHOUSE_LOG_COMMENT}\'/--log_comment=\'${LOG_COMMENT}\'} +CLICKHOUSE_CLIENT_WITH_SETTINGS=${CLICKHOUSE_CLIENT/--log_comment ${CLICKHOUSE_LOG_COMMENT}/--log_comment ${LOG_COMMENT}} CLICKHOUSE_CLIENT_WITH_SETTINGS+=" --output_format_parallel_formatting=0 " CLICKHOUSE_CLIENT_WITH_SETTINGS+=" --distributed_ddl_entry_format_version=2 " diff --git a/tests/queries/0_stateless/01186_conversion_to_nullable.reference b/tests/queries/0_stateless/01186_conversion_to_nullable.reference index 7a690240eb5..dc77029ec3b 100644 --- a/tests/queries/0_stateless/01186_conversion_to_nullable.reference +++ b/tests/queries/0_stateless/01186_conversion_to_nullable.reference @@ -12,7 +12,7 @@ \N 1970-01-01 \N -1970-01-01 +2149-06-06 2020-12-24 01:02:03 \N 1970-01-01 03:00:00 diff --git a/tests/queries/0_stateless/01187_set_profile_as_setting.sh b/tests/queries/0_stateless/01187_set_profile_as_setting.sh index ec07f4d3687..dacb609d790 100755 --- a/tests/queries/0_stateless/01187_set_profile_as_setting.sh +++ b/tests/queries/0_stateless/01187_set_profile_as_setting.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-settings unset CLICKHOUSE_LOG_COMMENT diff --git a/tests/queries/0_stateless/01191_rename_dictionary.sql b/tests/queries/0_stateless/01191_rename_dictionary.sql index e9fed1dd6b2..ed9bc8af61b 100644 --- a/tests/queries/0_stateless/01191_rename_dictionary.sql +++ b/tests/queries/0_stateless/01191_rename_dictionary.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-backward-compatibility-check DROP DATABASE IF EXISTS test_01191; CREATE DATABASE test_01191 ENGINE=Atomic; diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index f15032c6929..1604de6004a 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest, no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index de9c2df7f08..ff4918c0810 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,4 +1,4 @@ --- Tags: no-debug, no-parallel, long +-- Tags: no-debug, no-parallel, long, no-s3-storage DROP TABLE IF EXISTS table_with_single_pk; diff --git a/tests/queries/0_stateless/01221_system_settings.sql b/tests/queries/0_stateless/01221_system_settings.sql index 226be55503d..fcffd6c45fe 100644 --- a/tests/queries/0_stateless/01221_system_settings.sql +++ b/tests/queries/0_stateless/01221_system_settings.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage select * from system.settings where name = 'send_timeout'; select * from system.merge_tree_settings order by length(description) limit 1; diff --git a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py index addff72ce66..67c79778736 100755 --- a/tests/queries/0_stateless/01246_insert_into_watch_live_view.py +++ b/tests/queries/0_stateless/01246_insert_into_watch_live_view.py @@ -7,75 +7,81 @@ import time import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1, client(name='client2>', log=log) as client2, client(name='client3>', log=log) as client3: +with client(name="client1>", log=log) as client1, client( + name="client2>", log=log +) as client2, client(name="client3>", log=log) as client3: client1.expect(prompt) client2.expect(prompt) client3.expect(prompt) - client1.send('SET allow_experimental_live_view = 1') + client1.send("SET allow_experimental_live_view = 1") client1.expect(prompt) - client3.send('SET allow_experimental_live_view = 1') + client3.send("SET allow_experimental_live_view = 1") client3.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv') + client1.send("DROP TABLE IF EXISTS test.lv") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.lv_sums') + client1.send("DROP TABLE IF EXISTS test.lv_sums") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.mt') + client1.send("DROP TABLE IF EXISTS test.mt") client1.expect(prompt) - client1.send('DROP TABLE IF EXISTS test.sums') + client1.send("DROP TABLE IF EXISTS test.sums") client1.expect(prompt) - client1.send('CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()') + client1.send("CREATE TABLE test.mt (a Int32) Engine=MergeTree order by tuple()") client1.expect(prompt) - client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) AS s FROM test.mt') + client1.send("CREATE LIVE VIEW test.lv AS SELECT sum(a) AS s FROM test.mt") client1.expect(prompt) - client1.send('CREATE TABLE test.sums (s Int32, version Int32) Engine=MergeTree ORDER BY tuple()') + client1.send( + "CREATE TABLE test.sums (s Int32, version Int32) Engine=MergeTree ORDER BY tuple()" + ) client1.expect(prompt) - client3.send('CREATE LIVE VIEW test.lv_sums AS SELECT * FROM test.sums ORDER BY version') + client3.send( + "CREATE LIVE VIEW test.lv_sums AS SELECT * FROM test.sums ORDER BY version" + ) client3.expect(prompt) client3.send("WATCH test.lv_sums FORMAT CSVWithNames") - client1.send('INSERT INTO test.sums WATCH test.lv') - client1.expect(r'INSERT INTO') + client1.send("INSERT INTO test.sums WATCH test.lv") + client1.expect(r"INSERT INTO") - client3.expect('0,1.*\r\n') + client3.expect("0,1.*\r\n") - client2.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client2.send("INSERT INTO test.mt VALUES (1),(2),(3)") client2.expect(prompt) - client3.expect('6,2.*\r\n') + client3.expect("6,2.*\r\n") - client2.send('INSERT INTO test.mt VALUES (4),(5),(6)') + client2.send("INSERT INTO test.mt VALUES (4),(5),(6)") client2.expect(prompt) - client3.expect('21,3.*\r\n') + client3.expect("21,3.*\r\n") # send Ctrl-C - client3.send('\x03', eol='') - match = client3.expect('(%s)|([#\$] )' % prompt) + client3.send("\x03", eol="") + match = client3.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client3.send(client3.command) client3.expect(prompt) # send Ctrl-C - client1.send('\x03', eol='') - match = client1.expect('(%s)|([#\$] )' % prompt) + client1.send("\x03", eol="") + match = client1.expect("(%s)|([#\$] )" % prompt) if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client2.send('DROP TABLE test.lv') + client2.send("DROP TABLE test.lv") client2.expect(prompt) - client2.send('DROP TABLE test.lv_sums') + client2.send("DROP TABLE test.lv_sums") client2.expect(prompt) - client2.send('DROP TABLE test.sums') + client2.send("DROP TABLE test.sums") client2.expect(prompt) - client2.send('DROP TABLE test.mt') + client2.send("DROP TABLE test.mt") client2.expect(prompt) diff --git a/tests/queries/0_stateless/01249_flush_interactive.sh b/tests/queries/0_stateless/01249_flush_interactive.sh index 89167002ed5..2ab85e2fb6c 100755 --- a/tests/queries/0_stateless/01249_flush_interactive.sh +++ b/tests/queries/0_stateless/01249_flush_interactive.sh @@ -28,3 +28,5 @@ while true; do [[ $(test) == $(echo -ne "0\n1\n2\n3\n4\n---\n0\n1\n2\n3\n4\n---\n") ]] && break sleep 1 done + +clickhouse_test_wait_queries 60 diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql index 73b87817bb3..242a253e67c 100644 --- a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql +++ b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func_with_alias.sql @@ -1,4 +1,5 @@ set optimize_arithmetic_operations_in_aggregate_functions = 1; +SET convert_query_to_cnf = 0; explain syntax select min((n as a) + (1 as b)) c from (select number n from numbers(10)) where a > 0 and b > 0 having c > 0; select min((n as a) + (1 as b)) c from (select number n from numbers(10)) where a > 0 and b > 0 having c > 0; diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index b2b02b24cb0..06bd6ab04e4 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -68,9 +68,9 @@ CREATE ROLE [] GLOBAL ACCESS MANAGEMENT ALTER ROLE [] GLOBAL ACCESS MANAGEMENT DROP ROLE [] GLOBAL ACCESS MANAGEMENT ROLE ADMIN [] GLOBAL ACCESS MANAGEMENT -CREATE ROW POLICY ['CREATE POLICY'] GLOBAL ACCESS MANAGEMENT -ALTER ROW POLICY ['ALTER POLICY'] GLOBAL ACCESS MANAGEMENT -DROP ROW POLICY ['DROP POLICY'] GLOBAL ACCESS MANAGEMENT +CREATE ROW POLICY ['CREATE POLICY'] TABLE ACCESS MANAGEMENT +ALTER ROW POLICY ['ALTER POLICY'] TABLE ACCESS MANAGEMENT +DROP ROW POLICY ['DROP POLICY'] TABLE ACCESS MANAGEMENT CREATE QUOTA [] GLOBAL ACCESS MANAGEMENT ALTER QUOTA [] GLOBAL ACCESS MANAGEMENT DROP QUOTA [] GLOBAL ACCESS MANAGEMENT @@ -79,7 +79,7 @@ ALTER SETTINGS PROFILE ['ALTER PROFILE'] GLOBAL ACCESS MANAGEMENT DROP SETTINGS PROFILE ['DROP PROFILE'] GLOBAL ACCESS MANAGEMENT SHOW USERS ['SHOW CREATE USER'] GLOBAL SHOW ACCESS SHOW ROLES ['SHOW CREATE ROLE'] GLOBAL SHOW ACCESS -SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY'] GLOBAL SHOW ACCESS +SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY'] TABLE SHOW ACCESS SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS SHOW ACCESS [] \N ACCESS MANAGEMENT diff --git a/tests/queries/0_stateless/01273_arrow.reference b/tests/queries/0_stateless/01273_arrow.reference index 89eca82f8ef..b4d011754dc 100644 --- a/tests/queries/0_stateless/01273_arrow.reference +++ b/tests/queries/0_stateless/01273_arrow.reference @@ -32,29 +32,29 @@ 991 990 original: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789012 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789012 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789012 converted: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789012 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789012 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789012 diff: dest: -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 -80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 00:00:00 2005-02-03 04:05:06.789012 +80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 2007-02-03 04:05:06.789012 min: --128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 --108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 -79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 -127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 +-128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 2003-02-03 04:05:06.789012 +-108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 2002-02-03 04:05:06.789012 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 2005-02-03 04:05:06.789012 +127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 2004-02-03 04:05:06.789012 max: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 -80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 2003-04-05 00:00:00 2003-02-03 04:05:06 2003-02-03 04:05:06.789012 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 2001-02-03 00:00:00 2002-02-03 04:05:06 2002-02-03 04:05:06.789012 +80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 2007-02-03 04:05:06.789012 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 2004-06-07 00:00:00 2004-02-03 04:05:06 2004-02-03 04:05:06.789012 dest from null: --128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 --108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 -127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 -\N \N \N \N \N \N \N \N \N \N \N \N \N \N +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 2003-02-03 04:05:06.789 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 2002-02-03 04:05:06.789 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 2004-02-03 04:05:06.789 +\N \N \N \N \N \N \N \N \N \N \N \N \N \N \N diff --git a/tests/queries/0_stateless/01273_arrow.sh b/tests/queries/0_stateless/01273_arrow.sh index 01a27ac12d1..90638b37c3e 100755 --- a/tests/queries/0_stateless/01273_arrow.sh +++ b/tests/queries/0_stateless/01273_arrow.sh @@ -40,20 +40,20 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types4" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(6)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime, datetime64 DateTime64(6)) ENGINE = Memory" # convert min type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date, datetime64 DateTime64(6)) ENGINE = Memory" # convert max type -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime, datetime64 DateTime64(6)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06', toDateTime64('2002-02-03 04:05:06.789012', 6))" # min -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06', toDateTime64('2003-02-03 04:05:06.789012', 6))" # max -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06', toDateTime64('2004-02-03 04:05:06.789012', 6))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" @@ -67,11 +67,11 @@ echo diff: diff "${CLICKHOUSE_TMP}"/arrow_all_types_1.dump "${CLICKHOUSE_TMP}"/arrow_all_types_2.dump ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06', toDateTime64('2005-02-03 04:05:06.789012', 6))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types3 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 FORMAT Arrow" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12')" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12', toDateTime64('2007-02-03 04:05:06.789012', 6))" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 FORMAT Arrow" @@ -86,9 +86,9 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types5" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types6" ${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime), datetime64 Nullable(DateTime64)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" > "${CLICKHOUSE_TMP}"/arrow_all_types_5.arrow ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference index a5987acafde..9021ae2bb1a 100644 --- a/tests/queries/0_stateless/01275_parallel_mv.reference +++ b/tests/queries/0_stateless/01275_parallel_mv.reference @@ -2,8 +2,8 @@ set parallel_view_processing=1; insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } system flush logs; -select length(thread_ids) from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '1'; -8 +select length(thread_ids) >= 8 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '1'; +1 select count() from testX; 10 select count() from testXA; @@ -15,8 +15,8 @@ select count() from testXC; set parallel_view_processing=0; insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } system flush logs; -select length(thread_ids) from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '0'; -5 +select length(thread_ids) >= 5 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '0'; +1 select count() from testX; 20 select count() from testXA; diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql b/tests/queries/0_stateless/01275_parallel_mv.sql index 32b43ce616f..27b8ef96e0b 100644 --- a/tests/queries/0_stateless/01275_parallel_mv.sql +++ b/tests/queries/0_stateless/01275_parallel_mv.sql @@ -1,3 +1,5 @@ +set max_threads = 0; + drop table if exists testX; drop table if exists testXA; drop table if exists testXB; @@ -13,7 +15,7 @@ create materialized view testXC engine=MergeTree order by tuple() as select slee set parallel_view_processing=1; insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } system flush logs; -select length(thread_ids) from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '1'; +select length(thread_ids) >= 8 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '1'; select count() from testX; select count() from testXA; @@ -23,7 +25,7 @@ select count() from testXC; set parallel_view_processing=0; insert into testX select number from numbers(10) settings log_queries=1; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } system flush logs; -select length(thread_ids) from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '0'; +select length(thread_ids) >= 5 from system.query_log where current_database = currentDatabase() and type != 'QueryStart' and query like '%insert into testX %' and Settings['parallel_view_processing'] = '0'; select count() from testX; select count() from testXA; diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh index bf201187f45..c9c01455e31 100755 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-replicated-database, no-parallel, no-fasttest +# Tags: no-replicated-database, no-parallel, no-fasttest, no-tsan, no-asan # Tag no-fasttest: max_memory_usage_for_user can interfere another queries running concurrently # Regression for MemoryTracker that had been incorrectly accounted diff --git a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql index 61db4376c91..59d8605ba1c 100644 --- a/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql +++ b/tests/queries/0_stateless/01283_max_threads_simple_query_optimization.sql @@ -1,5 +1,7 @@ DROP TABLE IF EXISTS data_01283; +set remote_filesystem_read_method='read'; + CREATE TABLE data_01283 engine=MergeTree() ORDER BY key PARTITION BY key diff --git a/tests/queries/0_stateless/01287_max_execution_speed.sql b/tests/queries/0_stateless/01287_max_execution_speed.sql index 7e8f6681c84..7dbeab2d635 100644 --- a/tests/queries/0_stateless/01287_max_execution_speed.sql +++ b/tests/queries/0_stateless/01287_max_execution_speed.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0.1; SELECT count() FROM system.numbers; -- { serverError 160 } SELECT 'Ok (1)'; diff --git a/tests/queries/0_stateless/01293_show_settings.sql b/tests/queries/0_stateless/01293_show_settings.sql index 08f00ed201c..3e55ffb58d7 100644 --- a/tests/queries/0_stateless/01293_show_settings.sql +++ b/tests/queries/0_stateless/01293_show_settings.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-settings + show settings like 'send_timeout'; SHOW SETTINGS ILIKE '%CONNECT_timeout%'; SHOW CHANGED SETTINGS ILIKE '%MEMORY%'; diff --git a/tests/queries/0_stateless/01293_system_distribution_queue.sql b/tests/queries/0_stateless/01293_system_distribution_queue.sql index 34158fb081c..9997f18f61d 100644 --- a/tests/queries/0_stateless/01293_system_distribution_queue.sql +++ b/tests/queries/0_stateless/01293_system_distribution_queue.sql @@ -1,4 +1,5 @@ -- Tags: no-parallel +set prefer_localhost_replica = 1; drop table if exists null_01293; drop table if exists dist_01293; diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh index 3c11dc5f772..1235e013ff7 100755 --- a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh +++ b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh @@ -10,94 +10,67 @@ export CURR_DATABASE="test_lazy_01294_concurrent_${CLICKHOUSE_DATABASE}" function recreate_lazy_func1() { - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; + $CLICKHOUSE_CLIENT -nm -q " + DETACH TABLE $CURR_DATABASE.log; + ATTACH TABLE $CURR_DATABASE.log; "; - - while true; do - $CLICKHOUSE_CLIENT -q " - DETACH TABLE $CURR_DATABASE.log; - "; - - $CLICKHOUSE_CLIENT -q " - ATTACH TABLE $CURR_DATABASE.log; - "; - done } function recreate_lazy_func2() { - while true; do - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; - "; - - $CLICKHOUSE_CLIENT -q " - DROP TABLE $CURR_DATABASE.tlog; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; + DROP TABLE $CURR_DATABASE.tlog; + " } function recreate_lazy_func3() { - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; - "; - - while true; do - $CLICKHOUSE_CLIENT -q " - ATTACH TABLE $CURR_DATABASE.slog; - "; - - $CLICKHOUSE_CLIENT -q " - DETACH TABLE $CURR_DATABASE.slog; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + ATTACH TABLE $CURR_DATABASE.slog; + DETACH TABLE $CURR_DATABASE.slog; + " } function recreate_lazy_func4() { - while true; do - $CLICKHOUSE_CLIENT -q " - CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; - "; - - $CLICKHOUSE_CLIENT -q " - DROP TABLE $CURR_DATABASE.tlog2; - "; - done + $CLICKHOUSE_CLIENT -nm -q " + CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; + DROP TABLE $CURR_DATABASE.tlog2; + " } function test_func() { - while true; do - for table in log tlog slog tlog2; do - $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES $CURR_DATABASE.$table" >& /dev/null - done + for table in log tlog slog tlog2; do + $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES $CURR_DATABASE.$table" >& /dev/null done } -export -f recreate_lazy_func1; -export -f recreate_lazy_func2; -export -f recreate_lazy_func3; -export -f recreate_lazy_func4; -export -f test_func; +export -f recreate_lazy_func1 +export -f recreate_lazy_func2 +export -f recreate_lazy_func3 +export -f recreate_lazy_func4 +export -f test_func ${CLICKHOUSE_CLIENT} -n -q " DROP DATABASE IF EXISTS $CURR_DATABASE; CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); + + CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; + CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; " TIMEOUT=30 -timeout $TIMEOUT bash -c recreate_lazy_func1 2> /dev/null & -timeout $TIMEOUT bash -c recreate_lazy_func2 2> /dev/null & -timeout $TIMEOUT bash -c recreate_lazy_func3 2> /dev/null & -timeout $TIMEOUT bash -c recreate_lazy_func4 2> /dev/null & -timeout $TIMEOUT bash -c test_func 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func1 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func2 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func3 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func4 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT test_func 2> /dev/null & wait sleep 1 diff --git a/tests/queries/0_stateless/01300_group_by_other_keys.sql b/tests/queries/0_stateless/01300_group_by_other_keys.sql index 22cff012e71..0e37ef55a6a 100644 --- a/tests/queries/0_stateless/01300_group_by_other_keys.sql +++ b/tests/queries/0_stateless/01300_group_by_other_keys.sql @@ -1,3 +1,5 @@ +set max_block_size = 65505; + set optimize_group_by_function_keys = 1; SELECT round(max(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY number % 2, number % 3, (number % 2 + number % 3) % 2 ORDER BY k; diff --git a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh index 010c66b9bb1..ed7e5937bd0 100755 --- a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh +++ b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh @@ -5,15 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -function test() -{ - for _ in {1..1000}; do - $CLICKHOUSE_CLIENT --max_memory_usage 1G <<< "SELECT uniqExactState(number) FROM system.numbers_mt GROUP BY number % 10"; - done -} - -export -f test; - # If the memory leak exists, it will lead to OOM fairly quickly. -timeout 30 bash -c test 2>&1 | grep -o -F 'Memory limit (for query) exceeded' | uniq +clickhouse_client_loop_timeout 30 $CLICKHOUSE_CLIENT --max_memory_usage 1G -q 'SELECT uniqExactState(number) FROM system.numbers_mt GROUP BY number % 10' 2>&1 | grep -o -F 'Memory limit (for query) exceeded' | uniq echo 'Ok' diff --git a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh index 7e10ab475d4..52b635fe5df 100755 --- a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh +++ b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh @@ -7,12 +7,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function test() { - for _ in {1..250}; do - $CLICKHOUSE_CLIENT --query "SELECT groupArrayIfState(('Hello, world' AS s) || s || s || s || s || s || s || s || s || s, NOT throwIf(number > 10000000, 'Ok')) FROM system.numbers_mt GROUP BY number % 10"; - done + $CLICKHOUSE_CLIENT --query "SELECT groupArrayIfState(('Hello, world' AS s) || s || s || s || s || s || s || s || s || s, NOT throwIf(number > 10000000, 'Ok')) FROM system.numbers_mt GROUP BY number % 10"; } -export -f test; +export -f test # If the memory leak exists, it will lead to OOM fairly quickly. -timeout 30 bash -c test 2>&1 | grep -o -F 'Ok' | uniq +clickhouse_client_loop_timeout 30 test 2>&1 | grep -o -F 'Ok' | uniq diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 1d2d4516b9c..9b4159fa002 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -9,20 +9,17 @@ set -e function thread() { - while true; do - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | - grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|^\(query: ' - done + $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|^\(query: ' } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f thread; +export -f thread TIMEOUT=10 -timeout $TIMEOUT bash -c 'thread 1' & -timeout $TIMEOUT bash -c 'thread 2' & +clickhouse_client_loop_timeout $TIMEOUT thread 1 & +clickhouse_client_loop_timeout $TIMEOUT thread 2 & wait diff --git a/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql index cd41bb227eb..81bd2ad97a9 100644 --- a/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql +++ b/tests/queries/0_stateless/01308_row_policy_and_trivial_count_query.sql @@ -1,3 +1,5 @@ +SET optimize_move_to_prewhere = 1; + DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt8) ENGINE = MergeTree ORDER BY x; diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference index ed7daeb3609..d1e2cba5663 100644 --- a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference @@ -1 +1 @@ -CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' +CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index 758ec4825e0..d538cafd79d 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -12,19 +12,19 @@ $CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Diff function thread1() { - while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done + $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;" } function thread2() { - while true; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA test_01320.r" 2>/dev/null; done + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA test_01320.r" 2>/dev/null } export -f thread1 export -f thread2 -timeout 10 bash -c thread1 & -timeout 10 bash -c thread2 & +clickhouse_client_loop_timeout 10 thread1 & +clickhouse_client_loop_timeout 10 thread2 & wait diff --git a/tests/queries/0_stateless/01323_too_many_threads_bug.sql b/tests/queries/0_stateless/01323_too_many_threads_bug.sql index 6033fe66cd3..5dbb5aca2ec 100644 --- a/tests/queries/0_stateless/01323_too_many_threads_bug.sql +++ b/tests/queries/0_stateless/01323_too_many_threads_bug.sql @@ -1,5 +1,7 @@ drop table if exists table_01323_many_parts; +set remote_filesystem_read_method='read'; + create table table_01323_many_parts (x UInt64) engine = MergeTree order by x partition by x % 100; set max_partitions_per_insert_block = 100; insert into table_01323_many_parts select number from numbers(100000); diff --git a/tests/queries/0_stateless/01339_client_unrecognized_option.sh b/tests/queries/0_stateless/01339_client_unrecognized_option.sh index 00c153ec915..9f827ccb13e 100755 --- a/tests/queries/0_stateless/01339_client_unrecognized_option.sh +++ b/tests/queries/0_stateless/01339_client_unrecognized_option.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-random-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql index c8c49ea8c58..614629351ef 100644 --- a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql +++ b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS test_01343; CREATE TABLE test_01343 (x String) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01343 VALUES ('Hello, world'); diff --git a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql index 7805ecea392..2e5ec563641 100644 --- a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql +++ b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS test_01344; CREATE TABLE test_01344 (x String, INDEX idx (x) TYPE set(10) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01344 VALUES ('Hello, world'); diff --git a/tests/queries/0_stateless/01386_negative_float_constant_key_condition.sql b/tests/queries/0_stateless/01386_negative_float_constant_key_condition.sql index c2191d6ab96..b45b9c84b18 100644 --- a/tests/queries/0_stateless/01386_negative_float_constant_key_condition.sql +++ b/tests/queries/0_stateless/01386_negative_float_constant_key_condition.sql @@ -1,3 +1,5 @@ +SET convert_query_to_cnf = 0; + DROP TABLE IF EXISTS t0; CREATE TABLE t0 diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.sh b/tests/queries/0_stateless/01412_cache_dictionary_race.sh index 165a461193d..7d189518a58 100755 --- a/tests/queries/0_stateless/01412_cache_dictionary_race.sh +++ b/tests/queries/0_stateless/01412_cache_dictionary_race.sh @@ -26,45 +26,41 @@ LAYOUT(CACHE(SIZE_IN_CELLS 3)); function dict_get_thread() { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null - done + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null } function drop_create_table_thread() { - while true; do - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE ordinary_db.table_for_dict_real ( - key_column UInt64, - second_column UInt8, - third_column String - ) - ENGINE MergeTree() ORDER BY tuple(); - INSERT INTO ordinary_db.table_for_dict_real SELECT number, number, toString(number) from numbers(2); - CREATE VIEW ordinary_db.view_for_dict AS SELECT key_column, second_column, third_column from ordinary_db.table_for_dict_real WHERE sleepEachRow(1) == 0; + $CLICKHOUSE_CLIENT -n --query "CREATE TABLE ordinary_db.table_for_dict_real ( + key_column UInt64, + second_column UInt8, + third_column String + ) + ENGINE MergeTree() ORDER BY tuple(); + INSERT INTO ordinary_db.table_for_dict_real SELECT number, number, toString(number) from numbers(2); + CREATE VIEW ordinary_db.view_for_dict AS SELECT key_column, second_column, third_column from ordinary_db.table_for_dict_real WHERE sleepEachRow(1) == 0; " - sleep 10 + sleep 10 - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ordinary_db.table_for_dict_real" - sleep 10 - done + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ordinary_db.table_for_dict_real" + sleep 10 } -export -f dict_get_thread; -export -f drop_create_table_thread; +export -f dict_get_thread +export -f drop_create_table_thread TIMEOUT=30 -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c drop_create_table_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT bash -c drop_create_table_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01415_overlimiting_threads_for_repica_bug.sql b/tests/queries/0_stateless/01415_overlimiting_threads_for_repica_bug.sql index 306d94387a4..6b5c2ac8ffd 100644 --- a/tests/queries/0_stateless/01415_overlimiting_threads_for_repica_bug.sql +++ b/tests/queries/0_stateless/01415_overlimiting_threads_for_repica_bug.sql @@ -1,5 +1,6 @@ set log_queries = 1; set max_threads = 16; +set prefer_localhost_replica = 1; select sum(number) from remote('127.0.0.{1|2}', numbers_mt(1000000)) group by number % 2 order by number % 2; diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.sql b/tests/queries/0_stateless/01428_nullable_asof_join.sql index c812e6ecfab..30e5c51eb1c 100644 --- a/tests/queries/0_stateless/01428_nullable_asof_join.sql +++ b/tests/queries/0_stateless/01428_nullable_asof_join.sql @@ -18,13 +18,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } select 'left asof on'; @@ -44,13 +44,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b -ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ON a.dt >= b.dt AND a.pk = b.pk +ORDER BY a.dt; -- { serverError 48 } select 'asof using'; @@ -70,13 +70,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b USING(pk, dt) -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } select 'asof on'; @@ -96,10 +96,16 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b ON a.pk = b.pk AND a.dt >= b.dt -ORDER BY a.dt;-- { serverError 48 } +ORDER BY a.dt; -- { serverError 48 } + +SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt)) +FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a +ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b +ON a.dt >= b.dt AND a.pk = b.pk +ORDER BY a.dt; -- { serverError 48 } diff --git a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh index eb231e71525..51989685e05 100755 --- a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh +++ b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh @@ -11,18 +11,14 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function thread1() { - while true; do -# ${CLICKHOUSE_CLIENT} --query="SHOW TABLES FROM test_01444" - ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS test_01444" 2>&1| grep -F "Code: " | grep -Fv "Code: 219" - ${CLICKHOUSE_CLIENT} --query="CREATE DATABASE IF NOT EXISTS test_01444" - done + # ${CLICKHOUSE_CLIENT} --query="SHOW TABLES FROM test_01444" + ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS test_01444" 2>&1| grep -F "Code: " | grep -Fv "Code: 219" + ${CLICKHOUSE_CLIENT} --query="CREATE DATABASE IF NOT EXISTS test_01444" } function thread2() { - while true; do - ${CLICKHOUSE_CLIENT} --query="CREATE TABLE IF NOT EXISTS test_01444.t$RANDOM (x UInt8) ENGINE = MergeTree ORDER BY tuple()" 2>/dev/null - done + ${CLICKHOUSE_CLIENT} --query="CREATE TABLE IF NOT EXISTS test_01444.t$RANDOM (x UInt8) ENGINE = MergeTree ORDER BY tuple()" 2>/dev/null } export -f thread1 @@ -30,9 +26,9 @@ export -f thread2 TIMEOUT=10 -timeout $TIMEOUT bash -c thread1 & -timeout $TIMEOUT bash -c thread2 & -timeout $TIMEOUT bash -c thread2 & +clickhouse_client_loop_timeout $TIMEOUT thread1 & +clickhouse_client_loop_timeout $TIMEOUT thread2 & +clickhouse_client_loop_timeout $TIMEOUT thread2 & wait diff --git a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh index d83343b3cb3..27fa74f332c 100755 --- a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh +++ b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh @@ -10,14 +10,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mem" $CLICKHOUSE_CLIENT -q "CREATE TABLE mem (x UInt64) engine = Memory" -function f { - for _ in $(seq 1 300); do +function f() +{ $CLICKHOUSE_CLIENT -q "SELECT count() FROM (SELECT * FROM mem SETTINGS max_threads=2) FORMAT Null;" - done } -function g { - for _ in $(seq 1 100); do +function g() +{ $CLICKHOUSE_CLIENT -n -q " INSERT INTO mem SELECT number FROM numbers(1000000); INSERT INTO mem SELECT number FROM numbers(1000000); @@ -30,14 +29,13 @@ function g { INSERT INTO mem VALUES (1); TRUNCATE TABLE mem; " - done } -export -f f; -export -f g; +export -f f +export -f g -timeout 30 bash -c f > /dev/null & -timeout 30 bash -c g > /dev/null & +clickhouse_client_loop_timeout 30 f > /dev/null & +clickhouse_client_loop_timeout 30 g > /dev/null & wait $CLICKHOUSE_CLIENT -q "DROP TABLE mem" diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index 119642df395..e70506599ec 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -11,9 +11,11 @@ ===native=== {"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} {"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} -{"total spans":"2","unique spans":"2","unique non-zero parent spans":"2"} +{"total spans":"3","unique spans":"3","unique non-zero parent spans":"2"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} ===sampled=== diff --git a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql index d7c681dc615..bc677698d88 100644 --- a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql +++ b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql @@ -1,5 +1,7 @@ -- Tags: no-parallel +SET prefer_localhost_replica = 1; + DROP DATABASE IF EXISTS test_01457; CREATE DATABASE test_01457; diff --git a/tests/queries/0_stateless/01473_event_time_microseconds.sql b/tests/queries/0_stateless/01473_event_time_microseconds.sql index 9c6b392a8aa..932acf48cc4 100644 --- a/tests/queries/0_stateless/01473_event_time_microseconds.sql +++ b/tests/queries/0_stateless/01473_event_time_microseconds.sql @@ -1,4 +1,4 @@ --- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug +-- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-cpu-aarch64 -- This file contains tests for the event_time_microseconds field for various tables. -- Note: Only event_time_microseconds for asynchronous_metric_log table is tested via diff --git a/tests/queries/0_stateless/01475_read_subcolumns.sql b/tests/queries/0_stateless/01475_read_subcolumns.sql index 6e2c8d458ae..4724bec9eff 100644 --- a/tests/queries/0_stateless/01475_read_subcolumns.sql +++ b/tests/queries/0_stateless/01475_read_subcolumns.sql @@ -1,3 +1,7 @@ +-- Tags: no-s3-storage + +SET use_uncompressed_cache = 0; + SELECT '====array===='; DROP TABLE IF EXISTS t_arr; CREATE TABLE t_arr (a Array(UInt32)) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; diff --git a/tests/queries/bugs/01482_move_to_prewhere_and_cast.reference b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.reference similarity index 100% rename from tests/queries/bugs/01482_move_to_prewhere_and_cast.reference rename to tests/queries/0_stateless/01482_move_to_prewhere_and_cast.reference diff --git a/tests/queries/bugs/01482_move_to_prewhere_and_cast.sql b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql similarity index 91% rename from tests/queries/bugs/01482_move_to_prewhere_and_cast.sql rename to tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql index b81cf585b13..282363dcdd7 100644 --- a/tests/queries/bugs/01482_move_to_prewhere_and_cast.sql +++ b/tests/queries/0_stateless/01482_move_to_prewhere_and_cast.sql @@ -1,6 +1,3 @@ --- Tags: no-polymorphic-parts --- Tag no-polymorphic-parts: bug, shoud be fixed - DROP TABLE IF EXISTS APPLICATION; DROP TABLE IF EXISTS DATABASE_IO; @@ -22,9 +19,9 @@ ORDER BY Date; insert into table DATABASE_IO values ('AppA', 'BaseA', '2020-01-01 00:00:00', 1000); SELECT `APPLICATION`.`Name` AS `App`, - CAST(CAST(`DATABASE_IO`.`Date` AS DATE) AS DATE) AS `date` + CAST(CAST(`DATABASE_IO`.`Date` AS DATE) AS DATE) AS `date` FROM `DATABASE_IO` -INNER +INNER JOIN `APPLICATION` ON (`DATABASE_IO`.`Base` = `APPLICATION`.`Base`) WHERE ( CAST(CAST(`DATABASE_IO`.`Date` AS DATE) AS TIMESTAMP) >= toDateTime('2020-01-01 00:00:00') diff --git a/tests/queries/0_stateless/01492_format_readable_quantity.reference b/tests/queries/0_stateless/01492_format_readable_quantity.reference index e58a1954eee..247063b70a2 100644 --- a/tests/queries/0_stateless/01492_format_readable_quantity.reference +++ b/tests/queries/0_stateless/01492_format_readable_quantity.reference @@ -20,26 +20,26 @@ 178.48 million 178.48 million 178.48 million 485.17 million 485.17 million 485.17 million 1.32 billion 1.32 billion 1.32 billion -3.58 billion 3.58 billion -2.15 billion -9.74 billion 9.74 billion -2.15 billion -26.49 billion 26.49 billion -2.15 billion -72.00 billion 72.00 billion -2.15 billion -195.73 billion 195.73 billion -2.15 billion -532.05 billion 532.05 billion -2.15 billion -1.45 trillion 1.45 trillion -2.15 billion -3.93 trillion 3.93 trillion -2.15 billion -10.69 trillion 10.69 trillion -2.15 billion -29.05 trillion 29.05 trillion -2.15 billion -78.96 trillion 78.96 trillion -2.15 billion -214.64 trillion 214.64 trillion -2.15 billion -583.46 trillion 583.46 trillion -2.15 billion -1.59 quadrillion 1.59 quadrillion -2.15 billion -4.31 quadrillion 4.31 quadrillion -2.15 billion -11.72 quadrillion 11.72 quadrillion -2.15 billion -31.86 quadrillion 31.86 quadrillion -2.15 billion -86.59 quadrillion 86.59 quadrillion -2.15 billion -235.39 quadrillion 235.39 quadrillion -2.15 billion -639.84 quadrillion 639.84 quadrillion -2.15 billion -1739.27 quadrillion 1739.27 quadrillion -2.15 billion -4727.84 quadrillion 4727.84 quadrillion -2.15 billion -12851.60 quadrillion 12851.60 quadrillion -2.15 billion +3.58 billion 3.58 billion 2.15 billion +9.74 billion 9.74 billion 2.15 billion +26.49 billion 26.49 billion 2.15 billion +72.00 billion 72.00 billion 2.15 billion +195.73 billion 195.73 billion 2.15 billion +532.05 billion 532.05 billion 2.15 billion +1.45 trillion 1.45 trillion 2.15 billion +3.93 trillion 3.93 trillion 2.15 billion +10.69 trillion 10.69 trillion 2.15 billion +29.05 trillion 29.05 trillion 2.15 billion +78.96 trillion 78.96 trillion 2.15 billion +214.64 trillion 214.64 trillion 2.15 billion +583.46 trillion 583.46 trillion 2.15 billion +1.59 quadrillion 1.59 quadrillion 2.15 billion +4.31 quadrillion 4.31 quadrillion 2.15 billion +11.72 quadrillion 11.72 quadrillion 2.15 billion +31.86 quadrillion 31.86 quadrillion 2.15 billion +86.59 quadrillion 86.59 quadrillion 2.15 billion +235.39 quadrillion 235.39 quadrillion 2.15 billion +639.84 quadrillion 639.84 quadrillion 2.15 billion +1739.27 quadrillion 1739.27 quadrillion 2.15 billion +4727.84 quadrillion 4727.84 quadrillion 2.15 billion +12851.60 quadrillion 12851.60 quadrillion 2.15 billion diff --git a/tests/queries/0_stateless/01492_format_readable_quantity.sql b/tests/queries/0_stateless/01492_format_readable_quantity.sql index 3931cde49df..93aa570ccc8 100644 --- a/tests/queries/0_stateless/01492_format_readable_quantity.sql +++ b/tests/queries/0_stateless/01492_format_readable_quantity.sql @@ -1,4 +1,4 @@ -WITH round(exp(number), 6) AS x, toUInt64(x) AS y, toInt32(x) AS z +WITH round(exp(number), 6) AS x, toUInt64(x) AS y, toInt32(min2(x, 2147483647)) AS z SELECT formatReadableQuantity(x), formatReadableQuantity(y), formatReadableQuantity(z) FROM system.numbers LIMIT 45; diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference index 1eb57a24638..fe093e39a56 100644 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference @@ -1,4 +1,5 @@ +: Number of CPUs detected is not deterministic. Per-CPU arena disabled. 1 -: Number of CPUs is not deterministic -: Number of CPUs is not deterministic, but narenas is set. Hope you not what you are doing and you have set narenas to largest possible CPU ID. +: Number of CPUs detected is not deterministic. Per-CPU arena disabled. +100000000 1 diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh index 265ca4a6763..b3ea6eca3f4 100755 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -9,10 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ncpus="$(getconf _NPROCESSORS_ONLN)" -# In debug build the following settings enabled by default: -# - abort_conf -# - abort -# Disable them explicitly (will enable when required). +# Disable them explicitly, to avoid failing on "Number of CPUs is not deterministic". export MALLOC_CONF=abort_conf:false,abort:false # Regression for: @@ -20,17 +17,10 @@ export MALLOC_CONF=abort_conf:false,abort:false # $ taskset --cpu-list 8 ./clickhouse local -q 'select 1' # : ../contrib/jemalloc/src/jemalloc.c:321: Failed assertion: "ind <= narenas_total_get()" # Aborted (core dumped) -taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' -# just in case something more complicated -taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' +taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 -# this command should fail because percpu arena will be disabled, -# and with abort_conf:true it is not allowed -( - # subshell is required to suppress "Aborted" message from the shell. - MALLOC_CONF=abort_conf:true,abort:true - taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' -) |& grep -F 'Number of CPUs is not deterministic' +# just in case something more complicated +taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select count() from numbers_mt(100000000) settings max_threads=100' 2>&1 # this command should not fail because we specify narenas explicitly # (even with abort_conf:true) diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index ff22597c620..c2750ad31b2 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -5,14 +5,19 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -# In debug build abort_conf:true is set by default, disable it explicitly -# to avoid "Number of CPUs is not deterministic" error from jemalloc. -export MALLOC_CONF=abort_conf:false +# Avoid failures due to "Number of CPUs is not deterministic" error from jemalloc. +export MALLOC_CONF=abort_conf:false,abort:false # Regression for UAF in ThreadPool. # (Triggered under TSAN) for _ in {1..10}; do ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' # Binding to specific CPU is not required, but this makes the test more reliable. - taskset --cpu-list 0 ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' + taskset --cpu-list 0 ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' 2>&1 | { + # build with santiziers does not have jemalloc + # and for jemalloc we have separate test + # 01502_jemalloc_percpu_arena + grep -v ': Number of CPUs detected is not deterministic. Per-CPU arena disabled.' + } done +exit 0 diff --git a/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql index 8862037c82b..f9c227942ac 100644 --- a/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql +++ b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql @@ -5,12 +5,11 @@ CREATE TABLE buf_dest (timestamp DateTime) ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) ORDER BY (timestamp); -CREATE TABLE buf (timestamp DateTime) Engine = Buffer(currentDatabase(), buf_dest, 16, 0.1, 0.1, 2000000, 20000000, 100000000, 300000000);; +CREATE TABLE buf (timestamp DateTime) Engine = Buffer(currentDatabase(), buf_dest, 16, 86400, 86400, 2000000, 20000000, 100000000, 300000000);; INSERT INTO buf (timestamp) VALUES (toDateTime('2020-01-01 00:05:00')); ---- wait for buffer to flush -SELECT sleep(1) from numbers(1) settings max_block_size=1 format Null; +OPTIMIZE TABLE buf; ALTER TABLE buf_dest ADD COLUMN s String; ALTER TABLE buf ADD COLUMN s String; diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh index f3cdb70a074..55b6110918b 100755 --- a/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -# Tags: long, no-replicated-database +# Tags: long, no-replicated-database, no-s3-storage # Tag no-replicated-database: Fails due to additional replicas or shards +# Tag no-s3-storage: Merge assigned to replica 2, but replication queues are stopped for it + set -e diff --git a/tests/queries/0_stateless/01517_select_final_distributed.sql b/tests/queries/0_stateless/01517_select_final_distributed.sql index a3d1fcfc185..701828b0b38 100644 --- a/tests/queries/0_stateless/01517_select_final_distributed.sql +++ b/tests/queries/0_stateless/01517_select_final_distributed.sql @@ -1,5 +1,7 @@ -- Tags: distributed +SET allow_experimental_parallel_reading_from_replicas = 0; + DROP TABLE IF EXISTS test5346; CREATE TABLE test5346 (`Id` String, `Timestamp` DateTime, `updated` DateTime) diff --git a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql index 25c47c008bd..ca9f296b6bf 100644 --- a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql +++ b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS select_final; SET do_not_merge_across_partitions_select_final = 1; -CREATE TABLE select_final (t DateTime, x Int32, string String) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(t) ORDER BY (x, t); +CREATE TABLE select_final (t DateTime, x Int32, string String) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(t) ORDER BY (x, t); INSERT INTO select_final SELECT toDate('2000-01-01'), number, '' FROM numbers(2); INSERT INTO select_final SELECT toDate('2000-01-01'), number + 1, '' FROM numbers(2); @@ -31,6 +31,8 @@ INSERT INTO select_final SELECT toDate('2000-01-01'), number, '' FROM numbers(50 OPTIMIZE TABLE select_final FINAL; +SET remote_filesystem_read_method = 'read'; + SELECT max(x) FROM select_final FINAL; SYSTEM FLUSH LOGS; diff --git a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql index 3b6f77336fe..d02a2af6666 100644 --- a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql +++ b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql @@ -3,6 +3,6 @@ SELECT number FROM numbers(10) ORDER BY number DESC OFFSET 2 ROWS FETCH NEXT 3 R DROP TABLE IF EXISTS test_fetch; CREATE TABLE test_fetch(a Int32, b Int32) Engine = Memory; INSERT INTO test_fetch VALUES(1, 1), (2, 1), (3, 4), (3, 3), (5, 4), (0, 6), (5, 7); -SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY; +SELECT * FROM (SELECT * FROM test_fetch ORDER BY a, b OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY) ORDER BY a, b; SELECT * FROM (SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS WITH TIES) ORDER BY a, b; DROP TABLE test_fetch; diff --git a/tests/queries/0_stateless/01526_max_untracked_memory.sh b/tests/queries/0_stateless/01526_max_untracked_memory.sh index 20c986f14ca..45fdb314fb2 100755 --- a/tests/queries/0_stateless/01526_max_untracked_memory.sh +++ b/tests/queries/0_stateless/01526_max_untracked_memory.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan -# Tag no-tsan: requires TraceCollector, does not available under sanitizers -# Tag no-asan: requires TraceCollector, does not available under sanitizers -# Tag no-ubsan: requires TraceCollector, does not available under sanitizers -# Tag no-msan: requires TraceCollector, does not available under sanitizers +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-cpu-aarch64 +# requires TraceCollector, does not available under sanitizers and aarch64 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql b/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql index 8df6d0b314e..cf06af0113d 100644 --- a/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql +++ b/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql @@ -1,4 +1,4 @@ --- Tags: long, replica, no-replicated-database, no-parallel +-- Tags: long, replica, no-replicated-database, no-parallel, no-s3-storage -- Tag no-replicated-database: Fails due to additional replicas or shards -- Tag no-parallel: static zk path diff --git a/tests/queries/0_stateless/01533_multiple_nested.sql b/tests/queries/0_stateless/01533_multiple_nested.sql index 82049243006..03724ce0b46 100644 --- a/tests/queries/0_stateless/01533_multiple_nested.sql +++ b/tests/queries/0_stateless/01533_multiple_nested.sql @@ -1,6 +1,9 @@ +-- Tags: no-s3-storage +-- Temporary supressed DROP TABLE IF EXISTS nested; SET flatten_nested = 0; +SET use_uncompressed_cache = 0; CREATE TABLE nested ( diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh index 981beb785a5..39ef530f6e9 100755 --- a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh +++ b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh @@ -18,27 +18,25 @@ $CLICKHOUSE_CLIENT --query "CREATE DICTIONARY ordinary_db.dict1 ( key_column UIn function dict_get_thread() { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null - done + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null } -export -f dict_get_thread; +export -f dict_get_thread TIMEOUT=10 -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01543_collate_in_tuple.sql b/tests/queries/0_stateless/01543_collate_in_tuple.sql index 222f7762b32..e50b5e5223d 100644 --- a/tests/queries/0_stateless/01543_collate_in_tuple.sql +++ b/tests/queries/0_stateless/01543_collate_in_tuple.sql @@ -12,19 +12,19 @@ INSERT INTO collate_test1 VALUES (1, (1, 'Ё')), (1, (1, 'ё')), (1, (1, 'а')), INSERT INTO collate_test2 VALUES (1, (1, 'Ё')), (1, (1, 'ё')), (1, (1, 'а')), (2, (2, 'А')), (2, (1, 'я')), (2, (2, 'Я')), (1, (2, null)), (1, (3, 'я')), (1, (1, null)), (2, (2, null)); INSERT INTO collate_test3 VALUES (1, (1, (1, ['Ё']))), (1, (2, (1, ['ё']))), (1, (1, (2, ['а']))), (2, (1, (1, ['А']))), (2, (2, (1, ['я']))), (2, (1, (1, ['Я']))), (1, (2, (1, ['ё','а']))), (1, (1, (2, ['ё', 'я']))), (2, (1, (1, ['ё', 'а', 'а']))); -SELECT * FROM collate_test1 ORDER BY s COLLATE 'ru'; +SELECT * FROM collate_test1 ORDER BY s COLLATE 'ru', x; SELECT ''; SELECT * FROM collate_test1 ORDER BY x, s COLLATE 'ru'; SELECT ''; -SELECT * FROM collate_test2 ORDER BY s COLLATE 'ru'; +SELECT * FROM collate_test2 ORDER BY s COLLATE 'ru', x; SELECT ''; SELECT * FROM collate_test2 ORDER BY x, s COLLATE 'ru'; SELECT ''; -SELECT * FROM collate_test3 ORDER BY s COLLATE 'ru'; +SELECT * FROM collate_test3 ORDER BY s COLLATE 'ru', x; SELECT ''; SELECT * FROM collate_test3 ORDER BY x, s COLLATE 'ru'; diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql index a71c9f9a714..1d21d861e20 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + DROP TABLE IF EXISTS data_01551; CREATE TABLE data_01551 diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index 0143b8e46ed..bdcde1adbad 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check + -- force data path with the user/pass in it set use_compact_format_in_distributed_parts_names=0; -- use async send even for localhost diff --git a/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql b/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql index 2b1a66147a4..04777f5b31c 100644 --- a/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql +++ b/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql @@ -1,5 +1,7 @@ -- Tags: replica +SET allow_experimental_parallel_reading_from_replicas=0; + DROP TABLE IF EXISTS t; CREATE TABLE t (x String) ENGINE = MergeTree ORDER BY x; INSERT INTO t VALUES ('Hello'); diff --git a/tests/queries/0_stateless/01569_query_profiler_big_query_id.sh b/tests/queries/0_stateless/01569_query_profiler_big_query_id.sh index 118d0a4fb96..e54783e9655 100755 --- a/tests/queries/0_stateless/01569_query_profiler_big_query_id.sh +++ b/tests/queries/0_stateless/01569_query_profiler_big_query_id.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-cpu-aarch64 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh b/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh index fcea7f57cd3..27de10ab16a 100755 --- a/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh +++ b/tests/queries/0_stateless/01576_alter_low_cardinality_and_select.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-backward-compatibility-check CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01582_move_to_prewhere_compact_parts.sql b/tests/queries/0_stateless/01582_move_to_prewhere_compact_parts.sql index 788c99da76d..bd3e651e0dc 100644 --- a/tests/queries/0_stateless/01582_move_to_prewhere_compact_parts.sql +++ b/tests/queries/0_stateless/01582_move_to_prewhere_compact_parts.sql @@ -1,3 +1,6 @@ +SET optimize_move_to_prewhere = 1; +SET convert_query_to_cnf = 0; + DROP TABLE IF EXISTS prewhere_move; CREATE TABLE prewhere_move (x Int, y String) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO prewhere_move SELECT number, toString(number) FROM numbers(1000); diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 2f6077fceb3..655232fcdd4 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -39,7 +39,7 @@ select number, avg(number) over (order by number rows unbounded preceding) from 8 4 9 4.5 -- no order by -select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) from numbers(10); 0 0 1 1 2 1 @@ -51,7 +51,7 @@ select number, quantileExact(number) over (partition by intDiv(number, 3) AS val 8 7 9 9 -- can add an alias after window spec -select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) q from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) q from numbers(10); 0 0 1 1 2 1 @@ -198,7 +198,7 @@ select sum(number) over w1, sum(number) over w2 from numbers(10) window w1 as (rows unbounded preceding), - w2 as (partition by intDiv(number, 3) as value order by value rows unbounded preceding) + w2 as (partition by intDiv(number, 3) as value order by number rows unbounded preceding) ; 0 0 1 1 @@ -214,7 +214,7 @@ window -- EXPLAIN test for this. select sum(number) over w1, - sum(number) over (partition by intDiv(number, 3) as value order by value rows unbounded preceding) + sum(number) over (partition by intDiv(number, 3) as value order by number rows unbounded preceding) from numbers(10) window w1 as (partition by intDiv(number, 3) rows unbounded preceding) @@ -240,118 +240,118 @@ select sum(number) over () from numbers(3); -- interesting corner cases. select number, intDiv(number, 3) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 5 ; -0 0 0 2 +0 0 0 1 1 0 1 3 2 0 0 2 -3 1 1 3 +3 1 1 2 4 1 0 1 5 1 1 3 -6 2 0 2 +6 2 0 1 7 2 1 3 8 2 0 2 -9 3 1 3 +9 3 1 2 10 3 0 1 11 3 1 3 -12 4 0 2 +12 4 0 1 13 4 1 3 14 4 0 2 -15 5 1 3 +15 5 1 2 16 5 0 1 17 5 1 3 -18 6 0 2 +18 6 0 1 19 6 1 3 20 6 0 2 -21 7 1 3 +21 7 1 2 22 7 0 1 23 7 1 3 -24 8 0 2 +24 8 0 1 25 8 1 3 26 8 0 2 -27 9 1 3 +27 9 1 2 28 9 0 1 29 9 1 3 30 10 0 1 select number, intDiv(number, 5) p, mod(number, 3) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; -0 0 0 2 -1 0 1 4 +0 0 0 1 +1 0 1 3 2 0 2 5 3 0 0 2 4 0 1 4 -5 1 2 5 -6 1 0 2 +5 1 2 4 +6 1 0 1 7 1 1 3 8 1 2 5 9 1 0 2 -10 2 1 3 -11 2 2 5 +10 2 1 2 +11 2 2 4 12 2 0 1 13 2 1 3 14 2 2 5 -15 3 0 2 -16 3 1 4 +15 3 0 1 +16 3 1 3 17 3 2 5 18 3 0 2 19 3 1 4 -20 4 2 5 -21 4 0 2 +20 4 2 4 +21 4 0 1 22 4 1 3 23 4 2 5 24 4 0 2 -25 5 1 3 -26 5 2 5 +25 5 1 2 +26 5 2 4 27 5 0 1 28 5 1 3 29 5 2 5 30 6 0 1 select number, intDiv(number, 5) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; -0 0 0 3 -1 0 1 5 -2 0 0 3 +0 0 0 1 +1 0 1 4 +2 0 0 2 3 0 1 5 4 0 0 3 -5 1 1 5 -6 1 0 2 -7 1 1 5 +5 1 1 3 +6 1 0 1 +7 1 1 4 8 1 0 2 9 1 1 5 -10 2 0 3 -11 2 1 5 -12 2 0 3 +10 2 0 1 +11 2 1 4 +12 2 0 2 13 2 1 5 14 2 0 3 -15 3 1 5 -16 3 0 2 -17 3 1 5 +15 3 1 3 +16 3 0 1 +17 3 1 4 18 3 0 2 19 3 1 5 -20 4 0 3 -21 4 1 5 -22 4 0 3 +20 4 0 1 +21 4 1 4 +22 4 0 2 23 4 1 5 24 4 0 3 -25 5 1 5 -26 5 0 2 -27 5 1 5 +25 5 1 3 +26 5 0 1 +27 5 1 4 28 5 0 2 29 5 1 5 30 6 0 1 select number, intDiv(number, 3) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; @@ -388,7 +388,7 @@ settings max_block_size = 2 30 10 0 1 select number, intDiv(number, 2) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; @@ -975,39 +975,39 @@ select number, p, o, row_number() over w from (select number, intDiv(number, 5) p, mod(number, 3) o from numbers(31) order by o, number) t -window w as (partition by p order by o) +window w as (partition by p order by o, number) order by p, o, number settings max_block_size = 2; -0 0 0 2 1 1 1 -3 0 0 2 1 1 2 -1 0 1 4 3 2 3 -4 0 1 4 3 2 4 -2 0 2 5 5 3 5 -6 1 0 2 1 1 1 -9 1 0 2 1 1 2 -7 1 1 3 3 2 3 -5 1 2 5 4 3 4 -8 1 2 5 4 3 5 +0 0 0 1 1 1 1 +3 0 0 2 2 2 2 +1 0 1 3 3 3 3 +4 0 1 4 4 4 4 +2 0 2 5 5 5 5 +6 1 0 1 1 1 1 +9 1 0 2 2 2 2 +7 1 1 3 3 3 3 +5 1 2 4 4 4 4 +8 1 2 5 5 5 5 12 2 0 1 1 1 1 -10 2 1 3 2 2 2 -13 2 1 3 2 2 3 -11 2 2 5 4 3 4 -14 2 2 5 4 3 5 -15 3 0 2 1 1 2 -18 3 0 2 1 1 1 -16 3 1 4 3 2 3 -19 3 1 4 3 2 4 -17 3 2 5 5 3 5 -21 4 0 2 1 1 1 -24 4 0 2 1 1 2 -22 4 1 3 3 2 3 -20 4 2 5 4 3 5 -23 4 2 5 4 3 4 +10 2 1 2 2 2 2 +13 2 1 3 3 3 3 +11 2 2 4 4 4 4 +14 2 2 5 5 5 5 +15 3 0 1 1 1 1 +18 3 0 2 2 2 2 +16 3 1 3 3 3 3 +19 3 1 4 4 4 4 +17 3 2 5 5 5 5 +21 4 0 1 1 1 1 +24 4 0 2 2 2 2 +22 4 1 3 3 3 3 +20 4 2 4 4 4 4 +23 4 2 5 5 5 5 27 5 0 1 1 1 1 -25 5 1 3 2 2 2 -28 5 1 3 2 2 3 -26 5 2 5 4 3 4 -29 5 2 5 4 3 5 +25 5 1 2 2 2 2 +28 5 1 3 3 3 3 +26 5 2 4 4 4 4 +29 5 2 5 5 5 5 30 6 0 1 1 1 1 -- our replacement for lag/lead select @@ -1141,6 +1141,28 @@ from ( from numbers_mt(10000) ) settings max_block_size = 7; 49995000 +-- a test with aggregate function which is -state type +select bitmapCardinality(bs) +from + ( + select groupBitmapMergeState(bm) over (order by k asc rows between unbounded preceding and current row) as bs + from + ( + select + groupBitmapState(number) as bm, k + from + ( + select + number, + number % 3 as k + from numbers(3) + ) + group by k + ) + ); +1 +2 +3 -- -INT_MIN row offset that can lead to problems with negation, found when fuzzing -- under UBSan. Should be limited to at most INT_MAX. select count() over (rows between 2147483648 preceding and 2147493648 following) from numbers(2); -- { serverError 36 } @@ -1153,7 +1175,7 @@ select count() over () where null; select number, count() over (w1 rows unbounded preceding) from numbers(10) window w0 as (partition by intDiv(number, 5) as p), - w1 as (w0 order by mod(number, 3) as o) + w1 as (w0 order by mod(number, 3) as o, number) order by p, o, number ; 0 1 diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index eb8c28de719..4a900045c6d 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -1,3 +1,5 @@ +-- Tags: long + -- { echo } -- just something basic @@ -13,10 +15,10 @@ select number, abs(number) over (partition by toString(intDiv(number, 3)) rows u select number, avg(number) over (order by number rows unbounded preceding) from numbers(10); -- no order by -select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) from numbers(10); -- can add an alias after window spec -select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by value rows unbounded preceding) q from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) q from numbers(10); -- can't reference it yet -- the window functions are calculated at the -- last stage of select, after all other functions. @@ -81,14 +83,14 @@ select sum(number) over w1, sum(number) over w2 from numbers(10) window w1 as (rows unbounded preceding), - w2 as (partition by intDiv(number, 3) as value order by value rows unbounded preceding) + w2 as (partition by intDiv(number, 3) as value order by number rows unbounded preceding) ; -- FIXME both functions should use the same window, but they don't. Add an -- EXPLAIN test for this. select sum(number) over w1, - sum(number) over (partition by intDiv(number, 3) as value order by value rows unbounded preceding) + sum(number) over (partition by intDiv(number, 3) as value order by number rows unbounded preceding) from numbers(10) window w1 as (partition by intDiv(number, 3) rows unbounded preceding) @@ -103,35 +105,35 @@ select sum(number) over () from numbers(3); -- interesting corner cases. select number, intDiv(number, 3) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 5 ; select number, intDiv(number, 5) p, mod(number, 3) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; select number, intDiv(number, 5) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; select number, intDiv(number, 3) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; select number, intDiv(number, 2) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; @@ -349,7 +351,7 @@ select number, p, o, row_number() over w from (select number, intDiv(number, 5) p, mod(number, 3) o from numbers(31) order by o, number) t -window w as (partition by p order by o) +window w as (partition by p order by o, number) order by p, o, number settings max_block_size = 2; @@ -442,6 +444,26 @@ from ( from numbers_mt(10000) ) settings max_block_size = 7; +-- a test with aggregate function which is -state type +select bitmapCardinality(bs) +from + ( + select groupBitmapMergeState(bm) over (order by k asc rows between unbounded preceding and current row) as bs + from + ( + select + groupBitmapState(number) as bm, k + from + ( + select + number, + number % 3 as k + from numbers(3) + ) + group by k + ) + ); + -- -INT_MIN row offset that can lead to problems with negation, found when fuzzing -- under UBSan. Should be limited to at most INT_MAX. select count() over (rows between 2147483648 preceding and 2147493648 following) from numbers(2); -- { serverError 36 } @@ -456,7 +478,7 @@ select count() over () where null; select number, count() over (w1 rows unbounded preceding) from numbers(10) window w0 as (partition by intDiv(number, 5) as p), - w1 as (w0 order by mod(number, 3) as o) + w1 as (w0 order by mod(number, 3) as o, number) order by p, o, number ; diff --git a/tests/queries/0_stateless/01592_window_functions.sql b/tests/queries/0_stateless/01592_window_functions.sql index 1ef416aaa78..f0d173b1f20 100644 --- a/tests/queries/0_stateless/01592_window_functions.sql +++ b/tests/queries/0_stateless/01592_window_functions.sql @@ -36,7 +36,7 @@ SELECT price, rank() OVER (PARTITION BY group_name ORDER BY price) rank FROM products INNER JOIN product_groups USING (group_id) -order by group_name, rank, price; +order by group_name, rank, price, product_name; select '---- Q3 ----'; SELECT diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index d18fbe6bdd5..30e1eff87da 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -13,36 +13,32 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_kill SELECT number, to function alter_thread { - while true; do - TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}' and name='value'") - if [ "$TYPE" == "String" ]; then - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" - else - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" - fi - done + TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}' and name='value'") + if [ "$TYPE" == "String" ]; then + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" + else + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" + fi } function kill_mutation_thread { - while true; do - # find any mutation and kill it - mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done=0 and database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_kill' LIMIT 1") - if [ ! -z "$mutation_id" ]; then - $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null - sleep 1 - fi - done + # find any mutation and kill it + mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done=0 and database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_kill' LIMIT 1") + if [ ! -z "$mutation_id" ]; then + $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null + sleep 1 + fi } -export -f alter_thread; -export -f kill_mutation_thread; +export -f alter_thread +export -f kill_mutation_thread TIMEOUT=30 -timeout $TIMEOUT bash -c alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c kill_mutation_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT kill_mutation_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh index f8f3ccd6dd6..3ee321371bf 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh @@ -27,38 +27,34 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SELECT sum(toUInt64(value)) FROM concurrent_kill_$i" done -function alter_thread +function alter_thread() { - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_kill_$REPLICA' and database='${CLICKHOUSE_DATABASE}' and name='value'") - if [ "$TYPE" == "String" ]; then - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" - else - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" - fi - done + REPLICA=$(($RANDOM % 5 + 1)) + TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_kill_$REPLICA' and database='${CLICKHOUSE_DATABASE}' and name='value'") + if [ "$TYPE" == "String" ]; then + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" + else + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" + fi } -function kill_mutation_thread +function kill_mutation_thread() { - while true; do - # find any mutation and kill it - mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done = 0 and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}' LIMIT 1") - if [ ! -z "$mutation_id" ]; then - $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null - sleep 1 - fi - done + # find any mutation and kill it + mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done = 0 and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}' LIMIT 1") + if [ ! -z "$mutation_id" ]; then + $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null + sleep 1 + fi } -export -f alter_thread; -export -f kill_mutation_thread; +export -f alter_thread +export -f kill_mutation_thread TIMEOUT=30 -timeout $TIMEOUT bash -c alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c kill_mutation_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT kill_mutation_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh b/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh index 248e8a06fb2..7f77f8bb403 100755 --- a/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh +++ b/tests/queries/0_stateless/01599_multiline_input_and_singleline_comments.sh @@ -1,4 +1,6 @@ #!/usr/bin/expect -f +# Tags: no-fasttest +# Tag no-fasttest: 180 seconds running log_user 0 set timeout 60 diff --git a/tests/queries/0_stateless/01602_max_distributed_connections.sh b/tests/queries/0_stateless/01602_max_distributed_connections.sh index ed835a8768f..0869d2d3a68 100755 --- a/tests/queries/0_stateless/01602_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01602_max_distributed_connections.sh @@ -15,14 +15,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) i=0 retries=30 while [[ $i -lt $retries ]]; do - timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " + clickhouse_client_timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=0 && break ((++i)) done i=0 retries=30 while [[ $i -lt $retries ]]; do - timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " + clickhouse_client_timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=1 && break ((++i)) done @@ -30,10 +30,13 @@ done # If max_distributed_connections is low and async_socket_for_remote is disabled, # the concurrency of distributed queries will be also low. -timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " +clickhouse_client_timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " SELECT sleep(0.15) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=0 && echo 'Fail' -timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " +clickhouse_client_timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " SELECT sleep(0.15) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=1 && echo 'Fail' +# FIXME +clickhouse_test_wait_queries 5 + echo 'Ok' diff --git a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql index a73045f5a6f..750809da338 100644 --- a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql +++ b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql @@ -1,3 +1,5 @@ +SET use_uncompressed_cache = 0; + DROP TABLE IF EXISTS adaptive_table; --- If granularity of consequent blocks differs a lot, then adaptive @@ -20,6 +22,8 @@ OPTIMIZE TABLE adaptive_table FINAL; SELECT marks FROM system.parts WHERE table = 'adaptive_table' and database=currentDatabase() and active; +SET remote_fs_enable_cache = 0; + -- If we have computed granularity incorrectly than we will exceed this limit. SET max_memory_usage='30M'; diff --git a/tests/queries/0_stateless/01632_tinylog_read_write.sh b/tests/queries/0_stateless/01632_tinylog_read_write.sh index 3f41bcc5924..42211168d45 100755 --- a/tests/queries/0_stateless/01632_tinylog_read_write.sh +++ b/tests/queries/0_stateless/01632_tinylog_read_write.sh @@ -9,35 +9,32 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --multiquery --query "DROP TABLE IF EXISTS test; CREATE TABLE IF NOT EXISTS test (x UInt64, s Array(Nullable(String))) ENGINE = TinyLog;" -function thread_select { - while true; do - $CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" - sleep 0.0$RANDOM - done +function thread_select() +{ + $CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" + sleep 0.0$RANDOM } -function thread_insert { - while true; do - $CLICKHOUSE_CLIENT --query "INSERT INTO test VALUES (1, ['Hello'])" - sleep 0.0$RANDOM - done +function thread_insert() +{ + $CLICKHOUSE_CLIENT --query "INSERT INTO test VALUES (1, ['Hello'])" + sleep 0.0$RANDOM } export -f thread_select export -f thread_insert - # Do randomized queries and expect nothing extraordinary happens. -timeout 10 bash -c 'thread_select' & -timeout 10 bash -c 'thread_select' & -timeout 10 bash -c 'thread_select' & -timeout 10 bash -c 'thread_select' & +clickhouse_client_loop_timeout 10 thread_select & +clickhouse_client_loop_timeout 10 thread_select & +clickhouse_client_loop_timeout 10 thread_select & +clickhouse_client_loop_timeout 10 thread_select & -timeout 10 bash -c 'thread_insert' & -timeout 10 bash -c 'thread_insert' & -timeout 10 bash -c 'thread_insert' & -timeout 10 bash -c 'thread_insert' & +clickhouse_client_loop_timeout 10 thread_insert & +clickhouse_client_loop_timeout 10 thread_insert & +clickhouse_client_loop_timeout 10 thread_insert & +clickhouse_client_loop_timeout 10 thread_insert & wait echo "Done" diff --git a/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql index 6bbf6fcec6a..7ec3153886c 100644 --- a/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql +++ b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql @@ -2,6 +2,10 @@ drop table if exists data_01641; +-- Disable cache for s3 storage tests because it increases memory usage. +set remote_fs_enable_cache=0; +set remote_filesystem_read_method='read'; + create table data_01641 (key Int, value String) engine=MergeTree order by (key, repeat(value, 40)) settings old_parts_lifetime=0, min_bytes_for_wide_part=0; SET max_block_size = 1000, min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; @@ -9,6 +13,7 @@ insert into data_01641 select number, toString(number) from numbers(120000); -- Definitely should fail and it proves that memory is tracked in OPTIMIZE query. set max_memory_usage='10Mi', max_untracked_memory=0; + optimize table data_01641 final; -- { serverError 241 } drop table data_01641; diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference index f57d5df6efd..613c455fc59 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference @@ -12,3 +12,4 @@ memory in_memory_parts_insert_sync 1 wide fsync_part_directory,vertical 1 +2 diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql index 644cf063a33..598e1ef3c34 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql @@ -43,8 +43,10 @@ optimize table data_01643 final; drop table data_01643; select 'wide fsync_part_directory,vertical'; -create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0; insert into data_01643 values (1); -select * from data_01643; -optimize table data_01643 final; +insert into data_01643 values (2); +select * from data_01643 order by key; +-- vertical merge does not supports deduplicate, hence no FINAL +optimize table data_01643; drop table data_01643; diff --git a/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql b/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql index ce48ad6a02a..4357aa199dc 100644 --- a/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql +++ b/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-backward-compatibility-check DROP TABLE IF EXISTS test_01640; DROP TABLE IF EXISTS restore_01640; diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh index de3d3ac3eb6..b66d788a338 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations.sh @@ -64,7 +64,7 @@ $CLICKHOUSE_CLIENT -q " settings enable_optimize_predicate_expression=0" echo "> one condition of filter should be pushed down after aggregating, other two conditions are ANDed" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT --convert_query_to_cnf=0 -q " explain actions = 1 select s, y from ( select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y ) where y != 0 and s - 8 and s - 4 @@ -77,7 +77,7 @@ $CLICKHOUSE_CLIENT -q " settings enable_optimize_predicate_expression=0" echo "> two conditions of filter should be pushed down after aggregating and ANDed, one condition is aliased" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT --convert_query_to_cnf=0 -q " explain actions = 1 select s, y from ( select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y ) where y != 0 and s != 8 and y - 4 @@ -127,7 +127,7 @@ $CLICKHOUSE_CLIENT -q " settings enable_optimize_predicate_expression=0" echo "> filter is pushed down before sorting steps" -$CLICKHOUSE_CLIENT -q " +$CLICKHOUSE_CLIENT --convert_query_to_cnf=0 -q " explain actions = 1 select x, y from ( select number % 2 as x, number % 3 as y from numbers(6) order by y desc ) where x != 0 and y != 0 diff --git a/tests/queries/0_stateless/01662_date_ubsan.reference b/tests/queries/0_stateless/01662_date_ubsan.reference index 3c90ee8960e..b9bff160f6b 100644 --- a/tests/queries/0_stateless/01662_date_ubsan.reference +++ b/tests/queries/0_stateless/01662_date_ubsan.reference @@ -1,34 +1,5 @@ --- { echo } --- tests with INT64_MIN (via overflow) -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -2021-01-01 00:00:00 --- tests with INT64_MAX -SELECT addMinutes(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-31 23:59:00 -SELECT addHours(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-31 23:00:00 -SELECT addWeeks(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-25 00:00:00 -SELECT addDays(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-12-31 00:00:00 -SELECT addYears(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); 2019-01-01 00:00:00 --- tests with inf -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -2021-01-01 00:00:00 diff --git a/tests/queries/0_stateless/01662_date_ubsan.sql b/tests/queries/0_stateless/01662_date_ubsan.sql index 784853bef3d..fd197044fad 100644 --- a/tests/queries/0_stateless/01662_date_ubsan.sql +++ b/tests/queries/0_stateless/01662_date_ubsan.sql @@ -1,19 +1,18 @@ --- { echo } --- tests with INT64_MIN (via overflow) -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- tests with INT64_MAX SELECT addMinutes(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addHours(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addWeeks(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addDays(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); SELECT addYears(toDateTime('2020-01-01 00:00:00', 'GMT'), 9223372036854775807); +-- tests with INT64_MIN (via overflow) +SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } +SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), 9223372036854775808); -- { serverError DECIMAL_OVERFLOW } -- tests with inf -SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); +SELECT addMinutes(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addHours(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addWeeks(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addDays(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } +SELECT addYears(toDateTime('2021-01-01 00:00:00', 'GMT'), inf); -- { serverError DECIMAL_OVERFLOW } diff --git a/tests/queries/0_stateless/01666_blns_long.sql b/tests/queries/0_stateless/01666_blns_long.sql index fd959cf0a73..74054551b18 100644 --- a/tests/queries/0_stateless/01666_blns_long.sql +++ b/tests/queries/0_stateless/01666_blns_long.sql @@ -27,6 +27,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +SET max_insert_threads = 0; + DROP TABLE IF EXISTS test; CREATE TABLE test diff --git a/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference b/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference index eb246761f9c..49c1542f109 100644 --- a/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference +++ b/tests/queries/0_stateless/01670_dictionary_create_key_expression.reference @@ -1,8 +1,8 @@ Simple -5791441145865411458 Test2 -3450587330153346914 Test1 3111929972906540512 Test3 +3450587330153346914 Test1 +5791441145865411458 Test2 Complex 3111929972906540512 5 Test3 -5791441145865411458 5 Test2 3450587330153346914 5 Test1 +5791441145865411458 5 Test2 diff --git a/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql b/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql index 32e7dc17479..97c04ce445f 100644 --- a/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql +++ b/tests/queries/0_stateless/01670_dictionary_create_key_expression.sql @@ -17,7 +17,7 @@ SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'test_for LIFETIME(MIN 1 MAX 10) LAYOUT(HASHED()); -SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_simple; +SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_simple ORDER BY value_id; DROP DICTIONARY IF EXISTS database_dictionary_test_key_expression.test_query_log_dictionary_simple; @@ -34,7 +34,7 @@ SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'test_for LIFETIME(MIN 1 MAX 10) LAYOUT(COMPLEX_KEY_HASHED()); -SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_complex; +SELECT * FROM database_dictionary_test_key_expression.test_query_log_dictionary_complex ORDER BY value_id; DROP DICTIONARY IF EXISTS database_dictionary_test_key_expression.test_query_log_dictionary_complex; diff --git a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql index 9de544b111d..f216befbb06 100644 --- a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql +++ b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql @@ -37,7 +37,7 @@ FROM ( SELECT * FROM neighbor_test - ORDER BY val_string ASC + ORDER BY val_string, rowNr ) ORDER BY rowNr, val_string, str_m1, str_p1, val_low, low_m1, low_p1 format PrettyCompact; diff --git a/tests/queries/0_stateless/01671_aggregate_function_group_bitmap_data.sql b/tests/queries/0_stateless/01671_aggregate_function_group_bitmap_data.sql index 3f5c5c2f25b..d70665655ca 100644 --- a/tests/queries/0_stateless/01671_aggregate_function_group_bitmap_data.sql +++ b/tests/queries/0_stateless/01671_aggregate_function_group_bitmap_data.sql @@ -1,3 +1,5 @@ +SET group_by_two_level_threshold = 10000; + CREATE TABLE group_bitmap_data_test ( `pickup_date` Date, diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh index 93c4451524c..a06a9ec7057 100755 --- a/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh +++ b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh @@ -5,26 +5,24 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -function thread_create_drop_table { - while true; do - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01671', 'r_$REPLICA') order by x" 2>/dev/null - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t1" - done +function thread_create_drop_table() +{ + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01671', 'r_$REPLICA') order by x" 2>/dev/null + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t1" } -function thread_alter_table { - while true; do - $CLICKHOUSE_CLIENT --query "ALTER TABLE $CLICKHOUSE_DATABASE.t1 on cluster test_shard_localhost ADD COLUMN newcol UInt32" >/dev/null 2>&1 - sleep 0.0$RANDOM - done +function thread_alter_table() +{ + $CLICKHOUSE_CLIENT --query "ALTER TABLE $CLICKHOUSE_DATABASE.t1 on cluster test_shard_localhost ADD COLUMN newcol UInt32" >/dev/null 2>&1 + sleep 0.0$RANDOM } export -f thread_create_drop_table export -f thread_alter_table -timeout 20 bash -c "thread_create_drop_table" & -timeout 20 bash -c 'thread_alter_table' & +clickhouse_client_loop_timeout 20 thread_create_drop_table & +clickhouse_client_loop_timeout 20 thread_alter_table & wait sleep 1 diff --git a/tests/queries/0_stateless/01675_data_type_coroutine.reference b/tests/queries/0_stateless/01675_data_type_coroutine.reference index 7326d960397..541dab48def 100644 --- a/tests/queries/0_stateless/01675_data_type_coroutine.reference +++ b/tests/queries/0_stateless/01675_data_type_coroutine.reference @@ -1 +1,2 @@ Ok +Ok diff --git a/tests/queries/0_stateless/01675_data_type_coroutine.sh b/tests/queries/0_stateless/01675_data_type_coroutine.sh index 8e80d722a4c..9ae6dadd1dc 100755 --- a/tests/queries/0_stateless/01675_data_type_coroutine.sh +++ b/tests/queries/0_stateless/01675_data_type_coroutine.sh @@ -16,3 +16,15 @@ done #echo "I = ${I}" echo 'Ok' + +counter=0 +I=0 +while [[ $counter -lt $retries ]]; do + I=$((I + 1)) + TYPE=$(perl -e "print 'Array(' x $I; print 'UInt8'; print ')' x $I") + ${CLICKHOUSE_CLIENT} --prefer_localhost_replica=0 --max_parser_depth 1000000 --query "SELECT * FROM remote('127.0.0.{1,2}', generateRandom('x $TYPE', 1, 1, 1)) LIMIT 1 FORMAT Null" 2>&1 | grep -q -F 'Maximum parse depth' && break; + ((++counter)) +done + +#echo "I = ${I}" +echo 'Ok' diff --git a/tests/queries/0_stateless/01680_date_time_add_ubsan.sql b/tests/queries/0_stateless/01680_date_time_add_ubsan.sql index f4690116e1a..d2c443bddf9 100644 --- a/tests/queries/0_stateless/01680_date_time_add_ubsan.sql +++ b/tests/queries/0_stateless/01680_date_time_add_ubsan.sql @@ -1,2 +1,3 @@ -SELECT DISTINCT result FROM (SELECT toStartOfFifteenMinutes(toDateTime(toStartOfFifteenMinutes(toDateTime(1000.0001220703125) + (number * 65536))) + (number * 9223372036854775807)) AS result FROM system.numbers LIMIT 1048576) ORDER BY result DESC NULLS FIRST FORMAT Null; +SELECT DISTINCT result FROM (SELECT toStartOfFifteenMinutes(toDateTime(toStartOfFifteenMinutes(toDateTime(1000.0001220703125) + (number * 65536))) + (number * 9223372036854775807)) AS result FROM system.numbers LIMIT 1048576) ORDER BY result DESC NULLS FIRST FORMAT Null; -- { serverError 407 } +SELECT DISTINCT result FROM (SELECT toStartOfFifteenMinutes(toDateTime(toStartOfFifteenMinutes(toDateTime(1000.0001220703125) + (number * 65536))) + toInt64(number * 9223372036854775807)) AS result FROM system.numbers LIMIT 1048576) ORDER BY result DESC NULLS FIRST FORMAT Null; SELECT round(round(round(round(round(100)), round(round(round(round(NULL), round(65535)), toTypeName(now() + 9223372036854775807) LIKE 'DateTime%DateTime%DateTime%DateTime%', round(-2)), 255), round(NULL)))); diff --git a/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference b/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference index 75a0b4104b3..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference +++ b/tests/queries/0_stateless/01681_arg_min_max_if_fix.reference @@ -1 +0,0 @@ -0 0 2 diff --git a/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql b/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql index b0aab898536..5edd52e0841 100644 --- a/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql +++ b/tests/queries/0_stateless/01681_arg_min_max_if_fix.sql @@ -1 +1 @@ -SELECT bitAnd(number, toUInt64(pow(257, 20) - 1048576)) AS k, argMaxIf(k, if((number % 255) = 256, toInt256(65535), number), number > 42), uniq(number) AS u FROM numbers(2) GROUP BY toInt256(-2, NULL), k; +SELECT bitAnd(number, toUInt64(pow(257, 20) - 1048576)) AS k, argMaxIf(k, if((number % 255) = 256, toInt256(65535), number), number > 42), uniq(number) AS u FROM numbers(2) GROUP BY toInt256(-2, NULL), k FORMAT Null diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql index 59993f40774..683bd271405 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql @@ -1,4 +1,4 @@ --- Tags: zookeeper, no-replicated-database, no-parallel +-- Tags: zookeeper, no-replicated-database, no-parallel, no-s3-storage drop table if exists x; diff --git a/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql b/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql index d5eaa2617a6..fd17bdf1e3c 100644 --- a/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql +++ b/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage set allow_experimental_projection_optimization = 1, force_optimize_projection = 1; drop table if exists tp; diff --git a/tests/queries/0_stateless/01710_aggregate_projections.sh b/tests/queries/0_stateless/01710_aggregate_projections.sh index a8b3e6bf99d..f4b2c2189dd 100755 --- a/tests/queries/0_stateless/01710_aggregate_projections.sh +++ b/tests/queries/0_stateless/01710_aggregate_projections.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -14,8 +15,8 @@ $CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by $CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1" $CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2), intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc limit 5 settings allow_experimental_projection_optimization=1" -$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2), intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1" +$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" $CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1" $CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" diff --git a/tests/queries/0_stateless/01710_force_use_projection.sql b/tests/queries/0_stateless/01710_force_use_projection.sql index 8931c65e34e..90dd4a6bec3 100644 --- a/tests/queries/0_stateless/01710_force_use_projection.sql +++ b/tests/queries/0_stateless/01710_force_use_projection.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists tp; create table tp (d1 Int32, d2 Int32, eventcnt Int64, projection p (select sum(eventcnt) group by d1)) engine = MergeTree order by (d1, d2); diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index c0f2250cc0f..0792fe331bb 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists d; create table d (i int, j int) engine MergeTree partition by i % 2 order by tuple() settings index_granularity = 1; diff --git a/tests/queries/0_stateless/01710_normal_projection_fix1.sql b/tests/queries/0_stateless/01710_normal_projection_fix1.sql index b4d7c6e8734..0634a810619 100644 --- a/tests/queries/0_stateless/01710_normal_projection_fix1.sql +++ b/tests/queries/0_stateless/01710_normal_projection_fix1.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (i int, j int) engine MergeTree order by i; diff --git a/tests/queries/0_stateless/01710_normal_projections.sh b/tests/queries/0_stateless/01710_normal_projections.sh index 70e38b3722a..1ab659a76ab 100755 --- a/tests/queries/0_stateless/01710_normal_projections.sh +++ b/tests/queries/0_stateless/01710_normal_projections.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Tags: no-s3-storage + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01710_projection_array_join.sql b/tests/queries/0_stateless/01710_projection_array_join.sql index cd18d9282b9..fde355fe4eb 100644 --- a/tests/queries/0_stateless/01710_projection_array_join.sql +++ b/tests/queries/0_stateless/01710_projection_array_join.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage set allow_experimental_projection_optimization = 1; drop table if exists x; diff --git a/tests/queries/0_stateless/01710_projection_detach_part.sql b/tests/queries/0_stateless/01710_projection_detach_part.sql index e3e6c7ac165..73e801a11ea 100644 --- a/tests/queries/0_stateless/01710_projection_detach_part.sql +++ b/tests/queries/0_stateless/01710_projection_detach_part.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage set allow_experimental_projection_optimization = 1; drop table if exists t; diff --git a/tests/queries/0_stateless/01710_projection_drop_if_exists.sql b/tests/queries/0_stateless/01710_projection_drop_if_exists.sql index f21092e5491..93b121dd39c 100644 --- a/tests/queries/0_stateless/01710_projection_drop_if_exists.sql +++ b/tests/queries/0_stateless/01710_projection_drop_if_exists.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + drop table if exists tp; create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y; diff --git a/tests/queries/0_stateless/01710_projection_fetch_long.sql b/tests/queries/0_stateless/01710_projection_fetch_long.sql index 6c41c69254e..fd12b84c817 100644 --- a/tests/queries/0_stateless/01710_projection_fetch_long.sql +++ b/tests/queries/0_stateless/01710_projection_fetch_long.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage drop table if exists tp_1; drop table if exists tp_2; diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql index 9370e9d36ce..d45b959ccd5 100644 --- a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql +++ b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql @@ -1,3 +1,6 @@ +-- Tags: no-s3-storage + +DROP TABLE IF EXISTS t; drop table if exists tp; create table tp (type Int32, eventcnt UInt64, projection p (select sum(eventcnt), type group by type order by sum(eventcnt))) engine = MergeTree order by type; -- { serverError 583 } diff --git a/tests/queries/0_stateless/01710_projection_in_index.sql b/tests/queries/0_stateless/01710_projection_in_index.sql index 2669d69dc9f..e1ae3540705 100644 --- a/tests/queries/0_stateless/01710_projection_in_index.sql +++ b/tests/queries/0_stateless/01710_projection_in_index.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (i int, j int, k int, projection p (select * order by j)) engine MergeTree order by i settings index_granularity = 1; diff --git a/tests/queries/0_stateless/01710_projection_in_set.sql b/tests/queries/0_stateless/01710_projection_in_set.sql index 99fa2cab0c5..39b54db86e3 100644 --- a/tests/queries/0_stateless/01710_projection_in_set.sql +++ b/tests/queries/0_stateless/01710_projection_in_set.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists x; create table x (i UInt64, j UInt64, k UInt64, projection agg (select sum(j), avg(k) group by i), projection norm (select j, k order by i)) engine MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql b/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql index 28bf1c050d0..4b2675f9677 100644 --- a/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql +++ b/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists x; create table x (i int) engine MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/01710_projection_mutation.sql b/tests/queries/0_stateless/01710_projection_mutation.sql index ab3fbd117d0..2503d828880 100644 --- a/tests/queries/0_stateless/01710_projection_mutation.sql +++ b/tests/queries/0_stateless/01710_projection_mutation.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS t; CREATE TABLE t (`key` UInt32, `created_at` Date, `value` UInt32, PROJECTION xxx (SELECT key, created_at, sum(value) GROUP BY key, created_at)) ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY key; @@ -5,3 +6,5 @@ CREATE TABLE t (`key` UInt32, `created_at` Date, `value` UInt32, PROJECTION xxx INSERT INTO t SELECT 1 AS key, today() + (number % 30), number FROM numbers(1000); ALTER TABLE t UPDATE value = 0 WHERE (value > 0) AND (created_at >= '2021-12-21') SETTINGS allow_experimental_projection_optimization = 1; + +DROP TABLE IF EXISTS t; diff --git a/tests/queries/0_stateless/01710_projection_optimize_materialize.sql b/tests/queries/0_stateless/01710_projection_optimize_materialize.sql index d8251aabaf6..27252e2a171 100644 --- a/tests/queries/0_stateless/01710_projection_optimize_materialize.sql +++ b/tests/queries/0_stateless/01710_projection_optimize_materialize.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists z; create table z (pk Int64, d Date, id UInt64, c UInt64) Engine MergeTree partition by d order by pk ; diff --git a/tests/queries/0_stateless/01710_projection_part_check.sql b/tests/queries/0_stateless/01710_projection_part_check.sql index 1ccd9de5903..aa087169ad1 100644 --- a/tests/queries/0_stateless/01710_projection_part_check.sql +++ b/tests/queries/0_stateless/01710_projection_part_check.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists tp; create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32; diff --git a/tests/queries/0_stateless/01710_projection_vertical_merges.sql b/tests/queries/0_stateless/01710_projection_vertical_merges.sql index d54fef7e71d..74e3e3ac6ce 100644 --- a/tests/queries/0_stateless/01710_projection_vertical_merges.sql +++ b/tests/queries/0_stateless/01710_projection_vertical_merges.sql @@ -1,4 +1,4 @@ --- Tags: long, no-parallel +-- Tags: long, no-parallel, no-s3-storage drop table if exists t; diff --git a/tests/queries/0_stateless/01710_projection_with_joins.sql b/tests/queries/0_stateless/01710_projection_with_joins.sql index a54ba21fd27..472242e3043 100644 --- a/tests/queries/0_stateless/01710_projection_with_joins.sql +++ b/tests/queries/0_stateless/01710_projection_with_joins.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (s UInt16, l UInt16, projection p (select s, l order by l)) engine MergeTree order by s; diff --git a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql index 734aa659146..7986ca40095 100644 --- a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql +++ b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (x UInt32) engine = MergeTree order by tuple() settings index_granularity = 8; diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index c929bfaa273..54581b5ae11 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists projection_test; create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 alias length(domain), datetime DateTime, domain LowCardinality(String), x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64), projection p (select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) group by dt_m, domain)) engine MergeTree partition by toDate(datetime) order by (toStartOfTenMinutes(datetime), domain); diff --git a/tests/queries/0_stateless/01710_projections_group_by_no_key.sql b/tests/queries/0_stateless/01710_projections_group_by_no_key.sql index eefc03afb7a..b91d182e309 100644 --- a/tests/queries/0_stateless/01710_projections_group_by_no_key.sql +++ b/tests/queries/0_stateless/01710_projections_group_by_no_key.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists projection_without_key; create table projection_without_key (key UInt32, PROJECTION x (SELECT sum(key) group by key % 3)) engine MergeTree order by key; diff --git a/tests/queries/0_stateless/01710_projections_in_distributed_query.sql b/tests/queries/0_stateless/01710_projections_in_distributed_query.sql index fa734b605cd..71ac158577a 100644 --- a/tests/queries/0_stateless/01710_projections_in_distributed_query.sql +++ b/tests/queries/0_stateless/01710_projections_in_distributed_query.sql @@ -1,4 +1,4 @@ --- Tags: distributed +-- Tags: distributed, no-s3-storage drop table if exists projection_test; diff --git a/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh b/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh index 5bb10220f7f..1cd243675c6 100755 --- a/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh +++ b/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh @@ -11,47 +11,34 @@ set -o errexit set -o pipefail echo " - DROP TABLE IF EXISTS storage_join_race; - CREATE TABLE storage_join_race (x UInt64, y UInt64) Engine = Join(ALL, FULL, x); + DROP TABLE IF EXISTS storage_join_race; + CREATE TABLE storage_join_race (x UInt64, y UInt64) Engine = Join(ALL, FULL, x); " | $CLICKHOUSE_CLIENT -n function read_thread_big() { - while true; do - echo " - SELECT * FROM ( SELECT number AS x FROM numbers(100000) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null; - " | $CLICKHOUSE_CLIENT -n - done + $CLICKHOUSE_CLIENT -n -q "SELECT * FROM ( SELECT number AS x FROM numbers(100000) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null" } function read_thread_small() { - while true; do - echo " - SELECT * FROM ( SELECT number AS x FROM numbers(10) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null; - " | $CLICKHOUSE_CLIENT -n - done + $CLICKHOUSE_CLIENT -n -q "SELECT * FROM ( SELECT number AS x FROM numbers(10) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null" } function read_thread_select() { - while true; do - echo " - SELECT * FROM storage_join_race FORMAT Null; - " | $CLICKHOUSE_CLIENT -n - done + $CLICKHOUSE_CLIENT -n -q "SELECT * FROM storage_join_race FORMAT Null" } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f read_thread_big; -export -f read_thread_small; -export -f read_thread_select; +export -f read_thread_big +export -f read_thread_small +export -f read_thread_select TIMEOUT=20 -timeout $TIMEOUT bash -c read_thread_big 2> /dev/null & -timeout $TIMEOUT bash -c read_thread_small 2> /dev/null & -timeout $TIMEOUT bash -c read_thread_select 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT read_thread_big 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT read_thread_small 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT read_thread_select 2> /dev/null & echo " INSERT INTO storage_join_race SELECT number AS x, number AS y FROM numbers (10000000); diff --git a/tests/queries/0_stateless/01737_move_order_key_to_prewhere_select_final.sql b/tests/queries/0_stateless/01737_move_order_key_to_prewhere_select_final.sql index ecc11c625e3..789892dbd38 100644 --- a/tests/queries/0_stateless/01737_move_order_key_to_prewhere_select_final.sql +++ b/tests/queries/0_stateless/01737_move_order_key_to_prewhere_select_final.sql @@ -1,3 +1,6 @@ +SET optimize_move_to_prewhere = 1; +SET convert_query_to_cnf = 0; + DROP TABLE IF EXISTS prewhere_move_select_final; CREATE TABLE prewhere_move_select_final (x Int, y Int, z Int) ENGINE = ReplacingMergeTree() ORDER BY (x, y); diff --git a/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh b/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh index 02943cad583..e10032e04fd 100755 --- a/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh +++ b/tests/queries/0_stateless/01746_long_zstd_http_compression_json_format.sh @@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -${CLICKHOUSE_CURL} -sS -H 'Accept-Encoding: zstd' "${CLICKHOUSE_URL}&enable_http_compression=1" -d "SELECT toDate('2020-12-12') as datetime, 'test-pipeline' as pipeline, 'clickhouse-test-host-001.clickhouse.com' as host, 'clickhouse' as home, 'clickhouse' as detail, number as row_number FROM numbers(1000000) FORMAT JSON" | zstd -d | tail -n30 | head -n23 +${CLICKHOUSE_CURL} -sS -H 'Accept-Encoding: zstd' "${CLICKHOUSE_URL}&enable_http_compression=1" -d "SELECT toDate('2020-12-12') as datetime, 'test-pipeline' as pipeline, 'clickhouse-test-host-001.clickhouse.com' as host, 'clickhouse' as home, 'clickhouse' as detail, number as row_number FROM numbers(1000000) SETTINGS max_block_size=65505 FORMAT JSON" | zstd -d | tail -n30 | head -n23 diff --git a/tests/queries/0_stateless/01747_system_session_log_long.sh b/tests/queries/0_stateless/01747_system_session_log_long.sh index b41bf077b57..9b127e0b48d 100755 --- a/tests/queries/0_stateless/01747_system_session_log_long.sh +++ b/tests/queries/0_stateless/01747_system_session_log_long.sh @@ -369,4 +369,4 @@ GROUP BY user_name, interface, type ORDER BY user_name, interface, type; -EOF \ No newline at end of file +EOF diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference index 9b76ca91780..15e00db0231 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference @@ -12,6 +12,9 @@ WITH _CAST(\'default\', \'Nullable(String)\') AS `id_2` SELECT `one`.`dummy`, ig optimize_skip_unused_shards_rewrite_in(0,) 0 0 WITH _CAST(\'default\', \'Nullable(String)\') AS `id_0` SELECT `one`.`dummy`, ignore(`id_0`) FROM `system`.`one` WHERE `dummy` IN tuple(0) +signed column +WITH _CAST(\'default\', \'Nullable(String)\') AS `key_signed` SELECT `key`, ignore(`key_signed`) FROM `default`.`data_01756_signed` WHERE `key` IN tuple(-1) +WITH _CAST(\'default\', \'Nullable(String)\') AS `key_signed` SELECT `key`, ignore(`key_signed`) FROM `default`.`data_01756_signed` WHERE `key` IN tuple(-2) 0 0 errors diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql index 220d5d91a0b..a5090551c89 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql @@ -9,6 +9,7 @@ drop table if exists dist_01756; drop table if exists dist_01756_str; drop table if exists dist_01756_column; drop table if exists data_01756_str; +drop table if exists data_01756_signed; -- SELECT -- intHash64(0) % 2, @@ -83,6 +84,20 @@ select query from system.query_log where type = 'QueryFinish' order by query; +-- signed column +select 'signed column'; +create table data_01756_signed (key Int) engine=Null; +with (select currentDatabase()) as key_signed select *, ignore(key_signed) from cluster(test_cluster_two_shards, currentDatabase(), data_01756_signed, key) where key in (-1, -2); +system flush logs; +select query from system.query_log where + event_date >= yesterday() and + event_time > now() - interval 1 hour and + not is_initial_query and + query not like '%system%query_log%' and + query like concat('WITH%', currentDatabase(), '%AS `key_signed` %') and + type = 'QueryFinish' +order by query; + -- not tuple select * from dist_01756 where dummy in (0); select * from dist_01756 where dummy in ('0'); @@ -119,8 +134,8 @@ create table data_01756_str (key String) engine=Memory(); create table dist_01756_str as data_01756_str engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01756_str, cityHash64(key)); select * from dist_01756_str where key in ('0', '2'); select * from dist_01756_str where key in ('0', Null); -- { serverError 507 } -select * from dist_01756_str where key in (0, 2); -- { serverError 53 } -select * from dist_01756_str where key in (0, Null); -- { serverError 53 } +-- select * from dist_01756_str where key in (0, 2); -- { serverError 53 } +-- select * from dist_01756_str where key in (0, Null); -- { serverError 53 } -- different type #2 select 'different types -- conversion'; @@ -139,3 +154,4 @@ drop table dist_01756; drop table dist_01756_str; drop table dist_01756_column; drop table data_01756_str; +drop table data_01756_signed; diff --git a/tests/queries/0_stateless/01763_max_distributed_depth.sql b/tests/queries/0_stateless/01763_max_distributed_depth.sql index 12b2e368007..f50d15e7121 100644 --- a/tests/queries/0_stateless/01763_max_distributed_depth.sql +++ b/tests/queries/0_stateless/01763_max_distributed_depth.sql @@ -1,5 +1,7 @@ -- Tags: distributed +SET prefer_localhost_replica = 1; + DROP TABLE IF EXISTS tt6; CREATE TABLE tt6 @@ -13,6 +15,8 @@ CREATE TABLE tt6 ) ENGINE = Distributed('test_shard_localhost', '', 'tt7', rand()); +DROP TABLE IF EXISTS tt7; + CREATE TABLE tt7 as tt6 ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError 581 } @@ -28,3 +32,4 @@ INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError 306} SELECT * FROM tt6; -- { serverError 306 } DROP TABLE tt6; +DROP TABLE tt7; diff --git a/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference b/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference index aa47d0d46d4..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference +++ b/tests/queries/0_stateless/01773_datetime64_add_ubsan.reference @@ -1,2 +0,0 @@ -0 -0 diff --git a/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql b/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql index f7267f2b6b4..70dcd6a133f 100644 --- a/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql +++ b/tests/queries/0_stateless/01773_datetime64_add_ubsan.sql @@ -1,2 +1,2 @@ -- The result is unspecified but UBSan should not argue. -SELECT ignore(addHours(now64(3), inf)) FROM numbers(2); +SELECT ignore(addHours(now64(3), inf)) FROM numbers(2); -- { serverError 407 } diff --git a/tests/queries/0_stateless/01780_column_sparse_alter.reference b/tests/queries/0_stateless/01780_column_sparse_alter.reference index 4fb0122db96..38fa0bf446a 100644 --- a/tests/queries/0_stateless/01780_column_sparse_alter.reference +++ b/tests/queries/0_stateless/01780_column_sparse_alter.reference @@ -5,3 +5,6 @@ u Sparse id Default t Sparse 182 +id Default +t Sparse +182 diff --git a/tests/queries/0_stateless/01780_column_sparse_alter.sql b/tests/queries/0_stateless/01780_column_sparse_alter.sql index 7f9558bfc18..925b81ea2c2 100644 --- a/tests/queries/0_stateless/01780_column_sparse_alter.sql +++ b/tests/queries/0_stateless/01780_column_sparse_alter.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check + SET mutations_sync = 2; DROP TABLE IF EXISTS t_sparse_alter; @@ -5,7 +7,6 @@ DROP TABLE IF EXISTS t_sparse_alter; CREATE TABLE t_sparse_alter (id UInt64, u UInt64, s String) ENGINE = MergeTree ORDER BY id SETTINGS ratio_of_defaults_for_sparse_serialization = 0.5; - INSERT INTO t_sparse_alter SELECT number, if (number % 11 = 0, number, 0), @@ -23,4 +24,10 @@ SELECT column, serialization_kind FROM system.parts_columns WHERE database = cur SELECT uniqExact(t) FROM t_sparse_alter; +DETACH TABLE t_sparse_alter; +ATTACH TABLE t_sparse_alter; + +SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_sparse_alter' AND active ORDER BY column; +SELECT uniqExact(t) FROM t_sparse_alter; + DROP TABLE t_sparse_alter; diff --git a/tests/queries/0_stateless/01780_column_sparse_filter.reference b/tests/queries/0_stateless/01780_column_sparse_filter.reference new file mode 100644 index 00000000000..d673acfc89f --- /dev/null +++ b/tests/queries/0_stateless/01780_column_sparse_filter.reference @@ -0,0 +1,13 @@ +id Default +s Sparse +u Sparse +5000 +2000 +id Default +id Default +s Default +s Sparse +u Default +u Sparse +105000 +102000 diff --git a/tests/queries/0_stateless/01780_column_sparse_filter.sql b/tests/queries/0_stateless/01780_column_sparse_filter.sql new file mode 100644 index 00000000000..45958b5c4e0 --- /dev/null +++ b/tests/queries/0_stateless/01780_column_sparse_filter.sql @@ -0,0 +1,33 @@ +DROP TABLE IF EXISTS t_sparse; + +CREATE TABLE t_sparse (id UInt64, u UInt64, s String) +ENGINE = MergeTree ORDER BY id +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse SELECT + number, + if (number % 20 = 0, number, 0), + if (number % 50 = 0, toString(number), '') +FROM numbers(1, 100000); + +SELECT column, serialization_kind FROM system.parts_columns +WHERE table = 't_sparse' AND database = currentDatabase() +ORDER BY column, serialization_kind; + +SELECT count() FROM t_sparse WHERE u > 0; +SELECT count() FROM t_sparse WHERE notEmpty(s); + +SYSTEM STOP MERGES t_sparse; + +INSERT INTO t_sparse SELECT + number, number, toString(number) +FROM numbers (1, 100000); + +SELECT column, serialization_kind FROM system.parts_columns +WHERE table = 't_sparse' AND database = currentDatabase() +ORDER BY column, serialization_kind; + +SELECT count() FROM t_sparse WHERE u > 0; +SELECT count() FROM t_sparse WHERE notEmpty(s); + +DROP TABLE t_sparse; diff --git a/tests/queries/0_stateless/01780_column_sparse_full.reference b/tests/queries/0_stateless/01780_column_sparse_full.reference index 4d2d0a58798..dbe815800a9 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.reference +++ b/tests/queries/0_stateless/01780_column_sparse_full.reference @@ -91,9 +91,9 @@ all_2_2_0 u Default ====== 0 0 0 0 0 0 -1 1 1 1 0 -2 2 2 +1 1 1 +2 0 ====== 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/01780_column_sparse_full.sql b/tests/queries/0_stateless/01780_column_sparse_full.sql index af6fde116d9..59128223dba 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.sql +++ b/tests/queries/0_stateless/01780_column_sparse_full.sql @@ -81,7 +81,7 @@ INNER JOIN t_sparse_full USING(u) ORDER BY id, u, s LIMIT 5; SELECT '======'; SELECT id, u, s FROM (SELECT number * 2 AS u FROM numbers(10)) AS t1 -FULL JOIN t_sparse_full USING(u) ORDER BY id LIMIT 5; +FULL JOIN t_sparse_full USING(u) ORDER BY id, u, s LIMIT 5; SELECT '======'; diff --git a/tests/queries/0_stateless/01780_column_sparse_tuple.reference b/tests/queries/0_stateless/01780_column_sparse_tuple.reference index 22337838cff..743ded75a81 100644 --- a/tests/queries/0_stateless/01780_column_sparse_tuple.reference +++ b/tests/queries/0_stateless/01780_column_sparse_tuple.reference @@ -41,7 +41,7 @@ a a a id [] [] [] -t ['a','b','b.u','b.s'] ['UInt64','Tuple(u UInt32, s String)','UInt32','String'] ['Sparse','Default','Sparse','Default'] +t ['a','b.u','b.s'] ['UInt64','UInt32','String'] ['Sparse','Sparse','Default'] 0 0 0 @@ -58,7 +58,7 @@ aaaaaa a aaaaaa id [] [] [] -t ['a','b','b.u','b.s'] ['UInt64','Tuple(u UInt32, s String)','UInt32','String'] ['Sparse','Default','Sparse','Default'] +t ['a','b.u','b.s'] ['UInt64','UInt32','String'] ['Sparse','Sparse','Default'] aaaaaa a aaaaaa diff --git a/tests/queries/0_stateless/01780_column_sparse_tuple.sql b/tests/queries/0_stateless/01780_column_sparse_tuple.sql index da679f2c7eb..e3dfc16fc74 100644 --- a/tests/queries/0_stateless/01780_column_sparse_tuple.sql +++ b/tests/queries/0_stateless/01780_column_sparse_tuple.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS sparse_tuple; CREATE TABLE sparse_tuple (id UInt64, t Tuple(a UInt64, s String)) diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.sh b/tests/queries/0_stateless/01786_explain_merge_tree.sh index 6be86f9ce02..eb47f065044 100755 --- a/tests/queries/0_stateless/01786_explain_merge_tree.sh +++ b/tests/queries/0_stateless/01786_explain_merge_tree.sh @@ -4,6 +4,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0" + $CLICKHOUSE_CLIENT -q "drop table if exists test_index" $CLICKHOUSE_CLIENT -q "drop table if exists idx" diff --git a/tests/queries/0_stateless/01788_update_nested_type_subcolumn_check.sql b/tests/queries/0_stateless/01788_update_nested_type_subcolumn_check.sql index accb785ba03..efd8ea2a565 100644 --- a/tests/queries/0_stateless/01788_update_nested_type_subcolumn_check.sql +++ b/tests/queries/0_stateless/01788_update_nested_type_subcolumn_check.sql @@ -31,6 +31,8 @@ select * from test_wide_nested; alter table test_wide_nested update `info.id` = [100,200], `info.age` = [10,20,30], `info.name` = ['a','b','c'] where id = 0; -- { serverError 341 } +kill mutation where table = 'test_wide_nested' and database = currentDatabase() format Null; + -- Recreate table, because KILL MUTATION is not suitable for parallel tests execution. SELECT '********* test 2 **********'; DROP TABLE test_wide_nested; @@ -54,6 +56,8 @@ select * from test_wide_nested; alter table test_wide_nested update `info.id` = [100,200,300], `info.age` = [10,20,30] where id = 1; -- { serverError 341 } +kill mutation where table = 'test_wide_nested' and database = currentDatabase() format Null; + DROP TABLE test_wide_nested; SELECT '********* test 3 **********'; diff --git a/tests/queries/0_stateless/01798_uniq_theta_sketch.sql b/tests/queries/0_stateless/01798_uniq_theta_sketch.sql index bb400c5de14..eace83d5cfa 100644 --- a/tests/queries/0_stateless/01798_uniq_theta_sketch.sql +++ b/tests/queries/0_stateless/01798_uniq_theta_sketch.sql @@ -1,5 +1,7 @@ -- Tags: no-fasttest +SET max_block_size = 65505; + SELECT 'uniqTheta many agrs'; SELECT diff --git a/tests/queries/0_stateless/01822_short_circuit.sql b/tests/queries/0_stateless/01822_short_circuit.sql index 48fff04921b..c7379d210eb 100644 --- a/tests/queries/0_stateless/01822_short_circuit.sql +++ b/tests/queries/0_stateless/01822_short_circuit.sql @@ -1,4 +1,5 @@ set short_circuit_function_evaluation = 'enable'; +set convert_query_to_cnf = 0; select if(number > 0, intDiv(number + 100, number), throwIf(number)) from numbers(10); select multiIf(number == 0, 0, number == 1, intDiv(1, number), number == 2, intDiv(1, number - 1), number == 3, intDiv(1, number - 2), intDiv(1, number - 3)) from numbers(10); diff --git a/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql b/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql index e03972e818d..c4ef5516fc8 100644 --- a/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql +++ b/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql @@ -1,3 +1,6 @@ +SET optimize_move_to_prewhere = 1; +SET convert_query_to_cnf = 0; + DROP TABLE IF EXISTS t_move_to_prewhere; CREATE TABLE t_move_to_prewhere (id UInt32, a UInt8, b UInt8, c UInt8, fat_string String) diff --git a/tests/queries/0_stateless/01825_type_json_1.reference b/tests/queries/0_stateless/01825_type_json_1.reference new file mode 100644 index 00000000000..857c624fb9b --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_1.reference @@ -0,0 +1,27 @@ +1 aa bb c +2 ee ff +3 foo +all_1_1_0 data Tuple(k1 String, k2 Tuple(k3 String, k4 String), k5 String) +all_2_2_0 data Tuple(k5 String) +all_1_2_1 data Tuple(k1 String, k2 Tuple(k3 String, k4 String), k5 String) +============ +1 ['aaa','ddd'] [['bbb','ccc'],['eee','fff']] +all_3_3_0 data Tuple(k1 Nested(k2 String, k3 Nested(k4 String))) +============ +1 a 42 +2 b 4200 +4242 +all_4_4_0 data Tuple(name String, value Int16) +1 a 42 +2 b 4200 +3 a 42.123 +all_4_4_0 data Tuple(name String, value Int16) +all_5_5_0 data Tuple(name String, value Float64) +1 a 42 +2 b 4200 +3 a 42.123 +4 a some +all_4_4_0 data Tuple(name String, value Int16) +all_5_5_0 data Tuple(name String, value Float64) +all_6_6_0 data Tuple(name String, value String) +all_4_6_1 data Tuple(name String, value String) diff --git a/tests/queries/0_stateless/01825_type_json_1.sql b/tests/queries/0_stateless/01825_type_json_1.sql new file mode 100644 index 00000000000..e74faf2d4c7 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_1.sql @@ -0,0 +1,85 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json; + +CREATE TABLE t_json(id UInt64, data Object('JSON')) +ENGINE = MergeTree ORDER BY tuple(); + +SYSTEM STOP MERGES t_json; + +INSERT INTO t_json FORMAT JSONEachRow {"id": 1, "data": {"k1": "aa", "k2": {"k3": "bb", "k4": "c"}}} {"id": 2, "data": {"k1": "ee", "k5": "ff"}}; +INSERT INTO t_json FORMAT JSONEachRow {"id": 3, "data": {"k5":"foo"}}; + +SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM t_json ORDER BY id; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +SYSTEM START MERGES t_json; + +OPTIMIZE TABLE t_json FINAL; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +SELECT '============'; +TRUNCATE TABLE t_json; + +INSERT INTO t_json FORMAT JSONEachRow {"id": 1, "data": {"k1":[{"k2":"aaa","k3":[{"k4":"bbb"},{"k4":"ccc"}]},{"k2":"ddd","k3":[{"k4":"eee"},{"k4":"fff"}]}]}}; +SELECT id, data.k1.k2, data.k1.k3.k4 FROM t_json ORDER BY id; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +SELECT '============'; +TRUNCATE TABLE t_json; + +SYSTEM STOP MERGES t_json; + +INSERT INTO t_json FORMAT JSONEachRow {"id": 1, "data": {"name": "a", "value": 42 }}, {"id": 2, "data": {"name": "b", "value": 4200 }}; + +SELECT id, data.name, data.value FROM t_json ORDER BY id; +SELECT sum(data.value) FROM t_json; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +INSERT INTO t_json FORMAT JSONEachRow {"id": 3, "data": {"name": "a", "value": 42.123 }}; + +SELECT id, data.name, data.value FROM t_json ORDER BY id; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +INSERT INTO t_json FORMAT JSONEachRow {"id": 4, "data": {"name": "a", "value": "some" }}; + +SELECT id, data.name, data.value FROM t_json ORDER BY id; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +SYSTEM START MERGES t_json; +OPTIMIZE TABLE t_json FINAL; + +SELECT name, column, type +FROM system.parts_columns +WHERE table = 't_json' AND database = currentDatabase() AND active AND column = 'data' +ORDER BY name; + +DROP TABLE IF EXISTS t_json; + +CREATE TABLE t_json(id UInt64, data Object('JSON')) ENGINE = Log; -- { serverError 44 } diff --git a/tests/queries/0_stateless/01825_type_json_2.reference b/tests/queries/0_stateless/01825_type_json_2.reference new file mode 100644 index 00000000000..8524035a3a4 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_2.reference @@ -0,0 +1,24 @@ +1 (1,2,0) Tuple(k1 Int8, k2 Int8, k3 Int8) +2 (0,3,4) Tuple(k1 Int8, k2 Int8, k3 Int8) +1 1 2 0 +2 0 3 4 +1 (1,2,'0') Tuple(k1 Int8, k2 Int8, k3 String) +2 (0,3,'4') Tuple(k1 Int8, k2 Int8, k3 String) +3 (0,0,'10') Tuple(k1 Int8, k2 Int8, k3 String) +4 (0,5,'str') Tuple(k1 Int8, k2 Int8, k3 String) +1 1 2 0 +2 0 3 4 +3 0 0 10 +4 0 5 str +============ +1 ([1,2,3.3]) Tuple(k1 Array(Float64)) +1 [1,2,3.3] +1 (['1','2','3.3']) Tuple(k1 Array(String)) +2 (['a','4','b']) Tuple(k1 Array(String)) +1 ['1','2','3.3'] +2 ['a','4','b'] +============ +1 ([(11,0,0),(0,22,0)]) Tuple(k1 Nested(k2 Int8, k3 Int8, k4 Int8)) +2 ([(0,33,0),(0,0,44),(0,55,66)]) Tuple(k1 Nested(k2 Int8, k3 Int8, k4 Int8)) +1 [11,0] [0,22] [0,0] +2 [0,0,0] [33,0,55] [0,44,66] diff --git a/tests/queries/0_stateless/01825_type_json_2.sql b/tests/queries/0_stateless/01825_type_json_2.sql new file mode 100644 index 00000000000..d2d26ce4106 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_2.sql @@ -0,0 +1,41 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_2; + +CREATE TABLE t_json_2(id UInt64, data Object('JSON')) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_2/t_json_2', 'r1') ORDER BY tuple(); + +INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data": {"k1": 1, "k2" : 2}} {"id": 2, "data": {"k2": 3, "k3" : 4}}; + +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; +SELECT id, data.k1, data.k2, data.k3 FROM t_json_2 ORDER BY id; + +INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 3, "data": {"k3" : 10}} {"id": 4, "data": {"k2": 5, "k3" : "str"}}; + +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; +SELECT id, data.k1, data.k2, data.k3 FROM t_json_2 ORDER BY id; + +SELECT '============'; +TRUNCATE TABLE t_json_2; + +INSERT INTO TABLE t_json_2 FORMAT JSONEachRow {"id": 1, "data": {"k1" : [1, 2, 3.3]}}; + +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; +SELECT id, data.k1 FROM t_json_2 ORDEr BY id; + +INSERT INTO TABLE t_json_2 FORMAT JSONEachRow {"id": 2, "data": {"k1" : ["a", 4, "b"]}}; + +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; +SELECT id, data.k1 FROM t_json_2 ORDER BY id; + +SELECT '============'; +TRUNCATE TABLE t_json_2; + +INSERT INTO TABLE t_json_2 FORMAT JSONEachRow {"id": 1, "data": {"k1" : [{"k2" : 11}, {"k3" : 22}]}} {"id": 2, "data": {"k1" : [{"k3" : 33}, {"k4" : 44}, {"k3" : 55, "k4" : 66}]}}; + +SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id; +SELECT id, data.k1.k2, data.k1.k3, data.k1.k4 FROM t_json_2 ORDER BY id; + +DROP TABLE t_json_2; diff --git a/tests/queries/0_stateless/01825_type_json_3.reference.j2 b/tests/queries/0_stateless/01825_type_json_3.reference.j2 new file mode 100644 index 00000000000..23f38b74fd1 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_3.reference.j2 @@ -0,0 +1,35 @@ +{% for engine in ["ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_3/t_json_3', 'r1') ORDER BY tuple()", "Memory"] -%} +1 ('',0) Tuple(k1 String, k2 Int8) +2 ('v1',2) Tuple(k1 String, k2 Int8) +1 0 +2 v1 2 +======== +1 ([]) Tuple(k1 Nested(k2 String, k3 String)) +2 ([('v1','v3'),('v4','')]) Tuple(k1 Nested(k2 String, k3 String)) +1 [] [] +2 ['v1','v4'] ['v3',''] +1 ([]) Tuple(k1 Nested(k2 String, k3 String)) +2 ([('v1','v3'),('v4','')]) Tuple(k1 Nested(k2 String, k3 String)) +3 ([]) Tuple(k1 Nested(k2 String, k3 String)) +4 ([]) Tuple(k1 Nested(k2 String, k3 String)) +1 [] [] +2 ['v1','v4'] ['v3',''] +3 [] [] +4 [] [] +{%- if 'MergeTree' in engine %} +all_2_2_0 data Tuple(k1 Nested(k2 String, k3 String)) +all_3_3_0 data Tuple(_dummy UInt8) +data Tuple(k1 Nested(k2 String, k3 String)) +{%- endif %} +1 [] [] +2 ['v1','v4'] ['v3',''] +3 [] [] +4 [] [] +======== +1 ((1,'foo'),[]) Tuple(k1 Tuple(k2 Int8, k3 String), k4 Array(Int8)) +2 ((0,''),[1,2,3]) Tuple(k1 Tuple(k2 Int8, k3 String), k4 Array(Int8)) +3 ((10,''),[]) Tuple(k1 Tuple(k2 Int8, k3 String), k4 Array(Int8)) +1 1 foo [] +2 0 [1,2,3] +3 10 [] +{% endfor -%} diff --git a/tests/queries/0_stateless/01825_type_json_3.sql.j2 b/tests/queries/0_stateless/01825_type_json_3.sql.j2 new file mode 100644 index 00000000000..62d86c3efd4 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_3.sql.j2 @@ -0,0 +1,61 @@ +-- Tags: no-fasttest + +{% for engine in ["ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_3/t_json_3', 'r1') ORDER BY tuple()", "Memory"] -%} + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_3; + +CREATE TABLE t_json_3(id UInt64, data JSON) +ENGINE = {{ engine }}; + +{% if 'MergeTree' in engine %} + SYSTEM STOP MERGES t_json_3; +{% endif %} + +INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 1, "data": {"k1": null}}, {"id": 2, "data": {"k1": "v1", "k2" : 2}}; + +SELECT id, data, toTypeName(data) FROM t_json_3 ORDER BY id; +SELECT id, data.k1, data.k2 FROM t_json_3 ORDER BY id; + +SELECT '========'; +TRUNCATE TABLE t_json_3; + +INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 1, "data": {"k1" : []}} {"id": 2, "data": {"k1" : [{"k2" : "v1", "k3" : "v3"}, {"k2" : "v4"}]}}; + +SELECT id, data, toTypeName(data) FROM t_json_3 ORDER BY id; +SELECT id, `data.k1.k2`, `data.k1.k3` FROM t_json_3 ORDER BY id; + +INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 3, "data": {"k1" : []}} {"id": 4, "data": {"k1" : []}}; + +SELECT id, data, toTypeName(data) FROM t_json_3 ORDER BY id; +SELECT id, data.k1.k2, data.k1.k3 FROM t_json_3 ORDER BY id; + +{% if 'MergeTree' in engine %} + SELECT name, column, type + FROM system.parts_columns + WHERE table = 't_json_3' AND database = currentDatabase() AND active AND column = 'data' + ORDER BY name; + + SYSTEM START MERGES t_json_3; + OPTIMIZE TABLE t_json_3 FINAL; + + SELECT column, type + FROM system.parts_columns + WHERE table = 't_json_3' AND database = currentDatabase() AND active AND column = 'data' + ORDER BY name; +{% endif %} + +SELECT id, data.k1.k2, data.k1.k3 FROM t_json_3 ORDER BY id; + +SELECT '========'; +TRUNCATE TABLE t_json_3; + +INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 1, "data": {"k1" : {"k2" : 1, "k3" : "foo"}}} {"id": 2, "data": {"k1" : null, "k4" : [1, 2, 3]}}, {"id" : 3, "data": {"k1" : {"k2" : 10}, "k4" : []}}; + +SELECT id, data, toTypeName(data) FROM t_json_3 ORDER BY id; +SELECT id, data.k1.k2, data.k1.k3, data.k4 FROM t_json_3 ORDER BY id; + +DROP TABLE t_json_3; + +{% endfor -%} diff --git a/tests/queries/0_stateless/01825_type_json_4.reference b/tests/queries/0_stateless/01825_type_json_4.reference new file mode 100644 index 00000000000..1b23bf2213e --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_4.reference @@ -0,0 +1,5 @@ +Code: 645 +Code: 15 +Code: 53 +1 ('v1') Tuple(k1 String) +1 v1 diff --git a/tests/queries/0_stateless/01825_type_json_4.sh b/tests/queries/0_stateless/01825_type_json_4.sh new file mode 100755 index 00000000000..4d81e9516c9 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_4.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_4" + +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_4(id UInt64, data JSON) \ +ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1 + +echo '{"id": 1, "data": {"k1": "v1"}}, {"id": 2, "data": {"k1": [1, 2]}}' \ + | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_4 FORMAT JSONEachRow" 2>&1 | grep -o -m1 "Code: 645" + +echo '{"id": 1, "data": {"k1": "v1"}}, {"id": 2, "data": {"k1": [{"k2" : 1}, {"k2" : 2}]}}' \ + | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_4 FORMAT JSONEachRow" 2>&1 | grep -o -m1 "Code: 15" + +echo '{"id": 1, "data": {"k1": "v1"}}' \ + | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_4 FORMAT JSONEachRow" + +echo '{"id": 2, "data": {"k1": [1, 2]}}' \ + | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_4 FORMAT JSONEachRow" 2>&1 | grep -o -m1 "Code: 53" + +$CLICKHOUSE_CLIENT -q "SELECT id, data, toTypeName(data) FROM t_json_4" +$CLICKHOUSE_CLIENT -q "SELECT id, data.k1 FROM t_json_4 ORDER BY id" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_json_4" diff --git a/tests/queries/0_stateless/01825_type_json_5.reference b/tests/queries/0_stateless/01825_type_json_5.reference new file mode 100644 index 00000000000..4ac0aa26ffd --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_5.reference @@ -0,0 +1,5 @@ +{"a.b":1,"a.c":2} +{"s":{"a.b":1,"a.c":2}} +1 [22,33] +2 qqq [44] +Tuple(k1 Int8, k2 Tuple(k3 String, k4 Array(Int8))) diff --git a/tests/queries/0_stateless/01825_type_json_5.sql b/tests/queries/0_stateless/01825_type_json_5.sql new file mode 100644 index 00000000000..b939a960e32 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_5.sql @@ -0,0 +1,23 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +SELECT '{"a": {"b": 1, "c": 2}}'::JSON AS s; +SELECT '{"a": {"b": 1, "c": 2}}'::JSON AS s format JSONEachRow; + +DROP TABLE IF EXISTS t_json_5; +DROP TABLE IF EXISTS t_json_str_5; + +CREATE TABLE t_json_str_5 (data String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t_json_5 (data JSON) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_json_str_5 FORMAT JSONAsString {"k1": 1, "k2": {"k4": [22, 33]}}, {"k1": 2, "k2": {"k3": "qqq", "k4": [44]}} +; + +INSERT INTO t_json_5 SELECT data FROM t_json_str_5; + +SELECT data.k1, data.k2.k3, data.k2.k4 FROM t_json_5 ORDER BY data.k1; +SELECT DISTINCT toTypeName(data) FROM t_json_5; + +DROP TABLE t_json_5; +DROP TABLE t_json_str_5; diff --git a/tests/queries/0_stateless/01825_type_json_6.reference b/tests/queries/0_stateless/01825_type_json_6.reference new file mode 100644 index 00000000000..7fcd2a40826 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_6.reference @@ -0,0 +1,3 @@ +Tuple(key String, out Nested(outputs Nested(index Int32, n Int8), type Int8, value Int8)) +v1 [0,0] [1,2] [[],[1960131]] [[],[0]] +v2 [1,1] [4,3] [[1881212],[]] [[1],[]] diff --git a/tests/queries/0_stateless/01825_type_json_6.sh b/tests/queries/0_stateless/01825_type_json_6.sh new file mode 100755 index 00000000000..8bbb1abee4a --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_6.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_6;" + +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_6 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1 + +cat < notEmpty(x), outpoints)" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS btc" diff --git a/tests/queries/0_stateless/01825_type_json_describe.reference b/tests/queries/0_stateless/01825_type_json_describe.reference new file mode 100644 index 00000000000..629b60cb629 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_describe.reference @@ -0,0 +1,3 @@ +data Object(\'json\') +data Tuple(k1 Int8) +data Tuple(k1 String, k2 Array(Int8)) diff --git a/tests/queries/0_stateless/01825_type_json_describe.sql b/tests/queries/0_stateless/01825_type_json_describe.sql new file mode 100644 index 00000000000..cd7c4ff8c8c --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_describe.sql @@ -0,0 +1,21 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + + +DROP TABLE IF EXISTS t_json_desc; + +CREATE TABLE t_json_desc (data JSON) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_json_desc FORMAT JSONAsObject {"k1": 10} +; + +DESC TABLE t_json_desc; +DESC TABLE t_json_desc SETTINGS describe_extend_object_types = 1; + +INSERT INTO t_json_desc FORMAT JSONAsObject {"k1": "q", "k2": [1, 2, 3]} +; + +DESC TABLE t_json_desc SETTINGS describe_extend_object_types = 1; + +DROP TABLE IF EXISTS t_json_desc; diff --git a/tests/queries/0_stateless/01825_type_json_distributed.reference b/tests/queries/0_stateless/01825_type_json_distributed.reference new file mode 100644 index 00000000000..9ae85ac888c --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_distributed.reference @@ -0,0 +1,4 @@ +(2,('qqq',[44,55])) Tuple(k1 Int8, k2 Tuple(k3 String, k4 Array(Int8))) +(2,('qqq',[44,55])) Tuple(k1 Int8, k2 Tuple(k3 String, k4 Array(Int8))) +2 qqq [44,55] +2 qqq [44,55] diff --git a/tests/queries/0_stateless/01825_type_json_distributed.sql b/tests/queries/0_stateless/01825_type_json_distributed.sql new file mode 100644 index 00000000000..70cc0743556 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_distributed.sql @@ -0,0 +1,18 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_local; +DROP TABLE IF EXISTS t_json_dist; + +CREATE TABLE t_json_local(data JSON) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t_json_dist AS t_json_local ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_json_local); + +INSERT INTO t_json_local FORMAT JSONAsObject {"k1": 2, "k2": {"k3": "qqq", "k4": [44, 55]}} +; + +SELECT data, toTypeName(data) FROM t_json_dist; +SELECT data.k1, data.k2.k3, data.k2.k4 FROM t_json_dist; + +DROP TABLE IF EXISTS t_json_local; +DROP TABLE IF EXISTS t_json_dist; diff --git a/tests/queries/0_stateless/01825_type_json_field.reference b/tests/queries/0_stateless/01825_type_json_field.reference new file mode 100644 index 00000000000..b5637b1fbb7 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_field.reference @@ -0,0 +1,12 @@ +1 10 a +Tuple(a UInt8, s String) +1 10 a 0 +2 sss b 300 +3 20 c 0 +Tuple(a String, b UInt16, s String) +1 10 a 0 +2 sss b 300 +3 20 c 0 +4 30 400 +5 0 qqq 0 foo +Tuple(a String, b UInt16, s String, t String) diff --git a/tests/queries/0_stateless/01825_type_json_field.sql b/tests/queries/0_stateless/01825_type_json_field.sql new file mode 100644 index 00000000000..6c906023cef --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_field.sql @@ -0,0 +1,28 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_field; + +CREATE TABLE t_json_field (id UInt32, data JSON) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_json_field VALUES (1, (10, 'a')::Tuple(a UInt32, s String)); + +SELECT id, data.a, data.s FROM t_json_field ORDER BY id; +SELECT DISTINCT toTypeName(data) FROM t_json_field; + +INSERT INTO t_json_field VALUES (2, ('sss', 300, 'b')::Tuple(a String, b UInt64, s String)), (3, (20, 'c')::Tuple(a UInt32, s String)); + +SELECT id, data.a, data.s, data.b FROM t_json_field ORDER BY id; +SELECT DISTINCT toTypeName(data) FROM t_json_field; + +INSERT INTO t_json_field VALUES (4, map('a', 30, 'b', 400)), (5, map('s', 'qqq', 't', 'foo')); + +SELECT id, data.a, data.s, data.b, data.t FROM t_json_field ORDER BY id; +SELECT DISTINCT toTypeName(data) FROM t_json_field; + +INSERT INTO t_json_field VALUES (6, map(1, 2, 3, 4)); -- { clientError 53 } +INSERT INTO t_json_field VALUES (6, (1, 2, 3)); -- { clientError 53 } + +DROP TABLE t_json_field; diff --git a/tests/queries/0_stateless/01825_type_json_ghdata.reference b/tests/queries/0_stateless/01825_type_json_ghdata.reference new file mode 100644 index 00000000000..3418121ad43 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_ghdata.reference @@ -0,0 +1,20 @@ +5000 +839 +String 562 +Array 134 +UInt64 63 +Int32 47 +Int8 17 +Int16 15 +Int64 1 +leonardomso/33-js-concepts 3 +ytdl-org/youtube-dl 3 +Bogdanp/neko 2 +bminossi/AllVideoPocsFromHackerOne 2 +disclose/diodata 2 +Commit 182 +chipeo345 119 +phanwi346 114 +Nicholas Piggin 95 +direwolf-github 49 +2 diff --git a/tests/queries/0_stateless/01825_type_json_ghdata.sh b/tests/queries/0_stateless/01825_type_json_ghdata.sh new file mode 100755 index 00000000000..7486571cc22 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_ghdata.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1 + +cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata FORMAT JSONAsObject" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ghdata WHERE NOT ignore(*)" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT length(subcolumns.names) \ + FROM system.parts_columns \ + WHERE table = 'ghdata' AND database = '$CLICKHOUSE_DATABASE'" + +${CLICKHOUSE_CLIENT} -q "WITH position(full_type, '(') AS pos +SELECT if(pos = 0, full_type, substring(full_type, 1, pos - 1)) AS type, count() AS c \ + FROM system.parts_columns ARRAY JOIN subcolumns.types AS full_type \ + WHERE table = 'ghdata' AND database = '$CLICKHOUSE_DATABASE' \ + GROUP BY type ORDER BY c DESC" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT data.repo.name, count() AS stars FROM ghdata \ + WHERE data.type = 'WatchEvent' GROUP BY data.repo.name ORDER BY stars DESC, data.repo.name LIMIT 5" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT data.payload.commits.author.name AS name, count() AS c FROM ghdata \ + ARRAY JOIN data.payload.commits.author.name \ + GROUP BY name ORDER BY c DESC, name LIMIT 5" + +${CLICKHOUSE_CLIENT} -q "SELECT max(data.payload.pull_request.assignees.size0) FROM ghdata" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata" diff --git a/tests/queries/0_stateless/01825_type_json_insert_select.reference b/tests/queries/0_stateless/01825_type_json_insert_select.reference new file mode 100644 index 00000000000..8283cc5af48 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_insert_select.reference @@ -0,0 +1,12 @@ +Tuple(k1 Int8, k2 String) +1 (1,'foo') +Tuple(k1 Int8, k2 String, k3 String) +1 (1,'foo','') +2 (2,'bar','') +3 (3,'','aaa') +Tuple(arr Nested(k11 Int8, k22 String, k33 Int8), k1 Int8, k2 String, k3 String) +1 ([],1,'foo','') +2 ([],2,'bar','') +3 ([],3,'','aaa') +4 ([(5,'6',0),(7,'0',8)],0,'','') +5 ([(0,'str1',0)],0,'','') diff --git a/tests/queries/0_stateless/01825_type_json_insert_select.sql b/tests/queries/0_stateless/01825_type_json_insert_select.sql new file mode 100644 index 00000000000..8bb03f84f5a --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_insert_select.sql @@ -0,0 +1,36 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS type_json_src; +DROP TABLE IF EXISTS type_json_dst; + +CREATE TABLE type_json_src (id UInt32, data JSON) ENGINE = MergeTree ORDER BY id; +CREATE TABLE type_json_dst AS type_json_src; + +INSERT INTO type_json_src VALUES (1, '{"k1": 1, "k2": "foo"}'); +INSERT INTO type_json_dst SELECT * FROM type_json_src; + +SELECT DISTINCT toTypeName(data) FROM type_json_dst; +SELECT id, data FROM type_json_dst ORDER BY id; + +INSERT INTO type_json_src VALUES (2, '{"k1": 2, "k2": "bar"}') (3, '{"k1": 3, "k3": "aaa"}'); +INSERT INTO type_json_dst SELECT * FROM type_json_src WHERE id > 1; + +SELECT DISTINCT toTypeName(data) FROM type_json_dst; +SELECT id, data FROM type_json_dst ORDER BY id; + +INSERT INTO type_json_dst VALUES (4, '{"arr": [{"k11": 5, "k22": 6}, {"k11": 7, "k33": 8}]}'); + +INSERT INTO type_json_src VALUES (5, '{"arr": "not array"}'); +INSERT INTO type_json_dst SELECT * FROM type_json_src WHERE id = 5; -- { serverError 15 } + +TRUNCATE TABLE type_json_src; +INSERT INTO type_json_src VALUES (5, '{"arr": [{"k22": "str1"}]}') +INSERT INTO type_json_dst SELECT * FROM type_json_src WHERE id = 5; + +SELECT DISTINCT toTypeName(data) FROM type_json_dst; +SELECT id, data FROM type_json_dst ORDER BY id; + +DROP TABLE type_json_src; +DROP TABLE type_json_dst; diff --git a/tests/queries/0_stateless/01825_type_json_nbagames.reference b/tests/queries/0_stateless/01825_type_json_nbagames.reference new file mode 100644 index 00000000000..8f86bfe613e --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_nbagames.reference @@ -0,0 +1,12 @@ +1000 +data Tuple(_id Tuple(`$oid` String), date Tuple(`$date` String), teams Nested(abbreviation String, city String, home UInt64, name String, players Nested(ast Int8, blk Int8, drb Int8, fg Int8, fg3 Int8, fg3_pct String, fg3a Int8, fg_pct String, fga Int8, ft Int8, ft_pct String, fta Int8, mp String, orb Int8, pf Int8, player String, pts Int8, stl Int8, tov Int8, trb Int8), results Tuple(ast Int8, blk Int8, drb Int8, fg Int8, fg3 Int8, fg3_pct String, fg3a Int8, fg_pct String, fga Int8, ft Int8, ft_pct String, fta Int8, mp Int16, orb Int8, pf Int8, pts Int16, stl Int8, tov Int8, trb Int8), score Int16, won Int8)) +Boston Celtics 70 +Los Angeles Lakers 64 +Milwaukee Bucks 61 +Philadelphia 76ers 57 +Atlanta Hawks 55 +Larry Bird 10 +Clyde Drexler 4 +Alvin Robertson 3 +Magic Johnson 3 +Charles Barkley 2 diff --git a/tests/queries/0_stateless/01825_type_json_nbagames.sh b/tests/queries/0_stateless/01825_type_json_nbagames.sh new file mode 100755 index 00000000000..18e7c050680 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_nbagames.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1 + +cat $CUR_DIR/data_json/nbagames_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames FORMAT JSONAsObject" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM nbagames WHERE NOT ignore(*)" +${CLICKHOUSE_CLIENT} -q "DESC nbagames SETTINGS describe_extend_object_types = 1" + +${CLICKHOUSE_CLIENT} -q \ + "SELECT teams.name AS name, sum(teams.won) AS wins FROM nbagames \ + ARRAY JOIN data.teams AS teams GROUP BY name \ + ORDER BY wins DESC LIMIT 5;" + +${CLICKHOUSE_CLIENT} -q \ +"SELECT player, sum(triple_double) AS triple_doubles FROM \ +( \ + SELECT \ + tupleElement(players, 'player') AS player, \ + ((tupleElement(players, 'pts') >= 10) + \ + (tupleElement(players, 'ast') >= 10) + \ + (tupleElement(players, 'blk') >= 10) + \ + (tupleElement(players, 'stl') >= 10) + \ + (tupleElement(players, 'trb') >= 10)) >= 3 AS triple_double \ + FROM \ + ( \ + SELECT arrayJoin(arrayJoin(data.teams.players)) as players from nbagames \ + ) \ +) \ +GROUP BY player ORDER BY triple_doubles DESC, player LIMIT 5" + + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames" diff --git a/tests/queries/0_stateless/01825_type_json_nullable.reference b/tests/queries/0_stateless/01825_type_json_nullable.reference new file mode 100644 index 00000000000..587fb1b1bc9 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_nullable.reference @@ -0,0 +1,17 @@ +1 (1,2,NULL) Tuple(k1 Nullable(Int8), k2 Nullable(Int8), k3 Nullable(Int8)) +2 (NULL,3,4) Tuple(k1 Nullable(Int8), k2 Nullable(Int8), k3 Nullable(Int8)) +1 1 2 \N +2 \N 3 4 +1 (1,2,NULL) Tuple(k1 Nullable(Int8), k2 Nullable(Int8), k3 Nullable(String)) +2 (NULL,3,'4') Tuple(k1 Nullable(Int8), k2 Nullable(Int8), k3 Nullable(String)) +3 (NULL,NULL,'10') Tuple(k1 Nullable(Int8), k2 Nullable(Int8), k3 Nullable(String)) +4 (NULL,5,'str') Tuple(k1 Nullable(Int8), k2 Nullable(Int8), k3 Nullable(String)) +1 1 2 \N +2 \N 3 4 +3 \N \N 10 +4 \N 5 str +============ +1 ([(11,NULL,NULL),(NULL,22,NULL)]) Tuple(k1 Nested(k2 Nullable(Int8), k3 Nullable(Int8), k4 Nullable(Int8))) +2 ([(NULL,33,NULL),(NULL,NULL,44),(NULL,55,66)]) Tuple(k1 Nested(k2 Nullable(Int8), k3 Nullable(Int8), k4 Nullable(Int8))) +1 [11,NULL] [NULL,22] [NULL,NULL] +2 [NULL,NULL,NULL] [33,NULL,55] [NULL,44,66] diff --git a/tests/queries/0_stateless/01825_type_json_nullable.sql b/tests/queries/0_stateless/01825_type_json_nullable.sql new file mode 100644 index 00000000000..65589243f43 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_nullable.sql @@ -0,0 +1,28 @@ +-- Tags: no-fasttest + +SET allow_experimental_object_type = 1; + +DROP TABLE IF EXISTS t_json_null; + +CREATE TABLE t_json_null(id UInt64, data Object(Nullable('JSON'))) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_json_null FORMAT JSONEachRow {"id": 1, "data": {"k1": 1, "k2" : 2}} {"id": 2, "data": {"k2": 3, "k3" : 4}}; + +SELECT id, data, toTypeName(data) FROM t_json_null ORDER BY id; +SELECT id, data.k1, data.k2, data.k3 FROM t_json_null ORDER BY id; + +INSERT INTO t_json_null FORMAT JSONEachRow {"id": 3, "data": {"k3" : 10}} {"id": 4, "data": {"k2": 5, "k3" : "str"}}; + +SELECT id, data, toTypeName(data) FROM t_json_null ORDER BY id; +SELECT id, data.k1, data.k2, data.k3 FROM t_json_null ORDER BY id; + +SELECT '============'; +TRUNCATE TABLE t_json_null; + +INSERT INTO TABLE t_json_null FORMAT JSONEachRow {"id": 1, "data": {"k1" : [{"k2" : 11}, {"k3" : 22}]}} {"id": 2, "data": {"k1" : [{"k3" : 33}, {"k4" : 44}, {"k3" : 55, "k4" : 66}]}}; + +SELECT id, data, toTypeName(data) FROM t_json_null ORDER BY id; +SELECT id, data.k1.k2, data.k1.k3, data.k1.k4 FROM t_json_null ORDER BY id; + +DROP TABLE t_json_null; diff --git a/tests/queries/0_stateless/01825_type_json_parallel_insert.reference b/tests/queries/0_stateless/01825_type_json_parallel_insert.reference new file mode 100644 index 00000000000..ac512064a43 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_parallel_insert.reference @@ -0,0 +1 @@ +Tuple(k1 Int8, k2 String) 3000000 diff --git a/tests/queries/0_stateless/01825_type_json_parallel_insert.sql b/tests/queries/0_stateless/01825_type_json_parallel_insert.sql new file mode 100644 index 00000000000..f54004a6630 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_parallel_insert.sql @@ -0,0 +1,10 @@ +-- Tags: long +DROP TABLE IF EXISTS t_json_parallel; + +SET allow_experimental_object_type = 1, max_insert_threads = 20, max_threads = 20; +CREATE TABLE t_json_parallel (data JSON) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_json_parallel SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers_mt(3000000); +SELECT any(toTypeName(data)), count() FROM t_json_parallel; + +DROP TABLE t_json_parallel; diff --git a/tests/queries/0_stateless/01825_type_json_schema_race_long.reference b/tests/queries/0_stateless/01825_type_json_schema_race_long.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_schema_race_long.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/01825_type_json_schema_race_long.sh b/tests/queries/0_stateless/01825_type_json_schema_race_long.sh new file mode 100755 index 00000000000..38d1432cef6 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_schema_race_long.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_race" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_race (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1 + +function test_case() +{ + $CLICKHOUSE_CLIENT -q "TRUNCATE TABLE t_json_race" + + echo '{"data": {"k1": 1, "k2": 2}}' | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_race FORMAT JSONEachRow" + + pids=() + for _ in {1..5}; do + $CLICKHOUSE_CLIENT -q "SELECT * FROM t_json_race WHERE 0 IN (SELECT sleep(0.05)) FORMAT Null" & + pids+=($!) + done + + echo '{"data": {"k1": "str", "k2": "str1"}}' | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_race FORMAT JSONEachRow" & + + for pid in "${pids[@]}"; do + wait "$pid" || exit 1 + done +} + +for _ in {1..30}; do test_case; done + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_race" +echo OK diff --git a/tests/queries/0_stateless/01854_HTTP_dict_decompression.python b/tests/queries/0_stateless/01854_HTTP_dict_decompression.python index 929eaae8067..4f6878665aa 100644 --- a/tests/queries/0_stateless/01854_HTTP_dict_decompression.python +++ b/tests/queries/0_stateless/01854_HTTP_dict_decompression.python @@ -158,7 +158,7 @@ def test_select(dict_name="", schema="word String, counter UInt32", requests=[], COMPRESS_METHOD = requests[i] print(i, COMPRESS_METHOD, ADDING_ENDING, SEND_ENCODING) - check_answers("select * from {}".format(dict_name), answers[i]) + check_answers("SELECT * FROM {} ORDER BY word".format(dict_name), answers[i]) def main(): # first three for encoding, second three for url @@ -171,7 +171,7 @@ def main(): ] # This answers got experemently in non compressed mode and they are correct - answers = ['''This 152\nHello 1\nis 9283\ndata 555\nWorld 2\ntesting 2313213'''] * 5 + answers = ['''Hello 1\nThis 152\nWorld 2\ndata 555\nis 9283\ntesting 2313213'''] * 5 t = start_server(len(insert_requests)) t.start() diff --git a/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 b/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 index a51f4c856f3..1704fedb92b 100644 --- a/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 +++ b/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 @@ -24,9 +24,9 @@ SET join_algorithm = 'partial_merge'; SELECT '-- partial_merge --'; SELECT '--'; -SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2; +SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 ORDER BY t1.key, t1.key2; SELECT '--'; -SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2; +SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2 ORDER BY t1.key, t1.key2; SELECT '--'; SELECT t1.key FROM t1 INNER ANY JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2; diff --git a/tests/queries/0_stateless/01889_sqlite_read_write.sh b/tests/queries/0_stateless/01889_sqlite_read_write.sh index 247f44b61e7..fc87aa08fa7 100755 --- a/tests/queries/0_stateless/01889_sqlite_read_write.sh +++ b/tests/queries/0_stateless/01889_sqlite_read_write.sh @@ -19,7 +19,6 @@ DB_PATH2=$CUR_DIR/${CURR_DATABASE}_db2 function cleanup() { ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" - rm -r "${DB_PATH}" "${DB_PATH2}" } trap cleanup EXIT diff --git a/tests/queries/0_stateless/01917_prewhere_column_type.sql b/tests/queries/0_stateless/01917_prewhere_column_type.sql index 5147e6093a9..c0bc0c3e36b 100644 --- a/tests/queries/0_stateless/01917_prewhere_column_type.sql +++ b/tests/queries/0_stateless/01917_prewhere_column_type.sql @@ -1,3 +1,5 @@ +SET optimize_move_to_prewhere = 1; + DROP TABLE IF EXISTS t1; CREATE TABLE t1 ( s String, f Float32, e UInt16 ) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '100G'; diff --git a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh index 591305866a6..23df052a8d6 100755 --- a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh +++ b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh @@ -27,38 +27,34 @@ done function optimize_thread { - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE ttl_table$REPLICA FINAl" - done + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE ttl_table$REPLICA FINAl" } function insert_thread { - while true; do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" - $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" - $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" - done + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" + $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" + $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" } -export -f insert_thread; -export -f optimize_thread; +export -f insert_thread +export -f optimize_thread TIMEOUT=30 -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & wait for i in $(seq 1 $NUM_REPLICAS); do diff --git a/tests/queries/0_stateless/01921_datatype_date32.reference b/tests/queries/0_stateless/01921_datatype_date32.reference index 2114f6f6b1e..8beaefbeb38 100644 --- a/tests/queries/0_stateless/01921_datatype_date32.reference +++ b/tests/queries/0_stateless/01921_datatype_date32.reference @@ -221,13 +221,13 @@ 1925-04-01 1925-04-01 2283-03-31 -1925-01-01 +2283-11-11 2021-09-22 -------addYears--------- 1926-01-01 1926-01-01 2283-11-11 -1925-01-01 +2283-11-11 2022-06-22 -------subtractSeconds--------- 1925-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/01921_test_progress_bar.py b/tests/queries/0_stateless/01921_test_progress_bar.py index a95d5994607..3b0b429d396 100755 --- a/tests/queries/0_stateless/01921_test_progress_bar.py +++ b/tests/queries/0_stateless/01921_test_progress_bar.py @@ -4,16 +4,16 @@ import sys import signal CURDIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(CURDIR, 'helpers')) +sys.path.insert(0, os.path.join(CURDIR, "helpers")) from client import client, prompt, end_of_block log = None # uncomment the line below for debugging -#log=sys.stdout +# log=sys.stdout -with client(name='client1>', log=log) as client1: +with client(name="client1>", log=log) as client1: client1.expect(prompt) - client1.send('SELECT number FROM numbers(100) FORMAT Null') - client1.expect('Progress: 100\.00 rows, 800\.00 B.*' + end_of_block) - client1.expect('0 rows in set. Elapsed: [\\w]{1}\.[\\w]{3} sec.' + end_of_block) + client1.send("SELECT number FROM numbers(100) FORMAT Null") + client1.expect("Progress: 100\.00 rows, 800\.00 B.*" + end_of_block) + client1.expect("0 rows in set. Elapsed: [\\w]{1}\.[\\w]{3} sec." + end_of_block) diff --git a/tests/queries/0_stateless/01926_order_by_desc_limit.sql b/tests/queries/0_stateless/01926_order_by_desc_limit.sql index 7ea102e11e9..9f65cf73252 100644 --- a/tests/queries/0_stateless/01926_order_by_desc_limit.sql +++ b/tests/queries/0_stateless/01926_order_by_desc_limit.sql @@ -1,5 +1,9 @@ +-- Tags: no-random-settings + DROP TABLE IF EXISTS order_by_desc; +SET remote_fs_enable_cache=0; + CREATE TABLE order_by_desc (u UInt32, s String) ENGINE MergeTree ORDER BY u PARTITION BY u % 100 SETTINGS index_granularity = 1024; diff --git a/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference index b856b079327..9896f9396b6 100644 --- a/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference +++ b/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference @@ -38,8 +38,16 @@ select _shard_num, * from remote('127.{1..4}', view(select toUInt8(id) id from d 1 0 1 0 1 0 +1 128 2 1 4 127 +4 255 +4 255 +4 255 +4 255 +4 255 +4 255 +4 255 -- Int16, Int16 select _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toInt16(id)) where id in (0, 1, 0x7fff) order by _shard_num, id; 1 0 @@ -72,8 +80,14 @@ select _shard_num, * from remote('127.{1..4}', view(select toUInt16(id) id from 1 0 1 0 1 0 +1 32768 2 1 4 32767 +4 65535 +4 65535 +4 65535 +4 65535 +4 65535 -- Int32, Int32 select _shard_num, * from remote('127.{1..4}', view(select toInt32(id) id from data), toInt32(id)) where id in (0, 1, 0x7fffffff) order by _shard_num, id; 1 0 @@ -100,8 +114,12 @@ select _shard_num, * from remote('127.{1..4}', view(select toUInt32(id) id from select _shard_num, * from remote('127.{1..4}', view(select toUInt32(id) id from data), toInt32(id)) where id in (0, 1, 0x7fffffff, 0x80000000, 0xffffffff) order by _shard_num, id; 1 0 1 0 +1 2147483648 2 1 4 2147483647 +4 4294967295 +4 4294967295 +4 4294967295 -- Int64, Int64 select _shard_num, * from remote('127.{1..4}', view(select toInt64(id) id from data), toInt64(id)) where id in (0, 1, 0x7fffffffffffffff) order by _shard_num, id; 1 0 @@ -122,8 +140,10 @@ select _shard_num, * from remote('127.{1..4}', view(select toUInt64(id) id from -- UInt64, Int64 select _shard_num, * from remote('127.{1..4}', view(select toUInt64(id) id from data), toInt64(id)) where id in (0, 1, 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff) order by _shard_num, id; 1 0 +1 9223372036854775808 2 1 4 9223372036854775807 +4 18446744073709551615 -- modulo(Int8) select distinct _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toInt8(id)%255) where id in (-1) order by _shard_num, id; 4 -1 diff --git a/tests/queries/0_stateless/01942_create_table_with_sample.sql b/tests/queries/0_stateless/01942_create_table_with_sample.sql index 6320edd7a31..c4ededae4ca 100644 --- a/tests/queries/0_stateless/01942_create_table_with_sample.sql +++ b/tests/queries/0_stateless/01942_create_table_with_sample.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check:21.9.1.1 + CREATE TABLE IF NOT EXISTS sample_incorrect (`x` UUID) ENGINE = MergeTree diff --git a/tests/queries/0_stateless/01943_non_deterministic_order_key.sql b/tests/queries/0_stateless/01943_non_deterministic_order_key.sql index 200a88ec677..8b6abebe4da 100644 --- a/tests/queries/0_stateless/01943_non_deterministic_order_key.sql +++ b/tests/queries/0_stateless/01943_non_deterministic_order_key.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check:21.9.1.1 + CREATE TABLE a (number UInt64) ENGINE = MergeTree ORDER BY if(now() > toDateTime('2020-06-01 13:31:40'), toInt64(number), -number); -- { serverError 36 } CREATE TABLE b (number UInt64) ENGINE = MergeTree ORDER BY now() > toDateTime(number); -- { serverError 36 } CREATE TABLE c (number UInt64) ENGINE = MergeTree ORDER BY now(); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01943_query_id_check.sql b/tests/queries/0_stateless/01943_query_id_check.sql index cb2ef090854..ad9e88e0478 100644 --- a/tests/queries/0_stateless/01943_query_id_check.sql +++ b/tests/queries/0_stateless/01943_query_id_check.sql @@ -1,6 +1,8 @@ -- Tags: no-replicated-database -- Tag no-replicated-database: Different query_id +SET prefer_localhost_replica = 1; + DROP TABLE IF EXISTS tmp; CREATE TABLE tmp ENGINE = TinyLog AS SELECT queryID(); diff --git a/tests/queries/0_stateless/01951_distributed_push_down_limit.sql b/tests/queries/0_stateless/01951_distributed_push_down_limit.sql index fa2fc1800c1..184e6321988 100644 --- a/tests/queries/0_stateless/01951_distributed_push_down_limit.sql +++ b/tests/queries/0_stateless/01951_distributed_push_down_limit.sql @@ -1,5 +1,7 @@ -- Tags: distributed +set prefer_localhost_replica = 1; + -- { echo } explain select * from remote('127.{1,2}', view(select * from numbers(1e6))) order by number limit 10 settings distributed_push_down_limit=0; explain select * from remote('127.{1,2}', view(select * from numbers(1e6))) order by number limit 10 settings distributed_push_down_limit=1; diff --git a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql index d1f80b42e75..74b55b95315 100644 --- a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql +++ b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql @@ -2,6 +2,7 @@ set optimize_skip_unused_shards=1; set optimize_distributed_group_by_sharding_key=1; +set prefer_localhost_replica=1; -- { echo } explain select distinct k1 from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- not optimized diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.sql b/tests/queries/0_stateless/02006_test_positional_arguments.sql index 54b55c4a9f8..7442ca6bbf6 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.sql +++ b/tests/queries/0_stateless/02006_test_positional_arguments.sql @@ -1,3 +1,4 @@ +set group_by_two_level_threshold = 100000; set enable_positional_arguments = 1; drop table if exists test; diff --git a/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.reference b/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.reference index 8a4df1605fb..8da82a0726f 100644 --- a/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.reference +++ b/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.reference @@ -4,3 +4,4 @@ 2001:db8:0:85a3::ac1f:8001 String 0.0.0.0 IPv4 :: IPv6 +::ffff:1.1.1.1 IPv6 diff --git a/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.sql b/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.sql index 2fcc20b9811..b303d580e72 100644 --- a/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.sql +++ b/tests/queries/0_stateless/02007_ipv4_and_ipv6_to_and_from_string.sql @@ -4,10 +4,10 @@ SELECT CAST(toIPv4('127.0.0.1') as String) as v, toTypeName(v); SELECT CAST('2001:0db8:0000:85a3:0000:0000:ac1f:8001' as IPv6) as v, toTypeName(v); SELECT CAST(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001') as String) as v, toTypeName(v); -SELECT toIPv4('hello') as v, toTypeName(v); -SELECT toIPv6('hello') as v, toTypeName(v); +SELECT toIPv4OrDefault('hello') as v, toTypeName(v); +SELECT toIPv6OrDefault('hello') as v, toTypeName(v); SELECT CAST('hello' as IPv4) as v, toTypeName(v); -- { serverError CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING } SELECT CAST('hello' as IPv6) as v, toTypeName(v); -- { serverError CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING } -SELECT CAST('1.1.1.1' as IPv6) as v, toTypeName(v); -- { serverError CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING } +SELECT CAST('1.1.1.1' as IPv6) as v, toTypeName(v); diff --git a/tests/queries/0_stateless/02015_async_inserts_stress_long.sh b/tests/queries/0_stateless/02015_async_inserts_stress_long.sh index f9a58818404..6e17012a797 100755 --- a/tests/queries/0_stateless/02015_async_inserts_stress_long.sh +++ b/tests/queries/0_stateless/02015_async_inserts_stress_long.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash +# Tags: no-random-settings + set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh + function insert1() { url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=0" @@ -24,26 +27,28 @@ function insert2() done } +function insert3() +{ + url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=0" + while true; do + ${CLICKHOUSE_CURL} -sS "$url" -d "INSERT INTO FUNCTION remote('127.0.0.1', $CLICKHOUSE_DATABASE, async_inserts) VALUES (7, 'g') (8, 'h')" + done +} + function select1() { - while true; do - ${CLICKHOUSE_CLIENT} -q "SELECT * FROM async_inserts FORMAT Null" - done + ${CLICKHOUSE_CLIENT} -q "SELECT * FROM async_inserts FORMAT Null" } function select2() { - while true; do - ${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.asynchronous_inserts FORMAT Null" - done + ${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.asynchronous_inserts FORMAT Null" } function truncate1() { - while true; do - sleep 0.1 - ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE async_inserts" - done + sleep 0.1 + ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE async_inserts" } ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts" @@ -53,6 +58,7 @@ TIMEOUT=10 export -f insert1 export -f insert2 +export -f insert3 export -f select1 export -f select2 export -f truncate1 @@ -60,11 +66,12 @@ export -f truncate1 for _ in {1..5}; do timeout $TIMEOUT bash -c insert1 & timeout $TIMEOUT bash -c insert2 & + timeout $TIMEOUT bash -c insert3 & done -timeout $TIMEOUT bash -c select1 & -timeout $TIMEOUT bash -c select2 & -timeout $TIMEOUT bash -c truncate1 & +clickhouse_client_loop_timeout $TIMEOUT select1 & +clickhouse_client_loop_timeout $TIMEOUT select2 & +clickhouse_client_loop_timeout $TIMEOUT truncate1 & wait echo "OK" diff --git a/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh b/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh index 256297e253e..062b8512bff 100755 --- a/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh +++ b/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: shard +# Tags: shard, no-fasttest # This test reproduces crash in case of insufficient coroutines stack size @@ -19,26 +19,24 @@ $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zst function select_thread() { - while true; do - $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM local" >/dev/null - $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM distributed" >/dev/null - done + $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM local" >/dev/null + $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM distributed" >/dev/null } -export -f select_thread; +export -f select_thread TIMEOUT=30 -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & -timeout $TIMEOUT bash -c select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/02022_storage_filelog_one_file.sh b/tests/queries/0_stateless/02022_storage_filelog_one_file.sh index 8ae0ce0ec1c..2f47001eda9 100755 --- a/tests/queries/0_stateless/02022_storage_filelog_one_file.sh +++ b/tests/queries/0_stateless/02022_storage_filelog_one_file.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-backward-compatibility-check set -eu diff --git a/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh b/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh index a475913b7d2..f0faafe55d5 100755 --- a/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh +++ b/tests/queries/0_stateless/02025_storage_filelog_virtual_col.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-backward-compatibility-check set -eu diff --git a/tests/queries/0_stateless/02030_rocksdb_race_long.sh b/tests/queries/0_stateless/02030_rocksdb_race_long.sh index 88c30852c86..ec20007892a 100755 --- a/tests/queries/0_stateless/02030_rocksdb_race_long.sh +++ b/tests/queries/0_stateless/02030_rocksdb_race_long.sh @@ -11,38 +11,29 @@ set -o errexit set -o pipefail echo " - DROP TABLE IF EXISTS rocksdb_race; - CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); + DROP TABLE IF EXISTS rocksdb_race; + CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000); " | $CLICKHOUSE_CLIENT -n function read_stat_thread() { - while true; do - echo " - SELECT * FROM system.rocksdb FORMAT Null; - " | $CLICKHOUSE_CLIENT -n - done + $CLICKHOUSE_CLIENT -n -q 'SELECT * FROM system.rocksdb FORMAT Null' } function truncate_thread() { - while true; do - sleep 3s; - echo " - TRUNCATE TABLE rocksdb_race; - " | $CLICKHOUSE_CLIENT -n - done + sleep 3s; + $CLICKHOUSE_CLIENT -n -q 'TRUNCATE TABLE rocksdb_race' } -# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout -export -f read_stat_thread; -export -f truncate_thread; +export -f read_stat_thread +export -f truncate_thread TIMEOUT=20 -timeout $TIMEOUT bash -c read_stat_thread 2> /dev/null & -timeout $TIMEOUT bash -c truncate_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT read_stat_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT truncate_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/02030_tuple_filter.sql b/tests/queries/0_stateless/02030_tuple_filter.sql index 5efedeb8c0d..c19f538b8e1 100644 --- a/tests/queries/0_stateless/02030_tuple_filter.sql +++ b/tests/queries/0_stateless/02030_tuple_filter.sql @@ -5,6 +5,7 @@ CREATE TABLE test_tuple_filter (id UInt32, value String, log_date Date) Engine=M INSERT INTO test_tuple_filter VALUES (1,'A','2021-01-01'),(2,'B','2021-01-01'),(3,'C','2021-01-01'),(4,'D','2021-01-02'),(5,'E','2021-01-02'); SET force_primary_key = 1; +SET optimize_move_to_prewhere = 1; SELECT * FROM test_tuple_filter WHERE (id, value) = (1, 'A'); SELECT * FROM test_tuple_filter WHERE (1, 'A') = (id, value); diff --git a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh index 7e8579f7cbe..4b7d15c01c3 100755 --- a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh +++ b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh @@ -7,8 +7,15 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) i=0 retries=5 +client_opts=( + --http_max_tries 1 + --max_execution_time 3 + --max_threads 10 + --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" + --format Null +) # Connecting to wrong address and checking for race condition while [[ $i -lt $retries ]]; do - timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null 2>/dev/null + clickhouse_client_timeout 4s ${CLICKHOUSE_CLIENT} "${client_opts[@]}" 2>/dev/null ((++i)) done diff --git a/tests/queries/0_stateless/02050_client_profile_events.sh b/tests/queries/0_stateless/02050_client_profile_events.sh index 459e8505e22..f8bcea0d1bb 100755 --- a/tests/queries/0_stateless/02050_client_profile_events.sh +++ b/tests/queries/0_stateless/02050_client_profile_events.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # do not print any ProfileEvents packets $CLICKHOUSE_CLIENT -q 'select * from numbers(1e5) format Null' |& grep -c 'SelectedRows' # print only last (and also number of rows to provide more info in case of failures) -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select * from numbers(1e5)' 2> >(grep -o -e '\[ 0 \] SelectedRows: .*$' -e Exception) 1> >(wc -l) +$CLICKHOUSE_CLIENT --max_block_size=65505 --print-profile-events --profile-events-delay-ms=-1 -q 'select * from numbers(1e5)' 2> >(grep -o -e '\[ 0 \] SelectedRows: .*$' -e Exception) 1> >(wc -l) # print everything profile_events="$($CLICKHOUSE_CLIENT --max_block_size 1 --print-profile-events -q 'select sleep(1) from numbers(2) format Null' |& grep -c 'SelectedRows')" test "$profile_events" -gt 1 && echo OK || echo "FAIL ($profile_events)" diff --git a/tests/queries/0_stateless/02051_read_settings.sql.j2 b/tests/queries/0_stateless/02051_read_settings.sql.j2 index fa19fbd3036..69dd3c264ba 100644 --- a/tests/queries/0_stateless/02051_read_settings.sql.j2 +++ b/tests/queries/0_stateless/02051_read_settings.sql.j2 @@ -1,4 +1,7 @@ --- Tags: long +-- Tags: long, no-tsan, no-parallel, no-random-settings +-- Tag: no-tsan -- too slow under TSan (~5-6min) +-- Tag: no-random-settings -- to avoid settings overlaps +-- Tag: no-parallel -- to reduce test time -- -- Test for testing various read settings. diff --git a/tests/queries/0_stateless/02097_default_dict_get_add_database.sql b/tests/queries/0_stateless/02097_default_dict_get_add_database.sql index d4886c7b988..4a9bca00ad5 100644 --- a/tests/queries/0_stateless/02097_default_dict_get_add_database.sql +++ b/tests/queries/0_stateless/02097_default_dict_get_add_database.sql @@ -26,8 +26,8 @@ SOURCE(CLICKHOUSE(TABLE 'test_table')); CREATE TABLE test_table_default ( - data_1 DEFAULT dictGetUInt64('test_dictionary', 'data_column_1', toUInt64(0)), - data_2 DEFAULT dictGet(test_dictionary, 'data_column_2', toUInt64(0)) + data_1 DEFAULT dictGetUInt64('02097_db.test_dictionary', 'data_column_1', toUInt64(0)), + data_2 DEFAULT dictGet(02097_db.test_dictionary, 'data_column_2', toUInt64(0)) ) ENGINE=TinyLog; diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference index 993dd9b1cde..e749cc67187 100644 --- a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference @@ -1,10 +1,40 @@ 1 +=== Backward compatibility test +1 +=== Cannot resolve host +1 +1 +=== Bad arguments +1 +1 +=== Not alive host +1 1 1 1 1 1 1 +=== Code 210 with ipv6 +1 +1 +1 +1 +1 +1 +=== Values form config +1 +1 +1 +1 +=== Values form config 2 +1 +1 +1 +=== +1 +1 +1 1 1 1 diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh index 61244b80cf6..bf00fb4e44c 100755 --- a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -7,45 +8,100 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # default values test ${CLICKHOUSE_CLIENT} --query "SELECT 1" -# backward compatibility test +echo '=== Backward compatibility test' ${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +echo '=== Cannot resolve host' not_resolvable_host="notlocalhost" -exception_msg="Cannot resolve host (${not_resolvable_host}), error 0: ${not_resolvable_host}. -Code: 198. DB::Exception: Not found address of host: ${not_resolvable_host}. (DNS_ERROR) -" -error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" "${not_resolvable_host}" --query "SELECT 1" 2>&1 > /dev/null)"; -[ "${error}" == "${exception_msg}" ]; echo "$?" +error="$(${CLICKHOUSE_CLIENT} --host "${not_resolvable_host}" --query "SELECT 1" 2>&1 > /dev/null)"; +echo "${error}" | grep -Fc "DNS_ERROR" +echo "${error}" | grep -Fq "${not_resolvable_host}" && echo 1 || echo 0 +echo '=== Bad arguments' not_number_port="abc" -exception_msg="Bad arguments: the argument ('${CLICKHOUSE_HOST}:${not_number_port}') for option '--host' is invalid." -error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}:${not_number_port}" --query "SELECT 1" 2>&1 > /dev/null)"; -[ "${error}" == "${exception_msg}" ]; echo "$?" +error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_number_port}" --query "SELECT 1" 2>&1 > /dev/null)"; +echo "${error}" | grep -Fc "Bad arguments" +echo "${error}" | grep -Fc "${not_number_port}" + +echo '=== Not alive host' not_alive_host="10.100.0.0" -${CLICKHOUSE_CLIENT} --host "${not_alive_host}" "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --host "${not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; not_alive_port="1" -exception_msg="Code: 210. DB::NetException: Connection refused (${CLICKHOUSE_HOST}:${not_alive_port}). (NETWORK_ERROR) -" error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" -${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}:${not_alive_port}" "${CLICKHOUSE_HOST}" --query "SELECT 1"; -${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_TCP}" --port "${not_alive_port}" --query "SELECT 1"; +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${CLICKHOUSE_HOST}:${not_alive_port}" + +error="$(${CLICKHOUSE_CLIENT} --host "${not_alive_host}" --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:9000" + +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_alive_port}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" --port "${not_alive_port}" --query "SELECT 1"; + +echo '=== Code 210 with ipv6' ipv6_host_without_brackets="2001:3984:3989::1:1000" -exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_without_brackets}). (NETWORK_ERROR) -" error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_without_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${ipv6_host_without_brackets}" ipv6_host_with_brackets="[2001:3984:3989::1:1000]" -exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_with_brackets}). (NETWORK_ERROR) -" -error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" -exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_with_brackets}:${not_alive_port}). (NETWORK_ERROR) -" -error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}:${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" -[ "${error}" == "${exception_msg}" ]; echo "$?" +error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${ipv6_host_with_brackets}" + +error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --port "${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "Code: 210" +echo "${error}" | grep -Fc "${ipv6_host_with_brackets}:${not_alive_port}" + +echo '=== Values form config' + +CUSTOM_CONFIG="$CURDIR/02100_config_$(${CLICKHOUSE_LOCAL} -q 'SELECT rand()').xml" +rm -f ${CUSTOM_CONFIG} + +cat << EOF > ${CUSTOM_CONFIG} + + ${not_alive_host} + ${not_alive_port} + +EOF + +error="$(${CLICKHOUSE_CLIENT} --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:${not_alive_port}" + +error="$(${CLICKHOUSE_CLIENT} --host localhost --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "localhost:${not_alive_port}" + +rm -f ${CUSTOM_CONFIG} + +echo '=== Values form config 2' + +cat << EOF > ${CUSTOM_CONFIG} + + ${not_alive_host} + +EOF + +error="$(${CLICKHOUSE_CLIENT} --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:9000" + +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --config ${CUSTOM_CONFIG} --query "SELECT 1" + +rm -f ${CUSTOM_CONFIG} + +echo '===' + +${CLICKHOUSE_CLIENT} --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --host "{$not_alive_host}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1" 2> /dev/null; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; + diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set_ssl.reference b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set_ssl.reference new file mode 100644 index 00000000000..6674a55ac48 --- /dev/null +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set_ssl.reference @@ -0,0 +1,10 @@ +1 +1 +=== Values form config +1 +1 +1 +1 +=== Values form config 2 +1 +1 diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set_ssl.sh b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set_ssl.sh new file mode 100755 index 00000000000..e82414a05e1 --- /dev/null +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set_ssl.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Tags: use-ssl + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +not_alive_host="10.100.0.0" +not_alive_port="1" + +error="$(${CLICKHOUSE_CLIENT} --secure --host "${not_alive_host}" --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:9440" + +echo '=== Values form config' + +CUSTOM_CONFIG="$CURDIR/02100_config_$(${CLICKHOUSE_LOCAL} -q 'SELECT rand()').xml" +rm -f ${CUSTOM_CONFIG} + +cat << EOF > ${CUSTOM_CONFIG} + + ${not_alive_host} + ${not_alive_port} + +EOF + +error="$(${CLICKHOUSE_CLIENT} --secure --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:${not_alive_port}" + +error="$(${CLICKHOUSE_CLIENT} --secure --host localhost --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "localhost:${not_alive_port}" + +rm -f ${CUSTOM_CONFIG} + +echo '=== Values form config 2' + +cat << EOF > ${CUSTOM_CONFIG} + + ${not_alive_host} + +EOF + +error="$(${CLICKHOUSE_CLIENT} --secure --config ${CUSTOM_CONFIG} --query "SELECT 1" 2>&1 > /dev/null)" +echo "${error}" | grep -Fc "DB::NetException" +echo "${error}" | grep -Fc "${not_alive_host}:9440" + +rm -f ${CUSTOM_CONFIG} diff --git a/tests/queries/0_stateless/02100_replaceRegexpAll_bug.reference b/tests/queries/0_stateless/02100_replaceRegexpAll_bug.reference index 993dd9b1cde..4dff9ef38ef 100644 --- a/tests/queries/0_stateless/02100_replaceRegexpAll_bug.reference +++ b/tests/queries/0_stateless/02100_replaceRegexpAll_bug.reference @@ -9,3 +9,4 @@ 1 1 1 +1 diff --git a/tests/queries/0_stateless/02100_replaceRegexpAll_bug.sql b/tests/queries/0_stateless/02100_replaceRegexpAll_bug.sql index 32f7f63f6d0..66ccb044549 100644 --- a/tests/queries/0_stateless/02100_replaceRegexpAll_bug.sql +++ b/tests/queries/0_stateless/02100_replaceRegexpAll_bug.sql @@ -12,3 +12,5 @@ SELECT '1,,' == replaceRegexpOne('1,,', '^[,]*|[,]*$', '') x; SELECT '5935,5998,6014' == trim(BOTH ', ' FROM '5935,5998,6014, ') x; SELECT '5935,5998,6014' == replaceRegexpAll('5935,5998,6014, ', concat('^[', regexpQuoteMeta(', '), ']*|[', regexpQuoteMeta(', '), ']*$'), '') AS x; + +SELECT trim(BOTH '"' FROM '2') == '2' diff --git a/tests/queries/0_stateless/02104_overcommit_memory.reference b/tests/queries/0_stateless/02104_overcommit_memory.reference new file mode 100644 index 00000000000..b108f48e0fa --- /dev/null +++ b/tests/queries/0_stateless/02104_overcommit_memory.reference @@ -0,0 +1 @@ +OVERCOMMITED WITH USER LIMIT WAS KILLED diff --git a/tests/queries/0_stateless/02104_overcommit_memory.sh b/tests/queries/0_stateless/02104_overcommit_memory.sh new file mode 100755 index 00000000000..140557304c6 --- /dev/null +++ b/tests/queries/0_stateless/02104_overcommit_memory.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u1 IDENTIFIED WITH no_password' +$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u1' + +function overcommited() +{ + while true; do + $CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_guaranteed_memory_usage=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED" + done +} + +function expect_execution() +{ + while true; do + $CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,max_guaranteed_memory_usage=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null + done +} + +export -f overcommited +export -f expect_execution + +function user_test() +{ + for _ in {1..10}; + do + timeout 10 bash -c overcommited & + timeout 10 bash -c expect_execution & + done; + + wait +} + +output=$(user_test) + +if test -z "$output" +then + echo "OVERCOMMITED WITH USER LIMIT WAS NOT KILLED" +else + echo "OVERCOMMITED WITH USER LIMIT WAS KILLED" +fi + +$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u1' diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 678fe35fd96..a6a184b3d22 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -34,9 +34,9 @@ CREATE TABLE system.numbers_mt\n(\n `number` UInt64\n)\nENGINE = SystemNumber CREATE TABLE system.one\n(\n `dummy` UInt8\n)\nENGINE = SystemOne()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.part_moves_between_shards\n(\n `database` String,\n `table` String,\n `task_name` String,\n `task_uuid` UUID,\n `create_time` DateTime,\n `part_name` String,\n `part_uuid` UUID,\n `to_shard` String,\n `dst_part_name` String,\n `update_time` DateTime,\n `state` String,\n `rollback` UInt8,\n `num_tries` UInt32,\n `last_exception` String\n)\nENGINE = SystemShardMoves()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.parts\n(\n `partition` String,\n `name` String,\n `uuid` UUID,\n `part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `secondary_indices_compressed_bytes` UInt64,\n `secondary_indices_uncompressed_bytes` UInt64,\n `secondary_indices_marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `is_frozen` UInt8,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `hash_of_all_files` String,\n `hash_of_uncompressed_files` String,\n `uncompressed_hash_of_compressed_files` String,\n `delete_ttl_info_min` DateTime,\n `delete_ttl_info_max` DateTime,\n `move_ttl_info.expression` Array(String),\n `move_ttl_info.min` Array(DateTime),\n `move_ttl_info.max` Array(DateTime),\n `default_compression_codec` String,\n `recompression_ttl_info.expression` Array(String),\n `recompression_ttl_info.min` Array(DateTime),\n `recompression_ttl_info.max` Array(DateTime),\n `group_by_ttl_info.expression` Array(String),\n `group_by_ttl_info.min` Array(DateTime),\n `group_by_ttl_info.max` Array(DateTime),\n `rows_where_ttl_info.expression` Array(String),\n `rows_where_ttl_info.min` Array(DateTime),\n `rows_where_ttl_info.max` Array(DateTime),\n `projections` Array(String),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemParts()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.parts_columns\n(\n `partition` String,\n `name` String,\n `uuid` UUID,\n `part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `column` String,\n `type` String,\n `column_position` UInt64,\n `default_kind` String,\n `default_expression` String,\n `column_bytes_on_disk` UInt64,\n `column_data_compressed_bytes` UInt64,\n `column_data_uncompressed_bytes` UInt64,\n `column_marks_bytes` UInt64,\n `serialization_kind` String,\n `subcolumns.names` Array(String),\n `subcolumns.types` Array(String),\n `subcolumns.serializations` Array(String),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemPartsColumns()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.parts_columns\n(\n `partition` String,\n `name` String,\n `uuid` UUID,\n `part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `column` String,\n `type` String,\n `column_position` UInt64,\n `default_kind` String,\n `default_expression` String,\n `column_bytes_on_disk` UInt64,\n `column_data_compressed_bytes` UInt64,\n `column_data_uncompressed_bytes` UInt64,\n `column_marks_bytes` UInt64,\n `serialization_kind` String,\n `subcolumns.names` Array(String),\n `subcolumns.types` Array(String),\n `subcolumns.serializations` Array(String),\n `subcolumns.bytes_on_disk` Array(UInt64),\n `subcolumns.data_compressed_bytes` Array(UInt64),\n `subcolumns.data_uncompressed_bytes` Array(UInt64),\n `subcolumns.marks_bytes` Array(UInt64),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemPartsColumns()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.privileges\n(\n `privilege` Enum16(\'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToLineWithInlines\' = 120, \'addressToSymbol\' = 121, \'demangle\' = 122, \'INTROSPECTION\' = 123, \'FILE\' = 124, \'URL\' = 125, \'REMOTE\' = 126, \'MONGO\' = 127, \'MYSQL\' = 128, \'POSTGRES\' = 129, \'SQLITE\' = 130, \'ODBC\' = 131, \'JDBC\' = 132, \'HDFS\' = 133, \'S3\' = 134, \'SOURCES\' = 135, \'ALL\' = 136, \'NONE\' = 137),\n `aliases` Array(String),\n `level` Nullable(Enum8(\'GLOBAL\' = 0, \'DATABASE\' = 1, \'TABLE\' = 2, \'DICTIONARY\' = 3, \'VIEW\' = 4, \'COLUMN\' = 5)),\n `parent_group` Nullable(Enum16(\'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToLineWithInlines\' = 120, \'addressToSymbol\' = 121, \'demangle\' = 122, \'INTROSPECTION\' = 123, \'FILE\' = 124, \'URL\' = 125, \'REMOTE\' = 126, \'MONGO\' = 127, \'MYSQL\' = 128, \'POSTGRES\' = 129, \'SQLITE\' = 130, \'ODBC\' = 131, \'JDBC\' = 132, \'HDFS\' = 133, \'S3\' = 134, \'SOURCES\' = 135, \'ALL\' = 136, \'NONE\' = 137))\n)\nENGINE = SystemPrivileges()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.processes\n(\n `is_initial_query` UInt8,\n `user` String,\n `query_id` String,\n `address` IPv6,\n `port` UInt16,\n `initial_user` String,\n `initial_query_id` String,\n `initial_address` IPv6,\n `initial_port` UInt16,\n `interface` UInt8,\n `os_user` String,\n `client_hostname` String,\n `client_name` String,\n `client_revision` UInt64,\n `client_version_major` UInt64,\n `client_version_minor` UInt64,\n `client_version_patch` UInt64,\n `http_method` UInt8,\n `http_user_agent` String,\n `http_referer` String,\n `forwarded_for` String,\n `quota_key` String,\n `elapsed` Float64,\n `is_cancelled` UInt8,\n `read_rows` UInt64,\n `read_bytes` UInt64,\n `total_rows_approx` UInt64,\n `written_rows` UInt64,\n `written_bytes` UInt64,\n `memory_usage` Int64,\n `peak_memory_usage` Int64,\n `query` String,\n `thread_ids` Array(UInt64),\n `ProfileEvents` Map(String, UInt64),\n `Settings` Map(String, String),\n `current_database` String,\n `ProfileEvents.Names` Array(String),\n `ProfileEvents.Values` Array(UInt64),\n `Settings.Names` Array(String),\n `Settings.Values` Array(String)\n)\nENGINE = SystemProcesses()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.processes\n(\n `is_initial_query` UInt8,\n `user` String,\n `query_id` String,\n `address` IPv6,\n `port` UInt16,\n `initial_user` String,\n `initial_query_id` String,\n `initial_address` IPv6,\n `initial_port` UInt16,\n `interface` UInt8,\n `os_user` String,\n `client_hostname` String,\n `client_name` String,\n `client_revision` UInt64,\n `client_version_major` UInt64,\n `client_version_minor` UInt64,\n `client_version_patch` UInt64,\n `http_method` UInt8,\n `http_user_agent` String,\n `http_referer` String,\n `forwarded_for` String,\n `quota_key` String,\n `distributed_depth` UInt64,\n `elapsed` Float64,\n `is_cancelled` UInt8,\n `read_rows` UInt64,\n `read_bytes` UInt64,\n `total_rows_approx` UInt64,\n `written_rows` UInt64,\n `written_bytes` UInt64,\n `memory_usage` Int64,\n `peak_memory_usage` Int64,\n `query` String,\n `thread_ids` Array(UInt64),\n `ProfileEvents` Map(String, UInt64),\n `Settings` Map(String, String),\n `current_database` String,\n `ProfileEvents.Names` Array(String),\n `ProfileEvents.Values` Array(UInt64),\n `Settings.Names` Array(String),\n `Settings.Values` Array(String)\n)\nENGINE = SystemProcesses()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.projection_parts\n(\n `partition` String,\n `name` String,\n `part_type` String,\n `parent_name` String,\n `parent_uuid` UUID,\n `parent_part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `parent_marks` UInt64,\n `parent_rows` UInt64,\n `parent_bytes_on_disk` UInt64,\n `parent_data_compressed_bytes` UInt64,\n `parent_data_uncompressed_bytes` UInt64,\n `parent_marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `is_frozen` UInt8,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `hash_of_all_files` String,\n `hash_of_uncompressed_files` String,\n `uncompressed_hash_of_compressed_files` String,\n `delete_ttl_info_min` DateTime,\n `delete_ttl_info_max` DateTime,\n `move_ttl_info.expression` Array(String),\n `move_ttl_info.min` Array(DateTime),\n `move_ttl_info.max` Array(DateTime),\n `default_compression_codec` String,\n `recompression_ttl_info.expression` Array(String),\n `recompression_ttl_info.min` Array(DateTime),\n `recompression_ttl_info.max` Array(DateTime),\n `group_by_ttl_info.expression` Array(String),\n `group_by_ttl_info.min` Array(DateTime),\n `group_by_ttl_info.max` Array(DateTime),\n `rows_where_ttl_info.expression` Array(String),\n `rows_where_ttl_info.min` Array(DateTime),\n `rows_where_ttl_info.max` Array(DateTime),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemProjectionParts()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.projection_parts_columns\n(\n `partition` String,\n `name` String,\n `part_type` String,\n `parent_name` String,\n `parent_uuid` UUID,\n `parent_part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `parent_marks` UInt64,\n `parent_rows` UInt64,\n `parent_bytes_on_disk` UInt64,\n `parent_data_compressed_bytes` UInt64,\n `parent_data_uncompressed_bytes` UInt64,\n `parent_marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `column` String,\n `type` String,\n `column_position` UInt64,\n `default_kind` String,\n `default_expression` String,\n `column_bytes_on_disk` UInt64,\n `column_data_compressed_bytes` UInt64,\n `column_data_uncompressed_bytes` UInt64,\n `column_marks_bytes` UInt64,\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemProjectionPartsColumns()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.quota_limits\n(\n `quota_name` String,\n `duration` UInt32,\n `is_randomized_interval` UInt8,\n `max_queries` Nullable(UInt64),\n `max_query_selects` Nullable(UInt64),\n `max_query_inserts` Nullable(UInt64),\n `max_errors` Nullable(UInt64),\n `max_result_rows` Nullable(UInt64),\n `max_result_bytes` Nullable(UInt64),\n `max_read_rows` Nullable(UInt64),\n `max_read_bytes` Nullable(UInt64),\n `max_execution_time` Nullable(Float64)\n)\nENGINE = SystemQuotaLimits()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' @@ -60,7 +60,7 @@ CREATE TABLE system.table_functions\n(\n `name` String\n)\nENGINE = SystemTab CREATE TABLE system.tables\n(\n `database` String,\n `name` String,\n `uuid` UUID,\n `engine` String,\n `is_temporary` UInt8,\n `data_paths` Array(String),\n `metadata_path` String,\n `metadata_modification_time` DateTime,\n `dependencies_database` Array(String),\n `dependencies_table` Array(String),\n `create_table_query` String,\n `engine_full` String,\n `as_select` String,\n `partition_key` String,\n `sorting_key` String,\n `primary_key` String,\n `sampling_key` String,\n `storage_policy` String,\n `total_rows` Nullable(UInt64),\n `total_bytes` Nullable(UInt64),\n `lifetime_rows` Nullable(UInt64),\n `lifetime_bytes` Nullable(UInt64),\n `comment` String,\n `has_own_data` UInt8,\n `loading_dependencies_database` Array(String),\n `loading_dependencies_table` Array(String),\n `loading_dependent_database` Array(String),\n `loading_dependent_table` Array(String),\n `table` String\n)\nENGINE = SystemTables()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.time_zones\n(\n `time_zone` String\n)\nENGINE = SystemTimeZones()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.user_directories\n(\n `name` String,\n `type` String,\n `params` String,\n `precedence` UInt64\n)\nENGINE = SystemUserDirectories()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.users\n(\n `name` String,\n `id` UUID,\n `storage` String,\n `auth_type` Enum8(\'no_password\' = 0, \'plaintext_password\' = 1, \'sha256_password\' = 2, \'double_sha1_password\' = 3, \'ldap\' = 4, \'kerberos\' = 5),\n `auth_params` String,\n `host_ip` Array(String),\n `host_names` Array(String),\n `host_names_regexp` Array(String),\n `host_names_like` Array(String),\n `default_roles_all` UInt8,\n `default_roles_list` Array(String),\n `default_roles_except` Array(String),\n `grantees_any` UInt8,\n `grantees_list` Array(String),\n `grantees_except` Array(String),\n `default_database` String\n)\nENGINE = SystemUsers()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.users\n(\n `name` String,\n `id` UUID,\n `storage` String,\n `auth_type` Enum8(\'no_password\' = 0, \'plaintext_password\' = 1, \'sha256_password\' = 2, \'double_sha1_password\' = 3, \'ldap\' = 4, \'kerberos\' = 5, \'ssl_certificate\' = 6),\n `auth_params` String,\n `host_ip` Array(String),\n `host_names` Array(String),\n `host_names_regexp` Array(String),\n `host_names_like` Array(String),\n `default_roles_all` UInt8,\n `default_roles_list` Array(String),\n `default_roles_except` Array(String),\n `grantees_any` UInt8,\n `grantees_list` Array(String),\n `grantees_except` Array(String),\n `default_database` String\n)\nENGINE = SystemUsers()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.warnings\n(\n `message` String\n)\nENGINE = SystemWarnings()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.zeros\n(\n `zero` UInt8\n)\nENGINE = SystemZeros()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.zeros_mt\n(\n `zero` UInt8\n)\nENGINE = SystemZeros()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' diff --git a/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh b/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh index 2deaf788ecf..84a1befe421 100755 --- a/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh +++ b/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh @@ -17,28 +17,26 @@ function four_letter_thread() function create_drop_thread() { - while true; do - num=$(($RANDOM % 10 + 1)) - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table$num (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table$num', '0') ORDER BY key" - sleep 0.$RANDOM - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table$num" - done + num=$(($RANDOM % 10 + 1)) + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table$num (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table$num', '0') ORDER BY key" + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table$num" } -export -f four_letter_thread; -export -f create_drop_thread; +export -f four_letter_thread +export -f create_drop_thread TIMEOUT=15 -timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & -timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & -timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & -timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & +timeout $TIMEOUT four_letter_thread 2> /dev/null & +timeout $TIMEOUT four_letter_thread 2> /dev/null & +timeout $TIMEOUT four_letter_thread 2> /dev/null & +timeout $TIMEOUT four_letter_thread 2> /dev/null & -timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & -timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & -timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & -timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & +clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/02122_parallel_formatting.sh b/tests/queries/0_stateless/02122_parallel_formatting.sh index f0c24344329..97f7a3903e7 100755 --- a/tests/queries/0_stateless/02122_parallel_formatting.sh +++ b/tests/queries/0_stateless/02122_parallel_formatting.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh b/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh index 1b2197ef943..bc43f460289 100755 --- a/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh +++ b/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh @@ -9,16 +9,12 @@ $CLICKHOUSE_CLIENT -q "CREATE TABLE t_buffer_map(m1 Map(String, UInt64), m2 Map( function insert1 { - while true; do - $CLICKHOUSE_CLIENT -q "INSERT INTO t_buffer_map SELECT (range(10), range(10)), (range(10), range(10)) from numbers(100)" - done + $CLICKHOUSE_CLIENT -q "INSERT INTO t_buffer_map SELECT (range(10), range(10)), (range(10), range(10)) from numbers(100)" } function select1 { - while true; do - $CLICKHOUSE_CLIENT -q "SELECT * FROM t_buffer_map" 2> /dev/null > /dev/null - done + $CLICKHOUSE_CLIENT -q "SELECT * FROM t_buffer_map" 2> /dev/null > /dev/null } TIMEOUT=10 @@ -26,8 +22,8 @@ TIMEOUT=10 export -f insert1 export -f select1 -timeout $TIMEOUT bash -c insert1 & -timeout $TIMEOUT bash -c select1 & +clickhouse_client_loop_timeout $TIMEOUT insert1 & +clickhouse_client_loop_timeout $TIMEOUT select1 & wait diff --git a/tests/queries/0_stateless/02126_url_auth.python b/tests/queries/0_stateless/02126_url_auth.python index 60009624c76..57b16fb413e 100644 --- a/tests/queries/0_stateless/02126_url_auth.python +++ b/tests/queries/0_stateless/02126_url_auth.python @@ -121,18 +121,14 @@ class CSVHTTPServer(BaseHTTPRequestHandler): class HTTPServerV6(HTTPServer): address_family = socket.AF_INET6 -def start_server(requests_amount): +def start_server(): if IS_IPV6: httpd = HTTPServerV6(HTTP_SERVER_ADDRESS, CSVHTTPServer) else: httpd = HTTPServer(HTTP_SERVER_ADDRESS, CSVHTTPServer) - def real_func(): - for i in range(requests_amount): - httpd.handle_request() - - t = threading.Thread(target=real_func) - return t + t = threading.Thread(target=httpd.serve_forever) + return t, httpd # test section @@ -217,9 +213,10 @@ def main(): query : 'hello, world', } - t = start_server(len(list(select_requests_url_auth.keys()))) + t, httpd = start_server() t.start() test_select(requests=list(select_requests_url_auth.keys()), answers=list(select_requests_url_auth.values()), test_data=test_data) + httpd.shutdown() t.join() print("PASSED") diff --git a/tests/queries/0_stateless/02131_multiply_row_policies_on_same_column.sql b/tests/queries/0_stateless/02131_multiply_row_policies_on_same_column.sql index 75f7f737e85..d0a55c6ba65 100644 --- a/tests/queries/0_stateless/02131_multiply_row_policies_on_same_column.sql +++ b/tests/queries/0_stateless/02131_multiply_row_policies_on_same_column.sql @@ -1,3 +1,5 @@ +SET optimize_move_to_prewhere = 1; + DROP TABLE IF EXISTS 02131_multiply_row_policies_on_same_column; CREATE TABLE 02131_multiply_row_policies_on_same_column (x UInt8) ENGINE = MergeTree ORDER BY x; INSERT INTO 02131_multiply_row_policies_on_same_column VALUES (1), (2), (3), (4); diff --git a/tests/queries/0_stateless/02136_scalar_progress.sh b/tests/queries/0_stateless/02136_scalar_progress.sh index 4608031f83d..9f4429b0caa 100755 --- a/tests/queries/0_stateless/02136_scalar_progress.sh +++ b/tests/queries/0_stateless/02136_scalar_progress.sh @@ -4,4 +4,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CURL -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d "SELECT (SELECT max(number), count(number) FROM numbers(100000));" -v 2>&1 | grep -E "X-ClickHouse-Summary|X-ClickHouse-Progress" +$CLICKHOUSE_CURL -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d "SELECT (SELECT max(number), count(number) FROM numbers(100000) settings max_block_size=65505);" -v 2>&1 | grep -E "X-ClickHouse-Summary|X-ClickHouse-Progress" diff --git a/tests/queries/0_stateless/02136_scalar_read_rows_json.sh b/tests/queries/0_stateless/02136_scalar_read_rows_json.sh index d589cb60086..34b4b6909b5 100755 --- a/tests/queries/0_stateless/02136_scalar_read_rows_json.sh +++ b/tests/queries/0_stateless/02136_scalar_read_rows_json.sh @@ -7,4 +7,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo "#1" ${CLICKHOUSE_CLIENT} --query='SELECT count() FROM numbers(100) FORMAT JSON;' | grep -a -v "elapsed" echo "#2" -${CLICKHOUSE_CLIENT} --query='SELECT (SELECT max(number), count(number) FROM numbers(100000) as n) FORMAT JSON;' | grep -a -v "elapsed" | grep -v "_subquery" +${CLICKHOUSE_CLIENT} --query='SELECT (SELECT max(number), count(number) FROM numbers(100000) as n) SETTINGS max_block_size = 65505 FORMAT JSON;' | grep -a -v "elapsed" | grep -v "_subquery" diff --git a/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.reference b/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.reference index e8183f05f5d..da7b788b157 100644 --- a/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.reference +++ b/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.reference @@ -1,3 +1,18 @@ 1 1 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.sql b/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.sql index 6725fa04114..ebbc6ce97e0 100644 --- a/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.sql +++ b/tests/queries/0_stateless/02151_replace_regexp_all_empty_match_alternative.sql @@ -1,3 +1,21 @@ -select replaceRegexpAll(',,1,,', '^[,]*|[,]*$', '') x; -select replaceRegexpAll(',,1', '^[,]*|[,]*$', '') x; -select replaceRegexpAll('1,,', '^[,]*|[,]*$', '') x; +SELECT replaceRegexpAll(',,1,,', '^[,]*|[,]*$', ''); +SELECT replaceRegexpAll(',,1', '^[,]*|[,]*$', ''); +SELECT replaceRegexpAll('1,,', '^[,]*|[,]*$', ''); + +SELECT replaceRegexpAll(materialize(',,1,,'), '^[,]*|[,]*$', ''); +SELECT replaceRegexpAll(materialize(',,1'), '^[,]*|[,]*$', ''); +SELECT replaceRegexpAll(materialize('1,,'), '^[,]*|[,]*$', ''); + +SELECT replaceRegexpAll('a', 'z*', '') == 'a'; +SELECT replaceRegexpAll('aa', 'z*', '') == 'aa'; +SELECT replaceRegexpAll('aaq', 'z*', '') == 'aaq'; +SELECT replaceRegexpAll('aazq', 'z*', '') == 'aaq'; +SELECT replaceRegexpAll('aazzq', 'z*', '') == 'aaq'; +SELECT replaceRegexpAll('aazzqa', 'z*', '') == 'aaqa'; + +SELECT replaceRegexpAll(materialize('a'), 'z*', '') == 'a'; +SELECT replaceRegexpAll(materialize('aa'), 'z*', '') == 'aa'; +SELECT replaceRegexpAll(materialize('aaq'), 'z*', '') == 'aaq'; +SELECT replaceRegexpAll(materialize('aazq'), 'z*', '') == 'aaq'; +SELECT replaceRegexpAll(materialize('aazzq'), 'z*', '') == 'aaq'; +SELECT replaceRegexpAll(materialize('aazzqa'), 'z*', '') == 'aaqa'; diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh index 2801ec16a43..44de0e15370 100755 --- a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh +++ b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Tags: no-tsan -# ^^^^^^^ +# Tags: no-tsan, no-cpu-aarch64 # TSan does not supports tracing. +# trace_log doesn't work on aarch64 # Regression for proper release of Context, # via tracking memory of external tables. diff --git a/tests/queries/0_stateless/02156_storage_merge_prewhere.sql b/tests/queries/0_stateless/02156_storage_merge_prewhere.sql index 69fa9ac5ee2..b75d3fa22e5 100644 --- a/tests/queries/0_stateless/02156_storage_merge_prewhere.sql +++ b/tests/queries/0_stateless/02156_storage_merge_prewhere.sql @@ -1,3 +1,5 @@ +SET optimize_move_to_prewhere = 1; + DROP TABLE IF EXISTS t_02156_mt1; DROP TABLE IF EXISTS t_02156_mt2; DROP TABLE IF EXISTS t_02156_log; diff --git a/tests/queries/0_stateless/02158_proportions_ztest.reference b/tests/queries/0_stateless/02158_proportions_ztest.reference new file mode 100644 index 00000000000..3c5c5a13a34 --- /dev/null +++ b/tests/queries/0_stateless/02158_proportions_ztest.reference @@ -0,0 +1,2 @@ +(-0.20656724435948853,0.8363478437079654,-0.09345975390115283,0.07563797172293502) +(-0.20656724435948853,0.8363478437079654,-0.09345975390115283,0.07563797172293502) diff --git a/tests/queries/0_stateless/02158_proportions_ztest.sql b/tests/queries/0_stateless/02158_proportions_ztest.sql new file mode 100644 index 00000000000..bda50b43a97 --- /dev/null +++ b/tests/queries/0_stateless/02158_proportions_ztest.sql @@ -0,0 +1,13 @@ +SELECT proportionsZTest(10, 11, 100, 101, 0.95, 'unpooled'); + +DROP TABLE IF EXISTS proportions_ztest; +CREATE TABLE proportions_ztest (sx UInt64, sy UInt64, tx UInt64, ty UInt64) Engine = Memory(); +INSERT INTO proportions_ztest VALUES (10, 11, 100, 101); +SELECT proportionsZTest(sx, sy, tx, ty, 0.95, 'unpooled') FROM proportions_ztest; +DROP TABLE IF EXISTS proportions_ztest; + + +SELECT + NULL, + proportionsZTest(257, 1048575, 1048575, 257, -inf, NULL), + proportionsZTest(1024, 1025, 2, 2, 'unpooled'); -- { serverError 43 } \ No newline at end of file diff --git a/tests/queries/0_stateless/02158_proportions_ztest_cmp.python b/tests/queries/0_stateless/02158_proportions_ztest_cmp.python new file mode 100644 index 00000000000..d622004db28 --- /dev/null +++ b/tests/queries/0_stateless/02158_proportions_ztest_cmp.python @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import os +import sys +from math import sqrt, nan +from random import randrange +from scipy import stats +import pandas as pd +import numpy as np + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +from pure_http_client import ClickHouseClient + + +# unpooled variance z-test for proportions of two samples +def twosample_proportion_ztest(s1, s2, t1, t2, alpha): + if s1 == 0 or s2 == 0 or s1 > t1 or s2 > t2 or t1 + t2 == 0: + return nan, nan, nan, nan + + p1 = s1 / t1 + p2 = s2 / t2 + se = sqrt(p1 * (1 - p1) / t1 + p2 * (1 - p2) / t2) + if se == 0: + return nan, nan, nan, nan + z_stat = (p1 - p2) / se + + one_side = 1 - stats.norm.cdf(abs(z_stat)) + p_value = one_side * 2 + + z = stats.norm.ppf(1 - 0.5 * alpha) + ci_lower = (p1 - p2) - z * se + ci_upper = (p1 - p2) + z * se + + return z_stat, p_value, ci_lower, ci_upper + + +def test_and_check(name, z_stat, p_value, ci_lower, ci_upper, precision=1e-2): + client = ClickHouseClient() + real = client.query_return_df( + "SELECT roundBankers({}.1, 16) as z_stat, ".format(name) + + "roundBankers({}.2, 16) as p_value, ".format(name) + + "roundBankers({}.3, 16) as ci_lower, ".format(name) + + "roundBankers({}.4, 16) as ci_upper ".format(name) + + "FORMAT TabSeparatedWithNames;") + real_z_stat = real['z_stat'][0] + real_p_value = real['p_value'][0] + real_ci_lower = real['ci_lower'][0] + real_ci_upper = real['ci_upper'][0] + assert((np.isnan(real_z_stat) and np.isnan(z_stat)) or abs(real_z_stat - np.float64(z_stat)) < precision), "clickhouse_z_stat {}, py_z_stat {}".format(real_z_stat, z_stat) + assert((np.isnan(real_p_value) and np.isnan(p_value)) or abs(real_p_value - np.float64(p_value)) < precision), "clickhouse_p_value {}, py_p_value {}".format(real_p_value, p_value) + assert((np.isnan(real_ci_lower) and np.isnan(ci_lower)) or abs(real_ci_lower - np.float64(ci_lower)) < precision), "clickhouse_ci_lower {}, py_ci_lower {}".format(real_ci_lower, ci_lower) + assert((np.isnan(real_ci_upper) and np.isnan(ci_upper)) or abs(real_ci_upper - np.float64(ci_upper)) < precision), "clickhouse_ci_upper {}, py_ci_upper {}".format(real_ci_upper, ci_upper) + + +def test_mean_ztest(): + counts = [0, 0] + nobs = [0, 0] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(10, 10, 10, 10, 0.05) + + counts = [10, 10] + nobs = [10, 10] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(10, 10, 10, 10, 0.05) + + counts = [16, 16] + nobs = [16, 18] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + counts = [10, 20] + nobs = [30, 40] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + counts = [20, 10] + nobs = [40, 30] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + counts = [randrange(10,20), randrange(10,20)] + nobs = [randrange(counts[0] + 1, counts[0] * 2), randrange(counts[1], counts[1] * 2)] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + counts = [randrange(1,100), randrange(1,200)] + nobs = [randrange(counts[0], counts[0] * 2), randrange(counts[1], counts[1] * 3)] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + counts = [randrange(1,200), randrange(1,100)] + nobs = [randrange(counts[0], counts[0] * 3), randrange(counts[1], counts[1] * 2)] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + counts = [randrange(1,1000), randrange(1,1000)] + nobs = [randrange(counts[0], counts[0] * 2), randrange(counts[1], counts[1] * 2)] + z_stat, p_value, ci_lower, ci_upper = twosample_proportion_ztest(counts[0], counts[1], nobs[0], nobs[1], 0.05) + test_and_check("proportionsZTest(%d, %d, %d, %d, 0.95, 'unpooled')" % (counts[0], counts[1], nobs[0], nobs[1]), z_stat, p_value, ci_lower, ci_upper) + + +if __name__ == "__main__": + test_mean_ztest() + print("Ok.") + diff --git a/tests/queries/0_stateless/02158_proportions_ztest_cmp.reference b/tests/queries/0_stateless/02158_proportions_ztest_cmp.reference new file mode 100644 index 00000000000..587579af915 --- /dev/null +++ b/tests/queries/0_stateless/02158_proportions_ztest_cmp.reference @@ -0,0 +1 @@ +Ok. diff --git a/tests/queries/0_stateless/02158_proportions_ztest_cmp.sh b/tests/queries/0_stateless/02158_proportions_ztest_cmp.sh new file mode 100755 index 00000000000..64eeb513958 --- /dev/null +++ b/tests/queries/0_stateless/02158_proportions_ztest_cmp.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test + +python3 "$CURDIR"/02158_proportions_ztest_cmp.python diff --git a/tests/queries/0_stateless/02161_addressToLineWithInlines.sql b/tests/queries/0_stateless/02161_addressToLineWithInlines.sql index baddea30ae3..b6b497b4b55 100644 --- a/tests/queries/0_stateless/02161_addressToLineWithInlines.sql +++ b/tests/queries/0_stateless/02161_addressToLineWithInlines.sql @@ -1,6 +1,6 @@ --- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug - +-- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-cpu-aarch64 +SET allow_introspection_functions = 0; SELECT addressToLineWithInlines(1); -- { serverError 446 } SET allow_introspection_functions = 1; diff --git a/tests/queries/0_stateless/02169_map_functions.reference b/tests/queries/0_stateless/02169_map_functions.reference new file mode 100644 index 00000000000..160aebbc852 --- /dev/null +++ b/tests/queries/0_stateless/02169_map_functions.reference @@ -0,0 +1,33 @@ +{} +{} +{} +{'key3':103} +{} +{} +{} +{'key3':100,'key2':101,'key4':102} {'key4':102} +{'key3':101,'key2':102,'key4':103} {'key2':102,'key4':103} +{'key3':102,'key2':103,'key4':104} {'key3':102,'key2':103,'key4':104} +{'key3':103,'key2':104,'key4':105} {'key3':103,'key2':104,'key4':105} +{'key1':1111,'key2':2222} {'key2':2222} +{'key1':1112,'key2':2224} {'key1':1112,'key2':2224} +{'key1':1113,'key2':2226} {'key1':1113,'key2':2226} +{'key3':101,'key2':102,'key4':103} +{'key3':102,'key2':103,'key4':104} +{'key3':103,'key2':104,'key4':105} +{'key3':104,'key2':105,'key4':106} +{'key1':1112,'key2':2223} +{'key1':1113,'key2':2225} +{'key1':1114,'key2':2227} +{} +{} +{} +{} +{} +{} +{} +{3:2,1:0,2:0} +{1:2,2:3} +{1:2,2:3} +{'x':'y','x':'y'} +{'x':'y','x':'y'} diff --git a/tests/queries/0_stateless/02169_map_functions.sql b/tests/queries/0_stateless/02169_map_functions.sql new file mode 100644 index 00000000000..4cccaa56722 --- /dev/null +++ b/tests/queries/0_stateless/02169_map_functions.sql @@ -0,0 +1,39 @@ +DROP TABLE IF EXISTS table_map; +CREATE TABLE table_map (id UInt32, col Map(String, UInt64)) engine = MergeTree() ORDER BY tuple(); +INSERT INTO table_map SELECT number, map('key1', number, 'key2', number * 2) FROM numbers(1111, 3); +INSERT INTO table_map SELECT number, map('key3', number, 'key2', number + 1, 'key4', number + 2) FROM numbers(100, 4); + +SELECT mapFilter((k, v) -> k like '%3' and v > 102, col) FROM table_map ORDER BY id; +SELECT col, mapFilter((k, v) -> ((v % 10) > 1), col) FROM table_map ORDER BY id ASC; +SELECT mapApply((k, v) -> (k, v + 1), col) FROM table_map ORDER BY id; +SELECT mapFilter((k, v) -> 0, col) from table_map; +SELECT mapApply((k, v) -> tuple(v + 9223372036854775806), col) FROM table_map; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT mapUpdate(map(1, 3, 3, 2), map(1, 0, 2, 0)); +SELECT mapApply((x, y) -> (x, x + 1), map(1, 0, 2, 0)); +SELECT mapApply((x, y) -> (x, x + 1), materialize(map(1, 0, 2, 0))); +SELECT mapApply((x, y) -> ('x', 'y'), map(1, 0, 2, 0)); +SELECT mapApply((x, y) -> ('x', 'y'), materialize(map(1, 0, 2, 0))); + +SELECT mapApply(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT mapApply((x, y) -> (x), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapApply((x, y) -> ('x'), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapApply((x) -> (x, x), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapApply((x, y) -> (x, 1, 2), map(1, 0, 2, 0)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT mapApply((x, y) -> (x, x + 1)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT mapApply(map(1, 0, 2, 0), (x, y) -> (x, x + 1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapApply((x, y) -> (x, x+1), map(1, 0, 2, 0), map(1, 0, 2, 0)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT mapFilter(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT mapFilter((x, y) -> (toInt32(x)), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapFilter((x, y) -> ('x'), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapFilter((x) -> (x, x), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapFilter((x, y) -> (x, 1, 2), map(1, 0, 2, 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapFilter((x, y) -> (x, x + 1)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT mapFilter(map(1, 0, 2, 0), (x, y) -> (x > 0)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT mapFilter((x, y) -> (x, x + 1), map(1, 0, 2, 0), map(1, 0, 2, 0)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +SELECT mapUpdate(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT mapUpdate(map(1, 3, 3, 2), map(1, 0, 2, 0), map(1, 0, 2, 0)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +DROP TABLE table_map; diff --git a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference index 645cec31b47..5e8c7fc243f 100644 --- a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference +++ b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference @@ -5,4 +5,5 @@ -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -- at first and after -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings distributed_aggregation_memory_efficient=0; diff --git a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql index a86fd4357c8..35731c63f0d 100644 --- a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql +++ b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql @@ -1,6 +1,8 @@ drop table if exists data_02176; create table data_02176 (key Int) Engine=MergeTree() order by key; +set optimize_aggregation_in_order=1; + -- { echoOn } -- regression for optimize_aggregation_in_order with empty result set @@ -8,7 +10,8 @@ create table data_02176 (key Int) Engine=MergeTree() order by key; -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -- at first and after -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings distributed_aggregation_memory_efficient=0; -- { echoOff } drop table data_02176; diff --git a/tests/queries/0_stateless/02177_issue_31009_pt2.reference b/tests/queries/0_stateless/02177_issue_31009_pt2.reference new file mode 100644 index 00000000000..3c644f22b9b --- /dev/null +++ b/tests/queries/0_stateless/02177_issue_31009_pt2.reference @@ -0,0 +1,28 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02177_issue_31009_pt2.sql.j2 b/tests/queries/0_stateless/02177_issue_31009_pt2.sql.j2 new file mode 100644 index 00000000000..47940356302 --- /dev/null +++ b/tests/queries/0_stateless/02177_issue_31009_pt2.sql.j2 @@ -0,0 +1,40 @@ +-- Tags: long + +DROP TABLE IF EXISTS left; +DROP TABLE IF EXISTS right; + +SET join_algorithm = 'partial_merge'; + +{% for block_size in [10, 11, 128, 129, 65505, 65506, 70000] -%} +{% for join_block_size in range(block_size - 2, block_size + 2) -%} + +DROP TABLE IF EXISTS left; +DROP TABLE IF EXISTS right; + +CREATE TABLE left ( key UInt32, value String ) ENGINE = Memory; +CREATE TABLE right ( key UInt32, value String ) ENGINE = Memory; + +INSERT INTO left SELECT number, toString(number) FROM numbers({{ block_size * 2 + 1 }}); +INSERT INTO right SELECT number, toString(number) FROM numbers({{ block_size * 2 + 5 }}); + +SET max_joined_block_size_rows = {{ join_block_size }}; +SET max_block_size = {{ block_size }}; + +SELECT key, count(1) AS cnt +FROM ( + SELECT * + FROM ( SELECT key FROM left AS s ) AS data + ALL LEFT JOIN ( SELECT key FROM right GROUP BY key ) AS promo ON promo.key = data.key +) GROUP BY key HAVING count(1) > 1 +; + +SELECT count() == (SELECT count() from left) AND min(key == promo.key) == 1 +FROM ( SELECT key FROM left AS s ) AS data +ALL LEFT JOIN ( SELECT key FROM right GROUP BY key ) AS promo ON promo.key = data.key +; + +{% endfor -%} +{% endfor -%} + +DROP TABLE IF EXISTS left; +DROP TABLE IF EXISTS right; diff --git a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference index 00e893213c0..0345e05303c 100644 --- a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference +++ b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference @@ -2,5 +2,7 @@ -- regression for optimize_aggregation_in_order -- that cause "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" error -select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key; +2 +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings distributed_aggregation_memory_efficient=0; 2 diff --git a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql index 17c4a1dba29..9dc3a24213e 100644 --- a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql +++ b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql @@ -2,11 +2,14 @@ drop table if exists data_02177; create table data_02177 (key Int) Engine=MergeTree() order by key; insert into data_02177 values (1); +set optimize_aggregation_in_order=1; + -- { echoOn } -- regression for optimize_aggregation_in_order -- that cause "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" error -select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key; +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings distributed_aggregation_memory_efficient=0; -- { echoOff } drop table data_02177; diff --git a/tests/queries/0_stateless/02180_group_by_lowcardinality.reference b/tests/queries/0_stateless/02180_group_by_lowcardinality.reference new file mode 100644 index 00000000000..a7149390d1a --- /dev/null +++ b/tests/queries/0_stateless/02180_group_by_lowcardinality.reference @@ -0,0 +1,10 @@ +{"val":"1563.8","avg(toUInt32(val))":null} +{"val":"891.4","avg(toUInt32(val))":null} +{"val":"584.4","avg(toUInt32(val))":null} +{"val":"269","avg(toUInt32(val))":269} +{"val":"1233.4","avg(toUInt32(val))":null} +{"val":"1833","avg(toUInt32(val))":1833} +{"val":"1009.4","avg(toUInt32(val))":null} +{"val":"1178.6","avg(toUInt32(val))":null} +{"val":"372.6","avg(toUInt32(val))":null} +{"val":"232.4","avg(toUInt32(val))":null} diff --git a/tests/queries/0_stateless/02180_group_by_lowcardinality.sql b/tests/queries/0_stateless/02180_group_by_lowcardinality.sql new file mode 100644 index 00000000000..463753a624e --- /dev/null +++ b/tests/queries/0_stateless/02180_group_by_lowcardinality.sql @@ -0,0 +1,10 @@ +create table if not exists t_group_by_lowcardinality(p_date Date, val LowCardinality(Nullable(String))) +engine=MergeTree() partition by p_date order by tuple(); + +insert into t_group_by_lowcardinality select today() as p_date, toString(number/5) as val from numbers(10000); +insert into t_group_by_lowcardinality select today() as p_date, Null as val from numbers(100); + +select val, avg(toUInt32(val)) from t_group_by_lowcardinality group by val limit 10 settings max_threads=1, max_rows_to_group_by=100, group_by_overflow_mode='any' format JSONEachRow; + +drop table if exists t_group_by_lowcardinality; + diff --git a/tests/queries/0_stateless/02187_async_inserts_all_formats.python b/tests/queries/0_stateless/02187_async_inserts_all_formats.python index 0a909451259..65a323ef9db 100644 --- a/tests/queries/0_stateless/02187_async_inserts_all_formats.python +++ b/tests/queries/0_stateless/02187_async_inserts_all_formats.python @@ -28,7 +28,7 @@ def run_test(data_format, gen_data_template, settings): exit(1) formats = client.query("SELECT name FROM system.formats WHERE is_input AND is_output \ - AND name NOT IN ('CapnProto', 'RawBLOB', 'Template', 'ProtobufSingle', 'LineAsString', 'Protobuf') ORDER BY name").strip().split('\n') + AND name NOT IN ('CapnProto', 'RawBLOB', 'Template', 'ProtobufSingle', 'LineAsString', 'Protobuf', 'ProtobufList') ORDER BY name").strip().split('\n') # Generic formats client.query("DROP TABLE IF EXISTS t_async_insert") diff --git a/tests/queries/0_stateless/02205_HTTP_user_agent.python b/tests/queries/0_stateless/02205_HTTP_user_agent.python index 8fb9cea0845..397e06cbe82 100644 --- a/tests/queries/0_stateless/02205_HTTP_user_agent.python +++ b/tests/queries/0_stateless/02205_HTTP_user_agent.python @@ -124,7 +124,8 @@ def test_select(): check_answers(query, EXPECTED_ANSWER) def main(): - t = start_server(1) + # HEAD + GET + t = start_server(2) t.start() test_select() t.join() diff --git a/tests/queries/0_stateless/02205_ephemeral_1.reference b/tests/queries/0_stateless/02205_ephemeral_1.reference new file mode 100644 index 00000000000..6e98ffd2495 --- /dev/null +++ b/tests/queries/0_stateless/02205_ephemeral_1.reference @@ -0,0 +1,8 @@ +x UInt32 DEFAULT y +y UInt32 EPHEMERAL 17 +z UInt32 DEFAULT 5 +1 2 +17 2 +17 5 +7 5 +21 5 diff --git a/tests/queries/0_stateless/02205_ephemeral_1.sql b/tests/queries/0_stateless/02205_ephemeral_1.sql new file mode 100644 index 00000000000..5d0565cbfc0 --- /dev/null +++ b/tests/queries/0_stateless/02205_ephemeral_1.sql @@ -0,0 +1,40 @@ +DROP TABLE IF EXISTS t_ephemeral_02205_1; + +CREATE TABLE t_ephemeral_02205_1 (x UInt32 DEFAULT y, y UInt32 EPHEMERAL 17, z UInt32 DEFAULT 5) ENGINE = Memory; + +DESCRIBE t_ephemeral_02205_1; + +# Test INSERT without columns list - should participate only ordinary columns (x, z) +INSERT INTO t_ephemeral_02205_1 VALUES (1, 2); +# SELECT * should only return ordinary columns (x, z) - ephemeral is not stored in the table +SELECT * FROM t_ephemeral_02205_1; + +TRUNCATE TABLE t_ephemeral_02205_1; + +INSERT INTO t_ephemeral_02205_1 VALUES (DEFAULT, 2); +SELECT * FROM t_ephemeral_02205_1; + +TRUNCATE TABLE t_ephemeral_02205_1; + +# Test INSERT using ephemerals default +INSERT INTO t_ephemeral_02205_1 (x, y) VALUES (DEFAULT, DEFAULT); +SELECT * FROM t_ephemeral_02205_1; + +TRUNCATE TABLE t_ephemeral_02205_1; + +# Test INSERT using explicit ephemerals value +INSERT INTO t_ephemeral_02205_1 (x, y) VALUES (DEFAULT, 7); +SELECT * FROM t_ephemeral_02205_1; + +# Test ALTER TABLE DELETE +ALTER TABLE t_ephemeral_02205_1 DELETE WHERE x = 7; +SELECT * FROM t_ephemeral_02205_1; + +TRUNCATE TABLE t_ephemeral_02205_1; + +# Test INSERT into column, defaulted to ephemeral, but explicitly provided with value +INSERT INTO t_ephemeral_02205_1 (x, y) VALUES (21, 7); +SELECT * FROM t_ephemeral_02205_1; + + +DROP TABLE IF EXISTS t_ephemeral_02205_1; diff --git a/tests/queries/0_stateless/02206_array_starts_ends_with.reference b/tests/queries/0_stateless/02206_array_starts_ends_with.reference new file mode 100644 index 00000000000..e0dacfc06e0 --- /dev/null +++ b/tests/queries/0_stateless/02206_array_starts_ends_with.reference @@ -0,0 +1,30 @@ +1 +1 +0 +- +1 +1 +0 +1 +0 +- +1 +0 +1 +0 +- +1 +1 +0 +- +1 +1 +0 +1 +0 +- +1 +0 +- +1 +1 diff --git a/tests/queries/0_stateless/02206_array_starts_ends_with.sql b/tests/queries/0_stateless/02206_array_starts_ends_with.sql new file mode 100644 index 00000000000..39b02c29dc0 --- /dev/null +++ b/tests/queries/0_stateless/02206_array_starts_ends_with.sql @@ -0,0 +1,36 @@ +select startsWith([], []); +select startsWith([1], []); +select startsWith([], [1]); +select '-'; + +select startsWith([NULL], [NULL]); +select startsWith([NULL], []); +select startsWith([], [NULL]); +select startsWith([NULL, 1], [NULL]); +select startsWith([NULL, 1], [1]); +select '-'; + +select startsWith([1, 2, 3, 4], [1, 2, 3]); +select startsWith([1, 2, 3, 4], [1, 2, 4]); +select startsWith(['a', 'b', 'c'], ['a', 'b']); +select startsWith(['a', 'b', 'c'], ['b']); +select '-'; + +select endsWith([], []); +select endsWith([1], []); +select endsWith([], [1]); +select '-'; + +select endsWith([NULL], [NULL]); +select endsWith([NULL], []); +select endsWith([], [NULL]); +select endsWith([1, NULL], [NULL]); +select endsWith([NULL, 1], [NULL]); +select '-'; + +select endsWith([1, 2, 3, 4], [3, 4]); +select endsWith([1, 2, 3, 4], [3]); +select '-'; + +select startsWith([1], emptyArrayUInt8()); +select endsWith([1], emptyArrayUInt8()); diff --git a/tests/queries/0_stateless/02206_format_override.reference b/tests/queries/0_stateless/02206_format_override.reference new file mode 100644 index 00000000000..e1bb01eeb2f --- /dev/null +++ b/tests/queries/0_stateless/02206_format_override.reference @@ -0,0 +1,33 @@ +File generated: +Options: --input-format=CSV --output-format JSONEachRow --format TSV +{"num1":"0","num2":"0"} +{"num1":"1","num2":"2"} +{"num1":"2","num2":"4"} +{"num1":"3","num2":"6"} +{"num1":"4","num2":"8"} +{"num1":"5","num2":"10"} +{"num1":"6","num2":"12"} +Options: --input-format=CSV --format TSV +0 0 +1 2 +2 4 +3 6 +4 8 +5 10 +6 12 +Options: --output-format=JSONEachRow --format CSV +{"num1":"0","num2":"0"} +{"num1":"1","num2":"2"} +{"num1":"2","num2":"4"} +{"num1":"3","num2":"6"} +{"num1":"4","num2":"8"} +{"num1":"5","num2":"10"} +{"num1":"6","num2":"12"} +Options: --format CSV +0,0 +1,2 +2,4 +3,6 +4,8 +5,10 +6,12 diff --git a/tests/queries/0_stateless/02206_format_override.sh b/tests/queries/0_stateless/02206_format_override.sh new file mode 100755 index 00000000000..1359f1edeb8 --- /dev/null +++ b/tests/queries/0_stateless/02206_format_override.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +SAMPLE_FILE="$CURDIR/02206_sample_data.csv" + +echo 'File generated:' +${CLICKHOUSE_LOCAL} -q "SELECT number, number * 2 from numbers(7) FORMAT TSV" | tr '\t' ',' >"$SAMPLE_FILE" + + +echo "Options: --input-format=CSV --output-format JSONEachRow --format TSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --input-format CSV --output-format JSONEachRow --format TSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +echo "Options: --input-format=CSV --format TSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --input-format CSV --format TSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +echo "Options: --output-format=JSONEachRow --format CSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --output-format JSONEachRow --format CSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +echo "Options: --format CSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --format CSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +rm "$SAMPLE_FILE" \ No newline at end of file diff --git a/tests/queries/0_stateless/02206_information_schema_show_database.reference b/tests/queries/0_stateless/02206_information_schema_show_database.reference index 551186fa0ab..af437aca989 100644 --- a/tests/queries/0_stateless/02206_information_schema_show_database.reference +++ b/tests/queries/0_stateless/02206_information_schema_show_database.reference @@ -1 +1,4 @@ CREATE DATABASE INFORMATION_SCHEMA\nENGINE = Memory +CREATE VIEW INFORMATION_SCHEMA.COLUMNS\n(\n `table_catalog` String,\n `table_schema` String,\n `table_name` String,\n `column_name` String,\n `ordinal_position` UInt64,\n `column_default` String,\n `is_nullable` UInt8,\n `data_type` String,\n `character_maximum_length` Nullable(UInt64),\n `character_octet_length` Nullable(UInt64),\n `numeric_precision` Nullable(UInt64),\n `numeric_precision_radix` Nullable(UInt64),\n `numeric_scale` Nullable(UInt64),\n `datetime_precision` Nullable(UInt64),\n `character_set_catalog` Nullable(String),\n `character_set_schema` Nullable(String),\n `character_set_name` Nullable(String),\n `collation_catalog` Nullable(String),\n `collation_schema` Nullable(String),\n `collation_name` Nullable(String),\n `domain_catalog` Nullable(String),\n `domain_schema` Nullable(String),\n `domain_name` Nullable(String),\n `column_comment` String,\n `column_type` String,\n `TABLE_CATALOG` String ALIAS table_catalog,\n `TABLE_SCHEMA` String ALIAS table_schema,\n `TABLE_NAME` String ALIAS table_name,\n `COLUMN_NAME` String ALIAS column_name,\n `ORDINAL_POSITION` UInt64 ALIAS ordinal_position,\n `COLUMN_DEFAULT` String ALIAS column_default,\n `IS_NULLABLE` UInt8 ALIAS is_nullable,\n `DATA_TYPE` String ALIAS data_type,\n `CHARACTER_MAXIMUM_LENGTH` Nullable(UInt64) ALIAS character_maximum_length,\n `CHARACTER_OCTET_LENGTH` Nullable(UInt64) ALIAS character_octet_length,\n `NUMERIC_PRECISION` Nullable(UInt64) ALIAS numeric_precision,\n `NUMERIC_PRECISION_RADIX` Nullable(UInt64) ALIAS numeric_precision_radix,\n `NUMERIC_SCALE` Nullable(UInt64) ALIAS numeric_scale,\n `DATETIME_PRECISION` Nullable(UInt64) ALIAS datetime_precision,\n `CHARACTER_SET_CATALOG` Nullable(String) ALIAS character_set_catalog,\n `CHARACTER_SET_SCHEMA` Nullable(String) ALIAS character_set_schema,\n `CHARACTER_SET_NAME` Nullable(String) ALIAS character_set_name,\n `COLLATION_CATALOG` Nullable(String) ALIAS collation_catalog,\n `COLLATION_SCHEMA` Nullable(String) ALIAS collation_schema,\n `COLLATION_NAME` Nullable(String) ALIAS collation_name,\n `DOMAIN_CATALOG` Nullable(String) ALIAS domain_catalog,\n `DOMAIN_SCHEMA` Nullable(String) ALIAS domain_schema,\n `DOMAIN_NAME` Nullable(String) ALIAS domain_name,\n `COLUMN_COMMENT` String ALIAS column_comment,\n `COLUMN_TYPE` String ALIAS column_type\n) AS\nSELECT\n database AS table_catalog,\n database AS table_schema,\n table AS table_name,\n name AS column_name,\n position AS ordinal_position,\n default_expression AS column_default,\n type LIKE \'Nullable(%)\' AS is_nullable,\n type AS data_type,\n character_octet_length AS character_maximum_length,\n character_octet_length,\n numeric_precision,\n numeric_precision_radix,\n numeric_scale,\n datetime_precision,\n NULL AS character_set_catalog,\n NULL AS character_set_schema,\n NULL AS character_set_name,\n NULL AS collation_catalog,\n NULL AS collation_schema,\n NULL AS collation_name,\n NULL AS domain_catalog,\n NULL AS domain_schema,\n NULL AS domain_name,\n comment AS column_comment,\n type AS column_type\nFROM system.columns +CREATE VIEW INFORMATION_SCHEMA.TABLES (`table_catalog` String, `table_schema` String, `table_name` String, `table_type` Enum8(\'BASE TABLE\' = 1, \'VIEW\' = 2, \'FOREIGN TABLE\' = 3, \'LOCAL TEMPORARY\' = 4, \'SYSTEM VIEW\' = 5), `TABLE_CATALOG` String ALIAS table_catalog, `TABLE_SCHEMA` String ALIAS table_schema, `TABLE_NAME` String ALIAS table_name, `TABLE_TYPE` Enum8(\'BASE TABLE\' = 1, \'VIEW\' = 2, \'FOREIGN TABLE\' = 3, \'LOCAL TEMPORARY\' = 4, \'SYSTEM VIEW\' = 5) ALIAS table_type) AS SELECT database AS table_catalog, database AS table_schema, name AS table_name, multiIf(is_temporary, 4, engine LIKE \'%View\', 2, engine LIKE \'System%\', 5, has_own_data = 0, 3, 1) AS table_type FROM system.tables +CREATE VIEW information_schema.tables (`table_catalog` String, `table_schema` String, `table_name` String, `table_type` Enum8(\'BASE TABLE\' = 1, \'VIEW\' = 2, \'FOREIGN TABLE\' = 3, \'LOCAL TEMPORARY\' = 4, \'SYSTEM VIEW\' = 5), `TABLE_CATALOG` String ALIAS table_catalog, `TABLE_SCHEMA` String ALIAS table_schema, `TABLE_NAME` String ALIAS table_name, `TABLE_TYPE` Enum8(\'BASE TABLE\' = 1, \'VIEW\' = 2, \'FOREIGN TABLE\' = 3, \'LOCAL TEMPORARY\' = 4, \'SYSTEM VIEW\' = 5) ALIAS table_type) AS SELECT database AS table_catalog, database AS table_schema, name AS table_name, multiIf(is_temporary, 4, engine LIKE \'%View\', 2, engine LIKE \'System%\', 5, has_own_data = 0, 3, 1) AS table_type FROM system.tables diff --git a/tests/queries/0_stateless/02206_information_schema_show_database.sql b/tests/queries/0_stateless/02206_information_schema_show_database.sql index de5ca495e2e..91a8a0d1dea 100644 --- a/tests/queries/0_stateless/02206_information_schema_show_database.sql +++ b/tests/queries/0_stateless/02206_information_schema_show_database.sql @@ -1 +1,3 @@ SHOW CREATE DATABASE INFORMATION_SCHEMA; +SHOW CREATE INFORMATION_SCHEMA.COLUMNS; +SELECT create_table_query FROM system.tables WHERE database ILIKE 'INFORMATION_SCHEMA' AND table ILIKE 'TABLES'; -- supress style check: database = currentDatabase() diff --git a/tests/queries/0_stateless/02206_minimum_sample_size.reference b/tests/queries/0_stateless/02206_minimum_sample_size.reference new file mode 100644 index 00000000000..91d6d331007 --- /dev/null +++ b/tests/queries/0_stateless/02206_minimum_sample_size.reference @@ -0,0 +1,22 @@ +continous const 1 1569.78 19 21 +continous const 2 inf 0 0 +continous const 3 1569.78 19 21 +continous const 4 1569.78 19 21 +continous UInt64 1 15.7 190 210 +continous UInt64 1 1569.78 19 21 +continous UInt64 2 1569.78 19 21 +continous UInt64 2 1569.78 19 21 +continous UInt64 3 15.7 190 210 +continous UInt64 3 1569.78 19 21 +continous Float64 1 15.7 190 210 +continous Float64 1 1569.78 19 21 +continous Float64 2 1569.78 19 21 +continous Float64 2 1569.78 19 21 +continous UInt64 3 15.7 190 210 +continous UInt64 3 1569.78 19 21 +conversion const 1 13494.97 0.89 0.91 +conversion const 2 779.78 -0.01 0.01 +conversion Float64 1 13494.97 0.89 0.91 +conversion Float64 1 24640.38 0.79 0.81 +conversion Float64 2 13494.97 0.89 0.91 +conversion Float64 2 13494.97 0.89 0.91 diff --git a/tests/queries/0_stateless/02206_minimum_sample_size.sql b/tests/queries/0_stateless/02206_minimum_sample_size.sql new file mode 100644 index 00000000000..b8f153faafb --- /dev/null +++ b/tests/queries/0_stateless/02206_minimum_sample_size.sql @@ -0,0 +1,33 @@ +WITH minSampleSizeContinous(20, 10, 0.05, 0.8, 0.05) AS res SELECT 'continous const 1', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2); +WITH minSampleSizeContinous(0.0, 10.0, 0.05, 0.8, 0.05) AS res SELECT 'continous const 2', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2); +WITH minSampleSizeContinous(20, 10.0, 0.05, 0.8, 0.05) AS res SELECT 'continous const 3', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2); +WITH minSampleSizeContinous(20.0, 10, 0.05, 0.8, 0.05) AS res SELECT 'continous const 4', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2); + +DROP TABLE IF EXISTS minimum_sample_size_continuos; +CREATE TABLE minimum_sample_size_continuos (baseline UInt64, sigma UInt64) ENGINE = Memory(); +INSERT INTO minimum_sample_size_continuos VALUES (20, 10); +INSERT INTO minimum_sample_size_continuos VALUES (200, 10); +WITH minSampleSizeContinous(baseline, sigma, 0.05, 0.8, 0.05) AS res SELECT 'continous UInt64 1', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_continuos ORDER BY roundBankers(res.1, 2); +WITH minSampleSizeContinous(20, sigma, 0.05, 0.8, 0.05) AS res SELECT 'continous UInt64 2', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_continuos ORDER BY roundBankers(res.1, 2); +WITH minSampleSizeContinous(baseline, 10, 0.05, 0.8, 0.05) AS res SELECT 'continous UInt64 3', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_continuos ORDER BY roundBankers(res.1, 2); +DROP TABLE IF EXISTS minimum_sample_size_continuos; + +DROP TABLE IF EXISTS minimum_sample_size_continuos; +CREATE TABLE minimum_sample_size_continuos (baseline Float64, sigma Float64) ENGINE = Memory(); +INSERT INTO minimum_sample_size_continuos VALUES (20, 10); +INSERT INTO minimum_sample_size_continuos VALUES (200, 10); +WITH minSampleSizeContinous(baseline, sigma, 0.05, 0.8, 0.05) AS res SELECT 'continous Float64 1', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_continuos ORDER BY roundBankers(res.1, 2); +WITH minSampleSizeContinous(20, sigma, 0.05, 0.8, 0.05) AS res SELECT 'continous Float64 2', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_continuos ORDER BY roundBankers(res.1, 2); +WITH minSampleSizeContinous(baseline, 10, 0.05, 0.8, 0.05) AS res SELECT 'continous UInt64 3', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_continuos ORDER BY roundBankers(res.1, 2); +DROP TABLE IF EXISTS minimum_sample_size_continuos; + +WITH minSampleSizeConversion(0.9, 0.01, 0.8, 0.05) AS res SELECT 'conversion const 1', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2); +WITH minSampleSizeConversion(0.0, 0.01, 0.8, 0.05) AS res SELECT 'conversion const 2', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2); + +DROP TABLE IF EXISTS minimum_sample_size_conversion; +CREATE TABLE minimum_sample_size_conversion (p1 Float64) ENGINE = Memory(); +INSERT INTO minimum_sample_size_conversion VALUES (0.9); +INSERT INTO minimum_sample_size_conversion VALUES (0.8); +WITH minSampleSizeConversion(p1, 0.01, 0.8, 0.05) AS res SELECT 'conversion Float64 1', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_conversion ORDER BY roundBankers(res.1, 2); +WITH minSampleSizeConversion(0.9, 0.01, 0.8, 0.05) AS res SELECT 'conversion Float64 2', roundBankers(res.1, 2), roundBankers(res.2, 2), roundBankers(res.3, 2) FROM minimum_sample_size_conversion ORDER BY roundBankers(res.1, 2); +DROP TABLE IF EXISTS minimum_sample_size_conversion; diff --git a/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.config.xml b/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.config.xml new file mode 100644 index 00000000000..891fb45e4ba --- /dev/null +++ b/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.config.xml @@ -0,0 +1,24 @@ + + + + trace + true + + + 9000 + 0 + 0 + . + 0 + + + + + users.xml + + + + ./ + + + diff --git a/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.reference b/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.sh b/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.sh new file mode 100755 index 00000000000..693f1d817e3 --- /dev/null +++ b/tests/queries/0_stateless/02207_allow_plaintext_and_no_password.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-parallel, no-fasttest +# Tag no-tsan: requires jemalloc to track small allocations +# Tag no-asan: requires jemalloc to track small allocations +# Tag no-ubsan: requires jemalloc to track small allocations +# Tag no-msan: requires jemalloc to track small allocations + + + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +cp /etc/clickhouse-server/users.xml "$CURDIR"/users.xml +sed -i 's/<\/password>/c64c5e4e53ea1a9f1427d2713b3a22bbebe8940bc807adaf654744b1568c70ab<\/password_sha256_hex>/g' "$CURDIR"/users.xml + sed -i 's//1<\/access_management>/g' "$CURDIR"/users.xml + +server_opts=( + "--config-file=$CURDIR/$(basename "${BASH_SOURCE[0]}" .sh).config.xml" + "--" + # to avoid multiple listen sockets (complexity for port discovering) + "--listen_host=127.1" + # we will discover the real port later. + "--tcp_port=0" + "--shutdown_wait_unfinished=0" +) + +CLICKHOUSE_WATCHDOG_ENABLE=0 $CLICKHOUSE_SERVER_BINARY "${server_opts[@]}" &> clickhouse-server.stderr & +server_pid=$! + +server_port= +i=0 retries=300 +# wait until server will start to listen (max 30 seconds) +while [[ -z $server_port ]] && [[ $i -lt $retries ]]; do + server_port=$(lsof -n -a -P -i tcp -s tcp:LISTEN -p $server_pid 2>/dev/null | awk -F'[ :]' '/LISTEN/ { print $(NF-1) }') + ((++i)) + sleep 0.1 + if ! kill -0 $server_pid >& /dev/null; then + echo "No server (pid $server_pid)" + break + fi +done +if [[ -z $server_port ]]; then + echo "Cannot wait for LISTEN socket" >&2 + exit 1 +fi + +# wait for the server to start accepting tcp connections (max 30 seconds) +i=0 retries=300 +while ! $CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" --format Null -q 'select 1' 2>/dev/null && [[ $i -lt $retries ]]; do + sleep 0.1 + if ! kill -0 $server_pid >& /dev/null; then + echo "No server (pid $server_pid)" + break + fi +done + + +if ! $CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" --format Null -q 'select 1'; then + echo "Cannot wait until server will start accepting connections on " >&2 + exit 1 +fi + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q " DROP USER IF EXISTS u_02207, u1_02207"; + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q "CREATE USER u_02207 IDENTIFIED WITH double_sha1_hash BY '8DCDD69CE7D121DE8013062AEAEB2A148910D50E' +" + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q " CREATE USER u1_02207 IDENTIFIED BY 'qwe123'"; + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q "CREATE USER u2_02207 HOST IP '127.1' IDENTIFIED WITH plaintext_password BY 'qwerty' " " -- { serverError 516 } --" &> /dev/null ; + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q "CREATE USER u3_02207 HOST IP '127.1' IDENTIFIED WITH no_password " " -- { serverError 516 } --" &> /dev/null ; + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q "CREATE USER u4_02207 HOST IP '127.1' NOT IDENTIFIED " " -- { serverError 516 } --" &> /dev/null ; + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q "CREATE USER IF NOT EXISTS u5_02207 " " -- { serverError 516 } --" &> /dev/null ; + +$CLICKHOUSE_CLIENT_BINARY -u default --password='1w2swhb1' --host 127.1 --port "$server_port" -q " DROP USER u_02207, u1_02207"; + + +# no sleep, since flushing to stderr should not be buffered. + grep 'User is not allowed to Create users' clickhouse-server.stderr + + +# send TERM and save the error code to ensure that it is 0 (EXIT_SUCCESS) +kill $server_pid +wait $server_pid +return_code=$? + +rm -f clickhouse-server.stderr +rm -f "$CURDIR"/users.xml + +exit $return_code diff --git a/tests/queries/0_stateless/02207_s3_content_type.reference b/tests/queries/0_stateless/02207_s3_content_type.reference new file mode 100644 index 00000000000..b015e4a148c --- /dev/null +++ b/tests/queries/0_stateless/02207_s3_content_type.reference @@ -0,0 +1,2 @@ +ContentLength:6888890 +ContentType:binary/octet-stream diff --git a/tests/queries/0_stateless/02207_s3_content_type.sh b/tests/queries/0_stateless/02207_s3_content_type.sh new file mode 100755 index 00000000000..ca75b36c688 --- /dev/null +++ b/tests/queries/0_stateless/02207_s3_content_type.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: needs s3 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query " +INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/content-type.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10000, s3_truncate_on_insert = 1; +" + +aws --endpoint-url http://localhost:11111 s3api head-object --bucket test --key content-type.csv | grep Content | sed 's/[ \t,"]*//g' diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.reference b/tests/queries/0_stateless/02207_subseconds_intervals.reference new file mode 100644 index 00000000000..f7b91ff48b8 --- /dev/null +++ b/tests/queries/0_stateless/02207_subseconds_intervals.reference @@ -0,0 +1,62 @@ +test intervals +- test nanoseconds +1980-12-12 12:12:12.123456789 +1980-12-12 12:12:12.123456700 +1980-12-12 12:12:12.123456789 +1930-12-12 12:12:12.123456789 +1930-12-12 12:12:12.123456700 +2220-12-12 12:12:12.123456789 +2220-12-12 12:12:12.123456700 +- test microseconds +1980-12-12 12:12:12.123456 +1980-12-12 12:12:12.123400 +1980-12-12 12:12:12.123456 +1980-12-12 12:12:12.123456 +1930-12-12 12:12:12.123456 +1930-12-12 12:12:12.123400 +1930-12-12 12:12:12.123457 +2220-12-12 12:12:12.123456 +2220-12-12 12:12:12.123400 +2220-12-12 12:12:12.123456 +- test milliseconds +1980-12-12 12:12:12.123 +1980-12-12 12:12:12.120 +1980-12-12 12:12:12.123 +1980-12-12 12:12:12.123 +1930-12-12 12:12:12.123 +1930-12-12 12:12:12.120 +1930-12-12 12:12:12.124 +2220-12-12 12:12:12.123 +2220-12-12 12:12:12.120 +2220-12-12 12:12:12.123 +test add[...]seconds() +- test nanoseconds +1980-12-12 12:12:12.123456790 +1980-12-12 12:12:12.123456701 +1980-12-12 12:12:12.123456790 +1930-12-12 12:12:12.123456788 +1930-12-12 12:12:12.123456699 +2220-12-12 12:12:12.123456790 +2220-12-12 12:12:12.123456701 +- test microseconds +1980-12-12 12:12:12.123457 +1980-12-12 12:12:12.123401 +1980-12-12 12:12:12.12345778 +1980-12-12 12:12:12.123457 +1930-12-12 12:12:12.123455 +1930-12-12 12:12:12.123399 +1930-12-12 12:12:12.12345578 +2220-12-12 12:12:12.123457 +2220-12-12 12:12:12.123401 +2220-12-12 12:12:12.12345778 +- test milliseconds +1980-12-12 12:12:12.124 +1980-12-12 12:12:12.121 +1980-12-12 12:12:12.124456 +1980-12-12 12:12:12.124 +1930-12-12 12:12:12.122 +1930-12-12 12:12:12.119 +1930-12-12 12:12:12.122456 +2220-12-12 12:12:12.124 +2220-12-12 12:12:12.121 +2220-12-12 12:12:12.124456 diff --git a/tests/queries/0_stateless/02207_subseconds_intervals.sql b/tests/queries/0_stateless/02207_subseconds_intervals.sql new file mode 100644 index 00000000000..a7ce03d9330 --- /dev/null +++ b/tests/queries/0_stateless/02207_subseconds_intervals.sql @@ -0,0 +1,94 @@ +SELECT 'test intervals'; + +SELECT '- test nanoseconds'; +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.123456789', 9), INTERVAL 1 NANOSECOND); -- In normal range, source scale matches result +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.1234567', 7), INTERVAL 1 NANOSECOND); -- In normal range, source scale less than result + +select toStartOfInterval(a, INTERVAL 1 NANOSECOND) from ( select toDateTime64('1980-12-12 12:12:12.123456789', 9) AS a ); -- Non-constant argument + +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.123456789', 9), INTERVAL 1 NANOSECOND); -- Below normal range, source scale matches result +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.1234567', 7), INTERVAL 1 NANOSECOND); -- Below normal range, source scale less than result + +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.123456789', 9), INTERVAL 1 NANOSECOND); -- Above normal range, source scale matches result +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.1234567', 7), INTERVAL 1 NANOSECOND); -- Above normal range, source scale less than result + + +SELECT '- test microseconds'; +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.123456', 6), INTERVAL 1 MICROSECOND); -- In normal range, source scale matches result +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.1234', 4), INTERVAL 1 MICROSECOND); -- In normal range, source scale less than result +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.12345678', 8), INTERVAL 1 MICROSECOND); -- In normal range, source scale greater than result + +select toStartOfInterval(a, INTERVAL 1 MICROSECOND) from ( select toDateTime64('1980-12-12 12:12:12.12345678', 8) AS a ); -- Non-constant argument + +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.123456', 6), INTERVAL 1 MICROSECOND); -- Below normal range, source scale matches result +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.1234', 4), INTERVAL 1 MICROSECOND); -- Below normal range, source scale less than result +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.12345678', 8), INTERVAL 1 MICROSECOND); -- Below normal range, source scale greater than result + + +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.123456', 6), INTERVAL 1 MICROSECOND); -- Above normal range, source scale matches result +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.1234', 4), INTERVAL 1 MICROSECOND); -- Above normal range, source scale less than result +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.12345678', 8), INTERVAL 1 MICROSECOND); -- Above normal range, source scale greater than result + + +SELECT '- test milliseconds'; +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.123', 3), INTERVAL 1 MILLISECOND); -- In normal range, source scale matches result +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.12', 2), INTERVAL 1 MILLISECOND); -- In normal range, source scale less than result +select toStartOfInterval(toDateTime64('1980-12-12 12:12:12.123456', 6), INTERVAL 1 MILLISECOND); -- In normal range, source scale greater than result + +select toStartOfInterval(a, INTERVAL 1 MILLISECOND) from ( select toDateTime64('1980-12-12 12:12:12.123456', 6) AS a ); -- Non-constant argument + +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.123', 3), INTERVAL 1 MILLISECOND); -- Below normal range, source scale matches result +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.12', 2), INTERVAL 1 MILLISECOND); -- Below normal range, source scale less than result +select toStartOfInterval(toDateTime64('1930-12-12 12:12:12.123456', 6), INTERVAL 1 MILLISECOND); -- Below normal range, source scale greater than result + +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.123', 3), INTERVAL 1 MILLISECOND); -- Above normal range, source scale matches result +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.12', 2), INTERVAL 1 MILLISECOND); -- Above normal range, source scale less than result +select toStartOfInterval(toDateTime64('2220-12-12 12:12:12.123456', 6), INTERVAL 1 MILLISECOND); -- Above normal range, source scale greater than result + + +SELECT 'test add[...]seconds()'; + + +SELECT '- test nanoseconds'; +select addNanoseconds(toDateTime64('1980-12-12 12:12:12.123456789', 9), 1); -- In normal range, source scale matches result +select addNanoseconds(toDateTime64('1980-12-12 12:12:12.1234567', 7), 1); -- In normal range, source scale less than result + +select addNanoseconds(a, 1) from ( select toDateTime64('1980-12-12 12:12:12.123456789', 9) AS a ); -- Non-constant argument + +select addNanoseconds(toDateTime64('1930-12-12 12:12:12.123456789', 9), 1); -- Below normal range, source scale matches result +select addNanoseconds(toDateTime64('1930-12-12 12:12:12.1234567', 7), 1); -- Below normal range, source scale less than result + +select addNanoseconds(toDateTime64('2220-12-12 12:12:12.123456789', 9), 1); -- Above normal range, source scale matches result +select addNanoseconds(toDateTime64('2220-12-12 12:12:12.1234567', 7), 1); -- Above normal range, source scale less than result + + +SELECT '- test microseconds'; +select addMicroseconds(toDateTime64('1980-12-12 12:12:12.123456', 6), 1); -- In normal range, source scale matches result +select addMicroseconds(toDateTime64('1980-12-12 12:12:12.1234', 4), 1); -- In normal range, source scale less than result +select addMicroseconds(toDateTime64('1980-12-12 12:12:12.12345678', 8), 1); -- In normal range, source scale greater than result + +select addMicroseconds(a, 1) from ( select toDateTime64('1980-12-12 12:12:12.123456', 6) AS a ); -- Non-constant argument + +select addMicroseconds(toDateTime64('1930-12-12 12:12:12.123456', 6), 1); -- Below normal range, source scale matches result +select addMicroseconds(toDateTime64('1930-12-12 12:12:12.1234', 4), 1); -- Below normal range, source scale less than result +select addMicroseconds(toDateTime64('1930-12-12 12:12:12.12345678', 8), 1); -- Below normal range, source scale greater than result + +select addMicroseconds(toDateTime64('2220-12-12 12:12:12.123456', 6), 1); -- Above normal range, source scale matches result +select addMicroseconds(toDateTime64('2220-12-12 12:12:12.1234', 4), 1); -- Above normal range, source scale less than result +select addMicroseconds(toDateTime64('2220-12-12 12:12:12.12345678', 8), 1); -- Above normal range, source scale greater than result + + +SELECT '- test milliseconds'; +select addMilliseconds(toDateTime64('1980-12-12 12:12:12.123', 3), 1); -- In normal range, source scale matches result +select addMilliseconds(toDateTime64('1980-12-12 12:12:12.12', 2), 1); -- In normal range, source scale less than result +select addMilliseconds(toDateTime64('1980-12-12 12:12:12.123456', 6), 1); -- In normal range, source scale greater than result + +select addMilliseconds(a, 1) from ( select toDateTime64('1980-12-12 12:12:12.123', 3) AS a ); -- Non-constant argument + +select addMilliseconds(toDateTime64('1930-12-12 12:12:12.123', 3), 1); -- Below normal range, source scale matches result +select addMilliseconds(toDateTime64('1930-12-12 12:12:12.12', 2), 1); -- Below normal range, source scale less than result +select addMilliseconds(toDateTime64('1930-12-12 12:12:12.123456', 6), 1); -- Below normal range, source scale greater than result + +select addMilliseconds(toDateTime64('2220-12-12 12:12:12.123', 3), 1); -- Above normal range, source scale matches result +select addMilliseconds(toDateTime64('2220-12-12 12:12:12.12', 2), 1); -- Above normal range, source scale less than result +select addMilliseconds(toDateTime64('2220-12-12 12:12:12.123456', 6), 1); -- Above normal range, source scale greater than result diff --git a/tests/queries/0_stateless/02207_ttl_move_if_exists.reference b/tests/queries/0_stateless/02207_ttl_move_if_exists.reference new file mode 100644 index 00000000000..bedef1a5ceb --- /dev/null +++ b/tests/queries/0_stateless/02207_ttl_move_if_exists.reference @@ -0,0 +1 @@ +CREATE TABLE default.t_ttl_move_if_exists\n(\n `d` DateTime,\n `a` UInt32\n)\nENGINE = MergeTree\nORDER BY tuple()\nTTL d TO DISK IF EXISTS \'non_existing_disk\'\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02207_ttl_move_if_exists.sql b/tests/queries/0_stateless/02207_ttl_move_if_exists.sql new file mode 100644 index 00000000000..ab17d343e49 --- /dev/null +++ b/tests/queries/0_stateless/02207_ttl_move_if_exists.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS t_ttl_move_if_exists; + +CREATE TABLE t_ttl_move_if_exists (d DateTime, a UInt32) +ENGINE = MergeTree ORDER BY tuple() +TTL d TO DISK IF EXISTS 'non_existing_disk'; + +SHOW CREATE TABLE t_ttl_move_if_exists; + +DROP TABLE IF EXISTS t_ttl_move_if_exists; diff --git a/tests/queries/0_stateless/02211_shcema_inference_from_stdin.reference b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.reference new file mode 100644 index 00000000000..d176e0ee1ed --- /dev/null +++ b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.reference @@ -0,0 +1,15 @@ +x Nullable(Float64) +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +c1 Nullable(String) +c2 Nullable(String) +c3 Nullable(String) +1 2 3 diff --git a/tests/queries/0_stateless/02211_shcema_inference_from_stdin.sh b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.sh new file mode 100755 index 00000000000..2b469797f89 --- /dev/null +++ b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_LOCAL -q "select toUInt32(number) as x from numbers(10) format JSONEachRow" > data.jsoneachrow + +$CLICKHOUSE_LOCAL -q "desc table table" < data.jsoneachrow +$CLICKHOUSE_LOCAL -q "select * from table" < data.jsoneachrow + +rm data.jsoneachrow + +echo -e "1\t2\t3" | $CLICKHOUSE_LOCAL -q "desc table table" --file=- +echo -e "1\t2\t3" | $CLICKHOUSE_LOCAL -q "select * from table" --file=- + diff --git a/tests/queries/0_stateless/02212_cte_and_table_alias.reference b/tests/queries/0_stateless/02212_cte_and_table_alias.reference new file mode 100644 index 00000000000..1d3d2cc6415 --- /dev/null +++ b/tests/queries/0_stateless/02212_cte_and_table_alias.reference @@ -0,0 +1,4 @@ +5000 +5000 +5000 +5000 diff --git a/tests/queries/0_stateless/02212_cte_and_table_alias.sql b/tests/queries/0_stateless/02212_cte_and_table_alias.sql new file mode 100644 index 00000000000..ce0fba4bf56 --- /dev/null +++ b/tests/queries/0_stateless/02212_cte_and_table_alias.sql @@ -0,0 +1,41 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/19222 +SET enable_global_with_statement = 1; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE a.n < 5000; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE t.n < 5000; + + +SET enable_global_with_statement = 0; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE a.n < 5000; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE t.n < 5000; diff --git a/tests/queries/0_stateless/02212_h3_get_pentagon_indexes.reference b/tests/queries/0_stateless/02212_h3_get_pentagon_indexes.reference new file mode 100644 index 00000000000..843c488e8e1 --- /dev/null +++ b/tests/queries/0_stateless/02212_h3_get_pentagon_indexes.reference @@ -0,0 +1,28 @@ +[576636674163867647,576988517884755967,577340361605644287,577832942814887935,578219970907865087,578536630256664575,578712552117108735,579029211465908223,579416239558885375,579908820768129023,580260664489017343,580612508209905663] +[581109487465660415,581461331186548735,581813174907437055,582305756116680703,582692784209657855,583009443558457343,583185365418901503,583502024767700991,583889052860678143,584381634069921791,584733477790810111,585085321511698431] +[585609238802333695,585961082523222015,586312926244110335,586805507453353983,587192535546331135,587509194895130623,587685116755574783,588001776104374271,588388804197351423,588881385406595071,589233229127483391,589585072848371711] +[590112357393367039,590464201114255359,590816044835143679,591308626044387327,591695654137364479,592012313486163967,592188235346608127,592504894695407615,592891922788384767,593384503997628415,593736347718516735,594088191439405055] +[594615896891195391,594967740612083711,595319584332972031,595812165542215679,596199193635192831,596515852983992319,596691774844436479,597008434193235967,597395462286213119,597888043495456767,598239887216345087,598591730937233407] +[599119489002373119,599471332723261439,599823176444149759,600315757653393407,600702785746370559,601019445095170047,601195366955614207,601512026304413695,601899054397390847,602391635606634495,602743479327522815,603095323048411135] +[603623087690219519,603974931411107839,604326775131996159,604819356341239807,605206384434216959,605523043783016447,605698965643460607,606015624992260095,606402653085237247,606895234294480895,607247078015369215,607598921736257535] +[608126687200149503,608478530921037823,608830374641926143,609322955851169791,609709983944146943,610026643292946431,610202565153390591,610519224502190079,610906252595167231,611398833804410879,611750677525299199,612102521246187519] +[612630286812839935,612982130533728255,613333974254616575,613826555463860223,614213583556837375,614530242905636863,614706164766081023,615022824114880511,615409852207857663,615902433417101311,616254277137989631,616606120858877951] +[617133886438375423,617485730159263743,617837573880152063,618330155089395711,618717183182372863,619033842531172351,619209764391616511,619526423740415999,619913451833393151,620406033042636799,620757876763525119,621109720484413439] +[621637486065516543,621989329786404863,622341173507293183,622833754716536831,623220782809513983,623537442158313471,623713364018757631,624030023367557119,624417051460534271,624909632669777919,625261476390666239,625613320111554559] +[626141085692858367,626492929413746687,626844773134635007,627337354343878655,627724382436855807,628041041785655295,628216963646099455,628533622994898943,628920651087876095,629413232297119743,629765076018008063,630116919738896383] +[630644685320225279,630996529041113599,631348372762001919,631840953971245567,632227982064222719,632544641413022207,632720563273466367,633037222622265855,633424250715243007,633916831924486655,634268675645374975,634620519366263295] +[635148284947595327,635500128668483647,635851972389371967,636344553598615615,636731581691592767,637048241040392255,637224162900836415,637540822249635903,637927850342613055,638420431551856703,638772275272745023,639124118993633343] +[639651884574965767,640003728295854087,640355572016742407,640848153225986055,641235181318963207,641551840667762695,641727762528206855,642044421877006343,642431449969983495,642924031179227143,643275874900115463,643627718621003783] +[644155484202336256,644507327923224576,644859171644112896,645351752853356544,645738780946333696,646055440295133184,646231362155577344,646548021504376832,646935049597353984,647427630806597632,647779474527485952,648131318248374272] +-- test for const cols +[576636674163867647,576988517884755967,577340361605644287,577832942814887935,578219970907865087,578536630256664575,578712552117108735,579029211465908223,579416239558885375,579908820768129023,580260664489017343,580612508209905663] +[581109487465660415,581461331186548735,581813174907437055,582305756116680703,582692784209657855,583009443558457343,583185365418901503,583502024767700991,583889052860678143,584381634069921791,584733477790810111,585085321511698431] +[585609238802333695,585961082523222015,586312926244110335,586805507453353983,587192535546331135,587509194895130623,587685116755574783,588001776104374271,588388804197351423,588881385406595071,589233229127483391,589585072848371711] +[590112357393367039,590464201114255359,590816044835143679,591308626044387327,591695654137364479,592012313486163967,592188235346608127,592504894695407615,592891922788384767,593384503997628415,593736347718516735,594088191439405055] +[594615896891195391,594967740612083711,595319584332972031,595812165542215679,596199193635192831,596515852983992319,596691774844436479,597008434193235967,597395462286213119,597888043495456767,598239887216345087,598591730937233407] +[599119489002373119,599471332723261439,599823176444149759,600315757653393407,600702785746370559,601019445095170047,601195366955614207,601512026304413695,601899054397390847,602391635606634495,602743479327522815,603095323048411135] +[603623087690219519,603974931411107839,604326775131996159,604819356341239807,605206384434216959,605523043783016447,605698965643460607,606015624992260095,606402653085237247,606895234294480895,607247078015369215,607598921736257535] +[608126687200149503,608478530921037823,608830374641926143,609322955851169791,609709983944146943,610026643292946431,610202565153390591,610519224502190079,610906252595167231,611398833804410879,611750677525299199,612102521246187519] +[612630286812839935,612982130533728255,613333974254616575,613826555463860223,614213583556837375,614530242905636863,614706164766081023,615022824114880511,615409852207857663,615902433417101311,616254277137989631,616606120858877951] +[617133886438375423,617485730159263743,617837573880152063,618330155089395711,618717183182372863,619033842531172351,619209764391616511,619526423740415999,619913451833393151,620406033042636799,620757876763525119,621109720484413439] +[621637486065516543,621989329786404863,622341173507293183,622833754716536831,623220782809513983,623537442158313471,623713364018757631,624030023367557119,624417051460534271,624909632669777919,625261476390666239,625613320111554559] diff --git a/tests/queries/0_stateless/02212_h3_get_pentagon_indexes.sql b/tests/queries/0_stateless/02212_h3_get_pentagon_indexes.sql new file mode 100644 index 00000000000..c7a72fed6bc --- /dev/null +++ b/tests/queries/0_stateless/02212_h3_get_pentagon_indexes.sql @@ -0,0 +1,32 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS table1; + +CREATE TABLE table1 (resolution UInt8) ENGINE = Memory; + +INSERT INTO table1 VALUES(0); +INSERT INTO table1 VALUES(1); +INSERT INTO table1 VALUES(2); +INSERT INTO table1 VALUES(3); +INSERT INTO table1 VALUES(4); +INSERT INTO table1 VALUES(5); +INSERT INTO table1 VALUES(6); +INSERT INTO table1 VALUES(7); +INSERT INTO table1 VALUES(8); +INSERT INTO table1 VALUES(9); +INSERT INTO table1 VALUES(10); +INSERT INTO table1 VALUES(11); +INSERT INTO table1 VALUES(12); +INSERT INTO table1 VALUES(13); +INSERT INTO table1 VALUES(14); +INSERT INTO table1 VALUES(15); + + +SELECT h3GetPentagonIndexes(resolution) AS indexes from table1 order by indexes; +SELECT h3GetPentagonIndexes(20) AS indexes; -- { serverError 69 } + +DROP TABLE table1; + +-- tests for const cols +SELECT '-- test for const cols'; +SELECT h3GetPentagonIndexes(arrayJoin([0,1,2,3,4,5,6,7,8,9,10])); diff --git a/tests/queries/0_stateless/02212_h3_get_res0_indexes.reference b/tests/queries/0_stateless/02212_h3_get_res0_indexes.reference new file mode 100644 index 00000000000..299d6e9883f --- /dev/null +++ b/tests/queries/0_stateless/02212_h3_get_res0_indexes.reference @@ -0,0 +1,6 @@ +[576495936675512319,576531121047601151,576566305419689983,576601489791778815,576636674163867647,576671858535956479,576707042908045311,576742227280134143,576777411652222975,576812596024311807,576847780396400639,576882964768489471,576918149140578303,576953333512667135,576988517884755967,577023702256844799,577058886628933631,577094071001022463,577129255373111295,577164439745200127,577199624117288959,577234808489377791,577269992861466623,577305177233555455,577340361605644287,577375545977733119,577410730349821951,577445914721910783,577481099093999615,577516283466088447,577551467838177279,577586652210266111,577621836582354943,577657020954443775,577692205326532607,577727389698621439,577762574070710271,577797758442799103,577832942814887935,577868127186976767,577903311559065599,577938495931154431,577973680303243263,578008864675332095,578044049047420927,578079233419509759,578114417791598591,578149602163687423,578184786535776255,578219970907865087,578255155279953919,578290339652042751,578325524024131583,578360708396220415,578395892768309247,578431077140398079,578466261512486911,578501445884575743,578536630256664575,578571814628753407,578606999000842239,578642183372931071,578677367745019903,578712552117108735,578747736489197567,578782920861286399,578818105233375231,578853289605464063,578888473977552895,578923658349641727,578958842721730559,578994027093819391,579029211465908223,579064395837997055,579099580210085887,579134764582174719,579169948954263551,579205133326352383,579240317698441215,579275502070530047,579310686442618879,579345870814707711,579381055186796543,579416239558885375,579451423930974207,579486608303063039,579521792675151871,579556977047240703,579592161419329535,579627345791418367,579662530163507199,579697714535596031,579732898907684863,579768083279773695,579803267651862527,579838452023951359,579873636396040191,579908820768129023,579944005140217855,579979189512306687,580014373884395519,580049558256484351,580084742628573183,580119927000662015,580155111372750847,580190295744839679,580225480116928511,580260664489017343,580295848861106175,580331033233195007,580366217605283839,580401401977372671,580436586349461503,580471770721550335,580506955093639167,580542139465727999,580577323837816831,580612508209905663,580647692581994495,580682876954083327,580718061326172159,580753245698260991] +[576495936675512319,576531121047601151,576566305419689983,576601489791778815,576636674163867647,576671858535956479,576707042908045311,576742227280134143,576777411652222975,576812596024311807,576847780396400639,576882964768489471,576918149140578303,576953333512667135,576988517884755967,577023702256844799,577058886628933631,577094071001022463,577129255373111295,577164439745200127,577199624117288959,577234808489377791,577269992861466623,577305177233555455,577340361605644287,577375545977733119,577410730349821951,577445914721910783,577481099093999615,577516283466088447,577551467838177279,577586652210266111,577621836582354943,577657020954443775,577692205326532607,577727389698621439,577762574070710271,577797758442799103,577832942814887935,577868127186976767,577903311559065599,577938495931154431,577973680303243263,578008864675332095,578044049047420927,578079233419509759,578114417791598591,578149602163687423,578184786535776255,578219970907865087,578255155279953919,578290339652042751,578325524024131583,578360708396220415,578395892768309247,578431077140398079,578466261512486911,578501445884575743,578536630256664575,578571814628753407,578606999000842239,578642183372931071,578677367745019903,578712552117108735,578747736489197567,578782920861286399,578818105233375231,578853289605464063,578888473977552895,578923658349641727,578958842721730559,578994027093819391,579029211465908223,579064395837997055,579099580210085887,579134764582174719,579169948954263551,579205133326352383,579240317698441215,579275502070530047,579310686442618879,579345870814707711,579381055186796543,579416239558885375,579451423930974207,579486608303063039,579521792675151871,579556977047240703,579592161419329535,579627345791418367,579662530163507199,579697714535596031,579732898907684863,579768083279773695,579803267651862527,579838452023951359,579873636396040191,579908820768129023,579944005140217855,579979189512306687,580014373884395519,580049558256484351,580084742628573183,580119927000662015,580155111372750847,580190295744839679,580225480116928511,580260664489017343,580295848861106175,580331033233195007,580366217605283839,580401401977372671,580436586349461503,580471770721550335,580506955093639167,580542139465727999,580577323837816831,580612508209905663,580647692581994495,580682876954083327,580718061326172159,580753245698260991] +[576495936675512319,576531121047601151,576566305419689983,576601489791778815,576636674163867647,576671858535956479,576707042908045311,576742227280134143,576777411652222975,576812596024311807,576847780396400639,576882964768489471,576918149140578303,576953333512667135,576988517884755967,577023702256844799,577058886628933631,577094071001022463,577129255373111295,577164439745200127,577199624117288959,577234808489377791,577269992861466623,577305177233555455,577340361605644287,577375545977733119,577410730349821951,577445914721910783,577481099093999615,577516283466088447,577551467838177279,577586652210266111,577621836582354943,577657020954443775,577692205326532607,577727389698621439,577762574070710271,577797758442799103,577832942814887935,577868127186976767,577903311559065599,577938495931154431,577973680303243263,578008864675332095,578044049047420927,578079233419509759,578114417791598591,578149602163687423,578184786535776255,578219970907865087,578255155279953919,578290339652042751,578325524024131583,578360708396220415,578395892768309247,578431077140398079,578466261512486911,578501445884575743,578536630256664575,578571814628753407,578606999000842239,578642183372931071,578677367745019903,578712552117108735,578747736489197567,578782920861286399,578818105233375231,578853289605464063,578888473977552895,578923658349641727,578958842721730559,578994027093819391,579029211465908223,579064395837997055,579099580210085887,579134764582174719,579169948954263551,579205133326352383,579240317698441215,579275502070530047,579310686442618879,579345870814707711,579381055186796543,579416239558885375,579451423930974207,579486608303063039,579521792675151871,579556977047240703,579592161419329535,579627345791418367,579662530163507199,579697714535596031,579732898907684863,579768083279773695,579803267651862527,579838452023951359,579873636396040191,579908820768129023,579944005140217855,579979189512306687,580014373884395519,580049558256484351,580084742628573183,580119927000662015,580155111372750847,580190295744839679,580225480116928511,580260664489017343,580295848861106175,580331033233195007,580366217605283839,580401401977372671,580436586349461503,580471770721550335,580506955093639167,580542139465727999,580577323837816831,580612508209905663,580647692581994495,580682876954083327,580718061326172159,580753245698260991] +[576495936675512319,576531121047601151,576566305419689983,576601489791778815,576636674163867647,576671858535956479,576707042908045311,576742227280134143,576777411652222975,576812596024311807,576847780396400639,576882964768489471,576918149140578303,576953333512667135,576988517884755967,577023702256844799,577058886628933631,577094071001022463,577129255373111295,577164439745200127,577199624117288959,577234808489377791,577269992861466623,577305177233555455,577340361605644287,577375545977733119,577410730349821951,577445914721910783,577481099093999615,577516283466088447,577551467838177279,577586652210266111,577621836582354943,577657020954443775,577692205326532607,577727389698621439,577762574070710271,577797758442799103,577832942814887935,577868127186976767,577903311559065599,577938495931154431,577973680303243263,578008864675332095,578044049047420927,578079233419509759,578114417791598591,578149602163687423,578184786535776255,578219970907865087,578255155279953919,578290339652042751,578325524024131583,578360708396220415,578395892768309247,578431077140398079,578466261512486911,578501445884575743,578536630256664575,578571814628753407,578606999000842239,578642183372931071,578677367745019903,578712552117108735,578747736489197567,578782920861286399,578818105233375231,578853289605464063,578888473977552895,578923658349641727,578958842721730559,578994027093819391,579029211465908223,579064395837997055,579099580210085887,579134764582174719,579169948954263551,579205133326352383,579240317698441215,579275502070530047,579310686442618879,579345870814707711,579381055186796543,579416239558885375,579451423930974207,579486608303063039,579521792675151871,579556977047240703,579592161419329535,579627345791418367,579662530163507199,579697714535596031,579732898907684863,579768083279773695,579803267651862527,579838452023951359,579873636396040191,579908820768129023,579944005140217855,579979189512306687,580014373884395519,580049558256484351,580084742628573183,580119927000662015,580155111372750847,580190295744839679,580225480116928511,580260664489017343,580295848861106175,580331033233195007,580366217605283839,580401401977372671,580436586349461503,580471770721550335,580506955093639167,580542139465727999,580577323837816831,580612508209905663,580647692581994495,580682876954083327,580718061326172159,580753245698260991] +[576495936675512319,576531121047601151,576566305419689983,576601489791778815,576636674163867647,576671858535956479,576707042908045311,576742227280134143,576777411652222975,576812596024311807,576847780396400639,576882964768489471,576918149140578303,576953333512667135,576988517884755967,577023702256844799,577058886628933631,577094071001022463,577129255373111295,577164439745200127,577199624117288959,577234808489377791,577269992861466623,577305177233555455,577340361605644287,577375545977733119,577410730349821951,577445914721910783,577481099093999615,577516283466088447,577551467838177279,577586652210266111,577621836582354943,577657020954443775,577692205326532607,577727389698621439,577762574070710271,577797758442799103,577832942814887935,577868127186976767,577903311559065599,577938495931154431,577973680303243263,578008864675332095,578044049047420927,578079233419509759,578114417791598591,578149602163687423,578184786535776255,578219970907865087,578255155279953919,578290339652042751,578325524024131583,578360708396220415,578395892768309247,578431077140398079,578466261512486911,578501445884575743,578536630256664575,578571814628753407,578606999000842239,578642183372931071,578677367745019903,578712552117108735,578747736489197567,578782920861286399,578818105233375231,578853289605464063,578888473977552895,578923658349641727,578958842721730559,578994027093819391,579029211465908223,579064395837997055,579099580210085887,579134764582174719,579169948954263551,579205133326352383,579240317698441215,579275502070530047,579310686442618879,579345870814707711,579381055186796543,579416239558885375,579451423930974207,579486608303063039,579521792675151871,579556977047240703,579592161419329535,579627345791418367,579662530163507199,579697714535596031,579732898907684863,579768083279773695,579803267651862527,579838452023951359,579873636396040191,579908820768129023,579944005140217855,579979189512306687,580014373884395519,580049558256484351,580084742628573183,580119927000662015,580155111372750847,580190295744839679,580225480116928511,580260664489017343,580295848861106175,580331033233195007,580366217605283839,580401401977372671,580436586349461503,580471770721550335,580506955093639167,580542139465727999,580577323837816831,580612508209905663,580647692581994495,580682876954083327,580718061326172159,580753245698260991] +[576495936675512319,576531121047601151,576566305419689983,576601489791778815,576636674163867647,576671858535956479,576707042908045311,576742227280134143,576777411652222975,576812596024311807,576847780396400639,576882964768489471,576918149140578303,576953333512667135,576988517884755967,577023702256844799,577058886628933631,577094071001022463,577129255373111295,577164439745200127,577199624117288959,577234808489377791,577269992861466623,577305177233555455,577340361605644287,577375545977733119,577410730349821951,577445914721910783,577481099093999615,577516283466088447,577551467838177279,577586652210266111,577621836582354943,577657020954443775,577692205326532607,577727389698621439,577762574070710271,577797758442799103,577832942814887935,577868127186976767,577903311559065599,577938495931154431,577973680303243263,578008864675332095,578044049047420927,578079233419509759,578114417791598591,578149602163687423,578184786535776255,578219970907865087,578255155279953919,578290339652042751,578325524024131583,578360708396220415,578395892768309247,578431077140398079,578466261512486911,578501445884575743,578536630256664575,578571814628753407,578606999000842239,578642183372931071,578677367745019903,578712552117108735,578747736489197567,578782920861286399,578818105233375231,578853289605464063,578888473977552895,578923658349641727,578958842721730559,578994027093819391,579029211465908223,579064395837997055,579099580210085887,579134764582174719,579169948954263551,579205133326352383,579240317698441215,579275502070530047,579310686442618879,579345870814707711,579381055186796543,579416239558885375,579451423930974207,579486608303063039,579521792675151871,579556977047240703,579592161419329535,579627345791418367,579662530163507199,579697714535596031,579732898907684863,579768083279773695,579803267651862527,579838452023951359,579873636396040191,579908820768129023,579944005140217855,579979189512306687,580014373884395519,580049558256484351,580084742628573183,580119927000662015,580155111372750847,580190295744839679,580225480116928511,580260664489017343,580295848861106175,580331033233195007,580366217605283839,580401401977372671,580436586349461503,580471770721550335,580506955093639167,580542139465727999,580577323837816831,580612508209905663,580647692581994495,580682876954083327,580718061326172159,580753245698260991] diff --git a/tests/queries/0_stateless/02212_h3_get_res0_indexes.sql b/tests/queries/0_stateless/02212_h3_get_res0_indexes.sql new file mode 100644 index 00000000000..e84f1f43964 --- /dev/null +++ b/tests/queries/0_stateless/02212_h3_get_res0_indexes.sql @@ -0,0 +1,6 @@ +-- Tags: no-fasttest + +SELECT h3GetRes0Indexes(); +SELECT h3GetRes0Indexes(3); -- { serverError 42 } + +SELECT h3GetRes0Indexes() FROM system.numbers LIMIT 5; diff --git a/tests/queries/0_stateless/02212_h3_point_dist.reference b/tests/queries/0_stateless/02212_h3_point_dist.reference new file mode 100644 index 00000000000..00d316ab508 --- /dev/null +++ b/tests/queries/0_stateless/02212_h3_point_dist.reference @@ -0,0 +1,42 @@ +-- select h3PointDistM(lat1, lon1,lat2, lon2) AS k from table1 order by k; +111195.05197522961 +111228.91103262542 +111901.0753744776 +111901.07537447763 +111901.07537447763 +157225.60925091387 +157249.55851177874 +222457.78082261496 +223528.36944466401 +223528.36944466401 +400900.66882205213 +2223901.039504589 +-- select h3PointDistKm(lat1, lon1,lat2, lon2) AS k from table1 order by k; +111.1950519752296 +111.22891103262542 +111.9010753744776 +111.90107537447763 +111.90107537447763 +157.22560925091386 +157.24955851177873 +222.45778082261498 +223.528369444664 +223.528369444664 +400.9006688220521 +2223.901039504589 +-- select h3PointDistRads(lat1, lon1,lat2, lon2) AS k from table1 order by k; +0.01745329251994332 +0.017458607073268143 +0.017564110696598745 +0.01756411069659875 +0.01756411069659875 +0.024678297290546682 +0.02468205639176644 +0.034917207673048706 +0.03508524839120321 +0.03508524839120321 +0.06292579139178688 +0.3490658503988659 +-- test for non const cols +0.3490658503988659 +0.3490658503988659 diff --git a/tests/queries/0_stateless/02212_h3_point_dist.sql b/tests/queries/0_stateless/02212_h3_point_dist.sql new file mode 100644 index 00000000000..ccc806db75e --- /dev/null +++ b/tests/queries/0_stateless/02212_h3_point_dist.sql @@ -0,0 +1,32 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS table1; + +CREATE TABLE table1 (lat1 Float64, lon1 Float64, lat2 Float64, lon2 Float64) ENGINE = Memory; + +INSERT INTO table1 VALUES(-10.0 ,0.0, 10.0, 0.0); +INSERT INTO table1 VALUES(-1, -1, 2, 1); +INSERT INTO table1 VALUES(0, 2, 1, 3); +INSERT INTO table1 VALUES(-2, -3, -1, -2); +INSERT INTO table1 VALUES(-87, 0, -85, 3); +INSERT INTO table1 VALUES(-89, 1, -88, 2); +INSERT INTO table1 VALUES(-84, 1, -83, 2); +INSERT INTO table1 VALUES(-88, 90, -86, 91); +INSERT INTO table1 VALUES(-84, -91, -83, -90); +INSERT INTO table1 VALUES(-90, 181, -89, 182); +INSERT INTO table1 VALUES(-84, 181, -83, 182); +INSERT INTO table1 VALUES(-87, 0, -85, 3); + +select '-- select h3PointDistM(lat1, lon1,lat2, lon2) AS k from table1 order by k;'; +select h3PointDistM(lat1, lon1,lat2, lon2) AS k from table1 order by k; +select '-- select h3PointDistKm(lat1, lon1,lat2, lon2) AS k from table1 order by k;'; +select h3PointDistKm(lat1, lon1,lat2, lon2) AS k from table1 order by k; +select '-- select h3PointDistRads(lat1, lon1,lat2, lon2) AS k from table1 order by k;'; +select h3PointDistRads(lat1, lon1,lat2, lon2) AS k from table1 order by k; + +DROP TABLE table1; + +-- tests for const columns +select '-- test for non const cols'; +select h3PointDistRads(-10.0 ,0.0, 10.0, arrayJoin([0.0])) as h3PointDistRads; +select h3PointDistRads(-10.0 ,0.0, 10.0, toFloat64(0)) as h3PointDistRads; diff --git a/tests/queries/0_stateless/02220_array_join_format.reference b/tests/queries/0_stateless/02220_array_join_format.reference new file mode 100644 index 00000000000..b1978acfbfa --- /dev/null +++ b/tests/queries/0_stateless/02220_array_join_format.reference @@ -0,0 +1,11 @@ +SELECT + range_, + point_ +FROM +( + SELECT + range(0, 10) AS range_, + point_ + FROM system.one + ARRAY JOIN range(0, 10) AS point_ +) diff --git a/tests/queries/0_stateless/02220_array_join_format.sql b/tests/queries/0_stateless/02220_array_join_format.sql new file mode 100644 index 00000000000..afea6855877 --- /dev/null +++ b/tests/queries/0_stateless/02220_array_join_format.sql @@ -0,0 +1 @@ +explain syntax select * from (select range(0, 10) range_, point_ from system.one array join range_ as point_); diff --git a/tests/queries/0_stateless/02221_parallel_replicas_bug.reference b/tests/queries/0_stateless/02221_parallel_replicas_bug.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02221_parallel_replicas_bug.sh b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh new file mode 100755 index 00000000000..b4ac6817a54 --- /dev/null +++ b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 -nmT < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null diff --git a/tests/queries/0_stateless/02221_system_zookeeper_unrestricted.reference b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted.reference new file mode 100644 index 00000000000..bd0c9cee464 --- /dev/null +++ b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted.reference @@ -0,0 +1,74 @@ +1 +1 +alter_partition_version +alter_partition_version +block_numbers +block_numbers +blocks +blocks +columns +columns +columns +columns +failed_parts +failed_parts +flags +flags +host +host +is_active +is_active +is_lost +is_lost +last_part +last_part +leader_election +leader_election +leader_election-0 +leader_election-0 +log +log +log_pointer +log_pointer +max_processed_insert_time +max_processed_insert_time +metadata +metadata +metadata +metadata +metadata_version +metadata_version +min_unprocessed_insert_time +min_unprocessed_insert_time +mutation_pointer +mutation_pointer +mutations +mutations +nonincrement_block_numbers +nonincrement_block_numbers +parallel +parallel +part_moves_shard +part_moves_shard +parts +parts +pinned_part_uuids +pinned_part_uuids +queue +queue +quorum +quorum +replicas +replicas +shared +shared +shared +shared +table_shared_id +table_shared_id +temp +temp +zero_copy_hdfs +zero_copy_hdfs +zero_copy_s3 +zero_copy_s3 diff --git a/tests/queries/0_stateless/02221_system_zookeeper_unrestricted.sh b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted.sh new file mode 100755 index 00000000000..db94c59d2de --- /dev/null +++ b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database, zookeeper + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table_2" + +${CLICKHOUSE_CLIENT} -n -q" +CREATE TABLE sample_table ( + key UInt64 +) +ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted', '1') +ORDER BY tuple(); +" + +${CLICKHOUSE_CLIENT} -n -q" +CREATE TABLE sample_table_2 ( + key UInt64 +) +ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_2', '1') +ORDER BY tuple(); +" + +${CLICKHOUSE_CLIENT} --allow_unrestricted_reads_from_keeper=1 --query "SELECT name FROM (SELECT path, name FROM system.zookeeper ORDER BY name) WHERE path LIKE '%$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted%'"; + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table_2" diff --git a/tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.reference b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.reference new file mode 100644 index 00000000000..f95d60dc07b --- /dev/null +++ b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.reference @@ -0,0 +1,75 @@ +1 +alter_partition_version +block_numbers +blocks +columns +columns +failed_parts +flags +host +is_active +is_lost +last_part +leader_election +leader_election-0 +log +log_pointer +max_processed_insert_time +metadata +metadata +metadata_version +min_unprocessed_insert_time +mutation_pointer +mutations +nonincrement_block_numbers +parallel +part_moves_shard +parts +pinned_part_uuids +queue +quorum +replicas +shared +shared +table_shared_id +temp +zero_copy_hdfs +zero_copy_s3 +------------------------- +1 +alter_partition_version +block_numbers +blocks +columns +columns +failed_parts +flags +host +is_active +is_lost +last_part +leader_election +leader_election-0 +log +log_pointer +max_processed_insert_time +metadata +metadata +metadata_version +min_unprocessed_insert_time +mutation_pointer +mutations +nonincrement_block_numbers +parallel +part_moves_shard +parts +pinned_part_uuids +queue +quorum +replicas +shared +shared +table_shared_id +temp +zero_copy_hdfs +zero_copy_s3 diff --git a/tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.sh b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.sh new file mode 100755 index 00000000000..152d8344764 --- /dev/null +++ b/tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database, zookeeper + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table;" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table_2;" + +${CLICKHOUSE_CLIENT} -n --query="CREATE TABLE sample_table ( + key UInt64 +) +ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_like', '1') +ORDER BY tuple(); +DROP TABLE IF EXISTS sample_table;" + + +${CLICKHOUSE_CLIENT} -n --query "CREATE TABLE sample_table_2 ( + key UInt64 +) +ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_like_2', '1') +ORDER BY tuple();" + +${CLICKHOUSE_CLIENT} --allow_unrestricted_reads_from_keeper=1 --query="SELECT name FROM (SELECT path, name FROM system.zookeeper WHERE path LIKE '/clickhouse%' ORDER BY name) WHERE path LIKE '%$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_like%'" + +${CLICKHOUSE_CLIENT} --query="SELECT '-------------------------'" + +${CLICKHOUSE_CLIENT} --allow_unrestricted_reads_from_keeper=1 --query="SELECT name FROM (SELECT path, name FROM system.zookeeper WHERE path LIKE '/clickhouse/%' ORDER BY name) WHERE path LIKE '%$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_like%'" + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table;" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table_2;" diff --git a/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.reference b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.reference new file mode 100644 index 00000000000..4ace962aa0f --- /dev/null +++ b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.reference @@ -0,0 +1,14 @@ +-- { echoOn } +WITH + (SELECT * FROM data_02222) AS bm1, + (SELECT * FROM data_02222) AS bm2, + (SELECT * FROM data_02222) AS bm3, + (SELECT * FROM data_02222) AS bm4, + (SELECT * FROM data_02222) AS bm5, + (SELECT * FROM data_02222) AS bm6, + (SELECT * FROM data_02222) AS bm7, + (SELECT * FROM data_02222) AS bm8, + (SELECT * FROM data_02222) AS bm9, + (SELECT * FROM data_02222) AS bm10 +SELECT bm1, bm2, bm3, bm4, bm5, bm6, bm7, bm8, bm9, bm10 FROM data_02222; +0 0 0 0 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.sql b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.sql new file mode 100644 index 00000000000..f870b985d81 --- /dev/null +++ b/tests/queries/0_stateless/02222_allow_experimental_projection_optimization__enable_global_with_statement.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS data_02222; +CREATE TABLE data_02222 engine=MergeTree() ORDER BY dummy AS SELECT * FROM system.one; +-- { echoOn } +WITH + (SELECT * FROM data_02222) AS bm1, + (SELECT * FROM data_02222) AS bm2, + (SELECT * FROM data_02222) AS bm3, + (SELECT * FROM data_02222) AS bm4, + (SELECT * FROM data_02222) AS bm5, + (SELECT * FROM data_02222) AS bm6, + (SELECT * FROM data_02222) AS bm7, + (SELECT * FROM data_02222) AS bm8, + (SELECT * FROM data_02222) AS bm9, + (SELECT * FROM data_02222) AS bm10 +SELECT bm1, bm2, bm3, bm4, bm5, bm6, bm7, bm8, bm9, bm10 FROM data_02222; +-- { echoOff } +DROP TABLE data_02222; diff --git a/tests/queries/0_stateless/02222_create_table_without_columns_metadata.reference b/tests/queries/0_stateless/02222_create_table_without_columns_metadata.reference new file mode 100644 index 00000000000..9e9e0082cb3 --- /dev/null +++ b/tests/queries/0_stateless/02222_create_table_without_columns_metadata.reference @@ -0,0 +1,3 @@ +CREATE TABLE default.test\n(\n `y` Nullable(String),\n `x` Nullable(Float64)\n)\nENGINE = File(\'JSONEachRow\', \'data.jsonl\') +OK +OK diff --git a/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh b/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh new file mode 100755 index 00000000000..842c32cf243 --- /dev/null +++ b/tests/queries/0_stateless/02222_create_table_without_columns_metadata.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-backward-compatibility-check + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +$CLICKHOUSE_CLIENT -q "insert into table function file(data.jsonl, 'JSONEachRow', 'x UInt32 default 42, y String') select number as x, 'String' as y from numbers(10)" + +$CLICKHOUSE_CLIENT -q "drop table if exists test" +$CLICKHOUSE_CLIENT -q "create table test engine=File(JSONEachRow, 'data.jsonl')" +$CLICKHOUSE_CLIENT -q "show create table test" +$CLICKHOUSE_CLIENT -q "detach table test" + +rm $USER_FILES_PATH/data.jsonl + +$CLICKHOUSE_CLIENT -q "attach table test" +$CLICKHOUSE_CLIENT -q "select * from test" 2>&1 | grep -q "FILE_DOESNT_EXIST" && echo "OK" || echo "FAIL" + + +$CLICKHOUSE_CLIENT -q "drop table test" +$CLICKHOUSE_CLIENT -q "create table test (x UInt64) engine=Memory()" + +$CLICKHOUSE_CLIENT -q "drop table if exists test_dist" +$CLICKHOUSE_CLIENT -q "create table test_dist engine=Distributed('test_shard_localhost', currentDatabase(), 'test')" + +$CLICKHOUSE_CLIENT -q "detach table test_dist" +$CLICKHOUSE_CLIENT -q "drop table test" +$CLICKHOUSE_CLIENT -q "attach table test_dist" +$CLICKHOUSE_CLIENT -q "select * from test_dist" 2>&1 | grep -q "UNKNOWN_TABLE" && echo "OK" || echo "FAIL" + diff --git a/tests/queries/0_stateless/02223_h3_test_const_columns.reference b/tests/queries/0_stateless/02223_h3_test_const_columns.reference new file mode 100644 index 00000000000..5102199ef58 --- /dev/null +++ b/tests/queries/0_stateless/02223_h3_test_const_columns.reference @@ -0,0 +1,80 @@ +583031433791012863 +587531185127686143 +583031433791012863 +587531185127686143 +607221000000 +86745854035 +607220.9782 +86745.85403 +4106166334463.9233 +85294486110.07852 +12781831077.715292 +0.10116268528089567 +0.002101380838405832 +0.00031490306268786255 +0 +2 +3 +9.961887434044831 +3.7652395323603707 +1.4231267757782213 +1107712.591 +418676.0055 +158244.6558 +1107.712591 +418.6760055 +158.2446558 +(-173.6412167681162,-14.130272474941535) +(172.68095885060296,-83.6576608516349) +(-94.46556851304558,-69.1999982492279) +[(-25.60370257696877,-170.61932339479839),(-16.505947603561054,-161.6348206171839),(-5.762860491436932,-165.41674992858833),(-3.968796976609588,-176.05696384421356),(-11.54529597541476,175.98600155652952),(-22.19754138630238,177.51613498805204)] +[(-82.24829137508873,167.18203088800593),(-83.41761096812805,158.00531624510785),(-84.94207431820979,162.09183616506846),(-85.00324300064887,-178.60454506450245),(-83.46691212211444,-172.41232929697492),(-82.25118471750908,179.4928586395771)] +[(-69.70201806837188,-95.63006768303532),(-69.78121889088241,-93.8329499937899),(-69.26603652285242,-92.70414199751751),(-68.6908704290193,-93.35479180342097),(-68.62037380778602,-95.0614247833063),(-69.11663254992226,-96.20491957306085)] +[579275502070530047,579768083279773695,578888473977552895,579662530163507199,579205133326352383,578466261512486911,578712552117108735] +[578888473977552895,580225480116928511,579979189512306687,578114417791598591,578079233419509759,578712552117108735,579310686442618879,578606999000842239,578571814628753407,579205133326352383,579275502070530047,578466261512486911,579240317698441215,577727389698621439,579838452023951359,579662530163507199,579768083279773695,580331033233195007] +[577375545977733119,578431077140398079,579099580210085887,579732898907684863,580612508209905663,579275502070530047,580190295744839679,577094071001022463,578606999000842239,579029211465908223,577727389698621439,579240317698441215,579662530163507199,578571814628753407,580331033233195007,580295848861106175,579205133326352383,577903311559065599,578114417791598591,579838452023951359,577445914721910783,577868127186976767,578079233419509759,579592161419329535,578501445884575743,578712552117108735,580225480116928511,580471770721550335,580647692581994495,578466261512486911,579768083279773695,578888473977552895,579979189512306687,579310686442618879] +77 +121 +116 +1 +0 +1 +[603909588852408319,603909588986626047,603909589120843775,603909589255061503,603909589389279231,603909589523496959,603909589657714687] +[599405990164561919] +576918149140578303 +581395360488882175 +809bfffffffffff +82f39ffffffffff +83e9abfffffffff +579205133326352383 +589753847883235327 +594082350283882495 +0 +0 +1 +0 +1 +0 +1 +[7] +[1,6,11,7,2] +[7] +[1,6,11,7,2] +581496515558637567 +585996266895310847 +590499385486344191 +1263609.6633631135 +1263609.663363112 +1263609.6633631124 +1263609.6633631117 +1263.6096633631134 +1263.6096633631118 +1263.6096633631123 +1263.6096633631116 +0.19833750417794152 +0.19833750417794127 +0.19833750417794133 +0.19833750417794122 +842 +5882 +41162 diff --git a/tests/queries/0_stateless/02223_h3_test_const_columns.sql b/tests/queries/0_stateless/02223_h3_test_const_columns.sql new file mode 100644 index 00000000000..5bae85edb23 --- /dev/null +++ b/tests/queries/0_stateless/02223_h3_test_const_columns.sql @@ -0,0 +1,31 @@ +-- Tags: no-fasttest + +select geoToH3(toFloat64(0),toFloat64(1),arrayJoin([1,2])); +select h3ToParent(641573946153969375, arrayJoin([1,2])); +SELECT h3HexAreaM2(arrayJoin([1,2])); +SELECT h3HexAreaKm2(arrayJoin([1,2])); +SELECT h3CellAreaM2(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT NULL, toFloat64('-1'), -2147483648, h3CellAreaM2(arrayJoin([9223372036854775807, 65535, NULL])); -- { serverError 117 } +SELECT h3CellAreaRads2(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT NULL, toFloat64('-1'), -2147483648, h3CellAreaRads2(arrayJoin([9223372036854775807, 65535, NULL])); -- { serverError 117 } +SELECT h3GetResolution(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3EdgeAngle(arrayJoin([0,1,2])); +SELECT h3EdgeLengthM(arrayJoin([0,1,2])); +SELECT h3EdgeLengthKm(arrayJoin([0,1,2])); +SELECT h3ToGeo(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3ToGeoBoundary(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3kRing(arrayJoin([579205133326352383]), arrayJoin([toUInt16(1),toUInt16(2),toUInt16(3)])); +SELECT h3GetBaseCell(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3IndexesAreNeighbors(617420388351344639, arrayJoin([617420388352655359, 617420388351344639, 617420388352917503])); +SELECT h3ToChildren(599405990164561919, arrayJoin([6,5])); +SELECT h3ToParent(599405990164561919, arrayJoin([0,1])); +SELECT h3ToString(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT stringToH3(h3ToString(arrayJoin([579205133326352383,589753847883235327,594082350283882495]))); +SELECT h3IsResClassIII(arrayJoin([579205133326352383,589753847883235327,594082350283882495])); +SELECT h3IsPentagon(arrayJoin([stringToH3('8f28308280f18f2'),stringToH3('821c07fffffffff'),stringToH3('0x8f28308280f18f2L'),stringToH3('0x821c07fffffffffL')])); +SELECT h3GetFaces(arrayJoin([stringToH3('8f28308280f18f2'),stringToH3('821c07fffffffff'),stringToH3('0x8f28308280f18f2L'),stringToH3('0x821c07fffffffffL')])); +SELECT h3ToCenterChild(577023702256844799, arrayJoin([1,2,3])); +SELECT h3ExactEdgeLengthM(arrayJoin([1298057039473278975,1370114633511206911,1442172227549134847,1514229821587062783])); +SELECT h3ExactEdgeLengthKm(arrayJoin([1298057039473278975,1370114633511206911,1442172227549134847,1514229821587062783])); +SELECT h3ExactEdgeLengthRads(arrayJoin([1298057039473278975,1370114633511206911,1442172227549134847,1514229821587062783])); +SELECT h3NumHexagons(arrayJoin([1,2,3])); diff --git a/tests/queries/0_stateless/02223_insert_select_schema_inference.reference b/tests/queries/0_stateless/02223_insert_select_schema_inference.reference new file mode 100644 index 00000000000..ef1eea12112 --- /dev/null +++ b/tests/queries/0_stateless/02223_insert_select_schema_inference.reference @@ -0,0 +1,13 @@ +x UInt32 +y String +d Date +0 0 1970-01-01 +1 1 1970-01-02 +2 2 1970-01-03 +3 3 1970-01-04 +4 4 1970-01-05 +5 5 1970-01-06 +6 6 1970-01-07 +7 7 1970-01-08 +8 8 1970-01-09 +9 9 1970-01-10 diff --git a/tests/queries/0_stateless/02223_insert_select_schema_inference.sql b/tests/queries/0_stateless/02223_insert_select_schema_inference.sql new file mode 100644 index 00000000000..ff39ca83b9b --- /dev/null +++ b/tests/queries/0_stateless/02223_insert_select_schema_inference.sql @@ -0,0 +1,5 @@ +drop table if exists test; +create table test (x UInt32, y String, d Date) engine=Memory() as select number as x, toString(number) as y, toDate(number) as d from numbers(10); +insert into table function file('data.native.zst') select * from test; +desc file('data.native.zst'); +select * from file('data.native.zst'); diff --git a/tests/queries/0_stateless/02224_parallel_distributed_insert_select_cluster.reference b/tests/queries/0_stateless/02224_parallel_distributed_insert_select_cluster.reference new file mode 100644 index 00000000000..05fbb680c65 --- /dev/null +++ b/tests/queries/0_stateless/02224_parallel_distributed_insert_select_cluster.reference @@ -0,0 +1,27 @@ +-- { echoOn } +truncate table dst_02224; +insert into function cluster('test_cluster_two_shards', currentDatabase(), dst_02224, key) +select * from cluster('test_cluster_two_shards', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=1, max_distributed_depth=1; -- { serverError TOO_LARGE_DISTRIBUTED_DEPTH } +select * from dst_02224; +truncate table dst_02224; +insert into function cluster('test_cluster_two_shards', currentDatabase(), dst_02224, key) +select * from cluster('test_cluster_two_shards', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=1, max_distributed_depth=2; +select * from dst_02224; +1 +1 +truncate table dst_02224; +insert into function cluster('test_cluster_two_shards', currentDatabase(), dst_02224, key) +select * from cluster('test_cluster_two_shards', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02224; +1 +1 +truncate table dst_02224; +insert into function remote('127.{1,2}', currentDatabase(), dst_02224, key) +select * from remote('127.{1,2}', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02224; +1 +1 diff --git a/tests/queries/0_stateless/02224_parallel_distributed_insert_select_cluster.sql b/tests/queries/0_stateless/02224_parallel_distributed_insert_select_cluster.sql new file mode 100644 index 00000000000..023f220e930 --- /dev/null +++ b/tests/queries/0_stateless/02224_parallel_distributed_insert_select_cluster.sql @@ -0,0 +1,34 @@ +drop table if exists dst_02224; +drop table if exists src_02224; +create table dst_02224 (key Int) engine=Memory(); +create table src_02224 (key Int) engine=Memory(); +insert into src_02224 values (1); + +-- { echoOn } +truncate table dst_02224; +insert into function cluster('test_cluster_two_shards', currentDatabase(), dst_02224, key) +select * from cluster('test_cluster_two_shards', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=1, max_distributed_depth=1; -- { serverError TOO_LARGE_DISTRIBUTED_DEPTH } +select * from dst_02224; + +truncate table dst_02224; +insert into function cluster('test_cluster_two_shards', currentDatabase(), dst_02224, key) +select * from cluster('test_cluster_two_shards', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=1, max_distributed_depth=2; +select * from dst_02224; + +truncate table dst_02224; +insert into function cluster('test_cluster_two_shards', currentDatabase(), dst_02224, key) +select * from cluster('test_cluster_two_shards', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02224; + +truncate table dst_02224; +insert into function remote('127.{1,2}', currentDatabase(), dst_02224, key) +select * from remote('127.{1,2}', currentDatabase(), src_02224, key) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02224; +-- { echoOff } + +drop table src_02224; +drop table dst_02224; diff --git a/tests/queries/0_stateless/02224_s2_test_const_columns.reference b/tests/queries/0_stateless/02224_s2_test_const_columns.reference new file mode 100644 index 00000000000..9982596f097 --- /dev/null +++ b/tests/queries/0_stateless/02224_s2_test_const_columns.reference @@ -0,0 +1,19 @@ +4704772434919038107 +1527555102286129111 +(37.79506681471008,55.7129059052841) +(-73.98300293448631,40.755097559353004) +[1157339245694594831,1157339245694594835,1157339245694594931,1157339245694594827] +[5074766987100422144,5074766712222515200,5074767536856236032,5074767261978329088] +1 +0 +1 +1 +(4534655147792050737,60.2088283994957) +(5179062030687166815,5177056748191934217) +(5178914411069187297,5177057445452335297) +0 +0 +(5179062030687166815,5177056748191934217) +(5179062030687166815,5177057445452335297) +(5178914411069187297,5177056748191934217) +(6304347505408739331,8070450532247928833) diff --git a/tests/queries/0_stateless/02224_s2_test_const_columns.sql b/tests/queries/0_stateless/02224_s2_test_const_columns.sql new file mode 100644 index 00000000000..f33a7f2b696 --- /dev/null +++ b/tests/queries/0_stateless/02224_s2_test_const_columns.sql @@ -0,0 +1,12 @@ +-- Tags: no-fasttest + +SELECT geoToS2(37.79506683, arrayJoin([55.71290588,37.79506683])); +SELECT s2ToGeo(arrayJoin([4704772434919038107,9926594385212866560])); +SELECT s2GetNeighbors(arrayJoin([1157339245694594829, 5074766849661468672])); +SELECT s2CellsIntersect(9926595209846587392, arrayJoin([9926594385212866560, 5074766849661468672])); +SELECT s2CapContains(1157339245694594829, toFloat64(1), arrayJoin([1157347770437378819,1157347770437378389])); +SELECT s2CapUnion(3814912406305146967, toFloat64(1), 1157347770437378819, toFloat64(1)); +SELECT s2RectAdd(5178914411069187297, 5177056748191934217, arrayJoin([5179056748191934217,5177914411069187297])); +SELECT s2RectContains(5179062030687166815, 5177056748191934217, arrayJoin([5177914411069187297, 5177914411069187297])); +SELECT s2RectUnion(5178914411069187297, 5177056748191934217, 5179062030687166815, arrayJoin([5177056748191934217, 5177914411069187297])); +SELECT s2RectIntersection(5178914411069187297, 5177056748191934217, 5179062030687166815, arrayJoin([5177056748191934217,1157347770437378819])); diff --git a/tests/queries/0_stateless/02225_hints_for_indeices.reference b/tests/queries/0_stateless/02225_hints_for_indeices.reference new file mode 100644 index 00000000000..2c94e483710 --- /dev/null +++ b/tests/queries/0_stateless/02225_hints_for_indeices.reference @@ -0,0 +1,2 @@ +OK +OK diff --git a/tests/queries/0_stateless/02225_hints_for_indeices.sh b/tests/queries/0_stateless/02225_hints_for_indeices.sh new file mode 100755 index 00000000000..f4cfa17f8db --- /dev/null +++ b/tests/queries/0_stateless/02225_hints_for_indeices.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS t" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE t ENGINE=MergeTree ORDER BY n AS SELECT number AS n FROM numbers(10)" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE t ADD INDEX test_index n TYPE minmax GRANULARITY 32" + +$CLICKHOUSE_CLIENT --query="ALTER TABLE t DROP INDEX test_indes" 2>&1 | grep -q "may be you meant: \['test_index'\]" && echo 'OK' || echo 'FAIL' + +$CLICKHOUSE_CLIENT --query="ALTER TABLE t ADD INDEX test_index1 n TYPE minmax GRANULARITY 4 AFTER test_indes" 2>&1 | grep -q "may be you meant: \['test_index'\]" && echo 'OK' || echo 'FAIL' + +$CLICKHOUSE_CLIENT --query="DROP TABLE t" diff --git a/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference new file mode 100644 index 00000000000..98fb6a68656 --- /dev/null +++ b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh new file mode 100755 index 00000000000..376a49fd820 --- /dev/null +++ b/tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# NOTE: sh test is required since view() does not have current database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " +drop table if exists dst_02225; +drop table if exists src_02225; +create table dst_02225 (key Int) engine=Memory(); +create table src_02225 (key Int) engine=Memory(); +insert into src_02225 values (1); +" + +$CLICKHOUSE_CLIENT --param_database=$CLICKHOUSE_DATABASE -nm -q " +truncate table dst_02225; +insert into function remote('127.{1,2}', currentDatabase(), dst_02225, key) +select * from remote('127.{1,2}', view(select * from {database:Identifier}.src_02225), key) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02225; + +-- w/o sharding key +truncate table dst_02225; +insert into function remote('127.{1,2}', currentDatabase(), dst_02225, key) +select * from remote('127.{1,2}', view(select * from {database:Identifier}.src_02225)) +settings parallel_distributed_insert_select=2, max_distributed_depth=1; +select * from dst_02225; +" + +$CLICKHOUSE_CLIENT -nm -q " +drop table src_02225; +drop table dst_02225; +" diff --git a/tests/queries/0_stateless/02225_unwinder_dwarf_version.reference b/tests/queries/0_stateless/02225_unwinder_dwarf_version.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02225_unwinder_dwarf_version.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02225_unwinder_dwarf_version.sh b/tests/queries/0_stateless/02225_unwinder_dwarf_version.sh new file mode 100755 index 00000000000..59dcba401f2 --- /dev/null +++ b/tests/queries/0_stateless/02225_unwinder_dwarf_version.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +# In case if DWARF-5 debug info is generated during build, it cannot parse it and correctly show this exception. This is why we limit DWARF version to 4 max. +$CLICKHOUSE_LOCAL --query "SELECT throwIf(1)" 2>&1 | grep -c 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' diff --git a/tests/queries/0_stateless/02226_async_insert_table_function.reference b/tests/queries/0_stateless/02226_async_insert_table_function.reference new file mode 100644 index 00000000000..60d475a7393 --- /dev/null +++ b/tests/queries/0_stateless/02226_async_insert_table_function.reference @@ -0,0 +1,2 @@ +1 aaa +2 bbb diff --git a/tests/queries/0_stateless/02226_async_insert_table_function.sql b/tests/queries/0_stateless/02226_async_insert_table_function.sql new file mode 100644 index 00000000000..fc4aadfbfcd --- /dev/null +++ b/tests/queries/0_stateless/02226_async_insert_table_function.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t_async_insert_table_function; + +CREATE TABLE t_async_insert_table_function (id UInt32, s String) ENGINE = Memory; + +SET async_insert = 1; + +INSERT INTO function remote('127.0.0.1', currentDatabase(), t_async_insert_table_function) values (1, 'aaa') (2, 'bbb'); + +SELECT * FROM t_async_insert_table_function ORDER BY id; + +DROP TABLE t_async_insert_table_function; diff --git a/tests/queries/0_stateless/02226_in_untuple_issue_34810.reference b/tests/queries/0_stateless/02226_in_untuple_issue_34810.reference new file mode 100644 index 00000000000..84cc85e3715 --- /dev/null +++ b/tests/queries/0_stateless/02226_in_untuple_issue_34810.reference @@ -0,0 +1 @@ +2001 2 diff --git a/tests/queries/0_stateless/02226_in_untuple_issue_34810.sql b/tests/queries/0_stateless/02226_in_untuple_issue_34810.sql new file mode 100644 index 00000000000..a313d526e9d --- /dev/null +++ b/tests/queries/0_stateless/02226_in_untuple_issue_34810.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS calendar; +DROP TABLE IF EXISTS events32; + +CREATE TABLE calendar ( `year` Int64, `month` Int64 ) ENGINE = TinyLog; +INSERT INTO calendar VALUES (2000, 1), (2001, 2), (2000, 3); + +CREATE TABLE events32 ( `year` Int32, `month` Int32 ) ENGINE = TinyLog; +INSERT INTO events32 VALUES (2001, 2), (2001, 3); + +SELECT * FROM calendar WHERE (year, month) IN ( SELECT (year, month) FROM events32 ); + +DROP TABLE IF EXISTS calendar; +DROP TABLE IF EXISTS events32; diff --git a/tests/queries/0_stateless/02226_low_cardinality_text_bloom_filter_index.reference b/tests/queries/0_stateless/02226_low_cardinality_text_bloom_filter_index.reference new file mode 100644 index 00000000000..a1533c4e44a --- /dev/null +++ b/tests/queries/0_stateless/02226_low_cardinality_text_bloom_filter_index.reference @@ -0,0 +1,24 @@ +lc_bf_tokenbf +1 K1 K1ZZZZZZ +2 K2 K2ZZZZZZ +lc_fixed_bf_tokenbf +1 K1 K1ZZZZZZ +2 K2 K2ZZZZZZ +lc_ngram +1 K1 K1ZZZZZZ +2 K2 K2ZZZZZZ +lc_fixed_ngram +1 K1 K1ZZZZZZ +2 K2 K2ZZZZZZ +lc_bf_tokenbf +3 abCD3ef abCD3ef\0 +4 abCD4ef abCD4ef\0 +lc_fixed_bf_tokenbf +3 abCD3ef abCD3ef\0 +4 abCD4ef abCD4ef\0 +lc_ngram +3 abCD3ef abCD3ef\0 +4 abCD4ef abCD4ef\0 +lc_fixed_ngram +3 abCD3ef abCD3ef\0 +4 abCD4ef abCD4ef\0 diff --git a/tests/queries/0_stateless/02226_low_cardinality_text_bloom_filter_index.sql b/tests/queries/0_stateless/02226_low_cardinality_text_bloom_filter_index.sql new file mode 100644 index 00000000000..d2b30f5e8f4 --- /dev/null +++ b/tests/queries/0_stateless/02226_low_cardinality_text_bloom_filter_index.sql @@ -0,0 +1,69 @@ +DROP TABLE IF EXISTS bf_tokenbf_lowcard_test; +DROP TABLE IF EXISTS bf_ngram_lowcard_test; + +CREATE TABLE bf_tokenbf_lowcard_test +( + row_id UInt32, + lc LowCardinality(String), + lc_fixed LowCardinality(FixedString(8)), + INDEX lc_bf_tokenbf lc TYPE tokenbf_v1(256,2,0) GRANULARITY 1, + INDEX lc_fixed_bf_tokenbf lc_fixed TYPE tokenbf_v1(256,2,0) GRANULARITY 1 +) Engine=MergeTree() ORDER BY row_id SETTINGS index_granularity = 1; + +CREATE TABLE bf_ngram_lowcard_test +( + row_id UInt32, + lc LowCardinality(String), + lc_fixed LowCardinality(FixedString(8)), + INDEX lc_ngram lc TYPE ngrambf_v1(4,256,2,0) GRANULARITY 1, + INDEX lc_fixed_ngram lc_fixed TYPE ngrambf_v1(4,256,2,0) GRANULARITY 1 +) Engine=MergeTree() ORDER BY row_id SETTINGS index_granularity = 1; + +INSERT INTO bf_tokenbf_lowcard_test VALUES (1, 'K1', 'K1ZZZZZZ'), (2, 'K2', 'K2ZZZZZZ'); +INSERT INTO bf_ngram_lowcard_test VALUES (1, 'K1', 'K1ZZZZZZ'), (2, 'K2', 'K2ZZZZZZ'); +INSERT INTO bf_tokenbf_lowcard_test VALUES (3, 'abCD3ef', 'abCD3ef'), (4, 'abCD4ef', 'abCD4ef'); +INSERT INTO bf_ngram_lowcard_test VALUES (3, 'abCD3ef', 'abCD3ef'), (4, 'abCD4ef', 'abCD4ef'); + +SELECT 'lc_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc, 'K1') SETTINGS force_data_skipping_indices='lc_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc, 'K2') SETTINGS force_data_skipping_indices='lc_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc, 'K3') SETTINGS force_data_skipping_indices='lc_bf_tokenbf'; + +SELECT 'lc_fixed_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc_fixed, 'K1ZZZZZZ') SETTINGS force_data_skipping_indices='lc_fixed_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc_fixed, 'K2ZZZZZZ') SETTINGS force_data_skipping_indices='lc_fixed_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc_fixed, 'K3ZZZZZZ') SETTINGS force_data_skipping_indices='lc_fixed_bf_tokenbf'; + +SELECT 'lc_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc, 'K1') SETTINGS force_data_skipping_indices='lc_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc, 'K2') SETTINGS force_data_skipping_indices='lc_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc, 'K3') SETTINGS force_data_skipping_indices='lc_ngram'; + +SELECT 'lc_fixed_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc_fixed, 'K1ZZZZZZ') SETTINGS force_data_skipping_indices='lc_fixed_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc_fixed, 'K2ZZZZZZ') SETTINGS force_data_skipping_indices='lc_fixed_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc_fixed, 'K3ZZZZZZ') SETTINGS force_data_skipping_indices='lc_fixed_ngram'; + + +SELECT 'lc_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc, '%CD3%') SETTINGS force_data_skipping_indices='lc_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc, '%CD4%') SETTINGS force_data_skipping_indices='lc_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc, '%CD5%') SETTINGS force_data_skipping_indices='lc_bf_tokenbf'; + +SELECT 'lc_fixed_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc_fixed, '%CD3%') SETTINGS force_data_skipping_indices='lc_fixed_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc_fixed, '%CD4%') SETTINGS force_data_skipping_indices='lc_fixed_bf_tokenbf'; +SELECT * FROM bf_tokenbf_lowcard_test WHERE like(lc_fixed, '%CD5%') SETTINGS force_data_skipping_indices='lc_fixed_bf_tokenbf'; + +SELECT 'lc_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc, '%CD3%') SETTINGS force_data_skipping_indices='lc_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc, '%CD4%') SETTINGS force_data_skipping_indices='lc_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc, '%CD5%') SETTINGS force_data_skipping_indices='lc_ngram'; + +SELECT 'lc_fixed_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc_fixed, '%CD3%') SETTINGS force_data_skipping_indices='lc_fixed_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc_fixed, '%CD4%') SETTINGS force_data_skipping_indices='lc_fixed_ngram'; +SELECT * FROM bf_ngram_lowcard_test WHERE like(lc_fixed, '%CD5%') SETTINGS force_data_skipping_indices='lc_fixed_ngram'; + +DROP TABLE bf_tokenbf_lowcard_test; +DROP TABLE bf_ngram_lowcard_test; diff --git a/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.reference b/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh b/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh new file mode 100755 index 00000000000..2a163746e20 --- /dev/null +++ b/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " +drop table if exists data_02226; +create table data_02226 (key Int) engine=MergeTree() order by key +as select * from numbers(1); +" + +# Regression for: +# +# Logical error: 'Coordinator for parallel reading from replicas is not initialized'. +opts=( + --allow_experimental_parallel_reading_from_replicas 1 + --max_parallel_replicas 3 + + --iterations 1 +) +$CLICKHOUSE_BENCHMARK --query "select * from remote('127.1', $CLICKHOUSE_DATABASE, data_02226)" "${opts[@]}" >& /dev/null +ret=$? + +$CLICKHOUSE_CLIENT -nm -q " +drop table data_02226; +" + +exit $ret diff --git a/tests/queries/0_stateless/02226_s3_with_cache.reference b/tests/queries/0_stateless/02226_s3_with_cache.reference new file mode 100644 index 00000000000..214addac2d6 --- /dev/null +++ b/tests/queries/0_stateless/02226_s3_with_cache.reference @@ -0,0 +1,2 @@ +SELECT 1, * FROM test LIMIT 10 FORMAT Null; 1 0 1 +SELECT 2, * FROM test LIMIT 10 FORMAT Null; 0 1 0 diff --git a/tests/queries/0_stateless/02226_s3_with_cache.sql b/tests/queries/0_stateless/02226_s3_with_cache.sql new file mode 100644 index 00000000000..b3126a419df --- /dev/null +++ b/tests/queries/0_stateless/02226_s3_with_cache.sql @@ -0,0 +1,44 @@ +-- Tags: no-parallel, no-fasttest, long + +SET max_memory_usage='20G'; + +CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache'; +INSERT INTO test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000; + +SET remote_filesystem_read_method='threadpool'; + +SELECT 1, * FROM test LIMIT 10 FORMAT Null; + +SYSTEM FLUSH LOGS; +SELECT query, + ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read, + ProfileEvents['RemoteFSCacheReadBytes'] > 0 as remote_fs_cache_read, + ProfileEvents['RemoteFSCacheDownloadBytes'] > 0 as remote_fs_read_and_download +FROM system.query_log +WHERE query LIKE 'SELECT 1, * FROM test LIMIT%' +AND type = 'QueryFinish' +AND current_database = currentDatabase() +ORDER BY query_start_time DESC +LIMIT 1; + +SET remote_filesystem_read_method='read'; + +SELECT 2, * FROM test LIMIT 10 FORMAT Null; + +SYSTEM FLUSH LOGS; +SELECT query, + ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read, + ProfileEvents['RemoteFSCacheReadBytes'] > 0 as remote_fs_cache_read, + ProfileEvents['RemoteFSCacheDownloadBytes'] > 0 as remote_fs_read_and_download +FROM system.query_log +WHERE query LIKE 'SELECT 2, * FROM test LIMIT%' +AND type = 'QueryFinish' +AND current_database = currentDatabase() +ORDER BY query_start_time DESC +LIMIT 1; + +SET remote_filesystem_read_method='threadpool'; + +SELECT * FROM test WHERE value LIKE '%abc%' ORDER BY value LIMIT 10 FORMAT Null; + +DROP TABLE test; diff --git a/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.reference b/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.reference new file mode 100644 index 00000000000..472973e965a --- /dev/null +++ b/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.reference @@ -0,0 +1 @@ +table1 diff --git a/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh b/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh new file mode 100755 index 00000000000..253d3f3149d --- /dev/null +++ b/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# See 01658_read_file_to_string_column.sh +user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" + rm -r "${DB_PATH}" +} +trap cleanup EXIT + +export CURR_DATABASE="test_01889_sqllite_${CLICKHOUSE_DATABASE}" + +DB_PATH=${user_files_path}/${CURR_DATABASE}_db1 + +${CLICKHOUSE_CLIENT} --multiquery --multiline --query=""" +DROP DATABASE IF EXISTS ${CURR_DATABASE}; +CREATE DATABASE ${CURR_DATABASE} ENGINE = SQLite('${DB_PATH}'); +SHOW TABLES FROM ${CURR_DATABASE}; +""" + +sqlite3 "${DB_PATH}" 'CREATE TABLE table1 (col1 text, col2 smallint);' + +${CLICKHOUSE_CLIENT} --multiquery --multiline --query=""" +SHOW TABLES FROM ${CURR_DATABASE}; +""" diff --git a/tests/queries/0_stateless/02228_merge_tree_insert_memory_usage.reference b/tests/queries/0_stateless/02228_merge_tree_insert_memory_usage.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02228_merge_tree_insert_memory_usage.sql b/tests/queries/0_stateless/02228_merge_tree_insert_memory_usage.sql new file mode 100644 index 00000000000..7aa3be1990b --- /dev/null +++ b/tests/queries/0_stateless/02228_merge_tree_insert_memory_usage.sql @@ -0,0 +1,15 @@ +-- Tags: long, no-parallel + +-- regression for MEMORY_LIMIT_EXCEEDED error because of deferred final part flush + +drop table if exists data_02228; +create table data_02228 (key1 UInt32, sign Int8, s UInt64) engine = CollapsingMergeTree(sign) order by (key1) partition by key1 % 1024; +insert into data_02228 select number, 1, number from numbers_mt(100e3) settings max_memory_usage='300Mi', max_partitions_per_insert_block=1024; +insert into data_02228 select number, 1, number from numbers_mt(100e3) settings max_memory_usage='300Mi', max_partitions_per_insert_block=1024, max_insert_delayed_streams_for_parallel_write=10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } +drop table data_02228; + +drop table if exists data_rep_02228; +create table data_rep_02228 (key1 UInt32, sign Int8, s UInt64) engine = ReplicatedCollapsingMergeTree('/clickhouse/{database}', 'r1', sign) order by (key1) partition by key1 % 1024; +insert into data_rep_02228 select number, 1, number from numbers_mt(100e3) settings max_memory_usage='300Mi', max_partitions_per_insert_block=1024; +insert into data_rep_02228 select number, 1, number from numbers_mt(100e3) settings max_memory_usage='300Mi', max_partitions_per_insert_block=1024, max_insert_delayed_streams_for_parallel_write=10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } +drop table data_rep_02228; diff --git a/tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.reference b/tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.reference new file mode 100644 index 00000000000..5fd48ae580a --- /dev/null +++ b/tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.reference @@ -0,0 +1 @@ +c1 Nullable(String) diff --git a/tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.sh b/tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.sh new file mode 100755 index 00000000000..314a60d6491 --- /dev/null +++ b/tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +echo "2020-02-01 16:00:00" | $CLICKHOUSE_LOCAL -q "desc table table" --input-format "CSV" --file=- + diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.reference b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh new file mode 100755 index 00000000000..171dcc52c9c --- /dev/null +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -nm -q " + SELECT sleep(1) FROM numbers(100) FORMAT Null; + SELECT 'FAIL'; +" + +timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -nm -q " + SELECT sleep(1) FROM numbers(100) FORMAT Null; + SELECT 'FAIL'; +" + +exit 0 diff --git a/tests/queries/0_stateless/02230_create_table_as_ignore_ttl.reference b/tests/queries/0_stateless/02230_create_table_as_ignore_ttl.reference new file mode 100644 index 00000000000..5236875e209 --- /dev/null +++ b/tests/queries/0_stateless/02230_create_table_as_ignore_ttl.reference @@ -0,0 +1,32 @@ +CREATE TABLE default.data_02230_ttl +( + `date` Date, + `key` Int32 +) +ENGINE = MergeTree +ORDER BY key +TTL date + 14 +SETTINGS index_granularity = 8192 +CREATE TABLE default.null_02230_ttl +( + `date` Date, + `key` Int32 +) +ENGINE = Null +CREATE TABLE default.data_02230_column_ttl +( + `date` Date, + `value` Int32 TTL date + 7, + `key` Int32 +) +ENGINE = MergeTree +ORDER BY key +TTL date + 14 +SETTINGS index_granularity = 8192 +CREATE TABLE default.null_02230_column_ttl +( + `date` Date, + `value` Int32, + `key` Int32 +) +ENGINE = Null diff --git a/tests/queries/0_stateless/02230_create_table_as_ignore_ttl.sql b/tests/queries/0_stateless/02230_create_table_as_ignore_ttl.sql new file mode 100644 index 00000000000..8838f67ec83 --- /dev/null +++ b/tests/queries/0_stateless/02230_create_table_as_ignore_ttl.sql @@ -0,0 +1,18 @@ +drop table if exists data_02230_ttl; +drop table if exists null_02230_ttl; +create table data_02230_ttl (date Date, key Int) Engine=MergeTree() order by key TTL date + 14; +show create data_02230_ttl format TSVRaw; +create table null_02230_ttl engine=Null() as data_02230_ttl; +show create null_02230_ttl format TSVRaw; +drop table data_02230_ttl; +drop table null_02230_ttl; + +drop table if exists data_02230_column_ttl; +drop table if exists null_02230_column_ttl; +create table data_02230_column_ttl (date Date, value Int TTL date + 7, key Int) Engine=MergeTree() order by key TTL date + 14; +show create data_02230_column_ttl format TSVRaw; +create table null_02230_column_ttl engine=Null() as data_02230_column_ttl; +-- check that order of columns is the same +show create null_02230_column_ttl format TSVRaw; +drop table data_02230_column_ttl; +drop table null_02230_column_ttl; diff --git a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.reference b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql new file mode 100644 index 00000000000..a53b7f50e51 --- /dev/null +++ b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql @@ -0,0 +1,36 @@ +-- Tags: long + +drop table if exists buffer_02231; +drop table if exists out_02231; +drop table if exists in_02231; +drop table if exists mv_02231; + +-- To reproduce leak of memory tracking of aggregate states, +-- background flush is required. +create table buffer_02231 +( + key Int, + v1 AggregateFunction(groupArray, String) +) engine=Buffer(currentDatabase(), 'out_02231', + /* layers= */1, + /* min/max time */ 86400, 86400, + /* min/max rows */ 1e9, 1e9, + /* min/max bytes */ 1e12, 1e12, + /* flush time */ 1 +); +create table out_02231 as buffer_02231 engine=Null(); +create table in_02231 (number Int) engine=Null(); + +-- Create lots of INSERT blocks with MV +create materialized view mv_02231 to buffer_02231 as select + number as key, + groupArrayState(toString(number)) as v1 +from in_02231 +group by key; + +insert into in_02231 select * from numbers(10e6) settings max_memory_usage='300Mi'; + +drop table buffer_02231; +drop table out_02231; +drop table in_02231; +drop table mv_02231; diff --git a/tests/queries/0_stateless/02231_hierarchical_dictionaries_constant.reference b/tests/queries/0_stateless/02231_hierarchical_dictionaries_constant.reference new file mode 100644 index 00000000000..bccd9864b30 --- /dev/null +++ b/tests/queries/0_stateless/02231_hierarchical_dictionaries_constant.reference @@ -0,0 +1,32 @@ +Get hierarchy +[] +[1] +[2,1] +[3,1] +[4,2,1] +[] +Get is in hierarchy +1 +1 +0 +Get children +[1] +[2,3] +[4] +[] +[] +[] +Get all descendants +[1,2,3,4] +[2,3,4] +[4] +[] +[] +[] +Get descendants at first level +[1] +[2,3] +[4] +[] +[] +[] diff --git a/tests/queries/0_stateless/02231_hierarchical_dictionaries_constant.sql b/tests/queries/0_stateless/02231_hierarchical_dictionaries_constant.sql new file mode 100644 index 00000000000..bc01b447338 --- /dev/null +++ b/tests/queries/0_stateless/02231_hierarchical_dictionaries_constant.sql @@ -0,0 +1,54 @@ +DROP TABLE IF EXISTS hierarchy_source_table; +CREATE TABLE hierarchy_source_table (id UInt64, parent_id UInt64) ENGINE = TinyLog; +INSERT INTO hierarchy_source_table VALUES (1, 0), (2, 1), (3, 1), (4, 2); + +DROP DICTIONARY IF EXISTS hierarchy_flat_dictionary; +CREATE DICTIONARY hierarchy_flat_dictionary +( + id UInt64, + parent_id UInt64 HIERARCHICAL +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(TABLE 'hierarchy_source_table')) +LAYOUT(FLAT()) +LIFETIME(MIN 1 MAX 1000); + +SELECT 'Get hierarchy'; +SELECT dictGetHierarchy('hierarchy_flat_dictionary', 0); +SELECT dictGetHierarchy('hierarchy_flat_dictionary', 1); +SELECT dictGetHierarchy('hierarchy_flat_dictionary', 2); +SELECT dictGetHierarchy('hierarchy_flat_dictionary', 3); +SELECT dictGetHierarchy('hierarchy_flat_dictionary', 4); +SELECT dictGetHierarchy('hierarchy_flat_dictionary', 5); + +SELECT 'Get is in hierarchy'; +SELECT dictIsIn('hierarchy_flat_dictionary', 1, 1); +SELECT dictIsIn('hierarchy_flat_dictionary', 2, 1); +SELECT dictIsIn('hierarchy_flat_dictionary', 2, 0); + +SELECT 'Get children'; +SELECT dictGetChildren('hierarchy_flat_dictionary', 0); +SELECT dictGetChildren('hierarchy_flat_dictionary', 1); +SELECT dictGetChildren('hierarchy_flat_dictionary', 2); +SELECT dictGetChildren('hierarchy_flat_dictionary', 3); +SELECT dictGetChildren('hierarchy_flat_dictionary', 4); +SELECT dictGetChildren('hierarchy_flat_dictionary', 5); + +SELECT 'Get all descendants'; +SELECT dictGetDescendants('hierarchy_flat_dictionary', 0); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 1); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 2); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 3); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 4); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 5); + +SELECT 'Get descendants at first level'; +SELECT dictGetDescendants('hierarchy_flat_dictionary', 0, 1); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 1, 1); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 2, 1); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 3, 1); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 4, 1); +SELECT dictGetDescendants('hierarchy_flat_dictionary', 5, 1); + +DROP DICTIONARY hierarchy_flat_dictionary; +DROP TABLE hierarchy_source_table; diff --git a/tests/queries/0_stateless/02232_allow_only_replicated_engine.reference b/tests/queries/0_stateless/02232_allow_only_replicated_engine.reference new file mode 100644 index 00000000000..9b45eb31b7e --- /dev/null +++ b/tests/queries/0_stateless/02232_allow_only_replicated_engine.reference @@ -0,0 +1,3 @@ +Only table with Replicated engine +Only table with Replicated engine +Only table with Replicated engine diff --git a/tests/queries/0_stateless/02232_allow_only_replicated_engine.sh b/tests/queries/0_stateless/02232_allow_only_replicated_engine.sh new file mode 100755 index 00000000000..d04c8a98df4 --- /dev/null +++ b/tests/queries/0_stateless/02232_allow_only_replicated_engine.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Tags: replica + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "create table mute_stylecheck (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root', '1') order by x" + +${CLICKHOUSE_CLIENT} -q "CREATE USER user_${CLICKHOUSE_DATABASE} settings database_replicated_allow_only_replicated_engine=1" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}_db.* TO user_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --allow_experimental_database_replicated=1 --query "CREATE DATABASE ${CLICKHOUSE_DATABASE}_db engine = Replicated('/clickhouse/databases/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}_db', '{shard}', '{replica}')" +${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_memory (x UInt32) engine = Memory;" +${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "set distributed_ddl_entry_format_version=2; CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_mt (x UInt32) engine = MergeTree order by x;" 2>&1 | grep -o "Only table with Replicated engine" +${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none -n --query "set distributed_ddl_entry_format_version=2; CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_mt (x UInt32) engine = MergeTree order by x;" +${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "set distributed_ddl_entry_format_version=2; CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt (x UInt32) engine = ReplicatedMergeTree order by x;" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db" +${CLICKHOUSE_CLIENT} -q "DROP USER user_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/02232_functions_to_subcolumns_alias.reference b/tests/queries/0_stateless/02232_functions_to_subcolumns_alias.reference new file mode 100644 index 00000000000..f18e41e497e --- /dev/null +++ b/tests/queries/0_stateless/02232_functions_to_subcolumns_alias.reference @@ -0,0 +1,8 @@ +cnt +2 +t0 t0 +100 100 +0 0 +hit +1 +0 diff --git a/tests/queries/0_stateless/02232_functions_to_subcolumns_alias.sql b/tests/queries/0_stateless/02232_functions_to_subcolumns_alias.sql new file mode 100644 index 00000000000..89383ed4ba3 --- /dev/null +++ b/tests/queries/0_stateless/02232_functions_to_subcolumns_alias.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS t_functions_to_subcolumns_alias; + +CREATE TABLE t_functions_to_subcolumns_alias (id UInt64, t Tuple(UInt64, String), m Map(String, UInt64)) ENGINE = Memory; +INSERT INTO t_functions_to_subcolumns_alias VALUES (1, (100, 'abc'), map('foo', 1, 'bar', 2)) (2, NULL, map()); + +SELECT count(id) AS cnt FROM t_functions_to_subcolumns_alias FORMAT TSVWithNames; +SELECT tupleElement(t, 1) as t0, t0 FROM t_functions_to_subcolumns_alias FORMAT TSVWithNames; +SELECT mapContains(m, 'foo') AS hit FROM t_functions_to_subcolumns_alias FORMAT TSVWithNames; + +DROP TABLE t_functions_to_subcolumns_alias; diff --git a/tests/queries/0_stateless/02232_partition_pruner_mixed_constant_type.reference b/tests/queries/0_stateless/02232_partition_pruner_mixed_constant_type.reference new file mode 100644 index 00000000000..6fcbc14234d --- /dev/null +++ b/tests/queries/0_stateless/02232_partition_pruner_mixed_constant_type.reference @@ -0,0 +1,4 @@ +1647353101000 +1647353101001 +1647353101002 +1647353101003 diff --git a/tests/queries/0_stateless/02232_partition_pruner_mixed_constant_type.sql b/tests/queries/0_stateless/02232_partition_pruner_mixed_constant_type.sql new file mode 100644 index 00000000000..a0b58271764 --- /dev/null +++ b/tests/queries/0_stateless/02232_partition_pruner_mixed_constant_type.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS broken; + +CREATE TABLE broken (time UInt64) ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(time / 1000)) ORDER BY time; +INSERT INTO broken (time) VALUES (1647353101000), (1647353101001), (1647353101002), (1647353101003); +SELECT * FROM broken WHERE time>-1; + +DROP TABLE broken; diff --git a/tests/queries/0_stateless/02232_partition_pruner_single_point.reference b/tests/queries/0_stateless/02232_partition_pruner_single_point.reference new file mode 100644 index 00000000000..1191247b6d9 --- /dev/null +++ b/tests/queries/0_stateless/02232_partition_pruner_single_point.reference @@ -0,0 +1,2 @@ +1 +2 diff --git a/tests/queries/0_stateless/02232_partition_pruner_single_point.sql b/tests/queries/0_stateless/02232_partition_pruner_single_point.sql new file mode 100644 index 00000000000..0400d0e1b59 --- /dev/null +++ b/tests/queries/0_stateless/02232_partition_pruner_single_point.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS lower_test; + +CREATE TABLE lower_test ( + a Int32, + b String +) ENGINE=MergeTree +PARTITION BY b +ORDER BY a; + +INSERT INTO lower_test (a,b) VALUES (1,'A'),(2,'B'),(3,'C'); + +SELECT a FROM lower_test WHERE lower(b) IN ('a','b') order by a; + +DROP TABLE lower_test; diff --git a/tests/queries/0_stateless/02233_HTTP_ranged.python b/tests/queries/0_stateless/02233_HTTP_ranged.python new file mode 100644 index 00000000000..e0198210c16 --- /dev/null +++ b/tests/queries/0_stateless/02233_HTTP_ranged.python @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +from http.server import BaseHTTPRequestHandler, HTTPServer +import socket +import sys +import re +import threading +import os +import traceback +import urllib.request +import subprocess + + +def is_ipv6(host): + try: + socket.inet_aton(host) + return False + except: + return True + + +def get_local_port(host, ipv6): + if ipv6: + family = socket.AF_INET6 + else: + family = socket.AF_INET + + with socket.socket(family) as fd: + fd.bind((host, 0)) + return fd.getsockname()[1] + + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "localhost") +CLICKHOUSE_PORT_HTTP = os.environ.get("CLICKHOUSE_PORT_HTTP", "8123") + +# Server returns this JSON response. +SERVER_JSON_RESPONSE = """{ + "login": "ClickHouse", + "id": 54801242, + "name": "ClickHouse", + "company": null +}""" + +PAYLOAD_LEN = len(SERVER_JSON_RESPONSE) + +EXPECTED_ANSWER = """{\\n\\t"login": "ClickHouse",\\n\\t"id": 54801242,\\n\\t"name": "ClickHouse",\\n\\t"company": null\\n}""" + +##################################################################################### +# This test starts an HTTP server and serves data to clickhouse url-engine based table. +# The objective of this test is to check the ClickHouse server provides a User-Agent +# with HTTP requests. +# In order for it to work ip+port of http server (given below) should be +# accessible from clickhouse server. +##################################################################################### + +# IP-address of this host accessible from the outside world. Get the first one +HTTP_SERVER_HOST = ( + subprocess.check_output(["hostname", "-i"]).decode("utf-8").strip().split()[0] +) +IS_IPV6 = is_ipv6(HTTP_SERVER_HOST) +HTTP_SERVER_PORT = get_local_port(HTTP_SERVER_HOST, IS_IPV6) + +# IP address and port of the HTTP server started from this script. +HTTP_SERVER_ADDRESS = (HTTP_SERVER_HOST, HTTP_SERVER_PORT) +if IS_IPV6: + HTTP_SERVER_URL_STR = ( + "http://" + + f"[{str(HTTP_SERVER_ADDRESS[0])}]:{str(HTTP_SERVER_ADDRESS[1])}" + + "/" + ) +else: + HTTP_SERVER_URL_STR = ( + "http://" + f"{str(HTTP_SERVER_ADDRESS[0])}:{str(HTTP_SERVER_ADDRESS[1])}" + "/" + ) + + +def get_ch_answer(query): + host = CLICKHOUSE_HOST + if IS_IPV6: + host = f"[{host}]" + + url = os.environ.get( + "CLICKHOUSE_URL", + "http://{host}:{port}".format(host=CLICKHOUSE_HOST, port=CLICKHOUSE_PORT_HTTP), + ) + return urllib.request.urlopen(url, data=query.encode()).read().decode() + + +def check_answers(query, answer): + ch_answer = get_ch_answer(query) + if ch_answer.strip() != answer.strip(): + print("FAIL on query:", query, file=sys.stderr) + print("Expected answer:", answer, file=sys.stderr) + print("Fetched answer :", ch_answer, file=sys.stderr) + raise Exception("Fail on query") + + +BYTE_RANGE_RE = re.compile(r"bytes=(\d+)-(\d+)?$") + + +def parse_byte_range(byte_range): + """Returns the two numbers in 'bytes=123-456' or throws ValueError. + The last number or both numbers may be None. + """ + if byte_range.strip() == "": + return None, None + + m = BYTE_RANGE_RE.match(byte_range) + if not m: + raise ValueError(f"Invalid byte range {byte_range}") + + first, last = [x and int(x) for x in m.groups()] + if last and last < first: + raise ValueError(f"Invalid byte range {byte_range}") + return first, last + + +# Server with check for User-Agent headers. +class HttpProcessor(BaseHTTPRequestHandler): + allow_range = False + range_used = False + get_call_num = 0 + + def send_head(self): + if self.headers["Range"] and HttpProcessor.allow_range: + try: + self.range = parse_byte_range(self.headers["Range"]) + except ValueError as e: + self.send_error(400, "Invalid byte range") + return None + else: + self.range = None + + if self.range: + first, last = self.range + else: + first, last = None, None + + if first == None: + first = 0 + + payload = SERVER_JSON_RESPONSE.encode() + payload_len = len(payload) + if first and first >= payload_len: + self.send_error(416, "Requested Range Not Satisfiable") + return None + + self.send_response(206 if HttpProcessor.allow_range else 200) + self.send_header("Content-type", "application/json") + + if HttpProcessor.allow_range: + self.send_header("Accept-Ranges", "bytes") + + if last is None or last >= payload_len: + last = payload_len - 1 + + response_length = last - first + 1 + + if first or last: + self.send_header("Content-Range", f"bytes {first}-{last}/{payload_len}") + self.send_header( + "Content-Length", + str(response_length) if HttpProcessor.allow_range else str(payload_len), + ) + self.end_headers() + return payload + + def do_HEAD(self): + self.send_head() + + def do_GET(self): + result = self.send_head() + if result == None: + return + + HttpProcessor.get_call_num += 1 + + if not self.range: + self.wfile.write(SERVER_JSON_RESPONSE.encode()) + return + + HttpProcessor.range_used = True + payload = SERVER_JSON_RESPONSE.encode() + start, stop = self.range + if stop == None: + stop = len(payload) - 1 + if start == None: + start = 0 + self.wfile.write(SERVER_JSON_RESPONSE.encode()[start : stop + 1]) + + def log_message(self, format, *args): + return + + +class HTTPServerV6(HTTPServer): + address_family = socket.AF_INET6 + + +def start_server(): + if IS_IPV6: + httpd = HTTPServerV6(HTTP_SERVER_ADDRESS, HttpProcessor) + else: + httpd = HTTPServer(HTTP_SERVER_ADDRESS, HttpProcessor) + + t = threading.Thread(target=httpd.serve_forever) + return t, httpd + + +##################################################################### +# Testing area. +##################################################################### + + +def test_select(download_buffer_size): + global HTTP_SERVER_URL_STR + query = f"SELECT * FROM url('{HTTP_SERVER_URL_STR}','JSONAsString') SETTINGS max_download_buffer_size={download_buffer_size};" + check_answers(query, EXPECTED_ANSWER) + + +def run_test(allow_range, download_buffer_size=20): + HttpProcessor.range_used = False + HttpProcessor.get_call_num = 0 + HttpProcessor.allow_range = allow_range + + t, httpd = start_server() + t.start() + test_select(download_buffer_size) + + expected_get_call_num = (PAYLOAD_LEN - 1) // download_buffer_size + 1 + if allow_range: + if not HttpProcessor.range_used: + raise Exception("HTTP Range was not used when supported") + + if expected_get_call_num != HttpProcessor.get_call_num: + raise Exception( + f"Invalid amount of GET calls with Range. Expected {expected_get_call_num}, actual {HttpProcessor.get_call_num}" + ) + else: + if HttpProcessor.range_used: + raise Exception("HTTP Range used while not supported") + + httpd.shutdown() + t.join() + print("PASSED") + + +def main(): + run_test(allow_range=False) + run_test(allow_range=True, download_buffer_size=20) + run_test(allow_range=True, download_buffer_size=10) + + +if __name__ == "__main__": + try: + main() + except Exception as ex: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_tb(exc_traceback, file=sys.stderr) + print(ex, file=sys.stderr) + sys.stderr.flush() + + os._exit(1) diff --git a/tests/queries/0_stateless/02233_HTTP_ranged.reference b/tests/queries/0_stateless/02233_HTTP_ranged.reference new file mode 100644 index 00000000000..17f0fff172a --- /dev/null +++ b/tests/queries/0_stateless/02233_HTTP_ranged.reference @@ -0,0 +1,3 @@ +PASSED +PASSED +PASSED diff --git a/tests/queries/0_stateless/02233_HTTP_ranged.sh b/tests/queries/0_stateless/02233_HTTP_ranged.sh new file mode 100755 index 00000000000..b6fba098d10 --- /dev/null +++ b/tests/queries/0_stateless/02233_HTTP_ranged.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +python3 "$CURDIR"/02233_HTTP_ranged.python + diff --git a/tests/queries/0_stateless/02233_set_enable_with_statement_cte_perf.reference b/tests/queries/0_stateless/02233_set_enable_with_statement_cte_perf.reference new file mode 100644 index 00000000000..d97ba04b8c3 --- /dev/null +++ b/tests/queries/0_stateless/02233_set_enable_with_statement_cte_perf.reference @@ -0,0 +1,4 @@ +1000 +1000 +1000 +3 diff --git a/tests/queries/0_stateless/02233_set_enable_with_statement_cte_perf.sql b/tests/queries/0_stateless/02233_set_enable_with_statement_cte_perf.sql new file mode 100644 index 00000000000..71321b4dfe4 --- /dev/null +++ b/tests/queries/0_stateless/02233_set_enable_with_statement_cte_perf.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS ev; +DROP TABLE IF EXISTS idx; + +CREATE TABLE ev (a Int32, b Int32) Engine=MergeTree() ORDER BY a; +CREATE TABLE idx (a Int32) Engine=MergeTree() ORDER BY a; +INSERT INTO ev SELECT number, number FROM numbers(10000000); +INSERT INTO idx SELECT number * 5 FROM numbers(1000); + +-- test_enable_global_with_statement_performance_1 +WITH 'test' AS u SELECT count() FROM ev WHERE a IN (SELECT a FROM idx) SETTINGS enable_global_with_statement = 1; + +-- test_enable_global_with_statement_performance_2 +SELECT count() FROM ev WHERE a IN (SELECT a FROM idx) SETTINGS enable_global_with_statement = 1; + +-- test_enable_global_with_statement_performance_3 +WITH 'test' AS u SELECT count() FROM ev WHERE a IN (SELECT a FROM idx) SETTINGS enable_global_with_statement = 0; + +SYSTEM FLUSH LOGS; + +SELECT count(read_rows) FROM (SELECT read_rows FROM system.query_log WHERE current_database=currentDatabase() AND type='QueryFinish' AND query LIKE '-- test_enable_global_with_statement_performance%' ORDER BY initial_query_start_time_microseconds DESC LIMIT 3) GROUP BY read_rows; + +DROP TABLE IF EXISTS ev; +DROP TABLE IF EXISTS idx; diff --git a/tests/queries/0_stateless/02233_setting_input_format_use_lowercase_column_name.reference b/tests/queries/0_stateless/02233_setting_input_format_use_lowercase_column_name.reference new file mode 100644 index 00000000000..5c383cb3035 --- /dev/null +++ b/tests/queries/0_stateless/02233_setting_input_format_use_lowercase_column_name.reference @@ -0,0 +1,6 @@ +Parquet +123 1 +456 2 +ORC +123 1 +456 2 diff --git a/tests/queries/0_stateless/02233_setting_input_format_use_lowercase_column_name.sh b/tests/queries/0_stateless/02233_setting_input_format_use_lowercase_column_name.sh new file mode 100755 index 00000000000..b946addd01c --- /dev/null +++ b/tests/queries/0_stateless/02233_setting_input_format_use_lowercase_column_name.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +echo "Parquet" +DATA_FILE=$CUR_DIR/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_load (id String, score Int32) ENGINE = Memory" +cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_load FORMAT Parquet SETTINGS input_format_use_lowercase_column_name=true" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_load" +${CLICKHOUSE_CLIENT} --query="drop table parquet_load" + +echo "ORC" +DATA_FILE=$CUR_DIR/data_orc/test_setting_input_format_use_lowercase_column_name.orc +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (id String, score Int32) ENGINE = Memory" +cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "INSERT INTO orc_load FORMAT ORC SETTINGS input_format_use_lowercase_column_name=true" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_load" +${CLICKHOUSE_CLIENT} --query="drop table orc_load" diff --git a/tests/queries/0_stateless/02233_with_total_empty_chunk.reference b/tests/queries/0_stateless/02233_with_total_empty_chunk.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02233_with_total_empty_chunk.sql b/tests/queries/0_stateless/02233_with_total_empty_chunk.sql new file mode 100644 index 00000000000..bf9ce85b6ed --- /dev/null +++ b/tests/queries/0_stateless/02233_with_total_empty_chunk.sql @@ -0,0 +1 @@ +SELECT (NULL, NULL, NULL, NULL, NULL, NULL, NULL) FROM numbers(0) GROUP BY number WITH TOTALS HAVING sum(number) <= arrayJoin([]); diff --git a/tests/queries/0_stateless/02234_cast_to_ip_address.reference b/tests/queries/0_stateless/02234_cast_to_ip_address.reference new file mode 100644 index 00000000000..3a4c40a07cf --- /dev/null +++ b/tests/queries/0_stateless/02234_cast_to_ip_address.reference @@ -0,0 +1,45 @@ +IPv4 functions +0 +\N +2130706433 +2130706433 +2130706433 +-- +0.0.0.0 +\N +127.0.0.1 +127.0.0.1 +127.0.0.1 +-- +127.0.0.1 +-- +0 +0.0.0.0 +0 +0.0.0.0 +0.0.0.0 +0.0.0.0 +IPv6 functions +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0 +\N +\0\0\0\0\0\0\0\0\0\0\0\0 +\0\0\0\0\0\0\0\0\0\0\0\0 +\0\0\0\0\0\0\0\0\0\0\0\0 +-- +:: +\N +::ffff:127.0.0.1 +::ffff:127.0.0.1 +::ffff:127.0.0.1 +-- +::ffff:127.0.0.1 +-- +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0 +:: +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0 +:: +:: +:: +-- +::ffff:127.0.0.1 ::ffff:127.0.0.1 ::ffff:127.0.0.1 +::1\0\0 ::1 ::1 diff --git a/tests/queries/0_stateless/02234_cast_to_ip_address.sql b/tests/queries/0_stateless/02234_cast_to_ip_address.sql new file mode 100644 index 00000000000..d436c93b9db --- /dev/null +++ b/tests/queries/0_stateless/02234_cast_to_ip_address.sql @@ -0,0 +1,82 @@ +SELECT 'IPv4 functions'; + +SELECT IPv4StringToNum('test'); --{serverError 441} +SELECT IPv4StringToNumOrDefault('test'); +SELECT IPv4StringToNumOrNull('test'); + +SELECT IPv4StringToNum('127.0.0.1'); +SELECT IPv4StringToNumOrDefault('127.0.0.1'); +SELECT IPv4StringToNumOrNull('127.0.0.1'); + +SELECT '--'; + +SELECT toIPv4('test'); --{serverError 441} +SELECT toIPv4OrDefault('test'); +SELECT toIPv4OrNull('test'); + +SELECT toIPv4('127.0.0.1'); +SELECT toIPv4OrDefault('127.0.0.1'); +SELECT toIPv4OrNull('127.0.0.1'); + +SELECT '--'; + +SELECT cast('test' , 'IPv4'); --{serverError 441} +SELECT cast('127.0.0.1' , 'IPv4'); + +SELECT '--'; + +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + +SELECT IPv4StringToNum('test'); +SELECT toIPv4('test'); +SELECT IPv4StringToNum(''); +SELECT toIPv4(''); +SELECT cast('test' , 'IPv4'); +SELECT cast('' , 'IPv4'); + +SET cast_ipv4_ipv6_default_on_conversion_error = 0; + +SELECT 'IPv6 functions'; + +SELECT IPv6StringToNum('test'); --{serverError 441} +SELECT IPv6StringToNumOrDefault('test'); +SELECT IPv6StringToNumOrNull('test'); + +SELECT IPv6StringToNum('::ffff:127.0.0.1'); +SELECT IPv6StringToNumOrDefault('::ffff:127.0.0.1'); +SELECT IPv6StringToNumOrNull('::ffff:127.0.0.1'); + +SELECT '--'; + +SELECT toIPv6('test'); --{serverError 441} +SELECT toIPv6OrDefault('test'); +SELECT toIPv6OrNull('test'); + +SELECT toIPv6('::ffff:127.0.0.1'); +SELECT toIPv6OrDefault('::ffff:127.0.0.1'); +SELECT toIPv6OrNull('::ffff:127.0.0.1'); + +SELECT '--'; + +SELECT cast('test' , 'IPv6'); --{serverError 441} +SELECT cast('::ffff:127.0.0.1', 'IPv6'); + +SELECT '--'; + +SET cast_ipv4_ipv6_default_on_conversion_error = 1; + +SELECT IPv6StringToNum('test'); +SELECT toIPv6('test'); +SELECT IPv6StringToNum(''); +SELECT toIPv6(''); +SELECT cast('test' , 'IPv6'); +SELECT cast('' , 'IPv6'); + +SELECT '--'; + +SET cast_ipv4_ipv6_default_on_conversion_error = 0; + +SELECT toFixedString('::ffff:127.0.0.1', 16) as value, cast(value, 'IPv6'), toIPv6(value); +SELECT toFixedString('::1', 5) as value, cast(value, 'IPv6'), toIPv6(value); +SELECT toFixedString('', 16) as value, cast(value, 'IPv6'); --{serverError 441} +SELECT toFixedString('', 16) as value, toIPv6(value); --{serverError 441} diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh new file mode 100755 index 00000000000..6abe1e30334 --- /dev/null +++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_LOCAL --query="SELECT n" 2>&1 | grep -q "Code: 47. DB::Exception: Missing columns:" && echo 'OK' || echo 'FAIL' ||: +$CLICKHOUSE_LOCAL --testmode --query="SELECT n -- { serverError 47 }" + diff --git a/tests/queries/0_stateless/02234_column_function_short_circuit.reference b/tests/queries/0_stateless/02234_column_function_short_circuit.reference new file mode 100644 index 00000000000..2c08a29620e --- /dev/null +++ b/tests/queries/0_stateless/02234_column_function_short_circuit.reference @@ -0,0 +1,2 @@ +2.3 +4.3 diff --git a/tests/queries/0_stateless/02234_column_function_short_circuit.sql b/tests/queries/0_stateless/02234_column_function_short_circuit.sql new file mode 100644 index 00000000000..a6a36841073 --- /dev/null +++ b/tests/queries/0_stateless/02234_column_function_short_circuit.sql @@ -0,0 +1,43 @@ +DROP TABLE IF EXISTS dict_table; +DROP TABLE IF EXISTS data_table; +DROP DICTIONARY IF EXISTS dict; + +create table dict_table +( + `strField` String, + `dateField` Date, + `float64Field` Float64 +) Engine Log(); + +insert into dict_table values ('SomeStr', toDate('2021-01-01'), 1.1), ('SomeStr2', toDate('2021-01-02'), 2.2); + +create dictionary dict +( + `strField` String, + `dateField` Date, + `float64Field` Float64 +) +PRIMARY KEY strField, dateField +SOURCE (CLICKHOUSE(TABLE 'dict_table')) +LIFETIME(MIN 300 MAX 360) +LAYOUT (COMPLEX_KEY_HASHED()); + +create table data_table +( + `float64Field1` Float64, + `float64Field2` Float64, + `strField1` String, + `strField2` String +) Engine Log(); + +insert into data_table values (1.1, 1.2, 'SomeStr', 'SomeStr'), (2.1, 2.2, 'SomeStr2', 'SomeStr2'); + +select round( + float64Field1 * if(strField1 != '', 1.0, dictGetFloat64('dict', 'float64Field', (strField1, toDate('2021-01-01')))) + + if(strField2 != '', 1.0, dictGetFloat64('dict', 'float64Field', (strField2, toDate('2021-01-01')))) * if(isFinite(float64Field2), float64Field2, 0), + 2) +from data_table; + +DROP DICTIONARY dict; +DROP TABLE dict_table; +DROP TABLE data_table; diff --git a/tests/queries/0_stateless/02234_position_case_insensitive_utf8.reference b/tests/queries/0_stateless/02234_position_case_insensitive_utf8.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/02234_position_case_insensitive_utf8.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/02234_position_case_insensitive_utf8.sql b/tests/queries/0_stateless/02234_position_case_insensitive_utf8.sql new file mode 100644 index 00000000000..d77b13e7f97 --- /dev/null +++ b/tests/queries/0_stateless/02234_position_case_insensitive_utf8.sql @@ -0,0 +1,2 @@ +SELECT positionCaseInsensitiveUTF8('Hello', materialize('%\xF0%')); +SELECT DISTINCT positionCaseInsensitiveUTF8(materialize('Hello'), '%\xF0%') FROM numbers(1000); diff --git a/tests/queries/0_stateless/02235_brotli_bug.reference b/tests/queries/0_stateless/02235_brotli_bug.reference new file mode 100644 index 00000000000..d59d3c7902c --- /dev/null +++ b/tests/queries/0_stateless/02235_brotli_bug.reference @@ -0,0 +1 @@ +1000000 999999 diff --git a/tests/queries/0_stateless/02235_brotli_bug.sh b/tests/queries/0_stateless/02235_brotli_bug.sh new file mode 100755 index 00000000000..39b1e555ed7 --- /dev/null +++ b/tests/queries/0_stateless/02235_brotli_bug.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: depends on brotli and bzip2 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS file" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE file (x UInt64) ENGINE = File(TSV, '${CLICKHOUSE_DATABASE}/data.tsv.br')" +${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE file" +${CLICKHOUSE_CLIENT} --query "INSERT INTO file SELECT * FROM numbers(1000000)" +${CLICKHOUSE_CLIENT} --max_read_buffer_size=8 --query "SELECT count(), max(x) FROM file" +${CLICKHOUSE_CLIENT} --query "DROP TABLE file" diff --git a/tests/queries/0_stateless/02235_check_table_sparse_serialization.reference b/tests/queries/0_stateless/02235_check_table_sparse_serialization.reference new file mode 100644 index 00000000000..35e27925057 --- /dev/null +++ b/tests/queries/0_stateless/02235_check_table_sparse_serialization.reference @@ -0,0 +1,4 @@ +all_1_1_0 a Default +all_2_2_0 a Sparse +all_1_1_0 1 +all_2_2_0 1 diff --git a/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql b/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql new file mode 100644 index 00000000000..0ac97404c46 --- /dev/null +++ b/tests/queries/0_stateless/02235_check_table_sparse_serialization.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS t_sparse_02235; + +CREATE TABLE t_sparse_02235 (a UInt8) ENGINE = MergeTree ORDER BY tuple() +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +SYSTEM STOP MERGES t_sparse_02235; + +INSERT INTO t_sparse_02235 SELECT 1 FROM numbers(1000); +INSERT INTO t_sparse_02235 SELECT 0 FROM numbers(1000); + +SELECT name, column, serialization_kind FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_sparse_02235' +ORDER BY name, column; + +SET check_query_single_value_result = 0; +CHECK TABLE t_sparse_02235; + +DROP TABLE t_sparse_02235; diff --git a/tests/queries/0_stateless/02236_json_each_row_empty_map_schema_inference.reference b/tests/queries/0_stateless/02236_json_each_row_empty_map_schema_inference.reference new file mode 100644 index 00000000000..864f98fd923 --- /dev/null +++ b/tests/queries/0_stateless/02236_json_each_row_empty_map_schema_inference.reference @@ -0,0 +1,2 @@ +{} +{'b':1} diff --git a/tests/queries/0_stateless/02236_json_each_row_empty_map_schema_inference.sql b/tests/queries/0_stateless/02236_json_each_row_empty_map_schema_inference.sql new file mode 100644 index 00000000000..8241c8ead37 --- /dev/null +++ b/tests/queries/0_stateless/02236_json_each_row_empty_map_schema_inference.sql @@ -0,0 +1,3 @@ +-- Tags: no-fasttest + +select * from format(JSONEachRow, '{"a" : {}}, {"a" : {"b" : 1}}') diff --git a/tests/queries/0_stateless/02237_lzma_bug.reference b/tests/queries/0_stateless/02237_lzma_bug.reference new file mode 100644 index 00000000000..d59d3c7902c --- /dev/null +++ b/tests/queries/0_stateless/02237_lzma_bug.reference @@ -0,0 +1 @@ +1000000 999999 diff --git a/tests/queries/0_stateless/02237_lzma_bug.sh b/tests/queries/0_stateless/02237_lzma_bug.sh new file mode 100755 index 00000000000..163a530bf25 --- /dev/null +++ b/tests/queries/0_stateless/02237_lzma_bug.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS file" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE file (x UInt64) ENGINE = File(TSV, '${CLICKHOUSE_DATABASE}/data.tsv.xz')" +${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE file" +${CLICKHOUSE_CLIENT} --query "INSERT INTO file SELECT * FROM numbers(1000000)" +${CLICKHOUSE_CLIENT} --max_read_buffer_size=8 --query "SELECT count(), max(x) FROM file" +${CLICKHOUSE_CLIENT} --query "DROP TABLE file" + diff --git a/tests/queries/0_stateless/02238_lz4_bug.reference b/tests/queries/0_stateless/02238_lz4_bug.reference new file mode 100644 index 00000000000..d59d3c7902c --- /dev/null +++ b/tests/queries/0_stateless/02238_lz4_bug.reference @@ -0,0 +1 @@ +1000000 999999 diff --git a/tests/queries/0_stateless/02238_lz4_bug.sh b/tests/queries/0_stateless/02238_lz4_bug.sh new file mode 100755 index 00000000000..92d0aa1894f --- /dev/null +++ b/tests/queries/0_stateless/02238_lz4_bug.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: depends on brotli and bzip2 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS file" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE file (x UInt64) ENGINE = File(TSV, '${CLICKHOUSE_DATABASE}/data.tsv.lz4')" +${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE file" +${CLICKHOUSE_CLIENT} --query "INSERT INTO file SELECT * FROM numbers(1000000)" +${CLICKHOUSE_CLIENT} --max_read_buffer_size=2 --query "SELECT count(), max(x) FROM file" +${CLICKHOUSE_CLIENT} --query "DROP TABLE file" + diff --git a/tests/queries/0_stateless/02239_bzip2_bug.reference b/tests/queries/0_stateless/02239_bzip2_bug.reference new file mode 100644 index 00000000000..d59d3c7902c --- /dev/null +++ b/tests/queries/0_stateless/02239_bzip2_bug.reference @@ -0,0 +1 @@ +1000000 999999 diff --git a/tests/queries/0_stateless/02239_bzip2_bug.sh b/tests/queries/0_stateless/02239_bzip2_bug.sh new file mode 100755 index 00000000000..fc72a0178e0 --- /dev/null +++ b/tests/queries/0_stateless/02239_bzip2_bug.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: depends on brotli and bzip2 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS file" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE file (x UInt64) ENGINE = File(TSV, '${CLICKHOUSE_DATABASE}/data.tsv.bz2')" +${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE file" +${CLICKHOUSE_CLIENT} --query "INSERT INTO file SELECT * FROM numbers(1000000)" +${CLICKHOUSE_CLIENT} --max_read_buffer_size=2 --query "SELECT count(), max(x) FROM file" +${CLICKHOUSE_CLIENT} --query "DROP TABLE file" + diff --git a/tests/queries/0_stateless/02239_client_host_help.reference b/tests/queries/0_stateless/02239_client_host_help.reference new file mode 100644 index 00000000000..2c94e483710 --- /dev/null +++ b/tests/queries/0_stateless/02239_client_host_help.reference @@ -0,0 +1,2 @@ +OK +OK diff --git a/tests/queries/0_stateless/02239_client_host_help.sh b/tests/queries/0_stateless/02239_client_host_help.sh new file mode 100755 index 00000000000..7b31b9d4dd7 --- /dev/null +++ b/tests/queries/0_stateless/02239_client_host_help.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --help | grep -q "\-\-host" && echo "OK" || echo "FAIL" +${CLICKHOUSE_CLIENT} --help | grep -q "\-\-port arg" && echo "OK" || echo "FAIL" + diff --git a/tests/queries/0_stateless/02240_asof_join_biginteger.reference b/tests/queries/0_stateless/02240_asof_join_biginteger.reference new file mode 100644 index 00000000000..cac55eec430 --- /dev/null +++ b/tests/queries/0_stateless/02240_asof_join_biginteger.reference @@ -0,0 +1,4 @@ +0 18446744073709551617 +0 340282366920938463463374607431768211457 +0 18446744073709551617 +0 340282366920938463463374607431768211457 diff --git a/tests/queries/0_stateless/02240_asof_join_biginteger.sql b/tests/queries/0_stateless/02240_asof_join_biginteger.sql new file mode 100644 index 00000000000..6dc5b00f116 --- /dev/null +++ b/tests/queries/0_stateless/02240_asof_join_biginteger.sql @@ -0,0 +1,5 @@ +select * from (select 0 as k, toInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toInt128('18446744073709551616') as v) t2 using(k, v); +select * from (select 0 as k, toInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toInt256('340282366920938463463374607431768211456') as v) t2 using(k, v); + +select * from (select 0 as k, toUInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toUInt128('18446744073709551616') as v) t2 using(k, v); +select * from (select 0 as k, toUInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toUInt256('340282366920938463463374607431768211456') as v) t2 using(k, v); diff --git a/tests/queries/0_stateless/02240_protobuflist_format_persons.reference b/tests/queries/0_stateless/02240_protobuflist_format_persons.reference new file mode 100644 index 00000000000..3ff4a6e57df --- /dev/null +++ b/tests/queries/0_stateless/02240_protobuflist_format_persons.reference @@ -0,0 +1,569 @@ +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuflist1_format_persons:Person + +Binary representation: +00000000 ba 04 0a f4 01 0a 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +00000010 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +00000020 63 35 39 38 65 65 35 35 63 34 39 12 04 49 76 61 |c598ee55c49..Iva| +00000030 6e 1a 06 50 65 74 72 6f 76 20 01 28 af 1f 32 03 |n..Petrov .(..2.| +00000040 70 6e 67 3a 0d 2b 37 34 39 35 31 32 33 34 35 36 |png:.+7495123456| +00000050 37 00 40 01 4d fc d0 30 5c 50 26 58 09 62 09 59 |7.@.M..0\P&X.b.Y| +00000060 65 73 74 65 72 64 61 79 62 07 46 6c 6f 77 65 72 |esterdayb.Flower| +00000070 73 6a 04 ff 01 00 00 72 06 4d 6f 73 63 6f 77 7a |sj.....r.Moscowz| +00000080 08 4b 03 5f 42 72 7d 16 42 81 01 1f 85 eb 51 b8 |.K._Br}.B.....Q.| +00000090 1e 09 40 89 01 33 33 33 33 33 c3 6a 40 95 01 cd |..@..33333.j@...| +000000a0 cc cc 3d 9d 01 9a 99 b9 40 a0 01 80 c4 d7 8d 7f |..=.....@.......| +000000b0 aa 01 0c 0a 05 6d 65 74 65 72 15 00 00 80 3f aa |.....meter....?.| +000000c0 01 11 0a 0a 63 65 6e 74 69 6d 65 74 65 72 15 0a |....centimeter..| +000000d0 d7 23 3c aa 01 10 0a 09 6b 69 6c 6f 6d 65 74 65 |.#<.....kilomete| +000000e0 72 15 00 00 7a 44 b2 01 10 0a 0e a2 06 0b 0a 09 |r...zD..........| +000000f0 08 f4 03 12 04 f5 03 f6 03 0a 7e 0a 24 63 36 39 |..........~.$c69| +00000100 34 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d |4ad8a-f714-4ea3-| +00000110 39 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 |907d-fd54fb25d9b| +00000120 35 12 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f |5..Natalia..Soko| +00000130 6c 6f 76 61 28 a6 3f 32 03 6a 70 67 50 1a 58 0b |lova(.?2.jpgP.X.| +00000140 6a 04 64 c8 01 32 72 08 50 6c 79 6d 6f 75 74 68 |j.d..2r.Plymouth| +00000150 7a 08 6a 9d 49 42 46 8c 84 c0 81 01 6e 86 1b f0 |z.j.IBF.....n...| +00000160 f9 21 09 40 95 01 42 60 e5 3b 9d 01 cd cc ac 40 |.!.@..B`.;.....@| +00000170 a0 01 ff ff a9 ce 93 8c 09 0a c0 01 0a 24 61 37 |.............$a7| +00000180 64 61 31 61 61 36 2d 66 34 32 35 2d 34 37 38 39 |da1aa6-f425-4789| +00000190 2d 38 39 34 37 2d 62 30 33 34 37 38 36 65 64 33 |-8947-b034786ed3| +000001a0 37 34 12 06 56 61 73 69 6c 79 1a 07 53 69 64 6f |74..Vasily..Sido| +000001b0 72 6f 76 20 01 28 fb 48 32 03 62 6d 70 3a 0d 2b |rov .(.H2.bmp:.+| +000001c0 34 34 32 30 31 32 33 34 35 36 37 38 40 01 4d 50 |442012345678@.MP| +000001d0 e0 27 5c 50 17 58 04 62 05 53 75 6e 6e 79 6a 05 |.'\P.X.b.Sunnyj.| +000001e0 fa 01 f4 01 0a 72 08 4d 75 72 6d 61 6e 73 6b 7a |.....r.Murmanskz| +000001f0 08 fd f0 89 42 c8 4c 04 42 81 01 11 2d 44 54 fb |....B.L.B...-DT.| +00000200 21 09 40 89 01 00 00 00 e8 76 48 37 42 95 01 00 |!.@......vH7B...| +00000210 00 48 44 9d 01 cd cc 4c c0 a0 01 80 d4 9f 93 01 |.HD....L........| +00000220 aa 01 0c 0a 05 70 6f 75 6e 64 15 00 00 80 41 b2 |.....pound....A.| +00000230 01 0a 0a 08 a2 06 05 0a 03 08 f7 03 |............| +0000023c + +MESSAGE #1 AT 0x00000005 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: male +birthDate: 4015 +photo: "png" +phoneNumber: "+74951234567\000" +isOnline: true +visitTime: 1546703100 +age: 38 +zodiacSign: capricorn +songs: "Yesterday" +songs: "Flowers" +color: 255 +color: 0 +color: 0 +hometown: "Moscow" +location: 55.7532158 +location: 37.6225052 +pi: 3.14 +lotteryWin: 214.1 +someRatio: 0.1 +temperature: 5.8 +randomBigNumber: 17060000000 +measureUnits { + unit: "meter" + coef: 1 +} +measureUnits { + unit: "centimeter" + coef: 0.01 +} +measureUnits { + unit: "kilometer" + coef: 1000 +} +nestiness { + a { + b { + c { + d: 500 + e: 501 + e: 502 + } + } + } +} +MESSAGE #2 AT 0x000000FB +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +birthDate: 8102 +photo: "jpg" +age: 26 +zodiacSign: pisces +color: 100 +color: 200 +color: 50 +hometown: "Plymouth" +location: 50.4037247 +location: -4.14212322 +pi: 3.14159 +someRatio: 0.007 +temperature: 5.4 +randomBigNumber: -20000000000000 +MESSAGE #3 AT 0x0000017C +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: male +birthDate: 9339 +photo: "bmp" +phoneNumber: "+442012345678" +isOnline: true +visitTime: 1546117200 +age: 23 +zodiacSign: leo +songs: "Sunny" +color: 250 +color: 244 +color: 10 +hometown: "Murmansk" +location: 68.9706802 +location: 33.0749817 +pi: 3.14159265358979 +lotteryWin: 100000000000 +someRatio: 800 +temperature: -3.2 +randomBigNumber: 154400000 +measureUnits { + unit: "pound" + coef: 16 +} +nestiness { + a { + b { + c { + d: 503 + } + } + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.97068,33.074982] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuflist2_format_persons:AltPerson + +Binary representation: +00000000 f4 03 0a c4 01 08 01 12 04 49 76 61 6e 28 87 a8 |.........Ivan(..| +00000010 c4 9b 97 02 52 06 50 65 74 72 6f 76 72 0c 00 00 |....R.Petrovr...| +00000020 7f 43 00 00 00 00 00 00 00 00 79 fc d0 30 5c 00 |.C........y..0\.| +00000030 00 00 00 c8 02 0a c2 05 0c 00 00 80 3f 0a d7 23 |............?..#| +00000040 3c 00 00 7a 44 9a 06 05 6d 65 74 65 72 9a 06 0a |<..zD...meter...| +00000050 63 65 6e 74 69 6d 65 74 65 72 9a 06 09 6b 69 6c |centimeter...kil| +00000060 6f 6d 65 74 65 72 a1 06 00 00 00 a0 99 99 b9 3f |ometer.........?| +00000070 a8 06 37 a8 06 25 bd 06 c3 f5 48 40 fa 06 02 33 |..7..%....H@...3| +00000080 38 90 08 c6 09 e1 08 00 f1 da f8 03 00 00 00 b0 |8...............| +00000090 09 af 1f d0 0c d6 01 e2 12 24 61 37 35 32 32 31 |.........$a75221| +000000a0 35 38 2d 33 64 34 31 2d 34 62 37 37 2d 61 64 36 |58-3d41-4b77-ad6| +000000b0 39 2d 36 63 35 39 38 65 65 35 35 63 34 39 a0 38 |9-6c598ee55c49.8| +000000c0 f4 03 aa 38 04 f5 03 f6 03 0a 84 01 12 07 4e 61 |...8..........Na| +000000d0 74 61 6c 69 61 52 08 53 6f 6b 6f 6c 6f 76 61 72 |taliaR.Sokolovar| +000000e0 0c 00 00 c8 42 00 00 48 43 00 00 48 42 c8 02 0a |....B..HC..HB...| +000000f0 a1 06 00 00 00 40 08 ac 7c 3f a8 06 32 a8 06 fc |.....@..|?..2...| +00000100 ff ff ff ff ff ff ff ff 01 b0 06 01 bd 06 d0 0f |................| +00000110 49 40 fa 06 02 32 36 90 08 db 01 e1 08 00 c0 1a |I@...26.........| +00000120 63 cf ed ff ff b0 09 a6 3f e2 12 24 63 36 39 34 |c.......?..$c694| +00000130 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d 39 |ad8a-f714-4ea3-9| +00000140 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 35 |07d-fd54fb25d9b5| +00000150 0a a3 01 08 01 12 06 56 61 73 69 6c 79 28 ce ca |.......Vasily(..| +00000160 f4 cf ee 0c 52 07 53 69 64 6f 72 6f 76 72 0c 00 |....R.Sidorovr..| +00000170 00 7a 43 00 00 74 43 00 00 20 41 79 50 e0 27 5c |.zC..tC.. AyP.'\| +00000180 00 00 00 00 c8 02 05 c2 05 04 00 00 80 41 9a 06 |.............A..| +00000190 05 70 6f 75 6e 64 a1 06 00 00 00 00 00 00 89 40 |.pound.........@| +000001a0 a8 06 44 a8 06 21 bd 06 db 0f 49 40 fa 06 02 32 |..D..!....I@...2| +000001b0 33 90 08 d3 05 e1 08 00 f5 33 09 00 00 00 00 b0 |3........3......| +000001c0 09 fb 48 d0 0c 80 d0 db c3 f4 02 e2 12 24 61 37 |..H..........$a7| +000001d0 64 61 31 61 61 36 2d 66 34 32 35 2d 34 37 38 39 |da1aa6-f425-4789| +000001e0 2d 38 39 34 37 2d 62 30 33 34 37 38 36 65 64 33 |-8947-b034786ed3| +000001f0 37 34 a0 38 f7 03 |74.8..| +000001f6 + +MESSAGE #1 AT 0x00000005 +isOnline: online +name: "Ivan" +phoneNumber: 74951234567 +surname: "Petrov" +color: 255 +color: 0 +color: 0 +visitTime: 1546703100 +temperature: 5 +measureUnits_coef: 1 +measureUnits_coef: 0.01 +measureUnits_coef: 1000 +measureUnits_unit: "meter" +measureUnits_unit: "centimeter" +measureUnits_unit: "kilometer" +someRatio: 0.10000000149011612 +location: 55 +location: 37 +pi: 3.14 +age: "38" +zodiacSign: 1222 +randomBigNumber: 17060000000 +birthDate: 4015 +lotteryWin: 214 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +nestiness_a_b_c_d: 500 +nestiness_a_b_c_e: 501 +nestiness_a_b_c_e: 502 +MESSAGE #2 AT 0x000000CC +name: "Natalia" +surname: "Sokolova" +color: 100 +color: 200 +color: 50 +temperature: 5 +someRatio: 0.0070000002160668373 +location: 50 +location: -4 +gender: female +pi: 3.14159 +age: "26" +zodiacSign: 219 +randomBigNumber: -20000000000000 +birthDate: 8102 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +MESSAGE #3 AT 0x00000153 +isOnline: online +name: "Vasily" +phoneNumber: 442012345678 +surname: "Sidorov" +color: 250 +color: 244 +color: 10 +visitTime: 1546117200 +temperature: -3 +measureUnits_coef: 16 +measureUnits_unit: "pound" +someRatio: 800 +location: 68 +location: 33 +pi: 3.14159274 +age: "23" +zodiacSign: 723 +randomBigNumber: 154400000 +birthDate: 9339 +lotteryWin: 100000000000 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +nestiness_a_b_c_d: 503 + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N 74951234567\0\0 1 2019-01-05 18:45:00 38 capricorn [] [255,0,0] [55,37] 3.140000104904175 214 0.1 5 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] [50,-4] 3.141590118408203 \N 0.007 5 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N 442012345678\0 1 2018-12-30 00:00:00 23 leo [] [250,244,10] [68,33] 3.1415927410125732 100000000000 800 -3 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuflist3_format_persons:StrPerson as ProtobufList + +Binary representation: +00000000 e4 05 0a a6 02 0a 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +00000010 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +00000020 63 35 39 38 65 65 35 35 63 34 39 12 04 49 76 61 |c598ee55c49..Iva| +00000030 6e 1a 06 50 65 74 72 6f 76 22 04 6d 61 6c 65 2a |n..Petrov".male*| +00000040 0a 31 39 38 30 2d 31 32 2d 32 39 3a 0d 2b 37 34 |.1980-12-29:.+74| +00000050 39 35 31 32 33 34 35 36 37 00 42 01 31 4a 13 32 |951234567.B.1J.2| +00000060 30 31 39 2d 30 31 2d 30 35 20 31 38 3a 34 35 3a |019-01-05 18:45:| +00000070 30 30 52 02 33 38 5a 09 63 61 70 72 69 63 6f 72 |00R.38Z.capricor| +00000080 6e 62 09 59 65 73 74 65 72 64 61 79 62 07 46 6c |nb.Yesterdayb.Fl| +00000090 6f 77 65 72 73 6a 03 32 35 35 6a 01 30 6a 01 30 |owersj.255j.0j.0| +000000a0 72 06 4d 6f 73 63 6f 77 7a 09 35 35 2e 37 35 33 |r.Moscowz.55.753| +000000b0 32 31 35 7a 09 33 37 2e 36 32 32 35 30 34 82 01 |215z.37.622504..| +000000c0 04 33 2e 31 34 8a 01 05 32 31 34 2e 31 92 01 03 |.3.14...214.1...| +000000d0 30 2e 31 9a 01 03 35 2e 38 a2 01 0b 31 37 30 36 |0.1...5.8...1706| +000000e0 30 30 30 30 30 30 30 aa 01 2d 0a 05 6d 65 74 65 |0000000..-..mete| +000000f0 72 0a 0a 63 65 6e 74 69 6d 65 74 65 72 0a 09 6b |r..centimeter..k| +00000100 69 6c 6f 6d 65 74 65 72 12 01 31 12 04 30 2e 30 |ilometer..1..0.0| +00000110 31 12 04 31 30 30 30 b2 01 11 0a 0f 0a 03 35 30 |1..1000.......50| +00000120 30 12 03 35 30 31 12 03 35 30 32 0a b4 01 0a 24 |0..501..502....$| +00000130 63 36 39 34 61 64 38 61 2d 66 37 31 34 2d 34 65 |c694ad8a-f714-4e| +00000140 61 33 2d 39 30 37 64 2d 66 64 35 34 66 62 32 35 |a3-907d-fd54fb25| +00000150 64 39 62 35 12 07 4e 61 74 61 6c 69 61 1a 08 53 |d9b5..Natalia..S| +00000160 6f 6b 6f 6c 6f 76 61 22 06 66 65 6d 61 6c 65 2a |okolova".female*| +00000170 0a 31 39 39 32 2d 30 33 2d 30 38 42 01 30 52 02 |.1992-03-08B.0R.| +00000180 32 36 5a 06 70 69 73 63 65 73 6a 03 31 30 30 6a |26Z.piscesj.100j| +00000190 03 32 30 30 6a 02 35 30 72 08 50 6c 79 6d 6f 75 |.200j.50r.Plymou| +000001a0 74 68 7a 09 35 30 2e 34 30 33 37 32 34 7a 09 2d |thz.50.403724z.-| +000001b0 34 2e 31 34 32 31 32 33 82 01 07 33 2e 31 34 31 |4.142123...3.141| +000001c0 35 39 92 01 05 30 2e 30 30 37 9a 01 03 35 2e 34 |59...0.007...5.4| +000001d0 a2 01 0f 2d 32 30 30 30 30 30 30 30 30 30 30 30 |...-200000000000| +000001e0 30 30 0a 81 02 0a 24 61 37 64 61 31 61 61 36 2d |00....$a7da1aa6-| +000001f0 66 34 32 35 2d 34 37 38 39 2d 38 39 34 37 2d 62 |f425-4789-8947-b| +00000200 30 33 34 37 38 36 65 64 33 37 34 12 06 56 61 73 |034786ed374..Vas| +00000210 69 6c 79 1a 07 53 69 64 6f 72 6f 76 22 04 6d 61 |ily..Sidorov".ma| +00000220 6c 65 2a 0a 31 39 39 35 2d 30 37 2d 32 38 3a 0d |le*.1995-07-28:.| +00000230 2b 34 34 32 30 31 32 33 34 35 36 37 38 42 01 31 |+442012345678B.1| +00000240 4a 13 32 30 31 38 2d 31 32 2d 33 30 20 30 30 3a |J.2018-12-30 00:| +00000250 30 30 3a 30 30 52 02 32 33 5a 03 6c 65 6f 62 05 |00:00R.23Z.leob.| +00000260 53 75 6e 6e 79 6a 03 32 35 30 6a 03 32 34 34 6a |Sunnyj.250j.244j| +00000270 02 31 30 72 08 4d 75 72 6d 61 6e 73 6b 7a 09 36 |.10r.Murmanskz.6| +00000280 38 2e 39 37 30 36 38 32 7a 09 33 33 2e 30 37 34 |8.970682z.33.074| +00000290 39 38 31 82 01 10 33 2e 31 34 31 35 39 32 36 35 |981...3.14159265| +000002a0 33 35 38 39 37 39 8a 01 0c 31 30 30 30 30 30 30 |358979...1000000| +000002b0 30 30 30 30 30 92 01 03 38 30 30 9a 01 04 2d 33 |00000...800...-3| +000002c0 2e 32 a2 01 09 31 35 34 34 30 30 30 30 30 aa 01 |.2...154400000..| +000002d0 0b 0a 05 70 6f 75 6e 64 12 02 31 36 b2 01 07 0a |...pound..16....| +000002e0 05 0a 03 35 30 33 |...503| +000002e6 + +MESSAGE #1 AT 0x00000005 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: "male" +birthDate: "1980-12-29" +phoneNumber: "+74951234567\000" +isOnline: "1" +visitTime: "2019-01-05 18:45:00" +age: "38" +zodiacSign: "capricorn" +songs: "Yesterday" +songs: "Flowers" +color: "255" +color: "0" +color: "0" +hometown: "Moscow" +location: "55.753215" +location: "37.622504" +pi: "3.14" +lotteryWin: "214.1" +someRatio: "0.1" +temperature: "5.8" +randomBigNumber: "17060000000" +measureUnits { + unit: "meter" + unit: "centimeter" + unit: "kilometer" + coef: "1" + coef: "0.01" + coef: "1000" +} +nestiness_a { + b_c { + d: "500" + e: "501" + e: "502" + } +} +MESSAGE #2 AT 0x0000012E +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +gender: "female" +birthDate: "1992-03-08" +isOnline: "0" +age: "26" +zodiacSign: "pisces" +color: "100" +color: "200" +color: "50" +hometown: "Plymouth" +location: "50.403724" +location: "-4.142123" +pi: "3.14159" +someRatio: "0.007" +temperature: "5.4" +randomBigNumber: "-20000000000000" +MESSAGE #3 AT 0x000001E5 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: "male" +birthDate: "1995-07-28" +phoneNumber: "+442012345678" +isOnline: "1" +visitTime: "2018-12-30 00:00:00" +age: "23" +zodiacSign: "leo" +songs: "Sunny" +color: "250" +color: "244" +color: "10" +hometown: "Murmansk" +location: "68.970682" +location: "33.074981" +pi: "3.14159265358979" +lotteryWin: "100000000000" +someRatio: "800" +temperature: "-3.2" +randomBigNumber: "154400000" +measureUnits { + unit: "pound" + coef: "16" +} +nestiness_a { + b_c { + d: "503" + } +} + +Binary representation is as expected +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuf_format_syntax2:Syntax2Person + +Binary representation: +00000000 c0 04 0a f1 01 0a 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +00000010 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +00000020 63 35 39 38 65 65 35 35 63 34 39 12 04 49 76 61 |c598ee55c49..Iva| +00000030 6e 1a 06 50 65 74 72 6f 76 20 01 28 af 1f 32 03 |n..Petrov .(..2.| +00000040 70 6e 67 3a 0d 2b 37 34 39 35 31 32 33 34 35 36 |png:.+7495123456| +00000050 37 00 40 01 4d fc d0 30 5c 50 26 58 09 62 09 59 |7.@.M..0\P&X.b.Y| +00000060 65 73 74 65 72 64 61 79 62 07 46 6c 6f 77 65 72 |esterdayb.Flower| +00000070 73 68 ff 01 68 00 68 00 72 06 4d 6f 73 63 6f 77 |sh..h.h.r.Moscow| +00000080 7a 08 4b 03 5f 42 72 7d 16 42 81 01 1f 85 eb 51 |z.K._Br}.B.....Q| +00000090 b8 1e 09 40 89 01 33 33 33 33 33 c3 6a 40 95 01 |...@..33333.j@..| +000000a0 cd cc cc 3d 9d 01 9a 99 b9 40 a0 01 80 c4 d7 8d |...=.....@......| +000000b0 7f ab 01 0d 00 00 80 3f 0d 0a d7 23 3c 0d 00 00 |.......?...#<...| +000000c0 7a 44 12 05 6d 65 74 65 72 12 0a 63 65 6e 74 69 |zD..meter..centi| +000000d0 6d 65 74 65 72 12 09 6b 69 6c 6f 6d 65 74 65 72 |meter..kilometer| +000000e0 ac 01 b3 01 0b a2 06 0b 0b 08 f4 03 10 f5 03 10 |................| +000000f0 f6 03 0c 0c b4 01 0a 83 01 0a 24 63 36 39 34 61 |..........$c694a| +00000100 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d 39 30 |d8a-f714-4ea3-90| +00000110 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 35 12 |7d-fd54fb25d9b5.| +00000120 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f 6c 6f |.Natalia..Sokolo| +00000130 76 61 20 00 28 a6 3f 32 03 6a 70 67 40 00 50 1a |va .(.?2.jpg@.P.| +00000140 58 0b 68 64 68 c8 01 68 32 72 08 50 6c 79 6d 6f |X.hdh..h2r.Plymo| +00000150 75 74 68 7a 08 6a 9d 49 42 46 8c 84 c0 81 01 6e |uthz.j.IBF.....n| +00000160 86 1b f0 f9 21 09 40 95 01 42 60 e5 3b 9d 01 cd |....!.@..B`.;...| +00000170 cc ac 40 a0 01 ff ff a9 ce 93 8c 09 0a c3 01 0a |..@.............| +00000180 24 61 37 64 61 31 61 61 36 2d 66 34 32 35 2d 34 |$a7da1aa6-f425-4| +00000190 37 38 39 2d 38 39 34 37 2d 62 30 33 34 37 38 36 |789-8947-b034786| +000001a0 65 64 33 37 34 12 06 56 61 73 69 6c 79 1a 07 53 |ed374..Vasily..S| +000001b0 69 64 6f 72 6f 76 20 01 28 fb 48 32 03 62 6d 70 |idorov .(.H2.bmp| +000001c0 3a 0d 2b 34 34 32 30 31 32 33 34 35 36 37 38 40 |:.+442012345678@| +000001d0 01 4d 50 e0 27 5c 50 17 58 04 62 05 53 75 6e 6e |.MP.'\P.X.b.Sunn| +000001e0 79 68 fa 01 68 f4 01 68 0a 72 08 4d 75 72 6d 61 |yh..h..h.r.Murma| +000001f0 6e 73 6b 7a 08 fd f0 89 42 c8 4c 04 42 81 01 11 |nskz....B.L.B...| +00000200 2d 44 54 fb 21 09 40 89 01 00 00 00 e8 76 48 37 |-DT.!.@......vH7| +00000210 42 95 01 00 00 48 44 9d 01 cd cc 4c c0 a0 01 80 |B....HD....L....| +00000220 d4 9f 93 01 ab 01 0d 00 00 80 41 12 05 70 6f 75 |..........A..pou| +00000230 6e 64 ac 01 b3 01 0b a2 06 05 0b 08 f7 03 0c 0c |nd..............| +00000240 b4 01 |..| +00000242 + +MESSAGE #1 AT 0x00000005 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: male +birthDate: 4015 +photo: "png" +phoneNumber: "+74951234567\000" +isOnline: true +visitTime: 1546703100 +age: 38 +zodiacSign: capricorn +songs: "Yesterday" +songs: "Flowers" +color: 255 +color: 0 +color: 0 +hometown: "Moscow" +location: 55.7532158 +location: 37.6225052 +pi: 3.14 +lotteryWin: 214.1 +someRatio: 0.1 +temperature: 5.8 +randomBigNumber: 17060000000 +MeasureUnits { + coef: 1 + coef: 0.01 + coef: 1000 + unit: "meter" + unit: "centimeter" + unit: "kilometer" +} +Nestiness { + A { + b { + C { + d: 500 + e: 501 + e: 502 + } + } + } +} +MESSAGE #2 AT 0x000000F9 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +gender: female +birthDate: 8102 +photo: "jpg" +isOnline: false +age: 26 +zodiacSign: pisces +color: 100 +color: 200 +color: 50 +hometown: "Plymouth" +location: 50.4037247 +location: -4.14212322 +pi: 3.14159 +someRatio: 0.007 +temperature: 5.4 +randomBigNumber: -20000000000000 +MESSAGE #3 AT 0x0000017F +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: male +birthDate: 9339 +photo: "bmp" +phoneNumber: "+442012345678" +isOnline: true +visitTime: 1546117200 +age: 23 +zodiacSign: leo +songs: "Sunny" +color: 250 +color: 244 +color: 10 +hometown: "Murmansk" +location: 68.9706802 +location: 33.0749817 +pi: 3.14159265358979 +lotteryWin: 100000000000 +someRatio: 800 +temperature: -3.2 +randomBigNumber: 154400000 +MeasureUnits { + coef: 16 + unit: "pound" +} +Nestiness { + A { + b { + C { + d: 503 + } + } + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.97068,33.074982] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] diff --git a/tests/queries/0_stateless/02240_protobuflist_format_persons.sh b/tests/queries/0_stateless/02240_protobuflist_format_persons.sh new file mode 100755 index 00000000000..dec14b54eb2 --- /dev/null +++ b/tests/queries/0_stateless/02240_protobuflist_format_persons.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# End-to-end test of serialization/deserialization of a table with different +# data types to/from ProtobufList format. +# Cf. 00825_protobuf_format_persons.sh + +# To generate reference file for this test use the following commands: +# ninja ProtobufDelimitedMessagesSerializer +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -eo pipefail + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery < $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist1_format_persons:Person" --input "$BINARY_FILE_PATH" --format "protobuflist" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE roundtrip_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO roundtrip_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist1_format_persons:Person'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM roundtrip_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 02240_protobuflist2_format_persons:AltPerson +echo +echo "Schema 02240_protobuflist2_format_persons:AltPerson" +BINARY_FILE_PATH=$(mktemp "$CURDIR/02240_protobuflist2_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_02240 ORDER BY name FORMAT ProtobufList SETTINGS format_schema = '$SCHEMADIR/02240_protobuflist2_format_persons:AltPerson'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist2_format_persons:AltPerson" --input "$BINARY_FILE_PATH" --format="protobuflist" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alt_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO alt_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist2_format_persons:AltPerson'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM alt_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 02240_protobuflist3_format_persons:StrPerson +echo +echo "Schema 02240_protobuflist3_format_persons:StrPerson as ProtobufList" +BINARY_FILE_PATH=$(mktemp "$CURDIR/02240_protobuflist3_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_02240 ORDER BY name FORMAT ProtobufList SETTINGS format_schema = '$SCHEMADIR/02240_protobuflist3_format_persons:StrPerson'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist3_format_persons:StrPerson" --input "$BINARY_FILE_PATH" --format="protobuflist" +# echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE str_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO str_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist3_format_persons:StrPerson'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM str_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 02240_protobuflist_format_syntax2:Syntax2Person +echo +echo "Schema 02240_protobuf_format_syntax2:Syntax2Person" +BINARY_FILE_PATH=$(mktemp "$CURDIR/02240_protobuflist_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_02240 ORDER BY name FORMAT ProtobufList SETTINGS format_schema = '$SCHEMADIR/02240_protobuflist_format_persons_syntax2:Syntax2Person'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist_format_persons_syntax2:Syntax2Person" --input "$BINARY_FILE_PATH" --format="protobuflist" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE syntax2_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO syntax2_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist_format_persons_syntax2:Syntax2Person'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM syntax2_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +$CLICKHOUSE_CLIENT --multiquery <&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') +FILE_NAME=test_02240.data +DATA_FILE=${USER_FILES_PATH:?}/$FILE_NAME + +touch $DATA_FILE + +echo -e 'a=1\tb=s1\tc=\N +c=[2]\ta=2\tb=\N} + +a=\N +c=[3]\ta=\N' > $DATA_FILE +$CLICKHOUSE_CLIENT --max_read_buffer_size=4 -q "desc file('$FILE_NAME', 'TSKV')" +$CLICKHOUSE_CLIENT --max_read_buffer_size=4 -q "select * from file('$FILE_NAME', 'TSKV')" + diff --git a/tests/queries/0_stateless/02241_array_first_last_or_null.reference b/tests/queries/0_stateless/02241_array_first_last_or_null.reference new file mode 100644 index 00000000000..2906b04ecd0 --- /dev/null +++ b/tests/queries/0_stateless/02241_array_first_last_or_null.reference @@ -0,0 +1,18 @@ +ArrayFirst constant predicate +\N +\N +1 +\N +ArrayFirst non constant predicate +\N +2 +2 +ArrayLast constant predicate +\N +\N +3 +\N +ArrayLast non constant predicate +\N +3 +3 diff --git a/tests/queries/0_stateless/02241_array_first_last_or_null.sql b/tests/queries/0_stateless/02241_array_first_last_or_null.sql new file mode 100644 index 00000000000..3230e4d483a --- /dev/null +++ b/tests/queries/0_stateless/02241_array_first_last_or_null.sql @@ -0,0 +1,21 @@ +SELECT 'ArrayFirst constant predicate'; +SELECT arrayFirstOrNull(x -> 1, emptyArrayUInt8()); +SELECT arrayFirstOrNull(x -> 0, emptyArrayUInt8()); +SELECT arrayFirstOrNull(x -> 1, [1, 2, 3]); +SELECT arrayFirstOrNull(x -> 0, [1, 2, 3]); + +SELECT 'ArrayFirst non constant predicate'; +SELECT arrayFirstOrNull(x -> x >= 2, emptyArrayUInt8()); +SELECT arrayFirstOrNull(x -> x >= 2, [1, 2, 3]); +SELECT arrayFirstOrNull(x -> x >= 2, materialize([1, 2, 3])); + +SELECT 'ArrayLast constant predicate'; +SELECT arrayLastOrNull(x -> 1, emptyArrayUInt8()); +SELECT arrayLastOrNull(x -> 0, emptyArrayUInt8()); +SELECT arrayLastOrNull(x -> 1, [1, 2, 3]); +SELECT arrayLastOrNull(x -> 0, [1, 2, 3]); + +SELECT 'ArrayLast non constant predicate'; +SELECT arrayLastOrNull(x -> x >= 2, emptyArrayUInt8()); +SELECT arrayLastOrNull(x -> x >= 2, [1, 2, 3]); +SELECT arrayLastOrNull(x -> x >= 2, materialize([1, 2, 3])); diff --git a/tests/queries/0_stateless/02241_parquet_bad_column.reference b/tests/queries/0_stateless/02241_parquet_bad_column.reference new file mode 100644 index 00000000000..f599e28b8ab --- /dev/null +++ b/tests/queries/0_stateless/02241_parquet_bad_column.reference @@ -0,0 +1 @@ +10 diff --git a/tests/queries/0_stateless/02241_parquet_bad_column.sh b/tests/queries/0_stateless/02241_parquet_bad_column.sh new file mode 100755 index 00000000000..a160671a088 --- /dev/null +++ b/tests/queries/0_stateless/02241_parquet_bad_column.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists test_02241" +$CLICKHOUSE_CLIENT -q "create table test_02241 (image_path Nullable(String), + caption Nullable(String), + NSFW Nullable(String), + similarity Nullable(Float64), + LICENSE Nullable(String), + url Nullable(String), + key Nullable(UInt64), + shard_id Nullable(UInt64), + status Nullable(String), + width Nullable(UInt32), + height Nullable(UInt32), + exif Nullable(String), + original_width Nullable(UInt32), + original_height Nullable(UInt32)) engine=Memory" + +cat $CUR_DIR/data_parquet_bad_column/metadata_0.parquet | $CLICKHOUSE_CLIENT -q "insert into test_02241 format Parquet" + +$CLICKHOUSE_CLIENT -q "select count() from test_02241" +$CLICKHOUSE_CLIENT -q "drop table test_02241" diff --git a/tests/queries/0_stateless/02241_short_circuit_short_column.reference b/tests/queries/0_stateless/02241_short_circuit_short_column.reference new file mode 100644 index 00000000000..c25c8fb59d0 --- /dev/null +++ b/tests/queries/0_stateless/02241_short_circuit_short_column.reference @@ -0,0 +1,20 @@ +1 \N +1 \N +1 \N +1 \N +1 \N +1 \N +1 \N +1 \N +1 \N +1 \N +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/02241_short_circuit_short_column.sql b/tests/queries/0_stateless/02241_short_circuit_short_column.sql new file mode 100644 index 00000000000..311307fe505 --- /dev/null +++ b/tests/queries/0_stateless/02241_short_circuit_short_column.sql @@ -0,0 +1,2 @@ +SELECT 65536 AND 2147483647, throwIf((((1048575 AND throwIf((0. AND NULL) AND (((65536 AND 257) AND (1.1754943508222875e-38 AND 1024) AND -2147483649) AND NULL) AND (10 AND NULL)) AND (((65536 AND 257) AND (1.1754943508222875e-38 AND 1024) AND -1) AND NULL) AND 65535) AND -1) AND NULL) AND (NULL AND NULL), NULL < number) FROM numbers(10); +SELECT NULL AND throwIf((0 AND NULL) AND 2147483646 AND NULL AND NULL) AND -2147483649 AND (NULL AND NULL) AND NULL FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/02242_if_then_else_null_bug.reference b/tests/queries/0_stateless/02242_if_then_else_null_bug.reference new file mode 100644 index 00000000000..a3bd3b9256e --- /dev/null +++ b/tests/queries/0_stateless/02242_if_then_else_null_bug.reference @@ -0,0 +1,4 @@ +\N +1 +\N +2 diff --git a/tests/queries/0_stateless/02242_if_then_else_null_bug.sql b/tests/queries/0_stateless/02242_if_then_else_null_bug.sql new file mode 100644 index 00000000000..47b0f38d3dc --- /dev/null +++ b/tests/queries/0_stateless/02242_if_then_else_null_bug.sql @@ -0,0 +1,4 @@ +SELECT if(materialize(1) > 0, CAST(NULL, 'Nullable(Int64)'), materialize(toInt32(1))); +SELECT if(materialize(1) > 0, materialize(toInt32(1)), CAST(NULL, 'Nullable(Int64)')); +SELECT if(materialize(1) > 0, CAST(NULL, 'Nullable(Decimal(18, 4))'), materialize(CAST(2, 'Nullable(Decimal(9, 4))'))); +SELECT if(materialize(1) > 0, materialize(CAST(2, 'Nullable(Decimal(9, 4))')), CAST(NULL, 'Nullable(Decimal(18, 4))')); diff --git a/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.reference b/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql b/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql new file mode 100644 index 00000000000..e6e4663c5aa --- /dev/null +++ b/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql @@ -0,0 +1,3 @@ +SET optimize_functions_to_subcolumns = 1; +SELECT count(*) FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 +WHERE (n1.number = n2.number) AND (n2.number = n3.number); diff --git a/tests/queries/0_stateless/02242_subcolumns_sizes.reference b/tests/queries/0_stateless/02242_subcolumns_sizes.reference new file mode 100644 index 00000000000..124b6341116 --- /dev/null +++ b/tests/queries/0_stateless/02242_subcolumns_sizes.reference @@ -0,0 +1,8 @@ +arr size0 UInt64 1 +d k1 String 1 +d k2.k3 Array(String) 1 +d k2.k4 Array(String) 1 +d k2.k5 Array(Int8) 1 +d k2.size0 UInt64 1 +n null UInt8 1 +1 1 1 1 diff --git a/tests/queries/0_stateless/02242_subcolumns_sizes.sql b/tests/queries/0_stateless/02242_subcolumns_sizes.sql new file mode 100644 index 00000000000..8c3d8e69238 --- /dev/null +++ b/tests/queries/0_stateless/02242_subcolumns_sizes.sql @@ -0,0 +1,34 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS t_subcolumns_sizes; + +SET allow_experimental_object_type = 1; + +CREATE TABLE t_subcolumns_sizes (id UInt64, arr Array(UInt64), n Nullable(String), d JSON) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_subcolumns_sizes FORMAT JSONEachRow {"id": 1, "arr": [1, 2, 3], "n": null, "d": {"k1": "v1", "k2": [{"k3": 1, "k4": "v2"}, {"k3": 3}]}} +INSERT INTO t_subcolumns_sizes FORMAT JSONEachRow {"id": 2, "arr": [0], "n": "foo", "d": {"k1": "v3", "k2": [{"k4": "v4"}, {"k3": "v5", "k5": 5}]}} + +OPTIMIZE TABLE t_subcolumns_sizes FINAL; + +SELECT + column, + subcolumns.names AS sname, + subcolumns.types AS stype, + subcolumns.bytes_on_disk > 0 +FROM system.parts_columns ARRAY JOIN subcolumns +WHERE database = currentDatabase() AND table = 't_subcolumns_sizes' AND active +ORDER BY column, sname, stype; + +SELECT + any(column_bytes_on_disk) = sum(subcolumns.bytes_on_disk), + any(column_data_compressed_bytes) = sum(subcolumns.data_compressed_bytes), + any(column_data_uncompressed_bytes) = sum(subcolumns.data_uncompressed_bytes), + any(column_marks_bytes) = sum(subcolumns.marks_bytes) +FROM system.parts_columns ARRAY JOIN subcolumns +WHERE database = currentDatabase() AND table = 't_subcolumns_sizes' +AND active AND column = 'd'; + +DROP TABLE IF EXISTS t_subcolumns_sizes; diff --git a/tests/queries/0_stateless/02242_throw_if_constant_argument.reference b/tests/queries/0_stateless/02242_throw_if_constant_argument.reference new file mode 100644 index 00000000000..4521d575ff3 --- /dev/null +++ b/tests/queries/0_stateless/02242_throw_if_constant_argument.reference @@ -0,0 +1,10 @@ +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/02242_throw_if_constant_argument.sql b/tests/queries/0_stateless/02242_throw_if_constant_argument.sql new file mode 100644 index 00000000000..bdde059ef0f --- /dev/null +++ b/tests/queries/0_stateless/02242_throw_if_constant_argument.sql @@ -0,0 +1 @@ +SELECT throwIf(0 AND 2147483646) FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/02243_in_ip_address.reference b/tests/queries/0_stateless/02243_in_ip_address.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/02243_in_ip_address.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/02243_in_ip_address.sql b/tests/queries/0_stateless/02243_in_ip_address.sql new file mode 100644 index 00000000000..a2c8c37e585 --- /dev/null +++ b/tests/queries/0_stateless/02243_in_ip_address.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table (id UInt64, value_ipv4 IPv4, value_ipv6 IPv6) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0, '127.0.0.1', '127.0.0.1'); + +SELECT id FROM test_table WHERE value_ipv4 IN (SELECT value_ipv4 FROM test_table); +SELECT id FROM test_table WHERE value_ipv6 IN (SELECT value_ipv6 FROM test_table); + +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/data_json/btc_transactions.json b/tests/queries/0_stateless/data_json/btc_transactions.json new file mode 100644 index 00000000000..136f8ea29c1 Binary files /dev/null and b/tests/queries/0_stateless/data_json/btc_transactions.json differ diff --git a/tests/queries/0_stateless/data_json/ghdata_sample.json b/tests/queries/0_stateless/data_json/ghdata_sample.json new file mode 100644 index 00000000000..985b4f135b8 Binary files /dev/null and b/tests/queries/0_stateless/data_json/ghdata_sample.json differ diff --git a/tests/queries/0_stateless/data_json/nbagames_sample.json b/tests/queries/0_stateless/data_json/nbagames_sample.json new file mode 100644 index 00000000000..5082ca059b3 Binary files /dev/null and b/tests/queries/0_stateless/data_json/nbagames_sample.json differ diff --git a/tests/queries/0_stateless/data_orc/test.orc b/tests/queries/0_stateless/data_orc/test.orc deleted file mode 100644 index 1b2c9aa4922..00000000000 Binary files a/tests/queries/0_stateless/data_orc/test.orc and /dev/null differ diff --git a/tests/queries/0_stateless/data_orc/test_setting_input_format_use_lowercase_column_name.orc b/tests/queries/0_stateless/data_orc/test_setting_input_format_use_lowercase_column_name.orc new file mode 100644 index 00000000000..136f9980064 Binary files /dev/null and b/tests/queries/0_stateless/data_orc/test_setting_input_format_use_lowercase_column_name.orc differ diff --git a/tests/queries/0_stateless/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet b/tests/queries/0_stateless/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet new file mode 100644 index 00000000000..922def77caf Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet differ diff --git a/tests/queries/0_stateless/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet.columns b/tests/queries/0_stateless/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet.columns new file mode 100644 index 00000000000..e25da8f923d --- /dev/null +++ b/tests/queries/0_stateless/data_parquet/test_setting_input_format_use_lowercase_column_name.parquet.columns @@ -0,0 +1 @@ +`Id` Nullable(String), `Score` Nullable(Int32) diff --git a/tests/queries/0_stateless/data_parquet_bad_column/metadata_0.parquet b/tests/queries/0_stateless/data_parquet_bad_column/metadata_0.parquet new file mode 100644 index 00000000000..8dd5f1e76f7 Binary files /dev/null and b/tests/queries/0_stateless/data_parquet_bad_column/metadata_0.parquet differ diff --git a/tests/queries/0_stateless/format_schemas/02240_protobuflist1_format_persons.proto b/tests/queries/0_stateless/format_schemas/02240_protobuflist1_format_persons.proto new file mode 100644 index 00000000000..8692136200f --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/02240_protobuflist1_format_persons.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +enum Gender { + female = 0; + male = 1; +}; + +enum ZodiacSign { + aries = 0; + taurus = 1; + gemini = 2; + cancer = 3; + leo = 4; + virgo = 5; + libra = 6; + scorpius = 7; + sagittarius = 8; + capricorn = 9; + aquarius = 10; + pisces = 11; +}; + +message Nestiness +{ + message SubA { + message SubB { + message SubC { + uint32 d = 1; + repeated uint32 e = 2; + }; + SubC c = 1; + }; + SubB b = 100; + }; + SubA a = 1; +}; + +message Envelope { + message Person { + message MeasureUnit + { + string unit = 1; + float coef = 2; + }; + string uuid = 1; + string name = 2; + string surname = 3; + Gender gender = 4; + uint32 birthDate = 5; + bytes photo = 6; + string phoneNumber = 7; + bool isOnline = 8; + fixed32 visitTime = 9; + uint32 age = 10; + ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + string hometown = 14; + repeated float location = 15; + double pi = 16; + double lotteryWin = 17; + float someRatio = 18; + float temperature = 19; + sint64 randomBigNumber = 20; + repeated MeasureUnit measureUnits = 21; + Nestiness nestiness = 22; + }; + repeated Person person = 1; +}; + +// same as wrapped in Envelope, used for verification +message Person { + message MeasureUnit + { + string unit = 1; + float coef = 2; + }; + string uuid = 1; + string name = 2; + string surname = 3; + Gender gender = 4; + uint32 birthDate = 5; + bytes photo = 6; + string phoneNumber = 7; + bool isOnline = 8; + fixed32 visitTime = 9; + uint32 age = 10; + ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + string hometown = 14; + repeated float location = 15; + double pi = 16; + double lotteryWin = 17; + float someRatio = 18; + float temperature = 19; + sint64 randomBigNumber = 20; + repeated MeasureUnit measureUnits = 21; + Nestiness nestiness = 22; +}; diff --git a/tests/queries/0_stateless/format_schemas/02240_protobuflist2_format_persons.proto b/tests/queries/0_stateless/format_schemas/02240_protobuflist2_format_persons.proto new file mode 100644 index 00000000000..c718c3c28e4 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/02240_protobuflist2_format_persons.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +enum OnlineStatus { + offline = 0; + online = 1; +}; + +message Envelope { + message AltPerson { + enum Gender { + male = 0; + female = 1; + }; + message Dummy { + message Empty {}; + repeated Empty empty = 1; + float z = 2; + }; + repeated int32 location = 101 [packed=false]; + float pi = 103; + bytes uuid = 300; + bool newFieldBool = 299; + string name = 2; + Gender gender = 102; + int32 zodiacSign = 130; + int64 birthDate = 150; + bytes age = 111; + OnlineStatus isOnline = 1; + double someRatio = 100; + fixed64 visitTime = 15; + Dummy newMessage = 1000; + sfixed64 randomBigNumber = 140; + repeated int32 newFieldInt = 104; + repeated float color = 14; + uint64 lotteryWin = 202; + bytes surname = 10; + uint64 phoneNumber = 5; + sint32 temperature = 41; + string newFieldStr = 21; + repeated string measureUnits_unit = 99; + repeated float measureUnits_coef = 88; + uint32 nestiness_a_b_c_d = 900; + repeated uint32 nestiness_a_b_c_e = 901; + }; +}; + +// same as wrapped in Envelope, used for verification +message AltPerson { + enum Gender { + male = 0; + female = 1; + }; + message Dummy { + message Empty {}; + repeated Empty empty = 1; + float z = 2; + }; + repeated int32 location = 101 [packed=false]; + float pi = 103; + bytes uuid = 300; + bool newFieldBool = 299; + string name = 2; + Gender gender = 102; + int32 zodiacSign = 130; + int64 birthDate = 150; + bytes age = 111; + OnlineStatus isOnline = 1; + double someRatio = 100; + fixed64 visitTime = 15; + Dummy newMessage = 1000; + sfixed64 randomBigNumber = 140; + repeated int32 newFieldInt = 104; + repeated float color = 14; + uint64 lotteryWin = 202; + bytes surname = 10; + uint64 phoneNumber = 5; + sint32 temperature = 41; + string newFieldStr = 21; + repeated string measureUnits_unit = 99; + repeated float measureUnits_coef = 88; + uint32 nestiness_a_b_c_d = 900; + repeated uint32 nestiness_a_b_c_e = 901; +}; diff --git a/tests/queries/0_stateless/format_schemas/02240_protobuflist3_format_persons.proto b/tests/queries/0_stateless/format_schemas/02240_protobuflist3_format_persons.proto new file mode 100644 index 00000000000..7f0bab36007 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/02240_protobuflist3_format_persons.proto @@ -0,0 +1,78 @@ +syntax = "proto3"; + +message Envelope { + message StrPerson { + message MeasureUnits + { + repeated string unit = 1; + repeated string coef = 2; + }; + message NestinessA + { + message SubBC { + string d = 1; + repeated string e = 2; + }; + SubBC b_c = 1; + }; + string uuid = 1; + string name = 2; + string surname = 3; + string gender = 4; + string birthDate = 5; + string phoneNumber = 7; + string isOnline = 8; + string visitTime = 9; + string age = 10; + string zodiacSign = 11; + repeated string songs = 12; + repeated string color = 13; + string hometown = 14; + repeated string location = 15; + string pi = 16; + string lotteryWin = 17; + string someRatio = 18; + string temperature = 19; + string randomBigNumber = 20; + MeasureUnits measureUnits = 21; + NestinessA nestiness_a = 22; + }; +}; + +// same as wrapped in Envelope, used for verification +message StrPerson { + message MeasureUnits + { + repeated string unit = 1; + repeated string coef = 2; + }; + message NestinessA + { + message SubBC { + string d = 1; + repeated string e = 2; + }; + SubBC b_c = 1; + }; + string uuid = 1; + string name = 2; + string surname = 3; + string gender = 4; + string birthDate = 5; + string phoneNumber = 7; + string isOnline = 8; + string visitTime = 9; + string age = 10; + string zodiacSign = 11; + repeated string songs = 12; + repeated string color = 13; + string hometown = 14; + repeated string location = 15; + string pi = 16; + string lotteryWin = 17; + string someRatio = 18; + string temperature = 19; + string randomBigNumber = 20; + MeasureUnits measureUnits = 21; + NestinessA nestiness_a = 22; +}; diff --git a/tests/queries/0_stateless/format_schemas/02240_protobuflist_format_persons_syntax2.proto b/tests/queries/0_stateless/format_schemas/02240_protobuflist_format_persons_syntax2.proto new file mode 100644 index 00000000000..ff286e46785 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/02240_protobuflist_format_persons_syntax2.proto @@ -0,0 +1,128 @@ +syntax = "proto2"; + +message Envelope { + message Syntax2Person { + enum Gender { + female = 0; + male = 1; + }; + + enum ZodiacSign { + aries = 0; + taurus = 1; + gemini = 2; + cancer = 3; + leo = 4; + virgo = 5; + libra = 6; + scorpius = 7; + sagittarius = 8; + capricorn = 9; + aquarius = 10; + pisces = 11; + }; + + required string uuid = 1; + required string name = 2; + required string surname = 3; + required Gender gender = 4; + required uint32 birthDate = 5; + optional bytes photo = 6; + optional string phoneNumber = 7; + optional bool isOnline = 8; + optional fixed32 visitTime = 9; + optional uint32 age = 10; + optional ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + optional string hometown = 14 [default='Moscow']; + repeated float location = 15 [packed=true]; + optional double pi = 16; + optional double lotteryWin = 17; + optional float someRatio = 18; + optional float temperature = 19; + optional sint64 randomBigNumber = 20; + optional group MeasureUnits = 21 { + repeated float coef = 1; + repeated string unit = 2; + }; + optional group Nestiness = 22 + { + optional group A = 1 { + message SubB { + optional group C = 1 { + optional uint32 d = 1; + repeated uint32 e = 2; + }; + }; + optional SubB b = 100; + }; + }; + optional string newFieldStr = 23 [default='abc']; + optional int32 newFieldInt = 24 [default=-11]; + optional bool newBool = 25 [default=true]; + }; +}; + +// same as wrapped in Envelope, used for verification +message Syntax2Person { + enum Gender { + female = 0; + male = 1; + }; + + enum ZodiacSign { + aries = 0; + taurus = 1; + gemini = 2; + cancer = 3; + leo = 4; + virgo = 5; + libra = 6; + scorpius = 7; + sagittarius = 8; + capricorn = 9; + aquarius = 10; + pisces = 11; + }; + + required string uuid = 1; + required string name = 2; + required string surname = 3; + required Gender gender = 4; + required uint32 birthDate = 5; + optional bytes photo = 6; + optional string phoneNumber = 7; + optional bool isOnline = 8; + optional fixed32 visitTime = 9; + optional uint32 age = 10; + optional ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + optional string hometown = 14 [default='Moscow']; + repeated float location = 15 [packed=true]; + optional double pi = 16; + optional double lotteryWin = 17; + optional float someRatio = 18; + optional float temperature = 19; + optional sint64 randomBigNumber = 20; + optional group MeasureUnits = 21 { + repeated float coef = 1; + repeated string unit = 2; + }; + optional group Nestiness = 22 + { + optional group A = 1 { + message SubB { + optional group C = 1 { + optional uint32 d = 1; + repeated uint32 e = 2; + }; + }; + optional SubB b = 100; + }; + }; + optional string newFieldStr = 23 [default='abc']; + optional int32 newFieldInt = 24 [default=-11]; + optional bool newBool = 25 [default=true]; +}; diff --git a/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py b/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py index 92606c9cb26..b69bf7c8d11 100755 --- a/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py +++ b/tests/queries/0_stateless/helpers/00900_parquet_create_table_columns.py @@ -16,16 +16,19 @@ TYPE_PARQUET_PHYSICAL_TO_CLICKHOUSE = { "FLOAT": "Float32", "DOUBLE": "Float64", "BYTE_ARRAY": "String", - "INT96": "Int64", # TODO! + "INT96": "Int64", # TODO! } + def read_file(filename): with open(filename, "rb") as f: return f.read().decode("raw_unicode_escape") + def get_column_name(column): return column["Name"].split(".", 1)[0] + def resolve_clickhouse_column_type(column): column_name = get_column_name(column) logical_type = column.get("LogicalType", {}) @@ -35,23 +38,46 @@ def resolve_clickhouse_column_type(column): precision = int(logical_type["precision"]) scale = int(logical_type["scale"]) if precision < 1 or precision > 76: - raise RuntimeError("Column {} has invalid Decimal precision {}".format(column_name, precision)) + raise RuntimeError( + "Column {} has invalid Decimal precision {}".format( + column_name, precision + ) + ) if precision > 38: - raise RuntimeError("Column {} has unsupported Decimal precision {}".format(column_name, precision)) + raise RuntimeError( + "Column {} has unsupported Decimal precision {}".format( + column_name, precision + ) + ) if scale < 0 or scale > precision: - raise RuntimeError("Column {} has invalid Decimal scale {} for precision {}".format(column_name, scale, precision)) + raise RuntimeError( + "Column {} has invalid Decimal scale {} for precision {}".format( + column_name, scale, precision + ) + ) return "Decimal({}, {})".format(precision, scale) if converted_type and converted_type != "NONE": result_type = TYPE_PARQUET_CONVERTED_TO_CLICKHOUSE.get(converted_type) if result_type: return result_type - raise RuntimeError("Column {} has unknown ConvertedType: {}".format(column_name, converted_type)) + raise RuntimeError( + "Column {} has unknown ConvertedType: {}".format( + column_name, converted_type + ) + ) if physical_type and physical_type != "NONE": result_type = TYPE_PARQUET_PHYSICAL_TO_CLICKHOUSE.get(physical_type) if result_type: return result_type - raise RuntimeError("Column {} has unknown PhysicalType: {}".format(column_name, physical_type)) - raise RuntimeError("Column {} has invalid types: ConvertedType={}, PhysicalType={}".format(column_name, converted_type, physical_type)) + raise RuntimeError( + "Column {} has unknown PhysicalType: {}".format(column_name, physical_type) + ) + raise RuntimeError( + "Column {} has invalid types: ConvertedType={}, PhysicalType={}".format( + column_name, converted_type, physical_type + ) + ) + def dump_columns(obj): descr_by_column_name = {} @@ -78,11 +104,22 @@ def dump_columns(obj): else: return "Tuple({})".format(", ".join(types)) - print(", ".join(map(lambda descr: "`{}` {}".format(descr["name"], _format_type(descr["types"])), columns_descr))) + print( + ", ".join( + map( + lambda descr: "`{}` {}".format( + descr["name"], _format_type(descr["types"]) + ), + columns_descr, + ) + ) + ) + def dump_columns_from_file(filename): dump_columns(json.loads(read_file(filename), strict=False)) + if __name__ == "__main__": filename = sys.argv[1] dump_columns_from_file(filename) diff --git a/tests/queries/0_stateless/helpers/client.py b/tests/queries/0_stateless/helpers/client.py index 086d920d0b7..5c8589dfca1 100644 --- a/tests/queries/0_stateless/helpers/client.py +++ b/tests/queries/0_stateless/helpers/client.py @@ -8,29 +8,30 @@ sys.path.insert(0, os.path.join(CURDIR)) import uexpect -prompt = ':\) ' -end_of_block = r'.*\r\n.*\r\n' +prompt = ":\) " +end_of_block = r".*\r\n.*\r\n" + class client(object): - def __init__(self, command=None, name='', log=None): - self.client = uexpect.spawn(['/bin/bash','--noediting']) + def __init__(self, command=None, name="", log=None): + self.client = uexpect.spawn(["/bin/bash", "--noediting"]) if command is None: - command = os.environ.get('CLICKHOUSE_BINARY', 'clickhouse') + ' client' + command = os.environ.get("CLICKHOUSE_BINARY", "clickhouse") + " client" self.client.command = command - self.client.eol('\r') + self.client.eol("\r") self.client.logger(log, prefix=name) self.client.timeout(120) - self.client.expect('[#\$] ', timeout=60) + self.client.expect("[#\$] ", timeout=60) self.client.send(command) def __enter__(self): return self.client.__enter__() def __exit__(self, type, value, traceback): - self.client.reader['kill_event'].set() + self.client.reader["kill_event"].set() # send Ctrl-C - self.client.send('\x03', eol='') + self.client.send("\x03", eol="") time.sleep(0.3) - self.client.send('quit', eol='\r') - self.client.send('\x03', eol='') + self.client.send("quit", eol="\r") + self.client.send("\x03", eol="") return self.client.__exit__(type, value, traceback) diff --git a/tests/queries/0_stateless/helpers/httpclient.py b/tests/queries/0_stateless/helpers/httpclient.py index adbfbc7d287..00e6a8d164d 100644 --- a/tests/queries/0_stateless/helpers/httpclient.py +++ b/tests/queries/0_stateless/helpers/httpclient.py @@ -7,8 +7,11 @@ sys.path.insert(0, os.path.join(CURDIR)) import httpexpect -def client(request, name='', log=None): - client = httpexpect.spawn({'host':'localhost','port':8123,'timeout':30}, request) + +def client(request, name="", log=None): + client = httpexpect.spawn( + {"host": "localhost", "port": 8123, "timeout": 30}, request + ) client.logger(log, prefix=name) client.timeout(20) return client diff --git a/tests/queries/0_stateless/helpers/httpexpect.py b/tests/queries/0_stateless/helpers/httpexpect.py index 788e57499a8..6147118e793 100644 --- a/tests/queries/0_stateless/helpers/httpexpect.py +++ b/tests/queries/0_stateless/helpers/httpexpect.py @@ -23,6 +23,7 @@ import uexpect from threading import Thread, Event from queue import Queue, Empty + class IO(uexpect.IO): def __init__(self, connection, response, queue, reader): self.connection = connection @@ -33,10 +34,10 @@ class IO(uexpect.IO): raise NotImplementedError def close(self, force=True): - self.reader['kill_event'].set() + self.reader["kill_event"].set() self.connection.close() if self._logger: - self._logger.write('\n') + self._logger.write("\n") self._logger.flush() @@ -52,6 +53,7 @@ def reader(response, queue, kill_event): break raise + def spawn(connection, request): connection = http.client.HTTPConnection(**connection) connection.request(**request) @@ -63,11 +65,20 @@ def spawn(connection, request): thread.daemon = True thread.start() - return IO(connection, response, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) + return IO( + connection, + response, + queue, + reader={"thread": thread, "kill_event": reader_kill_event}, + ) -if __name__ == '__main__': - with spawn({'host':'localhost','port':8123},{'method':'GET', 'url':'?query=SELECT%201'}) as client: + +if __name__ == "__main__": + with spawn( + {"host": "localhost", "port": 8123}, + {"method": "GET", "url": "?query=SELECT%201"}, + ) as client: client.logger(sys.stdout) client.timeout(2) print(client.response.status, client.response.reason) - client.expect('1\n') + client.expect("1\n") diff --git a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py index 893180d6cc1..4a3f4613b6f 100755 --- a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py +++ b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py @@ -5,11 +5,13 @@ import argparse import os.path +import io import struct import subprocess import sys import tempfile + def read_varint(input): res = 0 multiplier = 1 @@ -26,29 +28,36 @@ def read_varint(input): multiplier *= 0x80 return res + def write_varint(output, value): while True: if value < 0x80: b = value - output.write(b.to_bytes(1, byteorder='little')) + output.write(b.to_bytes(1, byteorder="little")) break b = (value & 0x7F) + 0x80 - output.write(b.to_bytes(1, byteorder='little')) + output.write(b.to_bytes(1, byteorder="little")) value = value >> 7 + def write_hexdump(output, data): - with subprocess.Popen(["hexdump", "-C"], stdin=subprocess.PIPE, stdout=output, shell=False) as proc: + with subprocess.Popen( + ["hexdump", "-C"], stdin=subprocess.PIPE, stdout=output, shell=False + ) as proc: proc.communicate(data) if proc.returncode != 0: raise RuntimeError("hexdump returned code " + str(proc.returncode)) output.flush() + class FormatSchemaSplitted: def __init__(self, format_schema): self.format_schema = format_schema - splitted = self.format_schema.split(':') + splitted = self.format_schema.split(":") if len(splitted) < 2: - raise RuntimeError('The format schema must have the format "schemafile:MessageType"') + raise RuntimeError( + 'The format schema must have the format "schemafile:MessageType"' + ) path = splitted[0] self.schemadir = os.path.dirname(path) self.schemaname = os.path.basename(path) @@ -56,42 +65,58 @@ class FormatSchemaSplitted: self.schemaname = self.schemaname + ".proto" self.message_type = splitted[1] -def decode(input, output, format_schema): + +def decode(input, output, format_schema, format): if not type(format_schema) is FormatSchemaSplitted: format_schema = FormatSchemaSplitted(format_schema) msgindex = 1 + if format == "protobuflist": + read_varint(input) # envelope msg size while True: + if format == "protobuflist": + read_varint(input) # wiretype and field id of nested msg sz = read_varint(input) if sz is None: break - output.write("MESSAGE #{msgindex} AT 0x{msgoffset:08X}\n".format(msgindex=msgindex, msgoffset=input.tell()).encode()) + output.write( + "MESSAGE #{msgindex} AT 0x{msgoffset:08X}\n".format( + msgindex=msgindex, msgoffset=input.tell() + ).encode() + ) output.flush() msg = input.read(sz) if len(msg) < sz: - raise EOFError('Unexpected end of file') - protoc = os.getenv('PROTOC_BINARY', 'protoc') - with subprocess.Popen([protoc, - "--decode", format_schema.message_type, format_schema.schemaname], - cwd=format_schema.schemadir, - stdin=subprocess.PIPE, - stdout=output, - shell=False) as proc: + raise EOFError("Unexpected end of file") + protoc = os.getenv("PROTOC_BINARY", "protoc") + with subprocess.Popen( + [protoc, "--decode", format_schema.message_type, format_schema.schemaname], + cwd=format_schema.schemadir, + stdin=subprocess.PIPE, + stdout=output, + shell=False, + ) as proc: proc.communicate(msg) if proc.returncode != 0: raise RuntimeError("protoc returned code " + str(proc.returncode)) output.flush() msgindex = msgindex + 1 -def encode(input, output, format_schema): + +def encode(input, output, format_schema, format): if not type(format_schema) is FormatSchemaSplitted: format_schema = FormatSchemaSplitted(format_schema) line_offset = input.tell() line = input.readline() + buf = io.BytesIO() while True: if len(line) == 0: break if not line.startswith(b"MESSAGE #"): - raise RuntimeError("The line at 0x{line_offset:08X} must start with the text 'MESSAGE #'".format(line_offset=line_offset)) + raise RuntimeError( + "The line at 0x{line_offset:08X} must start with the text 'MESSAGE #'".format( + line_offset=line_offset + ) + ) msg = b"" while True: line_offset = input.tell() @@ -99,21 +124,30 @@ def encode(input, output, format_schema): if line.startswith(b"MESSAGE #") or len(line) == 0: break msg += line - protoc = os.getenv('PROTOC_BINARY', 'protoc') - with subprocess.Popen([protoc, - "--encode", format_schema.message_type, format_schema.schemaname], - cwd=format_schema.schemadir, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - shell=False) as proc: + protoc = os.getenv("PROTOC_BINARY", "protoc") + with subprocess.Popen( + [protoc, "--encode", format_schema.message_type, format_schema.schemaname], + cwd=format_schema.schemadir, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + shell=False, + ) as proc: msgbin = proc.communicate(msg)[0] if proc.returncode != 0: raise RuntimeError("protoc returned code " + str(proc.returncode)) - write_varint(output, len(msgbin)) - output.write(msgbin) - output.flush() + if format == "protobuflist": + field_number = 1 + wire_type = 2 # length-delimited + write_varint(buf, (field_number << 3) | wire_type) + write_varint(buf, len(msgbin)) + buf.write(msgbin) + if format == "protobuflist": + write_varint(output, len(buf.getvalue())) + output.write(buf.getvalue()) + output.flush() -def decode_and_check(input, output, format_schema): + +def decode_and_check(input, output, format_schema, format): input_data = input.read() output.write(b"Binary representation:\n") output.flush() @@ -125,13 +159,13 @@ def decode_and_check(input, output, format_schema): tmp_input.write(input_data) tmp_input.flush() tmp_input.seek(0) - decode(tmp_input, tmp_decoded, format_schema) + decode(tmp_input, tmp_decoded, format_schema, format) tmp_decoded.seek(0) decoded_text = tmp_decoded.read() output.write(decoded_text) output.flush() tmp_decoded.seek(0) - encode(tmp_decoded, tmp_encoded, format_schema) + encode(tmp_decoded, tmp_encoded, format_schema, format) tmp_encoded.seek(0) encoded_data = tmp_encoded.read() @@ -139,23 +173,56 @@ def decode_and_check(input, output, format_schema): output.write(b"\nBinary representation is as expected\n") output.flush() else: - output.write(b"\nBinary representation differs from the expected one (listed below):\n") + output.write( + b"\nBinary representation differs from the expected one (listed below):\n" + ) output.flush() write_hexdump(output, encoded_data) sys.exit(1) + if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Encodes or decodes length-delimited protobuf messages.') - parser.add_argument('--input', help='The input file, the standard input will be used if not specified.') - parser.add_argument('--output', help='The output file, the standard output will be used if not specified') - parser.add_argument('--format_schema', required=True, help='Format schema in the format "schemafile:MessageType"') + parser = argparse.ArgumentParser( + description="Encodes or decodes length-delimited protobuf messages." + ) + parser.add_argument( + "--input", + help="The input file, the standard input will be used if not specified.", + ) + parser.add_argument( + "--output", + help="The output file, the standard output will be used if not specified", + ) + parser.add_argument( + "--format_schema", + required=True, + help='Format schema in the format "schemafile:MessageType"', + ) + parser.add_argument( + "--format", + choices=["protobuf", "protobuflist"], + default="protobuf", + help='The input/output format, "protobuf" if not specified', + ) group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--encode', action='store_true', help='Specify to encode length-delimited messages.' - 'The utility will read text-format messages of the given type from the input and write it in binary to the output.') - group.add_argument('--decode', action='store_true', help='Specify to decode length-delimited messages.' - 'The utility will read messages in binary from the input and write text-format messages to the output.') - group.add_argument('--decode_and_check', action='store_true', help='The same as --decode, and the utility will then encode ' - ' the decoded data back to the binary form to check that the result of that encoding is the same as the input was.') + group.add_argument( + "--encode", + action="store_true", + help="Specify to encode length-delimited messages." + "The utility will read text-format messages of the given type from the input and write it in binary to the output.", + ) + group.add_argument( + "--decode", + action="store_true", + help="Specify to decode length-delimited messages." + "The utility will read messages in binary from the input and write text-format messages to the output.", + ) + group.add_argument( + "--decode_and_check", + action="store_true", + help="The same as --decode, and the utility will then encode " + " the decoded data back to the binary form to check that the result of that encoding is the same as the input was.", + ) args = parser.parse_args() custom_input_file = None @@ -169,11 +236,11 @@ if __name__ == "__main__": output = custom_output_file if custom_output_file else sys.stdout.buffer if args.encode: - encode(input, output, args.format_schema) + encode(input, output, args.format_schema, args.format) elif args.decode: - decode(input, output, args.format_schema) + decode(input, output, args.format_schema, args.format) elif args.decode_and_check: - decode_and_check(input, output, args.format_schema) + decode_and_check(input, output, args.format_schema, args.format) finally: if custom_input_file: diff --git a/tests/queries/0_stateless/helpers/pure_http_client.py b/tests/queries/0_stateless/helpers/pure_http_client.py index 3335f141bb5..0e7a4d27f4f 100644 --- a/tests/queries/0_stateless/helpers/pure_http_client.py +++ b/tests/queries/0_stateless/helpers/pure_http_client.py @@ -5,64 +5,75 @@ import requests import time import pandas as pd -CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1') -CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123') -CLICKHOUSE_SERVER_URL_STR = 'http://' + ':'.join(str(s) for s in [CLICKHOUSE_HOST, CLICKHOUSE_PORT_HTTP]) + "/" -CLICKHOUSE_DATABASE = os.environ.get('CLICKHOUSE_DATABASE', 'test') +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT_HTTP = os.environ.get("CLICKHOUSE_PORT_HTTP", "8123") +CLICKHOUSE_SERVER_URL_STR = ( + "http://" + ":".join(str(s) for s in [CLICKHOUSE_HOST, CLICKHOUSE_PORT_HTTP]) + "/" +) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "test") + class ClickHouseClient: - def __init__(self, host = CLICKHOUSE_SERVER_URL_STR): + def __init__(self, host=CLICKHOUSE_SERVER_URL_STR): self.host = host - def query(self, query, connection_timeout=1500, settings=dict(), binary_result=False): + def query( + self, query, connection_timeout=1500, settings=dict(), binary_result=False + ): NUMBER_OF_TRIES = 30 DELAY = 10 params = { - 'timeout_before_checking_execution_speed': 120, - 'max_execution_time': 6000, - 'database': CLICKHOUSE_DATABASE, + "timeout_before_checking_execution_speed": 120, + "max_execution_time": 6000, + "database": CLICKHOUSE_DATABASE, } # Add extra settings to params params = {**params, **settings} for i in range(NUMBER_OF_TRIES): - r = requests.post(self.host, params=params, timeout=connection_timeout, data=query) + r = requests.post( + self.host, params=params, timeout=connection_timeout, data=query + ) if r.status_code == 200: return r.content if binary_result else r.text else: - print('ATTENTION: try #%d failed' % i) - if i != (NUMBER_OF_TRIES-1): + print("ATTENTION: try #%d failed" % i) + if i != (NUMBER_OF_TRIES - 1): print(query) print(r.text) - time.sleep(DELAY*(i+1)) + time.sleep(DELAY * (i + 1)) else: raise ValueError(r.text) - def query_return_df(self, query, connection_timeout = 1500): + def query_return_df(self, query, connection_timeout=1500): data = self.query(query, connection_timeout) - df = pd.read_csv(io.StringIO(data), sep = '\t') + df = pd.read_csv(io.StringIO(data), sep="\t") return df def query_with_data(self, query, data, connection_timeout=1500, settings=dict()): params = { - 'query': query, - 'timeout_before_checking_execution_speed': 120, - 'max_execution_time': 6000, - 'database': CLICKHOUSE_DATABASE, + "query": query, + "timeout_before_checking_execution_speed": 120, + "max_execution_time": 6000, + "database": CLICKHOUSE_DATABASE, } - headers = { - "Content-Type": "application/binary" - } + headers = {"Content-Type": "application/binary"} # Add extra settings to params params = {**params, **settings} - r = requests.post(self.host, params=params, timeout=connection_timeout, data=data, headers=headers) + r = requests.post( + self.host, + params=params, + timeout=connection_timeout, + data=data, + headers=headers, + ) result = r.text if r.status_code == 200: return result else: - raise ValueError(r.text) \ No newline at end of file + raise ValueError(r.text) diff --git a/tests/queries/0_stateless/helpers/shell.py b/tests/queries/0_stateless/helpers/shell.py index 5c327a55d94..befb3dcd543 100644 --- a/tests/queries/0_stateless/helpers/shell.py +++ b/tests/queries/0_stateless/helpers/shell.py @@ -8,13 +8,14 @@ sys.path.insert(0, os.path.join(CURDIR)) import uexpect + class shell(object): - def __init__(self, command=None, name='', log=None, prompt='[#\$] '): + def __init__(self, command=None, name="", log=None, prompt="[#\$] "): if command is None: - command = ['/bin/bash', '--noediting'] + command = ["/bin/bash", "--noediting"] self.prompt = prompt self.client = uexpect.spawn(command) - self.client.eol('\r') + self.client.eol("\r") self.client.logger(log, prefix=name) self.client.timeout(20) self.client.expect(prompt, timeout=60) @@ -25,10 +26,10 @@ class shell(object): return io def __exit__(self, type, value, traceback): - self.client.reader['kill_event'].set() + self.client.reader["kill_event"].set() # send Ctrl-C - self.client.send('\x03', eol='') + self.client.send("\x03", eol="") time.sleep(0.3) - self.client.send('exit', eol='\r') - self.client.send('\x03', eol='') + self.client.send("exit", eol="\r") + self.client.send("\x03", eol="") return self.client.__exit__(type, value, traceback) diff --git a/tests/queries/0_stateless/helpers/uexpect.py b/tests/queries/0_stateless/helpers/uexpect.py index 7a633facc95..2e6d8aed19e 100644 --- a/tests/queries/0_stateless/helpers/uexpect.py +++ b/tests/queries/0_stateless/helpers/uexpect.py @@ -21,12 +21,14 @@ from threading import Thread, Event from subprocess import Popen from queue import Queue, Empty + class TimeoutError(Exception): def __init__(self, timeout): self.timeout = timeout def __str__(self): - return 'Timeout %.3fs' % float(self.timeout) + return "Timeout %.3fs" % float(self.timeout) + class ExpectTimeoutError(Exception): def __init__(self, pattern, timeout, buffer): @@ -35,14 +37,15 @@ class ExpectTimeoutError(Exception): self.buffer = buffer def __str__(self): - s = 'Timeout %.3fs ' % float(self.timeout) + s = "Timeout %.3fs " % float(self.timeout) if self.pattern: - s += 'for %s ' % repr(self.pattern.pattern) + s += "for %s " % repr(self.pattern.pattern) if self.buffer: - s += 'buffer %s' % repr(self.buffer[:]) - #s += ' or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) + s += "buffer %s" % repr(self.buffer[:]) + # s += ' or \'%s\'' % ','.join(['%x' % ord(c) for c in self.buffer[:]]) return s + class IO(object): class EOF(object): pass @@ -54,12 +57,12 @@ class IO(object): TIMEOUT = Timeout class Logger(object): - def __init__(self, logger, prefix=''): + def __init__(self, logger, prefix=""): self._logger = logger self._prefix = prefix def write(self, data): - self._logger.write(('\n' + data).replace('\n','\n' + self._prefix)) + self._logger.write(("\n" + data).replace("\n", "\n" + self._prefix)) def flush(self): self._logger.flush() @@ -76,7 +79,7 @@ class IO(object): self.reader = reader self._timeout = None self._logger = None - self._eol = '' + self._eol = "" def __enter__(self): return self @@ -84,7 +87,7 @@ class IO(object): def __exit__(self, type, value, traceback): self.close() - def logger(self, logger=None, prefix=''): + def logger(self, logger=None, prefix=""): if logger: self._logger = self.Logger(logger, prefix=prefix) return self._logger @@ -100,15 +103,15 @@ class IO(object): return self._eol def close(self, force=True): - self.reader['kill_event'].set() - os.system('pkill -TERM -P %d' % self.process.pid) + self.reader["kill_event"].set() + os.system("pkill -TERM -P %d" % self.process.pid) if force: self.process.kill() else: self.process.terminate() os.close(self.master) if self._logger: - self._logger.write('\n') + self._logger.write("\n") self._logger.flush() def send(self, data, eol=None): @@ -134,9 +137,9 @@ class IO(object): if self.buffer is not None: self.match = pattern.search(self.buffer, 0) if self.match is not None: - self.after = self.buffer[self.match.start():self.match.end()] - self.before = self.buffer[:self.match.start()] - self.buffer = self.buffer[self.match.end():] + self.after = self.buffer[self.match.start() : self.match.end()] + self.before = self.buffer[: self.match.start()] + self.buffer = self.buffer[self.match.end() :] break if timeleft < 0: break @@ -144,16 +147,16 @@ class IO(object): data = self.read(timeout=timeleft, raise_exception=True) except TimeoutError: if self._logger: - self._logger.write((self.buffer or '') + '\n') + self._logger.write((self.buffer or "") + "\n") self._logger.flush() exception = ExpectTimeoutError(pattern, timeout, self.buffer) self.buffer = None raise exception - timeleft -= (time.time() - start_time) + timeleft -= time.time() - start_time if data: self.buffer = (self.buffer + data) if self.buffer else data if self._logger: - self._logger.write((self.before or '') + (self.after or '')) + self._logger.write((self.before or "") + (self.after or "")) self._logger.flush() if self.match is None: exception = ExpectTimeoutError(pattern, timeout, self.buffer) @@ -162,15 +165,15 @@ class IO(object): return self.match def read(self, timeout=0, raise_exception=False): - data = '' + data = "" timeleft = timeout try: - while timeleft >= 0 : + while timeleft >= 0: start_time = time.time() data += self.queue.get(timeout=timeleft) if data: break - timeleft -= (time.time() - start_time) + timeleft -= time.time() - start_time except Empty: if data: return data @@ -182,9 +185,17 @@ class IO(object): return data + def spawn(command): master, slave = pty.openpty() - process = Popen(command, preexec_fn=os.setsid, stdout=slave, stdin=slave, stderr=slave, bufsize=1) + process = Popen( + command, + preexec_fn=os.setsid, + stdout=slave, + stdin=slave, + stderr=slave, + bufsize=1, + ) os.close(slave) queue = Queue() @@ -193,12 +204,18 @@ def spawn(command): thread.daemon = True thread.start() - return IO(process, master, queue, reader={'thread':thread, 'kill_event':reader_kill_event}) + return IO( + process, + master, + queue, + reader={"thread": thread, "kill_event": reader_kill_event}, + ) + def reader(process, out, queue, kill_event): while True: try: - data = os.read(out, 65536).decode(errors='replace') + data = os.read(out, 65536).decode(errors="replace") queue.put(data) except: if kill_event.is_set(): diff --git a/tests/queries/1_stateful/00011_sorting.sql b/tests/queries/1_stateful/00011_sorting.sql index 381be7b7dd4..3e451360e1b 100644 --- a/tests/queries/1_stateful/00011_sorting.sql +++ b/tests/queries/1_stateful/00011_sorting.sql @@ -1 +1 @@ -SELECT EventTime::DateTime('Europe/Moscow') FROM test.hits ORDER BY EventTime DESC LIMIT 10 +SELECT EventTime::DateTime('Asia/Dubai') FROM test.hits ORDER BY EventTime DESC LIMIT 10 diff --git a/tests/queries/1_stateful/00012_sorting_distributed.sql b/tests/queries/1_stateful/00012_sorting_distributed.sql index c71f643045d..2f852af1dba 100644 --- a/tests/queries/1_stateful/00012_sorting_distributed.sql +++ b/tests/queries/1_stateful/00012_sorting_distributed.sql @@ -1,3 +1,3 @@ -- Tags: distributed -SELECT EventTime::DateTime('Europe/Moscow') FROM remote('127.0.0.{1,2}', test, hits) ORDER BY EventTime DESC LIMIT 10 +SELECT EventTime::DateTime('Asia/Dubai') FROM remote('127.0.0.{1,2}', test, hits) ORDER BY EventTime DESC LIMIT 10 diff --git a/tests/queries/1_stateful/00066_sorting_distributed_many_replicas.sql b/tests/queries/1_stateful/00066_sorting_distributed_many_replicas.sql index 3e34d9d1348..63a833af114 100644 --- a/tests/queries/1_stateful/00066_sorting_distributed_many_replicas.sql +++ b/tests/queries/1_stateful/00066_sorting_distributed_many_replicas.sql @@ -1,4 +1,4 @@ -- Tags: replica, distributed SET max_parallel_replicas = 2; -SELECT EventTime::DateTime('Europe/Moscow') FROM remote('127.0.0.{1|2}', test, hits) ORDER BY EventTime DESC LIMIT 10 +SELECT EventTime::DateTime('Asia/Dubai') FROM remote('127.0.0.{1|2}', test, hits) ORDER BY EventTime DESC LIMIT 10 diff --git a/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql b/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql index 241f0f9b13b..16c0097bf21 100644 --- a/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql +++ b/tests/queries/1_stateful/00071_merge_tree_optimize_aio.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test.hits_snippet; -CREATE TABLE test.hits_snippet(EventTime DateTime('Europe/Moscow'), EventDate Date, CounterID UInt32, UserID UInt64, URL String, Referer String) ENGINE = MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); +CREATE TABLE test.hits_snippet(EventTime DateTime('Asia/Dubai'), EventDate Date, CounterID UInt32, UserID UInt64, URL String, Referer String) ENGINE = MergeTree(EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); SET min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; SET max_block_size = 4096; diff --git a/tests/queries/1_stateful/00072_compare_date_and_string_index.sql b/tests/queries/1_stateful/00072_compare_date_and_string_index.sql index af5d932fecb..d652b1bc559 100644 --- a/tests/queries/1_stateful/00072_compare_date_and_string_index.sql +++ b/tests/queries/1_stateful/00072_compare_date_and_string_index.sql @@ -15,7 +15,7 @@ SELECT count() FROM test.hits WHERE EventDate IN (toDate('2014-03-18'), toDate(' SELECT count() FROM test.hits WHERE EventDate = concat('2014-0', '3-18'); DROP TABLE IF EXISTS test.hits_indexed_by_time; -CREATE TABLE test.hits_indexed_by_time (EventDate Date, EventTime DateTime('Europe/Moscow')) ENGINE = MergeTree ORDER BY (EventDate, EventTime); +CREATE TABLE test.hits_indexed_by_time (EventDate Date, EventTime DateTime('Asia/Dubai')) ENGINE = MergeTree ORDER BY (EventDate, EventTime); INSERT INTO test.hits_indexed_by_time SELECT EventDate, EventTime FROM test.hits; SELECT count() FROM test.hits_indexed_by_time WHERE EventTime = '2014-03-18 01:02:03'; @@ -25,12 +25,12 @@ SELECT count() FROM test.hits_indexed_by_time WHERE EventTime <= '2014-03-18 01: SELECT count() FROM test.hits_indexed_by_time WHERE EventTime >= '2014-03-18 01:02:03'; SELECT count() FROM test.hits_indexed_by_time WHERE EventTime IN ('2014-03-18 01:02:03', '2014-03-19 04:05:06'); -SELECT count() FROM test.hits_indexed_by_time WHERE EventTime = toDateTime('2014-03-18 01:02:03', 'Europe/Moscow'); -SELECT count() FROM test.hits_indexed_by_time WHERE EventTime < toDateTime('2014-03-18 01:02:03', 'Europe/Moscow'); -SELECT count() FROM test.hits_indexed_by_time WHERE EventTime > toDateTime('2014-03-18 01:02:03', 'Europe/Moscow'); -SELECT count() FROM test.hits_indexed_by_time WHERE EventTime <= toDateTime('2014-03-18 01:02:03', 'Europe/Moscow'); -SELECT count() FROM test.hits_indexed_by_time WHERE EventTime >= toDateTime('2014-03-18 01:02:03', 'Europe/Moscow'); -SELECT count() FROM test.hits_indexed_by_time WHERE EventTime IN (toDateTime('2014-03-18 01:02:03', 'Europe/Moscow'), toDateTime('2014-03-19 04:05:06', 'Europe/Moscow')); +SELECT count() FROM test.hits_indexed_by_time WHERE EventTime = toDateTime('2014-03-18 01:02:03', 'Asia/Dubai'); +SELECT count() FROM test.hits_indexed_by_time WHERE EventTime < toDateTime('2014-03-18 01:02:03', 'Asia/Dubai'); +SELECT count() FROM test.hits_indexed_by_time WHERE EventTime > toDateTime('2014-03-18 01:02:03', 'Asia/Dubai'); +SELECT count() FROM test.hits_indexed_by_time WHERE EventTime <= toDateTime('2014-03-18 01:02:03', 'Asia/Dubai'); +SELECT count() FROM test.hits_indexed_by_time WHERE EventTime >= toDateTime('2014-03-18 01:02:03', 'Asia/Dubai'); +SELECT count() FROM test.hits_indexed_by_time WHERE EventTime IN (toDateTime('2014-03-18 01:02:03', 'Asia/Dubai'), toDateTime('2014-03-19 04:05:06', 'Asia/Dubai')); SELECT count() FROM test.hits_indexed_by_time WHERE EventTime = concat('2014-03-18 ', '01:02:03'); diff --git a/tests/queries/1_stateful/00075_left_array_join.sql b/tests/queries/1_stateful/00075_left_array_join.sql index 52a48462b9d..1fd045a26bf 100644 --- a/tests/queries/1_stateful/00075_left_array_join.sql +++ b/tests/queries/1_stateful/00075_left_array_join.sql @@ -1,2 +1,2 @@ -SELECT UserID, EventTime::DateTime('Europe/Moscow'), pp.Key1, pp.Key2, ParsedParams.Key1 FROM test.hits ARRAY JOIN ParsedParams AS pp WHERE CounterID = 1704509 ORDER BY UserID, EventTime, pp.Key1, pp.Key2 LIMIT 100; -SELECT UserID, EventTime::DateTime('Europe/Moscow'), pp.Key1, pp.Key2, ParsedParams.Key1 FROM test.hits LEFT ARRAY JOIN ParsedParams AS pp WHERE CounterID = 1704509 ORDER BY UserID, EventTime, pp.Key1, pp.Key2 LIMIT 100; +SELECT UserID, EventTime::DateTime('Asia/Dubai'), pp.Key1, pp.Key2, ParsedParams.Key1 FROM test.hits ARRAY JOIN ParsedParams AS pp WHERE CounterID = 1704509 ORDER BY UserID, EventTime, pp.Key1, pp.Key2 LIMIT 100; +SELECT UserID, EventTime::DateTime('Asia/Dubai'), pp.Key1, pp.Key2, ParsedParams.Key1 FROM test.hits LEFT ARRAY JOIN ParsedParams AS pp WHERE CounterID = 1704509 ORDER BY UserID, EventTime, pp.Key1, pp.Key2 LIMIT 100; diff --git a/tests/queries/1_stateful/00084_external_aggregation.sql b/tests/queries/1_stateful/00084_external_aggregation.sql index b3922eae049..816d95f4b8b 100644 --- a/tests/queries/1_stateful/00084_external_aggregation.sql +++ b/tests/queries/1_stateful/00084_external_aggregation.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-settings + SET max_bytes_before_external_group_by = 200000000; SET max_memory_usage = 1500000000; diff --git a/tests/queries/1_stateful/00091_prewhere_two_conditions.sql b/tests/queries/1_stateful/00091_prewhere_two_conditions.sql index c5952be83b6..1e476d3a27d 100644 --- a/tests/queries/1_stateful/00091_prewhere_two_conditions.sql +++ b/tests/queries/1_stateful/00091_prewhere_two_conditions.sql @@ -2,12 +2,12 @@ SET max_bytes_to_read = 600000000; SET optimize_move_to_prewhere = 1; -SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Europe/Moscow') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Europe/Moscow') < '2014-03-21 00:00:00'; -SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Europe/Moscow') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Europe/Moscow') < '2014-03-21 00:00:00'; -SELECT uniq(*) FROM test.hits WHERE toTimeZone(EventTime, 'Europe/Moscow') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Europe/Moscow') < '2014-03-21 00:00:00' AND EventDate = '2014-03-21'; -WITH toTimeZone(EventTime, 'Europe/Moscow') AS xyz SELECT uniq(*) FROM test.hits WHERE xyz >= '2014-03-20 00:00:00' AND xyz < '2014-03-21 00:00:00' AND EventDate = '2014-03-21'; +SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00'; +SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00'; +SELECT uniq(*) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00' AND EventDate = '2014-03-21'; +WITH toTimeZone(EventTime, 'Asia/Dubai') AS xyz SELECT uniq(*) FROM test.hits WHERE xyz >= '2014-03-20 00:00:00' AND xyz < '2014-03-21 00:00:00' AND EventDate = '2014-03-21'; SET optimize_move_to_prewhere = 0; -SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Europe/Moscow') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Europe/Moscow') < '2014-03-21 00:00:00'; -- { serverError 307 } -SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Europe/Moscow') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Europe/Moscow') < '2014-03-21 00:00:00'; -- { serverError 307 } +SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00'; -- { serverError 307 } +SELECT uniq(URL) FROM test.hits WHERE toTimeZone(EventTime, 'Asia/Dubai') >= '2014-03-20 00:00:00' AND URL != '' AND toTimeZone(EventTime, 'Asia/Dubai') < '2014-03-21 00:00:00'; -- { serverError 307 } diff --git a/tests/queries/1_stateful/00154_avro.sql b/tests/queries/1_stateful/00154_avro.sql index ea5d665a3b4..f608da629d2 100644 --- a/tests/queries/1_stateful/00154_avro.sql +++ b/tests/queries/1_stateful/00154_avro.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test.avro; -SET max_threads = 1, max_block_size = 8192, min_insert_block_size_rows = 8192, min_insert_block_size_bytes = 1048576; -- lower memory usage +SET max_threads = 1, max_insert_threads = 0, max_block_size = 8192, min_insert_block_size_rows = 8192, min_insert_block_size_bytes = 1048576; -- lower memory usage CREATE TABLE test.avro AS test.hits ENGINE = File(Avro); INSERT INTO test.avro SELECT * FROM test.hits LIMIT 10000; diff --git a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference index 04107d74341..3b7b346e7e8 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.reference @@ -1,8 +1,8 @@ CSV, false -ea1c740f03f5dcc43a3044528ad0a98f - +6929aaeac016d22c20464e3be38c64cd - CSV, true -ea1c740f03f5dcc43a3044528ad0a98f - +6929aaeac016d22c20464e3be38c64cd - CSVWithNames, false -e986f353467c87b07e7143d7bff2daff - +1610d7eac24fb923cd973c99ab7e3a8d - CSVWithNames, true -e986f353467c87b07e7143d7bff2daff - +1610d7eac24fb923cd973c99ab7e3a8d - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh index a6b5620812d..1476d2892bf 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_csv_and_friends.sh @@ -10,10 +10,10 @@ for format in "${FORMATS[@]}" do echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime::DateTime('Europe/Moscow') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=true -q \ - "SELECT ClientEventTime::DateTime('Europe/Moscow') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum done diff --git a/tests/queries/1_stateful/00159_parallel_formatting_http.reference b/tests/queries/1_stateful/00159_parallel_formatting_http.reference index 8eabf5d4f03..34ecd115748 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_http.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_http.reference @@ -1,12 +1,12 @@ TSV, false -6e4ce4996dd0e036d27cb0d2166c8e59 - +9e0a1b1db4d1e56b4b571a8824dde35b - TSV, true -6e4ce4996dd0e036d27cb0d2166c8e59 - +9e0a1b1db4d1e56b4b571a8824dde35b - CSV, false -ab6b3616f31e8a952c802ca92562e418 - +c9c6f633a59d349f9f8a14ee2f1cb1b3 - CSV, true -ab6b3616f31e8a952c802ca92562e418 - +c9c6f633a59d349f9f8a14ee2f1cb1b3 - JSONCompactEachRow, false -1651b540b43bd6c62446f4c340bf13c7 - +826e244bd6c547b52955dd69df61ea22 - JSONCompactEachRow, true -1651b540b43bd6c62446f4c340bf13c7 - +826e244bd6c547b52955dd69df61ea22 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_http.sh b/tests/queries/1_stateful/00159_parallel_formatting_http.sh index 1dcae50812e..ea4a4d12867 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_http.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_http.sh @@ -10,8 +10,8 @@ FORMATS=('TSV' 'CSV' 'JSONCompactEachRow') for format in "${FORMATS[@]}" do echo "$format, false"; - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+ClientEventTime::DateTime('Europe/Moscow')+as+a,MobilePhoneModel+as+b,ClientIP6+as+c+FROM+test.hits+ORDER+BY+a,b,c+LIMIT+1000000+Format+$format&output_format_parallel_formatting=false" -d' ' | md5sum + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+ClientEventTime::DateTime('Asia/Dubai')+as+a,MobilePhoneModel+as+b,ClientIP6+as+c+FROM+test.hits+ORDER+BY+a,b,c+LIMIT+1000000+Format+$format&output_format_parallel_formatting=false" -d' ' | md5sum echo "$format, true"; - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+ClientEventTime::DateTime('Europe/Moscow')+as+a,MobilePhoneModel+as+b,ClientIP6+as+c+FROM+test.hits+ORDER+BY+a,b,c+LIMIT+1000000+Format+$format&output_format_parallel_formatting=true" -d' ' | md5sum + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+ClientEventTime::DateTime('Asia/Dubai')+as+a,MobilePhoneModel+as+b,ClientIP6+as+c+FROM+test.hits+ORDER+BY+a,b,c+LIMIT+1000000+Format+$format&output_format_parallel_formatting=true" -d' ' | md5sum done diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference index 7ad5359a30e..42e69ea3a0d 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.reference @@ -1,28 +1,28 @@ JSONEachRow, false -e0a3c9978a92a277f2fff4664f3c1749 - +c6b89185cc5b3dff5d3779e2e1551b81 - JSONEachRow, true -e0a3c9978a92a277f2fff4664f3c1749 - +c6b89185cc5b3dff5d3779e2e1551b81 - JSONCompactEachRow, false -0c1efbbc25a5bd90a2ecea559d283667 - +5c838a00e22d943fa429c45106b7ff4d - JSONCompactEachRow, true -0c1efbbc25a5bd90a2ecea559d283667 - +5c838a00e22d943fa429c45106b7ff4d - JSONCompactStringsEachRow, false -0c1efbbc25a5bd90a2ecea559d283667 - +5c838a00e22d943fa429c45106b7ff4d - JSONCompactStringsEachRow, true -0c1efbbc25a5bd90a2ecea559d283667 - +5c838a00e22d943fa429c45106b7ff4d - JSONCompactEachRowWithNames, false -b9e4f8ecadbb650245d1762f4187ee0a - +e3231b1c8187de4da6752d692b2ddba9 - JSONCompactEachRowWithNames, true -b9e4f8ecadbb650245d1762f4187ee0a - +e3231b1c8187de4da6752d692b2ddba9 - JSONCompactStringsEachRowWithNames, false -b9e4f8ecadbb650245d1762f4187ee0a - +e3231b1c8187de4da6752d692b2ddba9 - JSONCompactStringsEachRowWithNames, true -b9e4f8ecadbb650245d1762f4187ee0a - +e3231b1c8187de4da6752d692b2ddba9 - JSONCompactEachRowWithNamesAndTypes, false -8b41f7375999b53d4c9607398456fe5b - +21302d11da0bf8d37ab599e28a51bac2 - JSONCompactEachRowWithNamesAndTypes, true -8b41f7375999b53d4c9607398456fe5b - +21302d11da0bf8d37ab599e28a51bac2 - JSONCompactStringsEachRowWithNamesAndTypes, false -8b41f7375999b53d4c9607398456fe5b - +21302d11da0bf8d37ab599e28a51bac2 - JSONCompactStringsEachRowWithNamesAndTypes, true -8b41f7375999b53d4c9607398456fe5b - +21302d11da0bf8d37ab599e28a51bac2 - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh index f6c87eabfde..a96ed0c9b96 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_json_and_friends.sh @@ -12,9 +12,9 @@ for format in "${FORMATS[@]}" do echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime::DateTime('Europe/Moscow') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=true -q \ - "SELECT ClientEventTime::DateTime('Europe/Moscow') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c LIMIT 3000000 Format $format" | md5sum done diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference index 04d6db3b4af..91e3af03db8 100644 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.reference @@ -1,12 +1,12 @@ TSV, false -8a984bbbfb127c430f67173f5371c6cb - +194d5061de4cae59489d989373f8effe - TSV, true -8a984bbbfb127c430f67173f5371c6cb - +194d5061de4cae59489d989373f8effe - TSVWithNames, false -ead321ed96754ff1aa39d112bc28c43d - +a6d327a3611288b3f973d00e6116f16e - TSVWithNames, true -ead321ed96754ff1aa39d112bc28c43d - +a6d327a3611288b3f973d00e6116f16e - TSKV, false -1735308ecea5c269846f36a55d5b335f - +c2e32a21c08aacf60bda21248ce4f73f - TSKV, true -1735308ecea5c269846f36a55d5b335f - +c2e32a21c08aacf60bda21248ce4f73f - diff --git a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh index 02d083c0498..9d48774dd2d 100755 --- a/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh +++ b/tests/queries/1_stateful/00159_parallel_formatting_tsv_and_friends.sh @@ -11,9 +11,9 @@ for format in "${FORMATS[@]}" do echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime::DateTime('Europe/Moscow') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=true -q \ - "SELECT ClientEventTime::DateTime('Europe/Moscow') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum + "SELECT ClientEventTime::DateTime('Asia/Dubai') as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits ORDER BY a, b, c Format $format" | md5sum done diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference index fb0ba75c148..cd8c2e21b09 100644 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference @@ -1,8 +1,8 @@ TSVWithNames, false -29caf86494f169d6339f6c5610b20731 - +0c6d493d47ff0aa1c6111c40c2b6cfcf - TSVWithNames, true -29caf86494f169d6339f6c5610b20731 - +0c6d493d47ff0aa1c6111c40c2b6cfcf - CSVWithNames, false -29caf86494f169d6339f6c5610b20731 - +0c6d493d47ff0aa1c6111c40c2b6cfcf - CSVWithNames, true -29caf86494f169d6339f6c5610b20731 - +0c6d493d47ff0aa1c6111c40c2b6cfcf - diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index 777d95fa0af..a1136a47319 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -10,21 +10,21 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" for format in "${FORMATS[@]}" do # Columns are permuted - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Europe/Moscow'), b String) ENGINE=Memory()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Asia/Dubai'), b String) ENGINE=Memory()" echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Europe/Moscow') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Asia/Dubai') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Europe/Moscow'), b String) ENGINE=Memory()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Asia/Dubai'), b String) ENGINE=Memory()" echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Europe/Moscow') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Asia/Dubai') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum diff --git a/tests/queries/1_stateful/00163_column_oriented_formats.reference b/tests/queries/1_stateful/00163_column_oriented_formats.reference index cb20aca4392..cf29a217fe4 100644 --- a/tests/queries/1_stateful/00163_column_oriented_formats.reference +++ b/tests/queries/1_stateful/00163_column_oriented_formats.reference @@ -1,12 +1,12 @@ Parquet -6b397d4643bc1f920f3eb8aa87ee180c - +093d0270733e505af52436f9df4a779f - 7fe6d8c57ddc5fe37bbdcb7f73c5fa78 - d8746733270cbeff7ab3550c9b944fb6 - Arrow -6b397d4643bc1f920f3eb8aa87ee180c - +093d0270733e505af52436f9df4a779f - 7fe6d8c57ddc5fe37bbdcb7f73c5fa78 - d8746733270cbeff7ab3550c9b944fb6 - ORC -6b397d4643bc1f920f3eb8aa87ee180c - +093d0270733e505af52436f9df4a779f - 7fe6d8c57ddc5fe37bbdcb7f73c5fa78 - d8746733270cbeff7ab3550c9b944fb6 - diff --git a/tests/queries/1_stateful/00163_column_oriented_formats.sh b/tests/queries/1_stateful/00163_column_oriented_formats.sh index 50ad20cbe92..803474c4fa7 100755 --- a/tests/queries/1_stateful/00163_column_oriented_formats.sh +++ b/tests/queries/1_stateful/00163_column_oriented_formats.sh @@ -11,7 +11,7 @@ for format in "${FORMATS[@]}" do echo $format $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 00163_column_oriented SYNC" - $CLICKHOUSE_CLIENT -q "CREATE TABLE 00163_column_oriented(ClientEventTime DateTime('Europe/Moscow'), MobilePhoneModel String, ClientIP6 FixedString(16)) ENGINE=File($format)" + $CLICKHOUSE_CLIENT -q "CREATE TABLE 00163_column_oriented(ClientEventTime DateTime('Asia/Dubai'), MobilePhoneModel String, ClientIP6 FixedString(16)) ENGINE=File($format)" $CLICKHOUSE_CLIENT -q "INSERT INTO 00163_column_oriented SELECT ClientEventTime, MobilePhoneModel, ClientIP6 FROM test.hits ORDER BY ClientEventTime, MobilePhoneModel, ClientIP6 LIMIT 100" $CLICKHOUSE_CLIENT -q "SELECT ClientEventTime from 00163_column_oriented" | md5sum $CLICKHOUSE_CLIENT -q "SELECT MobilePhoneModel from 00163_column_oriented" | md5sum diff --git a/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.reference b/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.reference index 0c0367694b2..a2c69c24fa2 100644 --- a/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.reference +++ b/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.reference @@ -1,20 +1,20 @@ TSVWithNamesAndTypes, false -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - TSVWithNamesAndTypes, true -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - CSVWithNamesAndTypes, false -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - CSVWithNamesAndTypes, true -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - JSONStringsEachRow, false -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - JSONStringsEachRow, true -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - JSONCompactEachRowWithNamesAndTypes, false -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - JSONCompactEachRowWithNamesAndTypes, true -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - JSONCompactStringsEachRowWithNamesAndTypes, false -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - JSONCompactStringsEachRowWithNamesAndTypes, true -7c1feeaae418e502d66fcc8e31946f2e - +0bd9fe2bc50147cd260bb58457329385 - diff --git a/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.sh b/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.sh index 9fdca20d097..33562918f67 100755 --- a/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.sh +++ b/tests/queries/1_stateful/00167_parallel_parsing_with_names_and_types.sh @@ -10,21 +10,21 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" for format in "${FORMATS[@]}" do # Columns are permuted - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Europe/Moscow'), b String) ENGINE=Memory()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Asia/Dubai'), b String) ENGINE=Memory()" echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Europe/Moscow') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 5000 Format $format" | \ + "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Asia/Dubai') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 5000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format SETTINGS input_format_null_as_default=0" $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Europe/Moscow'), b String) ENGINE=Memory()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime('Asia/Dubai'), b String) ENGINE=Memory()" echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Europe/Moscow') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 5000 Format $format" | \ + "SELECT URLRegions as d, toTimeZone(ClientEventTime, 'Asia/Dubai') as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 5000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format SETTINGS input_format_null_as_default=0" $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum diff --git a/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh b/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh index 699700bcd3e..276fc0274c2 100755 --- a/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh +++ b/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan +# Tags: no-tsan, no-random-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/1_stateful/00170_s3_cache.reference b/tests/queries/1_stateful/00170_s3_cache.reference new file mode 100644 index 00000000000..9c9c3bc537f --- /dev/null +++ b/tests/queries/1_stateful/00170_s3_cache.reference @@ -0,0 +1,270 @@ +-- { echo } +SET max_memory_usage='20G'; +SELECT count() FROM test.hits_s3; +8873898 +SELECT count() FROM test.hits_s3 WHERE AdvEngineID != 0; +30641 +SELECT sum(AdvEngineID), count(), avg(ResolutionWidth) FROM test.hits_s3 ; +329039 8873898 1400.8565027454677 +SELECT sum(UserID) FROM test.hits_s3 ; +15358948234638402412 +SELECT uniq(UserID) FROM test.hits_s3 ; +120665 +SELECT uniq(SearchPhrase) FROM test.hits_s3 ; +132591 +SELECT min(EventDate), max(EventDate) FROM test.hits_s3 ; +2014-03-17 2014-03-23 +SELECT AdvEngineID, count() FROM test.hits_s3 WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY AdvEngineID DESC; +62 7 +61 12 +58 83 +55 281 +52 454 +51 74 +50 353 +49 7 +48 224 +42 72 +41 76 +40 91 +35 2751 +32 141 +30 1832 +24 9 +22 3 +18 3 +16 1019 +12 1 +10 3 +4 10 +3 22948 +2 187 +SELECT RegionID, uniq(UserID) AS u FROM test.hits_s3 GROUP BY RegionID ORDER BY u DESC LIMIT 10; +196 9275 +8363 4624 +15887 4585 +241 4488 +207 3596 +3 3319 +12504 1594 +183 1592 +57 1251 +225 1177 +SELECT RegionID, sum(AdvEngineID), count() AS c, avg(ResolutionWidth), uniq(UserID) FROM test.hits_s3 GROUP BY RegionID ORDER BY c DESC LIMIT 10; +196 32570 1311992 1437.5239170665675 9275 +3 11425 428577 1424.2968801405582 3319 +241 8291 320659 1149.9956152797831 4488 +207 7360 285615 1264.5680093832607 3596 +15887 27514 197463 1392.8657064867848 4585 +8363 26522 197154 1361.9469247390364 4624 +183 13054 186914 1470.3840054784553 1592 +225 1817 164048 1404.8909831268898 1177 +40 1883 107154 1407.6735912798401 808 +57 2146 99424 1200.338721033151 1251 +SELECT MobilePhoneModel, uniq(UserID) AS u FROM test.hits_s3 WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +S820_ROW 7616 +iPhone 2 6111 +LG Optimus 4134 +Samsung Galaxy 813 +iPad HD 7 604 +Sams 558 +Samsung Galaxy Note 501 +iPad 2 434 +iPhone S720 393 +iPad 10 FHD 306 +SELECT MobilePhone, MobilePhoneModel, uniq(UserID) AS u FROM test.hits_s3 WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +1 S820_ROW 7613 +7 iPhone 2 5993 +1 LG Optimus 4098 +5 Samsung Galaxy Note 499 +5 Sams 346 +5 Samsung Galaxy 273 +7 iPad HD 7 240 +5 iPad 213 +4 Sams 210 +7 Samsung Galaxy 189 +SELECT uniq(SearchPhrase), count() AS c FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +1 3567 +1 2402 +1 2166 +1 1848 +1 1659 +1 1549 +1 1480 +1 1247 +1 1112 +1 1091 +SELECT uniq(SearchPhrase), uniq(UserID) AS u FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +1 786 +1 479 +1 320 +1 188 +1 181 +1 174 +1 173 +1 162 +1 159 +1 141 +SELECT SearchEngineID, uniq(SearchPhrase), count() AS c FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +3 1 3490 +3 1 2166 +3 1 1599 +3 1 1549 +3 1 1530 +3 1 1442 +3 1 1247 +3 1 1112 +3 1 1091 +3 1 1064 +SELECT UserID, count() FROM test.hits_s3 GROUP BY UserID ORDER BY count() DESC LIMIT 10; +1205491256153864188 31519 +3228040076666004453 20688 +2543118835429830843 16329 +1961021224905272484 13484 +4322253409885123546 11186 +2034549784946942048 10970 +397859646441652491 8229 +8032089779962875762 8149 +1839265440135330496 7816 +5548175707459682622 7806 +SELECT UserID, uniq(SearchPhrase) as m, count() as c FROM test.hits_s3 GROUP BY UserID, SearchPhrase ORDER BY UserID, m, c DESC LIMIT 10; +2961521519262 1 56 +87878526839192 1 414 +87878526839192 1 15 +87878526839192 1 6 +87878526839192 1 6 +87878526839192 1 5 +87878526839192 1 5 +87878526839192 1 5 +87878526839192 1 4 +87878526839192 1 3 +SELECT UserID, uniq(SearchPhrase) as m, count() as c FROM test.hits_s3 GROUP BY UserID, SearchPhrase ORDER BY UserID, m, c LIMIT 10; +2961521519262 1 56 +87878526839192 1 1 +87878526839192 1 1 +87878526839192 1 1 +87878526839192 1 2 +87878526839192 1 3 +87878526839192 1 4 +87878526839192 1 5 +87878526839192 1 5 +87878526839192 1 5 +SELECT UserID, toMinute(EventTime) AS m, uniq(SearchPhrase) as u, count() as c FROM test.hits_s3 GROUP BY UserID, m, SearchPhrase ORDER BY UserID DESC LIMIT 10 FORMAT Null; +SELECT UserID FROM test.hits_s3 WHERE UserID = 12345678901234567890; +SELECT count() FROM test.hits_s3 WHERE URL LIKE '%metrika%'; +2348 +SELECT uniq(SearchPhrase) as u, max(URL) as m, count() AS c FROM test.hits_s3 WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u, m, c DESC LIMIT 10; +1 goal://delive/812metrika.com/kizi-bulochkomna 4 +1 goal://delive/812metrika.com/kizi-bulochkomna 2 +1 goal://delive/812metrika.com/kizi-bulochkomna 2 +1 goal://delive/812metrika.com/kizi-bulochkomna 2 +1 goal://mail.yandex.ru/yrs/ekonometrika/kermosure-batakte 2 +1 http:%2F%2F%2F2014/03/18/cid=54&metrika.com 1 +1 http:%2F%2Ffiles&order=0&metrikancy-podar 1 +1 http:%2F%2Fiteme.metrika 1 +1 http:%2F%2Fproduct/shop.rbc.ru/rostometrikatuvali-k-pensadabuga/nauka_30_m_610_730641%2F01%2Fannovsk/dom-drugie_zhalujsta-s-social 1 +1 http:%2F%2Fwww.kirovanny/donnel_mart]=creative=0&metrika.ru/socialog 1 +SELECT uniq(SearchPhrase), max(URL), max(Title), count() AS c, uniq(UserID) FROM test.hits_s3 WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +1 http://korer.ru/categories.ru/?vkb Яндекс: нашлось 184 тыс изображений програница 27 тыс. ответов в России - 1245 1 +1 http://korer.ru/categories.ru/?vkb Яндекс.Картинках, поиск на AVITO.ru • Знакомства вакансии на дом электриса 710 1 +1 http://yandsearch[run][min]=200 одного подаров в Краснодателя » Страница 2 - современно в Яндекс: нашлось 8 мартфонарнажатие и последник Красность рисунки на AVITO.ru. Часы VU+ Uno 696 310 +1 http://korer.ru/categories.ru/?vkb Яндекс: нашем качествует о тебя не следников PRAJNA Cerator.org.com / Shopogody - Read izle, Diva.BY 668 1 +1 http://yandex.ru/chechristana.ru/clck/jsredircnt=1377554 Яндекс.Новости в Санкт-Петербурге: 228-135 тыс. ответов цифр трудников на Весная 572 1 +1 https://dns-state=AiuY0DBWFJ4ePaEs статися водят? - Испании туре за неделки игрушенко — Ирина домашних услуг Россия) - Яндекс: нашлось 236 тыс изображений 546 54 +1 http://korer.ru/categories.ru/?vkb Яндекс.Новоришь всё о купить модели Виннис, ЧП. Соболєв і 457 1 +1 https://my.mail.ru/appliancePotr 芒果 | ТЕЛЕГРАФ - Яндекс.Почта Mail.Ru: Из-за смотреть 439 221 +1 http://korer.ru/categories.ru/?vkb Продажа плании онлайн бесплатно в Яндекс.Маркетинг - новости менеджера, 61 438 1 +1 http://korer.ru/categories.ru/?vkb Яндекс: нашем качестве: почалась 396 Hp) 5-dr 200000 для зимние восписок тили 395 1 +SELECT * FROM test.hits_s3 WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10 format Null; +SELECT SearchPhrase FROM test.hits_s3 WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10 FORMAT Null; +SELECT SearchPhrase FROM test.hits_s3 WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10 FORMAT Null; +SELECT SearchPhrase FROM test.hits_s3 WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10 FORMAT Null; +SELECT CounterID, avg(length(URL)) AS l, count() AS c FROM test.hits_s3 WHERE URL != '' GROUP BY CounterID HAVING c > 100000 ORDER BY l DESC LIMIT 25; +25703952 185.35847185332617 147211 +732797 145.03929351646454 475142 +792887 123.97688315087015 252197 +3807842 78.46108053235935 196033 +1704509 60.11621475966243 523264 +598875 20.267298451681793 337140 +SELECT domainWithoutWWW(Referer) AS key, avg(length(Referer)) AS l, count() AS c, max(Referer) FROM test.hits_s3 WHERE Referer != '' GROUP BY key HAVING c > 100000 ORDER BY l DESC LIMIT 25; +vk.com.ua 670.6812170535467 205447 https://vk.com.ua/health.mail.yandsearch?lr=213&msid=87&redircnt=1310461&with_photorcycle/users/424246b7dcbba51/offers +avito.ru 89.56139198679928 243623 https://avito.ru/стих по биатлона +vk.com 88.93009846053418 680171 https://vk.com/video +yandex.ru 85.79982623523495 554773 https://yandex.ru/yandsearch + 81.39774471008556 2237229 httpvmkNCAErJlhPSHlqdmtsWFc4MXZtLUR1Q3Y9tM8jq5BkkHRyeFVKWTEJ6dE9iQnYCex9 +m.auto.ru 58.542011573622986 118027 https://m.auto.ru/yoshka-sokaklari-60.html#/battle-ru11 +SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM test.hits_s3; +12431057718 12439931616 12448805514 12457679412 12466553310 12475427208 12484301106 12493175004 12502048902 12510922800 12519796698 12528670596 12537544494 12546418392 12555292290 12564166188 12573040086 12581913984 12590787882 12599661780 12608535678 12617409576 12626283474 12635157372 12644031270 12652905168 12661779066 12670652964 12679526862 12688400760 12697274658 12706148556 12715022454 12723896352 12732770250 12741644148 12750518046 12759391944 12768265842 12777139740 12786013638 12794887536 12803761434 12812635332 12821509230 12830383128 12839257026 12848130924 12857004822 12865878720 12874752618 12883626516 12892500414 12901374312 12910248210 12919122108 12927996006 12936869904 12945743802 12954617700 12963491598 12972365496 12981239394 12990113292 12998987190 13007861088 13016734986 13025608884 13034482782 13043356680 13052230578 13061104476 13069978374 13078852272 13087726170 13096600068 13105473966 13114347864 13123221762 13132095660 13140969558 13149843456 13158717354 13167591252 13176465150 13185339048 13194212946 13203086844 13211960742 13220834640 +SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +3 1660732911 2564 21 1339 +3 1795610432 1808 49 1622 +3 442614592 1801 63 1622 +3 280750947 1722 92 1339 +3 1794713726 1565 143 1297 +3 2122160434 1449 29 1846 +3 2120191779 1431 117 1339 +3 3726560380 1338 37 1339 +3 1382059522 1212 25 1386 +3 2454020642 1108 25 1339 +SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c, WatchID DESC LIMIT 10; +9223343978848462524 807160513 1 0 1339 +9223311592760478486 622798371 1 0 1622 +9223290551912005343 1399751135 1 0 1386 +9223283743622263900 4248624768 1 0 1339 +9223277679551805964 2079360072 1 0 1639 +9223250576755718785 471654323 1 0 1622 +9223247301332594153 2030669591 1 0 1297 +9223246228500137980 2156909056 1 0 467 +9223227691645120897 91683468 1 0 1846 +9223220893120643152 1357136342 1 0 1297 +SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM test.hits_s3 GROUP BY WatchID, ClientIP ORDER BY c, WatchID DESC LIMIT 10; +9223371678237104442 1510763633 1 0 1622 +9223371583739401906 1316647510 1 0 1587 +9223369973176670469 1581144184 1 0 1297 +9223369447059354172 1759910327 1 0 1339 +9223368297061364285 1900808651 1 0 1339 +9223367627527921417 1250879542 1 0 1587 +9223367120605710467 818965311 1 0 1622 +9223365068732217887 287613368 1 0 1386 +9223364444623921469 697478885 1 0 1622 +9223363407092000972 76513606 1 0 1297 +SELECT URL, count() AS c FROM test.hits_s3 GROUP BY URL ORDER BY c DESC LIMIT 10; +http://public_search 311119 +http://auto.ru/chatay-barana.ru/traction.html#maybettaya 189442 +http://korer.ru/categories.ru/?vkb 142669 +http://main=hurriyet.com/iframe/frm_index.ru/photofunki-sayesilcipo-showthredir?from=&seatsTo=&purchynet.com/galaxy-nosti.ru/preso.tv/Archi.shtml?002 122598 +http://korablitz.ru/L_1OFFERS_CRD 45069 +http://bravoslava-230v 32907 +http://images.yandex.ru 22100 +http://doc/00003713844324&education.html?logi-38-rasstreferer_id 21145 +http://rutube.ru/patianu 19064 +http://search?win=11&pos=22&img_url=http:%2F%2Fcs411276 19060 +SELECT 1, URL, count() AS c FROM test.hits_s3 GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +1 http://public_search 311119 +1 http://auto.ru/chatay-barana.ru/traction.html#maybettaya 189442 +1 http://korer.ru/categories.ru/?vkb 142669 +1 http://main=hurriyet.com/iframe/frm_index.ru/photofunki-sayesilcipo-showthredir?from=&seatsTo=&purchynet.com/galaxy-nosti.ru/preso.tv/Archi.shtml?002 122598 +1 http://korablitz.ru/L_1OFFERS_CRD 45069 +1 http://bravoslava-230v 32907 +1 http://images.yandex.ru 22100 +1 http://doc/00003713844324&education.html?logi-38-rasstreferer_id 21145 +1 http://rutube.ru/patianu 19064 +1 http://search?win=11&pos=22&img_url=http:%2F%2Fcs411276 19060 +SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM test.hits_s3 GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10; +2950145570 2950145569 2950145568 2950145567 8149 +2408492821 2408492820 2408492819 2408492818 7770 +2494028488 2494028487 2494028486 2494028485 7696 +1688720600 1688720599 1688720598 1688720597 7681 +356903718 356903717 356903716 356903715 6817 +908127740 908127739 908127738 908127737 6624 +45907785 45907784 45907783 45907782 6556 +1567954933 1567954932 1567954931 1567954930 6203 +406416527 406416526 406416525 406416524 6015 +1410634230 1410634229 1410634228 1410634227 5742 +SELECT URL, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews, Title DESC LIMIT 10; +SELECT URL, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews, TraficSourceID DESC LIMIT 1000; +SELECT URLHash, EventDate, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100; +SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-02' AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; diff --git a/tests/queries/1_stateful/00170_s3_cache.sql b/tests/queries/1_stateful/00170_s3_cache.sql new file mode 100644 index 00000000000..af3fd402596 --- /dev/null +++ b/tests/queries/1_stateful/00170_s3_cache.sql @@ -0,0 +1,45 @@ +-- { echo } +SET max_memory_usage='20G'; +SELECT count() FROM test.hits_s3; +SELECT count() FROM test.hits_s3 WHERE AdvEngineID != 0; +SELECT sum(AdvEngineID), count(), avg(ResolutionWidth) FROM test.hits_s3 ; +SELECT sum(UserID) FROM test.hits_s3 ; +SELECT uniq(UserID) FROM test.hits_s3 ; +SELECT uniq(SearchPhrase) FROM test.hits_s3 ; +SELECT min(EventDate), max(EventDate) FROM test.hits_s3 ; +SELECT AdvEngineID, count() FROM test.hits_s3 WHERE AdvEngineID != 0 GROUP BY AdvEngineID ORDER BY AdvEngineID DESC; +SELECT RegionID, uniq(UserID) AS u FROM test.hits_s3 GROUP BY RegionID ORDER BY u DESC LIMIT 10; +SELECT RegionID, sum(AdvEngineID), count() AS c, avg(ResolutionWidth), uniq(UserID) FROM test.hits_s3 GROUP BY RegionID ORDER BY c DESC LIMIT 10; +SELECT MobilePhoneModel, uniq(UserID) AS u FROM test.hits_s3 WHERE MobilePhoneModel != '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT MobilePhone, MobilePhoneModel, uniq(UserID) AS u FROM test.hits_s3 WHERE MobilePhoneModel != '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +SELECT uniq(SearchPhrase), count() AS c FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT uniq(SearchPhrase), uniq(UserID) AS u FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +SELECT SearchEngineID, uniq(SearchPhrase), count() AS c FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT UserID, count() FROM test.hits_s3 GROUP BY UserID ORDER BY count() DESC LIMIT 10; +SELECT UserID, uniq(SearchPhrase) as m, count() as c FROM test.hits_s3 GROUP BY UserID, SearchPhrase ORDER BY UserID, m, c DESC LIMIT 10; +SELECT UserID, uniq(SearchPhrase) as m, count() as c FROM test.hits_s3 GROUP BY UserID, SearchPhrase ORDER BY UserID, m, c LIMIT 10; +SELECT UserID, toMinute(EventTime) AS m, uniq(SearchPhrase) as u, count() as c FROM test.hits_s3 GROUP BY UserID, m, SearchPhrase ORDER BY UserID DESC LIMIT 10 FORMAT Null; +SELECT UserID FROM test.hits_s3 WHERE UserID = 12345678901234567890; +SELECT count() FROM test.hits_s3 WHERE URL LIKE '%metrika%'; +SELECT uniq(SearchPhrase) as u, max(URL) as m, count() AS c FROM test.hits_s3 WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY u, m, c DESC LIMIT 10; +SELECT uniq(SearchPhrase), max(URL), max(Title), count() AS c, uniq(UserID) FROM test.hits_s3 WHERE Title LIKE '%Яндекс%' AND URL NOT LIKE '%.yandex.%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +SELECT * FROM test.hits_s3 WHERE URL LIKE '%metrika%' ORDER BY EventTime LIMIT 10 format Null; +SELECT SearchPhrase FROM test.hits_s3 WHERE SearchPhrase != '' ORDER BY EventTime LIMIT 10 FORMAT Null; +SELECT SearchPhrase FROM test.hits_s3 WHERE SearchPhrase != '' ORDER BY SearchPhrase LIMIT 10 FORMAT Null; +SELECT SearchPhrase FROM test.hits_s3 WHERE SearchPhrase != '' ORDER BY EventTime, SearchPhrase LIMIT 10 FORMAT Null; +SELECT CounterID, avg(length(URL)) AS l, count() AS c FROM test.hits_s3 WHERE URL != '' GROUP BY CounterID HAVING c > 100000 ORDER BY l DESC LIMIT 25; +SELECT domainWithoutWWW(Referer) AS key, avg(length(Referer)) AS l, count() AS c, max(Referer) FROM test.hits_s3 WHERE Referer != '' GROUP BY key HAVING c > 100000 ORDER BY l DESC LIMIT 25; +SELECT sum(ResolutionWidth), sum(ResolutionWidth + 1), sum(ResolutionWidth + 2), sum(ResolutionWidth + 3), sum(ResolutionWidth + 4), sum(ResolutionWidth + 5), sum(ResolutionWidth + 6), sum(ResolutionWidth + 7), sum(ResolutionWidth + 8), sum(ResolutionWidth + 9), sum(ResolutionWidth + 10), sum(ResolutionWidth + 11), sum(ResolutionWidth + 12), sum(ResolutionWidth + 13), sum(ResolutionWidth + 14), sum(ResolutionWidth + 15), sum(ResolutionWidth + 16), sum(ResolutionWidth + 17), sum(ResolutionWidth + 18), sum(ResolutionWidth + 19), sum(ResolutionWidth + 20), sum(ResolutionWidth + 21), sum(ResolutionWidth + 22), sum(ResolutionWidth + 23), sum(ResolutionWidth + 24), sum(ResolutionWidth + 25), sum(ResolutionWidth + 26), sum(ResolutionWidth + 27), sum(ResolutionWidth + 28), sum(ResolutionWidth + 29), sum(ResolutionWidth + 30), sum(ResolutionWidth + 31), sum(ResolutionWidth + 32), sum(ResolutionWidth + 33), sum(ResolutionWidth + 34), sum(ResolutionWidth + 35), sum(ResolutionWidth + 36), sum(ResolutionWidth + 37), sum(ResolutionWidth + 38), sum(ResolutionWidth + 39), sum(ResolutionWidth + 40), sum(ResolutionWidth + 41), sum(ResolutionWidth + 42), sum(ResolutionWidth + 43), sum(ResolutionWidth + 44), sum(ResolutionWidth + 45), sum(ResolutionWidth + 46), sum(ResolutionWidth + 47), sum(ResolutionWidth + 48), sum(ResolutionWidth + 49), sum(ResolutionWidth + 50), sum(ResolutionWidth + 51), sum(ResolutionWidth + 52), sum(ResolutionWidth + 53), sum(ResolutionWidth + 54), sum(ResolutionWidth + 55), sum(ResolutionWidth + 56), sum(ResolutionWidth + 57), sum(ResolutionWidth + 58), sum(ResolutionWidth + 59), sum(ResolutionWidth + 60), sum(ResolutionWidth + 61), sum(ResolutionWidth + 62), sum(ResolutionWidth + 63), sum(ResolutionWidth + 64), sum(ResolutionWidth + 65), sum(ResolutionWidth + 66), sum(ResolutionWidth + 67), sum(ResolutionWidth + 68), sum(ResolutionWidth + 69), sum(ResolutionWidth + 70), sum(ResolutionWidth + 71), sum(ResolutionWidth + 72), sum(ResolutionWidth + 73), sum(ResolutionWidth + 74), sum(ResolutionWidth + 75), sum(ResolutionWidth + 76), sum(ResolutionWidth + 77), sum(ResolutionWidth + 78), sum(ResolutionWidth + 79), sum(ResolutionWidth + 80), sum(ResolutionWidth + 81), sum(ResolutionWidth + 82), sum(ResolutionWidth + 83), sum(ResolutionWidth + 84), sum(ResolutionWidth + 85), sum(ResolutionWidth + 86), sum(ResolutionWidth + 87), sum(ResolutionWidth + 88), sum(ResolutionWidth + 89) FROM test.hits_s3; +SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM test.hits_s3 WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c, WatchID DESC LIMIT 10; +SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM test.hits_s3 GROUP BY WatchID, ClientIP ORDER BY c, WatchID DESC LIMIT 10; +SELECT URL, count() AS c FROM test.hits_s3 GROUP BY URL ORDER BY c DESC LIMIT 10; +SELECT 1, URL, count() AS c FROM test.hits_s3 GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM test.hits_s3 GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10; +SELECT URL, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews, Title DESC LIMIT 10; +SELECT URL, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews, TraficSourceID DESC LIMIT 1000; +SELECT URLHash, EventDate, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100; +SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM test.hits_s3 WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-02' AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; diff --git a/tests/queries/1_stateful/00171_grouping_aggregated_transform_bug.reference b/tests/queries/1_stateful/00171_grouping_aggregated_transform_bug.reference new file mode 100644 index 00000000000..e31a1e90d87 --- /dev/null +++ b/tests/queries/1_stateful/00171_grouping_aggregated_transform_bug.reference @@ -0,0 +1,2 @@ +10726001768429413598 +10726001768429413598 diff --git a/tests/queries/1_stateful/00171_grouping_aggregated_transform_bug.sql b/tests/queries/1_stateful/00171_grouping_aggregated_transform_bug.sql new file mode 100644 index 00000000000..7068780a1b1 --- /dev/null +++ b/tests/queries/1_stateful/00171_grouping_aggregated_transform_bug.sql @@ -0,0 +1,4 @@ +-- Tags: distributed + +SELECT sum(cityHash64(*)) FROM (SELECT CounterID, quantileTiming(0.5)(SendTiming), count() FROM remote('127.0.0.{1,2,3,4,5,6,7,8,9,10}', test.hits) WHERE SendTiming != -1 GROUP BY CounterID) SETTINGS max_block_size = 63169; +SELECT sum(cityHash64(*)) FROM (SELECT CounterID, quantileTiming(0.5)(SendTiming), count() FROM remote('127.0.0.{1,2,3,4,5,6,7,8,9,10}', test.hits) WHERE SendTiming != -1 GROUP BY CounterID) SETTINGS optimize_aggregation_in_order = 1, max_block_size = 63169; diff --git a/tests/queries/bugs/position_case_insensitive_utf8.sql b/tests/queries/bugs/position_case_insensitive_utf8.sql deleted file mode 100644 index 00ddd1b498d..00000000000 --- a/tests/queries/bugs/position_case_insensitive_utf8.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT positionCaseInsensitiveUTF8('Hello', materialize('%\xF0%')); -SELECT positionCaseInsensitiveUTF8(materialize('Hello'), '%\xF0%') FROM numbers(1000); diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index e95f8433636..23bf74139af 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -19,9 +19,9 @@ export CLICKHOUSE_TEST_UNIQUE_NAME="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABAS [ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_BENCHMARK_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " [ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " [ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " -[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' " +[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) " [ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_BENCHMARK_OPT0+=" --database=${CLICKHOUSE_DATABASE} " -[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' " +[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) " export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} # client @@ -129,3 +129,37 @@ function clickhouse_client_removed_host_parameter() # bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable. $(echo "$CLICKHOUSE_CLIENT" | python3 -c "import sys, re; print(re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read()))") "$@" } + +function clickhouse_client_timeout() +{ + local timeout=$1 && shift + timeout -s INT "$timeout" "$@" +} +# Helper function to stop the clickhouse-client after SIGINT properly. +function clickhouse_client_loop_timeout() +{ + local timeout=$1 && shift + + local cmd + cmd="$(printf '%q ' "$@")" + + timeout -s INT "$timeout" bash -c "trap 'STOP_THE_LOOP=1' INT; while true; do [ ! -v STOP_THE_LOOP ] || break; $cmd; done" +} +# wait for queries to be finished +function clickhouse_test_wait_queries() +{ + local timeout=${1:-"600"} && shift + local query_id="wait-$CLICKHOUSE_TEST_UNIQUE_NAME" + local query="SELECT count() FROM system.processes WHERE current_database = '$CLICKHOUSE_DATABASE' AND query_id != '$query_id'" + local i=0 + (( timeout*=2 )) + while [[ "$(${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query_id=$query_id" --data-binary "$query")" != "0" ]]; do + sleep 0.5 + + (( ++i )) + if [[ $i -gt $timeout ]]; then + echo "clickhouse_test_wait_queries: timeout exceeded" + exit 1 + fi + done +} diff --git a/tests/testflows/aes_encryption/configs/clickhouse/common.xml b/tests/testflows/aes_encryption/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/aes_encryption/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.xml deleted file mode 100644 index 9854f9f990e..00000000000 --- a/tests/testflows/aes_encryption/configs/clickhouse/config.xml +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 7200 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/aes_encryption/configs/clickhouse/users.xml b/tests/testflows/aes_encryption/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/aes_encryption/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py index 93add36cd1d..c12aaca861d 100755 --- a/tests/testflows/aes_encryption/regression.py +++ b/tests/testflows/aes_encryption/regression.py @@ -16,88 +16,126 @@ issue_24029 = "https://github.com/ClickHouse/ClickHouse/issues/24029" xfails = { # encrypt - "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too short": - [(Fail, "known issue")], - "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too long": - [(Fail, "known issue")], - "encrypt/invalid plaintext data type/data_type='IPv6', value=\"toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')\"": - [(Fail, "known issue as IPv6 is implemented as FixedString(16)")], + "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too short": [ + (Fail, "known issue") + ], + "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too long": [ + (Fail, "known issue") + ], + "encrypt/invalid plaintext data type/data_type='IPv6', value=\"toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')\"": [ + (Fail, "known issue as IPv6 is implemented as FixedString(16)") + ], # encrypt_mysql - "encrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None": - [(Fail, issue_18251)], - "encrypt_mysql/invalid parameters/iv not valid for mode": - [(Fail, issue_18251)], - "encrypt_mysql/invalid plaintext data type/data_type='IPv6', value=\"toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')\"": - [(Fail, "known issue as IPv6 is implemented as FixedString(16)")], + "encrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None": [ + (Fail, issue_18251) + ], + "encrypt_mysql/invalid parameters/iv not valid for mode": [(Fail, issue_18251)], + "encrypt_mysql/invalid plaintext data type/data_type='IPv6', value=\"toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')\"": [ + (Fail, "known issue as IPv6 is implemented as FixedString(16)") + ], # decrypt_mysql - "decrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None:": - [(Fail, issue_18251)], + "decrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None:": [ + (Fail, issue_18251) + ], # compatibility - "compatibility/insert/encrypt using materialized view/:": - [(Fail, issue_18249)], - "compatibility/insert/decrypt using materialized view/:": - [(Error, issue_18249)], - "compatibility/insert/aes encrypt mysql using materialized view/:": - [(Fail, issue_18249)], - "compatibility/insert/aes decrypt mysql using materialized view/:": - [(Error, issue_18249)], - "compatibility/select/decrypt unique": - [(Fail, issue_18249)], - "compatibility/mysql/:engine/decrypt/mysql_datatype='TEXT'/:": - [(Fail, issue_18250)], - "compatibility/mysql/:engine/decrypt/mysql_datatype='VARCHAR(100)'/:": - [(Fail, issue_18250)], - "compatibility/mysql/:engine/encrypt/mysql_datatype='TEXT'/:": - [(Fail, issue_18250)], - "compatibility/mysql/:engine/encrypt/mysql_datatype='VARCHAR(100)'/:": - [(Fail, issue_18250)], + "compatibility/insert/encrypt using materialized view/:": [(Fail, issue_18249)], + "compatibility/insert/decrypt using materialized view/:": [(Error, issue_18249)], + "compatibility/insert/aes encrypt mysql using materialized view/:": [ + (Fail, issue_18249) + ], + "compatibility/insert/aes decrypt mysql using materialized view/:": [ + (Error, issue_18249) + ], + "compatibility/select/decrypt unique": [(Fail, issue_18249)], + "compatibility/mysql/:engine/decrypt/mysql_datatype='TEXT'/:": [ + (Fail, issue_18250) + ], + "compatibility/mysql/:engine/decrypt/mysql_datatype='VARCHAR(100)'/:": [ + (Fail, issue_18250) + ], + "compatibility/mysql/:engine/encrypt/mysql_datatype='TEXT'/:": [ + (Fail, issue_18250) + ], + "compatibility/mysql/:engine/encrypt/mysql_datatype='VARCHAR(100)'/:": [ + (Fail, issue_18250) + ], # reinterpretAsFixedString for UUID stopped working - "decrypt/decryption/mode=:datatype=UUID:": - [(Fail, issue_24029)], - "encrypt/:/mode=:datatype=UUID:": - [(Fail, issue_24029)], - "decrypt/invalid ciphertext/mode=:/invalid ciphertext=reinterpretAsFixedString(toUUID:": - [(Fail, issue_24029)], - "encrypt_mysql/encryption/mode=:datatype=UUID:": - [(Fail, issue_24029)], - "decrypt_mysql/decryption/mode=:datatype=UUID:": - [(Fail, issue_24029)], - "decrypt_mysql/invalid ciphertext/mode=:/invalid ciphertext=reinterpretAsFixedString(toUUID:": - [(Fail, issue_24029)], + "decrypt/decryption/mode=:datatype=UUID:": [(Fail, issue_24029)], + "encrypt/:/mode=:datatype=UUID:": [(Fail, issue_24029)], + "decrypt/invalid ciphertext/mode=:/invalid ciphertext=reinterpretAsFixedString(toUUID:": [ + (Fail, issue_24029) + ], + "encrypt_mysql/encryption/mode=:datatype=UUID:": [(Fail, issue_24029)], + "decrypt_mysql/decryption/mode=:datatype=UUID:": [(Fail, issue_24029)], + "decrypt_mysql/invalid ciphertext/mode=:/invalid ciphertext=reinterpretAsFixedString(toUUID:": [ + (Fail, issue_24029) + ], } + @TestFeature @Name("aes encryption") @ArgumentParser(argparser) @Specifications(SRS_008_ClickHouse_AES_Encryption_Functions) @Requirements( - RQ_SRS008_AES_Functions("1.0"), - RQ_SRS008_AES_Functions_DifferentModes("1.0") + RQ_SRS008_AES_Functions("1.0"), RQ_SRS008_AES_Functions_DifferentModes("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path, stress=None): - """ClickHouse AES encryption functions regression module. - """ +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse AES encryption functions regression module.""" nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), } if stress is not None: self.context.stress = stress + self.context.clickhouse_version = clickhouse_version - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "aes_encryption_env")) as cluster: + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), "aes_encryption_env"), + ) as cluster: self.context.cluster = cluster with Pool(5) as pool: try: - Feature(run=load("aes_encryption.tests.encrypt", "feature"), flags=TE, parallel=True, executor=pool) - Feature(run=load("aes_encryption.tests.decrypt", "feature"), flags=TE, parallel=True, executor=pool) - Feature(run=load("aes_encryption.tests.encrypt_mysql", "feature"), flags=TE, parallel=True, executor=pool) - Feature(run=load("aes_encryption.tests.decrypt_mysql", "feature"), flags=TE, parallel=True, executor=pool) - Feature(run=load("aes_encryption.tests.compatibility.feature", "feature"), flags=TE, parallel=True, executor=pool) + Feature( + run=load("aes_encryption.tests.encrypt", "feature"), + flags=TE, + parallel=True, + executor=pool, + ) + Feature( + run=load("aes_encryption.tests.decrypt", "feature"), + flags=TE, + parallel=True, + executor=pool, + ) + Feature( + run=load("aes_encryption.tests.encrypt_mysql", "feature"), + flags=TE, + parallel=True, + executor=pool, + ) + Feature( + run=load("aes_encryption.tests.decrypt_mysql", "feature"), + flags=TE, + parallel=True, + executor=pool, + ) + Feature( + run=load("aes_encryption.tests.compatibility.feature", "feature"), + flags=TE, + parallel=True, + executor=pool, + ) finally: join() + if main(): regression() diff --git a/tests/testflows/aes_encryption/requirements/requirements.py b/tests/testflows/aes_encryption/requirements/requirements.py index 22259aef65e..0fbbea7e85a 100644 --- a/tests/testflows/aes_encryption/requirements/requirements.py +++ b/tests/testflows/aes_encryption/requirements/requirements.py @@ -9,1699 +9,1782 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS008_AES_Functions = Requirement( - name='RQ.SRS008.AES.Functions', - version='1.0', + name="RQ.SRS008.AES.Functions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n' - '\n' - ), + "[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n" + "\n" + ), link=None, level=3, - num='4.1.1') + num="4.1.1", +) RQ_SRS008_AES_Functions_Compatibility_MySQL = Requirement( - name='RQ.SRS008.AES.Functions.Compatibility.MySQL', - version='1.0', + name="RQ.SRS008.AES.Functions.Compatibility.MySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n' - '\n' - ), + "[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n" + "\n" + ), link=None, level=3, - num='4.2.1') + num="4.2.1", +) RQ_SRS008_AES_Functions_Compatibility_Dictionaries = Requirement( - name='RQ.SRS008.AES.Functions.Compatibility.Dictionaries', - version='1.0', + name="RQ.SRS008.AES.Functions.Compatibility.Dictionaries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n' - '[MySQL] servers using [MySQL Dictionary].\n' - '\n' - ), + "[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n" + "[MySQL] servers using [MySQL Dictionary].\n" + "\n" + ), link=None, level=3, - num='4.2.2') + num="4.2.2", +) RQ_SRS008_AES_Functions_Compatibility_Engine_Database_MySQL = Requirement( - name='RQ.SRS008.AES.Functions.Compatibility.Engine.Database.MySQL', - version='1.0', + name="RQ.SRS008.AES.Functions.Compatibility.Engine.Database.MySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n' - '\n' - ), + "[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n" + "\n" + ), link=None, level=3, - num='4.2.3') + num="4.2.3", +) RQ_SRS008_AES_Functions_Compatibility_Engine_Table_MySQL = Requirement( - name='RQ.SRS008.AES.Functions.Compatibility.Engine.Table.MySQL', - version='1.0', + name="RQ.SRS008.AES.Functions.Compatibility.Engine.Table.MySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n' - '\n' - ), + "[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n" + "\n" + ), link=None, level=3, - num='4.2.4') + num="4.2.4", +) RQ_SRS008_AES_Functions_Compatibility_TableFunction_MySQL = Requirement( - name='RQ.SRS008.AES.Functions.Compatibility.TableFunction.MySQL', - version='1.0', + name="RQ.SRS008.AES.Functions.Compatibility.TableFunction.MySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n' - '\n' - ), + "[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n" + "\n" + ), link=None, level=3, - num='4.2.5') + num="4.2.5", +) RQ_SRS008_AES_Functions_DifferentModes = Requirement( - name='RQ.SRS008.AES.Functions.DifferentModes', - version='1.0', + name="RQ.SRS008.AES.Functions.DifferentModes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n' - 'using explicit function parameters.\n' - '\n' - ), + "[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n" + "using explicit function parameters.\n" + "\n" + ), link=None, level=3, - num='4.3.1') + num="4.3.1", +) RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( - name='RQ.SRS008.AES.Functions.DataFromMultipleSources', - version='1.0', + name="RQ.SRS008.AES.Functions.DataFromMultipleSources", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources\n' - 'in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n' - '[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n' - 'with possibly different encryption schemes.\n' - '\n' - ), + "[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources\n" + "in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n" + "[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n" + "with possibly different encryption schemes.\n" + "\n" + ), link=None, level=3, - num='4.4.1') + num="4.4.1", +) RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( - name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', - version='1.0', + name="RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n' - 'error log, and `query_log` table to prevent leakage of sensitive values.\n' - '\n' - ), + "[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n" + "error log, and `query_log` table to prevent leakage of sensitive values.\n" + "\n" + ), link=None, level=3, - num='4.5.1') + num="4.5.1", +) RQ_SRS008_AES_Functions_InvalidParameters = Requirement( - name='RQ.SRS008.AES.Functions.InvalidParameters', - version='1.0', + name="RQ.SRS008.AES.Functions.InvalidParameters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when parameters are invalid.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when parameters are invalid.\n" "\n" + ), link=None, level=3, - num='4.6.1') + num="4.6.1", +) RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( - name='RQ.SRS008.AES.Functions.Mismatched.Key', - version='1.0', + name="RQ.SRS008.AES.Functions.Mismatched.Key", + version="1.0", priority=None, group=None, type=None, uid=None, - description=( - '[ClickHouse] SHALL return garbage for mismatched keys.\n' - '\n' - ), + description=("[ClickHouse] SHALL return garbage for mismatched keys.\n" "\n"), link=None, level=3, - num='4.7.1') + num="4.7.1", +) RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( - name='RQ.SRS008.AES.Functions.Mismatched.IV', - version='1.0', + name="RQ.SRS008.AES.Functions.Mismatched.IV", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n' - '\n' - ), + "[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n" + "\n" + ), link=None, level=3, - num='4.7.2') + num="4.7.2", +) RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( - name='RQ.SRS008.AES.Functions.Mismatched.AAD', - version='1.0', + name="RQ.SRS008.AES.Functions.Mismatched.AAD", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n' - '\n' - ), + "[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n" + "\n" + ), link=None, level=3, - num='4.7.3') + num="4.7.3", +) RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( - name='RQ.SRS008.AES.Functions.Mismatched.Mode', - version='1.0', + name="RQ.SRS008.AES.Functions.Mismatched.Mode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error or garbage for mismatched mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error or garbage for mismatched mode.\n" "\n" + ), link=None, level=3, - num='4.7.4') + num="4.7.4", +) RQ_SRS008_AES_Functions_Check_Performance = Requirement( - name='RQ.SRS008.AES.Functions.Check.Performance', - version='1.0', + name="RQ.SRS008.AES.Functions.Check.Performance", + version="1.0", priority=None, group=None, type=None, uid=None, - description=( - 'Performance of [AES] encryption functions SHALL be measured.\n' - '\n' - ), + description=("Performance of [AES] encryption functions SHALL be measured.\n" "\n"), link=None, level=3, - num='4.8.1') + num="4.8.1", +) RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( - name='RQ.SRS008.AES.Function.Check.Performance.BestCase', - version='1.0', + name="RQ.SRS008.AES.Function.Check.Performance.BestCase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - 'Performance of [AES] encryption functions SHALL be checked for the best case\n' - 'scenario where there is one key, one initialization vector, and one large stream of data.\n' - '\n' - ), + "Performance of [AES] encryption functions SHALL be checked for the best case\n" + "scenario where there is one key, one initialization vector, and one large stream of data.\n" + "\n" + ), link=None, level=3, - num='4.8.2') + num="4.8.2", +) RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( - name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', - version='1.0', + name="RQ.SRS008.AES.Function.Check.Performance.WorstCase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - 'Performance of [AES] encryption functions SHALL be checked for the worst case\n' - 'where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n' - '\n' - ), + "Performance of [AES] encryption functions SHALL be checked for the worst case\n" + "where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n" + "\n" + ), link=None, level=3, - num='4.8.3') + num="4.8.3", +) RQ_SRS008_AES_Functions_Check_Compression = Requirement( - name='RQ.SRS008.AES.Functions.Check.Compression', - version='1.0', + name="RQ.SRS008.AES.Functions.Check.Compression", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - 'Effect of [AES] encryption on column compression SHALL be measured.\n' - '\n' - ), + "Effect of [AES] encryption on column compression SHALL be measured.\n" "\n" + ), link=None, level=3, - num='4.8.4') + num="4.8.4", +) RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( - name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', - version='1.0', + name="RQ.SRS008.AES.Functions.Check.Compression.LowCardinality", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - 'Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n' - 'SHALL be measured.\n' - '\n' - ), + "Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n" + "SHALL be measured.\n" + "\n" + ), link=None, level=3, - num='4.8.5') + num="4.8.5", +) RQ_SRS008_AES_Encrypt_Function = Requirement( - name='RQ.SRS008.AES.Encrypt.Function', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `encrypt` function to encrypt data using [AES].\n' - '\n' - ), + "[ClickHouse] SHALL support `encrypt` function to encrypt data using [AES].\n" + "\n" + ), link=None, level=3, - num='4.9.1') + num="4.9.1", +) RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Syntax', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `encrypt` function\n' - '\n' - '```sql\n' - 'encrypt(mode, plaintext, key, [iv, aad])\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `encrypt` function\n" + "\n" + "```sql\n" + "encrypt(mode, plaintext, key, [iv, aad])\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.9.2') + num="4.9.2", +) RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] `encrypt` function output SHALL produce output that matches [NIST test vectors].\n' - '\n' - ), + "[ClickHouse] `encrypt` function output SHALL produce output that matches [NIST test vectors].\n" + "\n" + ), link=None, level=3, - num='4.9.3') + num="4.9.3", +) RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', - version='2.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `plaintext` with `String`, `FixedString`, `Nullable(String)`,\n' - '`Nullable(FixedString)`, `LowCardinality(String)`, or `LowCardinality(FixedString(N))` data types as\n' - 'the second parameter to the `encrypt` function that SHALL specify the data to be encrypted.\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL support `plaintext` with `String`, `FixedString`, `Nullable(String)`,\n" + "`Nullable(FixedString)`, `LowCardinality(String)`, or `LowCardinality(FixedString(N))` data types as\n" + "the second parameter to the `encrypt` function that SHALL specify the data to be encrypted.\n" + "\n" + "\n" + ), link=None, level=3, - num='4.9.4') + num="4.9.4", +) RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Key", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' - 'as the parameter to the `encrypt` function that SHALL specify the encryption key.\n' - '\n' - ), + "[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n" + "as the parameter to the `encrypt` function that SHALL specify the encryption key.\n" + "\n" + ), link=None, level=3, - num='4.9.5') + num="4.9.5", +) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n' - 'to the `encrypt` function that SHALL specify encryption key length and block encryption mode.\n' - '\n' - ), + "[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n" + "to the `encrypt` function that SHALL specify encryption key length and block encryption mode.\n" + "\n" + ), link=None, level=3, - num='4.9.6') + num="4.9.6", +) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' - 'of the `encrypt` function where\n' - 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' - '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' - 'mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB] as well as\n' - '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' - '\n' - ), + "[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n" + "of the `encrypt` function where\n" + "the `key_length` SHALL specifies the length of the key and SHALL accept\n" + "`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n" + "mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB] as well as\n" + "[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n" + "\n" + ), link=None, level=3, - num='4.9.7') + num="4.9.7", +) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `encrypt`\n' - 'function is not valid with the exception where such a mode is supported by the underlying\n' - '[OpenSSL] implementation.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `encrypt`\n" + "function is not valid with the exception where such a mode is supported by the underlying\n" + "[OpenSSL] implementation.\n" + "\n" + ), link=None, level=3, - num='4.9.8') + num="4.9.8", +) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Values = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Values', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n' - 'of the `encrypt` function:\n' - '\n' - '* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n' - '* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n' - '* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n' - '* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n' - '* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n' - '* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n' - '* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n' - '* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n' - '* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n' - '* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n' - '* `aes-128-gcm` that SHALL use [GCM] block mode encryption with 128 bit key\n' - ' and [AEAD] 16-byte tag is appended to the resulting ciphertext according to\n' - ' the [RFC5116]\n' - '* `aes-192-gcm` that SHALL use [GCM] block mode encryption with 192 bit key\n' - ' and [AEAD] 16-byte tag is appended to the resulting ciphertext according to\n' - ' the [RFC5116]\n' - '* `aes-256-gcm` that SHALL use [GCM] block mode encryption with 256 bit key\n' - ' and [AEAD] 16-byte tag is appended to the resulting ciphertext according to\n' - ' the [RFC5116]\n' - '* `aes-128-ctr` that SHALL use [CTR] block mode encryption with 128 bit key\n' - '* `aes-192-ctr` that SHALL use [CTR] block mode encryption with 192 bit key\n' - '* `aes-256-ctr` that SHALL use [CTR] block mode encryption with 256 bit key\n' - '\n' - ), + "[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n" + "of the `encrypt` function:\n" + "\n" + "* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n" + "* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n" + "* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n" + "* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n" + "* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n" + "* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n" + "* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n" + "* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n" + "* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n" + "* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n" + "* `aes-128-gcm` that SHALL use [GCM] block mode encryption with 128 bit key\n" + " and [AEAD] 16-byte tag is appended to the resulting ciphertext according to\n" + " the [RFC5116]\n" + "* `aes-192-gcm` that SHALL use [GCM] block mode encryption with 192 bit key\n" + " and [AEAD] 16-byte tag is appended to the resulting ciphertext according to\n" + " the [RFC5116]\n" + "* `aes-256-gcm` that SHALL use [GCM] block mode encryption with 256 bit key\n" + " and [AEAD] 16-byte tag is appended to the resulting ciphertext according to\n" + " the [RFC5116]\n" + "* `aes-128-ctr` that SHALL use [CTR] block mode encryption with 128 bit key\n" + "* `aes-192-ctr` that SHALL use [CTR] block mode encryption with 192 bit key\n" + "* `aes-256-ctr` that SHALL use [CTR] block mode encryption with 256 bit key\n" + "\n" + ), link=None, level=3, - num='4.9.9') + num="4.9.9", +) RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' - 'parameter to the `encrypt` function that SHALL specify the initialization vector for block modes that require\n' - 'it.\n' - '\n' - ), + "[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n" + "parameter to the `encrypt` function that SHALL specify the initialization vector for block modes that require\n" + "it.\n" + "\n" + ), link=None, level=3, - num='4.9.10') + num="4.9.10", +) RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' - 'parameter to the `encrypt` function that SHALL specify the additional authenticated data\n' - 'for block modes that require it.\n' - '\n' - ), + "[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n" + "parameter to the `encrypt` function that SHALL specify the additional authenticated data\n" + "for block modes that require it.\n" + "\n" + ), link=None, level=3, - num='4.9.11') + num="4.9.11", +) RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return the encrypted value of the data\n' - 'using `String` data type as the result of `encrypt` function.\n' - '\n' - ), + "[ClickHouse] SHALL return the encrypted value of the data\n" + "using `String` data type as the result of `encrypt` function.\n" + "\n" + ), link=None, level=3, - num='4.9.12') + num="4.9.12", +) RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `key` length is not exact for the `encrypt` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `key` length is not exact for the `encrypt` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.9.13') + num="4.9.13", +) RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `encrypt` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `encrypt` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.9.14') + num="4.9.14", +) RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` is specified for the `encrypt` function for a mode that does not need it.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` is specified for the `encrypt` function for a mode that does not need it.\n" + "\n" + ), link=None, level=3, - num='4.9.15') + num="4.9.15", +) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `aad` is specified for the `encrypt` function for a mode that does not need it.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `aad` is specified for the `encrypt` function for a mode that does not need it.\n" + "\n" + ), link=None, level=3, - num='4.9.16') + num="4.9.16", +) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `encrypt` function.\n' - '\n' - ), + "[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `encrypt` function.\n" + "\n" + ), link=None, level=3, - num='4.9.17') + num="4.9.17", +) RQ_SRS008_AES_Encrypt_Function_NonGCMMode_KeyAndInitializationVector_Length = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.NonGCMMode.KeyAndInitializationVector.Length', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.NonGCMMode.KeyAndInitializationVector.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the `encrypt` function is called with the following parameter values\n' - 'when using non-GCM modes\n' - '\n' - '* `aes-128-ecb` mode and `key` is not 16 bytes or `iv` or `aad` is specified\n' - '* `aes-192-ecb` mode and `key` is not 24 bytes or `iv` or `aad` is specified\n' - '* `aes-256-ecb` mode and `key` is not 32 bytes or `iv` or `aad` is specified\n' - '* `aes-128-cbc` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-cbc` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cbc` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-cfb1` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-cfb1` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cfb1` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-cfb8` mode and `key` is not 16 bytes and if specified `iv` is not 16 bytes\n' - '* `aes-192-cfb8` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cfb8` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-cfb128` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-cfb128` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cfb128` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-ofb` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-ofb` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-ofb` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-ctr` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes\n' - '* `aes-192-ctr` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes\n' - '* `aes-256-ctr` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the `encrypt` function is called with the following parameter values\n" + "when using non-GCM modes\n" + "\n" + "* `aes-128-ecb` mode and `key` is not 16 bytes or `iv` or `aad` is specified\n" + "* `aes-192-ecb` mode and `key` is not 24 bytes or `iv` or `aad` is specified\n" + "* `aes-256-ecb` mode and `key` is not 32 bytes or `iv` or `aad` is specified\n" + "* `aes-128-cbc` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-cbc` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cbc` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-cfb1` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-cfb1` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cfb1` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-cfb8` mode and `key` is not 16 bytes and if specified `iv` is not 16 bytes\n" + "* `aes-192-cfb8` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cfb8` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-cfb128` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-cfb128` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cfb128` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-ofb` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-ofb` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-ofb` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-ctr` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes\n" + "* `aes-192-ctr` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes\n" + "* `aes-256-ctr` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes\n" + "\n" + ), link=None, level=3, - num='4.9.18') + num="4.9.18", +) RQ_SRS008_AES_Encrypt_Function_GCMMode_KeyAndInitializationVector_Length = Requirement( - name='RQ.SRS008.AES.Encrypt.Function.GCMMode.KeyAndInitializationVector.Length', - version='1.0', + name="RQ.SRS008.AES.Encrypt.Function.GCMMode.KeyAndInitializationVector.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the `encrypt` function is called with the following parameter values\n' - 'when using GCM modes\n' - '\n' - '* `aes-128-gcm` mode and `key` is not 16 bytes or `iv` is not specified\n' - '* `aes-192-gcm` mode and `key` is not 24 bytes or `iv` is not specified\n' - '* `aes-256-gcm` mode and `key` is not 32 bytes or `iv` is not specified\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the `encrypt` function is called with the following parameter values\n" + "when using GCM modes\n" + "\n" + "* `aes-128-gcm` mode and `key` is not 16 bytes or `iv` is not specified\n" + "* `aes-192-gcm` mode and `key` is not 24 bytes or `iv` is not specified\n" + "* `aes-256-gcm` mode and `key` is not 32 bytes or `iv` is not specified\n" + "\n" + ), link=None, level=3, - num='4.9.19') + num="4.9.19", +) RQ_SRS008_AES_Decrypt_Function = Requirement( - name='RQ.SRS008.AES.Decrypt.Function', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `decrypt` function to decrypt data using [AES].\n' - '\n' - ), + "[ClickHouse] SHALL support `decrypt` function to decrypt data using [AES].\n" + "\n" + ), link=None, level=3, - num='4.10.1') + num="4.10.1", +) RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Syntax', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `decrypt` function\n' - '\n' - '```sql\n' - 'decrypt(mode, ciphertext, key, [iv, aad])\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `decrypt` function\n" + "\n" + "```sql\n" + "decrypt(mode, ciphertext, key, [iv, aad])\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.10.2') + num="4.10.2", +) RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n' - 'the second parameter to the `decrypt` function that SHALL specify the data to be decrypted.\n' - '\n' - ), + "[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n" + "the second parameter to the `decrypt` function that SHALL specify the data to be decrypted.\n" + "\n" + ), link=None, level=3, - num='4.10.3') + num="4.10.3", +) RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Key", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' - 'as the third parameter to the `decrypt` function that SHALL specify the encryption key.\n' - '\n' - ), + "[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n" + "as the third parameter to the `decrypt` function that SHALL specify the encryption key.\n" + "\n" + ), link=None, level=3, - num='4.10.4') + num="4.10.4", +) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n' - 'to the `decrypt` function that SHALL specify encryption key length and block encryption mode.\n' - '\n' - ), + "[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n" + "to the `decrypt` function that SHALL specify encryption key length and block encryption mode.\n" + "\n" + ), link=None, level=3, - num='4.10.5') + num="4.10.5", +) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' - 'of the `decrypt` function where\n' - 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' - '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' - 'mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB] as well as\n' - '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' - '\n' - ), + "[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n" + "of the `decrypt` function where\n" + "the `key_length` SHALL specifies the length of the key and SHALL accept\n" + "`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n" + "mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB] as well as\n" + "[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n" + "\n" + ), link=None, level=3, - num='4.10.6') + num="4.10.6", +) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `decrypt`\n' - 'function is not valid with the exception where such a mode is supported by the underlying\n' - '[OpenSSL] implementation.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `decrypt`\n" + "function is not valid with the exception where such a mode is supported by the underlying\n" + "[OpenSSL] implementation.\n" + "\n" + ), link=None, level=3, - num='4.10.7') + num="4.10.7", +) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Values = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Values', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n' - 'of the `decrypt` function:\n' - '\n' - '* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n' - '* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n' - '* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n' - '* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n' - '* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n' - '* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n' - '* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n' - '* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n' - '* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n' - '* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n' - '* `aes-128-gcm` that SHALL use [GCM] block mode encryption with 128 bit key\n' - ' and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' - ' the [RFC5116]\n' - '* `aes-192-gcm` that SHALL use [GCM] block mode encryption with 192 bit key\n' - ' and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' - ' the [RFC5116]\n' - '* `aes-256-gcm` that SHALL use [GCM] block mode encryption with 256 bit key\n' - ' and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' - ' the [RFC5116]\n' - '* `aes-128-ctr` that SHALL use [CTR] block mode encryption with 128 bit key\n' - '* `aes-192-ctr` that SHALL use [CTR] block mode encryption with 192 bit key\n' - '* `aes-256-ctr` that SHALL use [CTR] block mode encryption with 256 bit key\n' - '\n' - ), + "[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n" + "of the `decrypt` function:\n" + "\n" + "* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n" + "* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n" + "* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n" + "* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n" + "* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n" + "* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n" + "* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n" + "* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n" + "* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n" + "* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n" + "* `aes-128-gcm` that SHALL use [GCM] block mode encryption with 128 bit key\n" + " and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n" + " the [RFC5116]\n" + "* `aes-192-gcm` that SHALL use [GCM] block mode encryption with 192 bit key\n" + " and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n" + " the [RFC5116]\n" + "* `aes-256-gcm` that SHALL use [GCM] block mode encryption with 256 bit key\n" + " and [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n" + " the [RFC5116]\n" + "* `aes-128-ctr` that SHALL use [CTR] block mode encryption with 128 bit key\n" + "* `aes-192-ctr` that SHALL use [CTR] block mode encryption with 192 bit key\n" + "* `aes-256-ctr` that SHALL use [CTR] block mode encryption with 256 bit key\n" + "\n" + ), link=None, level=3, - num='4.10.8') + num="4.10.8", +) RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' - 'parameter to the `decrypt` function that SHALL specify the initialization vector for block modes that require\n' - 'it.\n' - '\n' - ), + "[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n" + "parameter to the `decrypt` function that SHALL specify the initialization vector for block modes that require\n" + "it.\n" + "\n" + ), link=None, level=3, - num='4.10.9') + num="4.10.9", +) RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' - 'parameter to the `decrypt` function that SHALL specify the additional authenticated data\n' - 'for block modes that require it.\n' - '\n' - ), + "[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n" + "parameter to the `decrypt` function that SHALL specify the additional authenticated data\n" + "for block modes that require it.\n" + "\n" + ), link=None, level=3, - num='4.10.10') + num="4.10.10", +) RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return the decrypted value of the data\n' - 'using `String` data type as the result of `decrypt` function.\n' - '\n' - ), + "[ClickHouse] SHALL return the decrypted value of the data\n" + "using `String` data type as the result of `decrypt` function.\n" + "\n" + ), link=None, level=3, - num='4.10.11') + num="4.10.11", +) RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `key` length is not exact for the `decrypt` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `key` length is not exact for the `decrypt` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.10.12') + num="4.10.12", +) RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` is specified and the length is not exact for the `decrypt` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` is specified and the length is not exact for the `decrypt` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.10.13') + num="4.10.13", +) RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` is specified for the `decrypt` function\n' - 'for a mode that does not need it.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` is specified for the `decrypt` function\n" + "for a mode that does not need it.\n" + "\n" + ), link=None, level=3, - num='4.10.14') + num="4.10.14", +) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `aad` is specified for the `decrypt` function\n' - 'for a mode that does not need it.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `aad` is specified for the `decrypt` function\n" + "for a mode that does not need it.\n" + "\n" + ), link=None, level=3, - num='4.10.15') + num="4.10.15", +) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `decrypt` function.\n' - '\n' - ), + "[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `decrypt` function.\n" + "\n" + ), link=None, level=3, - num='4.10.16') + num="4.10.16", +) RQ_SRS008_AES_Decrypt_Function_NonGCMMode_KeyAndInitializationVector_Length = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.NonGCMMode.KeyAndInitializationVector.Length', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.NonGCMMode.KeyAndInitializationVector.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the `decrypt` function is called with the following parameter values\n' - 'when using non-GCM modes\n' - '\n' - '* `aes-128-ecb` mode and `key` is not 16 bytes or `iv` or `aad` is specified\n' - '* `aes-192-ecb` mode and `key` is not 24 bytes or `iv` or `aad` is specified\n' - '* `aes-256-ecb` mode and `key` is not 32 bytes or `iv` or `aad` is specified\n' - '* `aes-128-cbc` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-cbc` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cbc` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-cfb1` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-cfb1` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cfb1` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-cfb8` mode and `key` is not 16 bytes and if specified `iv` is not 16 bytes\n' - '* `aes-192-cfb8` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cfb8` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-cfb128` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-cfb128` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-cfb128` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-ofb` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-192-ofb` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-256-ofb` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n' - '* `aes-128-ctr` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes\n' - '* `aes-192-ctr` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes\n' - '* `aes-256-ctr` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the `decrypt` function is called with the following parameter values\n" + "when using non-GCM modes\n" + "\n" + "* `aes-128-ecb` mode and `key` is not 16 bytes or `iv` or `aad` is specified\n" + "* `aes-192-ecb` mode and `key` is not 24 bytes or `iv` or `aad` is specified\n" + "* `aes-256-ecb` mode and `key` is not 32 bytes or `iv` or `aad` is specified\n" + "* `aes-128-cbc` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-cbc` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cbc` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-cfb1` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-cfb1` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cfb1` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-cfb8` mode and `key` is not 16 bytes and if specified `iv` is not 16 bytes\n" + "* `aes-192-cfb8` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cfb8` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-cfb128` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-cfb128` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-cfb128` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-ofb` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-192-ofb` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-256-ofb` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes or `aad` is specified\n" + "* `aes-128-ctr` mode and `key` is not 16 bytes or if specified `iv` is not 16 bytes\n" + "* `aes-192-ctr` mode and `key` is not 24 bytes or if specified `iv` is not 16 bytes\n" + "* `aes-256-ctr` mode and `key` is not 32 bytes or if specified `iv` is not 16 bytes\n" + "\n" + ), link=None, level=3, - num='4.10.17') + num="4.10.17", +) RQ_SRS008_AES_Decrypt_Function_GCMMode_KeyAndInitializationVector_Length = Requirement( - name='RQ.SRS008.AES.Decrypt.Function.GCMMode.KeyAndInitializationVector.Length', - version='1.0', + name="RQ.SRS008.AES.Decrypt.Function.GCMMode.KeyAndInitializationVector.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the `decrypt` function is called with the following parameter values\n' - 'when using GCM modes\n' - '\n' - '* `aes-128-gcm` mode and `key` is not 16 bytes or `iv` is not specified\n' - '* `aes-192-gcm` mode and `key` is not 24 bytes or `iv` is not specified\n' - '* `aes-256-gcm` mode and `key` is not 32 bytes or `iv` is not specified\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the `decrypt` function is called with the following parameter values\n" + "when using GCM modes\n" + "\n" + "* `aes-128-gcm` mode and `key` is not 16 bytes or `iv` is not specified\n" + "* `aes-192-gcm` mode and `key` is not 24 bytes or `iv` is not specified\n" + "* `aes-256-gcm` mode and `key` is not 32 bytes or `iv` is not specified\n" + "\n" + ), link=None, level=3, - num='4.10.18') + num="4.10.18", +) RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n' - '\n' - ), + "[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n" + "\n" + ), link=None, level=3, - num='4.11.1') + num="4.11.1", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function\n' - '\n' - '```sql\n' - 'aes_encrypt_mysql(mode, plaintext, key, [iv])\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function\n" + "\n" + "```sql\n" + "aes_encrypt_mysql(mode, plaintext, key, [iv])\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.11.2') + num="4.11.2", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', - version='2.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `plaintext` with `String`, `FixedString`, `Nullable(String)`,\n' - '`Nullable(FixedString)`, `LowCardinality(String)`, or `LowCardinality(FixedString(N))` data types as\n' - 'the second parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n' - '\n' - ), + "[ClickHouse] SHALL support `plaintext` with `String`, `FixedString`, `Nullable(String)`,\n" + "`Nullable(FixedString)`, `LowCardinality(String)`, or `LowCardinality(FixedString(N))` data types as\n" + "the second parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n" + "\n" + ), link=None, level=3, - num='4.11.3') + num="4.11.3", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' - 'as the third parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n' - '\n' - ), + "[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n" + "as the third parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n" + "\n" + ), link=None, level=3, - num='4.11.4') + num="4.11.4", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n' - 'to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' - '\n' - ), + "[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n" + "to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n" + "\n" + ), link=None, level=3, - num='4.11.5') + num="4.11.5", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' - 'of the `aes_encrypt_mysql` function where\n' - 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' - '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' - 'mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' - '\n' - ), + "[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n" + "of the `aes_encrypt_mysql` function where\n" + "the `key_length` SHALL specifies the length of the key and SHALL accept\n" + "`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n" + "mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n" + "\n" + ), link=None, level=3, - num='4.11.6') + num="4.11.6", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n' - 'function is not valid with the exception where such a mode is supported by the underlying\n' - '[OpenSSL] implementation.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n" + "function is not valid with the exception where such a mode is supported by the underlying\n" + "[OpenSSL] implementation.\n" + "\n" + ), link=None, level=3, - num='4.11.7') + num="4.11.7", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n' - 'of the `aes_encrypt_mysql` function:\n' - '\n' - '* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n' - '* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n' - '* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n' - '* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n' - '* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n' - '* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n' - '* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n' - '* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n' - '* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n' - '* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n' - '\n' - ), + "[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n" + "of the `aes_encrypt_mysql` function:\n" + "\n" + "* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n" + "* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n" + "* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n" + "* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n" + "* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n" + "* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n" + "* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n" + "* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n" + "* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n" + "* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n" + "\n" + ), link=None, level=3, - num='4.11.8') + num="4.11.8", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_GCM_Error = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.GCM.Error', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.GCM.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if any of the following [GCM] modes are specified as the value \n' - 'for the `mode` parameter of the `aes_encrypt_mysql` function\n' - '\n' - '* `aes-128-gcm`\n' - '* `aes-192-gcm`\n' - '* `aes-256-gcm`\n' - '\n' - ), + "[ClickHouse] SHALL return an error if any of the following [GCM] modes are specified as the value \n" + "for the `mode` parameter of the `aes_encrypt_mysql` function\n" + "\n" + "* `aes-128-gcm`\n" + "* `aes-192-gcm`\n" + "* `aes-256-gcm`\n" + "\n" + ), link=None, level=3, - num='4.11.9') + num="4.11.9", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_CTR_Error = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.CTR.Error', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.CTR.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if any of the following [CTR] modes are specified as the value \n' - 'for the `mode` parameter of the `aes_encrypt_mysql` function\n' - '\n' - '* `aes-128-ctr`\n' - '* `aes-192-ctr`\n' - '* `aes-256-ctr`\n' - '\n' - ), + "[ClickHouse] SHALL return an error if any of the following [CTR] modes are specified as the value \n" + "for the `mode` parameter of the `aes_encrypt_mysql` function\n" + "\n" + "* `aes-128-ctr`\n" + "* `aes-192-ctr`\n" + "* `aes-256-ctr`\n" + "\n" + ), link=None, level=3, - num='4.11.10') + num="4.11.10", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' - 'parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' - 'it.\n' - '\n' - ), + "[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n" + "parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n" + "it.\n" + "\n" + ), link=None, level=3, - num='4.11.11') + num="4.11.11", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return the encrypted value of the data\n' - 'using `String` data type as the result of `aes_encrypt_mysql` function.\n' - '\n' - ), + "[ClickHouse] SHALL return the encrypted value of the data\n" + "using `String` data type as the result of `aes_encrypt_mysql` function.\n" + "\n" + ), link=None, level=3, - num='4.11.12') + num="4.11.12", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n' - 'function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n" + "function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.11.13') + num="4.11.13", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n' - 'for the `aes_encrypt_mysql` function for a given block mode.\n' - '\n' - '```python\n' - 'def fold_key(key, cipher_key_size):\n' - ' key = list(key) if not isinstance(key, (list, tuple)) else key\n' - '\t folded_key = key[:cipher_key_size]\n' - '\t for i in range(cipher_key_size, len(key)):\n' - '\t\t print(i % cipher_key_size, i)\n' - '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' - '\t return folded_key\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n" + "for the `aes_encrypt_mysql` function for a given block mode.\n" + "\n" + "```python\n" + "def fold_key(key, cipher_key_size):\n" + " key = list(key) if not isinstance(key, (list, tuple)) else key\n" + "\t folded_key = key[:cipher_key_size]\n" + "\t for i in range(cipher_key_size, len(key)):\n" + "\t\t print(i % cipher_key_size, i)\n" + "\t\t folded_key[i % cipher_key_size] ^= key[i]\n" + "\t return folded_key\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.11.14') + num="4.11.14", +) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' - 'that is required for the `aes_encrypt_mysql` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n" + "that is required for the `aes_encrypt_mysql` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.11.15') + num="4.11.15", +) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' - 'its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n" + "its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.11.16') + num="4.11.16", +) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n' - 'function for a mode that does not need it.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n" + "function for a mode that does not need it.\n" + "\n" + ), link=None, level=3, - num='4.11.17') + num="4.11.17", +) RQ_SRS008_AES_MySQL_Encrypt_Function_Mode_KeyAndInitializationVector_Length = Requirement( - name='RQ.SRS008.AES.MySQL.Encrypt.Function.Mode.KeyAndInitializationVector.Length', - version='1.0', + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Mode.KeyAndInitializationVector.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the `aes_encrypt_mysql` function is called with the following parameter values\n' - '\n' - '* `aes-128-ecb` mode and `key` is less than 16 bytes or `iv` is specified\n' - '* `aes-192-ecb` mode and `key` is less than 24 bytes or `iv` is specified\n' - '* `aes-256-ecb` mode and `key` is less than 32 bytes or `iv` is specified\n' - '* `aes-128-cbc` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-cbc` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cbc` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-cfb1` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-cfb1` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cfb1` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-cfb8` mode and `key` is less than 16 bytes and if specified `iv` is less than 16 bytes\n' - '* `aes-192-cfb8` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cfb8` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-cfb128` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-cfb128` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cfb128` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-ofb` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-ofb` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-ofb` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the `aes_encrypt_mysql` function is called with the following parameter values\n" + "\n" + "* `aes-128-ecb` mode and `key` is less than 16 bytes or `iv` is specified\n" + "* `aes-192-ecb` mode and `key` is less than 24 bytes or `iv` is specified\n" + "* `aes-256-ecb` mode and `key` is less than 32 bytes or `iv` is specified\n" + "* `aes-128-cbc` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-cbc` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cbc` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-cfb1` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-cfb1` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cfb1` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-cfb8` mode and `key` is less than 16 bytes and if specified `iv` is less than 16 bytes\n" + "* `aes-192-cfb8` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cfb8` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-cfb128` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-cfb128` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cfb128` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-ofb` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-ofb` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-ofb` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "\n" + ), link=None, level=3, - num='4.11.18') + num="4.11.18", +) RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n' - '\n' - ), + "[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n" + "\n" + ), link=None, level=3, - num='4.12.1') + num="4.12.1", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function\n' - '\n' - '```sql\n' - 'aes_decrypt_mysql(mode, ciphertext, key, [iv])\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function\n" + "\n" + "```sql\n" + "aes_decrypt_mysql(mode, ciphertext, key, [iv])\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.12.2') + num="4.12.2", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `ciphertext` accepting any data type as\n' - 'the second parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n' - '\n' - ), + "[ClickHouse] SHALL support `ciphertext` accepting any data type as\n" + "the second parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n" + "\n" + ), link=None, level=3, - num='4.12.3') + num="4.12.3", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' - 'as the third parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n' - '\n' - ), + "[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n" + "as the third parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n" + "\n" + ), link=None, level=3, - num='4.12.4') + num="4.12.4", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n' - 'to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' - '\n' - ), + "[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the first parameter\n" + "to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n" + "\n" + ), link=None, level=3, - num='4.12.5') + num="4.12.5", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' - 'of the `aes_decrypt_mysql` function where\n' - 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' - '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' - 'mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' - '\n' - ), + "[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n" + "of the `aes_decrypt_mysql` function where\n" + "the `key_length` SHALL specifies the length of the key and SHALL accept\n" + "`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n" + "mode and SHALL accept [ECB], [CBC], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n" + "\n" + ), link=None, level=3, - num='4.12.6') + num="4.12.6", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n' - 'function is not valid with the exception where such a mode is supported by the underlying\n' - '[OpenSSL] implementation.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n" + "function is not valid with the exception where such a mode is supported by the underlying\n" + "[OpenSSL] implementation.\n" + "\n" + ), link=None, level=3, - num='4.12.7') + num="4.12.7", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n' - 'of the `aes_decrypt_mysql` function:\n' - '\n' - '* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n' - '* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n' - '* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n' - '* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n' - '* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n' - '* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n' - '* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n' - '* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n' - '* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n' - '* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n' - '* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n' - '\n' - ), + "[ClickHouse] SHALL support the following [AES] block encryption modes as the value for the `mode` parameter\n" + "of the `aes_decrypt_mysql` function:\n" + "\n" + "* `aes-128-ecb` that SHALL use [ECB] block mode encryption with 128 bit key\n" + "* `aes-192-ecb` that SHALL use [ECB] block mode encryption with 192 bit key\n" + "* `aes-256-ecb` that SHALL use [ECB] block mode encryption with 256 bit key\n" + "* `aes-128-cbc` that SHALL use [CBC] block mode encryption with 128 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 192 bit key\n" + "* `aes-192-cbc` that SHALL use [CBC] block mode encryption with 256 bit key\n" + "* `aes-128-cfb128` that SHALL use [CFB128] block mode encryption with 128 bit key\n" + "* `aes-192-cfb128` that SHALL use [CFB128] block mode encryption with 192 bit key\n" + "* `aes-256-cfb128` that SHALL use [CFB128] block mode encryption with 256 bit key\n" + "* `aes-128-ofb` that SHALL use [OFB] block mode encryption with 128 bit key\n" + "* `aes-192-ofb` that SHALL use [OFB] block mode encryption with 192 bit key\n" + "* `aes-256-ofb` that SHALL use [OFB] block mode encryption with 256 bit key\n" + "\n" + ), link=None, level=3, - num='4.12.8') + num="4.12.8", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_GCM_Error = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.GCM.Error', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.GCM.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if any of the following [GCM] modes are specified as the value \n' - 'for the `mode` parameter of the `aes_decrypt_mysql` function\n' - '\n' - '* `aes-128-gcm`\n' - '* `aes-192-gcm`\n' - '* `aes-256-gcm`\n' - '\n' - ), + "[ClickHouse] SHALL return an error if any of the following [GCM] modes are specified as the value \n" + "for the `mode` parameter of the `aes_decrypt_mysql` function\n" + "\n" + "* `aes-128-gcm`\n" + "* `aes-192-gcm`\n" + "* `aes-256-gcm`\n" + "\n" + ), link=None, level=3, - num='4.12.9') + num="4.12.9", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_CTR_Error = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.CTR.Error', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.CTR.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if any of the following [CTR] modes are specified as the value \n' - 'for the `mode` parameter of the `aes_decrypt_mysql` function\n' - '\n' - '* `aes-128-ctr`\n' - '* `aes-192-ctr`\n' - '* `aes-256-ctr`\n' - '\n' - ), + "[ClickHouse] SHALL return an error if any of the following [CTR] modes are specified as the value \n" + "for the `mode` parameter of the `aes_decrypt_mysql` function\n" + "\n" + "* `aes-128-ctr`\n" + "* `aes-192-ctr`\n" + "* `aes-256-ctr`\n" + "\n" + ), link=None, level=3, - num='4.12.10') + num="4.12.10", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' - 'parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' - 'it.\n' - '\n' - ), + "[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n" + "parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n" + "it.\n" + "\n" + ), link=None, level=3, - num='4.12.11') + num="4.12.11", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return the decrypted value of the data\n' - 'using `String` data type as the result of `aes_decrypt_mysql` function.\n' - '\n' - ), + "[ClickHouse] SHALL return the decrypted value of the data\n" + "using `String` data type as the result of `aes_decrypt_mysql` function.\n" + "\n" + ), link=None, level=3, - num='4.12.12') + num="4.12.12", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n' - 'function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n" + "function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.12.13') + num="4.12.13", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n' - 'for the `aes_decrypt_mysql` function for a given block mode.\n' - '\n' - '```python\n' - 'def fold_key(key, cipher_key_size):\n' - ' key = list(key) if not isinstance(key, (list, tuple)) else key\n' - '\t folded_key = key[:cipher_key_size]\n' - '\t for i in range(cipher_key_size, len(key)):\n' - '\t\t print(i % cipher_key_size, i)\n' - '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' - '\t return folded_key\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n" + "for the `aes_decrypt_mysql` function for a given block mode.\n" + "\n" + "```python\n" + "def fold_key(key, cipher_key_size):\n" + " key = list(key) if not isinstance(key, (list, tuple)) else key\n" + "\t folded_key = key[:cipher_key_size]\n" + "\t for i in range(cipher_key_size, len(key)):\n" + "\t\t print(i % cipher_key_size, i)\n" + "\t\t folded_key[i % cipher_key_size] ^= key[i]\n" + "\t return folded_key\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.12.14') + num="4.12.14", +) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' - 'that is required for the `aes_decrypt_mysql` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n" + "that is required for the `aes_decrypt_mysql` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.12.15') + num="4.12.15", +) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' - 'its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n' - '\n' - ), + "[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n" + "its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n" + "\n" + ), link=None, level=3, - num='4.12.16') + num="4.12.16", +) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n' - 'function for a mode that does not need it.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n" + "function for a mode that does not need it.\n" + "\n" + ), link=None, level=3, - num='4.12.17') + num="4.12.17", +) RQ_SRS008_AES_MySQL_Decrypt_Function_Mode_KeyAndInitializationVector_Length = Requirement( - name='RQ.SRS008.AES.MySQL.Decrypt.Function.Mode.KeyAndInitializationVector.Length', - version='1.0', + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Mode.KeyAndInitializationVector.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the `aes_decrypt_mysql` function is called with the following parameter values\n' - '\n' - '* `aes-128-ecb` mode and `key` is less than 16 bytes or `iv` is specified\n' - '* `aes-192-ecb` mode and `key` is less than 24 bytes or `iv` is specified\n' - '* `aes-256-ecb` mode and `key` is less than 32 bytes or `iv` is specified\n' - '* `aes-128-cbc` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-cbc` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cbc` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-cfb1` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-cfb1` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cfb1` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-cfb8` mode and `key` is less than 16 bytes and if specified `iv` is less than 16 bytes\n' - '* `aes-192-cfb8` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cfb8` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-cfb128` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-cfb128` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-cfb128` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-128-ofb` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-192-ofb` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n' - '* `aes-256-ofb` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the `aes_decrypt_mysql` function is called with the following parameter values\n" + "\n" + "* `aes-128-ecb` mode and `key` is less than 16 bytes or `iv` is specified\n" + "* `aes-192-ecb` mode and `key` is less than 24 bytes or `iv` is specified\n" + "* `aes-256-ecb` mode and `key` is less than 32 bytes or `iv` is specified\n" + "* `aes-128-cbc` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-cbc` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cbc` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-cfb1` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-cfb1` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cfb1` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-cfb8` mode and `key` is less than 16 bytes and if specified `iv` is less than 16 bytes\n" + "* `aes-192-cfb8` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cfb8` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-cfb128` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-cfb128` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-cfb128` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-128-ofb` mode and `key` is less than 16 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-192-ofb` mode and `key` is less than 24 bytes or if specified `iv` is less than 16 bytes\n" + "* `aes-256-ofb` mode and `key` is less than 32 bytes or if specified `iv` is less than 16 bytes\n" + "\n" + ), link=None, level=3, - num='4.12.18') + num="4.12.18", +) SRS_008_ClickHouse_AES_Encryption_Functions = Specification( - name='SRS-008 ClickHouse AES Encryption Functions', + name="SRS-008 ClickHouse AES Encryption Functions", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -1713,118 +1796,408 @@ SRS_008_ClickHouse_AES_Encryption_Functions = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='AES', level=2, num='3.1'), - Heading(name='AEAD', level=2, num='3.2'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='Generic', level=2, num='4.1'), - Heading(name='RQ.SRS008.AES.Functions', level=3, num='4.1.1'), - Heading(name='Compatibility', level=2, num='4.2'), - Heading(name='RQ.SRS008.AES.Functions.Compatibility.MySQL', level=3, num='4.2.1'), - Heading(name='RQ.SRS008.AES.Functions.Compatibility.Dictionaries', level=3, num='4.2.2'), - Heading(name='RQ.SRS008.AES.Functions.Compatibility.Engine.Database.MySQL', level=3, num='4.2.3'), - Heading(name='RQ.SRS008.AES.Functions.Compatibility.Engine.Table.MySQL', level=3, num='4.2.4'), - Heading(name='RQ.SRS008.AES.Functions.Compatibility.TableFunction.MySQL', level=3, num='4.2.5'), - Heading(name='Different Modes', level=2, num='4.3'), - Heading(name='RQ.SRS008.AES.Functions.DifferentModes', level=3, num='4.3.1'), - Heading(name='Multiple Sources', level=2, num='4.4'), - Heading(name='RQ.SRS008.AES.Functions.DataFromMultipleSources', level=3, num='4.4.1'), - Heading(name='Suppressing Sensitive Values', level=2, num='4.5'), - Heading(name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', level=3, num='4.5.1'), - Heading(name='Invalid Parameters', level=2, num='4.6'), - Heading(name='RQ.SRS008.AES.Functions.InvalidParameters', level=3, num='4.6.1'), - Heading(name='Mismatched Values', level=2, num='4.7'), - Heading(name='RQ.SRS008.AES.Functions.Mismatched.Key', level=3, num='4.7.1'), - Heading(name='RQ.SRS008.AES.Functions.Mismatched.IV', level=3, num='4.7.2'), - Heading(name='RQ.SRS008.AES.Functions.Mismatched.AAD', level=3, num='4.7.3'), - Heading(name='RQ.SRS008.AES.Functions.Mismatched.Mode', level=3, num='4.7.4'), - Heading(name='Performance', level=2, num='4.8'), - Heading(name='RQ.SRS008.AES.Functions.Check.Performance', level=3, num='4.8.1'), - Heading(name='RQ.SRS008.AES.Function.Check.Performance.BestCase', level=3, num='4.8.2'), - Heading(name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', level=3, num='4.8.3'), - Heading(name='RQ.SRS008.AES.Functions.Check.Compression', level=3, num='4.8.4'), - Heading(name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', level=3, num='4.8.5'), - Heading(name='Encrypt Function', level=2, num='4.9'), - Heading(name='RQ.SRS008.AES.Encrypt.Function', level=3, num='4.9.1'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Syntax', level=3, num='4.9.2'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', level=3, num='4.9.3'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', level=3, num='4.9.4'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', level=3, num='4.9.5'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', level=3, num='4.9.6'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', level=3, num='4.9.7'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', level=3, num='4.9.8'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Values', level=3, num='4.9.9'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', level=3, num='4.9.10'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', level=3, num='4.9.11'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', level=3, num='4.9.12'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', level=3, num='4.9.13'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', level=3, num='4.9.14'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', level=3, num='4.9.15'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', level=3, num='4.9.16'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', level=3, num='4.9.17'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.NonGCMMode.KeyAndInitializationVector.Length', level=3, num='4.9.18'), - Heading(name='RQ.SRS008.AES.Encrypt.Function.GCMMode.KeyAndInitializationVector.Length', level=3, num='4.9.19'), - Heading(name='Decrypt Function', level=2, num='4.10'), - Heading(name='RQ.SRS008.AES.Decrypt.Function', level=3, num='4.10.1'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Syntax', level=3, num='4.10.2'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', level=3, num='4.10.3'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', level=3, num='4.10.4'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', level=3, num='4.10.5'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', level=3, num='4.10.6'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', level=3, num='4.10.7'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Values', level=3, num='4.10.8'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', level=3, num='4.10.9'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', level=3, num='4.10.10'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', level=3, num='4.10.11'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', level=3, num='4.10.12'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', level=3, num='4.10.13'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', level=3, num='4.10.14'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', level=3, num='4.10.15'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', level=3, num='4.10.16'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.NonGCMMode.KeyAndInitializationVector.Length', level=3, num='4.10.17'), - Heading(name='RQ.SRS008.AES.Decrypt.Function.GCMMode.KeyAndInitializationVector.Length', level=3, num='4.10.18'), - Heading(name='MySQL Encrypt Function', level=2, num='4.11'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function', level=3, num='4.11.1'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', level=3, num='4.11.2'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', level=3, num='4.11.3'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', level=3, num='4.11.4'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', level=3, num='4.11.5'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', level=3, num='4.11.6'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', level=3, num='4.11.7'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values', level=3, num='4.11.8'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.GCM.Error', level=3, num='4.11.9'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.CTR.Error', level=3, num='4.11.10'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', level=3, num='4.11.11'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', level=3, num='4.11.12'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', level=3, num='4.11.13'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', level=3, num='4.11.14'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', level=3, num='4.11.15'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', level=3, num='4.11.16'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', level=3, num='4.11.17'), - Heading(name='RQ.SRS008.AES.MySQL.Encrypt.Function.Mode.KeyAndInitializationVector.Length', level=3, num='4.11.18'), - Heading(name='MySQL Decrypt Function', level=2, num='4.12'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function', level=3, num='4.12.1'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', level=3, num='4.12.2'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', level=3, num='4.12.3'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', level=3, num='4.12.4'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', level=3, num='4.12.5'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', level=3, num='4.12.6'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', level=3, num='4.12.7'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values', level=3, num='4.12.8'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.GCM.Error', level=3, num='4.12.9'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.CTR.Error', level=3, num='4.12.10'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', level=3, num='4.12.11'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', level=3, num='4.12.12'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', level=3, num='4.12.13'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', level=3, num='4.12.14'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', level=3, num='4.12.15'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', level=3, num='4.12.16'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', level=3, num='4.12.17'), - Heading(name='RQ.SRS008.AES.MySQL.Decrypt.Function.Mode.KeyAndInitializationVector.Length', level=3, num='4.12.18'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="AES", level=2, num="3.1"), + Heading(name="AEAD", level=2, num="3.2"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="Generic", level=2, num="4.1"), + Heading(name="RQ.SRS008.AES.Functions", level=3, num="4.1.1"), + Heading(name="Compatibility", level=2, num="4.2"), + Heading( + name="RQ.SRS008.AES.Functions.Compatibility.MySQL", level=3, num="4.2.1" ), + Heading( + name="RQ.SRS008.AES.Functions.Compatibility.Dictionaries", + level=3, + num="4.2.2", + ), + Heading( + name="RQ.SRS008.AES.Functions.Compatibility.Engine.Database.MySQL", + level=3, + num="4.2.3", + ), + Heading( + name="RQ.SRS008.AES.Functions.Compatibility.Engine.Table.MySQL", + level=3, + num="4.2.4", + ), + Heading( + name="RQ.SRS008.AES.Functions.Compatibility.TableFunction.MySQL", + level=3, + num="4.2.5", + ), + Heading(name="Different Modes", level=2, num="4.3"), + Heading(name="RQ.SRS008.AES.Functions.DifferentModes", level=3, num="4.3.1"), + Heading(name="Multiple Sources", level=2, num="4.4"), + Heading( + name="RQ.SRS008.AES.Functions.DataFromMultipleSources", level=3, num="4.4.1" + ), + Heading(name="Suppressing Sensitive Values", level=2, num="4.5"), + Heading( + name="RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues", + level=3, + num="4.5.1", + ), + Heading(name="Invalid Parameters", level=2, num="4.6"), + Heading(name="RQ.SRS008.AES.Functions.InvalidParameters", level=3, num="4.6.1"), + Heading(name="Mismatched Values", level=2, num="4.7"), + Heading(name="RQ.SRS008.AES.Functions.Mismatched.Key", level=3, num="4.7.1"), + Heading(name="RQ.SRS008.AES.Functions.Mismatched.IV", level=3, num="4.7.2"), + Heading(name="RQ.SRS008.AES.Functions.Mismatched.AAD", level=3, num="4.7.3"), + Heading(name="RQ.SRS008.AES.Functions.Mismatched.Mode", level=3, num="4.7.4"), + Heading(name="Performance", level=2, num="4.8"), + Heading(name="RQ.SRS008.AES.Functions.Check.Performance", level=3, num="4.8.1"), + Heading( + name="RQ.SRS008.AES.Function.Check.Performance.BestCase", + level=3, + num="4.8.2", + ), + Heading( + name="RQ.SRS008.AES.Function.Check.Performance.WorstCase", + level=3, + num="4.8.3", + ), + Heading(name="RQ.SRS008.AES.Functions.Check.Compression", level=3, num="4.8.4"), + Heading( + name="RQ.SRS008.AES.Functions.Check.Compression.LowCardinality", + level=3, + num="4.8.5", + ), + Heading(name="Encrypt Function", level=2, num="4.9"), + Heading(name="RQ.SRS008.AES.Encrypt.Function", level=3, num="4.9.1"), + Heading(name="RQ.SRS008.AES.Encrypt.Function.Syntax", level=3, num="4.9.2"), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors", level=3, num="4.9.3" + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText", + level=3, + num="4.9.4", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Key", level=3, num="4.9.5" + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode", level=3, num="4.9.6" + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat", + level=3, + num="4.9.7", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid", + level=3, + num="4.9.8", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Values", + level=3, + num="4.9.9", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector", + level=3, + num="4.9.10", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData", + level=3, + num="4.9.11", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue", + level=3, + num="4.9.12", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError", + level=3, + num="4.9.13", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError", + level=3, + num="4.9.14", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode", + level=3, + num="4.9.15", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode", + level=3, + num="4.9.16", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length", + level=3, + num="4.9.17", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.NonGCMMode.KeyAndInitializationVector.Length", + level=3, + num="4.9.18", + ), + Heading( + name="RQ.SRS008.AES.Encrypt.Function.GCMMode.KeyAndInitializationVector.Length", + level=3, + num="4.9.19", + ), + Heading(name="Decrypt Function", level=2, num="4.10"), + Heading(name="RQ.SRS008.AES.Decrypt.Function", level=3, num="4.10.1"), + Heading(name="RQ.SRS008.AES.Decrypt.Function.Syntax", level=3, num="4.10.2"), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText", + level=3, + num="4.10.3", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Key", level=3, num="4.10.4" + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode", level=3, num="4.10.5" + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat", + level=3, + num="4.10.6", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid", + level=3, + num="4.10.7", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Values", + level=3, + num="4.10.8", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector", + level=3, + num="4.10.9", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData", + level=3, + num="4.10.10", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue", + level=3, + num="4.10.11", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError", + level=3, + num="4.10.12", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError", + level=3, + num="4.10.13", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode", + level=3, + num="4.10.14", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode", + level=3, + num="4.10.15", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length", + level=3, + num="4.10.16", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.NonGCMMode.KeyAndInitializationVector.Length", + level=3, + num="4.10.17", + ), + Heading( + name="RQ.SRS008.AES.Decrypt.Function.GCMMode.KeyAndInitializationVector.Length", + level=3, + num="4.10.18", + ), + Heading(name="MySQL Encrypt Function", level=2, num="4.11"), + Heading(name="RQ.SRS008.AES.MySQL.Encrypt.Function", level=3, num="4.11.1"), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax", level=3, num="4.11.2" + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText", + level=3, + num="4.11.3", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key", + level=3, + num="4.11.4", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode", + level=3, + num="4.11.5", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat", + level=3, + num="4.11.6", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid", + level=3, + num="4.11.7", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values", + level=3, + num="4.11.8", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.GCM.Error", + level=3, + num="4.11.9", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Values.CTR.Error", + level=3, + num="4.11.10", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector", + level=3, + num="4.11.11", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue", + level=3, + num="4.11.12", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError", + level=3, + num="4.11.13", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong", + level=3, + num="4.11.14", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError", + level=3, + num="4.11.15", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong", + level=3, + num="4.11.16", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode", + level=3, + num="4.11.17", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Encrypt.Function.Mode.KeyAndInitializationVector.Length", + level=3, + num="4.11.18", + ), + Heading(name="MySQL Decrypt Function", level=2, num="4.12"), + Heading(name="RQ.SRS008.AES.MySQL.Decrypt.Function", level=3, num="4.12.1"), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax", level=3, num="4.12.2" + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText", + level=3, + num="4.12.3", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key", + level=3, + num="4.12.4", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode", + level=3, + num="4.12.5", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat", + level=3, + num="4.12.6", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid", + level=3, + num="4.12.7", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values", + level=3, + num="4.12.8", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.GCM.Error", + level=3, + num="4.12.9", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Values.CTR.Error", + level=3, + num="4.12.10", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector", + level=3, + num="4.12.11", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue", + level=3, + num="4.12.12", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError", + level=3, + num="4.12.13", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong", + level=3, + num="4.12.14", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError", + level=3, + num="4.12.15", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong", + level=3, + num="4.12.16", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode", + level=3, + num="4.12.17", + ), + Heading( + name="RQ.SRS008.AES.MySQL.Decrypt.Function.Mode.KeyAndInitializationVector.Length", + level=3, + num="4.12.18", + ), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS008_AES_Functions, RQ_SRS008_AES_Functions_Compatibility_MySQL, @@ -1918,8 +2291,8 @@ SRS_008_ClickHouse_AES_Encryption_Functions = Specification( RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong, RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode, RQ_SRS008_AES_MySQL_Decrypt_Function_Mode_KeyAndInitializationVector_Length, - ), - content=''' + ), + content=""" # SRS-008 ClickHouse AES Encryption Functions # Software Requirements Specification @@ -2892,4 +3265,5 @@ version: 1.0 [Revision history]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/aes_encryption/requirements/requirements.md [Git]: https://git-scm.com/ [NIST test vectors]: https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program -''') +""", +) diff --git a/tests/testflows/aes_encryption/tests/common.py b/tests/testflows/aes_encryption/tests/common.py index be1c0b98851..f0a10d34411 100644 --- a/tests/testflows/aes_encryption/tests/common.py +++ b/tests/testflows/aes_encryption/tests/common.py @@ -107,39 +107,51 @@ plaintexts = [ ("Decimal32", "reinterpretAsFixedString(toDecimal32(2, 4))"), ("Decimal64", "reinterpretAsFixedString(toDecimal64(2, 4))"), ("Decimal128", "reinterpretAsFixedString(toDecimal128(2, 4))"), - ("UUID", "reinterpretAsFixedString(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))"), + ( + "UUID", + "reinterpretAsFixedString(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))", + ), ("Date", "reinterpretAsFixedString(toDate('2020-01-01'))"), ("DateTime", "reinterpretAsFixedString(toDateTime('2020-01-01 20:01:02'))"), - ("DateTime64", "reinterpretAsFixedString(toDateTime64('2020-01-01 20:01:02.123', 3))"), + ( + "DateTime64", + "reinterpretAsFixedString(toDateTime64('2020-01-01 20:01:02.123', 3))", + ), ("LowCardinality", "toLowCardinality('1')"), ("LowCardinalityFixedString", "toLowCardinality(toFixedString('1',2))"), - #("Array", "[1,2]"), - not supported - #("Tuple", "(1,'a')") - not supported + # ("Array", "[1,2]"), - not supported + # ("Tuple", "(1,'a')") - not supported ("NULL", "reinterpretAsFixedString(toDateOrNull('foo'))"), ("NullableString", "toNullable('1')"), ("NullableStringNull", "toNullable(NULL)"), ("NullableFixedString", "toNullable(toFixedString('1',2))"), ("NullableFixedStringNull", "toNullable(toFixedString(NULL,2))"), ("IPv4", "reinterpretAsFixedString(toIPv4('171.225.130.45'))"), - ("IPv6", "reinterpretAsFixedString(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'))"), + ( + "IPv6", + "reinterpretAsFixedString(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'))", + ), ("Enum8", r"reinterpretAsFixedString(CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)'))"), ("Enum16", r"reinterpretAsFixedString(CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)'))"), ] _hex = hex + def hex(s): - """Convert string to hex. - """ + """Convert string to hex.""" if isinstance(s, str): - return "".join(['%X' % ord(c) for c in s]) + return "".join(["%X" % ord(c) for c in s]) if isinstance(s, bytes): - return "".join(['%X' % c for c in s]) + return "".join(["%X" % c for c in s]) return _hex(s) + def getuid(): if current().subtype == TestSubType.Example: - testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + testname = ( + f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + ) else: testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" - return testname + "_" + str(uuid.uuid1()).replace('-', '_') + return testname + "_" + str(uuid.uuid1()).replace("-", "_") diff --git a/tests/testflows/aes_encryption/tests/compatibility/feature.py b/tests/testflows/aes_encryption/tests/compatibility/feature.py index 5ef547e43f4..509bc894374 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/feature.py +++ b/tests/testflows/aes_encryption/tests/compatibility/feature.py @@ -2,16 +2,17 @@ from testflows.core import * from aes_encryption.requirements import * + @TestFeature @Name("compatibility") -@Requirements( - RQ_SRS008_AES_Functions_DataFromMultipleSources("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_DataFromMultipleSources("1.0")) def feature(self, node="clickhouse1"): - """Check encryption functions usage compatibility. - """ + """Check encryption functions usage compatibility.""" self.context.node = self.context.cluster.node(node) Feature(run=load("aes_encryption.tests.compatibility.insert", "feature"), flags=TE) Feature(run=load("aes_encryption.tests.compatibility.select", "feature"), flags=TE) - Feature(run=load("aes_encryption.tests.compatibility.mysql.feature", "feature"), flags=TE) \ No newline at end of file + Feature( + run=load("aes_encryption.tests.compatibility.mysql.feature", "feature"), + flags=TE, + ) diff --git a/tests/testflows/aes_encryption/tests/compatibility/insert.py b/tests/testflows/aes_encryption/tests/compatibility/insert.py index 6ddcc11b584..c4d80c85896 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/insert.py +++ b/tests/testflows/aes_encryption/tests/compatibility/insert.py @@ -10,6 +10,7 @@ from testflows.asserts import values, error, snapshot from aes_encryption.tests.common import modes, mysql_modes + @contextmanager def table(name): node = current().context.node @@ -33,6 +34,7 @@ def table(name): with Finally("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {name}") + @contextmanager def mv_transform(table, transform): node = current().context.node @@ -70,6 +72,7 @@ def mv_transform(table, transform): with And("dropping Null input table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table}_input") + @TestScenario def encrypt_using_materialized_view(self): """Check that we can use `encrypt` function when inserting @@ -82,7 +85,9 @@ def encrypt_using_materialized_view(self): aad = "some random aad" for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -92,21 +97,32 @@ def encrypt_using_materialized_view(self): with table("user_data"): with mv_transform("user_data", example_transform): with When("I insert encrypted data"): - node.query(f""" + node.query( + f""" INSERT INTO user_data_input (date, name, secret, mode, key) VALUES ('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), ('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), ('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) - """) + """ + ) with And("I read inserted data back"): - node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + node.query( + "SELECT date, name, hex(secret) FROM user_data ORDER BY date" + ) with Then("output must match the snapshot"): with values() as that: - assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_mv_example_{varname(basename(self.name))}")), error() + assert that( + snapshot( + r.output.strip(), + "insert", + name=f"encrypt_mv_example_{varname(basename(self.name))}", + ) + ), error() + @TestScenario def aes_encrypt_mysql_using_materialized_view(self): @@ -120,30 +136,45 @@ def aes_encrypt_mysql_using_materialized_view(self): aad = "some random aad" for mode, key_len, iv_len in mysql_modes: - with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" - example_transform = f"aes_encrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + example_transform = ( + f"aes_encrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + ) with table("user_data"): with mv_transform("user_data", example_transform): with When("I insert encrypted data"): - node.query(f""" + node.query( + f""" INSERT INTO user_data_input (date, name, secret, mode, key) VALUES ('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), ('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), ('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}) - """) + """ + ) with And("I read inserted data back"): - node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + node.query( + "SELECT date, name, hex(secret) FROM user_data ORDER BY date" + ) with Then("output must match the snapshot"): with values() as that: - assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_mv_example_{varname(basename(self.name))}")), error() + assert that( + snapshot( + r.output.strip(), + "insert", + name=f"aes_encrypt_mysql_mv_example_{varname(basename(self.name))}", + ) + ), error() + @TestScenario def encrypt_using_input_table_function(self): @@ -157,7 +188,9 @@ def encrypt_using_input_table_function(self): aad = "some random aad" for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -166,7 +199,8 @@ def encrypt_using_input_table_function(self): with table("user_data"): with When("I insert encrypted data"): - node.query(f""" + node.query( + f""" INSERT INTO user_data SELECT @@ -174,14 +208,24 @@ def encrypt_using_input_table_function(self): FROM input('date Date, name String, secret String') FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret') - """) + """ + ) with And("I read inserted data back"): - r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + r = node.query( + "SELECT date, name, hex(secret) FROM user_data ORDER BY date" + ) with Then("output must match the snapshot"): with values() as that: - assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_input_example_{varname(basename(example.name))}")), error() + assert that( + snapshot( + r.output.strip(), + "insert", + name=f"encrypt_input_example_{varname(basename(example.name))}", + ) + ), error() + @TestScenario def aes_encrypt_mysql_using_input_table_function(self): @@ -195,7 +239,9 @@ def aes_encrypt_mysql_using_input_table_function(self): aad = "some random aad" for mode, key_len, iv_len in mysql_modes: - with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -203,7 +249,8 @@ def aes_encrypt_mysql_using_input_table_function(self): with table("user_data"): with When("I insert encrypted data"): - node.query(f""" + node.query( + f""" INSERT INTO user_data SELECT @@ -211,14 +258,24 @@ def aes_encrypt_mysql_using_input_table_function(self): FROM input('date Date, name String, secret String') FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret') - """) + """ + ) with And("I read inserted data back"): - r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + r = node.query( + "SELECT date, name, hex(secret) FROM user_data ORDER BY date" + ) with Then("output must match the snapshot"): with values() as that: - assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_input_example_{varname(basename(example.name))}")), error() + assert that( + snapshot( + r.output.strip(), + "insert", + name=f"aes_encrypt_mysql_input_example_{varname(basename(example.name))}", + ) + ), error() + @TestScenario def decrypt_using_materialized_view(self): @@ -232,10 +289,15 @@ def decrypt_using_materialized_view(self): aad = "some random aad" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -244,28 +306,38 @@ def decrypt_using_materialized_view(self): with Given("I have ciphertexts"): example_name = basename(example.name) - ciphertexts = getattr(snapshot_module, varname(f"encrypt_mv_example_{example_name}")) - example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")] + ciphertexts = getattr( + snapshot_module, varname(f"encrypt_mv_example_{example_name}") + ) + example_ciphertexts = [ + "'{}'".format(l.split("\t")[-1].strup("'")) + for l in ciphertexts.split("\n") + ] with table("user_data"): with mv_transform("user_data", example_transform): with When("I insert encrypted data"): - node.query(f""" + node.query( + f""" INSERT INTO user_data_input (date, name, secret, mode, key) VALUES ('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), ('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), ('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) - """) + """ + ) with And("I read inserted data back"): - r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + r = node.query( + "SELECT date, name, secret FROM user_data ORDER BY date" + ) with Then("output must match the expected"): expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'""" assert r.output == expected, error() + @TestScenario def aes_decrypt_mysql_using_materialized_view(self): """Check that we can use `aes_decrypt_mysql` function when inserting @@ -278,40 +350,58 @@ def aes_decrypt_mysql_using_materialized_view(self): aad = "some random aad" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" example_aad = None if not aad_len else f"'{aad}'" - example_transform = f"aes_decrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + example_transform = ( + f"aes_decrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + ) with Given("I have ciphertexts"): example_name = basename(example.name) - ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_mv_example_{example_name}")) - example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")] + ciphertexts = getattr( + snapshot_module, + varname(f"aes_encrypt_mysql_mv_example_{example_name}"), + ) + example_ciphertexts = [ + "'{}'".format(l.split("\t")[-1].strup("'")) + for l in ciphertexts.split("\n") + ] with table("user_data"): with mv_transform("user_data", example_transform): with When("I insert encrypted data"): - node.query(f""" + node.query( + f""" INSERT INTO user_data_input (date, name, secret, mode, key) VALUES ('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), ('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), ('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}) - """) + """ + ) with And("I read inserted data back"): - r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + r = node.query( + "SELECT date, name, secret FROM user_data ORDER BY date" + ) with Then("output must match the expected"): expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'""" assert r.output == expected, error() + @TestScenario def decrypt_using_input_table_function(self): """Check that we can use `decrypt` function when inserting @@ -324,10 +414,15 @@ def decrypt_using_input_table_function(self): aad = "some random aad" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -336,12 +431,18 @@ def decrypt_using_input_table_function(self): with Given("I have ciphertexts"): example_name = basename(example.name) - ciphertexts = getattr(snapshot_module, varname(f"encrypt_input_example_{example_name}")) - example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")] + ciphertexts = getattr( + snapshot_module, varname(f"encrypt_input_example_{example_name}") + ) + example_ciphertexts = [ + l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n") + ] with table("user_data"): with When("I insert decrypted data"): - node.query(textwrap.dedent(f""" + node.query( + textwrap.dedent( + f""" INSERT INTO user_data SELECT @@ -349,15 +450,20 @@ def decrypt_using_input_table_function(self): FROM input('date Date, name String, secret String') FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}') - """)) + """ + ) + ) with And("I read inserted data back"): - r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + r = node.query( + "SELECT date, name, secret FROM user_data ORDER BY date" + ) expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret""" with Then("output must match the expected", description=expected): assert r.output == expected, error() + @TestScenario def aes_decrypt_mysql_using_input_table_function(self): """Check that we can use `aes_decrypt_mysql` function when inserting @@ -370,10 +476,15 @@ def aes_decrypt_mysql_using_input_table_function(self): aad = "some random aad" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot"), + ).load_module() for mode, key_len, iv_len in mysql_modes: - with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ) as example: example_key = f"'{key[:key_len]}'" example_mode = mode example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -381,12 +492,19 @@ def aes_decrypt_mysql_using_input_table_function(self): with Given("I have ciphertexts"): example_name = basename(example.name) - ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_input_example_{example_name}")) - example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")] + ciphertexts = getattr( + snapshot_module, + varname(f"aes_encrypt_mysql_input_example_{example_name}"), + ) + example_ciphertexts = [ + l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n") + ] with table("user_data"): with When("I insert decrypted data"): - node.query(textwrap.dedent(f""" + node.query( + textwrap.dedent( + f""" INSERT INTO user_data SELECT @@ -394,20 +512,24 @@ def aes_decrypt_mysql_using_input_table_function(self): FROM input('date Date, name String, secret String') FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}') - """)) + """ + ) + ) with And("I read inserted data back"): - r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + r = node.query( + "SELECT date, name, secret FROM user_data ORDER BY date" + ) expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret""" with Then("output must match the expected", description=expected): assert r.output == expected, error() + @TestFeature @Name("insert") def feature(self, node="clickhouse1"): - """Check encryption functions when used during data insertion into a table. - """ + """Check encryption functions when used during data insertion into a table.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py index 612e8bc450e..27884eb7cb3 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py @@ -7,10 +7,10 @@ from testflows.asserts import error from aes_encryption.requirements import * from aes_encryption.tests.common import mysql_modes, hex + @contextmanager def table(name, node, mysql_node, secret_type): - """Create a table that can be accessed using MySQL database engine. - """ + """Create a table that can be accessed using MySQL database engine.""" try: with Given("table in MySQL"): sql = f""" @@ -23,9 +23,15 @@ def table(name, node, mysql_node, secret_type): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I create a database using MySQL database engine"): sql = f""" @@ -43,15 +49,22 @@ def table(name, node, mysql_node, secret_type): node.query(f"DROP DATABASE IF EXISTS mysql_db") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def decrypt(self, mysql_datatype): """Check that when using a table provided by MySQL database engine that contains a column encrypted in MySQL stored using specified data type @@ -65,7 +78,7 @@ def decrypt(self, mysql_datatype): for func in ["decrypt", "aes_decrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -75,7 +88,9 @@ def decrypt(self, mysql_datatype): if key_len != exact_key_size: continue - with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with Example( + f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ): with table("user_data", node, mysql_node, mysql_datatype): example_mode = mode example_key = f"'{key[:key_len]}'" @@ -86,34 +101,51 @@ def decrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read encrypted data in MySQL to make sure it is valid"): sql = f""" SET block_encryption_mode = {example_mode}; SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read raw encrypted data in MySQL"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT id, date, name, hex(secret) as secret FROM user_data;"', + exitcode=0, + ) with And("I read raw data using MySQL database engine"): - output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_db.user_data") + output = node.query( + "SELECT id, date, name, hex(secret) AS secret FROM mysql_db.user_data" + ) with And("I read decrypted data using MySQL database engine"): - output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip() + output = node.query( + f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""" + ).output.strip() with Then("output should match the original plain text"): assert output == hex("secret"), error() + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def encrypt(self, mysql_datatype): """Check that when using a table provided by MySQL database engine that we can encrypt data during insert using the `aes_encrypt_mysql` function @@ -126,7 +158,7 @@ def encrypt(self, mysql_datatype): for func in ["encrypt", "aes_encrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -136,15 +168,21 @@ def encrypt(self, mysql_datatype): if key_len != exact_key_size: continue - with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with Example( + f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ): with table("user_data", node, mysql_node, mysql_datatype): example_mode = mode example_key = f"'{key[:key_len]}'" example_iv = None if not iv_len else f"'{iv[:iv_len]}'" example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" - with When("I insert encrypted data into a table provided by MySQL database engine"): - node.query(textwrap.dedent(f""" + with When( + "I insert encrypted data into a table provided by MySQL database engine" + ): + node.query( + textwrap.dedent( + f""" INSERT INTO mysql_db.user_data SELECT @@ -152,21 +190,36 @@ def encrypt(self, mysql_datatype): FROM input('id Int32, date Date, name String, secret String') FORMAT Values (1, '2020-01-01', 'user0', 'secret') - """)) + """ + ) + ) with And("I read decrypted data using MySQL database engine"): - output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip() + output = node.query( + f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""" + ).output.strip() - with Then("decrypted data from MySQL database engine should should match the original plain text"): + with Then( + "decrypted data from MySQL database engine should should match the original plain text" + ): assert output == hex("secret"), error() - with And("I read raw data using MySQL database engine to get expected raw data"): - expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_db.user_data").output.strip() + with And( + "I read raw data using MySQL database engine to get expected raw data" + ): + expected_raw_data = node.query( + "SELECT hex(secret) AS secret FROM mysql_db.user_data" + ).output.strip() with And("I read raw encrypted data in MySQL"): - output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + output = mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT hex(secret) as secret FROM user_data;"', + exitcode=0, + ).output.strip() - with Then("check that raw encryted data in MySQL matches the expected"): + with Then( + "check that raw encryted data in MySQL matches the expected" + ): assert expected_raw_data in output, error() with And("I decrypt data in MySQL to make sure it is valid"): @@ -174,16 +227,20 @@ def encrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; """ - output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + output = mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ).output.strip() - with Then("decryted data in MySQL should match the original plain text"): + with Then( + "decryted data in MySQL should match the original plain text" + ): assert hex("secret") in output, error() + @TestFeature @Name("database engine") -@Requirements( - RQ_SRS008_AES_Functions_Compatibility_Engine_Database_MySQL("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Compatibility_Engine_Database_MySQL("1.0")) def feature(self, node="clickhouse1", mysql_node="mysql1"): """Check usage of encryption functions with [MySQL database engine]. diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py index 812e0222866..89adcabd701 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py @@ -7,10 +7,10 @@ from testflows.asserts import error from aes_encryption.requirements import * from aes_encryption.tests.common import mysql_modes, hex + @contextmanager def dictionary(name, node, mysql_node, secret_type): - """Create a table in MySQL and use it a source for a dictionary. - """ + """Create a table in MySQL and use it a source for a dictionary.""" try: with Given("table in MySQL"): sql = f""" @@ -23,9 +23,15 @@ def dictionary(name, node, mysql_node, secret_type): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("dictionary that uses MySQL table as the external source"): with When("I drop the dictionary if exists"): @@ -59,7 +65,11 @@ def dictionary(name, node, mysql_node, secret_type): node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @contextmanager def parameters_dictionary(name, node, mysql_node): @@ -80,9 +90,15 @@ def parameters_dictionary(name, node, mysql_node): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("dictionary that uses MySQL table as the external source"): with When("I drop the dictionary if exists"): @@ -118,7 +134,11 @@ def parameters_dictionary(name, node, mysql_node): node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @TestScenario def parameter_values(self): @@ -134,16 +154,24 @@ def parameter_values(self): plaintext = "'secret'" for encrypt, decrypt in [ - ("encrypt", "decrypt"), - ("aes_encrypt_mysql", "aes_decrypt_mysql") - ]: - with Example(f"{encrypt} and {decrypt}", description=f"Check using dictionary for parameters of {encrypt} and {decrypt} functions."): - with parameters_dictionary("parameters_data", node, mysql_node) as dict_name: + ("encrypt", "decrypt"), + ("aes_encrypt_mysql", "aes_decrypt_mysql"), + ]: + with Example( + f"{encrypt} and {decrypt}", + description=f"Check using dictionary for parameters of {encrypt} and {decrypt} functions.", + ): + with parameters_dictionary( + "parameters_data", node, mysql_node + ) as dict_name: with When("I insert parameters values in MySQL"): sql = f""" INSERT INTO parameters_data VALUES (1, 'user0', {mode}, {key}, {iv}, {plaintext}); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I use dictionary values as parameters"): sql = f""" @@ -164,13 +192,17 @@ def parameter_values(self): with Then("output should match the plain text"): assert f"'{output}'" == plaintext, error() + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def decrypt(self, mysql_datatype): """Check that when using a dictionary that uses MySQL table as a source and contains a data encrypted in MySQL and stored using specified data type @@ -184,7 +216,7 @@ def decrypt(self, mysql_datatype): for func in ["decrypt", "aes_decrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -194,8 +226,12 @@ def decrypt(self, mysql_datatype): if key_len != exact_key_size: continue - with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): - with dictionary("user_data", node, mysql_node, mysql_datatype) as dict_name: + with Example( + f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ): + with dictionary( + "user_data", node, mysql_node, mysql_datatype + ) as dict_name: example_mode = mode example_key = f"'{key[:key_len]}'" example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -205,23 +241,36 @@ def decrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read encrypted data in MySQL to make sure it is valid"): sql = f""" SET block_encryption_mode = {example_mode}; SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read raw encrypted data in MySQL"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT id, date, name, hex(secret) as secret FROM user_data;"', + exitcode=0, + ) with And("I read raw data using MySQL dictionary"): - output = node.query(f"SELECT hex(dictGet('default.{dict_name}', 'secret', toUInt64(1))) AS secret") + output = node.query( + f"SELECT hex(dictGet('default.{dict_name}', 'secret', toUInt64(1))) AS secret" + ) with And("I read decrypted data using MySQL dictionary"): - output = node.query(textwrap.dedent(f""" + output = node.query( + textwrap.dedent( + f""" SELECT hex( {func}( {example_mode}, @@ -229,16 +278,17 @@ def decrypt(self, mysql_datatype): {example_key}{(", " + example_iv) if example_iv else ""} ) ) - """)).output.strip() + """ + ) + ).output.strip() with Then("output should match the original plain text"): assert output == hex("secret"), error() + @TestFeature @Name("dictionary") -@Requirements( - RQ_SRS008_AES_Functions_Compatibility_Dictionaries("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Compatibility_Dictionaries("1.0")) def feature(self, node="clickhouse1", mysql_node="mysql1"): """Check usage of encryption functions with [MySQL dictionary]. diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py index bc470dd13a7..ed5f47ee991 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py @@ -2,17 +2,27 @@ from testflows.core import * from aes_encryption.requirements import * + @TestFeature @Name("mysql") -@Requirements( - RQ_SRS008_AES_Functions_Compatibility_MySQL("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Compatibility_MySQL("1.0")) def feature(self, node="clickhouse1"): - """Check encryption functions usage compatibility with MySQL. - """ + """Check encryption functions usage compatibility with MySQL.""" self.context.node = self.context.cluster.node(node) - Feature(run=load("aes_encryption.tests.compatibility.mysql.table_engine", "feature"), flags=TE) - Feature(run=load("aes_encryption.tests.compatibility.mysql.database_engine", "feature"), flags=TE) - Feature(run=load("aes_encryption.tests.compatibility.mysql.table_function", "feature"), flags=TE) - Feature(run=load("aes_encryption.tests.compatibility.mysql.dictionary", "feature"), flags=TE) + Feature( + run=load("aes_encryption.tests.compatibility.mysql.table_engine", "feature"), + flags=TE, + ) + Feature( + run=load("aes_encryption.tests.compatibility.mysql.database_engine", "feature"), + flags=TE, + ) + Feature( + run=load("aes_encryption.tests.compatibility.mysql.table_function", "feature"), + flags=TE, + ) + Feature( + run=load("aes_encryption.tests.compatibility.mysql.dictionary", "feature"), + flags=TE, + ) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py index afc8b607a6f..7f7d5ada559 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py @@ -7,10 +7,10 @@ from testflows.asserts import error from aes_encryption.requirements import * from aes_encryption.tests.common import mysql_modes, hex + @contextmanager def table(name, node, mysql_node, secret_type): - """Create a table that can be accessed using MySQL table engine. - """ + """Create a table that can be accessed using MySQL table engine.""" try: with Given("table in MySQL"): sql = f""" @@ -23,9 +23,15 @@ def table(name, node, mysql_node, secret_type): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I create a table using MySQL table engine"): sql = f""" @@ -49,15 +55,22 @@ def table(name, node, mysql_node, secret_type): node.query(f"DROP TABLE IF EXISTS mysql_{name}") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def decrypt(self, mysql_datatype): """Check that when using a table with MySQL table engine that contains a column encrypted in MySQL stored using specified data type @@ -71,7 +84,7 @@ def decrypt(self, mysql_datatype): for func in ["decrypt", "aes_decrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -81,7 +94,9 @@ def decrypt(self, mysql_datatype): if key_len != exact_key_size: continue - with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with Example( + f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ): with table("user_data", node, mysql_node, mysql_datatype): example_mode = mode example_key = f"'{key[:key_len]}'" @@ -92,34 +107,51 @@ def decrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read encrypted data in MySQL to make sure it is valid"): sql = f""" SET block_encryption_mode = {example_mode}; SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read raw encrypted data in MySQL"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT id, date, name, hex(secret) as secret FROM user_data;"', + exitcode=0, + ) with And("I read raw data using MySQL table engine"): - output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_user_data") + output = node.query( + "SELECT id, date, name, hex(secret) AS secret FROM mysql_user_data" + ) with And("I read decrypted data via MySQL table engine"): - output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip() + output = node.query( + f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""" + ).output.strip() with Then("the output should match the original plain text"): assert output == hex("secret"), error() + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def encrypt(self, mysql_datatype): """Check that when using a table with MySQL table engine that we can encrypt data during insert using the `encrypt` and `aes_encrypt_mysql` @@ -132,7 +164,7 @@ def encrypt(self, mysql_datatype): for func in ["encrypt", "aes_encrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -150,7 +182,9 @@ def encrypt(self, mysql_datatype): example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" with When("I insert encrypted data into MySQL table engine"): - node.query(textwrap.dedent(f""" + node.query( + textwrap.dedent( + f""" INSERT INTO mysql_user_data SELECT @@ -158,21 +192,36 @@ def encrypt(self, mysql_datatype): FROM input('id Nullable(Int32), date Date, name String, secret String') FORMAT Values (null, '2020-01-01', 'user0', 'secret') - """)) + """ + ) + ) with And("I read decrypted data via MySQL table engine"): - output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip() + output = node.query( + f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""" + ).output.strip() - with Then("decrypted data from MySQL table engine should should match the original plain text"): + with Then( + "decrypted data from MySQL table engine should should match the original plain text" + ): assert output == hex("secret"), error() - with And("I read raw data using MySQL table engine to get expected raw data"): - expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_user_data").output.strip() + with And( + "I read raw data using MySQL table engine to get expected raw data" + ): + expected_raw_data = node.query( + "SELECT hex(secret) AS secret FROM mysql_user_data" + ).output.strip() with And("I read raw encrypted data in MySQL"): - output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + output = mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT hex(secret) as secret FROM user_data;"', + exitcode=0, + ).output.strip() - with Then("check that raw encryted data in MySQL matches the expected"): + with Then( + "check that raw encryted data in MySQL matches the expected" + ): assert expected_raw_data in output, error() with And("I decrypt data in MySQL to make sure it is valid"): @@ -180,16 +229,20 @@ def encrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; """ - output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + output = mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ).output.strip() - with Then("decryted data in MySQL should match the original plain text"): + with Then( + "decryted data in MySQL should match the original plain text" + ): assert hex("secret") in output, error() + @TestFeature @Name("table engine") -@Requirements( - RQ_SRS008_AES_Functions_Compatibility_Engine_Table_MySQL("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Compatibility_Engine_Table_MySQL("1.0")) def feature(self, node="clickhouse1", mysql_node="mysql1"): """Check usage of encryption functions with [MySQL table engine]. diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py index 91ea8956cad..9c38efd2807 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py @@ -7,10 +7,10 @@ from testflows.asserts import error from aes_encryption.requirements import * from aes_encryption.tests.common import mysql_modes, hex + @contextmanager def table(name, node, mysql_node, secret_type): - """Create a table that can be accessed using MySQL table function. - """ + """Create a table that can be accessed using MySQL table function.""" try: with Given("table in MySQL"): sql = f""" @@ -23,22 +23,35 @@ def table(name, node, mysql_node, secret_type): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) yield f"mysql('{mysql_node.name}:3306', 'db', 'user_data', 'user', 'password')" finally: with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def decrypt(self, mysql_datatype): """Check that when using a table accessed through MySQL table function that contains a column encrypted in MySQL stored using specified data type @@ -52,7 +65,7 @@ def decrypt(self, mysql_datatype): for func in ["decrypt", "aes_decrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -62,8 +75,12 @@ def decrypt(self, mysql_datatype): if key_len != exact_key_size: continue - with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): - with table("user_data", node, mysql_node, mysql_datatype) as table_function: + with Example( + f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ): + with table( + "user_data", node, mysql_node, mysql_datatype + ) as table_function: example_mode = mode example_key = f"'{key[:key_len]}'" example_iv = None if not iv_len else f"'{iv[:iv_len]}'" @@ -73,34 +90,51 @@ def decrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read encrypted data in MySQL to make sure it is valid"): sql = f""" SET block_encryption_mode = {example_mode}; SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I read raw encrypted data in MySQL"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT id, date, name, hex(secret) as secret FROM user_data;"', + exitcode=0, + ) with And("I read raw data using MySQL table function"): - output = node.query(f"SELECT id, date, name, hex(secret) AS secret FROM {table_function}") + output = node.query( + f"SELECT id, date, name, hex(secret) AS secret FROM {table_function}" + ) with And("I read decrypted data using MySQL table function"): - output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip() + output = node.query( + f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""" + ).output.strip() with Then("output should match the original plain text"): assert output == hex("secret"), error() + @TestOutline(Scenario) -@Examples("mysql_datatype", [ - ("VARBINARY(100)",), - #("VARCHAR(100)",), - ("BLOB", ), - #("TEXT",) -]) +@Examples( + "mysql_datatype", + [ + ("VARBINARY(100)",), + # ("VARCHAR(100)",), + ("BLOB",), + # ("TEXT",) + ], +) def encrypt(self, mysql_datatype): """Check that when using a table accessed through MySQL table function that we can encrypt data during insert using the `aes_encrypt_mysql` function @@ -113,7 +147,7 @@ def encrypt(self, mysql_datatype): for func in ["encrypt", "aes_encrypt_mysql"]: for mode, key_len, iv_len in mysql_modes: - exact_key_size = int(mode.split("-")[1])//8 + exact_key_size = int(mode.split("-")[1]) // 8 if "ecb" not in mode and not iv_len: continue @@ -123,15 +157,23 @@ def encrypt(self, mysql_datatype): if key_len != exact_key_size: continue - with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): - with table("user_data", node, mysql_node, mysql_datatype) as table_function: + with Example( + f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}""" + ): + with table( + "user_data", node, mysql_node, mysql_datatype + ) as table_function: example_mode = mode example_key = f"'{key[:key_len]}'" example_iv = None if not iv_len else f"'{iv[:iv_len]}'" example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" - with When("I insert encrypted data into a table provided by MySQL database engine"): - node.query(textwrap.dedent(f""" + with When( + "I insert encrypted data into a table provided by MySQL database engine" + ): + node.query( + textwrap.dedent( + f""" INSERT INTO TABLE FUNCTION {table_function} SELECT @@ -139,21 +181,36 @@ def encrypt(self, mysql_datatype): FROM input('id Int32, date Date, name String, secret String') FORMAT Values (1, '2020-01-01', 'user0', 'secret') - """)) + """ + ) + ) with And("I read decrypted data using MySQL database engine"): - output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip() + output = node.query( + f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""" + ).output.strip() - with Then("decrypted data from MySQL database engine should should match the original plain text"): + with Then( + "decrypted data from MySQL database engine should should match the original plain text" + ): assert output == hex("secret"), error() - with And("I read raw data using MySQL database engine to get expected raw data"): - expected_raw_data = node.query(f"SELECT hex(secret) AS secret FROM {table_function}").output.strip() + with And( + "I read raw data using MySQL database engine to get expected raw data" + ): + expected_raw_data = node.query( + f"SELECT hex(secret) AS secret FROM {table_function}" + ).output.strip() with And("I read raw encrypted data in MySQL"): - output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + output = mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "SELECT hex(secret) as secret FROM user_data;"', + exitcode=0, + ).output.strip() - with Then("check that raw encryted data in MySQL matches the expected"): + with Then( + "check that raw encryted data in MySQL matches the expected" + ): assert expected_raw_data in output, error() with And("I decrypt data in MySQL to make sure it is valid"): @@ -161,16 +218,20 @@ def encrypt(self, mysql_datatype): SET block_encryption_mode = {example_mode}; SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; """ - output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + output = mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ).output.strip() - with Then("decryted data in MySQL should match the original plain text"): + with Then( + "decryted data in MySQL should match the original plain text" + ): assert hex("secret") in output, error() + @TestFeature @Name("table function") -@Requirements( - RQ_SRS008_AES_Functions_Compatibility_TableFunction_MySQL("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Compatibility_TableFunction_MySQL("1.0")) def feature(self, node="clickhouse1", mysql_node="mysql1"): """Check usage of encryption functions with [MySQL table function]. diff --git a/tests/testflows/aes_encryption/tests/compatibility/select.py b/tests/testflows/aes_encryption/tests/compatibility/select.py index f81920c65d3..057eb2947bd 100644 --- a/tests/testflows/aes_encryption/tests/compatibility/select.py +++ b/tests/testflows/aes_encryption/tests/compatibility/select.py @@ -7,6 +7,7 @@ from testflows.asserts import values, error, snapshot from aes_encryption.tests.common import modes, mysql_modes + @contextmanager def table(name, sql): node = current().context.node @@ -22,18 +23,22 @@ def table(name, sql): with Finally("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {name}") + @TestScenario def decrypt(self): - """Check decrypting column when reading data from a table. - """ + """Check decrypting column when reading data from a table.""" node = self.context.node key = f"{'1' * 64}" iv = f"{'2' * 64}" aad = "some random aad" for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example: - with table("user_table", """ + with Example( + f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""" + ) as example: + with table( + "user_table", + """ CREATE TABLE {name} ( date Nullable(Date), @@ -41,7 +46,8 @@ def decrypt(self): secret Nullable(String) ) ENGINE = Memory() - """): + """, + ): example_mode = mode example_key = f"'{key[:key_len]}'" @@ -49,20 +55,29 @@ def decrypt(self): example_aad = None if not aad_len else f"'{aad}'" with When("I insert encrypted data"): - encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip() - node.query(textwrap.dedent(f""" + encrypted_secret = node.query( + f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""" + ).output.strip() + node.query( + textwrap.dedent( + f""" INSERT INTO user_table (date, name, secret) VALUES ('2020-01-01', 'user0', unhex('{encrypted_secret}')) - """)) + """ + ) + ) with And("I decrypt data during query"): - output = node.query(f"""SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table FORMAT JSONEachRow""").output.strip() + output = node.query( + f"""SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table FORMAT JSONEachRow""" + ).output.strip() with Then("I should get back the original plain text"): assert output == '{"name":"user0","secret":"secret"}', error() + @TestScenario def decrypt_multiple(self, count=1000): """Check decrypting column when reading multiple entries @@ -75,8 +90,12 @@ def decrypt_multiple(self, count=1000): aad = "some random aad" for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example: - with table("user_table", """ + with Example( + f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""" + ) as example: + with table( + "user_table", + """ CREATE TABLE {name} ( date Nullable(Date), @@ -84,7 +103,8 @@ def decrypt_multiple(self, count=1000): secret Nullable(String) ) ENGINE = Memory() - """): + """, + ): example_mode = mode example_key = f"'{key[:key_len]}'" @@ -92,19 +112,32 @@ def decrypt_multiple(self, count=1000): example_aad = None if not aad_len else f"'{aad}'" with When("I insert encrypted data"): - encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip() - values = [f"('2020-01-01', 'user0', unhex('{encrypted_secret}'))"] * count + encrypted_secret = node.query( + f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""" + ).output.strip() + values = [ + f"('2020-01-01', 'user0', unhex('{encrypted_secret}'))" + ] * count node.query( "INSERT INTO user_table\n" " (date, name, secret)\n" - f"VALUES {', '.join(values)}") + f"VALUES {', '.join(values)}" + ) - with And("I decrypt data", description="using a subquery and get the number of entries that match the plaintext"): - output = node.query(f"""SELECT count() AS count FROM (SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table) WHERE secret = 'secret' FORMAT JSONEachRow""").output.strip() + with And( + "I decrypt data", + description="using a subquery and get the number of entries that match the plaintext", + ): + output = node.query( + f"""SELECT count() AS count FROM (SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table) WHERE secret = 'secret' FORMAT JSONEachRow""" + ).output.strip() - with Then("I should get back the expected result", description=f"{count}"): + with Then( + "I should get back the expected result", description=f"{count}" + ): assert output == f'{{"count":"{count}"}}', error() + @TestScenario def decrypt_unique(self): """Check decrypting column when reading multiple entries @@ -116,7 +149,9 @@ def decrypt_unique(self): iv = f"{'2' * 64}" aad = "some random aad" - with table("user_table", """ + with table( + "user_table", + """ CREATE TABLE {name} ( id UInt64, @@ -125,7 +160,8 @@ def decrypt_unique(self): secret Nullable(String) ) ENGINE = Memory() - """): + """, + ): user_modes = [] user_keys = [] @@ -142,9 +178,11 @@ def decrypt_unique(self): with When(f"I get encrypted data for user {user_id}"): encrypted_secret = node.query( - f"""SELECT hex(encrypt({user_modes[-1]}, 'secret', {user_keys[-1]}))""" - ).output.strip() - values.append(f"({user_id}, '2020-01-01', 'user{user_id}', unhex('{encrypted_secret}'))") + f"""SELECT hex(encrypt({user_modes[-1]}, 'secret', {user_keys[-1]}))""" + ).output.strip() + values.append( + f"({user_id}, '2020-01-01', 'user{user_id}', unhex('{encrypted_secret}'))" + ) user_id += 1 @@ -152,10 +190,13 @@ def decrypt_unique(self): node.query( "INSERT INTO user_table\n" " (id, date, name, secret)\n" - f"VALUES {', '.join(values)}") + f"VALUES {', '.join(values)}" + ) with And("I read decrypted data for all users"): - output = node.query(textwrap.dedent(f""" + output = node.query( + textwrap.dedent( + f""" SELECT count() AS count FROM @@ -170,16 +211,18 @@ def decrypt_unique(self): WHERE secret = 'secret' FORMAT JSONEachRow - """)).output.strip() + """ + ) + ).output.strip() with Then("I should get back the expected result", description=f"{count}"): assert output == f'{{"count":"{count}"}}', error() + @TestFeature @Name("select") def feature(self, node="clickhouse1"): - """Check encryption functions when used during table querying. - """ + """Check encryption functions when used during table querying.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/aes_encryption/tests/decrypt.py b/tests/testflows/aes_encryption/tests/decrypt.py index 6c99c4d9d41..1c7d958737c 100644 --- a/tests/testflows/aes_encryption/tests/decrypt.py +++ b/tests/testflows/aes_encryption/tests/decrypt.py @@ -10,10 +10,24 @@ from testflows.asserts import error from aes_encryption.requirements.requirements import * from aes_encryption.tests.common import * + @TestOutline -def decrypt(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When, cast=None, endcast=None, compare=None, no_checks=False): - """Execute `decrypt` function with the specified parameters. - """ +def decrypt( + self, + ciphertext=None, + key=None, + mode=None, + iv=None, + aad=None, + exitcode=0, + message=None, + step=When, + cast=None, + endcast=None, + compare=None, + no_checks=False, +): + """Execute `decrypt` function with the specified parameters.""" params = [] if mode is not None: params.append(mode) @@ -33,7 +47,10 @@ def decrypt(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitc sql = f"{compare} = {sql}" sql = f"SELECT {sql}" - return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks) + return current().context.node.query( + sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks + ) + @TestScenario @Requirements( @@ -58,19 +75,33 @@ def invalid_ciphertext(self): continue with When(f"invalid ciphertext={ciphertext}"): if "cfb" in mode or "ofb" in mode or "ctr" in mode: - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, cast="hex") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, + iv=d_iv, + aad=d_aad, + cast="hex", + ) else: with When("I execute decrypt function"): - r = decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, no_checks=True, step=By) + r = decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, + iv=d_iv, + aad=d_aad, + no_checks=True, + step=By, + ) with Then("exitcode is not zero"): assert r.exitcode in [198, 36] with And("exception is present in the output"): assert "DB::Exception:" in r.output + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_InvalidParameters("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_InvalidParameters("1.0")) def invalid_parameters(self): """Check that `decrypt` function returns an error when we call it with invalid parameters. @@ -78,110 +109,236 @@ def invalid_parameters(self): ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" with Example("no parameters"): - decrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function decrypt provided 0, expected 3 to 5") + decrypt( + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 0, expected 3 to 5", + ) with Example("missing key and mode"): - decrypt(ciphertext=ciphertext, exitcode=42, - message="DB::Exception: Incorrect number of arguments for function decrypt provided 1") + decrypt( + ciphertext=ciphertext, + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 1", + ) with Example("missing mode"): - decrypt(ciphertext=ciphertext, key="'123'", exitcode=42, - message="DB::Exception: Incorrect number of arguments for function decrypt provided 2") + decrypt( + ciphertext=ciphertext, + key="'123'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 2", + ) with Example("bad key type - UInt8"): - decrypt(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43, - message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + decrypt( + ciphertext=ciphertext, + key="123", + mode="'aes-128-ecb'", + exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3", + ) with Example("bad mode type - forgot quotes"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, - message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="aes-128-ecb", + exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query", + ) with Example("bad mode type - UInt8"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43, - message="DB::Exception: Illegal type of argument #1 'mode'") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="128", + exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'", + ) with Example("bad iv type - UInt8"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, - message="DB::Exception: Illegal type of argument") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="128", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) with Example("bad aad type - UInt8"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43, - message="DB::Exception: Illegal type of argument") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-gcm'", + iv="'012345678912'", + aad="123", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) - with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, - message="DB::Exception: aes-128-ecb does not support IV") + with Example( + "iv not valid for mode", + requirements=[ + RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0") + ], + ): + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="'012345678912'", + exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV", + ) - with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): - decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36, - message="DB::Exception: aes-128-ecb does not support IV") + with Example( + "iv not valid for mode - size 0", + requirements=[ + RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0") + ], + ): + decrypt( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="''", + exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV", + ) - with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36, - message="DB::Exception: AAD can be only set for GCM-mode") + with Example( + "aad not valid for mode", + requirements=[ + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode( + "1.0" + ) + ], + ): + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="'0123456789123456'", + aad="'aad'", + exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode", + ) - with Example("invalid mode value", requirements=[RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with Example( + "invalid mode value", + requirements=[ + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0") + ], + ): with When("using unsupported cfb1 mode"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cfb1'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-cfb1") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-cfb1'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-cfb1", + ) with When("using unsupported cfb8 mode"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cfb8'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-cfb8") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-cfb8'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-cfb8", + ) with When("typo in the block algorithm"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-eeb") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-eeb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb", + ) with When("typo in the key size"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-127-ecb") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-127-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb", + ) with When("typo in the aes prefix"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aee-128-ecb") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aee-128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb", + ) with When("missing last dash"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128ecb") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb", + ) with When("missing first dash"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes128-ecb") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb", + ) with When("all capitals"): - decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, - message="DB::Exception: Invalid mode: AES-128-ECB") + decrypt( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'AES-128-ECB'", + exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB", + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"), - RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError( + "1.0" + ), RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"), - RQ_SRS008_AES_Decrypt_Function_NonGCMMode_KeyAndInitializationVector_Length("1.0") + RQ_SRS008_AES_Decrypt_Function_NonGCMMode_KeyAndInitializationVector_Length("1.0"), +) +@Examples( + "mode key_len iv_len aad", + [ + # ECB + ("'aes-128-ecb'", 16, None, None), + ("'aes-192-ecb'", 24, None, None), + ("'aes-256-ecb'", 32, None, None), + # CBC + ("'aes-128-cbc'", 16, 16, None), + ("'aes-192-cbc'", 24, 16, None), + ("'aes-256-cbc'", 32, 16, None), + # CFB128 + ("'aes-128-cfb128'", 16, 16, None), + ("'aes-192-cfb128'", 24, 16, None), + ("'aes-256-cfb128'", 32, 16, None), + # OFB + ("'aes-128-ofb'", 16, 16, None), + ("'aes-192-ofb'", 24, 16, None), + ("'aes-256-ofb'", 32, 16, None), + # CTR + ("'aes-128-ctr'", 16, 16, None), + ("'aes-192-ctr'", 24, 16, None), + ("'aes-256-ctr'", 32, 16, None), + ], + "%-16s %-10s %-10s %-10s", ) -@Examples("mode key_len iv_len aad", [ - # ECB - ("'aes-128-ecb'", 16, None, None), - ("'aes-192-ecb'", 24, None, None), - ("'aes-256-ecb'", 32, None, None), - # CBC - ("'aes-128-cbc'", 16, 16, None), - ("'aes-192-cbc'", 24, 16, None), - ("'aes-256-cbc'", 32, 16, None), - # CFB128 - ("'aes-128-cfb128'", 16, 16, None), - ("'aes-192-cfb128'", 24, 16, None), - ("'aes-256-cfb128'", 32, 16, None), - # OFB - ("'aes-128-ofb'", 16, 16, None), - ("'aes-192-ofb'", 24, 16, None), - ("'aes-256-ofb'", 32, 16, None), - # CTR - ("'aes-128-ctr'", 16, 16, None), - ("'aes-192-ctr'", 24, 16, None), - ("'aes-256-ctr'", 32, 16, None) -], "%-16s %-10s %-10s %-10s") def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): """Check that an error is returned when key or iv length does not match the expected value for the mode. @@ -191,42 +348,90 @@ def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): iv = "0123456789" * 4 with When("key is too short"): - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) with When("key is too long"): - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len+1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) if iv_len is not None: with When("iv is too short"): - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) with When("iv is too long"): - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) if aad is None: with When("aad is specified but not needed"): - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", + aad="'AAD'", + mode=mode, + exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode", + ) else: with When("iv is specified but not needed"): - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'"))) + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv}'", + mode=mode, + exitcode=36, + message="DB::Exception: {} does not support IV".format(mode.strip("'")), + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"), - RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError( + "1.0" + ), RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"), - RQ_SRS008_AES_Decrypt_Function_GCMMode_KeyAndInitializationVector_Length("1.0") + RQ_SRS008_AES_Decrypt_Function_GCMMode_KeyAndInitializationVector_Length("1.0"), +) +@Examples( + "mode key_len iv_len aad", + [ + # GCM + ("'aes-128-gcm'", 16, 8, "'hello there aad'"), + ("'aes-128-gcm'", 16, None, "'hello there aad'"), + ("'aes-192-gcm'", 24, 8, "''"), + ("'aes-192-gcm'", 24, None, "''"), + ("'aes-256-gcm'", 32, 8, "'a'"), + ("'aes-256-gcm'", 32, None, "'a'"), + ], + "%-16s %-10s %-10s %-10s", ) -@Examples("mode key_len iv_len aad", [ - # GCM - ("'aes-128-gcm'", 16, 8, "'hello there aad'"), - ("'aes-128-gcm'", 16, None, "'hello there aad'"), - ("'aes-192-gcm'", 24, 8, "''"), - ("'aes-192-gcm'", 24, None, "''"), - ("'aes-256-gcm'", 32, 8, "'a'"), - ("'aes-256-gcm'", 32, None, "'a'") -], "%-16s %-10s %-10s %-10s") def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): """Check that an error is returned when key or iv length does not match the expected value for the GCM mode. @@ -238,25 +443,57 @@ def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): with When("key is too short"): ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len-1]}'", + iv=f"'{iv[:iv_len]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) with When("key is too long"): ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len+1]}'", + iv=f"'{iv[:iv_len]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) if iv_len is not None: with When(f"iv is too short"): - ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=198, message="DB::Exception:") + ciphertext = ( + "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" + ) + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len-1]}'", + mode=mode, + exitcode=198, + message="DB::Exception:", + ) else: with When("iv is not specified"): - ciphertext = "unhex('1CD4EC93A4B0C687926E8F8C2AA3B4CE1943D006DAE3A774CB1AE5')" - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size 0 != expected size 12") + ciphertext = ( + "unhex('1CD4EC93A4B0C687926E8F8C2AA3B4CE1943D006DAE3A774CB1AE5')" + ) + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size 0 != expected size 12", + ) + @TestScenario @Requirements( RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"), - RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length("1.0") + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length("1.0"), ) def aad_parameter_types_and_length(self): """Check that `decrypt` function accepts `aad` parameter as the fifth argument @@ -269,36 +506,84 @@ def aad_parameter_types_and_length(self): with When("aad is specified using String type"): ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'aad'", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad="'aad'", + message=plaintext, + ) with When("aad is specified using String with UTF8 characters"): ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad="'Gãńdåłf_Thê_Gręât'", + message=plaintext, + ) with When("aad is specified using FixedString type"): ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad="toFixedString('aad', 3)", + message=plaintext, + ) with When("aad is specified using FixedString with UTF8 characters"): ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", + message=plaintext, + ) with When("aad is 0 bytes"): ciphertext = "unhex('19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="''", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad="''", + message=plaintext, + ) with When("aad is 1 byte"): ciphertext = "unhex('19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'1'", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad="'1'", + message=plaintext, + ) with When("aad is 256 bytes"): ciphertext = "unhex('19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7')" - decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message=plaintext) + decrypt( + ciphertext=ciphertext, + key=key, + mode=mode, + iv=iv, + aad=f"'{'1' * 256}'", + message=plaintext, + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector("1.0") -) +@Requirements(RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector("1.0")) def iv_parameter_types(self): """Check that `decrypt` function accepts `iv` parameter as the fourth argument of either `String` or `FixedString` types. @@ -308,21 +593,44 @@ def iv_parameter_types(self): key = "'0123456789123456'" with When("iv is specified using String type"): - decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there") + decrypt( + ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", + key=key, + mode=mode, + iv=iv, + message="hello there", + ) with When("iv is specified using String with UTF8 characters"): - decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there") + decrypt( + ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", + key=key, + mode=mode, + iv="'Gãńdåłf_Thê'", + message="hello there", + ) with When("iv is specified using FixedString type"): - decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there") + decrypt( + ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", + key=key, + mode=mode, + iv=f"toFixedString({iv}, 16)", + message="hello there", + ) with When("iv is specified using FixedString with UTF8 characters"): - decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there") + decrypt( + ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", + key=key, + mode=mode, + iv=f"toFixedString('Gãńdåłf_Thê', 16)", + message="hello there", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Decrypt_Function_Parameters_Key("1.0") -) +@Requirements(RQ_SRS008_AES_Decrypt_Function_Parameters_Key("1.0")) def key_parameter_types(self): """Check that `decrypt` function accepts `key` parameter as the second argument of either `String` or `FixedString` types. @@ -332,16 +640,37 @@ def key_parameter_types(self): key = "'0123456789123456'" with When("key is specified using String type"): - decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + decrypt( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=key, + mode=mode, + message="hello there", + ) with When("key is specified using String with UTF8 characters"): - decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there") + decrypt( + ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", + key="'Gãńdåłf_Thê'", + mode=mode, + message="hello there", + ) with When("key is specified using FixedString type"): - decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there") + decrypt( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=f"toFixedString({key}, 16)", + mode=mode, + message="hello there", + ) with When("key is specified using FixedString with UTF8 characters"): - decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there") + decrypt( + ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", + key=f"toFixedString('Gãńdåłf_Thê', 16)", + mode=mode, + message="hello there", + ) + @TestScenario @Requirements( @@ -355,25 +684,43 @@ def mode_parameter_types(self): key = "'0123456789123456'" with When("mode is specified using String type"): - decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + decrypt( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=key, + mode=mode, + message="hello there", + ) with When("mode is specified using FixedString type"): - decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there") + decrypt( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=key, + mode=f"toFixedString({mode}, 12)", + message="hello there", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue("1.0") -) +@Requirements(RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue("1.0")) def return_value(self): - """Check that `decrypt` functions returns String data type. - """ + """Check that `decrypt` functions returns String data type.""" ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')" iv = "'0123456789123456'" mode = "'aes-128-cbc'" key = "'0123456789123456'" with When("I get type of the return value"): - sql = "SELECT toTypeName(decrypt(" + mode + "," + ciphertext + "," + key + "," + iv + "))" + sql = ( + "SELECT toTypeName(decrypt(" + + mode + + "," + + ciphertext + + "," + + key + + "," + + iv + + "))" + ) r = self.context.node.query(sql) with Then("type should be String"): @@ -382,6 +729,7 @@ def return_value(self): with When("I get the return value"): decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there") + @TestScenario @Requirements( RQ_SRS008_AES_Decrypt_Function_Syntax("1.0"), @@ -397,12 +745,13 @@ def syntax(self): sql = f"SELECT decrypt('aes-128-gcm', unhex('{ciphertext}'), '0123456789123456', '012345678912', 'AAD')" self.context.node.query(sql, step=When, message="hello there") + @TestScenario @Requirements( RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"), RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"), RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"), - RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Values("1.0") + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Values("1.0"), ) def decryption(self): """Check that `decrypt` functions accepts `ciphertext` as the second parameter @@ -414,17 +763,23 @@ def decryption(self): aad = "some random aad" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), - "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: for datatype, plaintext in plaintexts: - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) cast = None endcast = None @@ -436,18 +791,23 @@ def decryption(self): cast = "isNull" compare = None - decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, + decrypt( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, iv=(None if not iv_len else f"'{iv[:iv_len]}'"), aad=(None if not aad_len else f"'{aad}'"), - cast=cast, endcast=endcast, compare=compare, message="1") + cast=cast, + endcast=endcast, + compare=compare, + message="1", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_Key("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_Key("1.0")) def mismatched_key(self): - """Check that `decrypt` function returns garbage or an error when key parameter does not match. - """ + """Check that `decrypt` function returns garbage or an error when key parameter does not match.""" key = f"{'1' * 36}" iv = f"{'2' * 16}" aad = "some random aad" @@ -455,34 +815,46 @@ def mismatched_key(self): plaintext = "'1'" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), - "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) with When("I decrypt using a mismatched key"): - r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode, + r = decrypt( + ciphertext=f"unhex({ciphertext})", + key=f"'a{key[:key_len-1]}'", + mode=mode, iv=(None if not iv_len else f"'{iv[:iv_len]}'"), - aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + aad=(None if not aad_len else f"'{aad}'"), + no_checks=True, + cast="hex", + ) with Then("exitcode shoud be 0 or 198"): assert r.exitcode in [0, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - assert "Exception: Failed to decrypt" in output or output != "31", error() + assert ( + "Exception: Failed to decrypt" in output or output != "31" + ), error() + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_IV("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_IV("1.0")) def mismatched_iv(self): - """Check that `decrypt` function returns garbage or an error when iv parameter does not match. - """ + """Check that `decrypt` function returns garbage or an error when iv parameter does not match.""" key = f"{'1' * 36}" iv = f"{'2' * 16}" aad = "some random aad" @@ -490,35 +862,48 @@ def mismatched_iv(self): plaintext = "'1'" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: if not iv_len: continue - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) with When("I decrypt using a mismatched iv"): - r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + r = decrypt( + ciphertext=f"unhex({ciphertext})", + key=f"'{key[:key_len]}'", + mode=mode, iv=f"'a{iv[:iv_len-1]}'", - aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + aad=(None if not aad_len else f"'{aad}'"), + no_checks=True, + cast="hex", + ) with Then("exitcode shoud be 0 or 198"): assert r.exitcode in [0, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - assert "Exception: Failed to decrypt" in output or output != "31", error() + assert ( + "Exception: Failed to decrypt" in output or output != "31" + ), error() + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_AAD("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_AAD("1.0")) def mismatched_aad(self): - """Check that `decrypt` function returns garbage or an error when aad parameter does not match. - """ + """Check that `decrypt` function returns garbage or an error when aad parameter does not match.""" key = f"{'1' * 36}" iv = f"{'2' * 16}" aad = "some random aad" @@ -526,76 +911,102 @@ def mismatched_aad(self): plaintext = "'1'" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: if not aad_len: continue - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) with When("I decrypt using a mismatched aad"): - r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + r = decrypt( + ciphertext=f"unhex({ciphertext})", + key=f"'{key[:key_len]}'", + mode=mode, iv=(None if not iv_len else f"'{iv[:iv_len]}'"), - aad=(None if not aad_len else f"'a{aad}'"), no_checks=True, cast="hex") + aad=(None if not aad_len else f"'a{aad}'"), + no_checks=True, + cast="hex", + ) with Then("exitcode shoud be 0 or 198"): assert r.exitcode in [0, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - assert "Exception: Failed to decrypt" in output or output != "31", error() + assert ( + "Exception: Failed to decrypt" in output or output != "31" + ), error() + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_Mode("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_Mode("1.0")) def mismatched_mode(self): - """Check that `decrypt` function returns garbage or an error when mode parameter does not match. - """ + """Check that `decrypt` function returns garbage or an error when mode parameter does not match.""" key = f"{'1' * 36}" iv = f"{'2' * 16}" aad = "some random aad" - plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8")) + plaintext = hex("Gãńdåłf_Thê_Gręât".encode("utf-8")) with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot"), + ).load_module() for mode, key_len, iv_len, aad_len in modes: - with Example(f"""mode={mode.strip("'")} datatype=utf8string iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype=utf8string iv={iv_len} aad={aad_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) for mismatched_mode, _, _, _ in modes: if mismatched_mode == mode: continue with When(f"I decrypt using mismatched mode {mismatched_mode}"): - r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode, + r = decrypt( + ciphertext=f"unhex({ciphertext})", + key=f"'{key[:key_len]}'", + mode=mismatched_mode, iv=(None if not iv_len else f"'{iv[:iv_len]}'"), - aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + aad=(None if not aad_len else f"'{aad}'"), + no_checks=True, + cast="hex", + ) with Then("exitcode shoud be 0 or 36 or 198"): assert r.exitcode in [0, 36, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - condition = "Exception: Failed to decrypt" in output \ - or 'Exception: Invalid key size' in output \ + condition = ( + "Exception: Failed to decrypt" in output + or "Exception: Invalid key size" in output or output != plaintext + ) assert condition, error() + @TestFeature @Name("decrypt") -@Requirements( - RQ_SRS008_AES_Decrypt_Function("1.0") -) +@Requirements(RQ_SRS008_AES_Decrypt_Function("1.0")) def feature(self, node="clickhouse1"): - """Check the behavior of the `decrypt` function. - """ + """Check the behavior of the `decrypt` function.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/aes_encryption/tests/decrypt_mysql.py b/tests/testflows/aes_encryption/tests/decrypt_mysql.py index 52236ae0910..1a8f53464b7 100644 --- a/tests/testflows/aes_encryption/tests/decrypt_mysql.py +++ b/tests/testflows/aes_encryption/tests/decrypt_mysql.py @@ -10,11 +10,24 @@ from testflows.asserts import error from aes_encryption.requirements.requirements import * from aes_encryption.tests.common import * + @TestOutline -def aes_decrypt_mysql(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, - step=When, cast=None, endcast=None, compare=None, no_checks=False): - """Execute `aes_decrypt_mysql` function with the specified parameters. - """ +def aes_decrypt_mysql( + self, + ciphertext=None, + key=None, + mode=None, + iv=None, + aad=None, + exitcode=0, + message=None, + step=When, + cast=None, + endcast=None, + compare=None, + no_checks=False, +): + """Execute `aes_decrypt_mysql` function with the specified parameters.""" params = [] if mode is not None: params.append(mode) @@ -34,7 +47,10 @@ def aes_decrypt_mysql(self, ciphertext=None, key=None, mode=None, iv=None, aad=N sql = f"{compare} = {sql}" sql = f"SELECT {sql}" - return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks) + return current().context.node.query( + sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks + ) + @TestScenario @Requirements( @@ -57,39 +73,60 @@ def invalid_ciphertext(self): continue with When(f"invalid ciphertext={ciphertext}"): if "cfb" in mode or "ofb" in mode or "ctr" in mode: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, cast="hex") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, + iv=d_iv, + cast="hex", + ) else: with When("I execute aes_decrypt_mysql function"): - r = aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, no_checks=True, step=By) + r = aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, + iv=d_iv, + no_checks=True, + step=By, + ) with Then("exitcode is not zero"): assert r.exitcode in [198, 36] with And("exception is present in the output"): assert "DB::Exception:" in r.output + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_GCM_Error("1.0"), - RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_CTR_Error("1.0") + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values_CTR_Error("1.0"), +) +@Examples( + "mode", + [ + ("'aes-128-gcm'",), + ("'aes-192-gcm'",), + ("'aes-256-gcm'",), + ("'aes-128-ctr'",), + ("'aes-192-ctr'",), + ("'aes-256-ctr'",), + ], ) -@Examples("mode", [ - ("'aes-128-gcm'",), - ("'aes-192-gcm'",), - ("'aes-256-gcm'",), - ("'aes-128-ctr'",), - ("'aes-192-ctr'",), - ("'aes-256-ctr'",) -]) def unsupported_modes(self, mode): - """Check that `aes_decrypt_mysql` function returns an error when unsupported modes are specified. - """ + """Check that `aes_decrypt_mysql` function returns an error when unsupported modes are specified.""" ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" - aes_decrypt_mysql(ciphertext=ciphertext, mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode") + aes_decrypt_mysql( + ciphertext=ciphertext, + mode=mode, + key=f"'{'1'* 32}'", + exitcode=36, + message="DB::Exception: Unsupported cipher mode", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_InvalidParameters("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_InvalidParameters("1.0")) def invalid_parameters(self): """Check that `aes_decrypt_mysql` function returns an error when we call it with invalid parameters. @@ -97,103 +134,215 @@ def invalid_parameters(self): ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" with Example("no parameters"): - aes_decrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 0, expected 3 to 4") + aes_decrypt_mysql( + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 0, expected 3 to 4", + ) with Example("missing key and mode"): - aes_decrypt_mysql(ciphertext=ciphertext, exitcode=42, - message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 1") + aes_decrypt_mysql( + ciphertext=ciphertext, + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 1", + ) with Example("missing mode"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'123'", exitcode=42, - message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 2") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'123'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 2", + ) with Example("bad key type - UInt8"): - aes_decrypt_mysql(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43, - message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="123", + mode="'aes-128-ecb'", + exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3", + ) with Example("bad mode type - forgot quotes"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, - message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="aes-128-ecb", + exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query", + ) with Example("bad mode type - UInt8"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43, - message="DB::Exception: Illegal type of argument #1 'mode'") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="128", + exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'", + ) with Example("bad iv type - UInt8"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, - message="DB::Exception: Illegal type of argument") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="128", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) - with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): - aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=0, - message=None) + with Example( + "iv not valid for mode", + requirements=[ + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode( + "1.0" + ) + ], + ): + aes_decrypt_mysql( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="'012345678912'", + exitcode=0, + message=None, + ) - with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): - aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0, - message=None) + with Example( + "iv not valid for mode - size 0", + requirements=[ + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode( + "1.0" + ) + ], + ): + aes_decrypt_mysql( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="''", + exitcode=0, + message=None, + ) with Example("aad passed by mistake"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=42, - message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="'0123456789123456'", + aad="'aad'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5", + ) with Example("aad passed by mistake type - UInt8"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=42, - message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-gcm'", + iv="'012345678912'", + aad="123", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5", + ) - with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with Example( + "invalid mode value", + requirements=[ + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0") + ], + ): with When("typo in the block algorithm"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-eeb") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128-eeb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb", + ) with When("typo in the key size"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-127-ecb") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-127-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb", + ) with When("typo in the aes prefix"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aee-128-ecb") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aee-128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb", + ) with When("missing last dash"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128ecb") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes-128ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb", + ) with When("missing first dash"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes128-ecb") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'aes128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb", + ) with When("all capitals"): - aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, - message="DB::Exception: Invalid mode: AES-128-ECB") + aes_decrypt_mysql( + ciphertext=ciphertext, + key="'0123456789123456'", + mode="'AES-128-ECB'", + exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB", + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError("1.0"), RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong("1.0"), - RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError( + "1.0" + ), RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong("1.0"), RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0"), - RQ_SRS008_AES_MySQL_Decrypt_Function_Mode_KeyAndInitializationVector_Length("1.0") + RQ_SRS008_AES_MySQL_Decrypt_Function_Mode_KeyAndInitializationVector_Length("1.0"), +) +@Examples( + "mode key_len iv_len", + [ + # ECB + ("'aes-128-ecb'", 16, None), + ("'aes-192-ecb'", 24, None), + ("'aes-256-ecb'", 32, None), + # CBC + ("'aes-128-cbc'", 16, 16), + ("'aes-192-cbc'", 24, 16), + ("'aes-256-cbc'", 32, 16), + # CFB128 + ("'aes-128-cfb128'", 16, 16), + ("'aes-192-cfb128'", 24, 16), + ("'aes-256-cfb128'", 32, 16), + # OFB + ("'aes-128-ofb'", 16, 16), + ("'aes-192-ofb'", 24, 16), + ("'aes-256-ofb'", 32, 16), + ], + "%-16s %-10s %-10s", ) -@Examples("mode key_len iv_len", [ - # ECB - ("'aes-128-ecb'", 16, None), - ("'aes-192-ecb'", 24, None), - ("'aes-256-ecb'", 32, None), - # CBC - ("'aes-128-cbc'", 16, 16), - ("'aes-192-cbc'", 24, 16), - ("'aes-256-cbc'", 32, 16), - # CFB128 - ("'aes-128-cfb128'", 16, 16), - ("'aes-192-cfb128'", 24, 16), - ("'aes-256-cfb128'", 32, 16), - # OFB - ("'aes-128-ofb'", 16, 16), - ("'aes-192-ofb'", 24, 16), - ("'aes-256-ofb'", 32, 16) -], "%-16s %-10s %-10s") def key_or_iv_length_for_mode(self, mode, key_len, iv_len): - """Check that key or iv length for mode. - """ + """Check that key or iv length for mode.""" ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3A')" if mode == "'aes-128-ecb'": ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3B')" @@ -205,29 +354,76 @@ def key_or_iv_length_for_mode(self, mode, key_len, iv_len): iv = "0123456789" * 4 with When("key is too short"): - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) with When("key is too long"): if "ecb" in mode or "cbc" in mode: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len+1]}'", + mode=mode, + exitcode=198, + message="DB::Exception: Failed to decrypt", + ) else: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, cast="hex") + aes_decrypt_mysql( + ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, cast="hex" + ) if iv_len is not None: with When("iv is too short"): - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) with When("iv is too long"): if "ecb" in mode or "cbc" in mode: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1]}'", + mode=mode, + exitcode=198, + message="DB::Exception: Failed to decrypt", + ) else: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, cast="hex") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1]}'", + mode=mode, + cast="hex", + ) else: with When("iv is specified but not needed"): if "ecb" in mode or "cbc" in mode: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv}'", + mode=mode, + exitcode=198, + message="DB::Exception: Failed to decrypt", + ) else: - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode) + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + iv=f"'{iv}'", + mode=mode, + ) + @TestScenario @Requirements( @@ -242,21 +438,44 @@ def iv_parameter_types(self): key = "'0123456789123456'" with When("iv is specified using String type"): - aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", + key=key, + mode=mode, + iv=iv, + message="hello there", + ) with When("iv is specified using String with UTF8 characters"): - aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", + key=key, + mode=mode, + iv="'Gãńdåłf_Thê'", + message="hello there", + ) with When("iv is specified using FixedString type"): - aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", + key=key, + mode=mode, + iv=f"toFixedString({iv}, 16)", + message="hello there", + ) with When("iv is specified using FixedString with UTF8 characters"): - aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", + key=key, + mode=mode, + iv=f"toFixedString('Gãńdåłf_Thê', 16)", + message="hello there", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key("1.0") -) +@Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key("1.0")) def key_parameter_types(self): """Check that `aes_decrypt` function accepts `key` parameter as the second argument of either `String` or `FixedString` types. @@ -266,16 +485,37 @@ def key_parameter_types(self): key = "'0123456789123456'" with When("key is specified using String type"): - aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=key, + mode=mode, + message="hello there", + ) with When("key is specified using String with UTF8 characters"): - aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", + key="'Gãńdåłf_Thê'", + mode=mode, + message="hello there", + ) with When("key is specified using FixedString type"): - aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=f"toFixedString({key}, 16)", + mode=mode, + message="hello there", + ) with When("key is specified using FixedString with UTF8 characters"): - aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", + key=f"toFixedString('Gãńdåłf_Thê', 16)", + mode=mode, + message="hello there", + ) + @TestScenario @Requirements( @@ -289,32 +529,53 @@ def mode_parameter_types(self): key = "'0123456789123456'" with When("mode is specified using String type"): - aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=key, + mode=mode, + message="hello there", + ) with When("mode is specified using FixedString type"): - aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there") + aes_decrypt_mysql( + ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", + key=key, + mode=f"toFixedString({mode}, 12)", + message="hello there", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue("1.0") -) +@Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue("1.0")) def return_value(self): - """Check that `aes_decrypt_mysql` functions returns String data type. - """ + """Check that `aes_decrypt_mysql` functions returns String data type.""" ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')" iv = "'0123456789123456'" mode = "'aes-128-cbc'" key = "'0123456789123456'" with When("I get type of the return value"): - sql = "SELECT toTypeName(aes_decrypt_mysql(" + mode + "," + ciphertext + "," + key + "," + iv + "))" + sql = ( + "SELECT toTypeName(aes_decrypt_mysql(" + + mode + + "," + + ciphertext + + "," + + key + + "," + + iv + + "))" + ) r = self.context.node.query(sql) with Then("type should be String"): assert r.output.strip() == "String", error() with When("I get the return value"): - aes_decrypt_mysql(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there") + aes_decrypt_mysql( + ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there" + ) + @TestScenario @Requirements( @@ -331,12 +592,13 @@ def syntax(self): sql = f"SELECT aes_decrypt_mysql('aes-128-ofb', unhex('{ciphertext}'), '0123456789123456', '0123456789123456')" self.context.node.query(sql, step=When, message="hello there") + @TestScenario @Requirements( RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"), RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"), RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"), - RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values("1.0") + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Values("1.0"), ) def decryption(self): """Check that `aes_decrypt_mysql` functions accepts `mode` as the first parameter @@ -347,17 +609,25 @@ def decryption(self): iv = f"{'2' * 64}" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), - "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join( + current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot" + ), + ).load_module() for mode, key_len, iv_len in mysql_modes: for datatype, plaintext in plaintexts: - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) cast = None endcast = None @@ -369,124 +639,169 @@ def decryption(self): cast = "isNull" compare = None - aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, + aes_decrypt_mysql( + ciphertext=ciphertext, + key=f"'{key[:key_len]}'", + mode=mode, iv=(None if not iv_len else f"'{iv[:iv_len]}'"), - cast=cast, endcast=endcast, compare=compare, message="1") + cast=cast, + endcast=endcast, + compare=compare, + message="1", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_Key("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_Key("1.0")) def mismatched_key(self): - """Check that `aes_decrypt_mysql` function returns garbage or an error when key parameter does not match. - """ + """Check that `aes_decrypt_mysql` function returns garbage or an error when key parameter does not match.""" key = f"{'1' * 64}" iv = f"{'2' * 64}" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), - "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join( + current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot" + ), + ).load_module() for mode, key_len, iv_len in mysql_modes: - with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) with When("I decrypt using a mismatched key"): - r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode, + r = aes_decrypt_mysql( + ciphertext=f"unhex({ciphertext})", + key=f"'a{key[:key_len-1]}'", + mode=mode, iv=(None if not iv_len else f"'{iv[:iv_len]}'"), - cast="hex", no_checks=True) + cast="hex", + no_checks=True, + ) with Then("exitcode shoud be 0 or 198"): assert r.exitcode in [0, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - assert "Exception: Failed to decrypt" in output or output != "31", error() + assert ( + "Exception: Failed to decrypt" in output or output != "31" + ), error() + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_IV("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_IV("1.0")) def mismatched_iv(self): - """Check that `aes_decrypt_mysql` function returns garbage or an error when iv parameter does not match. - """ + """Check that `aes_decrypt_mysql` function returns garbage or an error when iv parameter does not match.""" key = f"{'1' * 64}" iv = f"{'2' * 64}" with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), - "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join( + current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot" + ), + ).load_module() for mode, key_len, iv_len in mysql_modes: if not iv_len: continue - with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) with When("I decrypt using a mismatched key"): - r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + r = aes_decrypt_mysql( + ciphertext=f"unhex({ciphertext})", + key=f"'{key[:key_len]}'", + mode=mode, iv=f"'a{iv[:iv_len-1]}'", - cast="hex", no_checks=True) + cast="hex", + no_checks=True, + ) with Then("exitcode shoud be 0 or 198"): assert r.exitcode in [0, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - assert "Exception: Failed to decrypt" in output or output != "31", error() + assert ( + "Exception: Failed to decrypt" in output or output != "31" + ), error() + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_Mismatched_Mode("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_Mismatched_Mode("1.0")) def mismatched_mode(self): - """Check that `aes_decrypt_mysql` function returns garbage or an error when mode parameter does not match. - """ + """Check that `aes_decrypt_mysql` function returns garbage or an error when mode parameter does not match.""" key = f"{'1' * 64}" iv = f"{'2' * 64}" - plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8")) + plaintext = hex("Gãńdåłf_Thê_Gręât".encode("utf-8")) with Given("I load encrypt snapshots"): - snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), - "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + snapshot_module = SourceFileLoader( + "snapshot", + os.path.join( + current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot" + ), + ).load_module() for mode, key_len, iv_len in mysql_modes: if not iv_len: continue - with Example(f"""mode={mode.strip("'")} datatype=utf8string key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype=utf8string key={key_len} iv={iv_len}""" + ) as example: with Given("I have ciphertext"): example_name = basename(example.name) - ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + ciphertext = getattr( + snapshot_module, varname(f"example_{example_name}") + ) for mismatched_mode, _, _ in mysql_modes: if mismatched_mode == mode: continue with When(f"I decrypt using a mismatched mode {mismatched_mode}"): - r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode, + r = aes_decrypt_mysql( + ciphertext=f"unhex({ciphertext})", + key=f"'{key[:key_len]}'", + mode=mismatched_mode, iv=f"'{iv[:iv_len]}'", - cast="hex", no_checks=True) + cast="hex", + no_checks=True, + ) with Then("exitcode shoud be 0 or 36 or 198"): assert r.exitcode in [0, 36, 198], error() with And("output should be garbage or an error"): output = r.output.strip() - assert "Exception: Failed to decrypt" in output or output != plaintext, error() + assert ( + "Exception: Failed to decrypt" in output + or output != plaintext + ), error() + @TestFeature @Name("decrypt_mysql") -@Requirements( - RQ_SRS008_AES_MySQL_Decrypt_Function("1.0") -) +@Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function("1.0")) def feature(self, node="clickhouse1"): - """Check the behavior of the `aes_decrypt_mysql` function. - """ + """Check the behavior of the `aes_decrypt_mysql` function.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/aes_encryption/tests/encrypt.py b/tests/testflows/aes_encryption/tests/encrypt.py index dde27c9d454..b18a721d297 100644 --- a/tests/testflows/aes_encryption/tests/encrypt.py +++ b/tests/testflows/aes_encryption/tests/encrypt.py @@ -6,10 +6,20 @@ from testflows.asserts import values, error, snapshot from aes_encryption.requirements.requirements import * from aes_encryption.tests.common import * + @TestOutline -def encrypt(self, plaintext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When): - """Execute `encrypt` function with the specified parameters. - """ +def encrypt( + self, + plaintext=None, + key=None, + mode=None, + iv=None, + aad=None, + exitcode=0, + message=None, + step=When, +): + """Execute `encrypt` function with the specified parameters.""" params = [] if mode is not None: params.append(mode) @@ -24,156 +34,296 @@ def encrypt(self, plaintext=None, key=None, mode=None, iv=None, aad=None, exitco sql = "SELECT hex(encrypt(" + ", ".join(params) + "))" - return current().context.node.query(sql, step=step, exitcode=exitcode, message=message) + return current().context.node.query( + sql, step=step, exitcode=exitcode, message=message + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_InvalidParameters("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_InvalidParameters("1.0")) def invalid_parameters(self): """Check that `encrypt` function returns an error when we call it with invalid parameters. """ with Example("no parameters"): - encrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 0, expected 3 to 5") + encrypt( + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function encrypt provided 0, expected 3 to 5", + ) with Example("missing key and mode"): - encrypt(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 1") + encrypt( + plaintext="'hello there'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function encrypt provided 1", + ) with Example("missing mode"): - encrypt(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 2") + encrypt( + plaintext="'hello there'", + key="'123'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function encrypt provided 2", + ) with Example("bad key type - UInt8"): - encrypt(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43, - message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + encrypt( + plaintext="'hello there'", + key="123", + mode="'aes-128-ecb'", + exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3", + ) with Example("bad mode type - forgot quotes"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, - message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="aes-128-ecb", + exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query", + ) with Example("bad mode type - UInt8"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43, - message="DB::Exception: Illegal type of argument #1 'mode'") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="128", + exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'", + ) with Example("bad iv type - UInt8"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, - message="DB::Exception: Illegal type of argument") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="128", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) with Example("bad aad type - UInt8"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43, - message="DB::Exception: Illegal type of argument") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-gcm'", + iv="'012345678912'", + aad="123", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) - with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, - message="DB::Exception: aes-128-ecb does not support IV") + with Example( + "iv not valid for mode", + requirements=[ + RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0") + ], + ): + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="'012345678912'", + exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV", + ) - with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36, - message="DB::Exception: aes-128-ecb does not support IV") + with Example( + "iv not valid for mode - size 0", + requirements=[ + RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0") + ], + ): + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="''", + exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV", + ) - with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36, - message="DB::Exception: AAD can be only set for GCM-mode") + with Example( + "aad not valid for mode", + requirements=[ + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode( + "1.0" + ) + ], + ): + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="'0123456789123456'", + aad="'aad'", + exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode", + ) - with Example("invalid mode value", requirements=[RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with Example( + "invalid mode value", + requirements=[ + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0") + ], + ): with When("using unsupported cfb1 mode"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cfb1'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-cfb1") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-cfb1'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-cfb1", + ) with When("using unsupported cfb8 mode"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cfb8'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-cfb8") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-cfb8'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-cfb8", + ) with When("typo in the block algorithm"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-eeb") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-eeb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb", + ) with When("typo in the key size"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-127-ecb") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-127-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb", + ) with When("typo in the aes prefix"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aee-128-ecb") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aee-128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb", + ) with When("missing last dash"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128ecb") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb", + ) with When("missing first dash"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes128-ecb") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb", + ) with When("all capitals"): - encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, - message="DB::Exception: Invalid mode: AES-128-ECB") + encrypt( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'AES-128-ECB'", + exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB", + ) + @TestOutline(Scenario) -@Requirements( - RQ_SRS008_AES_Functions_InvalidParameters("1.0") +@Requirements(RQ_SRS008_AES_Functions_InvalidParameters("1.0")) +@Examples( + "data_type, value", + [ + ("UInt8", "toUInt8('1')"), + ("UInt16", "toUInt16('1')"), + ("UInt32", "toUInt32('1')"), + ("UInt64", "toUInt64('1')"), + ("Int8", "toInt8('1')"), + ("Int16", "toInt16('1')"), + ("Int32", "toInt32('1')"), + ("Int64", "toInt64('1')"), + ("Float32", "toFloat32('1.0')"), + ("Float64", "toFloat64('1.0')"), + ("Decimal32", "toDecimal32(2, 4)"), + ("Decimal64", "toDecimal64(2, 4)"), + ("Decimal128", "toDecimal128(2, 4)"), + ("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), + ("Date", "toDate('2020-01-01')"), + ("DateTime", "toDateTime('2020-01-01 20:01:02')"), + ("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"), + ("Array", "[1,2]"), + ("Tuple", "(1,'a')"), + ("IPv4", "toIPv4('171.225.130.45')"), + ("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"), + ("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"), + ("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')"), + ], ) -@Examples("data_type, value", [ - ("UInt8", "toUInt8('1')"), - ("UInt16", "toUInt16('1')"), - ("UInt32", "toUInt32('1')"), - ("UInt64", "toUInt64('1')"), - ("Int8", "toInt8('1')"), - ("Int16", "toInt16('1')"), - ("Int32", "toInt32('1')"), - ("Int64", "toInt64('1')"), - ("Float32", "toFloat32('1.0')"), - ("Float64", "toFloat64('1.0')"), - ("Decimal32", "toDecimal32(2, 4)"), - ("Decimal64", "toDecimal64(2, 4)"), - ("Decimal128", "toDecimal128(2, 4)"), - ("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), - ("Date", "toDate('2020-01-01')"), - ("DateTime", "toDateTime('2020-01-01 20:01:02')"), - ("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"), - ("Array", "[1,2]"), - ("Tuple", "(1,'a')"), - ("IPv4", "toIPv4('171.225.130.45')"), - ("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"), - ("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"), - ("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')") -]) def invalid_plaintext_data_type(self, data_type, value): """Check that encrypt function returns an error if the plaintext parameter has invalid data type. """ - with When("I try to encrypt plaintext with invalid data type", description=f"{data_type} with value {value}"): - encrypt(plaintext=value, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", - exitcode=43, message="DB::Exception: Illegal type of argument") + with When( + "I try to encrypt plaintext with invalid data type", + description=f"{data_type} with value {value}", + ): + encrypt( + plaintext=value, + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="'0123456789123456'", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"), - RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError( + "1.0" + ), RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"), - RQ_SRS008_AES_Encrypt_Function_NonGCMMode_KeyAndInitializationVector_Length("1.0") + RQ_SRS008_AES_Encrypt_Function_NonGCMMode_KeyAndInitializationVector_Length("1.0"), +) +@Examples( + "mode key_len iv_len aad", + [ + # ECB + ("'aes-128-ecb'", 16, None, None), + ("'aes-192-ecb'", 24, None, None), + ("'aes-256-ecb'", 32, None, None), + # CBC + ("'aes-128-cbc'", 16, 16, None), + ("'aes-192-cbc'", 24, 16, None), + ("'aes-256-cbc'", 32, 16, None), + # CFB128 + ("'aes-128-cfb128'", 16, 16, None), + ("'aes-192-cfb128'", 24, 16, None), + ("'aes-256-cfb128'", 32, 16, None), + # OFB + ("'aes-128-ofb'", 16, 16, None), + ("'aes-192-ofb'", 24, 16, None), + ("'aes-256-ofb'", 32, 16, None), + # CTR + ("'aes-128-ctr'", 16, 16, None), + ("'aes-192-ctr'", 24, 16, None), + ("'aes-256-ctr'", 32, 16, None), + ], + "%-16s %-10s %-10s %-10s", ) -@Examples("mode key_len iv_len aad", [ - # ECB - ("'aes-128-ecb'", 16, None, None), - ("'aes-192-ecb'", 24, None, None), - ("'aes-256-ecb'", 32, None, None), - # CBC - ("'aes-128-cbc'", 16, 16, None), - ("'aes-192-cbc'", 24, 16, None), - ("'aes-256-cbc'", 32, 16, None), - # CFB128 - ("'aes-128-cfb128'", 16, 16, None), - ("'aes-192-cfb128'", 24, 16, None), - ("'aes-256-cfb128'", 32, 16, None), - # OFB - ("'aes-128-ofb'", 16, 16, None), - ("'aes-192-ofb'", 24, 16, None), - ("'aes-256-ofb'", 32, 16, None), - # CTR - ("'aes-128-ctr'", 16, 16, None), - ("'aes-192-ctr'", 24, 16, None), - ("'aes-256-ctr'", 32, 16, None), -], "%-16s %-10s %-10s %-10s") def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): """Check that an error is returned when key or iv length does not match the expected value for the mode. @@ -183,39 +333,87 @@ def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): iv = "0123456789" * 4 with When("key is too short"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) with When("key is too long"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len+1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) if iv_len is not None: with When("iv is too short"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) with When("iv is too long"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) if aad is None: with When("aad is specified but not needed"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", + aad="'AAD'", + mode=mode, + exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode", + ) else: with When("iv is specified but not needed"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'"))) + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv}'", + mode=mode, + exitcode=36, + message="DB::Exception: {} does not support IV".format(mode.strip("'")), + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"), - RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError( + "1.0" + ), RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0"), - RQ_SRS008_AES_Encrypt_Function_GCMMode_KeyAndInitializationVector_Length("1.0") + RQ_SRS008_AES_Encrypt_Function_GCMMode_KeyAndInitializationVector_Length("1.0"), +) +@Examples( + "mode key_len iv_len aad", + [ + # GCM + ("'aes-128-gcm'", 16, 8, "'hello there aad'"), + ("'aes-192-gcm'", 24, 8, "''"), + ("'aes-256-gcm'", 32, 8, "'a'"), + ], + "%-16s %-10s %-10s %-10s", ) -@Examples("mode key_len iv_len aad", [ - # GCM - ("'aes-128-gcm'", 16, 8, "'hello there aad'"), - ("'aes-192-gcm'", 24, 8, "''"), - ("'aes-256-gcm'", 32, 8, "'a'"), -], "%-16s %-10s %-10s %-10s") def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): """Check that an error is returned when key or iv length does not match the expected value for the GCM mode. @@ -225,26 +423,59 @@ def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): iv = "0123456789" * 4 with When("key is too short"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len-1]}'", + iv=f"'{iv[:iv_len]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) with When("key is too long"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len+1]}'", + iv=f"'{iv[:iv_len]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) if iv_len is not None: with When(f"iv is too short"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=0) + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len-1]}'", + mode=mode, + exitcode=0, + ) else: with When("iv is not specified"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) if aad is not None: with When(f"aad is {aad}"): - encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len]}'", aad=f"{aad}", mode=mode) + encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len]}'", + aad=f"{aad}", + mode=mode, + ) + @TestScenario @Requirements( RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"), - RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length("1.0") + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length("1.0"), ) def aad_parameter_types_and_length(self): """Check that `encrypt` function accepts `aad` parameter as the fifth argument @@ -256,30 +487,78 @@ def aad_parameter_types_and_length(self): key = "'0123456789123456'" with When("aad is specified using String type"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'aad'", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad="'aad'", + message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526", + ) with When("aad is specified using String with UTF8 characters"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad="'Gãńdåłf_Thê_Gręât'", + message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39", + ) with When("aad is specified using FixedString type"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad="toFixedString('aad', 3)", + message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526", + ) with When("aad is specified using FixedString with UTF8 characters"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", + message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39", + ) with When("aad is 0 bytes"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="''", message="19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad="''", + message="19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9", + ) with When("aad is 1 byte"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'1'", message="19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad="'1'", + message="19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035", + ) with When("aad is 256 bytes"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message="19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + aad=f"'{'1' * 256}'", + message="19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector("1.0") -) +@Requirements(RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector("1.0")) def iv_parameter_types(self): """Check that `encrypt` function accepts `iv` parameter as the fourth argument of either `String` or `FixedString` types. @@ -290,21 +569,44 @@ def iv_parameter_types(self): key = "'0123456789123456'" with When("iv is specified using String type"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + message="F024F9372FA0D8B974894D29FFB8A7F7", + ) with When("iv is specified using String with UTF8 characters"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv="'Gãńdåłf_Thê'", + message="7A4EC0FF3796F46BED281F4778ACE1DC", + ) with When("iv is specified using FixedString type"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=f"toFixedString({iv}, 16)", + message="F024F9372FA0D8B974894D29FFB8A7F7", + ) with When("iv is specified using FixedString with UTF8 characters"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv="toFixedString('Gãńdåłf_Thê', 16)", + message="7A4EC0FF3796F46BED281F4778ACE1DC", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Encrypt_Function_Parameters_Key("1.0") -) +@Requirements(RQ_SRS008_AES_Encrypt_Function_Parameters_Key("1.0")) def key_parameter_types(self): """Check that `encrypt` function accepts `key` parameter as the second argument of either `String` or `FixedString` types. @@ -315,16 +617,37 @@ def key_parameter_types(self): key = "'0123456789123456'" with When("key is specified using String type"): - encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) with When("key is specified using String with UTF8 characters"): - encrypt(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + encrypt( + plaintext=plaintext, + key="'Gãńdåłf_Thê'", + mode=mode, + message="180086AA42AD57B71C706EEC372D0C3D", + ) with When("key is specified using FixedString type"): - encrypt(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + encrypt( + plaintext=plaintext, + key=f"toFixedString({key}, 16)", + mode=mode, + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) with When("key is specified using FixedString with UTF8 characters"): - encrypt(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + encrypt( + plaintext=plaintext, + key="toFixedString('Gãńdåłf_Thê', 16)", + mode=mode, + message="180086AA42AD57B71C706EEC372D0C3D", + ) + @TestScenario @Requirements( @@ -339,17 +662,28 @@ def mode_parameter_types(self): key = "'0123456789123456'" with When("mode is specified using String type"): - encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) with When("mode is specified using FixedString type"): - encrypt(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576") + encrypt( + plaintext=plaintext, + key=key, + mode=f"toFixedString({mode}, 12)", + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) + @TestScenario @Requirements( RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText("2.0"), RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"), RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"), - RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Values("1.0") + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Values("1.0"), ) def encryption(self): """Check that `encrypt` functions accepts `plaintext` as the second parameter @@ -361,37 +695,65 @@ def encryption(self): for mode, key_len, iv_len, aad_len in modes: for datatype, plaintext in plaintexts: - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""" + ) as example: - r = encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, - iv=(None if not iv_len else f"'{iv[:iv_len]}'"), aad=(None if not aad_len else f"'{aad}'")) + r = encrypt( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), + ) with Then("I check output against snapshot"): with values() as that: example_name = basename(example.name) - assert that(snapshot(r.output.strip(), "encrypt", name=f"example_{example_name.replace(' ', '_')}")), error() + assert that( + snapshot( + r.output.strip(), + "encrypt", + name=f"example_{example_name.replace(' ', '_')}", + ) + ), error() + @TestScenario -@Requirements( - RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue("1.0") -) +@Requirements(RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue("1.0")) def return_value(self): - """Check that `encrypt` functions returns String data type. - """ + """Check that `encrypt` functions returns String data type.""" plaintext = "'hello there'" iv = "'0123456789123456'" mode = "'aes-128-cbc'" key = "'0123456789123456'" with When("I get type of the return value"): - sql = "SELECT toTypeName(encrypt(" + mode + "," + plaintext + "," + key + "," + iv + "))" + sql = ( + "SELECT toTypeName(encrypt(" + + mode + + "," + + plaintext + + "," + + key + + "," + + iv + + "))" + ) r = self.context.node.query(sql) with Then("type should be String"): assert r.output.strip() == "String", error() with When("I get return ciphertext as hex"): - encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + encrypt( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + message="F024F9372FA0D8B974894D29FFB8A7F7", + ) + @TestScenario @Requirements( @@ -405,16 +767,16 @@ def syntax(self): ``` """ sql = "SELECT hex(encrypt('aes-128-gcm', 'hello there', '0123456789123456', '012345678912', 'AAD'))" - self.context.node.query(sql, step=When, message="19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614") + self.context.node.query( + sql, step=When, message="19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614" + ) + @TestFeature @Name("encrypt") -@Requirements( - RQ_SRS008_AES_Encrypt_Function("1.0") -) +@Requirements(RQ_SRS008_AES_Encrypt_Function("1.0")) def feature(self, node="clickhouse1"): - """Check the behavior of the `encrypt` function. - """ + """Check the behavior of the `encrypt` function.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/aes_encryption/tests/encrypt_mysql.py b/tests/testflows/aes_encryption/tests/encrypt_mysql.py index b831d6dda85..f43abfd28a2 100644 --- a/tests/testflows/aes_encryption/tests/encrypt_mysql.py +++ b/tests/testflows/aes_encryption/tests/encrypt_mysql.py @@ -5,10 +5,19 @@ from testflows.asserts import values, error, snapshot from aes_encryption.requirements.requirements import * from aes_encryption.tests.common import * + @TestOutline -def aes_encrypt_mysql(self, plaintext=None, key=None, mode=None, iv=None, exitcode=0, message=None, step=When): - """Execute `aes_encrypt_mysql` function with the specified parameters. - """ +def aes_encrypt_mysql( + self, + plaintext=None, + key=None, + mode=None, + iv=None, + exitcode=0, + message=None, + step=When, +): + """Execute `aes_encrypt_mysql` function with the specified parameters.""" params = [] if mode is not None: params.append(mode) @@ -21,178 +30,325 @@ def aes_encrypt_mysql(self, plaintext=None, key=None, mode=None, iv=None, exitco sql = "SELECT hex(aes_encrypt_mysql(" + ", ".join(params) + "))" - return current().context.node.query(sql, step=step, exitcode=exitcode, message=message) + return current().context.node.query( + sql, step=step, exitcode=exitcode, message=message + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_GCM_Error("1.0"), - RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_CTR_Error("1.0") + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values_CTR_Error("1.0"), +) +@Examples( + "mode", + [ + ("'aes-128-gcm'",), + ("'aes-192-gcm'",), + ("'aes-256-gcm'",), + ("'aes-128-ctr'",), + ("'aes-192-ctr'",), + ("'aes-256-ctr'",), + ], ) -@Examples("mode", [ - ("'aes-128-gcm'",), - ("'aes-192-gcm'",), - ("'aes-256-gcm'",), - ("'aes-128-ctr'",), - ("'aes-192-ctr'",), - ("'aes-256-ctr'",), -]) def unsupported_modes(self, mode): - """Check that `aes_encrypt_mysql` function returns an error when unsupported modes are specified. - """ - aes_encrypt_mysql(plaintext="'hello there'", mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode") + """Check that `aes_encrypt_mysql` function returns an error when unsupported modes are specified.""" + aes_encrypt_mysql( + plaintext="'hello there'", + mode=mode, + key=f"'{'1'* 32}'", + exitcode=36, + message="DB::Exception: Unsupported cipher mode", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_Functions_InvalidParameters("1.0") -) +@Requirements(RQ_SRS008_AES_Functions_InvalidParameters("1.0")) def invalid_parameters(self): """Check that `aes_encrypt_mysql` function returns an error when we call it with invalid parameters. """ with Example("no parameters"): - aes_encrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 0, expected 3 to 4") + aes_encrypt_mysql( + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 0, expected 3 to 4", + ) with Example("missing key and mode"): - aes_encrypt_mysql(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 1") + aes_encrypt_mysql( + plaintext="'hello there'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 1", + ) with Example("missing mode"): - aes_encrypt_mysql(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 2") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'123'", + exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 2", + ) with Example("bad key type - UInt8"): - aes_encrypt_mysql(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43, - message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + aes_encrypt_mysql( + plaintext="'hello there'", + key="123", + mode="'aes-128-ecb'", + exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3", + ) with Example("bad mode type - forgot quotes"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, - message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="aes-128-ecb", + exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query", + ) with Example("bad mode type - UInt8"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43, - message="DB::Exception: Illegal type of argument #1 'mode'") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="128", + exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'", + ) with Example("bad iv type - UInt8"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, - message="DB::Exception: Illegal type of argument") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="128", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) - with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, - message="DB::Exception: aes-128-ecb does not support IV") + with Example( + "iv not valid for mode", + requirements=[ + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode( + "1.0" + ) + ], + ): + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="'012345678912'", + exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV", + ) - with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0, - message=None) + with Example( + "iv not valid for mode - size 0", + requirements=[ + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode( + "1.0" + ) + ], + ): + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-ecb'", + iv="''", + exitcode=0, + message=None, + ) - with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with Example( + "invalid mode value", + requirements=[ + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0") + ], + ): with When("typo in the block algorithm"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128-eeb") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128-eeb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb", + ) with When("typo in the key size"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-127-ecb") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-127-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb", + ) with When("typo in the aes prefix"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aee-128-ecb") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aee-128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb", + ) with When("missing last dash"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes-128ecb") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes-128ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb", + ) with When("missing first dash"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, - message="DB::Exception: Invalid mode: aes128-ecb") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'aes128-ecb'", + exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb", + ) with When("all capitals"): - aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, - message="DB::Exception: Invalid mode: AES-128-ECB") + aes_encrypt_mysql( + plaintext="'hello there'", + key="'0123456789123456'", + mode="'AES-128-ECB'", + exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB", + ) + @TestOutline(Scenario) -@Requirements( - RQ_SRS008_AES_Functions_InvalidParameters("1.0") +@Requirements(RQ_SRS008_AES_Functions_InvalidParameters("1.0")) +@Examples( + "data_type, value", + [ + ("UInt8", "toUInt8('1')"), + ("UInt16", "toUInt16('1')"), + ("UInt32", "toUInt32('1')"), + ("UInt64", "toUInt64('1')"), + ("Int8", "toInt8('1')"), + ("Int16", "toInt16('1')"), + ("Int32", "toInt32('1')"), + ("Int64", "toInt64('1')"), + ("Float32", "toFloat32('1.0')"), + ("Float64", "toFloat64('1.0')"), + ("Decimal32", "toDecimal32(2, 4)"), + ("Decimal64", "toDecimal64(2, 4)"), + ("Decimal128", "toDecimal128(2, 4)"), + ("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), + ("Date", "toDate('2020-01-01')"), + ("DateTime", "toDateTime('2020-01-01 20:01:02')"), + ("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"), + ("Array", "[1,2]"), + ("Tuple", "(1,'a')"), + ("IPv4", "toIPv4('171.225.130.45')"), + ("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"), + ("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"), + ("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')"), + ], ) -@Examples("data_type, value", [ - ("UInt8", "toUInt8('1')"), - ("UInt16", "toUInt16('1')"), - ("UInt32", "toUInt32('1')"), - ("UInt64", "toUInt64('1')"), - ("Int8", "toInt8('1')"), - ("Int16", "toInt16('1')"), - ("Int32", "toInt32('1')"), - ("Int64", "toInt64('1')"), - ("Float32", "toFloat32('1.0')"), - ("Float64", "toFloat64('1.0')"), - ("Decimal32", "toDecimal32(2, 4)"), - ("Decimal64", "toDecimal64(2, 4)"), - ("Decimal128", "toDecimal128(2, 4)"), - ("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), - ("Date", "toDate('2020-01-01')"), - ("DateTime", "toDateTime('2020-01-01 20:01:02')"), - ("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"), - ("Array", "[1,2]"), - ("Tuple", "(1,'a')"), - ("IPv4", "toIPv4('171.225.130.45')"), - ("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"), - ("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"), - ("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')") -]) def invalid_plaintext_data_type(self, data_type, value): """Check that aes_encrypt_mysql function returns an error if the plaintext parameter has invalid data type. """ - with When("I try to encrypt plaintext with invalid data type", description=f"{data_type} with value {value}"): - aes_encrypt_mysql(plaintext=value, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", - exitcode=43, message="DB::Exception: Illegal type of argument") + with When( + "I try to encrypt plaintext with invalid data type", + description=f"{data_type} with value {value}", + ): + aes_encrypt_mysql( + plaintext=value, + key="'0123456789123456'", + mode="'aes-128-cbc'", + iv="'0123456789123456'", + exitcode=43, + message="DB::Exception: Illegal type of argument", + ) + @TestOutline(Scenario) @Requirements( RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError("1.0"), RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong("1.0"), - RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError( + "1.0" + ), RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong("1.0"), RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0"), - RQ_SRS008_AES_MySQL_Encrypt_Function_Mode_KeyAndInitializationVector_Length("1.0") + RQ_SRS008_AES_MySQL_Encrypt_Function_Mode_KeyAndInitializationVector_Length("1.0"), +) +@Examples( + "mode key_len iv_len", + [ + # ECB + ("'aes-128-ecb'", 16, None), + ("'aes-192-ecb'", 24, None), + ("'aes-256-ecb'", 32, None), + # CBC + ("'aes-128-cbc'", 16, 16), + ("'aes-192-cbc'", 24, 16), + ("'aes-256-cbc'", 32, 16), + # CFB128 + ("'aes-128-cfb128'", 16, 16), + ("'aes-192-cfb128'", 24, 16), + ("'aes-256-cfb128'", 32, 16), + # OFB + ("'aes-128-ofb'", 16, 16), + ("'aes-192-ofb'", 24, 16), + ("'aes-256-ofb'", 32, 16), + ], + "%-16s %-10s %-10s", ) -@Examples("mode key_len iv_len", [ - # ECB - ("'aes-128-ecb'", 16, None), - ("'aes-192-ecb'", 24, None), - ("'aes-256-ecb'", 32, None), - # CBC - ("'aes-128-cbc'", 16, 16), - ("'aes-192-cbc'", 24, 16), - ("'aes-256-cbc'", 32, 16), - # CFB128 - ("'aes-128-cfb128'", 16, 16), - ("'aes-192-cfb128'", 24, 16), - ("'aes-256-cfb128'", 32, 16), - # OFB - ("'aes-128-ofb'", 16, 16), - ("'aes-192-ofb'", 24, 16), - ("'aes-256-ofb'", 32, 16) -], "%-16s %-10s %-10s") def key_or_iv_length_for_mode(self, mode, key_len, iv_len): - """Check that key or iv length for mode. - """ + """Check that key or iv length for mode.""" plaintext = "'hello there'" key = "0123456789" * 4 iv = "0123456789" * 4 with When("key is too short"): - aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + aes_encrypt_mysql( + plaintext=plaintext, + key=f"'{key[:key_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid key size", + ) with When("key is too long"): aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode) if iv_len is not None: with When("iv is too short"): - aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + aes_encrypt_mysql( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len-1]}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) with When("iv is too long"): - aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode) + aes_encrypt_mysql( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv[:iv_len+1]}'", + mode=mode, + ) else: with When("iv is specified but not needed"): - aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + aes_encrypt_mysql( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + iv=f"'{iv}'", + mode=mode, + exitcode=36, + message="DB::Exception: Invalid IV size", + ) + @TestScenario @Requirements( @@ -208,22 +364,44 @@ def iv_parameter_types(self): key = "'0123456789123456'" with When("iv is specified using String type"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + message="F024F9372FA0D8B974894D29FFB8A7F7", + ) with When("iv is specified using String with UTF8 characters"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + iv="'Gãńdåłf_Thê'", + message="7A4EC0FF3796F46BED281F4778ACE1DC", + ) with When("iv is specified using FixedString type"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + iv=f"toFixedString({iv}, 16)", + message="F024F9372FA0D8B974894D29FFB8A7F7", + ) with When("iv is specified using FixedString with UTF8 characters"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + iv="toFixedString('Gãńdåłf_Thê', 16)", + message="7A4EC0FF3796F46BED281F4778ACE1DC", + ) @TestScenario -@Requirements( - RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key("1.0") -) +@Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key("1.0")) def key_parameter_types(self): """Check that `aes_encrypt_mysql` function accepts `key` parameter as the second argument of either `String` or `FixedString` types. @@ -234,16 +412,36 @@ def key_parameter_types(self): key = "'0123456789123456'" with When("key is specified using String type"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) with When("key is specified using String with UTF8 characters"): - aes_encrypt_mysql(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + aes_encrypt_mysql( + plaintext=plaintext, + key="'Gãńdåłf_Thê'", + mode=mode, + message="180086AA42AD57B71C706EEC372D0C3D", + ) with When("key is specified using FixedString type"): - aes_encrypt_mysql(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + aes_encrypt_mysql( + plaintext=plaintext, + key=f"toFixedString({key}, 16)", + mode=mode, + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) with When("key is specified using FixedString with UTF8 characters"): - aes_encrypt_mysql(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + aes_encrypt_mysql( + plaintext=plaintext, + key="toFixedString('Gãńdåłf_Thê', 16)", + mode=mode, + message="180086AA42AD57B71C706EEC372D0C3D", + ) @TestScenario @@ -259,32 +457,57 @@ def mode_parameter_types(self): key = "'0123456789123456'" with When("mode is specified using String type"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) with When("mode is specified using FixedString type"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=f"toFixedString({mode}, 12)", + message="49C9ADB81BA9B58C485E7ADB90E70576", + ) + @TestScenario -@Requirements( - RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue("1.0") -) +@Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue("1.0")) def return_value(self): - """Check that `aes_encrypt_mysql` functions returns String data type. - """ + """Check that `aes_encrypt_mysql` functions returns String data type.""" plaintext = "'hello there'" iv = "'0123456789123456'" mode = "'aes-128-cbc'" key = "'0123456789123456'" with When("I get type of the return value"): - sql = "SELECT toTypeName(aes_encrypt_mysql("+ mode + "," + plaintext + "," + key + "," + iv + "))" + sql = ( + "SELECT toTypeName(aes_encrypt_mysql(" + + mode + + "," + + plaintext + + "," + + key + + "," + + iv + + "))" + ) r = self.context.node.query(sql) with Then("type should be String"): assert r.output.strip() == "String", error() with When("I get return ciphertext as hex"): - aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + aes_encrypt_mysql( + plaintext=plaintext, + key=key, + mode=mode, + iv=iv, + message="F024F9372FA0D8B974894D29FFB8A7F7", + ) + @TestScenario @Requirements( @@ -300,12 +523,13 @@ def syntax(self): sql = "SELECT hex(aes_encrypt_mysql('aes-128-ofb', 'hello there', '0123456789123456', '0123456789123456'))" self.context.node.query(sql, step=When, message="70FE78410D6EE237C2DE4A") + @TestScenario @Requirements( RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText("2.0"), RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"), RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"), - RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values("1.0") + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Values("1.0"), ) def encryption(self): """Check that `aes_encrypt_mysql` functions accepts `plaintext` as the second parameter @@ -317,24 +541,34 @@ def encryption(self): for mode, key_len, iv_len in mysql_modes: for datatype, plaintext in plaintexts: - with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""") as example: + with Example( + f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""" + ) as example: - r = aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, - iv=(None if not iv_len else f"'{iv[:iv_len]}'")) + r = aes_encrypt_mysql( + plaintext=plaintext, + key=f"'{key[:key_len]}'", + mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + ) with Then("I check output against snapshot"): with values() as that: example_name = basename(example.name) - assert that(snapshot(r.output.strip(), "encrypt_mysql", name=f"example_{example_name.replace(' ', '_')}")), error() + assert that( + snapshot( + r.output.strip(), + "encrypt_mysql", + name=f"example_{example_name.replace(' ', '_')}", + ) + ), error() + @TestFeature @Name("encrypt_mysql") -@Requirements( - RQ_SRS008_AES_MySQL_Encrypt_Function("1.0") -) +@Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function("1.0")) def feature(self, node="clickhouse1"): - """Check the behavior of the `aes_encrypt_mysql` function. - """ + """Check the behavior of the `aes_encrypt_mysql` function.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/datetime64_extended_range/configs/clickhouse/common.xml b/tests/testflows/datetime64_extended_range/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/datetime64_extended_range/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/datetime64_extended_range/configs/clickhouse/config.xml b/tests/testflows/datetime64_extended_range/configs/clickhouse/config.xml deleted file mode 100644 index a9a37875273..00000000000 --- a/tests/testflows/datetime64_extended_range/configs/clickhouse/config.xml +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/datetime64_extended_range/configs/clickhouse/users.xml b/tests/testflows/datetime64_extended_range/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/datetime64_extended_range/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/datetime64_extended_range/regression.py b/tests/testflows/datetime64_extended_range/regression.py index 062c36660ed..69c4021df4c 100755 --- a/tests/testflows/datetime64_extended_range/regression.py +++ b/tests/testflows/datetime64_extended_range/regression.py @@ -28,62 +28,147 @@ from datetime64_extended_range.common import * # Juba and Monrovia timezones are damaged - probably, due to wrong DST shifts lookup tables xfails = { - "type conversion/to int 8 16 32 64 128 256/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/16581#issuecomment-804360350")], - "type conversion/to uint 8 16 32 64 256/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/16581#issuecomment-804360350")], - "non existent time/leap seconds/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/17079#issuecomment-783396589")], - "date time funcs/date diff/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22824")], - "date time funcs/format date time/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22852")], - "date time funcs/time slot/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22854")], - "date time funcs/to monday/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22854")], - "date time funcs/time slots/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/16260")], - "date time funcs/to relative :/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22927#issuecomment-816574952")], - "date time funcs/to start of :/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22928")], - "date time funcs/to unix timestamp/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22929")], - "date time funcs/to week/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22930")], - "date time funcs/to year week/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22948")], - "type conversion/to unix timestamp64 */:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22959")], - "type conversion/from unix timestamp64 */:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22959")], - "type conversion/to int 8 16 32 64 128 256/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/16581#issuecomment-804360350")], + "type conversion/to int 8 16 32 64 128 256/:": [ + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/issues/16581#issuecomment-804360350", + ) + ], + "type conversion/to uint 8 16 32 64 256/:": [ + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/issues/16581#issuecomment-804360350", + ) + ], + "non existent time/leap seconds/:": [ + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/issues/17079#issuecomment-783396589", + ) + ], + "date time funcs/date diff/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22824") + ], + "date time funcs/format date time/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22852") + ], + "date time funcs/time slot/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22854") + ], + "date time funcs/to monday/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22854") + ], + "date time funcs/time slots/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/16260") + ], + "date time funcs/to relative :/:": [ + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/issues/22927#issuecomment-816574952", + ) + ], + "date time funcs/to start of :/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22928") + ], + "date time funcs/to unix timestamp/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22929") + ], + "date time funcs/to week/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22930") + ], + "date time funcs/to year week/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22948") + ], + "type conversion/to unix timestamp64 */:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22959") + ], + "type conversion/from unix timestamp64 */:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22959") + ], + "type conversion/to int 8 16 32 64 128 256/:": [ + ( + Fail, + "https://github.com/ClickHouse/ClickHouse/issues/16581#issuecomment-804360350", + ) + ], "reference times/:": [(Fail, "check procedure unclear")], # need to investigate "type conversion/to datetime/cast=True": [(Fail, "need to investigate")], - "date time funcs/today": [(Fail, "need to investigate")] + "date time funcs/today": [(Fail, "need to investigate")], } @TestModule @Name("datetime64 extended range") @ArgumentParser(argparser) -@Specifications( - SRS_010_ClickHouse_DateTime64_Extended_Range -) +@Specifications(SRS_010_ClickHouse_DateTime64_Extended_Range) @Requirements( RQ_SRS_010_DateTime64_ExtendedRange("1.0"), ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path, stress=False): - """ClickHouse DateTime64 Extended Range regression module. - """ +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=False +): + """ClickHouse DateTime64 Extended Range regression module.""" nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), } if stress is not None: self.context.stress = stress + self.context.clickhouse_version = clickhouse_version - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "datetime64_extended_range_env")) as cluster: + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join( + current_dir(), "datetime64_extended_range_env" + ), + ) as cluster: self.context.cluster = cluster with Pool(2) as pool: try: - Scenario(run=load("datetime64_extended_range.tests.generic", "generic"), parallel=True, executor=pool) - Scenario(run=load("datetime64_extended_range.tests.non_existent_time", "feature"), parallel=True, executor=pool) - Scenario(run=load("datetime64_extended_range.tests.reference_times", "reference_times"), parallel=True, executor=pool) - Scenario(run=load("datetime64_extended_range.tests.date_time_functions", "date_time_funcs"), parallel=True, executor=pool) - Scenario(run=load("datetime64_extended_range.tests.type_conversion", "type_conversion"), parallel=True, executor=pool) + Scenario( + run=load("datetime64_extended_range.tests.generic", "generic"), + parallel=True, + executor=pool, + ) + Scenario( + run=load( + "datetime64_extended_range.tests.non_existent_time", "feature" + ), + parallel=True, + executor=pool, + ) + Scenario( + run=load( + "datetime64_extended_range.tests.reference_times", + "reference_times", + ), + parallel=True, + executor=pool, + ) + Scenario( + run=load( + "datetime64_extended_range.tests.date_time_functions", + "date_time_funcs", + ), + parallel=True, + executor=pool, + ) + Scenario( + run=load( + "datetime64_extended_range.tests.type_conversion", + "type_conversion", + ), + parallel=True, + executor=pool, + ) finally: join() + if main(): regression() diff --git a/tests/testflows/datetime64_extended_range/requirements/requirements.py b/tests/testflows/datetime64_extended_range/requirements/requirements.py index a9ba2c235f2..1bbaf3547d9 100644 --- a/tests/testflows/datetime64_extended_range/requirements/requirements.py +++ b/tests/testflows/datetime64_extended_range/requirements/requirements.py @@ -9,1631 +9,1730 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_010_DateTime64_ExtendedRange = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support extended range for the [DateTime64] data type that includes dates from the year **1925** to **2238**.\n' - '\n' - ), + "[ClickHouse] SHALL support extended range for the [DateTime64] data type that includes dates from the year **1925** to **2238**.\n" + "\n" + ), link=None, level=4, - num='4.1.0.1') + num="4.1.0.1", +) RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_Start = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper time handling around the normal date range that starts at `1970-01-01 00:00:00.000`\n' - 'expressed using the [ISO 8601 format].\n' - '\n' - ), + "[ClickHouse] SHALL support proper time handling around the normal date range that starts at `1970-01-01 00:00:00.000`\n" + "expressed using the [ISO 8601 format].\n" + "\n" + ), link=None, level=4, - num='4.1.0.2') + num="4.1.0.2", +) RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_Start_BeforeEpochForTimeZone = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start.BeforeEpochForTimeZone', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start.BeforeEpochForTimeZone", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper time handling around the start of the [normal date range]\n' - 'when this time for the time zone is before the start of the [normal date range].\n' - '\n' - ), + "[ClickHouse] SHALL support proper time handling around the start of the [normal date range]\n" + "when this time for the time zone is before the start of the [normal date range].\n" + "\n" + ), link=None, level=4, - num='4.1.0.3') + num="4.1.0.3", +) RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_End = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper time handling around the normal date range that ends at `2105-12-31T23:59:59.99999`\n' - 'expressed using the [ISO 8601 format].\n' - '\n' - ), + "[ClickHouse] SHALL support proper time handling around the normal date range that ends at `2105-12-31T23:59:59.99999`\n" + "expressed using the [ISO 8601 format].\n" + "\n" + ), link=None, level=4, - num='4.1.0.4') + num="4.1.0.4", +) RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_End_AfterEpochForTimeZone = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End.AfterEpochForTimeZone', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End.AfterEpochForTimeZone", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper time handling around the end of the [normal date range]\n' - 'when this time for the time zone is after the end of the [normal date range].\n' - '\n' - ), + "[ClickHouse] SHALL support proper time handling around the end of the [normal date range]\n" + "when this time for the time zone is after the end of the [normal date range].\n" + "\n" + ), link=None, level=4, - num='4.1.0.5') + num="4.1.0.5", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper conversion to and from [DateTime64] data type from other data types.\n' - '\n' - ), + "[ClickHouse] SHALL support proper conversion to and from [DateTime64] data type from other data types.\n" + "\n" + ), link=None, level=4, - num='4.1.0.6') + num="4.1.0.6", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [Dates and Times Functions] with the [DateTime64] data type\n' - 'when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [Dates and Times Functions] with the [DateTime64] data type\n" + "when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=4, - num='4.1.0.7') + num="4.1.0.7", +) RQ_SRS_010_DateTime64_ExtendedRange_TimeZones = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TimeZones', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TimeZones", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation with the [DateTime64] extended range data type\n' - 'when combined with a supported time zone.\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation with the [DateTime64] extended range data type\n" + "when combined with a supported time zone.\n" + "\n" + ), link=None, level=4, - num='4.1.0.8') + num="4.1.0.8", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of non-existent times when using [DateTime64] extended range data type.\n' - '\n' - ), + "[ClickHouse] SHALL support proper handling of non-existent times when using [DateTime64] extended range data type.\n" + "\n" + ), link=None, level=4, - num='4.1.0.9') + num="4.1.0.9", +) RQ_SRS_010_DateTime64_ExtendedRange_Comparison = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.Comparison', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.Comparison", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of time comparison when using [DateTime64] extended range data type.\n' + "[ClickHouse] SHALL support proper handling of time comparison when using [DateTime64] extended range data type.\n" "For example, `SELECT toDateTime64('2019-05-05 20:20:12.050', 3) < now()`.\n" - '\n' - ), + "\n" + ), link=None, level=4, - num='4.1.0.10') + num="4.1.0.10", +) RQ_SRS_010_DateTime64_ExtendedRange_SpecificTimestamps = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.SpecificTimestamps', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.SpecificTimestamps", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL properly work with the following timestamps in all supported timezones:\n' - '```\n' - '[9961200,73476000,325666800,354675600,370400400,386125200,388566010,401850000,417574811,496803600,528253200,624423614,636516015,671011200,717555600,752047218,859683600,922582800,1018173600,1035705600,1143334800,1162105223,1174784400,1194156000,1206838823,1224982823,1236495624,1319936400,1319936424,1425798025,1459040400,1509872400,2090451627,2140668000]\n' - '```\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL properly work with the following timestamps in all supported timezones:\n" + "```\n" + "[9961200,73476000,325666800,354675600,370400400,386125200,388566010,401850000,417574811,496803600,528253200,624423614,636516015,671011200,717555600,752047218,859683600,922582800,1018173600,1035705600,1143334800,1162105223,1174784400,1194156000,1206838823,1224982823,1236495624,1319936400,1319936424,1425798025,1459040400,1509872400,2090451627,2140668000]\n" + "```\n" + "\n" + "\n" + ), link=None, level=4, - num='4.1.0.11') + num="4.1.0.11", +) RQ_SRS_010_DateTime64_ExtendedRange_Start = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.Start', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.Start", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support extended range for the [DateTime64] data type that starts at `1925-01-01T00:00:00.000000`\n' - 'expressed using the [ISO 8601 format].\n' - '\n' - ), + "[ClickHouse] SHALL support extended range for the [DateTime64] data type that starts at `1925-01-01T00:00:00.000000`\n" + "expressed using the [ISO 8601 format].\n" + "\n" + ), link=None, level=4, - num='4.2.0.1') + num="4.2.0.1", +) RQ_SRS_010_DateTime64_ExtendedRange_End = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.End', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.End", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support extended range for the [DateTime64] data type that ends at `2238-12-31T23:59:59.999999`\n' - 'expressed using the [ISO 8601 format].\n' - '\n' - ), + "[ClickHouse] SHALL support extended range for the [DateTime64] data type that ends at `2238-12-31T23:59:59.999999`\n" + "expressed using the [ISO 8601 format].\n" + "\n" + ), link=None, level=4, - num='4.2.0.2') + num="4.2.0.2", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_InvalidDate = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidDate', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidDate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of invalid dates when using [DateTime64] extended range data type,\n' - 'such as:\n' - '\n' - '* `YYYY-04-31, YYYY-06-31, YYYY-09-31, YYYY-11-31`\n' - '* `1990-02-30 00:00:02`\n' - '\n' - ), + "[ClickHouse] SHALL support proper handling of invalid dates when using [DateTime64] extended range data type,\n" + "such as:\n" + "\n" + "* `YYYY-04-31, YYYY-06-31, YYYY-09-31, YYYY-11-31`\n" + "* `1990-02-30 00:00:02`\n" + "\n" + ), link=None, level=5, - num='4.2.0.3.1') + num="4.2.0.3.1", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_InvalidTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of invalid time for a timezone\n' - 'when using [DateTime64] extended range data type, for example,\n' - '\n' - '* `2002-04-07 02:30:00` never happened at all in the US/Eastern timezone ([Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime))\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL support proper handling of invalid time for a timezone\n" + "when using [DateTime64] extended range data type, for example,\n" + "\n" + "* `2002-04-07 02:30:00` never happened at all in the US/Eastern timezone ([Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime))\n" + "\n" + "\n" + ), link=None, level=5, - num='4.2.0.3.2') + num="4.2.0.3.2", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_TimeZoneSwitch = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.TimeZoneSwitch', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.TimeZoneSwitch", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of invalid time when using [DateTime64] extended range data type\n' - 'when the invalid time is caused when *countries switch timezone definitions with no\n' - 'daylight savings time switch* [Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime).\n' - '\n' - '>\n' - '> For example, in 1915 Warsaw switched from Warsaw time to Central European time with\n' - '> no daylight savings transition. So at the stroke of midnight on August 5th 1915 the clocks\n' - '> were wound back 24 minutes creating an ambiguous time period that cannot be specified without\n' - '> referring to the timezone abbreviation or the actual UTC offset. In this case midnight happened twice,\n' - '> neither time during a daylight saving time period. pytz handles this transition by treating the ambiguous\n' - '> period before the switch as daylight savings time, and the ambiguous period after as standard time.\n' - '>\n' - '> [Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime)\n' - '\n' - ), + "[ClickHouse] SHALL support proper handling of invalid time when using [DateTime64] extended range data type\n" + "when the invalid time is caused when *countries switch timezone definitions with no\n" + "daylight savings time switch* [Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime).\n" + "\n" + ">\n" + "> For example, in 1915 Warsaw switched from Warsaw time to Central European time with\n" + "> no daylight savings transition. So at the stroke of midnight on August 5th 1915 the clocks\n" + "> were wound back 24 minutes creating an ambiguous time period that cannot be specified without\n" + "> referring to the timezone abbreviation or the actual UTC offset. In this case midnight happened twice,\n" + "> neither time during a daylight saving time period. pytz handles this transition by treating the ambiguous\n" + "> period before the switch as daylight savings time, and the ambiguous period after as standard time.\n" + ">\n" + "> [Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime)\n" + "\n" + ), link=None, level=5, - num='4.2.0.3.3') + num="4.2.0.3.3", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_DaylightSavingTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of invalid time when using [DateTime64] extended range data type\n' - 'when for a given timezone time switches from standard to daylight saving.\n' - '\n' - '> For example, in the US/Eastern timezone on the last Sunday morning in October, the following sequence happens:\n' - '>\n' - '> 01:00 EDT occurs\n' - '> 1 hour later, instead of 2:00am the clock is turned back 1 hour and 01:00 happens again (this time 01:00 EST)\n' - '> In fact, every instant between 01:00 and 02:00 occurs twice.\n' - '> [Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime)\n' - '\n' - ), + "[ClickHouse] SHALL support proper handling of invalid time when using [DateTime64] extended range data type\n" + "when for a given timezone time switches from standard to daylight saving.\n" + "\n" + "> For example, in the US/Eastern timezone on the last Sunday morning in October, the following sequence happens:\n" + ">\n" + "> 01:00 EDT occurs\n" + "> 1 hour later, instead of 2:00am the clock is turned back 1 hour and 01:00 happens again (this time 01:00 EST)\n" + "> In fact, every instant between 01:00 and 02:00 occurs twice.\n" + "> [Stuart Bishop: pytz library](http://pytz.sourceforge.net/#problems-with-localtime)\n" + "\n" + ), link=None, level=5, - num='4.2.0.3.4') + num="4.2.0.3.4", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_DaylightSavingTime_Disappeared = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime.Disappeared', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime.Disappeared", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of invalid time when using [DateTime64] extended range data type\n' - 'for a given timezone when transition from the standard to daylight saving time causes an hour to disappear.\n' - '\n' + "[ClickHouse] SHALL support proper handling of invalid time when using [DateTime64] extended range data type\n" + "for a given timezone when transition from the standard to daylight saving time causes an hour to disappear.\n" + "\n" "Expected behavior: if DateTime64 initialized by a skipped time value, it is being treated as DST and resulting value will be an hour earlier, e.g. `SELECT toDateTime64('2020-03-08 02:34:00', 0, 'America/Denver')` returns `2020-03-08 01:34:00`.\n" - '\n' - ), + "\n" + ), link=None, level=5, - num='4.2.0.3.5') + num="4.2.0.3.5", +) RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_LeapSeconds = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.LeapSeconds', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.LeapSeconds", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support proper handling of leap seconds adjustments when using [DateTime64] extended range data type.\n' - '\n' - ), + "[ClickHouse] SHALL support proper handling of leap seconds adjustments when using [DateTime64] extended range data type.\n" + "\n" + ), link=None, level=5, - num='4.2.0.3.6') + num="4.2.0.3.6", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toTimeZone = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTimeZone', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTimeZone", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toTimeZone](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totimezone)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toTimeZone](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totimezone)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.1') + num="4.2.0.4.1", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toYear = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYear', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYear", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyear)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyear)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.2') + num="4.2.0.4.2", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toQuarter = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toQuarter', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toQuarter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toQuarter](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toquarter)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toQuarter](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toquarter)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.3') + num="4.2.0.4.3", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toMonth = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonth', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonth", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toMonth](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tomonth)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toMonth](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tomonth)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.4') + num="4.2.0.4.4", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toDayOfYear = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfYear', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfYear", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toDayOfYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#todayofyear)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toDayOfYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#todayofyear)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.5') + num="4.2.0.4.5", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toDayOfMonth = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfMonth', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfMonth", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toDayOfMonth](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#todayofmonth)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toDayOfMonth](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#todayofmonth)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.6') + num="4.2.0.4.6", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toDayOfWeek = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfWeek', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfWeek", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toDayOfWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#todayofweek)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toDayOfWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#todayofweek)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.7') + num="4.2.0.4.7", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toHour = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toHour', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toHour", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toHour](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tohour)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toHour](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tohour)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.8') + num="4.2.0.4.8", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toMinute = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMinute', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMinute", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toMinute](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tominute)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toMinute](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tominute)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.9') + num="4.2.0.4.9", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toSecond = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toSecond', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toSecond", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toSecond](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tosecond)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toSecond](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tosecond)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.10') + num="4.2.0.4.10", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toUnixTimestamp = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toUnixTimestamp', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toUnixTimestamp", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toUnitTimestamp](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#to-unix-timestamp)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - 'Timestamp value expected to be negative when DateTime64 value is prior to `1970-01-01` and positine otherwise.\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toUnitTimestamp](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#to-unix-timestamp)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "Timestamp value expected to be negative when DateTime64 value is prior to `1970-01-01` and positine otherwise.\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.11') + num="4.2.0.4.11", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfYear = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfYear', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfYear", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofyear)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofyear)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.12') + num="4.2.0.4.12", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfISOYear = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfISOYear', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfISOYear", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfISOYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofisoyear)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfISOYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofisoyear)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.13') + num="4.2.0.4.13", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfQuarter = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfQuarter', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfQuarter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfQuarter](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofquarter)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfQuarter](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofquarter)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.14') + num="4.2.0.4.14", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfMonth = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMonth', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMonth", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfMonth](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofmonth)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfMonth](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofmonth)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.15') + num="4.2.0.4.15", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toMonday = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonday', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonday", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toMonday](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tomonday)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toMonday](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tomonday)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.16') + num="4.2.0.4.16", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfWeek = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfWeek', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfWeek", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofweektmode)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofweektmode)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.17') + num="4.2.0.4.17", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfDay = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfDay', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfDay", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfDay](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofday)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfDay](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofday)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.18') + num="4.2.0.4.18", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfHour = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfHour', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfHour", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfHour](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofhour)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfHour](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofhour)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.19') + num="4.2.0.4.19", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfMinute = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMinute', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMinute", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfMinute](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofminute)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfMinute](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofminute)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.20') + num="4.2.0.4.20", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfSecond = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfSecond', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfSecond", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfSecond](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofsecond)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfSecond](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofsecond)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.21') + num="4.2.0.4.21", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfFiveMinute = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFiveMinute', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFiveMinute", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfFiveMinute](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartoffiveminute)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfFiveMinute](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartoffiveminute)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.22') + num="4.2.0.4.22", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfTenMinutes = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfTenMinutes', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfTenMinutes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfTenMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartoftenminutes)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfTenMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartoftenminutes)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.23') + num="4.2.0.4.23", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfFifteenMinutes = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFifteenMinutes', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFifteenMinutes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfFifteenMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartoffifteenminutes)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfFifteenMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartoffifteenminutes)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.24') + num="4.2.0.4.24", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfInterval = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfInterval', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfInterval", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toStartOfInterval](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofintervaltime-or-data-interval-x-unit-time-zone)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - 'More detailed description can be found [here](https://github.com/ClickHouse/ClickHouse/issues/1201).\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toStartOfInterval](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#tostartofintervaltime-or-data-interval-x-unit-time-zone)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "More detailed description can be found [here](https://github.com/ClickHouse/ClickHouse/issues/1201).\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.25') + num="4.2.0.4.25", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toTime](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totime)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toTime](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totime)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.26') + num="4.2.0.4.26", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeYearNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeYearNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeYearNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeYearNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativeyearnum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeYearNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativeyearnum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.27') + num="4.2.0.4.27", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeQuarterNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeQuarterNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeQuarterNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeQuarterNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativequarternum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeQuarterNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativequarternum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.28') + num="4.2.0.4.28", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeMonthNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMonthNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMonthNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeMonthNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativemonthnum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeMonthNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativemonthnum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.29') + num="4.2.0.4.29", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeWeekNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeWeekNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeWeekNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeWeekNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativeweeknum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeWeekNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativeweeknum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.30') + num="4.2.0.4.30", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeDayNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeDayNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeDayNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeDayNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativedaynum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeDayNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativedaynum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.31') + num="4.2.0.4.31", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeHourNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeHourNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeHourNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeHourNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativehournum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeHourNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativehournum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.32') + num="4.2.0.4.32", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeMinuteNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMinuteNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMinuteNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeMinuteNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativeminutenum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeMinuteNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativeminutenum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.33') + num="4.2.0.4.33", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeSecondNum = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeSecondNum', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeSecondNum", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toRelativeSecondNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativesecondnum)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toRelativeSecondNum](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#torelativesecondnum)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.34') + num="4.2.0.4.34", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toISOYear = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOYear', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOYear", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toISOYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toisoyear)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toISOYear](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toisoyear)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.35') + num="4.2.0.4.35", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toISOWeek = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOWeek', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOWeek", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toISOWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toisoweek)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toISOWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toisoweek)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.36') + num="4.2.0.4.36", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toWeek = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toWeek', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toWeek", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toweekdatemode)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toweekdatemode)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.37') + num="4.2.0.4.37", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toYearWeek = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYearWeek', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYearWeek", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toYearWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyearweekdatemode)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toYearWeek](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyearweekdatemode)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.38') + num="4.2.0.4.38", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_now = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.now', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.now", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support conversion of output from the [now](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#now)\n' - 'function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support conversion of output from the [now](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#now)\n" + "function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.39') + num="4.2.0.4.39", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_today = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.today', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.today", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support conversion of output from the [today](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#today)\n' - 'function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support conversion of output from the [today](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#today)\n" + "function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.40') + num="4.2.0.4.40", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_yesterday = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.yesterday', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.yesterday", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support conversion of output from the [yesterday](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#yesterday)\n' - 'function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support conversion of output from the [yesterday](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#yesterday)\n" + "function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.41') + num="4.2.0.4.41", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_timeSlot = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlot', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlot", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support conversion of output from the [timeSlot](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#timeslot)\n' - 'function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support conversion of output from the [timeSlot](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#timeslot)\n" + "function to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.42') + num="4.2.0.4.42", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toYYYYMM = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMM', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMM", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toYYYYMM](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyyyymm)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toYYYYMM](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyyyymm)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.43') + num="4.2.0.4.43", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toYYYYMMDD = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDD', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDD", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toYYYYMMDD](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyyyymmdd)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toYYYYMMDD](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyyyymmdd)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.44') + num="4.2.0.4.44", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toYYYYMMDDhhmmss = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDDhhmmss', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDDhhmmss", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [toYYYYMMDDhhmmss](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyyyymmddhhmmss)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [toYYYYMMDDhhmmss](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#toyyyymmddhhmmss)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.45') + num="4.2.0.4.45", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addYears = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addYears', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addYears", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addYears](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addYears](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.46') + num="4.2.0.4.46", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addMonths = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMonths', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMonths", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addMonths](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addMonths](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.47') + num="4.2.0.4.47", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addWeeks = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addWeeks', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addWeeks", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addWeeks](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addWeeks](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.48') + num="4.2.0.4.48", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addDays = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addDays', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addDays", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addDays](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addDays](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.49') + num="4.2.0.4.49", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addHours = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addHours', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addHours", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addHours](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addHours](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.50') + num="4.2.0.4.50", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addMinutes = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMinutes', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMinutes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.51') + num="4.2.0.4.51", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addSeconds = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addSeconds', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addSeconds", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addSeconds](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addSeconds](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.52') + num="4.2.0.4.52", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addQuarters = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addQuarters', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addQuarters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [addQuarters](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [addQuarters](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#addyears-addmonths-addweeks-adddays-addhours-addminutes-addseconds-addquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.53') + num="4.2.0.4.53", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractYears = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractYears', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractYears", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractYears](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractYears](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.54') + num="4.2.0.4.54", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractMonths = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMonths', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMonths", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractMonths](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractMonths](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.55') + num="4.2.0.4.55", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractWeeks = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractWeeks', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractWeeks", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractWeeks](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractWeeks](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.56') + num="4.2.0.4.56", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractDays = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractDays', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractDays", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractDays](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractDays](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.57') + num="4.2.0.4.57", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractHours = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractHours', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractHours", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractHours](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractHours](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.58') + num="4.2.0.4.58", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractMinutes = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMinutes', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMinutes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractMinutes](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.59') + num="4.2.0.4.59", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractSeconds = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractSeconds', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractSeconds", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractSeconds](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractSeconds](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.60') + num="4.2.0.4.60", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_subtractQuarters = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractQuarters', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractQuarters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [subtractQuarters](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [subtractQuarters](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#subtractyears-subtractmonths-subtractweeks-subtractdays-subtracthours-subtractminutes-subtractseconds-subtractquarters)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.61') + num="4.2.0.4.61", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_dateDiff = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.dateDiff', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.dateDiff", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [dateDiff](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#datediff)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [dateDiff](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#datediff)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.62') + num="4.2.0.4.62", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_timeSlots = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlots', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlots", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [timeSlots](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#timeslotsstarttime-duration-size)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [timeSlots](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#timeslotsstarttime-duration-size)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.63') + num="4.2.0.4.63", +) RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_formatDateTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.formatDateTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.formatDateTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of the [formatDateTime](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#formatdatetime)\n' - 'function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of the [formatDateTime](https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#formatdatetime)\n" + "function used with the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + "\n" + ), link=None, level=5, - num='4.2.0.4.64') + num="4.2.0.4.64", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toInt_8_16_32_64_128_256_ = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toInt(8|16|32|64|128|256)', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toInt(8|16|32|64|128|256)", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to integer types using [toInt(8|16|32|64|128|256)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#toint8163264128256) functions.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to integer types using [toInt(8|16|32|64|128|256)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#toint8163264128256) functions.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.1') + num="4.2.1.4.1", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUInt_8_16_32_64_256_ = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUInt(8|16|32|64|256)', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUInt(8|16|32|64|256)", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to unsigned integer types using [toUInt(8|16|32|64|256)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#touint8163264256) functions.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to unsigned integer types using [toUInt(8|16|32|64|256)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#touint8163264256) functions.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.2') + num="4.2.1.4.2", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toFloat_32_64_ = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toFloat(32|64)', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toFloat(32|64)", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to float types using [toFloat(32|64)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tofloat3264) functions.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to float types using [toFloat(32|64)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tofloat3264) functions.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.3') + num="4.2.1.4.3", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDate = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDate', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range]\n' - 'to the [Date](https://clickhouse.com/docs/en/sql-reference/data-types/date/) type using the [toDate](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#todate) function.\n' - 'This function is ONLY supposed to work in NORMAL RANGE.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range]\n" + "to the [Date](https://clickhouse.com/docs/en/sql-reference/data-types/date/) type using the [toDate](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#todate) function.\n" + "This function is ONLY supposed to work in NORMAL RANGE.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.4') + num="4.2.1.4.4", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to the [DateTime](https://clickhouse.com/docs/en/sql-reference/data-types/datetime/) type using the [toDateTime](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#todatetime) function.\n' - 'This function is ONLY supposed to work in NORMAL RANGE.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to the [DateTime](https://clickhouse.com/docs/en/sql-reference/data-types/datetime/) type using the [toDateTime](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#todatetime) function.\n" + "This function is ONLY supposed to work in NORMAL RANGE.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.5') + num="4.2.1.4.5", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime64 = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion from the data types supported by the [toDateTime64](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64/) function\n' - 'to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion from the data types supported by the [toDateTime64](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64/) function\n" + "to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range].\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.6') + num="4.2.1.4.6", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime64_FromString_MissingTime = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64.FromString.MissingTime', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64.FromString.MissingTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion from the [String](https://clickhouse.com/docs/en/sql-reference/data-types/string/)\n' - 'data type to the [DateTime64](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64/) data type\n' - 'when value of the string is missing the `hh:mm-ss.sss` part.\n' + "[ClickHouse] SHALL support correct conversion from the [String](https://clickhouse.com/docs/en/sql-reference/data-types/string/)\n" + "data type to the [DateTime64](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64/) data type\n" + "when value of the string is missing the `hh:mm-ss.sss` part.\n" "For example, `toDateTime64('2020-01-01', 3)`.\n" - '\n' - ), + "\n" + ), link=None, level=5, - num='4.2.1.4.7') + num="4.2.1.4.7", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDecimal_32_64_128_256_ = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDecimal(32|64|128|256)', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDecimal(32|64|128|256)", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to [Decimal](https://clickhouse.com/docs/en/sql-reference/data-types/decimal/) types using [toDecimal(32|64|128|256)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#todecimal3264128256) functions.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to [Decimal](https://clickhouse.com/docs/en/sql-reference/data-types/decimal/) types using [toDecimal(32|64|128|256)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#todecimal3264128256) functions.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.8') + num="4.2.1.4.8", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toString = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toString', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toString", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to the [String](https://clickhouse.com/docs/en/sql-reference/data-types/string/) type using the [toString](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tostring) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to the [String](https://clickhouse.com/docs/en/sql-reference/data-types/string/) type using the [toString](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tostring) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.9') + num="4.2.1.4.9", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_ = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.CAST(x,T)', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.CAST(x,T)", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to one of the supported data type using the [CAST(x,T)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to one of the supported data type using the [CAST(x,T)](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.10') + num="4.2.1.4.10", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Milli = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Milli', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Milli", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type using the [toUnixTimestamp64Milli](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tounixtimestamp64milli) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type using the [toUnixTimestamp64Milli](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tounixtimestamp64milli) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.11') + num="4.2.1.4.11", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Micro = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Micro', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Micro", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type using the [toUnixTimestamp64Micro](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tounixtimestamp64micro) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type using the [toUnixTimestamp64Micro](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tounixtimestamp64micro) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.12') + num="4.2.1.4.12", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Nano = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Nano', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Nano", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'to the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type using the [toUnixTimestamp64Nano](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tounixtimestamp64nano) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion of the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "to the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type using the [toUnixTimestamp64Nano](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#tounixtimestamp64nano) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.13') + num="4.2.1.4.13", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Milli = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Milli', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Milli", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion from the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type\n' - 'to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'using the [fromUnixTimestamp64Milli](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#fromunixtimestamp64milli) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion from the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type\n" + "to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "using the [fromUnixTimestamp64Milli](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#fromunixtimestamp64milli) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.14') + num="4.2.1.4.14", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Micro = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Micro', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Micro", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion from the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type\n' - 'to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'using the [fromUnixTimestamp64Micro](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#fromunixtimestamp64micro) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion from the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type\n" + "to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "using the [fromUnixTimestamp64Micro](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#fromunixtimestamp64micro) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.15') + num="4.2.1.4.15", +) RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Nano = Requirement( - name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Nano', - version='1.0', + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Nano", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct conversion from the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type\n' - 'to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n' - 'using the [fromUnixTimestamp64Nano](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#fromunixtimestamp64nano) function.\n' - '\n' - ), + "[ClickHouse] SHALL support correct conversion from the [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/) type\n" + "to the [DateTime64] data type when it stores dates within the [normal date range] and the [extended date range]\n" + "using the [fromUnixTimestamp64Nano](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#fromunixtimestamp64nano) function.\n" + "\n" + ), link=None, level=5, - num='4.2.1.4.16') + num="4.2.1.4.16", +) SRS_010_ClickHouse_DateTime64_Extended_Range = Specification( - name='SRS-010 ClickHouse DateTime64 Extended Range', + name="SRS-010 ClickHouse DateTime64 Extended Range", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -1645,119 +1744,503 @@ SRS_010_ClickHouse_DateTime64_Extended_Range = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='SRS', level=2, num='3.1'), - Heading(name='Normal Date Range', level=2, num='3.2'), - Heading(name='Extended Date Range', level=2, num='3.3'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='Generic', level=2, num='4.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange', level=4, num='4.1.0.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start', level=4, num='4.1.0.2'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start.BeforeEpochForTimeZone', level=4, num='4.1.0.3'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End', level=4, num='4.1.0.4'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End.AfterEpochForTimeZone', level=4, num='4.1.0.5'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions', level=4, num='4.1.0.6'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions', level=4, num='4.1.0.7'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TimeZones', level=4, num='4.1.0.8'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime', level=4, num='4.1.0.9'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.Comparison', level=4, num='4.1.0.10'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.SpecificTimestamps', level=4, num='4.1.0.11'), - Heading(name='Specific', level=2, num='4.2'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.Start', level=4, num='4.2.0.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.End', level=4, num='4.2.0.2'), - Heading(name='Non-Existent Time', level=4, num='4.2.0.3'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidDate', level=5, num='4.2.0.3.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidTime', level=5, num='4.2.0.3.2'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.TimeZoneSwitch', level=5, num='4.2.0.3.3'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime', level=5, num='4.2.0.3.4'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime.Disappeared', level=5, num='4.2.0.3.5'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.LeapSeconds', level=5, num='4.2.0.3.6'), - Heading(name='Dates And Times Functions', level=4, num='4.2.0.4'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTimeZone', level=5, num='4.2.0.4.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYear', level=5, num='4.2.0.4.2'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toQuarter', level=5, num='4.2.0.4.3'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonth', level=5, num='4.2.0.4.4'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfYear', level=5, num='4.2.0.4.5'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfMonth', level=5, num='4.2.0.4.6'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfWeek', level=5, num='4.2.0.4.7'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toHour', level=5, num='4.2.0.4.8'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMinute', level=5, num='4.2.0.4.9'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toSecond', level=5, num='4.2.0.4.10'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toUnixTimestamp', level=5, num='4.2.0.4.11'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfYear', level=5, num='4.2.0.4.12'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfISOYear', level=5, num='4.2.0.4.13'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfQuarter', level=5, num='4.2.0.4.14'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMonth', level=5, num='4.2.0.4.15'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonday', level=5, num='4.2.0.4.16'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfWeek', level=5, num='4.2.0.4.17'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfDay', level=5, num='4.2.0.4.18'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfHour', level=5, num='4.2.0.4.19'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMinute', level=5, num='4.2.0.4.20'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfSecond', level=5, num='4.2.0.4.21'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFiveMinute', level=5, num='4.2.0.4.22'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfTenMinutes', level=5, num='4.2.0.4.23'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFifteenMinutes', level=5, num='4.2.0.4.24'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfInterval', level=5, num='4.2.0.4.25'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTime', level=5, num='4.2.0.4.26'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeYearNum', level=5, num='4.2.0.4.27'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeQuarterNum', level=5, num='4.2.0.4.28'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMonthNum', level=5, num='4.2.0.4.29'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeWeekNum', level=5, num='4.2.0.4.30'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeDayNum', level=5, num='4.2.0.4.31'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeHourNum', level=5, num='4.2.0.4.32'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMinuteNum', level=5, num='4.2.0.4.33'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeSecondNum', level=5, num='4.2.0.4.34'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOYear', level=5, num='4.2.0.4.35'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOWeek', level=5, num='4.2.0.4.36'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toWeek', level=5, num='4.2.0.4.37'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYearWeek', level=5, num='4.2.0.4.38'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.now', level=5, num='4.2.0.4.39'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.today', level=5, num='4.2.0.4.40'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.yesterday', level=5, num='4.2.0.4.41'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlot', level=5, num='4.2.0.4.42'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMM', level=5, num='4.2.0.4.43'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDD', level=5, num='4.2.0.4.44'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDDhhmmss', level=5, num='4.2.0.4.45'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addYears', level=5, num='4.2.0.4.46'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMonths', level=5, num='4.2.0.4.47'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addWeeks', level=5, num='4.2.0.4.48'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addDays', level=5, num='4.2.0.4.49'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addHours', level=5, num='4.2.0.4.50'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMinutes', level=5, num='4.2.0.4.51'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addSeconds', level=5, num='4.2.0.4.52'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addQuarters', level=5, num='4.2.0.4.53'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractYears', level=5, num='4.2.0.4.54'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMonths', level=5, num='4.2.0.4.55'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractWeeks', level=5, num='4.2.0.4.56'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractDays', level=5, num='4.2.0.4.57'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractHours', level=5, num='4.2.0.4.58'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMinutes', level=5, num='4.2.0.4.59'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractSeconds', level=5, num='4.2.0.4.60'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractQuarters', level=5, num='4.2.0.4.61'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.dateDiff', level=5, num='4.2.0.4.62'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlots', level=5, num='4.2.0.4.63'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.formatDateTime', level=5, num='4.2.0.4.64'), - Heading(name='Type Conversion Functions', level=3, num='4.2.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toInt(8|16|32|64|128|256)', level=5, num='4.2.1.4.1'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUInt(8|16|32|64|256)', level=5, num='4.2.1.4.2'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toFloat(32|64)', level=5, num='4.2.1.4.3'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDate', level=5, num='4.2.1.4.4'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime', level=5, num='4.2.1.4.5'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64', level=5, num='4.2.1.4.6'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64.FromString.MissingTime', level=5, num='4.2.1.4.7'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDecimal(32|64|128|256)', level=5, num='4.2.1.4.8'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toString', level=5, num='4.2.1.4.9'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.CAST(x,T)', level=5, num='4.2.1.4.10'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Milli', level=5, num='4.2.1.4.11'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Micro', level=5, num='4.2.1.4.12'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Nano', level=5, num='4.2.1.4.13'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Milli', level=5, num='4.2.1.4.14'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Micro', level=5, num='4.2.1.4.15'), - Heading(name='RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Nano', level=5, num='4.2.1.4.16'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="SRS", level=2, num="3.1"), + Heading(name="Normal Date Range", level=2, num="3.2"), + Heading(name="Extended Date Range", level=2, num="3.3"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="Generic", level=2, num="4.1"), + Heading(name="RQ.SRS-010.DateTime64.ExtendedRange", level=4, num="4.1.0.1"), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start", + level=4, + num="4.1.0.2", ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.Start.BeforeEpochForTimeZone", + level=4, + num="4.1.0.3", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End", + level=4, + num="4.1.0.4", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NormalRange.End.AfterEpochForTimeZone", + level=4, + num="4.1.0.5", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions", + level=4, + num="4.1.0.6", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions", + level=4, + num="4.1.0.7", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TimeZones", level=4, num="4.1.0.8" + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime", + level=4, + num="4.1.0.9", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.Comparison", + level=4, + num="4.1.0.10", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.SpecificTimestamps", + level=4, + num="4.1.0.11", + ), + Heading(name="Specific", level=2, num="4.2"), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.Start", level=4, num="4.2.0.1" + ), + Heading(name="RQ.SRS-010.DateTime64.ExtendedRange.End", level=4, num="4.2.0.2"), + Heading(name="Non-Existent Time", level=4, num="4.2.0.3"), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidDate", + level=5, + num="4.2.0.3.1", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.InvalidTime", + level=5, + num="4.2.0.3.2", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.TimeZoneSwitch", + level=5, + num="4.2.0.3.3", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime", + level=5, + num="4.2.0.3.4", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.DaylightSavingTime.Disappeared", + level=5, + num="4.2.0.3.5", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.NonExistentTime.LeapSeconds", + level=5, + num="4.2.0.3.6", + ), + Heading(name="Dates And Times Functions", level=4, num="4.2.0.4"), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTimeZone", + level=5, + num="4.2.0.4.1", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYear", + level=5, + num="4.2.0.4.2", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toQuarter", + level=5, + num="4.2.0.4.3", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonth", + level=5, + num="4.2.0.4.4", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfYear", + level=5, + num="4.2.0.4.5", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfMonth", + level=5, + num="4.2.0.4.6", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toDayOfWeek", + level=5, + num="4.2.0.4.7", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toHour", + level=5, + num="4.2.0.4.8", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMinute", + level=5, + num="4.2.0.4.9", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toSecond", + level=5, + num="4.2.0.4.10", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toUnixTimestamp", + level=5, + num="4.2.0.4.11", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfYear", + level=5, + num="4.2.0.4.12", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfISOYear", + level=5, + num="4.2.0.4.13", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfQuarter", + level=5, + num="4.2.0.4.14", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMonth", + level=5, + num="4.2.0.4.15", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toMonday", + level=5, + num="4.2.0.4.16", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfWeek", + level=5, + num="4.2.0.4.17", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfDay", + level=5, + num="4.2.0.4.18", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfHour", + level=5, + num="4.2.0.4.19", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfMinute", + level=5, + num="4.2.0.4.20", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfSecond", + level=5, + num="4.2.0.4.21", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFiveMinute", + level=5, + num="4.2.0.4.22", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfTenMinutes", + level=5, + num="4.2.0.4.23", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfFifteenMinutes", + level=5, + num="4.2.0.4.24", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toStartOfInterval", + level=5, + num="4.2.0.4.25", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toTime", + level=5, + num="4.2.0.4.26", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeYearNum", + level=5, + num="4.2.0.4.27", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeQuarterNum", + level=5, + num="4.2.0.4.28", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMonthNum", + level=5, + num="4.2.0.4.29", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeWeekNum", + level=5, + num="4.2.0.4.30", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeDayNum", + level=5, + num="4.2.0.4.31", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeHourNum", + level=5, + num="4.2.0.4.32", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeMinuteNum", + level=5, + num="4.2.0.4.33", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toRelativeSecondNum", + level=5, + num="4.2.0.4.34", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOYear", + level=5, + num="4.2.0.4.35", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toISOWeek", + level=5, + num="4.2.0.4.36", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toWeek", + level=5, + num="4.2.0.4.37", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYearWeek", + level=5, + num="4.2.0.4.38", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.now", + level=5, + num="4.2.0.4.39", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.today", + level=5, + num="4.2.0.4.40", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.yesterday", + level=5, + num="4.2.0.4.41", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlot", + level=5, + num="4.2.0.4.42", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMM", + level=5, + num="4.2.0.4.43", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDD", + level=5, + num="4.2.0.4.44", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.toYYYYMMDDhhmmss", + level=5, + num="4.2.0.4.45", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addYears", + level=5, + num="4.2.0.4.46", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMonths", + level=5, + num="4.2.0.4.47", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addWeeks", + level=5, + num="4.2.0.4.48", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addDays", + level=5, + num="4.2.0.4.49", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addHours", + level=5, + num="4.2.0.4.50", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addMinutes", + level=5, + num="4.2.0.4.51", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addSeconds", + level=5, + num="4.2.0.4.52", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.addQuarters", + level=5, + num="4.2.0.4.53", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractYears", + level=5, + num="4.2.0.4.54", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMonths", + level=5, + num="4.2.0.4.55", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractWeeks", + level=5, + num="4.2.0.4.56", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractDays", + level=5, + num="4.2.0.4.57", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractHours", + level=5, + num="4.2.0.4.58", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractMinutes", + level=5, + num="4.2.0.4.59", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractSeconds", + level=5, + num="4.2.0.4.60", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.subtractQuarters", + level=5, + num="4.2.0.4.61", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.dateDiff", + level=5, + num="4.2.0.4.62", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.timeSlots", + level=5, + num="4.2.0.4.63", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.DatesAndTimesFunctions.formatDateTime", + level=5, + num="4.2.0.4.64", + ), + Heading(name="Type Conversion Functions", level=3, num="4.2.1"), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toInt(8|16|32|64|128|256)", + level=5, + num="4.2.1.4.1", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUInt(8|16|32|64|256)", + level=5, + num="4.2.1.4.2", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toFloat(32|64)", + level=5, + num="4.2.1.4.3", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDate", + level=5, + num="4.2.1.4.4", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime", + level=5, + num="4.2.1.4.5", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64", + level=5, + num="4.2.1.4.6", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDateTime64.FromString.MissingTime", + level=5, + num="4.2.1.4.7", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toDecimal(32|64|128|256)", + level=5, + num="4.2.1.4.8", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toString", + level=5, + num="4.2.1.4.9", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.CAST(x,T)", + level=5, + num="4.2.1.4.10", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Milli", + level=5, + num="4.2.1.4.11", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Micro", + level=5, + num="4.2.1.4.12", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.toUnixTimestamp64Nano", + level=5, + num="4.2.1.4.13", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Milli", + level=5, + num="4.2.1.4.14", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Micro", + level=5, + num="4.2.1.4.15", + ), + Heading( + name="RQ.SRS-010.DateTime64.ExtendedRange.TypeConversionFunctions.fromUnixTimestamp64Nano", + level=5, + num="4.2.1.4.16", + ), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_010_DateTime64_ExtendedRange, RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_Start, @@ -1858,8 +2341,8 @@ SRS_010_ClickHouse_DateTime64_Extended_Range = Specification( RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Milli, RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Micro, RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Nano, - ), - content=''' + ), + content=""" # SRS-010 ClickHouse DateTime64 Extended Range # Software Requirements Specification @@ -2665,4 +3148,5 @@ using the [fromUnixTimestamp64Nano](https://clickhouse.com/docs/en/sql-reference [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/datetime64_extended_range/requirements/requirements.md [Git]: https://git-scm.com/ [GitHub]: https://github.com -''') +""", +) diff --git a/tests/testflows/datetime64_extended_range/tests/common.py b/tests/testflows/datetime64_extended_range/tests/common.py index c3bee076bf4..1154cf21b92 100644 --- a/tests/testflows/datetime64_extended_range/tests/common.py +++ b/tests/testflows/datetime64_extended_range/tests/common.py @@ -8,30 +8,31 @@ from datetime64_extended_range.common import * def in_normal_range(dt: datetime.datetime): - """Check if DateTime is in normal range - """ - return dt <= datetime.datetime(2105, 12, 31, 23, 59, 59, 999999) and dt >= datetime.datetime(1970, 1, 1, 0, 0, 0) + """Check if DateTime is in normal range""" + return dt <= datetime.datetime( + 2105, 12, 31, 23, 59, 59, 999999 + ) and dt >= datetime.datetime(1970, 1, 1, 0, 0, 0) def years_range(stress=False, padding=(0, 0)): - """Returns a set of year values used for testing. - """ - return range(1925+padding[0], 2283-padding[1]) if stress else (1927, 2000, 2281) + """Returns a set of year values used for testing.""" + return range(1925 + padding[0], 2283 - padding[1]) if stress else (1927, 2000, 2281) def timezones_range(stress=False): - """Returns a set of timezone values used for testing. - """ + """Returns a set of timezone values used for testing.""" if stress: return pytz.all_timezones else: - return ['UTC', 'Asia/Novosibirsk', 'America/Denver'] + return ["UTC", "Asia/Novosibirsk", "America/Denver"] @contextmanager def create_table(timezone, node): try: - node.query(f"CREATE TABLE dt(timestamp DateTime64(3, {timezone})) Engine = TinyLog") + node.query( + f"CREATE TABLE dt(timestamp DateTime64(3, {timezone})) Engine = TinyLog" + ) yield finally: node.query("DROP TABLE dt") @@ -49,15 +50,16 @@ def insert_check_datetime(self, datetime, expected, precision=0, timezone="UTC") """ with create_table(timezone, self.context.node): with When("I use toDateTime64"): - r = self.context.node.query(f"SELECT toDateTime64('{datetime}', {precision}, '{timezone}')") + r = self.context.node.query( + f"SELECT toDateTime64('{datetime}', {precision}, '{timezone}')" + ) with Then(f"I expect {expected}"): assert r.output == expected, error() def datetime_generator(year, microseconds=False): - """Helper generator - """ + """Helper generator""" date = datetime.datetime(year, 1, 1, 0, 0, 0) if microseconds: date = datetime.datetime(year, 1, 1, 0, 0, 0, 123000) @@ -67,12 +69,17 @@ def datetime_generator(year, microseconds=False): def select_dates_in_year(year, stress=False, microseconds=False): - """Returns various datetimes in a year that are to be checked - """ + """Returns various datetimes in a year that are to be checked""" if not stress: - dates = [datetime.datetime(year, 1, 1, 0, 0, 0), datetime.datetime(year, 12, 31, 23, 59, 59)] + dates = [ + datetime.datetime(year, 1, 1, 0, 0, 0), + datetime.datetime(year, 12, 31, 23, 59, 59), + ] if microseconds: - dates = [datetime.datetime(year, 1, 1, 0, 0, 0, 123000), datetime.datetime(year, 12, 31, 23, 59, 59, 123000)] + dates = [ + datetime.datetime(year, 1, 1, 0, 0, 0, 123000), + datetime.datetime(year, 12, 31, 23, 59, 59, 123000), + ] if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0): dates.append(datetime.datetime(year, 2, 29, 11, 11, 11, 123000)) return dates @@ -91,7 +98,9 @@ def select_check_datetime(self, datetime, expected, precision=0, timezone="UTC") :param timezone: timezone, default: UTC """ with When("I use toDateTime64"): - r = self.context.node.query(f"SELECT toDateTime64('{datetime}', {precision}, '{timezone}')") + r = self.context.node.query( + f"SELECT toDateTime64('{datetime}', {precision}, '{timezone}')" + ) with Then(f"I expect {expected}"): assert r.output == expected, error() @@ -116,7 +125,9 @@ def exec_query(self, request, expected=None, exitcode=None): @TestStep -def walk_datetime_in_incrementing_steps(self, date, hrs_range=(0, 24), step=1, timezone="UTC", precision=0): +def walk_datetime_in_incrementing_steps( + self, date, hrs_range=(0, 24), step=1, timezone="UTC", precision=0 +): """Sweep time starting from some start date. The time is incremented in steps specified by the `step` parameter (default: 1 min). @@ -130,22 +141,38 @@ def walk_datetime_in_incrementing_steps(self, date, hrs_range=(0, 24), step=1, t with Pool(2) as pool: try: - with When(f"I loop through datetime range {hrs_range} starting from {date} in {step}min increments"): - for hrs in range(*hrs_range) if stress else (hrs_range[0], hrs_range[1]-1): + with When( + f"I loop through datetime range {hrs_range} starting from {date} in {step}min increments" + ): + for hrs in ( + range(*hrs_range) if stress else (hrs_range[0], hrs_range[1] - 1) + ): for mins in range(0, 60, step) if stress else (0, 59): - datetime = f"{date} {str(hrs).zfill(2)}:{str(mins).zfill(2)}:{secs}" + datetime = ( + f"{date} {str(hrs).zfill(2)}:{str(mins).zfill(2)}:{secs}" + ) expected = datetime with When(f"time is {datetime}"): - Test(name=f"{hrs}:{mins}:{secs}", test=select_check_datetime, parallel=True, executor=pool)( - datetime=datetime, precision=precision, timezone=timezone, - expected=expected) + Test( + name=f"{hrs}:{mins}:{secs}", + test=select_check_datetime, + parallel=True, + executor=pool, + )( + datetime=datetime, + precision=precision, + timezone=timezone, + expected=expected, + ) finally: join() @TestStep -def walk_datetime_in_decrementing_steps(self, date, hrs_range=(23, 0), step=1, timezone="UTC", precision=0): +def walk_datetime_in_decrementing_steps( + self, date, hrs_range=(23, 0), step=1, timezone="UTC", precision=0 +): """Sweep time starting from some start date. The time is decremented in steps specified by the `step` parameter (default: 1 min). @@ -160,15 +187,29 @@ def walk_datetime_in_decrementing_steps(self, date, hrs_range=(23, 0), step=1, t with Pool(2) as pool: try: - with When(f"I loop through datetime range {hrs_range} starting from {date} in {step}min decrements"): - for hrs in range(*hrs_range, -1) if stress else (hrs_range[1], hrs_range[0]): + with When( + f"I loop through datetime range {hrs_range} starting from {date} in {step}min decrements" + ): + for hrs in ( + range(*hrs_range, -1) if stress else (hrs_range[1], hrs_range[0]) + ): for mins in range(59, 0, -step) if stress else (59, 0): - datetime = f"{date} {str(hrs).zfill(2)}:{str(mins).zfill(2)}:{secs}" + datetime = ( + f"{date} {str(hrs).zfill(2)}:{str(mins).zfill(2)}:{secs}" + ) expected = datetime with When(f"time is {datetime}"): - Test(name=f"{hrs}:{mins}:{secs}", test=select_check_datetime, parallel=True, executor=pool)( - datetime=datetime, precision=precision, timezone=timezone, - expected=expected) + Test( + name=f"{hrs}:{mins}:{secs}", + test=select_check_datetime, + parallel=True, + executor=pool, + )( + datetime=datetime, + precision=precision, + timezone=timezone, + expected=expected, + ) finally: join() diff --git a/tests/testflows/datetime64_extended_range/tests/date_time_functions.py b/tests/testflows/datetime64_extended_range/tests/date_time_functions.py index f972caac95b..53add63e8f2 100644 --- a/tests/testflows/datetime64_extended_range/tests/date_time_functions.py +++ b/tests/testflows/datetime64_extended_range/tests/date_time_functions.py @@ -15,8 +15,7 @@ from datetime64_extended_range.tests.common import * RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toTimeZone("1.0") ) def to_time_zone(self): - """Check the toTimeZone() function with DateTime64 extended range. - """ + """Check the toTimeZone() function with DateTime64 extended range.""" stress = self.context.stress timezones = timezones_range(stress) @@ -40,8 +39,7 @@ def to_time_zone(self): @TestOutline def to_date_part(self, py_func, ch_func): - """Check the toYear/toMonth/toQuarter functions with DateTime64 extended range. - """ + """Check the toYear/toMonth/toQuarter functions with DateTime64 extended range.""" stress = self.context.stress for year in years_range(stress): @@ -56,9 +54,13 @@ def to_date_part(self, py_func, ch_func): with Given("I compute expected output using pytz"): with By(f"localizing {dt} using {tz1} timezone"): time_tz1 = pytz.timezone(tz1).localize(dt) - with And(f"converting {tz1} local datetime {dt} to {tz2} timezone"): + with And( + f"converting {tz1} local datetime {dt} to {tz2} timezone" + ): time_tz2 = time_tz1.astimezone(pytz.timezone(tz2)) - with And(f"calling the '{py_func}' method of the datetime object to get expected result"): + with And( + f"calling the '{py_func}' method of the datetime object to get expected result" + ): result = eval(f"(time_tz2.{py_func}") expected = f"{result}" with And(f"Forming a {ch_func} ClickHouse query"): @@ -71,7 +73,7 @@ def to_date_part(self, py_func, ch_func): @TestScenario @Requirements( RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toYear("1.0"), - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeYearNum("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeYearNum("1.0"), ) def to_year(self): """Check the toYear() and toRelativeYearNum() [which is just an alias for toYear] @@ -81,12 +83,9 @@ def to_year(self): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toMonth("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toMonth("1.0")) def to_month(self): - """Check the toMonth() function with DateTime64 extended range. - """ + """Check the toMonth() function with DateTime64 extended range.""" to_date_part(py_func="month)", ch_func="toMonth") @@ -106,8 +105,7 @@ def to_quarter(self): @TestOutline def to_day_of(self, py_func, ch_func): - """Check the toDayOf....() functions with DateTime64 extended range. - """ + """Check the toDayOf....() functions with DateTime64 extended range.""" stress = self.context.stress for year in years_range(stress): @@ -138,8 +136,7 @@ def to_day_of(self, py_func, ch_func): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toDayOfYear("1.0") ) def to_day_of_year(self): - """Check toDayOfYear() function with DateTime64 extended range date time. - """ + """Check toDayOfYear() function with DateTime64 extended range date time.""" to_day_of(py_func="tm_yday", ch_func="toDayOfYear") @@ -148,8 +145,7 @@ def to_day_of_year(self): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toDayOfMonth("1.0") ) def to_day_of_month(self): - """Check toDayOfMonth() function with DateTime64 extended range date time. - """ + """Check toDayOfMonth() function with DateTime64 extended range date time.""" to_day_of(py_func="tm_mday", ch_func="toDayOfMonth") @@ -158,15 +154,13 @@ def to_day_of_month(self): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toDayOfWeek("1.0") ) def to_day_of_week(self): - """Check toDayOfWeek() function with DateTime64 extended range date time. - """ + """Check toDayOfWeek() function with DateTime64 extended range date time.""" to_day_of(py_func="tm_wday", ch_func="toDayOfWeek") @TestOutline def to_time_part(self, py_func, ch_func): - """Check the functions like toHour/toMinute/toSecond with DateTime64 extended range. - """ + """Check the functions like toHour/toMinute/toSecond with DateTime64 extended range.""" stress = self.context.stress for year in years_range(stress): @@ -191,12 +185,9 @@ def to_time_part(self, py_func, ch_func): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toHour("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toHour("1.0")) def to_hour(self): - """Check toHour() function with DateTime64 extended range date time. - """ + """Check toHour() function with DateTime64 extended range date time.""" to_time_part(py_func="hour", ch_func="toHour") @@ -205,8 +196,7 @@ def to_hour(self): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toMinute("1.0") ) def to_minute(self): - """Check toMinute() function with DateTime64 extended range date time. - """ + """Check toMinute() function with DateTime64 extended range date time.""" to_time_part(py_func="minute", ch_func="toMinute") @@ -215,8 +205,7 @@ def to_minute(self): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toSecond("1.0") ) def to_second(self): - """Check toSecond() function with DateTime64 extended range date time. - """ + """Check toSecond() function with DateTime64 extended range date time.""" to_time_part(py_func="second", ch_func="toSecond") @@ -225,8 +214,7 @@ def to_second(self): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toUnixTimestamp("1.0") ) def to_unix_timestamp(self): - """Check the toUnixTimestamp() function with DateTime64 extended range - """ + """Check the toUnixTimestamp() function with DateTime64 extended range""" stress = self.context.stress for year in years_range(stress): @@ -277,7 +265,7 @@ def to_start_of_year(self): def iso_year_start(dt): """Helper to find the beginning of iso year.""" - dt_s = datetime.datetime(dt.year-1, 12, 23, 0, 0, 0) + dt_s = datetime.datetime(dt.year - 1, 12, 23, 0, 0, 0) while dt_s.isocalendar()[0] != dt.year: dt_s += datetime.timedelta(days=1) return dt_s @@ -306,7 +294,10 @@ def to_start_of_iso_year(self): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") query = f"SELECT toStartOfISOYear(toDateTime64('{dt_str}', 0, '{tz}'))" with Then("I execute toStartOfISOYear query"): - exec_query(request=query, expected=f"{expected.strftime('%Y-%m-%d')}") + exec_query( + request=query, + expected=f"{expected.strftime('%Y-%m-%d')}", + ) @TestScenario @@ -331,7 +322,9 @@ def to_start_of_quarter(self): with By("computing expected result with python"): time_tz1 = pytz.timezone(tz1).localize(dt) time_tz2 = time_tz1.astimezone(pytz.timezone(tz2)) - expected = f"{year}-{str(time_tz2.month//3 * 3 + 1).zfill(2)}-01" + expected = ( + f"{year}-{str(time_tz2.month//3 * 3 + 1).zfill(2)}-01" + ) with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") query = f"SELECT toStartOfQuarter(toDateTime64('{dt_str}', 0, '{tz1}'), '{tz2}')" @@ -391,7 +384,13 @@ def to_monday(self): with By("computing expected result with python"): time_tz1 = pytz.timezone(tz1).localize(dt) time_tz2 = time_tz1.astimezone(pytz.timezone(tz2)) - expected_date = time_tz2 + datetime.timedelta(days=(-dt.weekday() if dt.weekday() <= 3 else 7 - dt.weekday())) + expected_date = time_tz2 + datetime.timedelta( + days=( + -dt.weekday() + if dt.weekday() <= 3 + else 7 - dt.weekday() + ) + ) expected = f"{expected_date.strftime('%Y-%m-%d')}" with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") @@ -416,13 +415,21 @@ def to_start_of_week(self): with When(f"I check each of the datetimes in {year}"): for dt in datetimes: for tz1, tz2 in itertools.product(timezones, timezones): - for mode in (0, 1): # mode - week beginning, either 0 (Sunday) or 1 (Monday) + for mode in ( + 0, + 1, + ): # mode - week beginning, either 0 (Sunday) or 1 (Monday) with Step(f"{dt} {tz1} -> {tz2}"): with By("computing expected result with python"): time_tz1 = pytz.timezone(tz1).localize(dt) time_tz2 = time_tz1.astimezone(pytz.timezone(tz2)) expected_date = time_tz2 + datetime.timedelta( - days=(mode - dt.weekday() if dt.weekday() <= (3+mode) else (mode + 7) - dt.weekday())) + days=( + mode - dt.weekday() + if dt.weekday() <= (3 + mode) + else (mode + 7) - dt.weekday() + ) + ) expected = f"{expected_date.strftime('%Y-%m-%d')}" with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") @@ -520,7 +527,9 @@ def to_start_of_second(self): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -543,7 +552,9 @@ def to_start_of_minutes_interval(self, interval, func): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -551,17 +562,23 @@ def to_start_of_minutes_interval(self, interval, func): with Step(f"{dt} {tz}"): with By("Computing expected result using python"): mins = dt.minute // interval * interval - expected = f"{dt.strftime('%Y-%m-%d %H:')}{str(mins).zfill(2)}:00" + expected = ( + f"{dt.strftime('%Y-%m-%d %H:')}{str(mins).zfill(2)}:00" + ) with And(f"Forming a {func} query to ClickHouse"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") - query = f"SELECT {func}(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT {func}(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then(f"I execute {func} query"): exec_query(request=query, expected=f"{expected}") @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfFiveMinute("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfFiveMinute( + "1.0" + ) ) def to_start_of_five_minute(self): """Check the toStartOfFiveMinute with DateTime64 extended range.""" @@ -570,7 +587,9 @@ def to_start_of_five_minute(self): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfTenMinutes("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfTenMinutes( + "1.0" + ) ) def to_start_of_ten_minutes(self): """Check the toStartOfTenMinutes with DateTime64 extended range.""" @@ -579,7 +598,9 @@ def to_start_of_ten_minutes(self): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfFifteenMinutes("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfFifteenMinutes( + "1.0" + ) ) def to_start_of_fifteen_minutes(self): """Check the toStartOfFifteenMinutes with DateTime64 extended range.""" @@ -602,13 +623,21 @@ def to_start_of_interval_helper(dt: datetime.datetime, interval_type, interval_v :param interval_type: interval type selector, String :param interval_value: interval size, int """ - intervals_in_seconds = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600, "DAY": 68400, "WEEK": 604800} + intervals_in_seconds = { + "SECOND": 1, + "MINUTE": 60, + "HOUR": 3600, + "DAY": 68400, + "WEEK": 604800, + } zero_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0) delta = dt - zero_datetime if interval_type in intervals_in_seconds.keys(): divisor = interval_value * intervals_in_seconds[interval_type] - retval = (zero_datetime + datetime.timedelta(seconds=(delta.seconds // divisor * divisor))) + retval = zero_datetime + datetime.timedelta( + seconds=(delta.seconds // divisor * divisor) + ) if interval_type == "WEEK": return retval.strftime("%Y-%m-%d") return retval.strftime("%Y-%m-%d %H:%M:%S") @@ -616,16 +645,24 @@ def to_start_of_interval_helper(dt: datetime.datetime, interval_type, interval_v elif interval_type == "MONTH": diff = (dt.year - zero_datetime.year) * 12 + (dt.month - zero_datetime.month) result_diff = diff // interval_value * interval_value - return (zero_datetime + rd.relativedelta(months=result_diff)).strftime("%Y-%m-%d") + return (zero_datetime + rd.relativedelta(months=result_diff)).strftime( + "%Y-%m-%d" + ) elif interval_type == "QUARTER": - diff = (dt.year - zero_datetime.year) * 4 + (dt.month // 4 - zero_datetime.month // 4) + diff = (dt.year - zero_datetime.year) * 4 + ( + dt.month // 4 - zero_datetime.month // 4 + ) result_diff = diff // interval_value * interval_value - return (zero_datetime + rd.relativedelta(months=result_diff*4)).strftime("%Y-%m-%d") + return (zero_datetime + rd.relativedelta(months=result_diff * 4)).strftime( + "%Y-%m-%d" + ) elif interval_type == "YEAR": result_diff = (dt.year - zero_datetime.year) // interval_value * interval_value - return (zero_datetime + rd.relativedelta(years=result_diff)).strftime("%Y-%m-%d") + return (zero_datetime + rd.relativedelta(years=result_diff)).strftime( + "%Y-%m-%d" + ) @TestScenario @@ -633,13 +670,20 @@ def to_start_of_interval_helper(dt: datetime.datetime, interval_type, interval_v RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toStartOfInterval("1.0") ) def to_start_of_interval(self): - """Check the toStartOfInterval with DateTime64 extended range. - """ + """Check the toStartOfInterval with DateTime64 extended range.""" stress = self.context.stress timezones = timezones_range(stress) - intervals_testing_ranges = {"SECOND": range(1, 15), "MINUTE": range(1, 15), "HOUR": range(1, 10), "DAY": (1, 5, 10), - "WEEK": range(1, 5), "MONTH": range(1, 6), "QUARTER": range(1, 4), "YEAR": range(1, 5)} + intervals_testing_ranges = { + "SECOND": range(1, 15), + "MINUTE": range(1, 15), + "HOUR": range(1, 10), + "DAY": (1, 5, 10), + "WEEK": range(1, 5), + "MONTH": range(1, 6), + "QUARTER": range(1, 4), + "YEAR": range(1, 5), + } for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): @@ -652,8 +696,12 @@ def to_start_of_interval(self): for value in intervals_testing_ranges[interval]: with Step(f"{dt} {tz} {interval}: {value}"): with By("Computing expected result using python"): - expected = to_start_of_interval_helper(dt, interval, value) - with And(f"Forming a toStartOfInterval() query to ClickHouse"): + expected = to_start_of_interval_helper( + dt, interval, value + ) + with And( + f"Forming a toStartOfInterval() query to ClickHouse" + ): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") query = f"SELECT toStartOfInterval(toDateTime64('{dt_str}', 0, '{tz}'), INTERVAL {value} {interval})" with Then(f"I execute toStartOfInterval() query"): @@ -668,7 +716,9 @@ def to_iso(self, func, isocalendar_pos): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -678,7 +728,9 @@ def to_iso(self, func, isocalendar_pos): expected = f"{dt.isocalendar()[isocalendar_pos]}" with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") - query = f"SELECT {func}(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT {func}(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then("I execute query"): exec_query(request=query, expected=f"{expected}") @@ -702,9 +754,7 @@ def to_iso_week(self): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toTime("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toTime("1.0")) def to_time(self): """Check the toTime function with DateTime64 extended range.""" stress = self.context.stress @@ -712,7 +762,9 @@ def to_time(self): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -722,14 +774,18 @@ def to_time(self): expected = f"1970-01-02 {dt.strftime('%H:%M:%S')}" with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") - query = f"SELECT toTime(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT toTime(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then("I execute query"): exec_query(request=query, expected=f"{expected}") @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeQuarterNum("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeQuarterNum( + "1.0" + ) ) def to_relative_quarter_num(self): """Check the toRelativeQuarterNum function with DateTime64 extended range.""" @@ -738,7 +794,9 @@ def to_relative_quarter_num(self): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -764,14 +822,23 @@ def to_relative_week_num(self): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: for tz in timezones: with When(f"{dt} {tz}"): with By("computing expected result using python"): - week_num = ((dt + datetime.timedelta(days=8) - datetime.timedelta(days=dt.weekday())) - datetime.datetime(1970, 1, 1, 0, 0, 0)).days // 7 + week_num = ( + ( + dt + + datetime.timedelta(days=8) + - datetime.timedelta(days=dt.weekday()) + ) + - datetime.datetime(1970, 1, 1, 0, 0, 0) + ).days // 7 expected = f"{week_num}" with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") @@ -791,7 +858,9 @@ def to_relative_month_num(self): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -818,7 +887,9 @@ def to_relative_day_num(self): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: @@ -844,18 +915,24 @@ def to_relative_time(self, divisor, func): for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) with When(f"I check each of the datetimes in {year}"): for dt in datetimes: for tz in timezones: with When(f"{dt} {tz}"): with By("Computing the expected result using python"): - result = (dt - datetime.datetime(1970, 1, 1, 0, 0, 0)).total_seconds() // divisor + result = ( + dt - datetime.datetime(1970, 1, 1, 0, 0, 0) + ).total_seconds() // divisor expected = f"{result}" with And(f"Forming a {func} query to ClickHouse"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") - query = f"SELECT {func}(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT {func}(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then(f"I execute {func} query"): exec_query(request=query, expected=f"{expected}") @@ -873,7 +950,9 @@ def to_relative_hour_num(self): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeMinuteNum("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeMinuteNum( + "1.0" + ) ) def to_relative_minute_num(self): """Check the toRelativeMinuteNum function @@ -884,7 +963,9 @@ def to_relative_minute_num(self): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeSecondNum("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toRelativeSecondNum( + "1.0" + ) ) def to_relative_second_num(self): """Check the toRelativeSecondNum function @@ -904,72 +985,114 @@ def to_week_compute_expected(dt: datetime.datetime, mode: int, ret_year=False): while ex.weekday() != 0: ex += datetime.timedelta(days=1) - first_monday = ex.day-1 + first_monday = ex.day - 1 ex = datetime.datetime(year, 1, 1) while ex.weekday() != 6: ex += datetime.timedelta(days=1) - first_sunday = ex.day-1 + first_sunday = ex.day - 1 if mode == 0: # First day of week: Sunday, Week 1 is the first week with Sunday, range 0-53 - expected = (dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_sunday)).days // 7 + 1 + expected = ( + dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_sunday) + ).days // 7 + 1 elif mode == 1: # First day of week: Monday, Week 1 is the first week containing 4 or more days, range 0-53 if j1_weekday <= 3: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=7+j1_weekday)).days // 7 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=7 + j1_weekday) + ).days // 7 else: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday)).days // 7 + expected = ( + dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday) + ).days // 7 elif mode == 2: # First day of week: Sunday, Week 1 is the first week with Sunday, range 1-53 - expected = (dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_sunday)).days // 7 + 1 + expected = ( + dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_sunday) + ).days // 7 + 1 if expected == 0: - return to_week_compute_expected(datetime.datetime(dt.year-1, 12, 31), 2) + return to_week_compute_expected(datetime.datetime(dt.year - 1, 12, 31), 2) elif mode == 3: # First day of week: Monday, Week 1 is the first week containing 4 or more days, range 1-53 if j1_weekday <= 3: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=7+j1_weekday)).days // 7 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=7 + j1_weekday) + ).days // 7 else: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday)).days // 7 + expected = ( + dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday) + ).days // 7 if expected == 0: - return to_week_compute_expected(datetime.datetime(dt.year-1, 12, 31), 3) + return to_week_compute_expected(datetime.datetime(dt.year - 1, 12, 31), 3) elif mode == 4: # First day of week: Sunday, Week 1 is the first week containing 4 or more days, range 0-53 if j1_weekday <= 3: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=8+j1_weekday)).days // 7 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=8 + j1_weekday) + ).days // 7 else: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday+1)).days // 7 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=j1_weekday + 1) + ).days // 7 elif mode == 5: # First day of week: Monday, Week 1 is the first week with a Monday, range 0-53 - expected = (dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_monday)).days // 7 + 1 + expected = ( + dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_monday) + ).days // 7 + 1 elif mode == 6: # First day of week: Sunday, Week 1 is the first week containing 4 or more days, range 1-53 if j1_weekday <= 3: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=8+j1_weekday)).days // 7 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=8 + j1_weekday) + ).days // 7 else: - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday+1)).days // 7 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=j1_weekday + 1) + ).days // 7 if expected == 0: - return to_week_compute_expected(datetime.datetime(dt.year-1, 12, 31), 6) + return to_week_compute_expected(datetime.datetime(dt.year - 1, 12, 31), 6) elif mode == 7: # First day of week: Monday, Week 1 is the first week with a Monday, range 1-53 - expected = (dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_monday)).days // 7 + 1 + expected = ( + dt - datetime.datetime(year, 1, 1) - datetime.timedelta(days=first_monday) + ).days // 7 + 1 if expected == 0: - return to_week_compute_expected(datetime.datetime(dt.year-1, 12, 31), 7) + return to_week_compute_expected(datetime.datetime(dt.year - 1, 12, 31), 7) elif mode == 8: # First day of week: Sunday, Week 1 is the week containing January 1, range 1-53 - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=(j1_weekday+1)%7)).days // 7 + 1 + expected = ( + dt + - datetime.datetime(year, 1, 1) + + datetime.timedelta(days=(j1_weekday + 1) % 7) + ).days // 7 + 1 elif mode == 9: # First day of week: Monday, Week 1 is the week containing January 1, range 1-53 - expected = (dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday%7)).days // 7 + 1 + expected = ( + dt - datetime.datetime(year, 1, 1) + datetime.timedelta(days=j1_weekday % 7) + ).days // 7 + 1 return f"{dt.year}{str(expected).zfill(2)}" if ret_year else f"{expected}" @@ -996,7 +1119,9 @@ def to_week_year_week(self, clh_func, ret_year): for mode in range(0, 10): with When(f"{dt} {tz}, mode={mode}"): with By("Computing expected output using python"): - expected = to_week_compute_expected(dt=dt, mode=mode, ret_year=ret_year) + expected = to_week_compute_expected( + dt=dt, mode=mode, ret_year=ret_year + ) with And(f"Forming a {clh_func} query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") query = f"SELECT {clh_func}(toDateTime64('{dt_str}', 0, '{tz}'), {mode})" @@ -1005,9 +1130,7 @@ def to_week_year_week(self, clh_func, ret_year): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toWeek("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_toWeek("1.0")) def to_week(self): """Check the toWeek function with DateTime64 extended range.""" to_week_year_week(clh_func="toWeek", ret_year=False) @@ -1045,7 +1168,9 @@ def to_yyyymm(self): expected = f"{dt.strftime('%Y%m')}" with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") - query = f"SELECT toYYYYMM(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT toYYYYMM(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then("I execute query"): exec_query(request=query, expected=f"{expected}") @@ -1074,7 +1199,11 @@ def to_yyyymmdd(self): with And("forming ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") query = f"SELECT toYYYYMMDD(toDateTime64('{dt_str}', 0, '{tz}'))" - with Then("I execute query", description=f"expected {expected}", flags=TE): + with Then( + "I execute query", + description=f"expected {expected}", + flags=TE, + ): exec_query(request=query, expected=f"{expected}") @@ -1107,9 +1236,7 @@ def to_yyyymmddhhmmss(self): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_now("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_now("1.0")) def now(self): """Check the now() conversion to DateTime64 extended range. In this test, we cannot assure that pytz now() and ClickHouse now() will be executed at the same time, so we need @@ -1127,9 +1254,13 @@ def now(self): with When("I execute query and format its result to string"): r = self.context.node.query(f"SELECT toDateTime64(now(), 0, '{tz}')") query_result = r.output - received_dt = datetime.datetime.strptime(query_result, '%Y-%m-%d %H:%M:%S') + received_dt = datetime.datetime.strptime( + query_result, "%Y-%m-%d %H:%M:%S" + ) - with Then("I compute the difference between ClickHouse query result and pytz result"): + with Then( + "I compute the difference between ClickHouse query result and pytz result" + ): dt = dt.replace(tzinfo=None) if dt < received_dt: diff = (received_dt - dt).total_seconds() @@ -1141,12 +1272,9 @@ def now(self): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_today("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_today("1.0")) def today(self): - """Check the today() conversion to DateTime64 extended range. - """ + """Check the today() conversion to DateTime64 extended range.""" stress = self.context.stress timezones = timezones_range(stress) @@ -1159,9 +1287,13 @@ def today(self): with When("I execute query and format its result to string"): r = self.context.node.query(f"SELECT toDateTime64(today(), 0, '{tz}')") query_result = r.output - received_dt = datetime.datetime.strptime(query_result, '%Y-%m-%d %H:%M:%S') + received_dt = datetime.datetime.strptime( + query_result, "%Y-%m-%d %H:%M:%S" + ) - with Then("I compute the difference between ClickHouse query result and pytz result"): + with Then( + "I compute the difference between ClickHouse query result and pytz result" + ): dt = dt.replace(tzinfo=None) if dt < received_dt: diff = (received_dt - dt).total_seconds() @@ -1177,8 +1309,7 @@ def today(self): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_yesterday("1.0") ) def yesterday(self): - """Check the yesterday() conversion to DateTime64 extended range. - """ + """Check the yesterday() conversion to DateTime64 extended range.""" stress = self.context.stress timezones = timezones_range(stress) @@ -1190,11 +1321,17 @@ def yesterday(self): with Step(f"{dt} {tz}"): with When("I execute query and format its result to string"): - r = self.context.node.query(f"SELECT toDateTime64(yesterday(), 0, '{tz}')") + r = self.context.node.query( + f"SELECT toDateTime64(yesterday(), 0, '{tz}')" + ) query_result = r.output - received_dt = datetime.datetime.strptime(query_result, '%Y-%m-%d %H:%M:%S') + received_dt = datetime.datetime.strptime( + query_result, "%Y-%m-%d %H:%M:%S" + ) - with Then("I compute the difference between ClickHouse query result and pytz result"): + with Then( + "I compute the difference between ClickHouse query result and pytz result" + ): dt = dt.replace(tzinfo=None) dt -= datetime.timedelta(days=1) if dt < received_dt: @@ -1207,7 +1344,9 @@ def yesterday(self): @TestOutline -def add_subtract_functions(self, clh_func, py_key, test_range, years_padding=(1, 1), modifier=1, mult=1): +def add_subtract_functions( + self, clh_func, py_key, test_range, years_padding=(1, 1), modifier=1, mult=1 +): """Check the addYears/addMonths/addWeeks/addDays/addHours/addMinutes/addSeconds with DateTime64 extended range. Calculating expected result using eval() to avoid writing 9000+ comparisons and just parse string as object field name. :param self: self @@ -1233,7 +1372,9 @@ def add_subtract_functions(self, clh_func, py_key, test_range, years_padding=(1, with By("converting datetime to string"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") with And("computing the expected result using pytz"): - dt_new = dt + rd.relativedelta(**{py_key: mult*incr*modifier}) + dt_new = dt + rd.relativedelta( + **{py_key: mult * incr * modifier} + ) tzone = pytz.timezone(tz) dt_norm = tzone.normalize(tzone.localize(dt_new)) expected = f"{dt_norm.strftime('%Y-%m-%d %H:%M:%S')}" @@ -1249,7 +1390,9 @@ def add_subtract_functions(self, clh_func, py_key, test_range, years_padding=(1, ) def add_years(self): """Check the addYears function with DateTime64 extended range.""" - add_subtract_functions(clh_func="addYears", py_key="years", test_range=(0, 1), years_padding=(0, 1)) + add_subtract_functions( + clh_func="addYears", py_key="years", test_range=(0, 1), years_padding=(0, 1) + ) @TestScenario @@ -1258,7 +1401,13 @@ def add_years(self): ) def subtract_years(self): """Check the subtractYears function with DateTime64 extended range.""" - add_subtract_functions(clh_func="subtractYears", py_key="years", test_range=(0, 1), years_padding=(1, 0), mult=-1) + add_subtract_functions( + clh_func="subtractYears", + py_key="years", + test_range=(0, 1), + years_padding=(1, 0), + mult=-1, + ) @TestScenario @@ -1267,7 +1416,13 @@ def subtract_years(self): ) def add_quarters(self): """Check the addQuarters function with DateTime64 extended range.""" - add_subtract_functions(clh_func="addQuarters", py_key="months", test_range=range(1, 5), years_padding=(0, 1), modifier=3) + add_subtract_functions( + clh_func="addQuarters", + py_key="months", + test_range=range(1, 5), + years_padding=(0, 1), + modifier=3, + ) @TestScenario @@ -1276,7 +1431,14 @@ def add_quarters(self): ) def subtract_quarters(self): """Check the subtractQuarters function with DateTime64 extended range.""" - add_subtract_functions(clh_func="subtractQuarters", py_key="months", test_range=range(1, 5), years_padding=(1, 0), modifier=3, mult=-1) + add_subtract_functions( + clh_func="subtractQuarters", + py_key="months", + test_range=range(1, 5), + years_padding=(1, 0), + modifier=3, + mult=-1, + ) @TestScenario @@ -1285,7 +1447,12 @@ def subtract_quarters(self): ) def add_months(self): """Check the addMonths function with DateTime64 extended range.""" - add_subtract_functions(clh_func="addMonths", py_key="months", test_range=range(1, 13), years_padding=(0, 1)) + add_subtract_functions( + clh_func="addMonths", + py_key="months", + test_range=range(1, 13), + years_padding=(0, 1), + ) @TestScenario @@ -1294,7 +1461,13 @@ def add_months(self): ) def subtract_months(self): """Check the subtractMonths function with DateTime64 extended range.""" - add_subtract_functions(clh_func="subtractMonths", py_key="months", test_range=range(1, 13), years_padding=(1, 0), mult=-1) + add_subtract_functions( + clh_func="subtractMonths", + py_key="months", + test_range=range(1, 13), + years_padding=(1, 0), + mult=-1, + ) @TestScenario @@ -1303,7 +1476,13 @@ def subtract_months(self): ) def add_weeks(self): """Check the addWeeks function with DateTime64 extended range.""" - add_subtract_functions(clh_func="addWeeks", py_key="days", test_range=range(6), years_padding=(0, 1), modifier=7) + add_subtract_functions( + clh_func="addWeeks", + py_key="days", + test_range=range(6), + years_padding=(0, 1), + modifier=7, + ) @TestScenario @@ -1312,14 +1491,18 @@ def add_weeks(self): ) def subtract_weeks(self): """Check the subtractWeeks function with DateTime64 extended range.""" - add_subtract_functions(clh_func="subtractWeeks", py_key="days", test_range=range(6), years_padding=(1, 0), modifier=7, mult=-1) - + add_subtract_functions( + clh_func="subtractWeeks", + py_key="days", + test_range=range(6), + years_padding=(1, 0), + modifier=7, + mult=-1, + ) @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addDays("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_addDays("1.0")) def add_days(self): """Check the addDays function work with DateTime64 extended range""" add_subtract_functions(clh_func="addDays", py_key="days", test_range=range(50)) @@ -1331,7 +1514,9 @@ def add_days(self): ) def subtract_days(self): """Check the subtractDays function work with DateTime64 extended range""" - add_subtract_functions(clh_func="subtractDays", py_key="days", test_range=range(50), mult=-1) + add_subtract_functions( + clh_func="subtractDays", py_key="days", test_range=range(50), mult=-1 + ) @TestScenario @@ -1349,7 +1534,9 @@ def add_hours(self): ) def subtract_hours(self): """Check the subtractHours function work with DateTime64 extended range""" - add_subtract_functions(clh_func="subtractHours", py_key="hours", test_range=range(25), mult=-1) + add_subtract_functions( + clh_func="subtractHours", py_key="hours", test_range=range(25), mult=-1 + ) @TestScenario @@ -1358,7 +1545,9 @@ def subtract_hours(self): ) def add_minutes(self): """Check the addMinutes function work with DateTime64 extended range""" - add_subtract_functions(clh_func="addMinutes", py_key="minutes", test_range=range(60)) + add_subtract_functions( + clh_func="addMinutes", py_key="minutes", test_range=range(60) + ) @TestScenario @@ -1367,7 +1556,9 @@ def add_minutes(self): ) def subtract_minutes(self): """Check the subtractMinutes function work with DateTime64 extended range""" - add_subtract_functions(clh_func="subtractMinutes", py_key="minutes", test_range=range(60), mult=-1) + add_subtract_functions( + clh_func="subtractMinutes", py_key="minutes", test_range=range(60), mult=-1 + ) @TestScenario @@ -1376,7 +1567,9 @@ def subtract_minutes(self): ) def add_seconds(self): """Check the addSeconds function work with DateTime64 extended range""" - add_subtract_functions(clh_func="addSeconds", py_key="seconds", test_range=range(60)) + add_subtract_functions( + clh_func="addSeconds", py_key="seconds", test_range=range(60) + ) @TestScenario @@ -1385,12 +1578,13 @@ def add_seconds(self): ) def subtract_seconds(self): """Check the subtractSeconds function work with DateTime64 extended range""" - add_subtract_functions(clh_func="subtractSeconds", py_key="seconds", test_range=range(60), mult=-1) + add_subtract_functions( + clh_func="subtractSeconds", py_key="seconds", test_range=range(60), mult=-1 + ) def date_diff_helper(dt1, dt2: datetime.datetime, unit: str): - """Helper for computing dateDiff expected result using Python. - """ + """Helper for computing dateDiff expected result using Python.""" delta = dt2 - dt1 if unit == "second": return delta.total_seconds() @@ -1415,10 +1609,18 @@ def date_diff_helper(dt1, dt2: datetime.datetime, unit: str): RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions_dateDiff("1.0") ) def date_diff(self): - """Check how dateDiff works with DateTime64 extended range. - """ + """Check how dateDiff works with DateTime64 extended range.""" stress = self.context.stress - compare_units = ("second", "minute", "hour", "day", "week", "month", "quarter", "year") + compare_units = ( + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year", + ) timezones = timezones_range(stress=stress) with Background("I select a set of datetimes to be compared"): @@ -1448,13 +1650,37 @@ def date_diff(self): ) def format_date_time(self): """Test formatDateTime() when DateTime64 is out of normal range. - This function formats DateTime according to a given Format string. - """ + This function formats DateTime according to a given Format string. + """ stress = self.context.stress timezones = timezones_range(stress) - modes = ('C', 'd', 'D', 'e', 'F', 'G', 'g', 'H', 'I', 'j', 'm', 'M', 'n', - 'p', 'R', 'S', 't', 'T', 'u', 'V', 'w', 'y', 'Y', '%') + modes = ( + "C", + "d", + "D", + "e", + "F", + "G", + "g", + "H", + "I", + "j", + "m", + "M", + "n", + "p", + "R", + "S", + "t", + "T", + "u", + "V", + "w", + "y", + "Y", + "%", + ) for year in years_range(stress=stress): with Given(f"I choose datetimes in {year}"): @@ -1476,11 +1702,17 @@ def format_date_time(self): def time_slots_get_expected(dt: datetime.datetime, duration, size=1800): - """Helper to compute expected array for timeSlots(). - """ + """Helper to compute expected array for timeSlots().""" zero_time = datetime.datetime(1970, 1, 1, 0, 0, 0) - result = [(zero_time + datetime.timedelta(seconds=((dt - zero_time).total_seconds() // size * size))).strftime("%Y-%m-%d %H:%M:%S")] + result = [ + ( + zero_time + + datetime.timedelta( + seconds=((dt - zero_time).total_seconds() // size * size) + ) + ).strftime("%Y-%m-%d %H:%M:%S") + ] s = 1 while s <= duration: @@ -1516,24 +1748,26 @@ def time_slots(self): for size in range(1, 50, 3): with Step(f"{dt}, dur={duration}, size={size}"): with By("getting an expected array using python"): - expected = time_slots_get_expected(dt=dt, duration=duration, size=size) + expected = time_slots_get_expected( + dt=dt, duration=duration, size=size + ) with And("forming a ClickHouse query"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") query = f"SELECT timeSlots(toDateTime64('{dt_str}', 0, 'UTC'), toUInt32({duration}), {size})" with Then("I execute query"): try: - assert eval(self.context.node.query(query).output) == expected, error() + assert ( + eval(self.context.node.query(query).output) + == expected + ), error() except SyntaxError: assert False @TestFeature -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_DatesAndTimesFunctions("1.0")) def date_time_funcs(self, node="clickhouse1"): - """Check the basic operations with DateTime64 - """ + """Check the basic operations with DateTime64""" self.context.node = self.context.cluster.node(node) with Pool(4) as pool: diff --git a/tests/testflows/datetime64_extended_range/tests/generic.py b/tests/testflows/datetime64_extended_range/tests/generic.py index 6eb117553e0..9ac2975e5a2 100644 --- a/tests/testflows/datetime64_extended_range/tests/generic.py +++ b/tests/testflows/datetime64_extended_range/tests/generic.py @@ -7,51 +7,59 @@ from datetime64_extended_range.tests.common import * import pytz import itertools + @TestScenario @Requirements( RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_Start("1.0"), ) def normal_range_start(self): - """Check DateTime64 can accept a dates around the start of the normal range that begins at 1970-01-01 00:00:00.000. - """ - with When("I do incrementing time sweep", description="check different time points in the first 24 hours at given date"): - walk_datetime_in_incrementing_steps(date="1970-01-01", precision=3, hrs_range=(0, 24)) + """Check DateTime64 can accept a dates around the start of the normal range that begins at 1970-01-01 00:00:00.000.""" + with When( + "I do incrementing time sweep", + description="check different time points in the first 24 hours at given date", + ): + walk_datetime_in_incrementing_steps( + date="1970-01-01", precision=3, hrs_range=(0, 24) + ) @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_End("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_NormalRange_End("1.0")) def normal_range_end(self): - """Check DateTime64 can accept a dates around the end of the normal range that ends at 2105-12-31 23:59:59.99999. - """ - with When("I do decrementing time sweep", - description="check different time points in the last 24 hours at given date"): - walk_datetime_in_decrementing_steps(date="2105-12-31", precision=3, hrs_range=(23, 0)) + """Check DateTime64 can accept a dates around the end of the normal range that ends at 2105-12-31 23:59:59.99999.""" + with When( + "I do decrementing time sweep", + description="check different time points in the last 24 hours at given date", + ): + walk_datetime_in_decrementing_steps( + date="2105-12-31", precision=3, hrs_range=(23, 0) + ) @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_Start("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_Start("1.0")) def extended_range_start(self): - """Check DateTime64 supports dates around the beginning of the extended range that begins at 1698-01-01 00:00:00.000000. - """ - with When("I do incrementing time sweep", - description="check different time points in the first 24 hours at given date"): - walk_datetime_in_incrementing_steps(date="1925-01-01", precision=5, hrs_range=(0, 24)) + """Check DateTime64 supports dates around the beginning of the extended range that begins at 1698-01-01 00:00:00.000000.""" + with When( + "I do incrementing time sweep", + description="check different time points in the first 24 hours at given date", + ): + walk_datetime_in_incrementing_steps( + date="1925-01-01", precision=5, hrs_range=(0, 24) + ) @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_End("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_End("1.0")) def extended_range_end(self, precision=3): - """Check DateTime64 supports dates around the beginning of the extended range that ends at 2377-12-31T23:59:59.999999. - """ - with When("I do decrementing time sweep", - description="check different time points in the last 24 hours at given date"): - walk_datetime_in_decrementing_steps(date="2238-12-31", precision=5, hrs_range=(23, 0)) + """Check DateTime64 supports dates around the beginning of the extended range that ends at 2377-12-31T23:59:59.999999.""" + with When( + "I do decrementing time sweep", + description="check different time points in the last 24 hours at given date", + ): + walk_datetime_in_decrementing_steps( + date="2238-12-31", precision=5, hrs_range=(23, 0) + ) @TestScenario @@ -62,9 +70,13 @@ def timezone_local_below_normal_range(self): """Check how UTC normal range time value treated when current timezone time value is out of normal range. """ - with When("I do incrementing time sweep", - description="check different time points when UTC datetime fits normal range but below it for local datetime"): - walk_datetime_in_incrementing_steps(date="1969-12-31", hrs_range=(17, 24), timezone='America/Phoenix') + with When( + "I do incrementing time sweep", + description="check different time points when UTC datetime fits normal range but below it for local datetime", + ): + walk_datetime_in_incrementing_steps( + date="1969-12-31", hrs_range=(17, 24), timezone="America/Phoenix" + ) @TestScenario @@ -75,18 +87,19 @@ def timezone_local_above_normal_range(self): """Check how UTC normal range time value treated when current timezone time value is out of normal range. """ - with When("I do decrementing time sweep", - description="check different time points when UTC datetime fits normal range but above it for local datetime"): - walk_datetime_in_decrementing_steps(date="2106-01-01", hrs_range=(6, 0), timezone='Asia/Novosibirsk') + with When( + "I do decrementing time sweep", + description="check different time points when UTC datetime fits normal range but above it for local datetime", + ): + walk_datetime_in_decrementing_steps( + date="2106-01-01", hrs_range=(6, 0), timezone="Asia/Novosibirsk" + ) @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_Comparison("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_Comparison("1.0")) def comparison_check(self): - """Check how comparison works with DateTime64 extended range. - """ + """Check how comparison works with DateTime64 extended range.""" stress = self.context.stress comparators = (">", "<", "==", "<=", ">=", "!=") timezones = timezones_range(stress=stress) @@ -112,12 +125,9 @@ def comparison_check(self): @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TimeZones("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TimeZones("1.0")) def timezones_support(self): - """Check how timezones work with DateTime64 extended range. - """ + """Check how timezones work with DateTime64 extended range.""" stress = self.context.stress timezones = timezones_range(stress=stress) @@ -138,8 +148,7 @@ def timezones_support(self): @TestFeature def generic(self, node="clickhouse1"): - """Check the basic operations with DateTime64 - """ + """Check the basic operations with DateTime64""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario, Suite): diff --git a/tests/testflows/datetime64_extended_range/tests/non_existent_time.py b/tests/testflows/datetime64_extended_range/tests/non_existent_time.py index 1036302b61b..0e3e180fe23 100644 --- a/tests/testflows/datetime64_extended_range/tests/non_existent_time.py +++ b/tests/testflows/datetime64_extended_range/tests/non_existent_time.py @@ -6,9 +6,7 @@ from datetime64_extended_range.tests.common import * @TestScenario -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_InvalidDate("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_InvalidDate("1.0")) def invalid_date(self): """Check how non-existent date is treated. For example, check 31st day in month that only has 30 days. @@ -29,16 +27,16 @@ def invalid_date(self): @TestOutline(Suite) -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_InvalidTime("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_InvalidTime("1.0")) @Examples( - "datetime expected timezone", [ - ('2002-04-07 02:30:00', '2002-04-07 01:30:00', 'America/New_York'), - ('2020-03-29 02:30:00', '2020-03-29 01:30:00', 'Europe/Zurich'), - ('2017-03-26 02:30:00', '2017-03-26 01:30:00', 'Europe/Berlin') - ]) -def invalid_time(self, datetime, expected, timezone='UTC'): + "datetime expected timezone", + [ + ("2002-04-07 02:30:00", "2002-04-07 01:30:00", "America/New_York"), + ("2020-03-29 02:30:00", "2020-03-29 01:30:00", "Europe/Zurich"), + ("2017-03-26 02:30:00", "2017-03-26 01:30:00", "Europe/Berlin"), + ], +) +def invalid_time(self, datetime, expected, timezone="UTC"): """proper handling of invalid time for a timezone when using DateTime64 extended range data type, for example, 2:30am on 7th April 2002 never happened at all in the US/Eastern timezone, @@ -50,13 +48,26 @@ def invalid_time(self, datetime, expected, timezone='UTC'): @TestOutline(Scenario) @Requirements( RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_DaylightSavingTime("1.0"), - RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_DaylightSavingTime_Disappeared("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_DaylightSavingTime_Disappeared( + "1.0" + ), ) @Examples( - "tz time_dates", [ - ('America/Denver', {'02:30:00': ('2018-03-11', '2020-03-08', '1980-04-27', '1942-02-09')}), - ('Europe/Zurich', {'02:30:00': ('2016-03-27', '2020-03-29', '1981-03-29'), '01:30:00': ('1942-05-04', )}) -]) + "tz time_dates", + [ + ( + "America/Denver", + {"02:30:00": ("2018-03-11", "2020-03-08", "1980-04-27", "1942-02-09")}, + ), + ( + "Europe/Zurich", + { + "02:30:00": ("2016-03-27", "2020-03-29", "1981-03-29"), + "01:30:00": ("1942-05-04",), + }, + ), + ], +) def dst_disappeared(self, tz, time_dates): """Proper handling of switching DST, when an hour is being skipped. Testing in 2 steps: first, try to make a DateTime64 with skipped time value. @@ -72,7 +83,9 @@ def dst_disappeared(self, tz, time_dates): dt -= datetime.timedelta(hours=1) expected = dt.strftime("%Y-%m-%d %H:%M:%S") with Then(f"I check skipped hour"): - select_check_datetime(datetime=dt_str, expected=expected, timezone=tz) + select_check_datetime( + datetime=dt_str, expected=expected, timezone=tz + ) with Step("Addition test"): with When("computing expected result"): dt += datetime.timedelta(hours=2) @@ -83,14 +96,37 @@ def dst_disappeared(self, tz, time_dates): @TestOutline(Scenario) -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_LeapSeconds("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_LeapSeconds("1.0")) @Examples( - "datet years", [ - ("06-30 23:59:55", [1972, 1981, 1982, 1983, 1985, 1992, 1993, 1994, 1997, 2012, 2015]), - ("12-31 23:59:55", [1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1987, 1989, 1990, 1995, 1998, 2005, 2008, 2016]) -]) + "datet years", + [ + ( + "06-30 23:59:55", + [1972, 1981, 1982, 1983, 1985, 1992, 1993, 1994, 1997, 2012, 2015], + ), + ( + "12-31 23:59:55", + [ + 1972, + 1973, + 1974, + 1975, + 1976, + 1977, + 1978, + 1979, + 1987, + 1989, + 1990, + 1995, + 1998, + 2005, + 2008, + 2016, + ], + ), + ], +) def leap_seconds(self, datet, years): """Test proper handling of leap seconds. Read more: https://de.wikipedia.org/wiki/Schaltsekunde Being checked by selecting a timestamp prior to leap second and adding seconds so that the result is after it. @@ -99,7 +135,7 @@ def leap_seconds(self, datet, years): with When(f"{datet}, {year}"): with By("forming an expected result using python"): dt_str = f"{year}-{datet}" - dt = datetime.datetime.strptime(dt_str, '%Y-%m-%d %H:%M:%S') + dt = datetime.datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S") dt += datetime.timedelta(seconds=9) expected = dt.strftime("%Y-%m-%d %H:%M:%S") with And(f"forming a query"): @@ -111,7 +147,7 @@ def leap_seconds(self, datet, years): @TestScenario @Requirements( RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_DaylightSavingTime("1.0"), - RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_TimeZoneSwitch("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime_TimeZoneSwitch("1.0"), ) def dst_time_zone_switch(self): """Check how ClickHouse supports handling of invalid time when using DateTime64 extended range data type @@ -122,15 +158,19 @@ def dst_time_zone_switch(self): utc = pytz.timezone("UTC") for timezone in timezones: - if timezone == 'UTC': + if timezone == "UTC": continue with Step(f"{timezone}"): tz = pytz.timezone(timezone) transition_times = tz._utc_transition_times transition_info = tz._transition_info - for i in range(len(transition_times)-1, 0, -1): - if (transition_times[i] > datetime.datetime.now()) or (transition_times[i].year < 1925) or (transition_times[i].year > 2238): + for i in range(len(transition_times) - 1, 0, -1): + if ( + (transition_times[i] > datetime.datetime.now()) + or (transition_times[i].year < 1925) + or (transition_times[i].year > 2238) + ): continue with Step(f"{transition_times[i]}"): with By("localize python datetime"): @@ -138,7 +178,9 @@ def dst_time_zone_switch(self): dt0 = dt - datetime.timedelta(hours=4) dt0 = utc.localize(dt0).astimezone(tz).replace(tzinfo=None) with And("compute expected result using Pytz"): - seconds_shift = transition_info[i][0] - transition_info[i-1][0] + seconds_shift = ( + transition_info[i][0] - transition_info[i - 1][0] + ) dt1 = dt0 + datetime.timedelta(hours=8) + seconds_shift dt0_str = dt0.strftime("%Y-%m-%d %H:%M:%S") dt1_str = dt1.strftime("%Y-%m-%d %H:%M:%S") @@ -150,12 +192,9 @@ def dst_time_zone_switch(self): @TestFeature @Name("non existent time") -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_NonExistentTime("1.0")) def feature(self, node="clickhouse1"): - """Check how ClickHouse treats non-existent time in DateTime64 data type. - """ + """Check how ClickHouse treats non-existent time in DateTime64 data type.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario, Suite): diff --git a/tests/testflows/datetime64_extended_range/tests/reference_times.py b/tests/testflows/datetime64_extended_range/tests/reference_times.py index cdec3eb260c..9cd9fadc35c 100644 --- a/tests/testflows/datetime64_extended_range/tests/reference_times.py +++ b/tests/testflows/datetime64_extended_range/tests/reference_times.py @@ -8,19 +8,49 @@ from datetime64_extended_range.tests.common import * @TestSuite -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_SpecificTimestamps("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_SpecificTimestamps("1.0")) def reference_times(self, node="clickhouse1"): """Check how ClickHouse converts a set of particular timestamps to DateTime64 for all timezones and compare the result to pytz. """ self.context.node = self.context.cluster.node(node) - timestamps = [9961200, 73476000, 325666800, 354675600, 370400400, 386125200, 388566010, 401850000, 417574811, - 496803600, 528253200, 624423614, 636516015, 671011200, 717555600, 752047218, 859683600, 922582800, - 1018173600, 1035705600, 1143334800, 1162105223, 1174784400, 1194156000, 1206838823, 1224982823, - 1236495624, 1319936400, 1319936424, 1425798025, 1459040400, 1509872400, 2090451627, 2140668000] + timestamps = [ + 9961200, + 73476000, + 325666800, + 354675600, + 370400400, + 386125200, + 388566010, + 401850000, + 417574811, + 496803600, + 528253200, + 624423614, + 636516015, + 671011200, + 717555600, + 752047218, + 859683600, + 922582800, + 1018173600, + 1035705600, + 1143334800, + 1162105223, + 1174784400, + 1194156000, + 1206838823, + 1224982823, + 1236495624, + 1319936400, + 1319936424, + 1425798025, + 1459040400, + 1509872400, + 2090451627, + 2140668000, + ] query = "" diff --git a/tests/testflows/datetime64_extended_range/tests/type_conversion.py b/tests/testflows/datetime64_extended_range/tests/type_conversion.py index 85582b82d7b..c52ecdce582 100644 --- a/tests/testflows/datetime64_extended_range/tests/type_conversion.py +++ b/tests/testflows/datetime64_extended_range/tests/type_conversion.py @@ -14,13 +14,29 @@ from datetime64_extended_range.tests.common import * @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toInt_8_16_32_64_128_256_("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toInt_8_16_32_64_128_256_( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_int_8_16_32_64_128_256(self, cast): - """Check the toInt(8|16|32|64|128|256) functions with DateTime64 extended range - """ + """Check the toInt(8|16|32|64|128|256) functions with DateTime64 extended range""" stress = self.context.stress timezones = timezones_range(stress) @@ -45,7 +61,9 @@ def to_int_8_16_32_64_128_256(self, cast): np_res = py_res if np_res == py_res: with Given(f"{py_res} fits int{int_type}"): - with When(f"making a query string for ClickHouse if py_res fits int{int_type}"): + with When( + f"making a query string for ClickHouse if py_res fits int{int_type}" + ): if cast: query = f"SELECT cast(toDateTime64('{dt_str}', 0, '{tz}'), 'Int{int_type}')" else: @@ -55,13 +73,29 @@ def to_int_8_16_32_64_128_256(self, cast): @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUInt_8_16_32_64_256_("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUInt_8_16_32_64_256_( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_uint_8_16_32_64_256(self, cast): - """Check the toUInt(8|16|32|64|256) functions with DateTime64 extended range - """ + """Check the toUInt(8|16|32|64|256) functions with DateTime64 extended range""" stress = self.context.stress timezones = timezones_range(stress) @@ -86,7 +120,9 @@ def to_uint_8_16_32_64_256(self, cast): np_res = py_res if np_res == py_res: with Given(f"{py_res} fits int{int_type}"): - with When(f"making a query string for ClickHouse if py_res fits int{int_type}"): + with When( + f"making a query string for ClickHouse if py_res fits int{int_type}" + ): if cast: query = f"SELECT cast(toDateTime64('{dt_str}', 0, '{tz}'), 'UInt{int_type}')" else: @@ -96,13 +132,29 @@ def to_uint_8_16_32_64_256(self, cast): @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toFloat_32_64_("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toFloat_32_64_( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_float_32_64(self, cast): - """Check the toFloat(32|64) functions with DateTime64 extended range - """ + """Check the toFloat(32|64) functions with DateTime64 extended range""" stress = self.context.stress timezones = timezones_range(stress) @@ -133,11 +185,12 @@ def to_float_32_64(self, cast): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime64_FromString_MissingTime("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime64_FromString_MissingTime( + "1.0" + ) ) def to_datetime64_from_string_missing_time(self): - """Check the toDateTime64() with DateTime64 extended range conversion when string is missing the time part. - """ + """Check the toDateTime64() with DateTime64 extended range conversion when string is missing the time part.""" stress = self.context.stress timezones = timezones_range(stress) @@ -163,8 +216,7 @@ def to_datetime64_from_string_missing_time(self): RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime64("1.0") ) def to_datetime64(self): - """Check the toDateTime64() conversion with DateTime64. This is supposed to work in normal range ONLY. - """ + """Check the toDateTime64() conversion with DateTime64. This is supposed to work in normal range ONLY.""" stress = self.context.stress timezones = timezones_range(stress) @@ -185,13 +237,29 @@ def to_datetime64(self): @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDate("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDate( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_date(self, cast): - """Check the toDate() conversion with DateTime64. This is supposed to work in normal range ONLY. - """ + """Check the toDate() conversion with DateTime64. This is supposed to work in normal range ONLY.""" stress = self.context.stress timezones = timezones_range(stress) @@ -202,7 +270,7 @@ def to_date(self, cast): for dt in datetimes: for tz in timezones: with Step(f"{dt} {tz}"): - expected = None # by default - not checked, checking the exitcode + expected = None # by default - not checked, checking the exitcode with By("converting datetime to string"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S") @@ -214,20 +282,38 @@ def to_date(self, cast): if cast: query = f"SELECT CAST(toDateTime64('{dt_str}', 0, '{tz}'), 'Date')" else: - query = f"SELECT toDate(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT toDate(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then(f"I execute toDate() query and check return/exitcode"): exec_query(request=query, expected=expected, exitcode=0) @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDateTime( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_datetime(self, cast): - """Check the toDateTime() conversion with DateTime64. This is supposed to work in normal range ONLY. - """ + """Check the toDateTime() conversion with DateTime64. This is supposed to work in normal range ONLY.""" stress = self.context.stress timezones = timezones_range(stress) @@ -247,7 +333,9 @@ def to_datetime(self, cast): dt_transformed = dt_local.astimezone(tzlocal()) expected = f"{dt_transformed.strftime('%Y-%m-%d %H:%M:%S')}" else: - query = f"SELECT toDateTime(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT toDateTime(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with When("figure out expected result in python"): expected = f"{dt.strftime('%Y-%m-%d %H:%M:%S')}" @@ -260,13 +348,29 @@ def to_datetime(self, cast): @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toString("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toString( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_string(self, cast): - """Check the toString() with DateTime64 extended range. - """ + """Check the toString() with DateTime64 extended range.""" stress = self.context.stress timezones = timezones_range(stress) @@ -283,22 +387,45 @@ def to_string(self, cast): if cast: query = f"SELECT cast(toDateTime64('{dt_str}', 0, '{tz}'), 'String')" else: - query = f"SELECT toString(toDateTime64('{dt_str}', 0, '{tz}'))" + query = ( + f"SELECT toString(toDateTime64('{dt_str}', 0, '{tz}'))" + ) with Then(f"I execute toDateTime64() query"): exec_query(request=query, expected=f"{dt_str}") def valid_decimal_range(bit_depth, S): """A helper to find valid range for Decimal(32|64|128|256) with given scale (S)""" - return {32: -1 * 10 ** (9 - S), 64: -1 * 10 ** (18 - S), 128: -1 * 10 ** (38 - S), 256: -1 * 10 ** (76 - S)}[ - bit_depth] + return { + 32: -1 * 10 ** (9 - S), + 64: -1 * 10 ** (18 - S), + 128: -1 * 10 ** (38 - S), + 256: -1 * 10 ** (76 - S), + }[bit_depth] @TestOutline(Scenario) -@Examples("cast", [ - (False, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDecimal_32_64_128_256_("1.0"))), - (True, Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_("1.0"))) -]) +@Examples( + "cast", + [ + ( + False, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toDecimal_32_64_128_256_( + "1.0" + ) + ), + ), + ( + True, + Requirements( + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_CAST_x_T_( + "1.0" + ) + ), + ), + ], +) def to_decimal_32_64_128_256(self, cast): """Check the toDecimal(32|64|128|256) functions with DateTime64 extended range. Decimal32(S) - ( -1 * 10^(9 - S), 1 * 10^(9 - S) ) @@ -320,7 +447,9 @@ def to_decimal_32_64_128_256(self, cast): for decimal_type in (32, 64, 128, 256): for scale in range(scales[decimal_type]): with When(f"{dt} {tz}, Decimal{decimal_type}({scale})"): - valid_range = valid_decimal_range(bit_depth=decimal_type, S=scale) + valid_range = valid_decimal_range( + bit_depth=decimal_type, S=scale + ) with By("computing the expected result using python"): expected = decimal.Decimal(time.mktime(dt.timetuple())) if -valid_range < expected < valid_range: @@ -342,11 +471,13 @@ def to_unix_timestamp64_milli_micro_nano(self, scale): """ stress = self.context.stress timezones = timezones_range(stress) - func = {3: 'Milli', 6: 'Micro', 9: 'Nano'} + func = {3: "Milli", 6: "Micro", 9: "Nano"} for year in years_range(stress): with Given(f"I select datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) for d in datetimes: for tz in timezones: @@ -355,7 +486,7 @@ def to_unix_timestamp64_milli_micro_nano(self, scale): with By("converting datetime to string"): dt_str = dt.strftime("%Y-%m-%d %H:%M:%S.%f") with And("converting DateTime to UTC"): - dt = dt.astimezone(pytz.timezone('UTC')) + dt = dt.astimezone(pytz.timezone("UTC")) with And("computing the expected result using python"): expected = int(dt.timestamp() * (10**scale)) if expected >= 0: @@ -370,31 +501,34 @@ def to_unix_timestamp64_milli_micro_nano(self, scale): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Milli("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Milli( + "1.0" + ) ) def to_unix_timestamp64_milli(self): - """Check the toUnixTimestamp64Milli functions with DateTime64 extended range. - """ + """Check the toUnixTimestamp64Milli functions with DateTime64 extended range.""" to_unix_timestamp64_milli_micro_nano(scale=3) @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Micro("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Micro( + "1.0" + ) ) def to_unix_timestamp64_micro(self): - """Check the toUnixTimestamp64Micro functions with DateTime64 extended range. - """ + """Check the toUnixTimestamp64Micro functions with DateTime64 extended range.""" to_unix_timestamp64_milli_micro_nano(scale=6) @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Nano("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_toUnixTimestamp64Nano( + "1.0" + ) ) def to_unix_timestamp64_nano(self): - """Check the toUnixTimestamp64Nano functions with DateTime64 extended range. - """ + """Check the toUnixTimestamp64Nano functions with DateTime64 extended range.""" to_unix_timestamp64_milli_micro_nano(scale=9) @@ -405,11 +539,13 @@ def from_unix_timestamp64_milli_micro_nano(self, scale): """ stress = self.context.stress timezones = timezones_range(stress) - func = {3: 'Milli', 6: 'Micro', 9: 'Nano'} + func = {3: "Milli", 6: "Micro", 9: "Nano"} for year in years_range(stress): with Given(f"I select datetimes in {year}"): - datetimes = select_dates_in_year(year=year, stress=stress, microseconds=True) + datetimes = select_dates_in_year( + year=year, stress=stress, microseconds=True + ) for d in datetimes: for tz in timezones: @@ -417,9 +553,9 @@ def from_unix_timestamp64_milli_micro_nano(self, scale): with When(f"{dt} {tz}"): with By("converting datetime to string"): d_str = d.strftime("%Y-%m-%d %H:%M:%S.%f") - d_str += "0" * (scale-3) + d_str += "0" * (scale - 3) with And("converting DateTime64 to UTC"): - dt = dt.astimezone(pytz.timezone('UTC')) + dt = dt.astimezone(pytz.timezone("UTC")) with And("computing the expected result using python"): ts = int(dt.timestamp() * (10**scale)) if ts >= 0: @@ -434,38 +570,39 @@ def from_unix_timestamp64_milli_micro_nano(self, scale): @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Milli("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Milli( + "1.0" + ) ) def from_unix_timestamp64_milli(self): - """Check the fromUnixTimestamp64Milli functions with DateTime64 extended range. - """ + """Check the fromUnixTimestamp64Milli functions with DateTime64 extended range.""" from_unix_timestamp64_milli_micro_nano(scale=3) @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Micro("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Micro( + "1.0" + ) ) def from_unix_timestamp64_micro(self): - """Check the fromUnixTimestamp64Micro functions with DateTime64 extended range. - """ + """Check the fromUnixTimestamp64Micro functions with DateTime64 extended range.""" from_unix_timestamp64_milli_micro_nano(scale=6) @TestScenario @Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Nano("1.0") + RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions_fromUnixTimestamp64Nano( + "1.0" + ) ) def from_unix_timestamp64_nano(self): - """Check the fromUnixTimestamp64Nano functions with DateTime64 extended range. - """ + """Check the fromUnixTimestamp64Nano functions with DateTime64 extended range.""" from_unix_timestamp64_milli_micro_nano(scale=9) @TestFeature -@Requirements( - RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions("1.0") -) +@Requirements(RQ_SRS_010_DateTime64_ExtendedRange_TypeConversionFunctions("1.0")) def type_conversion(self, node="clickhouse1"): """Check the type conversion operations with DateTime64. Cast can be set as Requirement thereby as the module diff --git a/tests/testflows/example/configs/clickhouse/common.xml b/tests/testflows/example/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/example/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/example/configs/clickhouse/config.xml b/tests/testflows/example/configs/clickhouse/config.xml deleted file mode 100644 index 9854f9f990e..00000000000 --- a/tests/testflows/example/configs/clickhouse/config.xml +++ /dev/null @@ -1,436 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 7200 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/example/configs/clickhouse/users.xml b/tests/testflows/example/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/example/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/example/example_env_arm64/clickhouse-service.yml b/tests/testflows/example/example_env_arm64/clickhouse-service.yml new file mode 100644 index 00000000000..a73d31421c8 --- /dev/null +++ b/tests/testflows/example/example_env_arm64/clickhouse-service.yml @@ -0,0 +1,29 @@ +version: '2.3' + +services: + clickhouse: + image: registry.gitlab.com/altinity-public/container-images/test/clickhouse-integration-test:21.12 + privileged: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 10 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/example/example_env_arm64/docker-compose.yml b/tests/testflows/example/example_env_arm64/docker-compose.yml new file mode 100644 index 00000000000..4edb415824f --- /dev/null +++ b/tests/testflows/example/example_env_arm64/docker-compose.yml @@ -0,0 +1,31 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + zookeeper: + condition: service_healthy diff --git a/tests/testflows/example/example_env_arm64/zookeeper-service.yml b/tests/testflows/example/example_env_arm64/zookeeper-service.yml new file mode 100644 index 00000000000..ca732a48dbd --- /dev/null +++ b/tests/testflows/example/example_env_arm64/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.6.2 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/example/regression.py b/tests/testflows/example/regression.py index 7a0c94a7cd4..8c6cb4f29b9 100755 --- a/tests/testflows/example/regression.py +++ b/tests/testflows/example/regression.py @@ -7,25 +7,39 @@ append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser +from platform import processor as current_cpu + @TestFeature @Name("example") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path, stress=None): - """Simple example of how you can use TestFlows to test ClickHouse. - """ +def regression(self, local, clickhouse_binary_path, clickhouse_version, stress=None): + """Simple example of how you can use TestFlows to test ClickHouse.""" nodes = { "clickhouse": ("clickhouse1",), } + self.context.clickhouse_version = clickhouse_version + if stress is not None: self.context.stress = stress - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "example_env")) as cluster: + folder_name = os.path.basename(current_dir()) + if current_cpu() == "aarch64": + env = f"{folder_name}_env_arm64" + else: + env = f"{folder_name}_env" + + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), env), + ) as cluster: self.context.cluster = cluster Scenario(run=load("example.tests.example", "scenario")) + if main(): regression() diff --git a/tests/testflows/example/requirements/requirements.py b/tests/testflows/example/requirements/requirements.py index 5b4765eb90e..92b6d912335 100644 --- a/tests/testflows/example/requirements/requirements.py +++ b/tests/testflows/example/requirements/requirements.py @@ -9,76 +9,79 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_001_Example = Requirement( - name='RQ.SRS-001.Example', - version='1.0', + name="RQ.SRS-001.Example", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - 'This is a long description of the requirement that can include any\n' - 'relevant information. \n' - '\n' - 'The one-line block that follows the requirement defines the `version` \n' - 'of the requirement. The version is controlled manually and is used\n' - 'to indicate material changes to the requirement that would \n' - 'require tests that cover this requirement to be updated.\n' - '\n' - 'It is a good practice to use requirement names that are broken\n' - 'up into groups. It is not recommended to use only numbers\n' - 'because if the requirement must be moved the numbering will not match.\n' - 'Therefore, the requirement name should start with the group\n' - 'name which is then followed by a number if any. For example,\n' - '\n' - ' RQ.SRS-001.Group.Subgroup.1\n' - '\n' + "This is a long description of the requirement that can include any\n" + "relevant information. \n" + "\n" + "The one-line block that follows the requirement defines the `version` \n" + "of the requirement. The version is controlled manually and is used\n" + "to indicate material changes to the requirement that would \n" + "require tests that cover this requirement to be updated.\n" + "\n" + "It is a good practice to use requirement names that are broken\n" + "up into groups. It is not recommended to use only numbers\n" + "because if the requirement must be moved the numbering will not match.\n" + "Therefore, the requirement name should start with the group\n" + "name which is then followed by a number if any. For example,\n" + "\n" + " RQ.SRS-001.Group.Subgroup.1\n" + "\n" "To keep names short, try to use abbreviations for the requirement's group name.\n" - '\n' - ), + "\n" + ), link=None, level=2, - num='4.1') + num="4.1", +) RQ_SRS_001_Example_Subgroup = Requirement( - name='RQ.SRS-001.Example.Subgroup', - version='1.0', + name="RQ.SRS-001.Example.Subgroup", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - 'This an example of a sub-requirement of the [RQ.SRS-001.Example](#rqsrs-001example).\n' - '\n' - ), + "This an example of a sub-requirement of the [RQ.SRS-001.Example](#rqsrs-001example).\n" + "\n" + ), link=None, level=2, - num='4.2') + num="4.2", +) RQ_SRS_001_Example_Select_1 = Requirement( - name='RQ.SRS-001.Example.Select.1', - version='1.0', + name="RQ.SRS-001.Example.Select.1", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return `1` when user executes query\n' - '\n' - '```sql\n' - 'SELECT 1\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return `1` when user executes query\n" + "\n" + "```sql\n" + "SELECT 1\n" + "```\n" + "\n" + ), link=None, level=2, - num='4.3') + num="4.3", +) SRS_001_ClickHouse_Software_Requirements_Specification_Template = Specification( - name='SRS-001 ClickHouse Software Requirements Specification Template', + name="SRS-001 ClickHouse Software Requirements Specification Template", description=None, - author='[name of the author]', - date='[date]', - status=None, + author="[name of the author]", + date="[date]", + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -90,26 +93,26 @@ SRS_001_ClickHouse_Software_Requirements_Specification_Template = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Table of Contents', level=2, num='2.1'), - Heading(name='Generating HTML version', level=2, num='2.2'), - Heading(name='Generating Python Requirements', level=2, num='2.3'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='SRS', level=2, num='3.1'), - Heading(name='Some term that you will use', level=2, num='3.2'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='RQ.SRS-001.Example', level=2, num='4.1'), - Heading(name='RQ.SRS-001.Example.Subgroup', level=2, num='4.2'), - Heading(name='RQ.SRS-001.Example.Select.1', level=2, num='4.3'), - Heading(name='References', level=1, num='5'), - ), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Table of Contents", level=2, num="2.1"), + Heading(name="Generating HTML version", level=2, num="2.2"), + Heading(name="Generating Python Requirements", level=2, num="2.3"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="SRS", level=2, num="3.1"), + Heading(name="Some term that you will use", level=2, num="3.2"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="RQ.SRS-001.Example", level=2, num="4.1"), + Heading(name="RQ.SRS-001.Example.Subgroup", level=2, num="4.2"), + Heading(name="RQ.SRS-001.Example.Select.1", level=2, num="4.3"), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_001_Example, RQ_SRS_001_Example_Subgroup, RQ_SRS_001_Example_Select_1, - ), - content=''' + ), + content=""" # SRS-001 ClickHouse Software Requirements Specification Template **Author:** [name of the author] @@ -245,4 +248,5 @@ SELECT 1 [Some term that you will use]: #Sometermthatyouwilluse [ClickHouse]: https://clickhouse.com [Git]: https://git-scm.com/ -''') +""", +) diff --git a/tests/testflows/example/tests/example.py b/tests/testflows/example/tests/example.py index ea77f8b0235..9977e973ede 100644 --- a/tests/testflows/example/tests/example.py +++ b/tests/testflows/example/tests/example.py @@ -3,14 +3,12 @@ from testflows.asserts import error from example.requirements import * + @TestScenario @Name("select 1") -@Requirements( - RQ_SRS_001_Example_Select_1("1.0") -) +@Requirements(RQ_SRS_001_Example_Select_1("1.0")) def scenario(self, node="clickhouse1"): - """Check that ClickHouse returns 1 when user executes `SELECT 1` query. - """ + """Check that ClickHouse returns 1 when user executes `SELECT 1` query.""" node = self.context.cluster.node(node) with When("I execute query select 1"): diff --git a/tests/testflows/extended_precision_data_types/common.py b/tests/testflows/extended_precision_data_types/common.py index ebd0a6cac45..959ff96a536 100644 --- a/tests/testflows/extended_precision_data_types/common.py +++ b/tests/testflows/extended_precision_data_types/common.py @@ -10,49 +10,59 @@ from helpers.common import * rounding_precision = 7 + @contextmanager def allow_experimental_bigint(node): - """Enable experimental big int setting in Clickhouse. - """ + """Enable experimental big int setting in Clickhouse.""" setting = ("allow_experimental_bigint_types", 1) default_query_settings = None try: with Given("I add allow_experimental_bigint to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(setting) yield finally: - with Finally("I remove allow_experimental_bigint from the default query settings"): + with Finally( + "I remove allow_experimental_bigint from the default query settings" + ): if default_query_settings: try: default_query_settings.pop(default_query_settings.index(setting)) except ValueError: pass + @TestStep(Given) def allow_experimental_map_type(self): - """Set allow_experimental_map_type = 1 - """ + """Set allow_experimental_map_type = 1""" setting = ("allow_experimental_map_type", 1) default_query_settings = None try: with By("adding allow_experimental_map_type to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(setting) yield finally: - with Finally("I remove allow_experimental_map_type from the default query settings"): + with Finally( + "I remove allow_experimental_map_type from the default query settings" + ): if default_query_settings: try: default_query_settings.pop(default_query_settings.index(setting)) except ValueError: pass -def execute_query(sql, expected=None, format="TabSeparatedWithNames", compare_func=None): - """Execute SQL query and compare the output to the snapshot. - """ + +def execute_query( + sql, expected=None, format="TabSeparatedWithNames", compare_func=None +): + """Execute SQL query and compare the output to the snapshot.""" name = basename(current().name) with When("I execute query", description=sql): @@ -70,12 +80,16 @@ def execute_query(sql, expected=None, format="TabSeparatedWithNames", compare_fu else: with Then("I check output against snapshot"): with values() as that: - assert that(snapshot("\n" + r.output.strip() + "\n", "tests", name=name, encoder=str)), error() + assert that( + snapshot( + "\n" + r.output.strip() + "\n", "tests", name=name, encoder=str + ) + ), error() + @TestStep(Given) def table(self, data_type, name="table0"): - """Create a table. - """ + """Create a table.""" node = current().context.node try: @@ -87,34 +101,51 @@ def table(self, data_type, name="table0"): with Finally("drop the table"): node.query(f"DROP TABLE IF EXISTS {name}") + def getuid(): - """Create a unique variable name based on the test it is called from. - """ + """Create a unique variable name based on the test it is called from.""" if current().subtype == TestSubType.Example: - testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + testname = ( + f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + ) else: testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" - for char in ['(', ')', '[', ']','\'']: - testname = testname.replace(f'{char}', '') + for char in ["(", ")", "[", "]", "'"]: + testname = testname.replace(f"{char}", "") + + return testname + "_" + str(uuid.uuid1()).replace("-", "_") - return testname + "_" + str(uuid.uuid1()).replace('-', '_') def to_data_type(data_type, value): - """Return a conversion statement based on the data type provided - """ - if data_type in ['Decimal256(0)']: - return f'toDecimal256(\'{value}\',0)' + """Return a conversion statement based on the data type provided""" + if data_type in ["Decimal256(0)"]: + return f"toDecimal256('{value}',0)" else: - return f'to{data_type}(\'{value}\')' + return f"to{data_type}('{value}')" data_types = [ - ('Int128', '-170141183460469231731687303715884105728', '170141183460469231731687303715884105727'), - ('Int256', '-57896044618658097711785492504343953926634992332820282019728792003956564819968', '57896044618658097711785492504343953926634992332820282019728792003956564819967'), - ('UInt128','0','340282366920938463463374607431768211455'), - ('UInt256', '0', '115792089237316195423570985008687907853269984665640564039457584007913129639935'), + ( + "Int128", + "-170141183460469231731687303715884105728", + "170141183460469231731687303715884105727", + ), + ( + "Int256", + "-57896044618658097711785492504343953926634992332820282019728792003956564819968", + "57896044618658097711785492504343953926634992332820282019728792003956564819967", + ), + ("UInt128", "0", "340282366920938463463374607431768211455"), + ( + "UInt256", + "0", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + ), ] -Decimal256_min_max = -1000000000000000000000000000000000000000000000000000000000000000000000000000,1000000000000000000000000000000000000000000000000000000000000000000000000000 +Decimal256_min_max = ( + -1000000000000000000000000000000000000000000000000000000000000000000000000000, + 1000000000000000000000000000000000000000000000000000000000000000000000000000, +) diff --git a/tests/testflows/extended_precision_data_types/configs/clickhouse/common.xml b/tests/testflows/extended_precision_data_types/configs/clickhouse/common.xml deleted file mode 100644 index 0ba01589b90..00000000000 --- a/tests/testflows/extended_precision_data_types/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - :: - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/extended_precision_data_types/configs/clickhouse/config.xml b/tests/testflows/extended_precision_data_types/configs/clickhouse/config.xml deleted file mode 100644 index 842a0573d49..00000000000 --- a/tests/testflows/extended_precision_data_types/configs/clickhouse/config.xml +++ /dev/null @@ -1,448 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - 0.0.0.0 - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/extended_precision_data_types/configs/clickhouse/users.xml b/tests/testflows/extended_precision_data_types/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/extended_precision_data_types/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/extended_precision_data_types/errors.py b/tests/testflows/extended_precision_data_types/errors.py index 63b82f3368d..a38b3ce571d 100644 --- a/tests/testflows/extended_precision_data_types/errors.py +++ b/tests/testflows/extended_precision_data_types/errors.py @@ -1,11 +1,14 @@ def not_implemented_bigints(name): - return(48, f"Exception: {name} is not implemented for big integers") + return (48, f"Exception: {name} is not implemented for big integers") + def bigints_not_implements(name): - return(48, f'Exception: {name} for big integers is not implemented') + return (48, f"Exception: {name} for big integers is not implemented") + def illegal_type(): - return(43, 'Exception: Illegal type') + return (43, "Exception: Illegal type") + def illegal_column(): - return(44, 'Exception: Illegal column') \ No newline at end of file + return (44, "Exception: Illegal column") diff --git a/tests/testflows/extended_precision_data_types/regression.py b/tests/testflows/extended_precision_data_types/regression.py index 5572381d817..f185a5e4ecb 100755 --- a/tests/testflows/extended_precision_data_types/regression.py +++ b/tests/testflows/extended_precision_data_types/regression.py @@ -10,38 +10,42 @@ from helpers.cluster import Cluster from helpers.argparser import argparser from extended_precision_data_types.requirements import * -xfails = { -} +xfails = {} + +xflags = {} -xflags = { -} @TestModule @ArgumentParser(argparser) @XFails(xfails) @XFlags(xflags) @Name("extended precision data types") -@Specifications( - SRS020_ClickHouse_Extended_Precision_Data_Types -) +@Specifications(SRS020_ClickHouse_Extended_Precision_Data_Types) @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision("1.0"), ) -def regression(self, local, clickhouse_binary_path, stress=None): - """Extended precision data type regression. - """ - nodes = { - "clickhouse": - ("clickhouse1",) - } +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """Extended precision data type regression.""" + nodes = {"clickhouse": ("clickhouse1",)} + if stress is not None: + self.context.stress = stress + self.context.clickhouse_version = clickhouse_version - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "extended-precision-data-type_env")) as cluster: + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join( + current_dir(), "extended-precision-data-type_env" + ), + ) as cluster: self.context.cluster = cluster - self.context.stress = stress Feature(run=load("extended_precision_data_types.tests.feature", "feature")) + if main(): regression() diff --git a/tests/testflows/extended_precision_data_types/requirements/__init__.py b/tests/testflows/extended_precision_data_types/requirements/__init__.py index 75e9d5b4bb8..02f7d430154 100644 --- a/tests/testflows/extended_precision_data_types/requirements/__init__.py +++ b/tests/testflows/extended_precision_data_types/requirements/__init__.py @@ -1 +1 @@ -from .requirements import * \ No newline at end of file +from .requirements import * diff --git a/tests/testflows/extended_precision_data_types/requirements/requirements.py b/tests/testflows/extended_precision_data_types/requirements/requirements.py index fa828897f66..3b1aa89d0e2 100644 --- a/tests/testflows/extended_precision_data_types/requirements/requirements.py +++ b/tests/testflows/extended_precision_data_types/requirements/requirements.py @@ -9,757 +9,787 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_020_ClickHouse_Extended_Precision = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using [Extended Precision Data Types].\n' - '\n' - ), + "[ClickHouse] SHALL support using [Extended Precision Data Types].\n" "\n" + ), link=None, level=2, - num='4.1') + num="4.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt128 = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt128', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt128", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting values to `Int128` using the `toInt128` function.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT toInt128(1)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support converting values to `Int128` using the `toInt128` function.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT toInt128(1)\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.1') + num="4.2.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toUInt128 = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt128', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt128", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting values to `UInt128` format using `toUInt128` function.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT toUInt128(1)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support converting values to `UInt128` format using `toUInt128` function.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT toUInt128(1)\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.2') + num="4.2.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt256 = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt256', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt256", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting values to `Int256` using `toInt256` function.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT toInt256(1)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support converting values to `Int256` using `toInt256` function.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT toInt256(1)\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.3') + num="4.2.3", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toUInt256 = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt256', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt256", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting values to `UInt256` format using `toUInt256` function.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT toUInt256(1)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support converting values to `UInt256` format using `toUInt256` function.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT toUInt256(1)\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.4') + num="4.2.4", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toDecimal256 = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toDecimal256', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toDecimal256", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting values to `Decimal256` format using `toDecimal256` function.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT toDecimal256(1,2)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support converting values to `Decimal256` format using `toDecimal256` function.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT toDecimal256(1,2)\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.5') + num="4.2.5", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_FromMySQL = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.FromMySQL', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.FromMySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting to [Extended Precision Data Types] from MySQL.\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL support converting to [Extended Precision Data Types] from MySQL.\n" + "\n" + "\n" + ), link=None, level=3, - num='4.2.6') + num="4.2.6", +) RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_ToMySQL = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.ToMySQL', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.ToMySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support converting from [Extended Precision Data Types] to MySQL.\n' - '\n' - ), + "[ClickHouse] MAY not support converting from [Extended Precision Data Types] to MySQL.\n" + "\n" + ), link=None, level=3, - num='4.2.7') + num="4.2.7", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arithmetic_Int_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Int.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Int.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using [Arithmetic functions] with Int128, UInt128, Int256, and UInt256.\n' - '\n' - 'Arithmetic functions:\n' - '* plus\n' - '* minus\n' - '* multiply\n' - '* divide\n' - '* intDiv\n' - '* intDivOrZero\n' - '* modulo\n' - '* moduloOrZero\n' - '* negate\n' - '* abs\n' - '* gcd\n' - '* lcm\n' - '\n' - ), + "[ClickHouse] SHALL support using [Arithmetic functions] with Int128, UInt128, Int256, and UInt256.\n" + "\n" + "Arithmetic functions:\n" + "* plus\n" + "* minus\n" + "* multiply\n" + "* divide\n" + "* intDiv\n" + "* intDivOrZero\n" + "* modulo\n" + "* moduloOrZero\n" + "* negate\n" + "* abs\n" + "* gcd\n" + "* lcm\n" + "\n" + ), link=None, level=3, - num='4.3.1') + num="4.3.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arithmetic_Dec_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Arithmetic functions] with Decimal256:\n' - '\n' - '* plus\n' - '* minus\n' - '* multiply\n' - '* divide\n' - '* intDiv\n' - '* intDivOrZero\n' - '* negate\n' - '* abs\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Arithmetic functions] with Decimal256:\n" + "\n" + "* plus\n" + "* minus\n" + "* multiply\n" + "* divide\n" + "* intDiv\n" + "* intDivOrZero\n" + "* negate\n" + "* abs\n" + "\n" + ), link=None, level=3, - num='4.3.2') + num="4.3.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arithmetic_Dec_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Arithmetic functions] with Decimal256:\n' - '\n' - '* modulo\n' - '* moduloOrZero\n' - '* gcd\n' - '* lcm\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Arithmetic functions] with Decimal256:\n" + "\n" + "* modulo\n" + "* moduloOrZero\n" + "* gcd\n" + "* lcm\n" + "\n" + ), link=None, level=3, - num='4.3.3') + num="4.3.3", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arrays_Int_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Array functions] with Int128, UInt128, Int256, and UInt256.\n' - '\n' - '* empty\n' - '* notEmpty\n' - '* length\n' - '* arrayCount\n' - '* arrayPopBack\n' - '* arrayPopFront\n' - '* arraySort\n' - '* arrayReverseSort\n' - '* arrayUniq\n' - '* arrayJoin\n' - '* arrayDistinct\n' - '* arrayEnumerate\n' - '* arrayEnumerateDense\n' - '* arrayEnumerateUniq\n' - '* arrayReverse\n' - '* reverse\n' - '* arrayFlatten\n' - '* arrayCompact\n' - '* arrayExists\n' - '* arrayAll\n' - '* arrayMin\n' - '* arrayMax\n' - '* arraySum\n' - '* arrayAvg\n' - '* arrayReduce\n' - '* arrayReduceInRanges\n' - '* arrayZip\n' - '* arrayMap\n' - '* arrayFilter\n' - '* arrayFill\n' - '* arrayReverseFill\n' - '* arraySplit\n' - '* arrayFirst\n' - '* arrayFirstIndex\n' - '* arrayConcat\n' - '* hasAll\n' - '* hasAny\n' - '* hasSubstr\n' - '* arrayElement\n' - '* has\n' - '* indexOf\n' - '* countEqual\n' - '* arrayPushBack\n' - '* arrayPushFront\n' - '* arrayResize\n' - '* arraySlice\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Array functions] with Int128, UInt128, Int256, and UInt256.\n" + "\n" + "* empty\n" + "* notEmpty\n" + "* length\n" + "* arrayCount\n" + "* arrayPopBack\n" + "* arrayPopFront\n" + "* arraySort\n" + "* arrayReverseSort\n" + "* arrayUniq\n" + "* arrayJoin\n" + "* arrayDistinct\n" + "* arrayEnumerate\n" + "* arrayEnumerateDense\n" + "* arrayEnumerateUniq\n" + "* arrayReverse\n" + "* reverse\n" + "* arrayFlatten\n" + "* arrayCompact\n" + "* arrayExists\n" + "* arrayAll\n" + "* arrayMin\n" + "* arrayMax\n" + "* arraySum\n" + "* arrayAvg\n" + "* arrayReduce\n" + "* arrayReduceInRanges\n" + "* arrayZip\n" + "* arrayMap\n" + "* arrayFilter\n" + "* arrayFill\n" + "* arrayReverseFill\n" + "* arraySplit\n" + "* arrayFirst\n" + "* arrayFirstIndex\n" + "* arrayConcat\n" + "* hasAll\n" + "* hasAny\n" + "* hasSubstr\n" + "* arrayElement\n" + "* has\n" + "* indexOf\n" + "* countEqual\n" + "* arrayPushBack\n" + "* arrayPushFront\n" + "* arrayResize\n" + "* arraySlice\n" + "\n" + ), link=None, level=3, - num='4.4.1') + num="4.4.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arrays_Int_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Array functions] with Int128, UInt128, Int256, and UInt256:\n' - '\n' - '* arrayDifference\n' - '* arrayCumSum\n' - '* arrayCumSumNonNegative\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Array functions] with Int128, UInt128, Int256, and UInt256:\n" + "\n" + "* arrayDifference\n" + "* arrayCumSum\n" + "* arrayCumSumNonNegative\n" + "\n" + ), link=None, level=3, - num='4.4.2') + num="4.4.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arrays_Dec_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Array functions] with Decimal256:\n' - '\n' - '* empty\n' - '* notEmpty\n' - '* length\n' - '* arrayCount\n' - '* arrayPopBack\n' - '* arrayPopFront\n' - '* arraySort\n' - '* arrayReverseSort\n' - '* arrayUniq\n' - '* arrayJoin\n' - '* arrayDistinct\n' - '* arrayEnumerate\n' - '* arrayEnumerateDense\n' - '* arrayEnumerateUniq\n' - '* arrayReverse\n' - '* reverse\n' - '* arrayFlatten\n' - '* arrayCompact\n' - '* arrayExists\n' - '* arrayAll\n' - '* arrayReduce\n' - '* arrayReduceInRanges\n' - '* arrayZip\n' - '* arrayMap\n' - '* arrayFilter\n' - '* arrayFill\n' - '* arrayReverseFill\n' - '* arraySplit\n' - '* arrayFirst\n' - '* arrayFirstIndex\n' - '* arrayConcat\n' - '* hasAll\n' - '* hasAny\n' - '* hasSubstr\n' - '* arrayElement\n' - '* has\n' - '* indexOf\n' - '* countEqual\n' - '* arrayPushBack\n' - '* arrayPushFront\n' - '* arrayResize\n' - '* arraySlice\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Array functions] with Decimal256:\n" + "\n" + "* empty\n" + "* notEmpty\n" + "* length\n" + "* arrayCount\n" + "* arrayPopBack\n" + "* arrayPopFront\n" + "* arraySort\n" + "* arrayReverseSort\n" + "* arrayUniq\n" + "* arrayJoin\n" + "* arrayDistinct\n" + "* arrayEnumerate\n" + "* arrayEnumerateDense\n" + "* arrayEnumerateUniq\n" + "* arrayReverse\n" + "* reverse\n" + "* arrayFlatten\n" + "* arrayCompact\n" + "* arrayExists\n" + "* arrayAll\n" + "* arrayReduce\n" + "* arrayReduceInRanges\n" + "* arrayZip\n" + "* arrayMap\n" + "* arrayFilter\n" + "* arrayFill\n" + "* arrayReverseFill\n" + "* arraySplit\n" + "* arrayFirst\n" + "* arrayFirstIndex\n" + "* arrayConcat\n" + "* hasAll\n" + "* hasAny\n" + "* hasSubstr\n" + "* arrayElement\n" + "* has\n" + "* indexOf\n" + "* countEqual\n" + "* arrayPushBack\n" + "* arrayPushFront\n" + "* arrayResize\n" + "* arraySlice\n" + "\n" + ), link=None, level=3, - num='4.4.3') + num="4.4.3", +) RQ_SRS_020_ClickHouse_Extended_Precision_Arrays_Dec_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Array functions] with Decimal256:\n' - '\n' - '* arrayMin\n' - '* arrayMax\n' - '* arraaySum\n' - '* arrayAvg\n' - '* arrayDifference\n' - '* arrayCumSum\n' - '* arrayCumSumNonNegative\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Array functions] with Decimal256:\n" + "\n" + "* arrayMin\n" + "* arrayMax\n" + "* arraaySum\n" + "* arrayAvg\n" + "* arrayDifference\n" + "* arrayCumSum\n" + "* arrayCumSumNonNegative\n" + "\n" + ), link=None, level=3, - num='4.4.4') + num="4.4.4", +) RQ_SRS_020_ClickHouse_Extended_Precision_Comparison = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Comparison', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Comparison", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using [Comparison functions] with [Extended Precision Data Types].\n' - '\n' - 'Comparison functions:\n' - '* equals\n' - '* notEquals\n' - '* less\n' - '* greater\n' - '* lessOrEquals\n' - '* greaterOrEquals\n' - '\n' - ), + "[ClickHouse] SHALL support using [Comparison functions] with [Extended Precision Data Types].\n" + "\n" + "Comparison functions:\n" + "* equals\n" + "* notEquals\n" + "* less\n" + "* greater\n" + "* lessOrEquals\n" + "* greaterOrEquals\n" + "\n" + ), link=None, level=3, - num='4.5.1') + num="4.5.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Logical = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Logical', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Logical", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using [Logical functions] with [Extended Precision Data Types].\n' - '\n' - 'Logical functions:\n' - '* and\n' - '* or\n' - '* not\n' - '* xor\n' - '\n' - ), + "[ClickHouse] MAY not support using [Logical functions] with [Extended Precision Data Types].\n" + "\n" + "Logical functions:\n" + "* and\n" + "* or\n" + "* not\n" + "* xor\n" + "\n" + ), link=None, level=3, - num='4.6.1') + num="4.6.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Mathematical_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Mathematical functions] with [Extended Precision Data Types]:\n' - '\n' - '* exp\n' - '* log, ln\n' - '* exp2\n' - '* log2\n' - '* exp10\n' - '* log10\n' - '* sqrt\n' - '* cbrt\n' - '* erf\n' - '* erfc\n' - '* lgamma\n' - '* tgamma\n' - '* sin\n' - '* cos\n' - '* tan\n' - '* asin\n' - '* acos\n' - '* atan\n' - '* cosh\n' - '* acosh\n' - '* sinh\n' - '* asinh\n' - '* tanh\n' - '* atanh\n' - '* log1p\n' - '* sign\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Mathematical functions] with [Extended Precision Data Types]:\n" + "\n" + "* exp\n" + "* log, ln\n" + "* exp2\n" + "* log2\n" + "* exp10\n" + "* log10\n" + "* sqrt\n" + "* cbrt\n" + "* erf\n" + "* erfc\n" + "* lgamma\n" + "* tgamma\n" + "* sin\n" + "* cos\n" + "* tan\n" + "* asin\n" + "* acos\n" + "* atan\n" + "* cosh\n" + "* acosh\n" + "* sinh\n" + "* asinh\n" + "* tanh\n" + "* atanh\n" + "* log1p\n" + "* sign\n" + "\n" + ), link=None, level=3, - num='4.7.1') + num="4.7.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Mathematical_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Mathematical functions] with [Extended Precision Data Types]:\n' - '\n' - '* pow, power\n' - '* intExp2\n' - '* intExp10\n' - '* atan2\n' - '* hypot\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Mathematical functions] with [Extended Precision Data Types]:\n" + "\n" + "* pow, power\n" + "* intExp2\n" + "* intExp10\n" + "* atan2\n" + "* hypot\n" + "\n" + ), link=None, level=3, - num='4.7.2') + num="4.7.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Rounding_Int_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Rounding functions] with Int128, UInt128, Int256, and UInt256:\n' - '\n' - '* floor\n' - '* ceil\n' - '* trunc\n' - '* round\n' - '* roundBankers\n' - '* roundDuration\n' - '* roundAge\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Rounding functions] with Int128, UInt128, Int256, and UInt256:\n" + "\n" + "* floor\n" + "* ceil\n" + "* trunc\n" + "* round\n" + "* roundBankers\n" + "* roundDuration\n" + "* roundAge\n" + "\n" + ), link=None, level=3, - num='4.8.1') + num="4.8.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Rounding_Int_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Rounding functions] with Int128, UInt128, Int256, and UInt256:\n' - '\n' - '* roundDown\n' - '* roundToExp2\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Rounding functions] with Int128, UInt128, Int256, and UInt256:\n" + "\n" + "* roundDown\n" + "* roundToExp2\n" + "\n" + ), link=None, level=3, - num='4.8.2') + num="4.8.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Rounding_Dec_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Rounding functions] with Decimal256:\n' - '\n' - '* floor\n' - '* ceil\n' - '* trunc\n' - '* round\n' - '* roundBankers\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Rounding functions] with Decimal256:\n" + "\n" + "* floor\n" + "* ceil\n" + "* trunc\n" + "* round\n" + "* roundBankers\n" + "\n" + ), link=None, level=3, - num='4.8.3') + num="4.8.3", +) RQ_SRS_020_ClickHouse_Extended_Precision_Rounding_Dec_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Rounding functions] with Decimal256:\n' - '\n' - '* roundDuration\n' - '* roundAge\n' - '* roundDown\n' - '* roundToExp2\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Rounding functions] with Decimal256:\n" + "\n" + "* roundDuration\n" + "* roundAge\n" + "* roundDown\n" + "* roundToExp2\n" + "\n" + ), link=None, level=3, - num='4.8.4') + num="4.8.4", +) RQ_SRS_020_ClickHouse_Extended_Precision_Bit_Int_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Bit functions] with Int128, UInt128, Int256, and UInt256:\n' - '\n' - '* bitAnd\n' - '* bitOr\n' - '* bitXor\n' - '* bitNot\n' - '* bitShiftLeft\n' - '* bitShiftRight\n' - '* bitCount\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Bit functions] with Int128, UInt128, Int256, and UInt256:\n" + "\n" + "* bitAnd\n" + "* bitOr\n" + "* bitXor\n" + "* bitNot\n" + "* bitShiftLeft\n" + "* bitShiftRight\n" + "* bitCount\n" + "\n" + ), link=None, level=3, - num='4.9.1') + num="4.9.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Bit_Int_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Bit functions] with Int128, UInt128, Int256, and UInt256:\n' - '\n' - '* bitRotateLeft\n' - '* bitRotateRight\n' - '* bitTest\n' - '* bitTestAll\n' - '* bitTestAny\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Bit functions] with Int128, UInt128, Int256, and UInt256:\n" + "\n" + "* bitRotateLeft\n" + "* bitRotateRight\n" + "* bitTest\n" + "* bitTestAll\n" + "* bitTestAny\n" + "\n" + ), link=None, level=3, - num='4.9.2') + num="4.9.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Bit_Dec_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Dec.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Dec.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using [Bit functions] with Decimal256.\n' - '\n' - 'Bit functions:\n' - '* bitAnd\n' - '* bitOr\n' - '* bitXor\n' - '* bitNot\n' - '* bitShiftLeft\n' - '* bitShiftRight\n' - '* bitCount\n' - '* bitRotateLeft\n' - '* bitRotateRight\n' - '* bitTest\n' - '* bitTestAll\n' - '* bitTestAny\n' - '\n' - ), + "[ClickHouse] MAY not support using [Bit functions] with Decimal256.\n" + "\n" + "Bit functions:\n" + "* bitAnd\n" + "* bitOr\n" + "* bitXor\n" + "* bitNot\n" + "* bitShiftLeft\n" + "* bitShiftRight\n" + "* bitCount\n" + "* bitRotateLeft\n" + "* bitRotateRight\n" + "* bitTest\n" + "* bitTestAll\n" + "* bitTestAny\n" + "\n" + ), link=None, level=3, - num='4.9.3') + num="4.9.3", +) RQ_SRS_020_ClickHouse_Extended_Precision_Null = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Null', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Null", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using [Null functions] with [Extended Precision Data Types].\n' - '\n' - 'Null functions:\n' - '* isNull\n' - '* isNotNull\n' - '* coalesce\n' - '* ifNull\n' - '* nullIf\n' - '* assumeNotNull\n' - '* toNullable\n' - '\n' - ), + "[ClickHouse] SHALL support using [Null functions] with [Extended Precision Data Types].\n" + "\n" + "Null functions:\n" + "* isNull\n" + "* isNotNull\n" + "* coalesce\n" + "* ifNull\n" + "* nullIf\n" + "* assumeNotNull\n" + "* toNullable\n" + "\n" + ), link=None, level=3, - num='4.10.1') + num="4.10.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Tuple = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Tuple', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Tuple", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using [Tuple functions] with [Extended Precision Data Types].\n' - '\n' - 'Tuple functions:\n' - '* tuple\n' - '* tupleElement\n' - '* untuple\n' - '\n' - ), + "[ClickHouse] SHALL support using [Tuple functions] with [Extended Precision Data Types].\n" + "\n" + "Tuple functions:\n" + "* tuple\n" + "* tupleElement\n" + "* untuple\n" + "\n" + ), link=None, level=3, - num='4.11.1') + num="4.11.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Map_Supported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Map.Supported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Map.Supported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the following [Map functions] with [Extended Precision Data Types]:\n' - '\n' - '* map\n' - '* mapContains\n' - '* mapKeys\n' - '* mapValues\n' - '\n' - ), + "[ClickHouse] SHALL support using the following [Map functions] with [Extended Precision Data Types]:\n" + "\n" + "* map\n" + "* mapContains\n" + "* mapKeys\n" + "* mapValues\n" + "\n" + ), link=None, level=3, - num='4.12.1') + num="4.12.1", +) RQ_SRS_020_ClickHouse_Extended_Precision_Map_NotSupported = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Map.NotSupported', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Map.NotSupported", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using the following [Map functions] with [Extended Precision Data Types]:\n' - '\n' - '* mapAdd\n' - '* mapSubtract\n' - '* mapPopulateSeries\n' - '\n' - ), + "[ClickHouse] MAY not support using the following [Map functions] with [Extended Precision Data Types]:\n" + "\n" + "* mapAdd\n" + "* mapSubtract\n" + "* mapPopulateSeries\n" + "\n" + ), link=None, level=3, - num='4.12.2') + num="4.12.2", +) RQ_SRS_020_ClickHouse_Extended_Precision_Create_Table = Requirement( - name='RQ.SRS-020.ClickHouse.Extended.Precision.Create.Table', - version='1.0', + name="RQ.SRS-020.ClickHouse.Extended.Precision.Create.Table", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating table with columns that use [Extended Precision Data Types].\n' - '\n' - ), + "[ClickHouse] SHALL support creating table with columns that use [Extended Precision Data Types].\n" + "\n" + ), link=None, level=3, - num='4.13.1') + num="4.13.1", +) SRS020_ClickHouse_Extended_Precision_Data_Types = Specification( - name='SRS020 ClickHouse Extended Precision Data Types', + name="SRS020 ClickHouse Extended Precision Data Types", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -771,56 +801,172 @@ SRS020_ClickHouse_Extended_Precision_Data_Types = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='Extended Precision Data Types', level=2, num='3.1'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision', level=2, num='4.1'), - Heading(name='Conversion', level=2, num='4.2'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt128', level=3, num='4.2.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt128', level=3, num='4.2.2'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt256', level=3, num='4.2.3'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt256', level=3, num='4.2.4'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toDecimal256', level=3, num='4.2.5'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.FromMySQL', level=3, num='4.2.6'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.ToMySQL', level=3, num='4.2.7'), - Heading(name='Arithmetic', level=2, num='4.3'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Int.Supported', level=3, num='4.3.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.Supported', level=3, num='4.3.2'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.NotSupported', level=3, num='4.3.3'), - Heading(name='Arrays', level=2, num='4.4'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.Supported', level=3, num='4.4.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.NotSupported', level=3, num='4.4.2'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.Supported', level=3, num='4.4.3'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.NotSupported', level=3, num='4.4.4'), - Heading(name='Comparison', level=2, num='4.5'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Comparison', level=3, num='4.5.1'), - Heading(name='Logical Functions', level=2, num='4.6'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Logical', level=3, num='4.6.1'), - Heading(name='Mathematical Functions', level=2, num='4.7'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.Supported', level=3, num='4.7.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.NotSupported', level=3, num='4.7.2'), - Heading(name='Rounding Functions', level=2, num='4.8'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.Supported', level=3, num='4.8.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.NotSupported', level=3, num='4.8.2'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.Supported', level=3, num='4.8.3'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.NotSupported', level=3, num='4.8.4'), - Heading(name='Bit Functions', level=2, num='4.9'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.Supported', level=3, num='4.9.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.NotSupported', level=3, num='4.9.2'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Dec.NotSupported', level=3, num='4.9.3'), - Heading(name='Null Functions', level=2, num='4.10'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Null', level=3, num='4.10.1'), - Heading(name='Tuple Functions', level=2, num='4.11'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Tuple', level=3, num='4.11.1'), - Heading(name='Map Functions', level=2, num='4.12'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Map.Supported', level=3, num='4.12.1'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Map.NotSupported', level=3, num='4.12.2'), - Heading(name='Create', level=2, num='4.13'), - Heading(name='RQ.SRS-020.ClickHouse.Extended.Precision.Create.Table', level=3, num='4.13.1'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="Extended Precision Data Types", level=2, num="3.1"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="RQ.SRS-020.ClickHouse.Extended.Precision", level=2, num="4.1"), + Heading(name="Conversion", level=2, num="4.2"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt128", + level=3, + num="4.2.1", ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt128", + level=3, + num="4.2.2", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toInt256", + level=3, + num="4.2.3", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toUInt256", + level=3, + num="4.2.4", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.toDecimal256", + level=3, + num="4.2.5", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.FromMySQL", + level=3, + num="4.2.6", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Conversion.ToMySQL", + level=3, + num="4.2.7", + ), + Heading(name="Arithmetic", level=2, num="4.3"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Int.Supported", + level=3, + num="4.3.1", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.Supported", + level=3, + num="4.3.2", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arithmetic.Dec.NotSupported", + level=3, + num="4.3.3", + ), + Heading(name="Arrays", level=2, num="4.4"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.Supported", + level=3, + num="4.4.1", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Int.NotSupported", + level=3, + num="4.4.2", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.Supported", + level=3, + num="4.4.3", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Arrays.Dec.NotSupported", + level=3, + num="4.4.4", + ), + Heading(name="Comparison", level=2, num="4.5"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Comparison", + level=3, + num="4.5.1", + ), + Heading(name="Logical Functions", level=2, num="4.6"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Logical", + level=3, + num="4.6.1", + ), + Heading(name="Mathematical Functions", level=2, num="4.7"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.Supported", + level=3, + num="4.7.1", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Mathematical.NotSupported", + level=3, + num="4.7.2", + ), + Heading(name="Rounding Functions", level=2, num="4.8"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.Supported", + level=3, + num="4.8.1", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Int.NotSupported", + level=3, + num="4.8.2", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.Supported", + level=3, + num="4.8.3", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Rounding.Dec.NotSupported", + level=3, + num="4.8.4", + ), + Heading(name="Bit Functions", level=2, num="4.9"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.Supported", + level=3, + num="4.9.1", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Int.NotSupported", + level=3, + num="4.9.2", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Bit.Dec.NotSupported", + level=3, + num="4.9.3", + ), + Heading(name="Null Functions", level=2, num="4.10"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Null", level=3, num="4.10.1" + ), + Heading(name="Tuple Functions", level=2, num="4.11"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Tuple", level=3, num="4.11.1" + ), + Heading(name="Map Functions", level=2, num="4.12"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Map.Supported", + level=3, + num="4.12.1", + ), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Map.NotSupported", + level=3, + num="4.12.2", + ), + Heading(name="Create", level=2, num="4.13"), + Heading( + name="RQ.SRS-020.ClickHouse.Extended.Precision.Create.Table", + level=3, + num="4.13.1", + ), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_020_ClickHouse_Extended_Precision, RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt128, @@ -853,8 +999,8 @@ SRS020_ClickHouse_Extended_Precision_Data_Types = Specification( RQ_SRS_020_ClickHouse_Extended_Precision_Map_Supported, RQ_SRS_020_ClickHouse_Extended_Precision_Map_NotSupported, RQ_SRS_020_ClickHouse_Extended_Precision_Create_Table, - ), - content=''' + ), + content=""" # SRS020 ClickHouse Extended Precision Data Types # Software Requirements Specification @@ -1420,4 +1566,5 @@ version: 1.0 [Revision History]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/extended_precision_data_types/requirements/requirements.md [Git]: https://git-scm.com/ [GitHub]: https://github.com -''') +""", +) diff --git a/tests/testflows/extended_precision_data_types/tests/arithmetic.py b/tests/testflows/extended_precision_data_types/tests/arithmetic.py index c57f3d7d8e1..e949ef65f53 100644 --- a/tests/testflows/extended_precision_data_types/tests/arithmetic.py +++ b/tests/testflows/extended_precision_data_types/tests/arithmetic.py @@ -5,198 +5,256 @@ from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * funcs = [ - ('plus', '2'), - ('minus', '0'), - ('multiply', '1'), - ('divide', '1'), - ('intDiv', '1'), - ('intDivOrZero', '1'), - ('modulo', '0'), - ('moduloOrZero', '0'), - ('negate', '-1'), - ('abs', '1'), - ('gcd', '1'), - ('lcm', '1'), + ("plus", "2"), + ("minus", "0"), + ("multiply", "1"), + ("divide", "1"), + ("intDiv", "1"), + ("intDivOrZero", "1"), + ("modulo", "0"), + ("moduloOrZero", "0"), + ("negate", "-1"), + ("abs", "1"), + ("gcd", "1"), + ("lcm", "1"), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]} - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_dec_list = [ + tuple(list(func) + [Name(f"{func[0]} - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_dec_list = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline -@Examples('arithmetic_func expected_result int_type min max', Examples_list) +@Examples("arithmetic_func expected_result int_type min max", Examples_list) def inline_check(self, arithmetic_func, expected_result, int_type, min, max, node=None): - """Check that arithmetic functions work using inline tests with Int128, UInt128, Int256, and UInt256. - """ + """Check that arithmetic functions work using inline tests with Int128, UInt128, Int256, and UInt256.""" if node is None: node = self.context.node - if arithmetic_func in ['negate','abs']: + if arithmetic_func in ["negate", "abs"]: with When(f"I check {arithmetic_func} with {int_type}"): output = node.query(f"SELECT {arithmetic_func}(to{int_type}(1))").output assert output == expected_result, error() with When(f"I check {arithmetic_func} with {int_type} max and min value"): - execute_query(f""" + execute_query( + f""" SELECT {arithmetic_func}(to{int_type}(\'{max}\')), {arithmetic_func}(to{int_type}(\'{min}\')) - """) + """ + ) else: with When(f"I check {arithmetic_func} with {int_type}"): - output = node.query(f"SELECT {arithmetic_func}(to{int_type}(1), to{int_type}(1))").output + output = node.query( + f"SELECT {arithmetic_func}(to{int_type}(1), to{int_type}(1))" + ).output assert output == expected_result, error() - if arithmetic_func in ['gcd','lcm']: + if arithmetic_func in ["gcd", "lcm"]: - if int_type in ['UInt128','UInt256']: - exitcode=153 + if int_type in ["UInt128", "UInt256"]: + exitcode = 153 else: - exitcode=151 + exitcode = 151 with When(f"I check {arithmetic_func} with {int_type} max and min value"): - node.query(f"SELECT {arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1)), {arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1))", - exitcode = exitcode, message = 'Exception:') + node.query( + f"SELECT {arithmetic_func}(to{int_type}('{max}'), to{int_type}(1)), {arithmetic_func}(to{int_type}('{min}'), to{int_type}(1))", + exitcode=exitcode, + message="Exception:", + ) else: with When(f"I check {arithmetic_func} with {int_type} max and min value"): - execute_query(f""" + execute_query( + f""" SELECT round({arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1)), {rounding_precision}), round({arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1)), {rounding_precision}) - """) + """ + ) + @TestOutline -@Examples('arithmetic_func expected_result int_type min max', Examples_list) +@Examples("arithmetic_func expected_result int_type min max", Examples_list) def table_check(self, arithmetic_func, expected_result, int_type, min, max, node=None): - """Check that arithmetic functions work using tables with Int128, UInt128, Int256, and UInt256. - """ + """Check that arithmetic functions work using tables with Int128, UInt128, Int256, and UInt256.""" if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = int_type) + table(name=table_name, data_type=int_type) - if arithmetic_func in ['negate','abs']: + if arithmetic_func in ["negate", "abs"]: for value in [1, min, max]: - with When(f"I insert {arithmetic_func} with {int_type} {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}(\'{value}\'))") + with When( + f"I insert {arithmetic_func} with {int_type} {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}('{value}'))" + ) with Then(f"I check the table output of {arithmetic_func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) else: with When(f"I insert {arithmetic_func} with {int_type} into the table"): - node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(1), to{int_type}(1)), {rounding_precision})") + node.query( + f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(1), to{int_type}(1)), {rounding_precision})" + ) with Then("I check that the output matches the expected value"): output = node.query(f"SELECT * FROM {table_name}").output assert output == expected_result, error() - if arithmetic_func in ['gcd', 'lcm']: + if arithmetic_func in ["gcd", "lcm"]: - if int_type in ['UInt128', 'UInt256']: + if int_type in ["UInt128", "UInt256"]: - with When(f"I insert {arithmetic_func} with {int_type} {min} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}(\'{min}\'), to{int_type}(1))", - exitcode = 153, message = 'Exception:') + with When( + f"I insert {arithmetic_func} with {int_type} {min} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}('{min}'), to{int_type}(1))", + exitcode=153, + message="Exception:", + ) - with And(f"I insert {arithmetic_func} with {int_type} {max} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}(\'{max}\'), to{int_type}(1))") + with And( + f"I insert {arithmetic_func} with {int_type} {max} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}('{max}'), to{int_type}(1))" + ) else: for value in [min, max]: - with When(f"I insert {arithmetic_func} with {int_type} {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}(\'{value}\'), to{int_type}(1))", - exitcode = 151, message = 'Exception:') + with When( + f"I insert {arithmetic_func} with {int_type} {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {arithmetic_func}(to{int_type}('{value}'), to{int_type}(1))", + exitcode=151, + message="Exception:", + ) else: for value in [min, max]: - with When(f"I insert {arithmetic_func} with {int_type} {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}(\'{value}\'), to{int_type}(1)), {rounding_precision})") + with When( + f"I insert {arithmetic_func} with {int_type} {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT round({arithmetic_func}(to{int_type}('{value}'), to{int_type}(1)), {rounding_precision})" + ) with Then(f"I check the table output of {arithmetic_func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestOutline -@Examples('arithmetic_func expected_result', Examples_dec_list) +@Examples("arithmetic_func expected_result", Examples_dec_list) def inline_check_dec(self, arithmetic_func, expected_result, node=None): - """Check that arithmetic functions work using inline with Decimal256. - """ + """Check that arithmetic functions work using inline with Decimal256.""" if node is None: node = self.context.node - if arithmetic_func in ['negate','abs']: + if arithmetic_func in ["negate", "abs"]: with When(f"I check {arithmetic_func} with toDecimal256"): output = node.query(f"SELECT {arithmetic_func}(toDecimal256(1,0))").output assert output == expected_result, error() - elif arithmetic_func in ['modulo', 'moduloOrZero', 'gcd', 'lcm']: + elif arithmetic_func in ["modulo", "moduloOrZero", "gcd", "lcm"]: with When(f"I check {arithmetic_func} with toDecimal256"): - node.query(f"SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))", - exitcode=43, message = 'Exception:') + node.query( + f"SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))", + exitcode=43, + message="Exception:", + ) else: with When(f"I check {arithmetic_func} with toDecimal256"): - output = node.query(f"SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))").output + output = node.query( + f"SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))" + ).output assert output == expected_result, error() + @TestOutline -@Examples('arithmetic_func expected_result', Examples_dec_list) +@Examples("arithmetic_func expected_result", Examples_dec_list) def table_check_dec(self, arithmetic_func, expected_result, node=None): - """Check that arithmetic functions work using tables with Decimal256. - """ + """Check that arithmetic functions work using tables with Decimal256.""" if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = 'Decimal256(0)') + table(name=table_name, data_type="Decimal256(0)") - if arithmetic_func in ['negate','abs']: + if arithmetic_func in ["negate", "abs"]: with When(f"I insert {arithmetic_func} with toDecimal256 into the table"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(toDecimal256(1,0))") + node.query( + f"INSERT INTO {table_name} SELECT {arithmetic_func}(toDecimal256(1,0))" + ) with Then(f"I check the table for output of {arithmetic_func} with Decimal256"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) - elif arithmetic_func in ['modulo', 'moduloOrZero', 'gcd', 'lcm']: + elif arithmetic_func in ["modulo", "moduloOrZero", "gcd", "lcm"]: with When(f"I check {arithmetic_func} with toDecimal256"): - node.query(f"INSERT INTO {table_name} SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))", - exitcode=43, message = 'Exception:') + node.query( + f"INSERT INTO {table_name} SELECT {arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0))", + exitcode=43, + message="Exception:", + ) else: with When(f"I insert {arithmetic_func} with toDecimal256 into the table"): - node.query(f"INSERT INTO {table_name} SELECT round({arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0)), {rounding_precision})") + node.query( + f"INSERT INTO {table_name} SELECT round({arithmetic_func}(toDecimal256(1,0), toDecimal256(1,0)), {rounding_precision})" + ) with Then("I check that the output matches the expected value"): output = node.query(f"SELECT * FROM {table_name}").output assert output == expected_result, error() + @TestFeature @Name("arithmetic") @Requirements( @@ -205,13 +263,12 @@ def table_check_dec(self, arithmetic_func, expected_result, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Arithmetic_Dec_NotSupported("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that arithmetic functions work with extended precision data types. - """ + """Check that arithmetic functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) with allow_experimental_bigint(self.context.node): - Scenario(run = inline_check) - Scenario(run = table_check) - Scenario(run = inline_check_dec) - Scenario(run = table_check_dec) + Scenario(run=inline_check) + Scenario(run=table_check) + Scenario(run=inline_check_dec) + Scenario(run=table_check_dec) diff --git a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py index c39574ba75e..106458d58bc 100644 --- a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py +++ b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py @@ -3,8 +3,10 @@ import uuid from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * + def get_table_name(): - return "table" + "_" + str(uuid.uuid1()).replace('-', '_') + return "table" + "_" + str(uuid.uuid1()).replace("-", "_") + @TestOutline(Suite) @Requirements( @@ -14,65 +16,74 @@ def get_table_name(): RQ_SRS_020_ClickHouse_Extended_Precision_Arrays_Dec_NotSupported("1.0"), ) def array_func(self, data_type, node=None): - """Check array functions with extended precision data types. - """ + """Check array functions with extended precision data types.""" if node is None: node = self.context.node - for func in ['arrayPopBack(', - 'arrayPopFront(', - 'arraySort(', - 'arrayReverseSort(', - 'arrayDistinct(', - 'arrayEnumerate(', - 'arrayEnumerateDense(', - 'arrayEnumerateUniq(', - 'arrayReverse(', - 'reverse(', - 'arrayFlatten(', - 'arrayCompact(', - 'arrayReduceInRanges(\'sum\', [(1, 5)],', - 'arrayMap(x -> (x + 2),', - 'arrayFill(x -> x=3,', - 'arrayReverseFill(x -> x=3,', - f'arrayConcat([{to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}],', - 'arrayFilter(x -> x == 1, ']: + for func in [ + "arrayPopBack(", + "arrayPopFront(", + "arraySort(", + "arrayReverseSort(", + "arrayDistinct(", + "arrayEnumerate(", + "arrayEnumerateDense(", + "arrayEnumerateUniq(", + "arrayReverse(", + "reverse(", + "arrayFlatten(", + "arrayCompact(", + "arrayReduceInRanges('sum', [(1, 5)],", + "arrayMap(x -> (x + 2),", + "arrayFill(x -> x=3,", + "arrayReverseFill(x -> x=3,", + f"arrayConcat([{to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}],", + "arrayFilter(x -> x == 1, ", + ]: with Scenario(f"Inline - {data_type} - {func})"): - execute_query(f""" + execute_query( + f""" SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)})) - """) + """ + ) with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() - table(name = table_name, data_type = f'Array({data_type})') + table(name=table_name, data_type=f"Array({data_type})") with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + node.query( + f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['arraySplit((x, y) -> x=y, [0, 0, 0],']: + for func in ["arraySplit((x, y) -> x=y, [0, 0, 0],"]: with Scenario(f"Inline - {data_type} - {func})"): - execute_query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}," - f"{to_data_type(data_type,1)}))") + execute_query( + f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}," + f"{to_data_type(data_type,1)}))" + ) with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() - table(name = table_name, data_type = f'Array(Array({data_type}))') + table(name=table_name, data_type=f"Array(Array({data_type}))") with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + node.query( + f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in [f'arrayZip([{to_data_type(data_type,1)}],']: + for func in [f"arrayZip([{to_data_type(data_type,1)}],"]: with Scenario(f"Inline - {data_type} - {func})"): execute_query(f"SELECT {func}array({to_data_type(data_type,3)}))") @@ -80,47 +91,62 @@ def array_func(self, data_type, node=None): with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() - table(name = table_name, data_type = f'Array(Tuple({data_type}, {data_type}))') + table(name=table_name, data_type=f"Array(Tuple({data_type}, {data_type}))") with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,1)}))") + node.query( + f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,1)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['empty(', - 'notEmpty(', - 'length(', - 'arrayCount(x -> x == 1, ', - 'arrayUniq(', - 'arrayJoin(', - 'arrayExists(x -> x==1,', - 'arrayAll(x -> x==1,', - 'arrayMin(', - 'arrayMax(', - 'arraySum(', - 'arrayAvg(', - 'arrayReduce(\'max\', ', - 'arrayFirst(x -> x==3,', - 'arrayFirstIndex(x -> x==3,', - f'hasAll([{to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}], ', - f'hasAny([{to_data_type(data_type,2)}, {to_data_type(data_type,1)}], ', - f'hasSubstr([{to_data_type(data_type,2)}, {to_data_type(data_type,1)}], ']: + for func in [ + "empty(", + "notEmpty(", + "length(", + "arrayCount(x -> x == 1, ", + "arrayUniq(", + "arrayJoin(", + "arrayExists(x -> x==1,", + "arrayAll(x -> x==1,", + "arrayMin(", + "arrayMax(", + "arraySum(", + "arrayAvg(", + "arrayReduce('max', ", + "arrayFirst(x -> x==3,", + "arrayFirstIndex(x -> x==3,", + f"hasAll([{to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}], ", + f"hasAny([{to_data_type(data_type,2)}, {to_data_type(data_type,1)}], ", + f"hasSubstr([{to_data_type(data_type,2)}, {to_data_type(data_type,1)}], ", + ]: - if func in ['arrayMin(','arrayMax(','arraySum(', 'arrayAvg('] and data_type in ['Decimal256(0)']: + if func in [ + "arrayMin(", + "arrayMax(", + "arraySum(", + "arrayAvg(", + ] and data_type in ["Decimal256(0)"]: with Scenario(f"Inline - {data_type} - {func})"): - node.query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", - exitcode = 44, message = 'Exception:') + node.query( + f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", + exitcode=44, + message="Exception:", + ) with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + node.query( + f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", - exitcode = 44, message = 'Exception:') + exitcode=44, + message="Exception:", + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") @@ -128,155 +154,185 @@ def array_func(self, data_type, node=None): with Scenario(f"Inline - {data_type} - {func})"): - execute_query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + execute_query( + f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))" + ) with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + node.query( + f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['arrayDifference(', - 'arrayCumSum(', - 'arrayCumSumNonNegative(']: + for func in ["arrayDifference(", "arrayCumSum(", "arrayCumSumNonNegative("]: - if data_type in ['Decimal256(0)']: + if data_type in ["Decimal256(0)"]: exitcode = 44 else: exitcode = 43 with Scenario(f"Inline - {data_type} - {func})"): - node.query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", - exitcode = exitcode, message = 'Exception:') + node.query( + f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", + exitcode=exitcode, + message="Exception:", + ) with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + node.query( + f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", - exitcode = exitcode, message = 'Exception:') + exitcode=exitcode, + message="Exception:", + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['arrayElement']: + for func in ["arrayElement"]: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f""" + execute_query( + f""" SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1) - """) + """ + ) with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") + node.query( + f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['arrayPushBack', - 'arrayPushFront']: + for func in ["arrayPushBack", "arrayPushFront"]: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f"SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}," - f"{to_data_type(data_type,1)}), {to_data_type(data_type,1)})") + execute_query( + f"SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}," + f"{to_data_type(data_type,1)}), {to_data_type(data_type,1)})" + ) with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() - table(name = table_name, data_type = f'Array({data_type})') + table(name=table_name, data_type=f"Array({data_type})") with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), {to_data_type(data_type,1)})") + node.query( + f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), {to_data_type(data_type,1)})" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['arrayResize', - 'arraySlice']: + for func in ["arrayResize", "arraySlice"]: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f"SELECT {func}(array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") + execute_query( + f"SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)" + ) with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() - table(name = table_name, data_type = f'Array({data_type})') + table(name=table_name, data_type=f"Array({data_type})") with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") + node.query( + f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") - for func in ['has', - 'indexOf', - 'countEqual']: + for func in ["has", "indexOf", "countEqual"]: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f"SELECT {func}(array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)") + execute_query( + f"SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)" + ) with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)") + node.query( + f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") + @TestOutline(Suite) @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Tuple("1.0"), ) def tuple_func(self, data_type, node=None): - """Check tuple functions with extended precision data types. - """ + """Check tuple functions with extended precision data types.""" if node is None: node = self.context.node with Scenario(f"Creating a tuple with {data_type}"): - node.query(f"SELECT tuple({to_data_type(data_type,1)}, {to_data_type(data_type,1)}, {to_data_type(data_type,1)})") + node.query( + f"SELECT tuple({to_data_type(data_type,1)}, {to_data_type(data_type,1)}, {to_data_type(data_type,1)})" + ) with Scenario(f"Creating a tuple with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = f'Tuple({data_type}, {data_type}, {data_type})') + table( + name=table_name, data_type=f"Tuple({data_type}, {data_type}, {data_type})" + ) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT tuple({to_data_type(data_type,1)}," - f"{to_data_type(data_type,1)}, {to_data_type(data_type,1)})") + node.query( + f"INSERT INTO {table_name} SELECT tuple({to_data_type(data_type,1)}," + f"{to_data_type(data_type,1)}, {to_data_type(data_type,1)})" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"tupleElement with {data_type}"): - node.query(f"SELECT tupleElement(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), 1)") + node.query( + f"SELECT tupleElement(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), 1)" + ) with Scenario(f"tupleElement with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT tupleElement(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), 1)") + node.query( + f"INSERT INTO {table_name} SELECT tupleElement(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), 1)" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") @@ -286,58 +342,70 @@ def tuple_func(self, data_type, node=None): with Scenario(f"untuple with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT untuple(({to_data_type(data_type,1)},))") + node.query( + f"INSERT INTO {table_name} SELECT untuple(({to_data_type(data_type,1)},))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"tupleHammingDistance with {data_type}"): - node.query(f"SELECT tupleHammingDistance(({to_data_type(data_type,1)}, {to_data_type(data_type,1)})," - f"({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))") + node.query( + f"SELECT tupleHammingDistance(({to_data_type(data_type,1)}, {to_data_type(data_type,1)})," + f"({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))" + ) with Scenario(f"tupleHammingDistance with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT tupleHammingDistance(({to_data_type(data_type,1)}," - f"{to_data_type(data_type,1)}), ({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))") + node.query( + f"INSERT INTO {table_name} SELECT tupleHammingDistance(({to_data_type(data_type,1)}," + f"{to_data_type(data_type,1)}), ({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") + @TestOutline(Suite) @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Map_Supported("1.0"), RQ_SRS_020_ClickHouse_Extended_Precision_Map_NotSupported("1.0"), ) def map_func(self, data_type, node=None): - """Check Map functions with extended precision data types. - """ + """Check Map functions with extended precision data types.""" if node is None: node = self.context.node with Scenario(f"Creating a map with {data_type}"): - node.query(f"SELECT map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)})") + node.query( + f"SELECT map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)})" + ) with Scenario(f"Creating a map with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = f'Map(String, {data_type})') + table(name=table_name, data_type=f"Map(String, {data_type})") with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)})") + node.query( + f"INSERT INTO {table_name} SELECT map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)})" + ) execute_query(f"SELECT * FROM {table_name}") with Scenario(f"mapAdd with {data_type}"): - sql = (f"SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + sql = ( + f"SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))" + ) if data_type.startswith("Decimal"): node.query(sql, exitcode=43, message="Exception:") else: @@ -346,14 +414,18 @@ def map_func(self, data_type, node=None): with Scenario(f"mapAdd with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') + table( + name=table_name, data_type=f"Tuple(Array({data_type}), Array({data_type}))" + ) with When("I insert the output into a table"): - sql = (f"INSERT INTO {table_name} SELECT mapAdd((" + sql = ( + f"INSERT INTO {table_name} SELECT mapAdd((" f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))" + ) exitcode, message = 0, None if data_type.startswith("Decimal"): @@ -363,11 +435,13 @@ def map_func(self, data_type, node=None): execute_query(f"""SELECT * FROM {table_name} ORDER BY a ASC""") with Scenario(f"mapSubtract with {data_type}"): - sql = (f"SELECT mapSubtract((" + sql = ( + f"SELECT mapSubtract((" f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))" + ) if data_type.startswith("Decimal"): node.query(sql, exitcode=43, message="Exception:") @@ -377,13 +451,17 @@ def map_func(self, data_type, node=None): with Scenario(f"mapSubtract with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') + table( + name=table_name, data_type=f"Tuple(Array({data_type}), Array({data_type}))" + ) with When("I insert the output into a table"): - sql = (f"INSERT INTO {table_name} SELECT mapSubtract(([{to_data_type(data_type,1)}," + sql = ( + f"INSERT INTO {table_name} SELECT mapSubtract(([{to_data_type(data_type,1)}," f"{to_data_type(data_type,2)}], [{to_data_type(data_type,1)}," f"{to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}," - f"{to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + f"{to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))" + ) exitcode, message = 0, None if data_type.startswith("Decimal"): @@ -393,8 +471,10 @@ def map_func(self, data_type, node=None): execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapPopulateSeries with {data_type}"): - sql = (f"SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}," - f"{to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)") + sql = ( + f"SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)" + ) exitcode, message = 0, None if data_type.startswith("Decimal"): @@ -404,11 +484,15 @@ def map_func(self, data_type, node=None): with Scenario(f"mapPopulateSeries with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') + table( + name=table_name, data_type=f"Tuple(Array({data_type}), Array({data_type}))" + ) with When("I insert the output into a table"): - sql = (f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)") + sql = ( + f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)" + ) exitcode, message = 0, None if data_type.startswith("Decimal"): @@ -418,57 +502,73 @@ def map_func(self, data_type, node=None): execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapContains with {data_type}"): - node.query(f"SELECT mapContains( map('key1', {to_data_type(data_type,1)}," - f"'key2', {to_data_type(data_type,2)}), 'key1')") + node.query( + f"SELECT mapContains( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}), 'key1')" + ) with Scenario(f"mapContains with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = data_type) + table(name=table_name, data_type=data_type) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapContains( map('key1', {to_data_type(data_type,1)}," - f"'key2', {to_data_type(data_type,2)}), 'key1')") + node.query( + f"INSERT INTO {table_name} SELECT mapContains( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}), 'key1')" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapKeys with {data_type}"): - node.query(f"SELECT mapKeys( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))") + node.query( + f"SELECT mapKeys( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))" + ) with Scenario(f"mapKeys with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = 'Array(String)') + table(name=table_name, data_type="Array(String)") with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapKeys( map('key1', {to_data_type(data_type,1)}," - f"'key2', {to_data_type(data_type,2)}))") + node.query( + f"INSERT INTO {table_name} SELECT mapKeys( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapValues with {data_type}"): - node.query(f"SELECT mapValues( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))") + node.query( + f"SELECT mapValues( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))" + ) with Scenario(f"mapValues with {data_type} on a table"): table_name = get_table_name() - table(name = table_name, data_type = f'Array({data_type})') + table(name=table_name, data_type=f"Array({data_type})") with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapValues( map('key1', {to_data_type(data_type,1)}," - f"'key2', {to_data_type(data_type,2)}))") + node.query( + f"INSERT INTO {table_name} SELECT mapValues( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}))" + ) execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") + @TestFeature @Name("array, tuple, map") -@Examples("data_type",[ - ('Int128',), - ('Int256',), - ('UInt128',), - ('UInt256',), - ('Decimal256(0)',), -]) +@Examples( + "data_type", + [ + ("Int128",), + ("Int256",), + ("UInt128",), + ("UInt256",), + ("Decimal256(0)",), + ], +) def feature(self, node="clickhouse1", stress=None, parallel=None): """Check that array, tuple, and map functions work with extended precision data types. @@ -477,7 +577,7 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): with allow_experimental_bigint(self.context.node): for example in self.examples: - data_type, = example + (data_type,) = example with Feature(data_type): diff --git a/tests/testflows/extended_precision_data_types/tests/bit.py b/tests/testflows/extended_precision_data_types/tests/bit.py index 24f63532c74..f32ae093607 100644 --- a/tests/testflows/extended_precision_data_types/tests/bit.py +++ b/tests/testflows/extended_precision_data_types/tests/bit.py @@ -3,31 +3,37 @@ from extended_precision_data_types.common import * from extended_precision_data_types.errors import * funcs = [ - ('bitAnd', True, None), - ('bitOr', True, None), - ('bitXor', True, None), - ('bitShiftLeft', True, None), - ('bitShiftRight', True, None), - ('bitRotateLeft', False, not_implemented_bigints('Bit rotate')), - ('bitRotateRight', False, not_implemented_bigints('Bit rotate')), - ('bitTest', False, not_implemented_bigints('bitTest')), - ('bitTestAll', False, illegal_column()), - ('bitTestAny', False, illegal_column()), - ('bitNot', True, None), - ('bitCount', True, None) + ("bitAnd", True, None), + ("bitOr", True, None), + ("bitXor", True, None), + ("bitShiftLeft", True, None), + ("bitShiftRight", True, None), + ("bitRotateLeft", False, not_implemented_bigints("Bit rotate")), + ("bitRotateRight", False, not_implemented_bigints("Bit rotate")), + ("bitTest", False, not_implemented_bigints("bitTest")), + ("bitTestAll", False, illegal_column()), + ("bitTestAny", False, illegal_column()), + ("bitNot", True, None), + ("bitCount", True, None), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]} - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_dec_list = [ + tuple(list(func) + [Name(f"{func[0]} - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_dec_list = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline(Scenario) -@Examples('func supported error int_type min max', Examples_list) +@Examples("func supported error int_type min max", Examples_list) def bit_int_inline(self, func, supported, error, int_type, min, max, node=None): - """ Check bit functions with Int128, UInt128, Int256, and UInt256 using inline tests. - """ + """Check bit functions with Int128, UInt128, Int256, and UInt256 using inline tests.""" if error is not None: - exitcode,message = error + exitcode, message = error if node is None: node = self.context.node @@ -35,28 +41,35 @@ def bit_int_inline(self, func, supported, error, int_type, min, max, node=None): if func in ["bitNot", "bitCount"]: with When(f"Check {func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT {func}(to{int_type}(1)), {func}(to{int_type}(\'{max}\')), {func}(to{int_type}(\'{min}\')) - """) + """ + ) elif supported: with When(f"I check {func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT {func}(to{int_type}(1), 1), {func}(to{int_type}(\'{max}\'), 1), {func}(to{int_type}(\'{min}\'), 1) - """) + """ + ) else: with When(f"I check {func} with {int_type}"): - node.query(f"SELECT {func}(to{int_type}(1), 1), {func}(to{int_type}(\'{max}\'), 1), {func}(to{int_type}(\'{min}\'), 1)", - exitcode=exitcode, message = message) + node.query( + f"SELECT {func}(to{int_type}(1), 1), {func}(to{int_type}('{max}'), 1), {func}(to{int_type}('{min}'), 1)", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) -@Examples('func supported error int_type min max', Examples_list) +@Examples("func supported error int_type min max", Examples_list) def bit_int_table(self, func, supported, error, int_type, min, max, node=None): - """ Check bit functions with Int128, UInt128, Int256, and UInt256 using table tests. - """ + """Check bit functions with Int128, UInt128, Int256, and UInt256 using table tests.""" table_name = f"table_{getuid()}" @@ -64,48 +77,59 @@ def bit_int_table(self, func, supported, error, int_type, min, max, node=None): node = self.context.node if error is not None: - exitcode,message = error + exitcode, message = error with Given(f"I have a table"): - table(name = table_name, data_type = int_type) + table(name=table_name, data_type=int_type) if func in ["bitNot", "bitCount"]: for value in [1, min, max]: with When(f"I insert the output of {func} with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(\'{value}\'))") + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}('{value}'))" + ) with Then(f"I check the table with values of {func} and {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) elif supported: for value in [1, min, max]: with When(f"I insert the output of {func} with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(\'{value}\'), 1)") + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}('{value}'), 1)" + ) with Then(f"I check the table with values of {func} and {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) else: for value in [1, min, max]: with When(f"I insert the output of {func} with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(\'{value}\'), 1)", - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}('{value}'), 1)", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) -@Examples('func supported error', Examples_dec_list) +@Examples("func supported error", Examples_dec_list) def bit_dec_inline(self, func, supported, error, node=None): - """ Check bit functions with Decimal256 using inline tests. - """ + """Check bit functions with Decimal256 using inline tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -117,20 +141,26 @@ def bit_dec_inline(self, func, supported, error, node=None): if func in ["bitNot", "bitCount"]: with When(f"Check {func} with Decimal256"): - node.query(f"SELECT {func}(toDecimal256(1,0)), {func}(toDecimal256(\'{max}\',0)), {func}(toDecimal256(\'{min}\',0))", - exitcode=exitcode, message = message) + node.query( + f"SELECT {func}(toDecimal256(1,0)), {func}(toDecimal256('{max}',0)), {func}(toDecimal256('{min}',0))", + exitcode=exitcode, + message=message, + ) else: with When(f"I check {func} with Decimal256"): - node.query(f"SELECT {func}(toDecimal256(1,0), 1), {func}(toDecimal256(\'{max}\',0), 1), {func}(toDecimal256(\'{min}\',0), 1)", - exitcode=exitcode, message = message) + node.query( + f"SELECT {func}(toDecimal256(1,0), 1), {func}(toDecimal256('{max}',0), 1), {func}(toDecimal256('{min}',0), 1)", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) -@Examples('func supported error', Examples_dec_list) +@Examples("func supported error", Examples_dec_list) def bit_dec_table(self, func, supported, error, node=None): - """ Check bit functions with Decimal256 using table tests. - """ + """Check bit functions with Decimal256 using table tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -141,23 +171,30 @@ def bit_dec_table(self, func, supported, error, node=None): node = self.context.node with Given(f"I have a table"): - table(name = table_name, data_type = 'Decimal256(0)') + table(name=table_name, data_type="Decimal256(0)") if func in ["bitNot", "bitCount"]: for value in [1, min, max]: with When(f"I insert the output of {func} with Decimal256 and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(toDecimal256(\'{value}\',0))", - exitcode=exitcode, message = message) + node.query( + f"INSERT INTO {table_name} SELECT {func}(toDecimal256('{value}',0))", + exitcode=exitcode, + message=message, + ) else: for value in [1, min, max]: with When(f"I insert the output of {func} with Decimal256 and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(toDecimal256(\'{value}\',0), 1)", - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} SELECT {func}(toDecimal256('{value}',0), 1)", + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("bit") @@ -167,8 +204,7 @@ def bit_dec_table(self, func, supported, error, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Bit_Dec_NotSupported("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that bit functions work with extended precision data types. - """ + """Check that bit functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/comparison.py b/tests/testflows/extended_precision_data_types/tests/comparison.py index 6f715e35b91..70d5abdd6a0 100644 --- a/tests/testflows/extended_precision_data_types/tests/comparison.py +++ b/tests/testflows/extended_precision_data_types/tests/comparison.py @@ -2,60 +2,74 @@ from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * funcs = [ - ('equals',), - ('notEquals',), - ('less',), - ('greater',), - ('lessOrEquals',), - ('greaterOrEquals',) + ("equals",), + ("notEquals",), + ("less",), + ("greater",), + ("lessOrEquals",), + ("greaterOrEquals",), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]} - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_list_dec = [ + tuple(list(func) + [Name(f"{func[0]} - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_list_dec = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline(Scenario) -@Examples('func int_type min max', Examples_list) +@Examples("func int_type min max", Examples_list) def comp_int_inline(self, func, int_type, min, max, node=None): - """Check comparison functions with Int128, UInt128, Int256, and UInt256 using inline tests. - """ + """Check comparison functions with Int128, UInt128, Int256, and UInt256 using inline tests.""" if node is None: node = self.context.node with When(f"I check {func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT {func}(to{int_type}(1), to{int_type}(1)), {func}(to{int_type}(\'{max}\'), to{int_type}(\'{min}\')) - """) + """ + ) + @TestOutline(Scenario) -@Examples('func int_type min max', Examples_list) +@Examples("func int_type min max", Examples_list) def comp_int_table(self, func, int_type, min, max, node=None): - """Check comparison functions with Int128, UInt128, Int256, and UInt256 using table tests. - """ + """Check comparison functions with Int128, UInt128, Int256, and UInt256 using table tests.""" if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = int_type) + table(name=table_name, data_type=int_type) for value in [1, max, min]: - with When(f"I insert into a table the output {func} with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(\'{value}\'), to{int_type}(1))") + with When( + f"I insert into a table the output {func} with {int_type} and {value}" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}('{value}'), to{int_type}(1))" + ) with Then(f"I check the table for the output of {func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestOutline(Scenario) -@Examples('func', Examples_list_dec) +@Examples("func", Examples_list_dec) def comp_dec_inline(self, func, node=None): - """Check comparison functions with Decimal256 using inline tests. - """ + """Check comparison functions with Decimal256 using inline tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -63,35 +77,44 @@ def comp_dec_inline(self, func, node=None): node = self.context.node with When(f"I check {func} with Decimal256"): - execute_query(f""" + execute_query( + f""" SELECT {func}(toDecimal256(1,0), toDecimal256(1,0)), {func}(toDecimal256(\'{max}\',0), toDecimal256(\'{min}\',0)) - """) + """ + ) + @TestOutline(Scenario) -@Examples('func', Examples_list_dec) +@Examples("func", Examples_list_dec) def comp_dec_table(self, func, node=None): - """Check comparison functions with Decimal256 using table tests. - """ + """Check comparison functions with Decimal256 using table tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = 'Decimal256(0)') + table(name=table_name, data_type="Decimal256(0)") for value in [1, max, min]: - with When(f"I insert into a table the output {func} with Decimal256 and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(toDecimal256(\'{value}\',0), toDecimal256(1,0))") + with When( + f"I insert into a table the output {func} with Decimal256 and {value}" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func}(toDecimal256('{value}',0), toDecimal256(1,0))" + ) with Then(f"I check the table for the output of {func} with Decimal256"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestFeature @Name("comparison") @@ -99,8 +122,7 @@ def comp_dec_table(self, func, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Comparison("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that comparison functions work with extended precision data types. - """ + """Check that comparison functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/conversion.py b/tests/testflows/extended_precision_data_types/tests/conversion.py index b98958009a0..942f40c91de 100644 --- a/tests/testflows/extended_precision_data_types/tests/conversion.py +++ b/tests/testflows/extended_precision_data_types/tests/conversion.py @@ -4,10 +4,10 @@ import textwrap from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * + @contextmanager def dictionary(name, node, mysql_node): - """Create a table in MySQL and use it a source for a dictionary. - """ + """Create a table in MySQL and use it a source for a dictionary.""" try: with Given("table in MySQL"): sql = f""" @@ -22,9 +22,15 @@ def dictionary(name, node, mysql_node): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("dictionary that uses MySQL table as the external source"): with When("I drop the dictionary if exists"): @@ -60,12 +66,15 @@ def dictionary(name, node, mysql_node): node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @contextmanager def table(name, node, mysql_node): - """Create a table in MySQL and use it a source for a table in ClickHouse. - """ + """Create a table in MySQL and use it a source for a table in ClickHouse.""" try: with Given("table in MySQL"): sql = f""" @@ -80,10 +89,16 @@ def table(name, node, mysql_node): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("table that uses MySQL table as the external source"): @@ -111,12 +126,15 @@ def table(name, node, mysql_node): node.query(f"DROP TABLE IF EXISTS {name}") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @contextmanager def table_func(name, node, mysql_node): - """Create a table in MySQL and use it a source for a table using mysql table function. - """ + """Create a table in MySQL and use it a source for a table using mysql table function.""" try: with Given("table in MySQL"): sql = f""" @@ -131,9 +149,15 @@ def table_func(name, node, mysql_node): ); """ with When("I drop the table if exists"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) with And("I create a table"): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) yield f"mysql('{mysql_node.name}:3306', 'db', '{name}', 'user', 'password')" @@ -143,33 +167,73 @@ def table_func(name, node, mysql_node): node.query(f"DROP TABLE IF EXISTS {name}") with And("I drop a table in MySQL", flags=TE): - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + mysql_node.command( + f'MYSQL_PWD=password mysql -D db -u user -e "DROP TABLE IF EXISTS {name};"', + exitcode=0, + ) + @TestOutline(Scenario) -@Examples('int_type min max',[ - ('Int128', '-170141183460469231731687303715884105728', '170141183460469231731687303715884105727', Requirements(RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt128("1.0")), Name('Int128')), - ('Int256', '-57896044618658097711785492504343953926634992332820282019728792003956564819968', '57896044618658097711785492504343953926634992332820282019728792003956564819967', Requirements(RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt256("1.0")), Name('Int256')), - ('UInt128','0','340282366920938463463374607431768211455', Requirements(RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toUInt128("1.0")), Name('UInt128')), - ('UInt256', '0', '115792089237316195423570985008687907853269984665640564039457584007913129639935', Requirements(RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toUInt256("1.0")), Name('UInt256')), -]) +@Examples( + "int_type min max", + [ + ( + "Int128", + "-170141183460469231731687303715884105728", + "170141183460469231731687303715884105727", + Requirements( + RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt128("1.0") + ), + Name("Int128"), + ), + ( + "Int256", + "-57896044618658097711785492504343953926634992332820282019728792003956564819968", + "57896044618658097711785492504343953926634992332820282019728792003956564819967", + Requirements( + RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toInt256("1.0") + ), + Name("Int256"), + ), + ( + "UInt128", + "0", + "340282366920938463463374607431768211455", + Requirements( + RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toUInt128("1.0") + ), + Name("UInt128"), + ), + ( + "UInt256", + "0", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + Requirements( + RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toUInt256("1.0") + ), + Name("UInt256"), + ), + ], +) def int_conversion(self, int_type, min, max, node=None): - """Check that ClickHouse converts values to Int128. - """ + """Check that ClickHouse converts values to Int128.""" if node is None: node = self.context.node with When(f"I convert {min}, {max}, 1 to {int_type}"): - output = node.query(f"SELECT to{int_type}(\'{min}\'), to{int_type}(\'{max}\'), to{int_type}(1) format TabSeparatedRaw").output - assert output == f'{min}\t{max}\t1', error() + output = node.query( + f"SELECT to{int_type}('{min}'), to{int_type}('{max}'), to{int_type}(1) format TabSeparatedRaw" + ).output + assert output == f"{min}\t{max}\t1", error() + @TestScenario @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_toDecimal256("1.0"), ) def to_decimal256(self, node=None): - """Check that ClickHouse converts values to Int128. - """ + """Check that ClickHouse converts values to Int128.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -178,28 +242,32 @@ def to_decimal256(self, node=None): with When(f"I check toDecimal256 with 0 scale with 1, {max}, and {min}"): - for value in [1,min,max]: - output = node.query(f"SELECT toDecimal256(\'{value}\',0)").output + for value in [1, min, max]: + output = node.query(f"SELECT toDecimal256('{value}',0)").output assert output == str(value), error() - for scale in range(1,76): + for scale in range(1, 76): with When(f"I check toDecimal256 with {scale} scale with its max"): - output = node.query(f"SELECT toDecimal256(\'{10**(76-scale)-1}\',{scale})").output - assert float(output) == float(10**(76-scale)-1), error() + output = node.query( + f"SELECT toDecimal256('{10**(76-scale)-1}',{scale})" + ).output + assert float(output) == float(10 ** (76 - scale) - 1), error() with And(f"I check toDecimal256 with {scale} scale with its min"): - output = node.query(f"SELECT toDecimal256(\'{-10**(76-scale)+1}\',{scale})").output - assert float(output) == float(-10**(76-scale)+1), error() + output = node.query( + f"SELECT toDecimal256('{-10**(76-scale)+1}',{scale})" + ).output + assert float(output) == float(-(10 ** (76 - scale)) + 1), error() + @TestScenario @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_ToMySQL("1.0"), ) def MySQL_table(self, node=None): - """Check that ClickHouse converts MySQL values from MySQL table into ClickHouse table. - """ - table_name = f'table_{getuid()}' + """Check that ClickHouse converts MySQL values from MySQL table into ClickHouse table.""" + table_name = f"table_{getuid()}" node = self.context.node mysql_node = self.context.mysql_node @@ -210,20 +278,22 @@ def MySQL_table(self, node=None): sql = f""" INSERT INTO {table_name}(int128, uint128, int256, uint256, dec256) VALUES (1,1,1,1,1); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with Then("I select from the table on top of the mysql table"): - node.query(f"SELECT * FROM {table_name}", - exitcode=50, message='Exception:') + node.query(f"SELECT * FROM {table_name}", exitcode=50, message="Exception:") + @TestScenario @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_FromMySQL("1.0"), ) def MySQL_func(self, node=None): - """Check that ClickHouse converts MySQL values into a ClickHouse table using the MySQL table function. - """ - table_name = f'table_{getuid()}' + """Check that ClickHouse converts MySQL values into a ClickHouse table using the MySQL table function.""" + table_name = f"table_{getuid()}" node = self.context.node mysql_node = self.context.mysql_node @@ -234,33 +304,38 @@ def MySQL_func(self, node=None): sql = f""" INSERT INTO {table_name}(int128, uint128, int256, uint256, dec256) VALUES (1,1,1,1,1); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with And("I make sure the table doesn't exist"): node.query(f"DROP TABLE IF EXISTS {table_name}") with And("I create the table"): - node.query(f"CREATE TABLE {table_name} (id UInt8, int128 Int128, uint128 UInt128, int256 Int256, uint256 UInt256, dec256 Decimal256(0)) Engine = Memory") + node.query( + f"CREATE TABLE {table_name} (id UInt8, int128 Int128, uint128 UInt128, int256 Int256, uint256 UInt256, dec256 Decimal256(0)) Engine = Memory" + ) with And("I insert into the clickhouse table from the mysql table"): node.query(f"INSERT INTO {table_name} SELECT * FROM {table_function}") with Then("I select from the clickhouse table"): output = node.query(f"SELECT * FROM {table_name}").output - assert output == '1\t1\t1\t1\t1\t1', error() + assert output == "1\t1\t1\t1\t1\t1", error() + @TestScenario @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Conversion_ToMySQL("1.0"), ) def MySQL_dict(self, node=None): - """Check that ClickHouse converts MySQL values from MySQL table into ClickHouse dictionary. - """ + """Check that ClickHouse converts MySQL values from MySQL table into ClickHouse dictionary.""" node = self.context.node mysql_node = self.context.mysql_node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with dictionary(table_name, node, mysql_node): @@ -268,17 +343,21 @@ def MySQL_dict(self, node=None): sql = f""" INSERT INTO {table_name}(int128, uint128, int256, uint256, dec256) VALUES (1,1,1,1,1); """ - mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + mysql_node.command( + f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", + exitcode=0, + ) with Then("I select from the table on top of the mysql table"): - node.query(f"SELECT * FROM dict_{table_name}", - exitcode=50, message='Exception:') + node.query( + f"SELECT * FROM dict_{table_name}", exitcode=50, message="Exception:" + ) + @TestFeature @Name("conversion") def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check the conversion of extended precision data types. - """ + """Check the conversion of extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/feature.py b/tests/testflows/extended_precision_data_types/tests/feature.py index 83293b61f35..dc08ee4adde 100644 --- a/tests/testflows/extended_precision_data_types/tests/feature.py +++ b/tests/testflows/extended_precision_data_types/tests/feature.py @@ -2,11 +2,11 @@ from testflows.core import * from testflows.core.name import basename, parentname from testflows._core.testtype import TestSubType + @TestFeature @Name("tests") def feature(self): - """Check functions with Int128, Int256, UInt256, and Decimal256. - """ + """Check functions with Int128, Int256, UInt256, and Decimal256.""" Feature(run=load("extended_precision_data_types.tests.conversion", "feature")) Feature(run=load("extended_precision_data_types.tests.arithmetic", "feature")) Feature(run=load("extended_precision_data_types.tests.array_tuple_map", "feature")) diff --git a/tests/testflows/extended_precision_data_types/tests/logical.py b/tests/testflows/extended_precision_data_types/tests/logical.py index 18dc33f062e..56ade9c4c3c 100644 --- a/tests/testflows/extended_precision_data_types/tests/logical.py +++ b/tests/testflows/extended_precision_data_types/tests/logical.py @@ -2,54 +2,66 @@ from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * funcs = [ - ('and',), - ('or',), - ('not',), - ('xor',), + ("and",), + ("or",), + ("not",), + ("xor",), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]} - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_list_dec = [ + tuple(list(func) + [Name(f"{func[0]} - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_list_dec = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline(Scenario) -@Examples('func int_type min max', Examples_list) +@Examples("func int_type min max", Examples_list) def log_int_inline(self, func, int_type, min, max, node=None): - """Check logical functions with Int128, Int256, and UInt256 using inline tests. - """ - table_name = f'table_{getuid()}' + """Check logical functions with Int128, Int256, and UInt256 using inline tests.""" + table_name = f"table_{getuid()}" if node is None: node = self.context.node with When(f"Check {func} with {int_type}"): - node.query(f"SELECT {func}(to{int_type}(1), to{int_type}(1)), {func}(to{int_type}(\'{max}\'), to{int_type}(1)), {func}(to{int_type}(\'{min}\'), to{int_type}(1))", - exitcode=43, message = 'Exception: Illegal type ') + node.query( + f"SELECT {func}(to{int_type}(1), to{int_type}(1)), {func}(to{int_type}('{max}'), to{int_type}(1)), {func}(to{int_type}('{min}'), to{int_type}(1))", + exitcode=43, + message="Exception: Illegal type ", + ) + @TestOutline(Scenario) -@Examples('func int_type min max', Examples_list) +@Examples("func int_type min max", Examples_list) def log_int_table(self, func, int_type, min, max, node=None): - """Check logical functions with Int128, Int256, and UInt256 using table tests. - """ + """Check logical functions with Int128, Int256, and UInt256 using table tests.""" if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = int_type) + table(name=table_name, data_type=int_type) for value in [1, min, max]: with When(f"Check {func} with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(\'{value}\'), to{int_type}(\'{value}\'))", - exitcode=43, message = 'Exception: Illegal type') + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}('{value}'), to{int_type}('{value}'))", + exitcode=43, + message="Exception: Illegal type", + ) + @TestOutline(Scenario) -@Examples('func', funcs) +@Examples("func", funcs) def log_dec_inline(self, func, node=None): - """Check logical functions with Decimal256 using inline tests. - """ + """Check logical functions with Decimal256 using inline tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -57,30 +69,37 @@ def log_dec_inline(self, func, node=None): node = self.context.node with When(f"Check {func} with Decimal256"): - node.query(f"SELECT {func}(toDecimal256(1,0), toDecimal256(1,0)), {func}(toDecimal256(\'{max}\',0), toDecimal256(1)), {func}(toDecimal256(\'{min}\',0), toDecimal256(1))", - exitcode=43, message = 'Exception: Illegal type ') + node.query( + f"SELECT {func}(toDecimal256(1,0), toDecimal256(1,0)), {func}(toDecimal256('{max}',0), toDecimal256(1)), {func}(toDecimal256('{min}',0), toDecimal256(1))", + exitcode=43, + message="Exception: Illegal type ", + ) + @TestOutline(Scenario) -@Examples('func', funcs) +@Examples("func", funcs) def log_dec_table(self, func, node=None): - """Check logical functions with Decimal256 using table tests. - """ + """Check logical functions with Decimal256 using table tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = 'Decimal256(0)') + table(name=table_name, data_type="Decimal256(0)") for value in [1, min, max]: with When(f"Check {func} with Decimal256 and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func}(toDecimal256(\'{value}\',0), toDecimal256(\'{value}\',0))", - exitcode=43, message = 'Exception: Illegal type ') + node.query( + f"INSERT INTO {table_name} SELECT {func}(toDecimal256('{value}',0), toDecimal256('{value}',0))", + exitcode=43, + message="Exception: Illegal type ", + ) + @TestFeature @Name("logical") @@ -88,8 +107,7 @@ def log_dec_table(self, func, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Logical("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that comparison functions work with extended precision data types. - """ + """Check that comparison functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/mathematical.py b/tests/testflows/extended_precision_data_types/tests/mathematical.py index 65872b766dd..612db532944 100644 --- a/tests/testflows/extended_precision_data_types/tests/mathematical.py +++ b/tests/testflows/extended_precision_data_types/tests/mathematical.py @@ -2,171 +2,214 @@ from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * funcs = [ - ('exp(', 3, 0), - ('log(', 0, 0), - ('ln(', 0, 0), - ('exp2(', 2, 0), - ('log2(', 0, 0), - ('exp10(', 10, 0), - ('log10(', 0, 0), - ('sqrt(', 1, 0), - ('cbrt(', 1, 0), - ('erf(', 1, 0), - ('erfc(', 0, 0), - ('lgamma(', 0, 0), - ('tgamma(', 1, 0), - ('sin(', 1, 0), - ('cos(', 1, 0), - ('tan(', 2, 0), - ('asin(', 2, 0), - ('acos(', 0, 0), - ('atan(', 1, 0), - ('intExp2(', 2, 48), - ('intExp10(', 10, 48), - ('cosh(', 2, 0), - ('acosh(', 0, 0), - ('sinh(', 1, 0), - ('asinh(', 1, 0), - ('tanh(', 1, 0), - ('atanh(', 'inf', 0), - ('log1p(', 1, 0), - ('sign(', 1, 0), - ('pow(1,', 1, 43), - ('power(1,', 1, 43), - ('atan2(1,', 1, 43), - ('hypot(1,', 1, 43), + ("exp(", 3, 0), + ("log(", 0, 0), + ("ln(", 0, 0), + ("exp2(", 2, 0), + ("log2(", 0, 0), + ("exp10(", 10, 0), + ("log10(", 0, 0), + ("sqrt(", 1, 0), + ("cbrt(", 1, 0), + ("erf(", 1, 0), + ("erfc(", 0, 0), + ("lgamma(", 0, 0), + ("tgamma(", 1, 0), + ("sin(", 1, 0), + ("cos(", 1, 0), + ("tan(", 2, 0), + ("asin(", 2, 0), + ("acos(", 0, 0), + ("atan(", 1, 0), + ("intExp2(", 2, 48), + ("intExp10(", 10, 48), + ("cosh(", 2, 0), + ("acosh(", 0, 0), + ("sinh(", 1, 0), + ("asinh(", 1, 0), + ("tanh(", 1, 0), + ("atanh(", "inf", 0), + ("log1p(", 1, 0), + ("sign(", 1, 0), + ("pow(1,", 1, 43), + ("power(1,", 1, 43), + ("atan2(1,", 1, 43), + ("hypot(1,", 1, 43), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]}) - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_dec_list = [ + tuple(list(func) + [Name(f"{func[0]}) - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]}) - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_dec_list = [tuple(list(func)+[Name(f'{func[0]}) - Decimal256')]) for func in funcs] @TestOutline(Scenario) -@Examples('func expected_result exitcode int_type min max', Examples_list) -def math_int_inline(self, func, expected_result, exitcode, int_type, min, max, node=None): - """Check mathematical functions with Int128, UInt128, Int256, and UInt256 using inline tests. - """ +@Examples("func expected_result exitcode int_type min max", Examples_list) +def math_int_inline( + self, func, expected_result, exitcode, int_type, min, max, node=None +): + """Check mathematical functions with Int128, UInt128, Int256, and UInt256 using inline tests.""" if node is None: node = self.context.node - if func in ['intExp2(', 'intExp10(', 'pow(1,', 'power(1,', 'atan2(1,', 'hypot(1,']: + if func in ["intExp2(", "intExp10(", "pow(1,", "power(1,", "atan2(1,", "hypot(1,"]: with When(f"I check {func} with {int_type} using 1, max, and min"): - node.query(f"SELECT {func} to{int_type}(1)), {func} to{int_type}(\'{max}\')), {func} to{int_type}(\'{min}\'))", - exitcode=exitcode, message = 'Exception:') + node.query( + f"SELECT {func} to{int_type}(1)), {func} to{int_type}('{max}')), {func} to{int_type}('{min}'))", + exitcode=exitcode, + message="Exception:", + ) else: with When(f"I check {func} with {int_type} using 1"): output = node.query(f"SELECT {func} to{int_type}(1))").output - if output == 'inf': + if output == "inf": pass else: assert round(float(output)) == expected_result, error() with And(f"I check {func} with {int_type} using max and min"): - execute_query(f""" + execute_query( + f""" SELECT round({func} to{int_type}(\'{max}\')), {rounding_precision}), round({func} to{int_type}(\'{min}\')), {rounding_precision}) - """) + """ + ) + @TestOutline(Scenario) -@Examples('func expected_result exitcode int_type min max', Examples_list) -def math_int_table(self, func, expected_result, exitcode, int_type, min, max, node=None): - """Check mathematical functions with Int128, UInt128, Int256, and UInt256 using table tests. - """ +@Examples("func expected_result exitcode int_type min max", Examples_list) +def math_int_table( + self, func, expected_result, exitcode, int_type, min, max, node=None +): + """Check mathematical functions with Int128, UInt128, Int256, and UInt256 using table tests.""" if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = f'Nullable({int_type})') + table(name=table_name, data_type=f"Nullable({int_type})") - if func in ['intExp2(', 'intExp10(', 'pow(1,', 'power(1,', 'atan2(1,', 'hypot(1,']: + if func in ["intExp2(", "intExp10(", "pow(1,", "power(1,", "atan2(1,", "hypot(1,"]: for value in [1, max, min]: - with When(f"I insert the output of {func} with {int_type} using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT {func} to{int_type}(\'{value}\'))", - exitcode=exitcode, message = 'Exception:') + with When( + f"I insert the output of {func} with {int_type} using {value} into a table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func} to{int_type}('{value}'))", + exitcode=exitcode, + message="Exception:", + ) else: for value in [1, max, min]: - with And(f"I insert the output of {func} with {int_type} using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT round(to{int_type}OrZero( toString({func} to{int_type}(\'{value}\')))), {rounding_precision})") + with And( + f"I insert the output of {func} with {int_type} using {value} into a table" + ): + node.query( + f"INSERT INTO {table_name} SELECT round(to{int_type}OrZero( toString({func} to{int_type}('{value}')))), {rounding_precision})" + ) with Then(f"I check the outputs of {func} with {int_type}"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestOutline(Scenario) -@Examples('func expected_result exitcode', Examples_dec_list) +@Examples("func expected_result exitcode", Examples_dec_list) def math_dec_inline(self, func, expected_result, exitcode, node=None): - """Check mathematical functions with Decimal256 using inline tests. - """ + """Check mathematical functions with Decimal256 using inline tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] if node is None: node = self.context.node - if func in ['intExp2(', 'intExp10(', 'pow(1,', 'power(1,', 'atan2(1,', 'hypot(1,']: + if func in ["intExp2(", "intExp10(", "pow(1,", "power(1,", "atan2(1,", "hypot(1,"]: with When(f"I check {func} with Decimal256 using 1, max, and min"): - node.query(f"SELECT {func} toDecimal256(1,0)), {func} toDecimal256(\'{max}\',0)), {func} toDecimal256(\'{min}\',0))", - exitcode=43, message = 'Exception: Illegal type ') + node.query( + f"SELECT {func} toDecimal256(1,0)), {func} toDecimal256('{max}',0)), {func} toDecimal256('{min}',0))", + exitcode=43, + message="Exception: Illegal type ", + ) else: with When(f"I check {func} with Decimal256 using 1"): output = node.query(f"SELECT {func} toDecimal256(1,0))").output - if output == 'inf': + if output == "inf": pass else: assert round(float(output)) == expected_result, error() with And(f"I check {func} with Decimal256 using max and min"): - execute_query(f""" + execute_query( + f""" SELECT round({func} toDecimal256(\'{max}\',0)),{rounding_precision}), round({func} toDecimal256(\'{min}\',0)),{rounding_precision}) - """) + """ + ) + @TestOutline(Scenario) -@Examples('func expected_result exitcode', Examples_dec_list) +@Examples("func expected_result exitcode", Examples_dec_list) def math_dec_table(self, func, expected_result, exitcode, node=None): - """Check mathematical functions with Decimal256 using table tests. - """ + """Check mathematical functions with Decimal256 using table tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] if node is None: node = self.context.node - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given(f"I have a table"): - table(name = table_name, data_type = 'Decimal256(0)') + table(name=table_name, data_type="Decimal256(0)") - if func in ['intExp2(', 'intExp10(', 'pow(1,', 'power(1,', 'atan2(1,', 'hypot(1,']: + if func in ["intExp2(", "intExp10(", "pow(1,", "power(1,", "atan2(1,", "hypot(1,"]: for value in [1, max, min]: - with When(f"I insert the output of {func} with Decimal256 using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT {func} toDecimal256(\'{value}\',0))", - exitcode=43, message = 'Exception: Illegal type ') + with When( + f"I insert the output of {func} with Decimal256 using {value} into a table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func} toDecimal256('{value}',0))", + exitcode=43, + message="Exception: Illegal type ", + ) else: for value in [1, max, min]: - with When(f"I insert the output of {func} with Decimal256 using {value} into a table"): - node.query(f"INSERT INTO {table_name} SELECT round(toDecimal256OrZero( toString({func} toDecimal256(\'{value}\',0))),0), 7)") + with When( + f"I insert the output of {func} with Decimal256 using {value} into a table" + ): + node.query( + f"INSERT INTO {table_name} SELECT round(toDecimal256OrZero( toString({func} toDecimal256('{value}',0))),0), 7)" + ) with Then(f"I check the outputs of {func} with Decimal256"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestFeature @Name("mathematical") @@ -175,8 +218,7 @@ def math_dec_table(self, func, expected_result, exitcode, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Mathematical_NotSupported("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that mathematical functions work with extended precision data types. - """ + """Check that mathematical functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/null.py b/tests/testflows/extended_precision_data_types/tests/null.py index f9b93f874bc..2d9f3cedef5 100644 --- a/tests/testflows/extended_precision_data_types/tests/null.py +++ b/tests/testflows/extended_precision_data_types/tests/null.py @@ -2,23 +2,29 @@ from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * funcs = [ - ('isNull(', 0), - ('isNotNull(', 1), - ('coalesce(', 1), - ('assumeNotNull(', 1), - ('toNullable(', 1), - ('ifNull(1,', 1), - ('nullIf(1,', '\\N'), + ("isNull(", 0), + ("isNotNull(", 1), + ("coalesce(", 1), + ("assumeNotNull(", 1), + ("toNullable(", 1), + ("ifNull(1,", 1), + ("nullIf(1,", "\\N"), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]}) - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_list_dec = [ + tuple(list(func) + [Name(f"{func[0]}) - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]}) - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_list_dec = [tuple(list(func)+[Name(f'{func[0]}) - Decimal256')]) for func in funcs] @TestOutline(Scenario) -@Examples('func expected_result int_type min max', Examples_list) +@Examples("func expected_result int_type min max", Examples_list) def null_int_inline(self, func, expected_result, int_type, min, max, node=None): - """Check null function with Int128, UInt128, Int256, and UInt256 using inline tests. - """ + """Check null function with Int128, UInt128, Int256, and UInt256 using inline tests.""" if node is None: node = self.context.node @@ -28,15 +34,17 @@ def null_int_inline(self, func, expected_result, int_type, min, max, node=None): assert output == str(expected_result), error() with And(f"I check {func} with {int_type} using min and max"): - execute_query(f""" + execute_query( + f""" SELECT {func} to{int_type}(\'{min}\')), {func} to{int_type}(\'{max}\')) - """) + """ + ) + @TestOutline(Scenario) -@Examples('func expected_result int_type min max', Examples_list) +@Examples("func expected_result int_type min max", Examples_list) def null_int_table(self, func, expected_result, int_type, min, max, node=None): - """Check null function with Int128, UInt128, Int256, and UInt256 using table tests. - """ + """Check null function with Int128, UInt128, Int256, and UInt256 using table tests.""" table_name = f"table_{getuid()}" @@ -44,23 +52,27 @@ def null_int_table(self, func, expected_result, int_type, min, max, node=None): node = self.context.node with Given("I have a table"): - table(name = table_name, data_type = f'Nullable({int_type})') + table(name=table_name, data_type=f"Nullable({int_type})") for value in [1, min, max]: with When(f"I insert the output of {func} with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func} to{int_type}(\'{value}\'))") + node.query( + f"INSERT INTO {table_name} SELECT {func} to{int_type}('{value}'))" + ) with Then(f"I check {func} with {int_type} on the table"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestOutline(Scenario) -@Examples('func expected_result', Examples_list_dec) +@Examples("func expected_result", Examples_list_dec) def null_dec_inline(self, func, expected_result, node=None): - """Check null function with Decimal256 using inline tests. - """ + """Check null function with Decimal256 using inline tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -72,15 +84,17 @@ def null_dec_inline(self, func, expected_result, node=None): assert output == str(expected_result), error() with And(f"I check {func} with Decimal256 using min and max"): - execute_query(f""" + execute_query( + f""" SELECT {func} toDecimal256(\'{min}\',0)), {func} toDecimal256(\'{max}\',0)) - """) + """ + ) + @TestOutline(Scenario) -@Examples('func expected_result', Examples_list_dec) +@Examples("func expected_result", Examples_list_dec) def null_dec_table(self, func, expected_result, node=None): - """Check null function with Decimal256 using table tests. - """ + """Check null function with Decimal256 using table tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -90,17 +104,22 @@ def null_dec_table(self, func, expected_result, node=None): node = self.context.node with Given("I have a table"): - table(name = table_name, data_type = 'Nullable(Decimal256(0))') + table(name=table_name, data_type="Nullable(Decimal256(0))") for value in [1, min, max]: with When(f"I insert the output of {func} with Decimal256 and {value}"): - node.query(f"INSERT INTO {table_name} SELECT {func} toDecimal256(\'{value}\',0))") + node.query( + f"INSERT INTO {table_name} SELECT {func} toDecimal256('{value}',0))" + ) with Then(f"I check {func} with Decimal256 on the table"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) + @TestFeature @Name("null") @@ -108,8 +127,7 @@ def null_dec_table(self, func, expected_result, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Null("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that null functions work with extended precision data types. - """ + """Check that null functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/rounding.py b/tests/testflows/extended_precision_data_types/tests/rounding.py index e32f4e941d3..489c545187c 100644 --- a/tests/testflows/extended_precision_data_types/tests/rounding.py +++ b/tests/testflows/extended_precision_data_types/tests/rounding.py @@ -2,34 +2,45 @@ from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * funcs = [ - ('ceil', 1, True), - ('floor', 1, True), - ('trunc', 1, True), - ('round', 1, True), - ('roundBankers', 1, True), - ('roundToExp2', 1, False), - ('roundDuration', 1, True), - ('roundAge', 17, True), - ('roundDown', 1, False) + ("ceil", 1, True), + ("floor", 1, True), + ("trunc", 1, True), + ("round", 1, True), + ("roundBankers", 1, True), + ("roundToExp2", 1, False), + ("roundDuration", 1, True), + ("roundAge", 17, True), + ("roundDown", 1, False), +] + +Examples_list = [ + tuple(list(func) + list(data_type) + [Name(f"{func[0]} - {data_type[0]}")]) + for func in funcs + for data_type in data_types +] +Examples_dec_list = [ + tuple(list(func) + [Name(f"{func[0]} - Decimal256")]) for func in funcs ] -Examples_list = [tuple(list(func)+list(data_type)+[Name(f'{func[0]} - {data_type[0]}')]) for func in funcs for data_type in data_types] -Examples_dec_list = [tuple(list(func)+[Name(f'{func[0]} - Decimal256')]) for func in funcs] @TestOutline(Scenario) -@Examples('func expected_result supported int_type min max', Examples_list) -def round_int_inline(self, func, expected_result, supported, int_type, min, max, node=None): - """Check rounding functions with Int128, UInt128, Int256, and UInt256 using inline tests. - """ +@Examples("func expected_result supported int_type min max", Examples_list) +def round_int_inline( + self, func, expected_result, supported, int_type, min, max, node=None +): + """Check rounding functions with Int128, UInt128, Int256, and UInt256 using inline tests.""" if node is None: node = self.context.node - if func == 'roundDown': + if func == "roundDown": with When(f"I check roundDown with {int_type}"): - node.query(f"SELECT roundDown(to{int_type}(1), [0,2]), roundDown(to{int_type}(\'{max}\'), [0,2]), roundDown(to{int_type}(\'{min}\'), [0,2])", - exitcode=44, message=f'Exception: Illegal column {int_type} of first argument of function roundDown') + node.query( + f"SELECT roundDown(to{int_type}(1), [0,2]), roundDown(to{int_type}('{max}'), [0,2]), roundDown(to{int_type}('{min}'), [0,2])", + exitcode=44, + message=f"Exception: Illegal column {int_type} of first argument of function roundDown", + ) elif supported: @@ -37,22 +48,29 @@ def round_int_inline(self, func, expected_result, supported, int_type, min, max, output = node.query(f"SELECT {func}(to{int_type}(1))").output assert output == str(expected_result), error() - with And(f'I check {func} with {int_type} using min and max values'): - execute_query(f""" + with And(f"I check {func} with {int_type} using min and max values"): + execute_query( + f""" SELECT {func}(to{int_type}(\'{min}\')), {func}(to{int_type}(\'{max}\')) - """) + """ + ) else: with When(f"I check {func} with {int_type}"): - node.query(f"SELECT {func}(to{int_type}(1)), {func}(to{int_type}(\'{max}\')), {func}(to{int_type}(\'{min}\'))", - exitcode=48, message=f'Exception: {func}() for big integers is not implemented:') + node.query( + f"SELECT {func}(to{int_type}(1)), {func}(to{int_type}('{max}')), {func}(to{int_type}('{min}'))", + exitcode=48, + message=f"Exception: {func}() for big integers is not implemented:", + ) + @TestOutline(Scenario) -@Examples('func expected_result supported int_type min max', Examples_list) -def round_int_table(self, func, expected_result, supported, int_type, min, max, node=None): - """Check rounding functions with Int128, UInt128, Int256, and UInt256 using table tests. - """ +@Examples("func expected_result supported int_type min max", Examples_list) +def round_int_table( + self, func, expected_result, supported, int_type, min, max, node=None +): + """Check rounding functions with Int128, UInt128, Int256, and UInt256 using table tests.""" table_name = f"table_{getuid()}" @@ -60,77 +78,99 @@ def round_int_table(self, func, expected_result, supported, int_type, min, max, node = self.context.node with Given("I have a table"): - table(name = table_name, data_type = int_type) + table(name=table_name, data_type=int_type) - if func == 'roundDown': + if func == "roundDown": - for value in [1,max,min]: + for value in [1, max, min]: with When(f"I check roundDown with {int_type} and {value}"): - node.query(f"INSERT INTO {table_name} SELECT roundDown(to{int_type}(\'{value}\'), [0,2])", - exitcode=44, message=f'Exception: Illegal column {int_type} of first argument of function roundDown') + node.query( + f"INSERT INTO {table_name} SELECT roundDown(to{int_type}('{value}'), [0,2])", + exitcode=44, + message=f"Exception: Illegal column {int_type} of first argument of function roundDown", + ) elif supported: - for value in [1,max,min]: + for value in [1, max, min]: - with When(f"I insert the output of {func} with {int_type} and {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(\'{value}\'))") + with When( + f"I insert the output of {func} with {int_type} and {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}('{value}'))" + ) with Then(f"I select the output of {func} with {int_type} from the table"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) else: - for value in [1,max,min]: + for value in [1, max, min]: + + with When( + f"I insert the output of {func} with {int_type} and {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func}(to{int_type}(1))", + exitcode=48, + message=f"Exception: {func}() for big integers is not implemented:", + ) - with When(f"I insert the output of {func} with {int_type} and {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(to{int_type}(1))", - exitcode=48, message=f'Exception: {func}() for big integers is not implemented:') @TestOutline(Scenario) -@Examples('func expected_result supported', Examples_dec_list) +@Examples("func expected_result supported", Examples_dec_list) def round_dec_inline(self, func, expected_result, supported, node=None): - """Check rounding functions with Decimal256 using inline tests. - """ + """Check rounding functions with Decimal256 using inline tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] if node is None: node = self.context.node - if func == 'roundDown': + if func == "roundDown": with When(f"I check roundDown with Decimal256"): - node.query(f"""SELECT roundDown(toDecimal256(1,0), [toDecimal256(0,0),toDecimal256(2,0)]), + node.query( + f"""SELECT roundDown(toDecimal256(1,0), [toDecimal256(0,0),toDecimal256(2,0)]), roundDown(toDecimal256(\'{max}\',0), [toDecimal256(0,0),toDecimal256(2,0)]), roundDown(toDecimal256(\'{min}\',0), [toDecimal256(0,0),toDecimal256(2,0)])""", - exitcode=44, message=f'Exception: Illegal column Decimal256 of first argument of function roundDown') + exitcode=44, + message=f"Exception: Illegal column Decimal256 of first argument of function roundDown", + ) - elif func not in ['roundDuration', 'roundAge', 'roundToExp2']: + elif func not in ["roundDuration", "roundAge", "roundToExp2"]: with When(f"I check {func} with Decimal256"): output = node.query(f"SELECT {func}(toDecimal256(1,0))").output assert output == str(expected_result), error() - with And(f'I check {func} with Decimal256 using min and max values'): - execute_query(f""" + with And(f"I check {func} with Decimal256 using min and max values"): + execute_query( + f""" SELECT {func}(toDecimal256(\'{min}\',0)), {func}(toDecimal256(\'{max}\',0)) - """) + """ + ) else: with When(f"I check {func} with Decimal256"): - node.query(f"SELECT {func}(toDecimal256(1,0)), {func}(toDecimal256(\'{max}\',0)), {func}(toDecimal256(\'{min}\',0))", - exitcode=43, message=f'Exception: Illegal type Decimal(76, 0)') + node.query( + f"SELECT {func}(toDecimal256(1,0)), {func}(toDecimal256('{max}',0)), {func}(toDecimal256('{min}',0))", + exitcode=43, + message=f"Exception: Illegal type Decimal(76, 0)", + ) + @TestOutline(Scenario) -@Examples('func expected_result supported', Examples_dec_list) +@Examples("func expected_result supported", Examples_dec_list) def round_dec_table(self, func, expected_result, supported, node=None): - """Check rounding functions with Decimal256 using table tests. - """ + """Check rounding functions with Decimal256 using table tests.""" min = Decimal256_min_max[0] max = Decimal256_min_max[1] @@ -140,35 +180,50 @@ def round_dec_table(self, func, expected_result, supported, node=None): node = self.context.node with Given("I have a table"): - table(name = table_name, data_type = 'Decimal256(0)') + table(name=table_name, data_type="Decimal256(0)") - if func == 'roundDown': + if func == "roundDown": for value in [1, max, min]: with When(f"I check roundDown with Decimal256 and {value}"): - node.query(f"INSERT INTO {table_name} SELECT roundDown(toDecimal256(\'{value}\',0), [toDecimal256(0,0),toDecimal256(2,0)])", - exitcode=44, message=f'Exception: Illegal column Decimal256 of first argument of function roundDown') + node.query( + f"INSERT INTO {table_name} SELECT roundDown(toDecimal256('{value}',0), [toDecimal256(0,0),toDecimal256(2,0)])", + exitcode=44, + message=f"Exception: Illegal column Decimal256 of first argument of function roundDown", + ) - elif func not in ['roundDuration', 'roundAge', 'roundToExp2']: + elif func not in ["roundDuration", "roundAge", "roundToExp2"]: for value in [1, max, min]: - with When(f"I insert the output of {func} with Decimal256 and {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(toDecimal256(\'{value}\',0))") + with When( + f"I insert the output of {func} with Decimal256 and {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func}(toDecimal256('{value}',0))" + ) with Then(f"I select the output of {func} with Decimal256 from the table"): - execute_query(f""" + execute_query( + f""" SELECT * FROM {table_name} ORDER BY a ASC - """) + """ + ) else: for value in [1, max, min]: - with When(f"I insert the output of {func} with Decimal256 and {value} into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(toDecimal256(\'{value}\',0))", - exitcode=43, message=f'Exception: Illegal type Decimal(76, 0)') + with When( + f"I insert the output of {func} with Decimal256 and {value} into the table" + ): + node.query( + f"INSERT INTO {table_name} SELECT {func}(toDecimal256('{value}',0))", + exitcode=43, + message=f"Exception: Illegal type Decimal(76, 0)", + ) + @TestFeature @Name("rounding") @@ -179,8 +234,7 @@ def round_dec_table(self, func, expected_result, supported, node=None): RQ_SRS_020_ClickHouse_Extended_Precision_Rounding_Dec_NotSupported("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that rounding functions work with extended precision data types. - """ + """Check that rounding functions work with extended precision data types.""" self.context.node = self.context.cluster.node(node) self.context.mysql_node = self.context.cluster.node(mysql_node) diff --git a/tests/testflows/extended_precision_data_types/tests/table.py b/tests/testflows/extended_precision_data_types/tests/table.py index 1548d6b20c2..58ec41f8e82 100644 --- a/tests/testflows/extended_precision_data_types/tests/table.py +++ b/tests/testflows/extended_precision_data_types/tests/table.py @@ -5,14 +5,14 @@ from contextlib import contextmanager from extended_precision_data_types.requirements import * from extended_precision_data_types.common import * + @TestFeature @Name("table") @Requirements( RQ_SRS_020_ClickHouse_Extended_Precision_Create_Table("1.0"), ) def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel=None): - """Check that clickhouse is able to create a table with extended precision data types. - """ + """Check that clickhouse is able to create a table with extended precision data types.""" node = self.context.cluster.node(node) table_name = f"table_{getuid()}" @@ -20,15 +20,21 @@ def feature(self, node="clickhouse1", mysql_node="mysql1", stress=None, parallel with allow_experimental_bigint(node): try: - with When("I create a table with Int128, UInt128, Int256, UInt256, Decimal256"): - node.query(f"CREATE TABLE {table_name}(a Int128, b UInt128, c Int256, d UInt256, e Decimal256(0)) ENGINE = Memory") + with When( + "I create a table with Int128, UInt128, Int256, UInt256, Decimal256" + ): + node.query( + f"CREATE TABLE {table_name}(a Int128, b UInt128, c Int256, d UInt256, e Decimal256(0)) ENGINE = Memory" + ) with And("I insert values into the table"): - node.query(f"INSERT INTO {table_name} VALUES (toInt128(1), toUInt128(1), toInt256(1), toUInt256(1), toDecimal256(1,0))") + node.query( + f"INSERT INTO {table_name} VALUES (toInt128(1), toUInt128(1), toInt256(1), toUInt256(1), toDecimal256(1,0))" + ) with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert output == '1\t1\t1\t1\t1', error() + assert output == "1\t1\t1\t1\t1", error() finally: with Finally("I drop the table"): diff --git a/tests/testflows/helpers/argparser.py b/tests/testflows/helpers/argparser.py index 63012601e3b..ec26b8f654b 100644 --- a/tests/testflows/helpers/argparser.py +++ b/tests/testflows/helpers/argparser.py @@ -1,16 +1,36 @@ import os + def argparser(parser): - """Default argument parser for regressions. - """ - parser.add_argument("--local", + """Default argument parser for regressions.""" + parser.add_argument( + "--local", action="store_true", - help="run regression in local mode", default=False) + help="run regression in local mode", + default=False, + ) - parser.add_argument("--clickhouse-binary-path", - type=str, dest="clickhouse_binary_path", - help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path", - default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) + parser.add_argument( + "--clickhouse-version", + type=str, + dest="clickhouse_version", + help="clickhouse server version", + metavar="version", + default=os.getenv("CLICKHOUSE_TESTS_SERVER_VERSION", None), + ) - parser.add_argument("--stress", action="store_true", default=False, - help="enable stress testing (might take a long time)") + parser.add_argument( + "--clickhouse-binary-path", + type=str, + dest="clickhouse_binary_path", + help="path to ClickHouse binary, default: /usr/bin/clickhouse", + metavar="path", + default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse"), + ) + + parser.add_argument( + "--stress", + action="store_true", + default=False, + help="enable stress testing (might take a long time)", + ) diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 5b987c1e376..ae9f9d6623e 100755 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -1,9 +1,12 @@ import os +import uuid import time import inspect import threading import tempfile +from testflows._core.cli.arg.common import description + import testflows.settings as settings from testflows.core import * @@ -12,6 +15,22 @@ from testflows.connect import Shell as ShellBase from testflows.uexpect import ExpectTimeoutError from testflows._core.testtype import TestSubType +MESSAGES_TO_RETRY = [ + "DB::Exception: ZooKeeper session has been expired", + "DB::Exception: Connection loss", + "Coordination::Exception: Session expired", + "Coordination::Exception: Connection loss", + "Coordination::Exception: Operation timeout", + "DB::Exception: Operation timeout", + "Operation timed out", + "ConnectionPoolWithFailover: Connection failed at try", + "DB::Exception: New table appeared in database being dropped or detached. Try again", + "is already started to be removing by another replica right now", + "Shutdown is called for table", # happens in SYSTEM SYNC REPLICA query if session with ZooKeeper is being reinitialized. + "is executing longer than distributed_ddl_task_timeout", # distributed TTL timeout message +] + + class Shell(ShellBase): def __exit__(self, type, value, traceback): # send exit and Ctrl-D repeatedly @@ -24,20 +43,22 @@ class Shell(ShellBase): for i in range(10): if self.child is not None: try: - self.send('exit\r', eol='') - self.send('\x04\r', eol='') + self.send("exit\r", eol="") + self.send("\x04\r", eol="") except OSError: pass return super(Shell, self).__exit__(type, value, traceback) + class QueryRuntimeException(Exception): - """Exception during query execution on the server. - """ + """Exception during query execution on the server.""" + pass + class Node(object): - """Generic cluster node. - """ + """Generic cluster node.""" + config_d_dir = "/etc/clickhouse-server/config.d/" def __init__(self, cluster, name): @@ -48,47 +69,69 @@ class Node(object): return f"Node(name='{self.name}')" def close_bashes(self): - """Close all active bashes to the node. - """ + """Close all active bashes to the node.""" with self.cluster.lock: for key in list(self.cluster._bash.keys()): if key.endswith(f"-{self.name}"): shell = self.cluster._bash.pop(key) shell.__exit__(None, None, None) - def restart(self, timeout=300, retries=5, safe=True): - """Restart node. - """ + def wait_healthy(self, timeout=300): + with By(f"waiting until container {self.name} is healthy"): + for attempt in retries(timeout=timeout, delay=1): + with attempt: + if self.command("echo 1", no_checks=1, steps=False).exitcode != 0: + fail("container is not healthy") + + def restart(self, timeout=300, retry_count=5, safe=True): + """Restart node.""" + self.close_bashes() + retry(self.cluster.command, retry_count)( + None, + f"{self.cluster.docker_compose} restart {self.name}", + timeout=timeout, + exitcode=0, + steps=False, + ) + + def start(self, timeout=300, retry_count=5): + """Start node.""" + retry(self.cluster.command, retry_count)( + None, + f"{self.cluster.docker_compose} start {self.name}", + timeout=timeout, + exitcode=0, + steps=False, + ) + + def stop(self, timeout=300, retry_count=5, safe=True): + """Stop node.""" self.close_bashes() - for retry in range(retries): - r = self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) - if r.exitcode == 0: - break - - def start(self, timeout=300, retries=5): - """Start node. - """ - for retry in range(retries): - r = self.cluster.command(None, f'{self.cluster.docker_compose} start {self.name}', timeout=timeout) - if r.exitcode == 0: - break - - def stop(self, timeout=300, retries=5, safe=True): - """Stop node. - """ - self.close_bashes() - - for retry in range(retries): - r = self.cluster.command(None, f'{self.cluster.docker_compose} stop {self.name}', timeout=timeout) - if r.exitcode == 0: - break + retry(self.cluster.command, retry_count)( + None, + f"{self.cluster.docker_compose} stop {self.name}", + timeout=timeout, + exitcode=0, + steps=False, + ) def command(self, *args, **kwargs): return self.cluster.command(self.name, *args, **kwargs) - def cmd(self, cmd, message=None, exitcode=None, steps=True, shell_command="bash --noediting", no_checks=False, - raise_on_exception=False, step=By, *args, **kwargs): + def cmd( + self, + cmd, + message=None, + exitcode=None, + steps=True, + shell_command="bash --noediting", + no_checks=False, + raise_on_exception=False, + step=By, + *args, + **kwargs, + ): """Execute and check command. :param cmd: command :param message: expected message that should be in the output, default: None @@ -96,9 +139,13 @@ class Node(object): """ command = f"{cmd}" - with Step("executing command", description=command, format_description=False) if steps else NullStep(): + with step( + "executing command", description=command, format_description=False + ) if steps else NullStep(): try: - r = self.cluster.bash(self.name, command=shell_command)(command, *args, **kwargs) + r = self.cluster.bash(self.name, command=shell_command)( + command, *args, **kwargs + ) except ExpectTimeoutError: self.cluster.close_bash(self.name) raise @@ -111,37 +158,90 @@ class Node(object): assert r.exitcode == exitcode, error(r.output) if message is not None: - with Then(f"output should contain message", description=message) if steps else NullStep(): + with Then( + f"output should contain message", description=message + ) if steps else NullStep(): assert message in r.output, error(r.output) - if message is None or "Exception:" not in message: - with Then("check if output has exception") if steps else NullStep(): - if "Exception:" in r.output: - if raise_on_exception: - raise QueryRuntimeException(r.output) - assert False, error(r.output) - return r - class ClickHouseNode(Node): - """Node with ClickHouse server. - """ - def wait_healthy(self, timeout=300): - with By(f"waiting until container {self.name} is healthy"): - start_time = time.time() - while True: - if self.query("select 1", no_checks=1, timeout=300, steps=False).exitcode == 0: - break - if time.time() - start_time < timeout: - time.sleep(2) - continue - assert False, "container is not healthy" + """Node with ClickHouse server.""" - def stop(self, timeout=300, safe=True, retries=5): - """Stop node. + def thread_fuzzer(self): + with Given("exporting THREAD_FUZZER"): + self.command("export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000") + self.command("export THREAD_FUZZER_SLEEP_PROBABILITY=0.1") + self.command("export THREAD_FUZZER_SLEEP_TIME_US=100000") + + self.command( + "export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_MIGRATE_PROBABILITY=1" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_unlock_AFTER_MIGRATE_PROBABILITY=1" + ) + + self.command( + "export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000" + ) + self.command( + "export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000" + ) + + def wait_clickhouse_healthy(self, timeout=300): + with By(f"waiting until ClickHouse server on {self.name} is healthy"): + for attempt in retries(timeout=timeout, delay=1): + with attempt: + if ( + self.query( + "SELECT version()", no_checks=1, steps=False + ).exitcode + != 0 + ): + fail("ClickHouse server is not healthy") + node_version = self.query( + "SELECT version()", no_checks=1, steps=False + ).output + if current().context.clickhouse_version is None: + current().context.clickhouse_version = node_version + else: + assert current().context.clickhouse_version == node_version, error() + + def clickhouse_pid(self): + """Return ClickHouse server pid if present + otherwise return None. """ + if self.command("ls /tmp/clickhouse-server.pid").exitcode == 0: + return self.command("cat /tmp/clickhouse-server.pid").output.strip() + return None + + def stop_clickhouse(self, timeout=300, safe=True): + """Stop ClickHouse server.""" if safe: self.query("SYSTEM STOP MOVES") self.query("SYSTEM STOP MERGES") @@ -149,89 +249,364 @@ class ClickHouseNode(Node): with By("waiting for 5 sec for moves and merges to stop"): time.sleep(5) with And("forcing to sync everything to disk"): - self.command("sync", timeout=300) + self.command("sync", timeout=300, exitcode=0) - self.close_bashes() + with By(f"sending kill -TERM to ClickHouse server process on {self.name}"): + pid = self.clickhouse_pid() + self.command(f"kill -TERM {pid}", exitcode=0, steps=False) - for retry in range(retries): - r = self.cluster.command(None, f'{self.cluster.docker_compose} stop {self.name}', timeout=timeout) - if r.exitcode == 0: - break + with And("checking pid does not exist"): + for i, attempt in enumerate(retries(timeout=100, delay=3)): + with attempt: + if i > 0 and i % 20 == 0: + self.command(f"kill -KILL {pid}", steps=False) + if ( + self.command(f"ps {pid}", steps=False, no_checks=True).exitcode + != 1 + ): + fail("pid still alive") - def start(self, timeout=300, wait_healthy=True, retries=5): - """Start node. - """ - for retry in range(retries): - r = self.cluster.command(None, f'{self.cluster.docker_compose} start {self.name}', timeout=timeout) - if r.exitcode == 0: - break + with And("deleting ClickHouse server pid file"): + self.command("rm -rf /tmp/clickhouse-server.pid", exitcode=0, steps=False) + + def start_clickhouse( + self, + timeout=300, + wait_healthy=True, + retry_count=5, + user=None, + thread_fuzzer=False, + ): + """Start ClickHouse server.""" + pid = self.clickhouse_pid() + if pid: + raise RuntimeError(f"ClickHouse server already running with pid {pid}") + + if thread_fuzzer: + self.thread_fuzzer() + + if user is None: + with By("starting ClickHouse server process"): + self.command( + "clickhouse server --config-file=/etc/clickhouse-server/config.xml" + " --log-file=/var/log/clickhouse-server/clickhouse-server.log" + " --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + " --pidfile=/tmp/clickhouse-server.pid --daemon", + exitcode=0, + steps=False, + ) + else: + with By(f"starting ClickHouse server process from {user}"): + self.command( + f"su {user} -c" + '"clickhouse server --config-file=/etc/clickhouse-server/config.xml' + " --log-file=/var/log/clickhouse-server/clickhouse-server.log" + " --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + ' --pidfile=/tmp/clickhouse-server.pid --daemon"', + exitcode=0, + steps=False, + ) + + with And("checking that ClickHouse server pid file was created"): + for attempt in retries(timeout=timeout, delay=1): + with attempt: + if ( + self.command( + "ls /tmp/clickhouse-server.pid", steps=False, no_checks=True + ).exitcode + != 0 + ): + fail("no pid file yet") if wait_healthy: - self.wait_healthy(timeout) + self.wait_clickhouse_healthy(timeout=timeout) - def restart(self, timeout=300, safe=True, wait_healthy=True, retries=5): - """Restart node. - """ - if safe: - self.query("SYSTEM STOP MOVES") - self.query("SYSTEM STOP MERGES") - self.query("SYSTEM FLUSH LOGS") - with By("waiting for 5 sec for moves and merges to stop"): - time.sleep(5) - with And("forcing to sync everything to disk"): - self.command("sync", timeout=300) + def restart_clickhouse( + self, timeout=300, safe=True, wait_healthy=True, retry_count=5, user=None + ): + """Restart ClickHouse server.""" + if self.clickhouse_pid(): + self.stop_clickhouse(timeout=timeout, safe=safe) - self.close_bashes() + self.start_clickhouse(timeout=timeout, wait_healthy=wait_healthy, user=user) - for retry in range(retries): - r = self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) - if r.exitcode == 0: - break + def stop(self, timeout=300, safe=True, retry_count=5): + """Stop node.""" + if self.clickhouse_pid(): + self.stop_clickhouse(timeout=timeout, safe=safe) - if wait_healthy: - self.wait_healthy(timeout) + return super(ClickHouseNode, self).stop( + timeout=timeout, retry_count=retry_count + ) + + def start( + self, + timeout=300, + start_clickhouse=True, + wait_healthy=True, + retry_count=5, + user=None, + ): + """Start node.""" + super(ClickHouseNode, self).start(timeout=timeout, retry_count=retry_count) + + if start_clickhouse: + self.start_clickhouse( + timeout=timeout, + wait_healthy=wait_healthy, + user=user, + ) + + def restart( + self, + timeout=300, + safe=True, + start_clickhouse=True, + wait_healthy=True, + retry_count=5, + user=None, + ): + """Restart node.""" + if self.clickhouse_pid(): + self.stop_clickhouse(timeout=timeout, safe=safe) + + super(ClickHouseNode, self).restart(timeout=timeout, retry_count=retry_count) + + if start_clickhouse: + self.start_clickhouse(timeout=timeout, wait_healthy=wait_healthy, user=user) + + def hash_query( + self, + sql, + hash_utility="sha1sum", + steps=True, + step=By, + settings=None, + secure=False, + *args, + **kwargs, + ): + """Execute sql query inside the container and return the hash of the output. - def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, - raise_on_exception=False, step=By, settings=None, *args, **kwargs): - """Execute and check query. :param sql: sql query - :param message: expected message that should be in the output, default: None - :param exitcode: expected exitcode, default: None + :param hash_utility: hash function which used to compute hash """ settings = list(settings or []) + query_settings = list(settings) if hasattr(current().context, "default_query_settings"): - settings += current().context.default_query_settings + query_settings += current().context.default_query_settings + + client = "clickhouse client -n" + if secure: + client += " -s" if len(sql) > 1024: with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: query.write(sql) query.flush() - command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n" - for setting in settings: + command = f'set -o pipefail && cat "{query.name}" | {self.cluster.docker_compose} exec -T {self.name} {client} | {hash_utility}' + for setting in query_settings: name, value = setting - command += f" --{name} \"{value}\"" + command += f' --{name} "{value}"' description = f""" - echo -e \"{sql[:100]}...\" > {query.name} - {command} - """ - with Step("executing command", description=description, format_description=False) if steps else NullStep(): + echo -e \"{sql[:100]}...\" > {query.name} + {command} + """ + with step( + "executing command", + description=description, + format_description=False, + ) if steps else NullStep(): try: r = self.cluster.bash(None)(command, *args, **kwargs) except ExpectTimeoutError: self.cluster.close_bash(None) else: - command = f"echo -e \"{sql}\" | clickhouse client -n" - for setting in settings: + command = f'set -o pipefail && echo -e "{sql}" | {client} | {hash_utility}' + for setting in query_settings: name, value = setting - command += f" --{name} \"{value}\"" - with Step("executing command", description=command, format_description=False) if steps else NullStep(): + command += f' --{name} "{value}"' + with step( + "executing command", description=command, format_description=False + ) if steps else NullStep(): + try: + r = self.cluster.bash(self.name)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(self.name) + + with Then(f"exitcode should be 0") if steps else NullStep(): + assert r.exitcode == 0, error(r.output) + + return r.output + + def diff_query( + self, + sql, + expected_output, + steps=True, + step=By, + settings=None, + secure=False, + *args, + **kwargs, + ): + """Execute inside the container but from the host and compare its output + to file that is located on the host. + + For example: + diff <(echo "SELECT * FROM myints FORMAT CSVWithNames" | clickhouse-client -mn) select.out + + :param sql: sql query + :param expected_output: path to the expected output + """ + settings = list(settings or []) + query_settings = list(settings) + + if hasattr(current().context, "default_query_settings"): + query_settings += current().context.default_query_settings + + client = "clickhouse client -n" + if secure: + client += " -s" + + if len(sql) > 1024: + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: + query.write(sql) + query.flush() + command = f'diff <(cat "{query.name}" | {self.cluster.docker_compose} exec -T {self.name} {client}) {expected_output}' + for setting in query_settings: + name, value = setting + command += f' --{name} "{value}"' + description = f""" + echo -e \"{sql[:100]}...\" > {query.name} + {command} + """ + with step( + "executing command", + description=description, + format_description=False, + ) if steps else NullStep(): + try: + r = self.cluster.bash(None)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(None) + else: + command = f'diff <(echo -e "{sql}" | {self.cluster.docker_compose} exec -T {self.name} {client}) {expected_output}' + for setting in query_settings: + name, value = setting + command += f' --{name} "{value}"' + with step( + "executing command", description=command, format_description=False + ) if steps else NullStep(): + try: + r = self.cluster.bash(None)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(None) + + with Then(f"exitcode should be 0") if steps else NullStep(): + assert r.exitcode == 0, error(r.output) + + def query( + self, + sql, + message=None, + exitcode=None, + steps=True, + no_checks=False, + raise_on_exception=False, + step=By, + settings=None, + retry_count=5, + messages_to_retry=None, + retry_delay=5, + secure=False, + *args, + **kwargs, + ): + """Execute and check query. + :param sql: sql query + :param message: expected message that should be in the output, default: None + :param exitcode: expected exitcode, default: None + :param steps: wrap query execution in a step, default: True + :param no_check: disable exitcode and message checks, default: False + :param step: wrapping step class, default: By + :param settings: list of settings to be used for the query in the form [(name, value),...], default: None + :param retry_count: number of retries, default: 5 + :param messages_to_retry: list of messages in the query output for + which retry should be triggered, default: MESSAGES_TO_RETRY + :param retry_delay: number of seconds to sleep before retry, default: 5 + :param secure: use secure connection, default: False + """ + retry_count = max(0, int(retry_count)) + retry_delay = max(0, float(retry_delay)) + settings = list(settings or []) + query_settings = list(settings) + + if messages_to_retry is None: + messages_to_retry = MESSAGES_TO_RETRY + + if hasattr(current().context, "default_query_settings"): + query_settings += current().context.default_query_settings + + client = "clickhouse client -n" + if secure: + client += " -s" + + if len(sql) > 1024: + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: + query.write(sql) + query.flush() + command = f'cat "{query.name}" | {self.cluster.docker_compose} exec -T {self.name} {client}' + for setting in query_settings: + name, value = setting + command += f' --{name} "{value}"' + description = f""" + echo -e \"{sql[:100]}...\" > {query.name} + {command} + """ + with step( + "executing command", + description=description, + format_description=False, + ) if steps else NullStep(): + try: + r = self.cluster.bash(None)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(None) + raise + else: + command = f'echo -e "{sql}" | {client}' + for setting in query_settings: + name, value = setting + command += f' --{name} "{value}"' + with step( + "executing command", description=command, format_description=False + ) if steps else NullStep(): try: r = self.cluster.bash(self.name)(command, *args, **kwargs) except ExpectTimeoutError: self.cluster.close_bash(self.name) raise + if retry_count and retry_count > 0: + if any(msg in r.output for msg in messages_to_retry): + time.sleep(retry_delay) + return self.query( + sql=sql, + message=message, + exitcode=exitcode, + steps=steps, + no_checks=no_checks, + raise_on_exception=raise_on_exception, + step=step, + settings=settings, + retry_count=retry_count - 1, + messages_to_retry=messages_to_retry, + *args, + **kwargs, + ) + if no_checks: return r @@ -240,7 +615,9 @@ class ClickHouseNode(Node): assert r.exitcode == exitcode, error(r.output) if message is not None: - with Then(f"output should contain message", description=message) if steps else NullStep(): + with Then( + f"output should contain message", description=message + ) if steps else NullStep(): assert message in r.output, error(r.output) if message is None or "Exception:" not in message: @@ -252,19 +629,28 @@ class ClickHouseNode(Node): return r + class Cluster(object): - """Simple object around docker-compose cluster. - """ - def __init__(self, local=False, - clickhouse_binary_path=None, configs_dir=None, - nodes=None, - docker_compose="docker-compose", docker_compose_project_dir=None, - docker_compose_file="docker-compose.yml"): + """Simple object around docker-compose cluster.""" + + def __init__( + self, + local=False, + clickhouse_binary_path=None, + clickhouse_odbc_bridge_binary_path=None, + configs_dir=None, + nodes=None, + docker_compose="docker-compose", + docker_compose_project_dir=None, + docker_compose_file="docker-compose.yml", + environ=None, + ): self._bash = {} self._control_shell = None - self.environ = {} + self.environ = {} if (environ is None) else environ self.clickhouse_binary_path = clickhouse_binary_path + self.clickhouse_odbc_bridge_binary_path = clickhouse_odbc_bridge_binary_path self.configs_dir = configs_dir self.local = local self.nodes = nodes or {} @@ -282,24 +668,88 @@ class Cluster(object): if not os.path.exists(self.configs_dir): raise TypeError(f"configs directory '{self.configs_dir}' does not exist") - # auto set docker-compose project directory if docker_compose_project_dir is None: - caller_project_dir = os.path.join(caller_dir, "docker-compose") - if os.path.exists(caller_project_dir): - docker_compose_project_dir = caller_project_dir + raise TypeError("docker compose directory must be specified.") - docker_compose_file_path = os.path.join(docker_compose_project_dir or "", docker_compose_file) + docker_compose_file_path = os.path.join( + docker_compose_project_dir or "", docker_compose_file + ) if not os.path.exists(docker_compose_file_path): - raise TypeError("docker compose file '{docker_compose_file_path}' does not exist") + raise TypeError( + f"docker compose file '{docker_compose_file_path}' does not exist" + ) - self.docker_compose += f" --ansi never --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" + if self.clickhouse_binary_path and self.clickhouse_binary_path.startswith( + "docker://" + ): + if current().context.clickhouse_version is None: + try: + current().context.clickhouse_version = ( + self.clickhouse_binary_path.split(":")[2] + ) + debug( + f"auto setting clickhouse version to {current().context.clickhouse_version}" + ) + except IndexError: + current().context.clickhouse_version = None + ( + self.clickhouse_binary_path, + self.clickhouse_odbc_bridge_binary_path, + ) = self.get_clickhouse_binary_from_docker_container( + self.clickhouse_binary_path + ) + + self.docker_compose += f' --ansi never --project-directory "{docker_compose_project_dir}" --file "{docker_compose_file_path}"' self.lock = threading.Lock() + def get_clickhouse_binary_from_docker_container( + self, + docker_image, + container_clickhouse_binary_path="/usr/bin/clickhouse", + container_clickhouse_odbc_bridge_binary_path="/usr/bin/clickhouse-odbc-bridge", + host_clickhouse_binary_path=None, + host_clickhouse_odbc_bridge_binary_path=None, + ): + """Get clickhouse-server and clickhouse-odbc-bridge binaries + from some Docker container. + """ + docker_image = docker_image.split("docker://", 1)[-1] + docker_container_name = str(uuid.uuid1()) + + if host_clickhouse_binary_path is None: + host_clickhouse_binary_path = os.path.join( + tempfile.gettempdir(), + f"{docker_image.rsplit('/',1)[-1].replace(':','_')}", + ) + + if host_clickhouse_odbc_bridge_binary_path is None: + host_clickhouse_odbc_bridge_binary_path = ( + host_clickhouse_binary_path + "_odbc_bridge" + ) + + with Given( + "I get ClickHouse server binary from docker container", + description=f"{docker_image}", + ): + with Shell() as bash: + bash.timeout = 300 + bash( + f'docker run -d --name "{docker_container_name}" {docker_image} | tee' + ) + bash( + f'docker cp "{docker_container_name}:{container_clickhouse_binary_path}" "{host_clickhouse_binary_path}"' + ) + bash( + f'docker cp "{docker_container_name}:{container_clickhouse_odbc_bridge_binary_path}" "{host_clickhouse_odbc_bridge_binary_path}"' + ) + bash(f'docker stop "{docker_container_name}"') + + return host_clickhouse_binary_path, host_clickhouse_odbc_bridge_binary_path + @property def control_shell(self, timeout=300): - """Must be called with self.lock.acquired. - """ + """Must be called with self.lock.acquired.""" if self._control_shell is not None: return self._control_shell @@ -310,30 +760,48 @@ class Cluster(object): shell.timeout = 30 shell("echo 1") break - except: + except IOError: + raise + except Exception as exc: shell.__exit__(None, None, None) if time.time() - time_start > timeout: raise RuntimeError(f"failed to open control shell") self._control_shell = shell return self._control_shell + def close_control_shell(self): + """Must be called with self.lock.acquired.""" + if self._control_shell is None: + return + shell = self._control_shell + self._control_shell = None + shell.__exit__(None, None, None) + def node_container_id(self, node, timeout=300): - """Must be called with self.lock acquired. - """ + """Must be called with self.lock acquired.""" container_id = None time_start = time.time() while True: - c = self.control_shell(f"{self.docker_compose} ps -q {node}") - container_id = c.output.strip() - if c.exitcode == 0 and len(container_id) > 1: - break - if time.time() - time_start > timeout: - raise RuntimeError(f"failed to get docker container id for the {node} service") + try: + c = self.control_shell( + f"{self.docker_compose} ps -q {node}", timeout=timeout + ) + container_id = c.output.strip() + if c.exitcode == 0 and len(container_id) > 1: + break + except IOError: + raise + except ExpectTimeoutError: + self.close_control_shell() + timeout = timeout - (time.time() - time_start) + if timeout <= 0: + raise RuntimeError( + f"failed to get docker container id for the {node} service" + ) return container_id def shell(self, node, timeout=300): - """Returns unique shell terminal to be used. - """ + """Returns unique shell terminal to be used.""" container_id = None if node is not None: @@ -346,13 +814,21 @@ class Cluster(object): if node is None: shell = Shell() else: - shell = Shell(command=[ - "/bin/bash", "--noediting", "-c", f"docker exec -it {container_id} bash --noediting" - ], name=node) + shell = Shell( + command=[ + "/bin/bash", + "--noediting", + "-c", + f"docker exec -it {container_id} bash --noediting", + ], + name=node, + ) shell.timeout = 30 shell("echo 1") break - except: + except IOError: + raise + except Exception as exc: shell.__exit__(None, None, None) if time.time() - time_start > timeout: raise RuntimeError(f"failed to open bash to node {node}") @@ -381,19 +857,27 @@ class Cluster(object): if node is None: self._bash[id] = Shell() else: - self._bash[id] = Shell(command=[ - "/bin/bash", "--noediting", "-c", f"docker exec -it {container_id} {command}" - ], name=node).__enter__() + self._bash[id] = Shell( + command=[ + "/bin/bash", + "--noediting", + "-c", + f"docker exec -it {container_id} {command}", + ], + name=node, + ).__enter__() self._bash[id].timeout = 30 self._bash[id]("echo 1") break - except: + except IOError: + raise + except Exception as exc: self._bash[id].__exit__(None, None, None) if time.time() - time_start > timeout: raise RuntimeError(f"failed to open bash to node {node}") if node is None: - for name,value in self.environ.items(): + for name, value in self.environ.items(): self._bash[id](f"export {name}={value}") self._bash[id].timeout = timeout @@ -447,7 +931,10 @@ class Cluster(object): # add message to each clickhouse-server.log if settings.debug: for node in self.nodes["clickhouse"]: - self.command(node=node, command=f"echo -e \"\n-- sending stop to: {node} --\n\" >> /var/log/clickhouse-server/clickhouse-server.log") + self.command( + node=node, + command=f'echo -e "\n-- sending stop to: {node} --\n" >> /var/log/clickhouse-server/clickhouse-server.log', + ) try: bash = self.bash(None) with self.lock: @@ -459,26 +946,50 @@ class Cluster(object): else: self._bash[id] = shell finally: - cmd = self.command(None, f"{self.docker_compose} down --timeout 60", bash=bash, timeout=timeout) + cmd = self.command( + None, + f"{self.docker_compose} down -v --remove-orphans --timeout 60", + bash=bash, + timeout=timeout, + ) with self.lock: if self._control_shell: self._control_shell.__exit__(None, None, None) self._control_shell = None return cmd - def up(self, timeout=30*60): + def temp_path(self): + """Return temporary folder path.""" + p = f"{self.environ['CLICKHOUSE_TESTS_DIR']}/_temp" + if not os.path.exists(p): + os.mkdir(p) + return p + + def temp_file(self, name): + """Return absolute temporary file path.""" + return f"{os.path.join(self.temp_path(), name)}" + + def up(self, timeout=30 * 60): if self.local: with Given("I am running in local mode"): with Then("check --clickhouse-binary-path is specified"): - assert self.clickhouse_binary_path, "when running in local mode then --clickhouse-binary-path must be specified" + assert ( + self.clickhouse_binary_path + ), "when running in local mode then --clickhouse-binary-path must be specified" with And("path should exist"): assert os.path.exists(self.clickhouse_binary_path) with And("I set all the necessary environment variables"): self.environ["COMPOSE_HTTP_TIMEOUT"] = "300" - self.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = self.clickhouse_binary_path - self.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = os.path.join( - os.path.dirname(self.clickhouse_binary_path), "clickhouse-odbc-bridge") + self.environ[ + "CLICKHOUSE_TESTS_SERVER_BIN_PATH" + ] = self.clickhouse_binary_path + self.environ[ + "CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH" + ] = self.clickhouse_odbc_bridge_binary_path or os.path.join( + os.path.dirname(self.clickhouse_binary_path), + "clickhouse-odbc-bridge", + ) self.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir with And("I list environment variables to show their values"): @@ -491,7 +1002,12 @@ class Cluster(object): for attempt in range(max_attempts): with When(f"attempt {attempt}/{max_attempts}"): with By("pulling images for all the services"): - cmd = self.command(None, f"{self.docker_compose} pull 2>&1 | tee", exitcode=None, timeout=timeout) + cmd = self.command( + None, + f"{self.docker_compose} pull 2>&1 | tee", + exitcode=None, + timeout=timeout, + ) if cmd.exitcode != 0: continue @@ -499,7 +1015,12 @@ class Cluster(object): self.command(None, f"{self.docker_compose} ps | tee") with And("executing docker-compose down just in case it is up"): - cmd = self.command(None, f"{self.docker_compose} down 2>&1 | tee", exitcode=None, timeout=timeout) + cmd = self.command( + None, + f"{self.docker_compose} down 2>&1 | tee", + exitcode=None, + timeout=timeout, + ) if cmd.exitcode != 0: continue @@ -509,27 +1030,55 @@ class Cluster(object): with And("executing docker-compose up"): for up_attempt in range(max_up_attempts): with By(f"attempt {up_attempt}/{max_up_attempts}"): - cmd = self.command(None, f"{self.docker_compose} up --renew-anon-volumes --force-recreate --timeout 300 -d 2>&1 | tee", timeout=timeout) + cmd = self.command( + None, + f"{self.docker_compose} up --renew-anon-volumes --force-recreate --timeout 300 -d 2>&1 | tee", + timeout=timeout, + ) if "is unhealthy" not in cmd.output: break with Then("check there are no unhealthy containers"): - ps_cmd = self.command(None, f"{self.docker_compose} ps | tee | grep -v \"Exit 0\"") + ps_cmd = self.command( + None, f'{self.docker_compose} ps | tee | grep -v "Exit 0"' + ) if "is unhealthy" in cmd.output or "Exit" in ps_cmd.output: self.command(None, f"{self.docker_compose} logs | tee") continue - if cmd.exitcode == 0 and "is unhealthy" not in cmd.output and "Exit" not in ps_cmd.output: + if ( + cmd.exitcode == 0 + and "is unhealthy" not in cmd.output + and "Exit" not in ps_cmd.output + ): break - if cmd.exitcode != 0 or "is unhealthy" in cmd.output or "Exit" in ps_cmd.output: + if ( + cmd.exitcode != 0 + or "is unhealthy" in cmd.output + or "Exit" in ps_cmd.output + ): fail("could not bring up docker-compose cluster") - with Then("wait all nodes report healhy"): + with Then("wait all nodes report healthy"): for name in self.nodes["clickhouse"]: self.node(name).wait_healthy() + if name.startswith("clickhouse"): + self.node(name).start_clickhouse() - def command(self, node, command, message=None, exitcode=None, steps=True, bash=None, *args, **kwargs): + def command( + self, + node, + command, + message=None, + exitcode=None, + steps=True, + bash=None, + no_checks=False, + use_error=True, + *args, + **kwargs, + ): """Execute and check command. :param node: name of the service :param command: command @@ -537,7 +1086,9 @@ class Cluster(object): :param exitcode: expected exitcode, default: None :param steps: don't break command into steps, default: True """ - with By("executing command", description=command, format_description=False) if steps else NullStep(): + with By( + "executing command", description=command, format_description=False + ) if steps else NullStep(): if bash is None: bash = self.bash(node) try: @@ -545,10 +1096,22 @@ class Cluster(object): except ExpectTimeoutError: self.close_bash(node) raise + + if no_checks: + return r + if exitcode is not None: - with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep(): + with Then( + f"exitcode should be {exitcode}", format_name=False + ) if steps else NullStep(): assert r.exitcode == exitcode, error(r.output) + if message is not None: - with Then(f"output should contain message", description=message, format_description=False) if steps else NullStep(): + with Then( + f"output should contain message", + description=message, + format_description=False, + ) if steps else NullStep(): assert message in r.output, error(r.output) + return r diff --git a/tests/testflows/helpers/common.py b/tests/testflows/helpers/common.py index 6110074b137..2ba6aef11ee 100644 --- a/tests/testflows/helpers/common.py +++ b/tests/testflows/helpers/common.py @@ -1,42 +1,568 @@ +import os +import uuid +import time +import xml.etree.ElementTree as xmltree +import packaging.version as pkg_version +from collections import namedtuple + import testflows.settings as settings from testflows.core import * +from testflows.asserts import error +from testflows.core.name import basename, parentname +from testflows._core.testtype import TestSubType + + +def check_clickhouse_version(version): + """Compare ClickHouse version.""" + + def check(test): + if getattr(test.context, "clickhouse_version", None) is None: + return False + + clickhouse_version = pkg_version.parse(str(test.context.clickhouse_version)) + + if version.startswith("=="): + return clickhouse_version == pkg_version.parse( + str(version.split("==", 1)[-1]) + ) + elif version.startswith(">="): + return clickhouse_version >= pkg_version.parse( + str(version.split(">=", 1)[-1]) + ) + elif version.startswith("<="): + return clickhouse_version <= pkg_version.parse( + str(version.split("<=", 1)[-1]) + ) + elif version.startswith("="): + return clickhouse_version == pkg_version.parse( + str(version.split("=", 1)[-1]) + ) + elif version.startswith(">"): + return clickhouse_version > pkg_version.parse( + str(version.split(">", 1)[-1]) + ) + elif version.startswith("<"): + return clickhouse_version < pkg_version.parse( + str(version.split("<", 1)[-1]) + ) + else: + return clickhouse_version == pkg_version.parse(str(version)) + + return check + + +def getuid(with_test_name=False): + if not with_test_name: + return str(uuid.uuid1()).replace("-", "_") + + if current().subtype == TestSubType.Example: + testname = ( + f"{basename(parentname(current().name)).replace(' ', '_').replace(',', '')}" + ) + else: + testname = f"{basename(current().name).replace(' ', '_').replace(',', '')}" + + return testname + "_" + str(uuid.uuid1()).replace("-", "_") + @TestStep(Given) -def instrument_clickhouse_server_log(self, node=None, test=None, - clickhouse_server_log="/var/log/clickhouse-server/clickhouse-server.log"): +def instrument_clickhouse_server_log( + self, + node=None, + test=None, + clickhouse_server_log="/var/log/clickhouse-server/clickhouse-server.log", + always_dump=False, +): """Instrument clickhouse-server.log for the current test (default) by adding start and end messages that include test name to log of the specified node. If we are in the debug mode and the test fails then dump the messages from the log for this test. + + :param always_dump: always dump clickhouse log after test, default: `False` """ if test is None: - test = current() + test = current() if node is None: node = self.context.node with By("getting current log size"): - cmd = node.command(f"stat --format=%s {clickhouse_server_log}") - start_logsize = cmd.output.split(" ")[0].strip() + cmd = node.command(f"stat --format=%s {clickhouse_server_log}") + if ( + cmd.output + == f"stat: cannot stat '{clickhouse_server_log}': No such file or directory" + ): + start_logsize = 0 + else: + start_logsize = cmd.output.split(" ")[0].strip() try: with And("adding test name start message to the clickhouse-server.log"): - node.command(f"echo -e \"\\n-- start: {test.name} --\\n\" >> {clickhouse_server_log}") + node.command( + f'echo -e "\\n-- start: {test.name} --\\n" >> {clickhouse_server_log}' + ) yield finally: if test.terminating is True: return - with Finally("adding test name end message to the clickhouse-server.log", flags=TE): - node.command(f"echo -e \"\\n-- end: {test.name} --\\n\" >> {clickhouse_server_log}") + with Finally( + "adding test name end message to the clickhouse-server.log", flags=TE + ): + node.command( + f'echo -e "\\n-- end: {test.name} --\\n" >> {clickhouse_server_log}' + ) with And("getting current log size at the end of the test"): - cmd = node.command(f"stat --format=%s {clickhouse_server_log}") - end_logsize = cmd.output.split(" ")[0].strip() + cmd = node.command(f"stat --format=%s {clickhouse_server_log}") + end_logsize = cmd.output.split(" ")[0].strip() - with And("checking if test has failing result"): - if settings.debug and not self.parent.result: - with Then("dumping clickhouse-server.log for this test"): - node.command(f"tail -c +{start_logsize} {clickhouse_server_log}" - f" | head -c {int(end_logsize) - int(start_logsize)}") + dump_log = always_dump or (settings.debug and not self.parent.result) + + if dump_log: + with Then("dumping clickhouse-server.log for this test"): + node.command( + f"tail -c +{start_logsize} {clickhouse_server_log}" + f" | head -c {int(end_logsize) - int(start_logsize)}" + ) + + +xml_with_utf8 = '\n' + + +def xml_indent(elem, level=0, by=" "): + i = "\n" + level * by + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + by + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for elem in elem: + xml_indent(elem, level + 1) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = i + + +def xml_append(root, tag, text): + element = xmltree.Element(tag) + element.text = text + root.append(element) + return element + + +class Config: + def __init__(self, content, path, name, uid, preprocessed_name): + self.content = content + self.path = path + self.name = name + self.uid = uid + self.preprocessed_name = preprocessed_name + + +class KeyWithAttributes: + def __init__(self, name, attributes): + """XML key with attributes. + + :param name: key name + :param attributes: dictionary of attributes {name: value, ...} + """ + self.name = name + self.attributes = dict(attributes) + + +def create_xml_config_content( + entries, config_file, config_d_dir="/etc/clickhouse-server/config.d" +): + """Create XML configuration file from a dictionary. + + :param entries: dictionary that defines xml + :param config_file: name of the config file + :param config_d_dir: config.d directory path, default: `/etc/clickhouse-server/config.d` + """ + uid = getuid() + path = os.path.join(config_d_dir, config_file) + name = config_file + root = xmltree.Element("clickhouse") + root.append(xmltree.Comment(text=f"config uid: {uid}")) + + def create_xml_tree(entries, root): + for k, v in entries.items(): + if isinstance(k, KeyWithAttributes): + xml_element = xmltree.Element(k.name) + for attr_name, attr_value in k.attributes.items(): + xml_element.set(attr_name, attr_value) + if type(v) is dict: + create_xml_tree(v, xml_element) + elif type(v) in (list, tuple): + for e in v: + create_xml_tree(e, xml_element) + else: + xml_element.text = v + root.append(xml_element) + elif type(v) is dict: + xml_element = xmltree.Element(k) + create_xml_tree(v, xml_element) + root.append(xml_element) + elif type(v) in (list, tuple): + xml_element = xmltree.Element(k) + for e in v: + create_xml_tree(e, xml_element) + root.append(xml_element) + else: + xml_append(root, k, v) + + create_xml_tree(entries, root) + xml_indent(root) + content = xml_with_utf8 + str( + xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8" + ) + + return Config(content, path, name, uid, "config.xml") + + +def add_invalid_config( + config, message, recover_config=None, tail=30, timeout=300, restart=True, user=None +): + """Check that ClickHouse errors when trying to load invalid configuration file.""" + cluster = current().context.cluster + node = current().context.node + + try: + with Given("I prepare the error log by writing empty lines into it"): + node.command( + 'echo -e "%s" > /var/log/clickhouse-server/clickhouse-server.err.log' + % ("-\\n" * tail) + ) + + with When("I add the config", description=config.path): + command = f"cat < {config.path}\n{config.content}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + + with Then( + f"{config.preprocessed_name} should be updated", + description=f"timeout {timeout}", + ): + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + if restart: + with When("I restart ClickHouse to apply the config changes"): + node.restart_clickhouse(safe=False, wait_healthy=False, user=user) + + finally: + if recover_config is None: + with Finally(f"I remove {config.name}"): + with By("removing invalid configuration file"): + system_config_path = os.path.join( + cluster.environ["CLICKHOUSE_TESTS_DIR"], + "configs", + node.name, + "config.d", + config.path.split("config.d/")[-1], + ) + cluster.command( + None, + f"rm -rf {system_config_path}", + timeout=timeout, + exitcode=0, + ) + + if restart: + with And("restarting ClickHouse"): + node.restart_clickhouse(safe=False, user=user) + node.restart_clickhouse(safe=False, user=user) + else: + with Finally(f"I change {config.name}"): + with By("changing invalid configuration file"): + system_config_path = os.path.join( + cluster.environ["CLICKHOUSE_TESTS_DIR"], + "configs", + node.name, + "config.d", + config.path.split("config.d/")[-1], + ) + cluster.command( + None, + f"rm -rf {system_config_path}", + timeout=timeout, + exitcode=0, + ) + command = f"cat < {system_config_path}\n{recover_config.content}\nHEREDOC" + cluster.command(None, command, timeout=timeout, exitcode=0) + + if restart: + with And("restarting ClickHouse"): + node.restart_clickhouse(safe=False, user=user) + + with Then("error log should contain the expected error message"): + started = time.time() + command = f'tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep "{message}"' + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + +def add_config( + config, + timeout=300, + restart=False, + modify=False, + node=None, + user=None, + wait_healthy=True, + check_preprocessed=True, +): + """Add dynamic configuration file to ClickHouse. + + :param config: configuration file description + :param timeout: timeout, default: 300 sec + :param restart: restart server, default: False + :param modify: only modify configuration file, default: False + """ + if node is None: + node = current().context.node + cluster = current().context.cluster + + def check_preprocessed_config_is_updated(after_removal=False): + """Check that preprocessed config is updated.""" + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if after_removal: + if exitcode == 1: + break + else: + if exitcode == 0: + break + time.sleep(1) + + if settings.debug: + node.command( + f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}" + ) + + if after_removal: + assert exitcode == 1, error() + else: + assert exitcode == 0, error() + + def wait_for_config_to_be_loaded(user=None): + """Wait for config to be loaded.""" + if restart: + with When("I close terminal to the node to be restarted"): + bash.close() + + with And("I stop ClickHouse to apply the config changes"): + node.stop_clickhouse(safe=False) + + with And("I get the current log size"): + cmd = node.cluster.command( + None, + f"stat --format=%s {cluster.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log", + ) + logsize = cmd.output.split(" ")[0].strip() + + with And("I start ClickHouse back up"): + node.start_clickhouse(user=user, wait_healthy=wait_healthy) + + with Then("I tail the log file from using previous log size as the offset"): + bash.prompt = bash.__class__.prompt + bash.open() + bash.send( + f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log" + ) + + with Then("I wait for config reload message in the log file"): + if restart: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", + timeout=timeout, + ) + else: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", + timeout=timeout, + ) + + try: + with Given(f"{config.name}"): + if settings.debug: + with When("I output the content of the config"): + debug(config.content) + + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + bash.send( + "tail -v -n 0 -f /var/log/clickhouse-server/clickhouse-server.log" + ) + # make sure tail process is launched and started to follow the file + bash.expect("<==") + bash.expect("\n") + + with When("I add the config", description=config.path): + command = ( + f"cat < {config.path}\n{config.content}\nHEREDOC" + ) + node.command(command, steps=False, exitcode=0) + + if check_preprocessed: + with Then( + f"{config.preprocessed_name} should be updated", + description=f"timeout {timeout}", + ): + check_preprocessed_config_is_updated() + + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded(user=user) + + yield + finally: + if not modify: + with Finally(f"I remove {config.name} on {node.name}"): + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + bash.send( + "tail -v -n 0 -f /var/log/clickhouse-server/clickhouse-server.log" + ) + # make sure tail process is launched and started to follow the file + bash.expect("<==") + bash.expect("\n") + + with By("removing the config file", description=config.path): + node.command(f"rm -rf {config.path}", exitcode=0) + + with Then( + f"{config.preprocessed_name} should be updated", + description=f"timeout {timeout}", + ): + check_preprocessed_config_is_updated(after_removal=True) + + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() + + +@TestStep(When) +def copy( + self, + dest_node, + src_path, + dest_path, + bash=None, + binary=False, + eof="EOF", + src_node=None, +): + """Copy file from source to destination node.""" + if binary: + raise NotImplementedError("not yet implemented; need to use base64 encoding") + + bash = self.context.cluster.bash(node=src_node) + + cmd = bash(f"cat {src_path}") + + assert cmd.exitcode == 0, error() + contents = cmd.output + + dest_node.command(f"cat << {eof} > {dest_path}\n{contents}\n{eof}") + + +@TestStep(Given) +def add_user_to_group_on_node( + self, node=None, group="clickhouse", username="clickhouse" +): + """Add user {username} into group {group}.""" + if node is None: + node = self.context.node + + node.command(f"usermode -g {group} {username}", exitcode=0) + + +@TestStep(Given) +def change_user_on_node(self, node=None, username="clickhouse"): + """Change user on node.""" + if node is None: + node = self.context.node + try: + node.command(f"su {username}", exitcode=0) + yield + finally: + node.command("exit", exitcode=0) + + +@TestStep(Given) +def add_user_on_node(self, node=None, groupname=None, username="clickhouse"): + """Create user on node with group specifying.""" + if node is None: + node = self.context.node + try: + if groupname is None: + node.command(f"useradd -s /bin/bash {username}", exitcode=0) + else: + node.command(f"useradd -g {groupname} -s /bin/bash {username}", exitcode=0) + yield + finally: + node.command(f"deluser {username}", exitcode=0) + + +@TestStep(Given) +def add_group_on_node(self, node=None, groupname="clickhouse"): + """Create group on node""" + if node is None: + node = self.context.node + try: + node.command(f"groupadd {groupname}", exitcode=0) + yield + finally: + node.command(f"delgroup clickhouse") + + +@TestStep(Given) +def create_file_on_node(self, path, content, node=None): + """Create file on node. + + :param path: file path + :param content: file content + """ + if node is None: + node = self.context.node + try: + with By(f"creating file {path}"): + node.command(f"cat < {path}\n{content}\nHEREDOC", exitcode=0) + yield path + finally: + with Finally(f"I remove {path}"): + node.command(f"rm -rf {path}", exitcode=0) + + +@TestStep(Given) +def set_envs_on_node(self, envs, node=None): + """Set environment variables on node. + + :param envs: dictionary of env variables key=value + """ + if node is None: + node = self.context.node + try: + with By("setting envs"): + for key, value in envs.items(): + node.command(f"export {key}={value}", exitcode=0) + yield + finally: + with Finally(f"I unset envs"): + for key in envs: + node.command(f"unset {key}", exitcode=0) diff --git a/tests/testflows/kerberos/configs/clickhouse/common.xml b/tests/testflows/kerberos/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/kerberos/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/kerberos/configs/clickhouse/config.xml b/tests/testflows/kerberos/configs/clickhouse/config.xml deleted file mode 100644 index 0d2904eed48..00000000000 --- a/tests/testflows/kerberos/configs/clickhouse/config.xml +++ /dev/null @@ -1,440 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - 0.0.0.0 - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/kerberos/configs/clickhouse/users.xml b/tests/testflows/kerberos/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/kerberos/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/kerberos/regression.py b/tests/testflows/kerberos/regression.py index d1b13acc1c9..0206cd3bf00 100755 --- a/tests/testflows/kerberos/regression.py +++ b/tests/testflows/kerberos/regression.py @@ -10,35 +10,42 @@ from helpers.argparser import argparser from kerberos.requirements.requirements import * xfails = { - "config/principal and realm specified/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/26197")], + "config/principal and realm specified/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/26197") + ], } @TestModule @Name("kerberos") @ArgumentParser(argparser) -@Requirements( - RQ_SRS_016_Kerberos("1.0") -) +@Requirements(RQ_SRS_016_Kerberos("1.0")) @XFails(xfails) -def regression(self, local, clickhouse_binary_path, stress=None): - """ClickHouse Kerberos authentication test regression module. - """ +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse Kerberos authentication test regression module.""" nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), - "kerberos": ("kerberos", ), + "kerberos": ("kerberos",), } if stress is not None: self.context.stress = stress + self.context.clickhouse_version = clickhouse_version - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "kerberos_env")) as cluster: + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), "kerberos_env"), + ) as cluster: self.context.cluster = cluster Feature(run=load("kerberos.tests.generic", "generic"), flags=TE) Feature(run=load("kerberos.tests.config", "config"), flags=TE) Feature(run=load("kerberos.tests.parallel", "parallel"), flags=TE) + if main(): regression() diff --git a/tests/testflows/kerberos/requirements/requirements.py b/tests/testflows/kerberos/requirements/requirements.py index 774f533373a..07f3e1edf42 100644 --- a/tests/testflows/kerberos/requirements/requirements.py +++ b/tests/testflows/kerberos/requirements/requirements.py @@ -9,434 +9,454 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_016_Kerberos = Requirement( - name='RQ.SRS-016.Kerberos', - version='1.0', + name="RQ.SRS-016.Kerberos", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using [Kerberos] server.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using [Kerberos] server.\n" "\n" + ), link=None, level=3, - num='4.1.1') + num="4.1.1", +) RQ_SRS_016_Kerberos_Ping = Requirement( - name='RQ.SRS-016.Kerberos.Ping', - version='1.0', + name="RQ.SRS-016.Kerberos.Ping", + version="1.0", priority=None, group=None, type=None, uid=None, - description=( - 'Docker containers SHALL be able to ping each other.\n' - '\n' - ), + description=("Docker containers SHALL be able to ping each other.\n" "\n"), link=None, level=3, - num='4.2.1') + num="4.2.1", +) RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods", + version="1.0", priority=None, group=None, type=None, uid=None, description=( "[ClickHouse] SHALL generate an exception and TERMINATE in case some user in `users.xml` has a `` section specified alongside with any other authentication method's section, e.g. `ldap`, `password`.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.3.1') + num="4.3.1", +) RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reject [Kerberos] authentication in case user is properly configured for using Kerberos, but Kerberos itself is not enabled in `config.xml`. For example:\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - '\n' - '```\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' HTTP/clickhouse.example.com@EXAMPLE.COM\n' - ' \n' - '\n' - '```\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' EXAMPLE.COM\n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL reject [Kerberos] authentication in case user is properly configured for using Kerberos, but Kerberos itself is not enabled in `config.xml`. For example:\n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + "\n" + "```\n" + "```xml\n" + "\n" + " \n" + " \n" + " HTTP/clickhouse.example.com@EXAMPLE.COM\n" + " \n" + "\n" + "```\n" + "```xml\n" + "\n" + " \n" + " \n" + " EXAMPLE.COM\n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.3.2') + num="4.3.2", +) RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL disable [Kerberos] and reject [Kerberos] authentication in case multiple `kerberos` sections are present in `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL disable [Kerberos] and reject [Kerberos] authentication in case multiple `kerberos` sections are present in `config.xml`.\n" + "\n" + ), link=None, level=3, - num='4.3.3') + num="4.3.3", +) RQ_SRS_016_Kerberos_Configuration_WrongUserRealm = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.WrongUserRealm", + version="1.0", priority=None, group=None, type=None, uid=None, description=( "[ClickHouse] SHALL reject [Kerberos] authentication if user's realm specified in `users.xml` doesn't match the realm of the principal trying to authenticate.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.3.4') + num="4.3.4", +) RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL generate an exception and disable [Kerberos] in case both `realm` and `principal` sections are defined in `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL generate an exception and disable [Kerberos] in case both `realm` and `principal` sections are defined in `config.xml`.\n" + "\n" + ), link=None, level=3, - num='4.3.5') + num="4.3.5", +) RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `principal` sections are specified inside `kerberos` section in `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `principal` sections are specified inside `kerberos` section in `config.xml`.\n" + "\n" + ), link=None, level=3, - num='4.3.6') + num="4.3.6", +) RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections = Requirement( - name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', - version='1.0', + name="RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `realm` sections are specified inside `kerberos` section in `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `realm` sections are specified inside `kerberos` section in `config.xml`.\n" + "\n" + ), link=None, level=3, - num='4.3.7') + num="4.3.7", +) RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser = Requirement( - name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', - version='1.0', + name="RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL accept [Kerberos] authentication for a user that is configured in `users.xml` and has [Kerberos] enabled, i.e.:\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' \n' - ' \n' - ' \n' - ' \n' - ' EXAMPLE.COM\n' - ' \n' - ' \n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL accept [Kerberos] authentication for a user that is configured in `users.xml` and has [Kerberos] enabled, i.e.:\n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " EXAMPLE.COM\n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.4.1') + num="4.4.1", +) RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser = Requirement( - name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', - version='1.0', + name="RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL accept [Kerberos] authentication if user is configured to authenticate via [Kerberos] using SQL queries\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL accept [Kerberos] authentication if user is configured to authenticate via [Kerberos] using SQL queries\n" + "\n" + "```sql\n" "CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'\n" - '```\n' - '\n' - 'or\n' - '\n' - '```sql\n' - 'CREATE USER my_user IDENTIFIED WITH kerberos\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "or\n" + "\n" + "```sql\n" + "CREATE USER my_user IDENTIFIED WITH kerberos\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.4.2') + num="4.4.2", +) RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured = Requirement( - name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', - version='1.0', + name="RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reject [Kerberos] authentication if username is valid but [ClickHouse] user is not configured to be authenticated using [Kerberos].\n' - '\n' - ), + "[ClickHouse] SHALL reject [Kerberos] authentication if username is valid but [ClickHouse] user is not configured to be authenticated using [Kerberos].\n" + "\n" + ), link=None, level=3, - num='4.4.3') + num="4.4.3", +) RQ_SRS_016_Kerberos_InvalidUser = Requirement( - name='RQ.SRS-016.Kerberos.InvalidUser', - version='1.0', + name="RQ.SRS-016.Kerberos.InvalidUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reject [Kerberos] authentication if name of the principal attempting to authenticate does not translate to a valid [ClickHouse] username configured in `users.xml` or via SQL workflow.\n' - '\n' - ), + "[ClickHouse] SHALL reject [Kerberos] authentication if name of the principal attempting to authenticate does not translate to a valid [ClickHouse] username configured in `users.xml` or via SQL workflow.\n" + "\n" + ), link=None, level=3, - num='4.5.1') + num="4.5.1", +) RQ_SRS_016_Kerberos_InvalidUser_UserDeleted = Requirement( - name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', - version='1.0', + name="RQ.SRS-016.Kerberos.InvalidUser.UserDeleted", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user was removed from the database using an SQL query.\n' - '\n' - ), + "[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user was removed from the database using an SQL query.\n" + "\n" + ), link=None, level=3, - num='4.5.2') + num="4.5.2", +) RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket = Requirement( - name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', - version='1.0', + name="RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket", + version="1.0", priority=None, group=None, type=None, uid=None, description=( "[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but [ClickHouse] doesn't have a valid Kerberos ticket or the ticket is expired.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.6.1') + num="4.6.1", +) RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket = Requirement( - name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', - version='1.0', + name="RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket", + version="1.0", priority=None, group=None, type=None, uid=None, description=( "[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but the client doesn't have a valid Kerberos ticket or the ticket is expired.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.6.2') + num="4.6.2", +) RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets = Requirement( - name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', - version='1.0', + name="RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL accept [Kerberos] authentication if no [Kerberos] server is reachable, but [ClickHouse] is configured to use valid credentials and [ClickHouse] has already processed some valid kerberized request (so it was granted a ticket), and the client has a valid ticket as well.\n' - '\n' - ), + "[ClickHouse] SHALL accept [Kerberos] authentication if no [Kerberos] server is reachable, but [ClickHouse] is configured to use valid credentials and [ClickHouse] has already processed some valid kerberized request (so it was granted a ticket), and the client has a valid ticket as well.\n" + "\n" + ), link=None, level=3, - num='4.6.3') + num="4.6.3", +) RQ_SRS_016_Kerberos_KerberosServerRestarted = Requirement( - name='RQ.SRS-016.Kerberos.KerberosServerRestarted', - version='1.0', + name="RQ.SRS-016.Kerberos.KerberosServerRestarted", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL accept [Kerberos] authentication if [Kerberos] server was restarted.\n' - '\n' - ), + "[ClickHouse] SHALL accept [Kerberos] authentication if [Kerberos] server was restarted.\n" + "\n" + ), link=None, level=3, - num='4.7.1') + num="4.7.1", +) RQ_SRS_016_Kerberos_Performance = Requirement( - name='RQ.SRS-016.Kerberos.Performance', - version='1.0', + name="RQ.SRS-016.Kerberos.Performance", + version="1.0", priority=None, group=None, type=None, uid=None, description=( "[ClickHouse]'s performance for [Kerberos] authentication SHALL be comparable to regular authentication.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.8.1') + num="4.8.1", +) RQ_SRS_016_Kerberos_Parallel = Requirement( - name='RQ.SRS-016.Kerberos.Parallel', - version='1.0', + name="RQ.SRS-016.Kerberos.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication using [Kerberos].\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication using [Kerberos].\n" "\n" + ), link=None, level=3, - num='4.9.1') + num="4.9.1", +) RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos = Requirement( - name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', - version='1.0', + name="RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support processing of simultaneous kerberized (for users configured to authenticate via [Kerberos]) and non-kerberized (for users configured to authenticate with any other means) requests.\n' - '\n' - ), + "[ClickHouse] SHALL support processing of simultaneous kerberized (for users configured to authenticate via [Kerberos]) and non-kerberized (for users configured to authenticate with any other means) requests.\n" + "\n" + ), link=None, level=3, - num='4.9.2') + num="4.9.2", +) RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials = Requirement( - name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', - version='1.0', + name="RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under the same credentials.\n' - '\n' - ), + "[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under the same credentials.\n" + "\n" + ), link=None, level=3, - num='4.9.3') + num="4.9.3", +) RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials = Requirement( - name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', - version='1.0', + name="RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under different credentials.\n' - '\n' - ), + "[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under different credentials.\n" + "\n" + ), link=None, level=3, - num='4.9.4') + num="4.9.4", +) RQ_SRS_016_Kerberos_Parallel_ValidInvalid = Requirement( - name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', - version='1.0', + name="RQ.SRS-016.Kerberos.Parallel.ValidInvalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( "[ClickHouse] SHALL support parallel authentication of users using [Kerberos] server, some of which are valid and some invalid. Valid users' authentication should not be affected by invalid users' attempts.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.9.5') + num="4.9.5", +) RQ_SRS_016_Kerberos_Parallel_Deletion = Requirement( - name='RQ.SRS-016.Kerberos.Parallel.Deletion', - version='1.0', + name="RQ.SRS-016.Kerberos.Parallel.Deletion", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not crash when two or more [Kerberos] users are simultaneously deleting one another.\n' - '\n' - ), + "[ClickHouse] SHALL not crash when two or more [Kerberos] users are simultaneously deleting one another.\n" + "\n" + ), link=None, level=3, - num='4.9.6') + num="4.9.6", +) QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( - name='QA-SRS016 ClickHouse Kerberos Authentication', + name="QA-SRS016 ClickHouse Kerberos Authentication", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -448,46 +468,112 @@ QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='Generic', level=2, num='4.1'), - Heading(name='RQ.SRS-016.Kerberos', level=3, num='4.1.1'), - Heading(name='Ping', level=2, num='4.2'), - Heading(name='RQ.SRS-016.Kerberos.Ping', level=3, num='4.2.1'), - Heading(name='Configuration', level=2, num='4.3'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', level=3, num='4.3.1'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', level=3, num='4.3.2'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', level=3, num='4.3.3'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', level=3, num='4.3.4'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', level=3, num='4.3.5'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', level=3, num='4.3.6'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', level=3, num='4.3.7'), - Heading(name='Valid User', level=2, num='4.4'), - Heading(name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', level=3, num='4.4.1'), - Heading(name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', level=3, num='4.4.2'), - Heading(name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', level=3, num='4.4.3'), - Heading(name='Invalid User', level=2, num='4.5'), - Heading(name='RQ.SRS-016.Kerberos.InvalidUser', level=3, num='4.5.1'), - Heading(name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', level=3, num='4.5.2'), - Heading(name='Kerberos Not Available', level=2, num='4.6'), - Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', level=3, num='4.6.1'), - Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', level=3, num='4.6.2'), - Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', level=3, num='4.6.3'), - Heading(name='Kerberos Restarted', level=2, num='4.7'), - Heading(name='RQ.SRS-016.Kerberos.KerberosServerRestarted', level=3, num='4.7.1'), - Heading(name='Performance', level=2, num='4.8'), - Heading(name='RQ.SRS-016.Kerberos.Performance', level=3, num='4.8.1'), - Heading(name='Parallel Requests processing', level=2, num='4.9'), - Heading(name='RQ.SRS-016.Kerberos.Parallel', level=3, num='4.9.1'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', level=3, num='4.9.2'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', level=3, num='4.9.3'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', level=3, num='4.9.4'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', level=3, num='4.9.5'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.Deletion', level=3, num='4.9.6'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="Generic", level=2, num="4.1"), + Heading(name="RQ.SRS-016.Kerberos", level=3, num="4.1.1"), + Heading(name="Ping", level=2, num="4.2"), + Heading(name="RQ.SRS-016.Kerberos.Ping", level=3, num="4.2.1"), + Heading(name="Configuration", level=2, num="4.3"), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods", + level=3, + num="4.3.1", ), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled", + level=3, + num="4.3.2", + ), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections", + level=3, + num="4.3.3", + ), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.WrongUserRealm", + level=3, + num="4.3.4", + ), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified", + level=3, + num="4.3.5", + ), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections", + level=3, + num="4.3.6", + ), + Heading( + name="RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections", + level=3, + num="4.3.7", + ), + Heading(name="Valid User", level=2, num="4.4"), + Heading( + name="RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser", level=3, num="4.4.1" + ), + Heading( + name="RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser", + level=3, + num="4.4.2", + ), + Heading( + name="RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured", + level=3, + num="4.4.3", + ), + Heading(name="Invalid User", level=2, num="4.5"), + Heading(name="RQ.SRS-016.Kerberos.InvalidUser", level=3, num="4.5.1"), + Heading( + name="RQ.SRS-016.Kerberos.InvalidUser.UserDeleted", level=3, num="4.5.2" + ), + Heading(name="Kerberos Not Available", level=2, num="4.6"), + Heading( + name="RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket", + level=3, + num="4.6.1", + ), + Heading( + name="RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket", + level=3, + num="4.6.2", + ), + Heading( + name="RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets", + level=3, + num="4.6.3", + ), + Heading(name="Kerberos Restarted", level=2, num="4.7"), + Heading( + name="RQ.SRS-016.Kerberos.KerberosServerRestarted", level=3, num="4.7.1" + ), + Heading(name="Performance", level=2, num="4.8"), + Heading(name="RQ.SRS-016.Kerberos.Performance", level=3, num="4.8.1"), + Heading(name="Parallel Requests processing", level=2, num="4.9"), + Heading(name="RQ.SRS-016.Kerberos.Parallel", level=3, num="4.9.1"), + Heading( + name="RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos", + level=3, + num="4.9.2", + ), + Heading( + name="RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials", + level=3, + num="4.9.3", + ), + Heading( + name="RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials", + level=3, + num="4.9.4", + ), + Heading(name="RQ.SRS-016.Kerberos.Parallel.ValidInvalid", level=3, num="4.9.5"), + Heading(name="RQ.SRS-016.Kerberos.Parallel.Deletion", level=3, num="4.9.6"), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_016_Kerberos, RQ_SRS_016_Kerberos_Ping, @@ -514,8 +600,8 @@ QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials, RQ_SRS_016_Kerberos_Parallel_ValidInvalid, RQ_SRS_016_Kerberos_Parallel_Deletion, - ), - content=''' + ), + content=""" # QA-SRS016 ClickHouse Kerberos Authentication # Software Requirements Specification @@ -806,4 +892,5 @@ version: 1.0 [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/kerberos/requirements/requirements.md [Git]: https://git-scm.com/ [Kerberos terminology]: https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm -''') +""", +) diff --git a/tests/testflows/kerberos/tests/common.py b/tests/testflows/kerberos/tests/common.py index 5dd0f734d8f..0e0f7f2ebc2 100644 --- a/tests/testflows/kerberos/tests/common.py +++ b/tests/testflows/kerberos/tests/common.py @@ -8,7 +8,7 @@ import uuid def getuid(): - return str(uuid.uuid1()).replace('-', '_') + return str(uuid.uuid1()).replace("-", "_") def xml_append(root, tag, text=Null): @@ -31,18 +31,21 @@ def xml_parse_file(filename): def create_default_config(filename): contents = "" if "kerberos_users.xml" in filename: - contents = "EXAMPLE.COM" \ - "" + contents = ( + "EXAMPLE.COM" + "" + ) elif "kerberos.xml" in filename: - contents = "EXAMPLE.COM" + contents = ( + "EXAMPLE.COM" + ) with open(filename, "w") as f: f.write(contents) def test_select_query(node, krb_auth=True, req="SELECT currentUser()"): - """ Helper forming a HTTP query to ClickHouse server - """ + """Helper forming a HTTP query to ClickHouse server""" if krb_auth: return f"echo '{req}' | curl --negotiate -u : 'http://{node.name}:8123/' --data-binary @-" else: @@ -51,11 +54,10 @@ def test_select_query(node, krb_auth=True, req="SELECT currentUser()"): @TestStep(Given) def kinit_no_keytab(self, node, principal="kerberos_user", lifetime_option="-l 10:00"): - """ Helper for obtaining Kerberos ticket for client - """ + """Helper for obtaining Kerberos ticket for client""" try: node.cmd("echo pwd | kinit admin/admin") - node.cmd(f"kadmin -w pwd -q \"add_principal -pw pwd {principal}\"") + node.cmd(f'kadmin -w pwd -q "add_principal -pw pwd {principal}"') node.cmd(f"echo pwd | kinit {lifetime_option} {principal}") yield finally: @@ -64,12 +66,15 @@ def kinit_no_keytab(self, node, principal="kerberos_user", lifetime_option="-l 1 @TestStep(Given) def create_server_principal(self, node): - """ Helper for obtaining Kerberos ticket for server - """ + """Helper for obtaining Kerberos ticket for server""" try: node.cmd("echo pwd | kinit admin/admin") - node.cmd(f"kadmin -w pwd -q \"add_principal -randkey HTTP/kerberos_env_{node.name}_1.krbnet\"") - node.cmd(f"kadmin -w pwd -q \"ktadd -k /etc/krb5.keytab HTTP/kerberos_env_{node.name}_1.krbnet\"") + node.cmd( + f'kadmin -w pwd -q "add_principal -randkey HTTP/kerberos_env_{node.name}_1.krbnet"' + ) + node.cmd( + f'kadmin -w pwd -q "ktadd -k /etc/krb5.keytab HTTP/kerberos_env_{node.name}_1.krbnet"' + ) yield finally: node.cmd("kdestroy") @@ -78,47 +83,48 @@ def create_server_principal(self, node): @TestStep(Given) def save_file_state(self, node, filename): - """ Save current file and then restore it, restarting the node - """ + """Save current file and then restore it, restarting the node""" try: with When("I save file state"): - with open(filename, 'r') as f: + with open(filename, "r") as f: a = f.read() yield finally: with Finally("I restore initial state"): - with open(filename, 'w') as f: + with open(filename, "w") as f: f.write(a) node.restart() @TestStep(Given) def temp_erase(self, node, filename=None): - """ Temporary erasing config file and restarting the node - """ + """Temporary erasing config file and restarting the node""" if filename is None: filename = f"kerberos/configs/{node.name}/config.d/kerberos.xml" with When("I save file state"): - with open(filename, 'r') as f: + with open(filename, "r") as f: a = f.read() try: with Then("I overwrite file to be dummy"): - with open(filename, 'w') as f: + with open(filename, "w") as f: f.write("\n") node.restart() yield finally: with Finally("I restore initial file state"): - with open(filename, 'w') as f: + with open(filename, "w") as f: f.write(a) node.restart() def restart(node, config_path, safe=False, timeout=60): - """Restart ClickHouse server and wait for config to be reloaded. - """ + """Restart ClickHouse server and wait for config to be reloaded.""" - filename = '/etc/clickhouse-server/config.xml' if 'config.d' in config_path else '/etc/clickhouse-server/users.xml' + filename = ( + "/etc/clickhouse-server/config.xml" + if "config.d" in config_path + else "/etc/clickhouse-server/users.xml" + ) with When("I restart ClickHouse server node"): with node.cluster.shell(node.name) as bash: bash.expect(bash.prompt) @@ -127,52 +133,82 @@ def restart(node, config_path, safe=False, timeout=60): bash.close() with And("getting current log size"): - logsize = \ - node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[0].strip() + logsize = ( + node.command( + "stat --format=%s /var/log/clickhouse-server/clickhouse-server.log" + ) + .output.split(" ")[0] + .strip() + ) with And("restarting ClickHouse server"): node.restart(safe=safe) - with Then("tailing the log file from using previous log size as the offset"): + with Then( + "tailing the log file from using previous log size as the offset" + ): bash.prompt = bash.__class__.prompt bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + bash.send( + f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log" + ) with And("waiting for config reload message in the log file"): bash.expect( f"ConfigReloader: Loaded config '{filename}', performed update on configuration", - timeout=timeout) + timeout=timeout, + ) @TestStep -def check_wrong_config(self, node, client, config_path, modify_file, log_error="", output="", - tail=120, timeout=60, healthy_on_restart=True): - """Check that ClickHouse errors when trying to load invalid configuration file. - """ +def check_wrong_config( + self, + node, + client, + config_path, + modify_file, + log_error="", + output="", + tail=120, + timeout=60, + healthy_on_restart=True, +): + """Check that ClickHouse errors when trying to load invalid configuration file.""" preprocessed_name = "config.xml" if "config.d" in config_path else "users.xml" - full_config_path = "/etc/clickhouse-server/config.d/kerberos.xml" if "config.d" in config_path else "/etc/clickhouse-server/users.d/kerberos-users.xml" + full_config_path = ( + "/etc/clickhouse-server/config.d/kerberos.xml" + if "config.d" in config_path + else "/etc/clickhouse-server/users.d/kerberos-users.xml" + ) uid = getuid() try: with Given("I save config file to restore it later"): - with open(config_path, 'r') as f: + with open(config_path, "r") as f: initial_contents = f.read() with And("I prepare the error log by writing empty lines into it"): - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + node.command( + 'echo -e "%s" > /var/log/clickhouse-server/clickhouse-server.err.log' + % ("-\\n" * tail) + ) with When("I modify xml file"): root = xml_parse_file(config_path) root = modify_file(root) root.append(xmltree.fromstring(f"{uid}")) - config_contents = xmltree.tostring(root, encoding='utf8', method='xml').decode('utf-8') + config_contents = xmltree.tostring( + root, encoding="utf8", method="xml" + ).decode("utf-8") command = f"cat < {full_config_path}\n{config_contents}\nHEREDOC" node.command(command, steps=False, exitcode=0) time.sleep(1) - with Then(f"{preprocessed_name} should be updated", description=f"timeout {timeout}"): + with Then( + f"{preprocessed_name} should be updated", description=f"timeout {timeout}" + ): started = time.time() command = f"cat /var/lib/clickhouse/preprocessed_configs/{preprocessed_name} | grep {uid} > /dev/null" while time.time() - started < timeout: @@ -190,7 +226,6 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" else: node.restart(safe=False, wait_healthy=False) - if output != "": with Then(f"check {output} is in output"): time.sleep(5) @@ -209,7 +244,7 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" finally: with Finally("I restore original config"): with By("restoring the (correct) config file"): - with open(config_path, 'w') as f: + with open(config_path, "w") as f: f.write(initial_contents) with And("restarting the node"): node.restart(safe=False) @@ -217,7 +252,7 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" if log_error != "": with Then("error log should contain the expected error message"): started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{log_error}\"" + command = f'tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep "{log_error}"' while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode if exitcode == 0: @@ -227,7 +262,9 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" @TestStep(Given) -def instrument_clickhouse_server_log(self, clickhouse_server_log="/var/log/clickhouse-server/clickhouse-server.log"): +def instrument_clickhouse_server_log( + self, clickhouse_server_log="/var/log/clickhouse-server/clickhouse-server.log" +): """Instrument clickhouse-server.log for the current test by adding start and end messages that include current test name to the clickhouse-server.log of the specified node and @@ -239,6 +276,10 @@ def instrument_clickhouse_server_log(self, clickhouse_server_log="/var/log/click for node in all_nodes: if node.name != "kerberos": with When(f"output stats for {node.repr()}"): - node.command(f"echo -e \"\\n-- {current().name} -- top --\\n\" && top -bn1") - node.command(f"echo -e \"\\n-- {current().name} -- df --\\n\" && df -h") - node.command(f"echo -e \"\\n-- {current().name} -- free --\\n\" && free -mh") + node.command( + f'echo -e "\\n-- {current().name} -- top --\\n" && top -bn1' + ) + node.command(f'echo -e "\\n-- {current().name} -- df --\\n" && df -h') + node.command( + f'echo -e "\\n-- {current().name} -- free --\\n" && free -mh' + ) diff --git a/tests/testflows/kerberos/tests/config.py b/tests/testflows/kerberos/tests/config.py index 35cec9527d8..e682858d557 100644 --- a/tests/testflows/kerberos/tests/config.py +++ b/tests/testflows/kerberos/tests/config.py @@ -8,9 +8,7 @@ import itertools @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled("1.0")) def kerberos_not_enabled(self): """ClickHouse SHALL reject Kerberos authentication if user is properly configured for Kerberos, but Kerberos itself is not enabled in config.xml. @@ -21,17 +19,19 @@ def kerberos_not_enabled(self): def modify_file(root): return xmltree.fromstring("") - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - output="Kerberos is not enabled") + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + output="Kerberos is not enabled", + ) @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections("1.0")) def multiple_kerberos(self): - """ClickHouse SHALL disable Kerberos authentication if more than one kerberos sections specified in config.xml. - """ + """ClickHouse SHALL disable Kerberos authentication if more than one kerberos sections specified in config.xml.""" ch_nodes = self.context.ch_nodes config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" @@ -40,14 +40,18 @@ def multiple_kerberos(self): root.append(xmltree.fromstring(second_section)) return root - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - log_error="Multiple kerberos sections are not allowed", healthy_on_restart=False) + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + log_error="Multiple kerberos sections are not allowed", + healthy_on_restart=False, + ) @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_WrongUserRealm("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_WrongUserRealm("1.0")) def wrong_user_realm(self): """ClickHouse SHALL reject Kerberos authentication if user's realm specified in users.xml doesn't match the realm of the principal trying to authenticate. @@ -57,18 +61,21 @@ def wrong_user_realm(self): config_path = f"kerberos/configs/{ch_nodes[0].name}/users.d/kerberos-users.xml" def modify_file(root): - krb = root.find('users').find('kerberos_user') - krb.find('kerberos').find('realm').text = "OTHER.COM" + krb = root.find("users").find("kerberos_user") + krb.find("kerberos").find("realm").text = "OTHER.COM" return root - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - output="Authentication failed") + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + output="Authentication failed", + ) @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods("1.0")) def multiple_auth_methods(self): """ClickHouse SHALL reject Kerberos authentication if other auth method is specified for user alongside with Kerberos. @@ -77,83 +84,98 @@ def multiple_auth_methods(self): config_path = f"kerberos/configs/{ch_nodes[0].name}/users.d/kerberos-users.xml" def modify_file(root): - krb = root.find('users').find('kerberos_user') - xml_append(krb, 'password', 'qwerty') + krb = root.find("users").find("kerberos_user") + xml_append(krb, "password", "qwerty") return root - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - log_error="More than one field of", healthy_on_restart=False) + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + log_error="More than one field of", + healthy_on_restart=False, + ) @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified("1.0")) def principal_and_realm_specified(self): - """ClickHouse SHALL drop an exception if both realm and principal fields are specified in config.xml. - """ + """ClickHouse SHALL drop an exception if both realm and principal fields are specified in config.xml.""" ch_nodes = self.context.ch_nodes config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" def modify_file(root): - krb = root.find('kerberos') - xml_append(krb, 'principal', 'HTTP/srv1@EXAMPLE.COM') + krb = root.find("kerberos") + xml_append(krb, "principal", "HTTP/srv1@EXAMPLE.COM") return root - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - log_error="Realm and principal name cannot be specified simultaneously", - output="Kerberos is not enabled") + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + log_error="Realm and principal name cannot be specified simultaneously", + output="Kerberos is not enabled", + ) @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections("1.0")) def multiple_realm(self): - """ClickHouse SHALL throw an exception and disable Kerberos if more than one realm is specified in config.xml. - """ + """ClickHouse SHALL throw an exception and disable Kerberos if more than one realm is specified in config.xml.""" ch_nodes = self.context.ch_nodes config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" def modify_file(root): - krb = root.find('kerberos') - xml_append(krb, 'realm', 'EXAM.COM') + krb = root.find("kerberos") + xml_append(krb, "realm", "EXAM.COM") return root - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - log_error="Multiple realm sections are not allowed") + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + log_error="Multiple realm sections are not allowed", + ) @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections("1.0")) def multiple_principal(self): - """ClickHouse SHALL throw an exception and disable Kerberos if more than one principal is specified in config.xml. - """ + """ClickHouse SHALL throw an exception and disable Kerberos if more than one principal is specified in config.xml.""" ch_nodes = self.context.ch_nodes config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" def modify_file(root): - krb = root.find('kerberos') - krb.remove(krb.find('realm')) - xml_append(krb, 'principal', 'HTTP/s1@EXAMPLE.COM') - xml_append(krb, 'principal', 'HTTP/s2@EXAMPLE.COM') + krb = root.find("kerberos") + krb.remove(krb.find("realm")) + xml_append(krb, "principal", "HTTP/s1@EXAMPLE.COM") + xml_append(krb, "principal", "HTTP/s2@EXAMPLE.COM") return root - check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, - log_error="Multiple principal sections are not allowed") + check_wrong_config( + node=ch_nodes[0], + client=ch_nodes[2], + config_path=config_path, + modify_file=modify_file, + log_error="Multiple principal sections are not allowed", + ) @TestFeature @Name("config") def config(self): - """Perform ClickHouse Kerberos authentication testing for incorrect configuration files - """ + """Perform ClickHouse Kerberos authentication testing for incorrect configuration files""" - self.context.ch_nodes = [self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4)] + self.context.ch_nodes = [ + self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4) + ] self.context.krb_server = self.context.cluster.node("kerberos") - self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] + self.context.clients = [ + self.context.cluster.node(f"krb-client{i}") for i in range(1, 6) + ] for scenario in loads(current_module(), Scenario, Suite): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/kerberos/tests/generic.py b/tests/testflows/kerberos/tests/generic.py index 642b99b4fc3..03629a7bdd7 100644 --- a/tests/testflows/kerberos/tests/generic.py +++ b/tests/testflows/kerberos/tests/generic.py @@ -6,12 +6,9 @@ import time @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Ping("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Ping("1.0")) def ping(self): - """Containers should be reachable - """ + """Containers should be reachable""" ch_nodes = self.context.ch_nodes for i in range(3): @@ -22,12 +19,9 @@ def ping(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser("1.0")) def xml_configured_user(self): - """ClickHouse SHALL accept Kerberos authentication for valid XML-configured user - """ + """ClickHouse SHALL accept Kerberos authentication for valid XML-configured user""" ch_nodes = self.context.ch_nodes with Given("kinit for client"): @@ -44,12 +38,9 @@ def xml_configured_user(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser("1.0")) def rbac_configured_user(self): - """ClickHouse SHALL accept Kerberos authentication for valid RBAC-configured user - """ + """ClickHouse SHALL accept Kerberos authentication for valid RBAC-configured user""" ch_nodes = self.context.ch_nodes with Given("kinit for client"): @@ -59,7 +50,9 @@ def rbac_configured_user(self): create_server_principal(node=ch_nodes[0]) with When("I create a RBAC user"): - ch_nodes[0].query("CREATE USER krb_rbac IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query( + "CREATE USER krb_rbac IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'" + ) with When("I attempt to authenticate"): r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) @@ -72,9 +65,7 @@ def rbac_configured_user(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket("1.0")) def invalid_server_ticket(self): """ClickHouse SHALL reject Kerberos authentication no Kerberos server is reachable and CH-server has no valid ticket (or the existing ticket is outdated). @@ -99,7 +90,10 @@ def invalid_server_ticket(self): while True: kinit_no_keytab(node=ch_nodes[2]) create_server_principal(node=ch_nodes[0]) - if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + if ( + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output + == "kerberos_user" + ): break debug(test_select_query(node=ch_nodes[0])) ch_nodes[2].cmd("kdestroy") @@ -109,12 +103,10 @@ def invalid_server_ticket(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket("1.0")) def invalid_client_ticket(self): """ClickHouse SHALL reject Kerberos authentication in case client has - no valid ticket (or the existing ticket is outdated). + no valid ticket (or the existing ticket is outdated). """ ch_nodes = self.context.ch_nodes @@ -142,15 +134,16 @@ def invalid_client_ticket(self): ch_nodes[2].cmd(f"echo pwd | kinit -l 10:00 kerberos_user") while True: time.sleep(1) - if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + if ( + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output + == "kerberos_user" + ): break ch_nodes[2].cmd("kdestroy") @TestCase -@Requirements( - RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets("1.0")) def kerberos_unreachable_valid_tickets(self): """ClickHouse SHALL accept Kerberos authentication if no Kerberos server is reachable but both CH-server and client have valid tickets. @@ -180,28 +173,30 @@ def kerberos_unreachable_valid_tickets(self): ch_nodes[2].cmd("kdestroy") while True: kinit_no_keytab(node=ch_nodes[2]) - if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + if ( + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output + == "kerberos_user" + ): break ch_nodes[2].cmd("kdestroy") @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured("1.0")) def kerberos_not_configured(self): - """ClickHouse SHALL reject Kerberos authentication if user is not a kerberos-auth user. - """ + """ClickHouse SHALL reject Kerberos authentication if user is not a kerberos-auth user.""" ch_nodes = self.context.ch_nodes with Given("kinit for client"): kinit_no_keytab(node=ch_nodes[2], principal="unkerberized") - with And('Kinit for server'): + with And("Kinit for server"): create_server_principal(node=ch_nodes[0]) with By("I add non-Kerberos user to ClickHouse"): - ch_nodes[0].query("CREATE USER unkerberized IDENTIFIED WITH plaintext_password BY 'qwerty'") + ch_nodes[0].query( + "CREATE USER unkerberized IDENTIFIED WITH plaintext_password BY 'qwerty'" + ) with When("I attempt to authenticate"): r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) @@ -214,12 +209,9 @@ def kerberos_not_configured(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_KerberosServerRestarted("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_KerberosServerRestarted("1.0")) def kerberos_server_restarted(self): - """ClickHouse SHALL accept Kerberos authentication if Kerberos server was restarted. - """ + """ClickHouse SHALL accept Kerberos authentication if Kerberos server was restarted.""" ch_nodes = self.context.ch_nodes krb_server = self.context.krb_server @@ -241,7 +233,10 @@ def kerberos_server_restarted(self): ch_nodes[2].cmd("kdestroy") while True: kinit_no_keytab(node=ch_nodes[2]) - if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + if ( + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output + == "kerberos_user" + ): break with Then(f"I expect kerberos_user"): @@ -249,12 +244,9 @@ def kerberos_server_restarted(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_InvalidUser("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_InvalidUser("1.0")) def invalid_user(self): - """ClickHouse SHALL reject Kerberos authentication for invalid principal - """ + """ClickHouse SHALL reject Kerberos authentication for invalid principal""" ch_nodes = self.context.ch_nodes with Given("I obtain keytab for invalid user"): @@ -267,16 +259,16 @@ def invalid_user(self): r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) with Then(f"I expect default"): - assert "Authentication failed: password is incorrect or there is no user with such name" in r.output, error() + assert ( + "Authentication failed: password is incorrect or there is no user with such name" + in r.output + ), error() @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_InvalidUser_UserDeleted("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_InvalidUser_UserDeleted("1.0")) def user_deleted(self): - """ClickHouse SHALL reject Kerberos authentication if Kerberos user was deleted prior to query. - """ + """ClickHouse SHALL reject Kerberos authentication if Kerberos user was deleted prior to query.""" ch_nodes = self.context.ch_nodes with Given("I obtain keytab for a user"): @@ -286,23 +278,25 @@ def user_deleted(self): create_server_principal(node=ch_nodes[0]) with And("I create and then delete kerberized user"): - ch_nodes[0].query("CREATE USER krb_rbac IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query( + "CREATE USER krb_rbac IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'" + ) ch_nodes[0].query("DROP USER krb_rbac") with When("I attempt to authenticate"): r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) with Then(f"I expect error"): - assert "Authentication failed: password is incorrect or there is no user with such name" in r.output, error() + assert ( + "Authentication failed: password is incorrect or there is no user with such name" + in r.output + ), error() @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Performance("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Performance("1.0")) def authentication_performance(self): - """ClickHouse's performance for Kerberos authentication SHALL shall be comparable to regular authentication. - """ + """ClickHouse's performance for Kerberos authentication SHALL shall be comparable to regular authentication.""" ch_nodes = self.context.ch_nodes with Given("I obtain keytab for a user"): @@ -312,7 +306,9 @@ def authentication_performance(self): create_server_principal(node=ch_nodes[0]) with And("I create a password-identified user"): - ch_nodes[0].query("CREATE USER pwd_user IDENTIFIED WITH plaintext_password BY 'pwd'") + ch_nodes[0].query( + "CREATE USER pwd_user IDENTIFIED WITH plaintext_password BY 'pwd'" + ) with When("I measure kerberos auth time"): start_time_krb = time.time() @@ -323,11 +319,17 @@ def authentication_performance(self): with And("I measure password auth time"): start_time_usual = time.time() for i in range(100): - ch_nodes[2].cmd(f"echo 'SELECT 1' | curl 'http://pwd_user:pwd@clickhouse1:8123/' -d @-") + ch_nodes[2].cmd( + f"echo 'SELECT 1' | curl 'http://pwd_user:pwd@clickhouse1:8123/' -d @-" + ) usual_time = (time.time() - start_time_usual) / 100 with Then("measuring the performance compared to password auth"): - metric("percentage_improvement", units="%", value=100*(krb_time - usual_time)/usual_time) + metric( + "percentage_improvement", + units="%", + value=100 * (krb_time - usual_time) / usual_time, + ) with Finally("I drop pwd_user"): ch_nodes[0].query("DROP USER pwd_user") @@ -335,12 +337,15 @@ def authentication_performance(self): @TestFeature def generic(self): - """Perform ClickHouse Kerberos authentication testing - """ + """Perform ClickHouse Kerberos authentication testing""" - self.context.ch_nodes = [self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4)] + self.context.ch_nodes = [ + self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4) + ] self.context.krb_server = self.context.cluster.node("kerberos") - self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] + self.context.clients = [ + self.context.cluster.node(f"krb-client{i}") for i in range(1, 6) + ] for scenario in loads(current_module(), Scenario, Suite): - Scenario(run=scenario, flags=TE) #, setup=instrument_clickhouse_server_log) + Scenario(run=scenario, flags=TE) # , setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/kerberos/tests/parallel.py b/tests/testflows/kerberos/tests/parallel.py index 5d352af7df4..4c1c988baff 100644 --- a/tests/testflows/kerberos/tests/parallel.py +++ b/tests/testflows/kerberos/tests/parallel.py @@ -2,20 +2,18 @@ from testflows.core import * from kerberos.tests.common import * from kerberos.requirements.requirements import * + @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials("1.0")) def valid_requests_same_credentials(self): - """ClickHouse should be able to process parallel requests sent under the same credentials. - """ + """ClickHouse should be able to process parallel requests sent under the same credentials.""" ch_nodes = self.context.ch_nodes with Given("kinit for clients"): kinit_no_keytab(node=ch_nodes[1]) kinit_no_keytab(node=ch_nodes[2]) - with And('create server principal'): + with And("create server principal"): create_server_principal(node=ch_nodes[0]) def helper(cmd): @@ -25,8 +23,8 @@ def valid_requests_same_credentials(self): tasks = [] with Pool(2) as pool: with When("I try simultaneous authentication"): - tasks.append(pool.submit(helper, (ch_nodes[1].cmd, ))) - tasks.append(pool.submit(helper, (ch_nodes[2].cmd, ))) + tasks.append(pool.submit(helper, (ch_nodes[1].cmd,))) + tasks.append(pool.submit(helper, (ch_nodes[2].cmd,))) tasks[0].result(timeout=200) tasks[1].result(timeout=200) @@ -36,12 +34,9 @@ def valid_requests_same_credentials(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials("1.0")) def valid_requests_different_credentials(self): - """ClickHouse should be able to process parallel requests by different users. - """ + """ClickHouse should be able to process parallel requests by different users.""" ch_nodes = self.context.ch_nodes with Given("kinit for clients"): @@ -59,12 +54,16 @@ def valid_requests_different_credentials(self): tasks = [] with Pool(2) as pool: with And("add 2 kerberos users via RBAC"): - ch_nodes[0].query("CREATE USER krb1 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") - ch_nodes[0].query("CREATE USER krb2 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query( + "CREATE USER krb1 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'" + ) + ch_nodes[0].query( + "CREATE USER krb2 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'" + ) with When("I try simultaneous authentication for valid and invalid"): - tasks.append(pool.submit(helper, (ch_nodes[1].cmd, ))) - tasks.append(pool.submit(helper, (ch_nodes[2].cmd, ))) + tasks.append(pool.submit(helper, (ch_nodes[1].cmd,))) + tasks.append(pool.submit(helper, (ch_nodes[2].cmd,))) tasks[0].result(timeout=200) tasks[1].result(timeout=200) @@ -78,19 +77,16 @@ def valid_requests_different_credentials(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Parallel_ValidInvalid("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Parallel_ValidInvalid("1.0")) def valid_invalid(self): - """Valid users' Kerberos authentication should not be affected by invalid users' attempts. - """ + """Valid users' Kerberos authentication should not be affected by invalid users' attempts.""" ch_nodes = self.context.ch_nodes with Given("kinit for clients"): kinit_no_keytab(node=ch_nodes[2]) kinit_no_keytab(node=ch_nodes[1], principal="invalid_user") - with And('create server principal'): + with And("create server principal"): create_server_principal(node=ch_nodes[0]) def helper(cmd): @@ -100,8 +96,8 @@ def valid_invalid(self): tasks = [] with Pool(2) as pool: with When("I try simultaneous authentication for valid and invalid"): - tasks.append(pool.submit(helper, (ch_nodes[1].cmd,))) # invalid - tasks.append(pool.submit(helper, (ch_nodes[2].cmd,))) # valid + tasks.append(pool.submit(helper, (ch_nodes[1].cmd,))) # invalid + tasks.append(pool.submit(helper, (ch_nodes[2].cmd,))) # valid with Then(f"I expect have auth failure"): assert tasks[1].result(timeout=300).output == "kerberos_user", error() @@ -109,12 +105,9 @@ def valid_invalid(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Parallel_Deletion("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Parallel_Deletion("1.0")) def deletion(self): - """ClickHouse SHALL NOT crash when 2 Kerberos users are simultaneously deleting one another. - """ + """ClickHouse SHALL NOT crash when 2 Kerberos users are simultaneously deleting one another.""" ch_nodes = self.context.ch_nodes with Given("kinit for clients"): @@ -125,18 +118,24 @@ def deletion(self): create_server_principal(node=ch_nodes[0]) def helper(cmd, todel): - return cmd(test_select_query(node=ch_nodes[0], req=f"DROP USER {todel}"), no_checks=True) + return cmd( + test_select_query(node=ch_nodes[0], req=f"DROP USER {todel}"), + no_checks=True, + ) for i in range(15): tasks = [] with Pool(2) as pool: with And("add 2 kerberos users via RBAC"): - ch_nodes[0].query("CREATE USER krb1 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") - ch_nodes[0].query("CREATE USER krb2 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query( + "CREATE USER krb1 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'" + ) + ch_nodes[0].query( + "CREATE USER krb2 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'" + ) ch_nodes[0].query("GRANT ACCESS MANAGEMENT ON *.* TO krb1") ch_nodes[0].query("GRANT ACCESS MANAGEMENT ON *.* TO krb2") - with When("I try simultaneous authentication for valid and invalid"): tasks.append(pool.submit(helper, (ch_nodes[1].cmd, "krb2"))) tasks.append(pool.submit(helper, (ch_nodes[2].cmd, "krb1"))) @@ -152,28 +151,29 @@ def deletion(self): @TestScenario -@Requirements( - RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos("1.0")) def kerberos_and_nonkerberos(self): - """ClickHouse SHALL support processing of simultaneous kerberized and non-kerberized requests. - """ + """ClickHouse SHALL support processing of simultaneous kerberized and non-kerberized requests.""" ch_nodes = self.context.ch_nodes with Given("kinit for clients"): kinit_no_keytab(node=ch_nodes[2]) - with And('create server principal'): + with And("create server principal"): create_server_principal(node=ch_nodes[0]) def helper(cmd, krb_auth): - return cmd(test_select_query(node=ch_nodes[0], krb_auth=krb_auth), no_checks=True) + return cmd( + test_select_query(node=ch_nodes[0], krb_auth=krb_auth), no_checks=True + ) for i in range(15): tasks = [] with Pool(2) as pool: with When("I try simultaneous authentication for valid and invalid"): - tasks.append(pool.submit(helper, (ch_nodes[1].cmd, False))) # non-kerberos + tasks.append( + pool.submit(helper, (ch_nodes[1].cmd, False)) + ) # non-kerberos tasks.append(pool.submit(helper, (ch_nodes[2].cmd, True))) # kerberos with Then(f"I expect have auth failure"): @@ -182,16 +182,17 @@ def kerberos_and_nonkerberos(self): @TestFeature -@Requirements( - RQ_SRS_016_Kerberos_Parallel("1.0") -) +@Requirements(RQ_SRS_016_Kerberos_Parallel("1.0")) def parallel(self): - """Perform ClickHouse Kerberos authentication testing for incorrect configuration files - """ + """Perform ClickHouse Kerberos authentication testing for incorrect configuration files""" - self.context.ch_nodes = [self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4)] + self.context.ch_nodes = [ + self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4) + ] self.context.krb_server = self.context.cluster.node("kerberos") - self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] + self.context.clients = [ + self.context.cluster.node(f"krb-client{i}") for i in range(1, 6) + ] for scenario in loads(current_module(), Scenario, Suite): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/ldap/authentication/authentication_env/clickhouse-service.yml b/tests/testflows/ldap/authentication/authentication_env/clickhouse-service.yml new file mode 100644 index 00000000000..74661f6fa04 --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env/clickhouse-service.yml @@ -0,0 +1,29 @@ +version: '2.3' + +services: + clickhouse: + image: clickhouse/integration-test + init: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "tail -f /dev/null" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/authentication/authentication_env/docker-compose.yml b/tests/testflows/ldap/authentication/authentication_env/docker-compose.yml new file mode 100644 index 00000000000..36e25ef766e --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env/docker-compose.yml @@ -0,0 +1,162 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/authentication/authentication_env/openldap-service.yml b/tests/testflows/ldap/authentication/authentication_env/openldap-service.yml new file mode 100644 index 00000000000..606ea3f723f --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env/openldap-service.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + environment: + PHPLDAPADMIN_HTTPS=false: + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/authentication/authentication_env/zookeeper-service.yml b/tests/testflows/ldap/authentication/authentication_env/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/authentication/authentication_env_arm64/clickhouse-service.yml b/tests/testflows/ldap/authentication/authentication_env_arm64/clickhouse-service.yml new file mode 100644 index 00000000000..a73d31421c8 --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env_arm64/clickhouse-service.yml @@ -0,0 +1,29 @@ +version: '2.3' + +services: + clickhouse: + image: registry.gitlab.com/altinity-public/container-images/test/clickhouse-integration-test:21.12 + privileged: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 10 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/authentication/authentication_env_arm64/docker-compose.yml b/tests/testflows/ldap/authentication/authentication_env_arm64/docker-compose.yml new file mode 100644 index 00000000000..36e25ef766e --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env_arm64/docker-compose.yml @@ -0,0 +1,162 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/authentication/authentication_env_arm64/openldap-service.yml b/tests/testflows/ldap/authentication/authentication_env_arm64/openldap-service.yml new file mode 100644 index 00000000000..606ea3f723f --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env_arm64/openldap-service.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + environment: + PHPLDAPADMIN_HTTPS=false: + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/authentication/authentication_env_arm64/zookeeper-service.yml b/tests/testflows/ldap/authentication/authentication_env_arm64/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/authentication/authentication_env_arm64/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/authentication/configs/clickhouse/common.xml b/tests/testflows/ldap/authentication/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/ldap/authentication/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/ldap/authentication/configs/clickhouse/config.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml deleted file mode 100644 index 53ffa10384e..00000000000 --- a/tests/testflows/ldap/authentication/configs/clickhouse/config.xml +++ /dev/null @@ -1,442 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 7200 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/ldap/authentication/configs/clickhouse/users.xml b/tests/testflows/ldap/authentication/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/ldap/authentication/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index 177f486e18a..d2e541598ea 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -11,46 +11,64 @@ from ldap.authentication.requirements import * # Cross-outs of known fails xfails = { - "connection protocols/tls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls require cert default demand": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls with custom port": - [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], - "connection protocols/tls cipher suite": - [(Fail, "can't get it to work")] + "connection protocols/tls/tls_require_cert='try'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/tls/tls_require_cert='demand'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/starttls/tls_require_cert='try'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/starttls/tls_require_cert='demand'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/tls require cert default demand": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/starttls with custom port": [ + ( + Fail, + "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server", + ) + ], + "connection protocols/tls cipher suite": [(Fail, "can't get it to work")], } + @TestFeature @Name("authentication") @ArgumentParser(argparser) -@Specifications( - SRS_007_ClickHouse_Authentication_of_Users_via_LDAP -) -@Requirements( - RQ_SRS_007_LDAP_Authentication("1.0") -) +@Specifications(SRS_007_ClickHouse_Authentication_of_Users_via_LDAP) +@Requirements(RQ_SRS_007_LDAP_Authentication("1.0")) @XFails(xfails) -def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): - """ClickHouse integration with LDAP regression module. - """ +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse integration with LDAP regression module.""" nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), } + self.context.clickhouse_version = clickhouse_version + if stress is not None: self.context.stress = stress - if parallel is not None: - self.context.parallel = parallel - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "ldap_authentication_env")) as cluster: + from platform import processor as current_cpu + + folder_name = os.path.basename(current_dir()) + if current_cpu() == "aarch64": + env = f"{folder_name}_env_arm64" + else: + env = f"{folder_name}_env" + + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), env), + ) as cluster: self.context.cluster = cluster Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) @@ -60,5 +78,6 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): Feature(run=load("ldap.authentication.tests.user_config", "feature")) Feature(run=load("ldap.authentication.tests.authentications", "feature")) + if main(): regression() diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 97c85d93c86..6ee904bd40e 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -9,1204 +9,1269 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_007_LDAP_Authentication = Requirement( - name='RQ.SRS-007.LDAP.Authentication', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication via an [LDAP] server.\n" "\n" + ), link=None, level=3, - num='4.1.1') + num="4.1.1", +) RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( - name='RQ.SRS-007.LDAP.Authentication.MultipleServers', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.MultipleServers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' - 'users.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n" + "users.\n" + "\n" + ), link=None, level=3, - num='4.1.2') + num="4.1.2", +) RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Protocol.PlainText", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n" + "\n" + ), link=None, level=3, - num='4.1.3') + num="4.1.3", +) RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Protocol.TLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n" + "\n" + ), link=None, level=3, - num='4.1.4') + num="4.1.4", +) RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' - 'plain text `ldap://` protocol that is upgraded to [TLS].\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n" + "plain text `ldap://` protocol that is upgraded to [TLS].\n" + "\n" + ), link=None, level=3, - num='4.1.5') + num="4.1.5", +) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( - name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' - '\n' - ), + "[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n" + "\n" + ), link=None, level=3, - num='4.1.6') + num="4.1.6", +) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( - name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' - '\n' - ), + "[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n" + "\n" + ), link=None, level=3, - num='4.1.7') + num="4.1.7", +) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( - name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' - '\n' - ), + "[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n" + "\n" + ), link=None, level=3, - num='4.1.8') + num="4.1.8", +) RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( - name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', - version='1.0', + name="RQ.SRS-007.LDAP.Server.Configuration.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n" + "\n" + ), link=None, level=3, - num='4.1.9') + num="4.1.9", +) RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( - name='RQ.SRS-007.LDAP.User.Configuration.Invalid', - version='1.0', + name="RQ.SRS-007.LDAP.User.Configuration.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n" + "\n" + ), link=None, level=3, - num='4.1.10') + num="4.1.10", +) RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n" + "authentication mechanism.\n" + "\n" + ), link=None, level=3, - num='4.1.11') + num="4.1.11", +) RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n" + "authentication mechanism.\n" + "\n" + ), link=None, level=3, - num='4.1.12') + num="4.1.12", +) RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism.\n' - '\n' - ), + "[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n" + "authentication mechanism.\n" + "\n" + ), link=None, level=3, - num='4.1.13') + num="4.1.13", +) RQ_SRS_007_LDAP_Authentication_Valid = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Valid', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Valid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' - 'user name and password match [LDAP] server records for the user.\n' - '\n' - ), + "[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n" + "user name and password match [LDAP] server records for the user.\n" + "\n" + ), link=None, level=3, - num='4.1.14') + num="4.1.14", +) RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Invalid', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' - 'do not match [LDAP] server records for the user.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n" + "do not match [LDAP] server records for the user.\n" + "\n" + ), link=None, level=3, - num='4.1.15') + num="4.1.15", +) RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' - 'has been deleted from the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if the user\n" + "has been deleted from the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.1.16') + num="4.1.16", +) RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( - name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.UsernameChanged", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' - 'on the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n" + "on the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.1.17') + num="4.1.17", +) RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( - name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.PasswordChanged", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' - 'for the user is changed on the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if the password\n" + "for the user is changed on the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.1.18') + num="4.1.18", +) RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( - name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.LDAPServerRestart", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n" + "\n" + ), link=None, level=3, - num='4.1.19') + num="4.1.19", +) RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( - name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users after server is restarted.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users after server is restarted.\n" + "\n" + ), link=None, level=3, - num='4.1.20') + num="4.1.20", +) RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Parallel', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.1.21') + num="4.1.21", +) RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authentication of valid users and\n' - 'prohibit authentication of invalid users using [LDAP] server\n' - 'in parallel without having invalid attempts affecting valid authentications.\n' - '\n' - ), + "[ClickHouse] SHALL support authentication of valid users and\n" + "prohibit authentication of invalid users using [LDAP] server\n" + "in parallel without having invalid attempts affecting valid authentications.\n" + "\n" + ), link=None, level=3, - num='4.1.22') + num="4.1.22", +) RQ_SRS_007_LDAP_UnreachableServer = Requirement( - name='RQ.SRS-007.LDAP.UnreachableServer', - version='1.0', + name="RQ.SRS-007.LDAP.UnreachableServer", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n" + "\n" + ), link=None, level=3, - num='4.2.1') + num="4.2.1", +) RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Name', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.Name", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support empty string as a server name.\n' - '\n' - ), + "[ClickHouse] SHALL not support empty string as a server name.\n" "\n" + ), link=None, level=3, - num='4.2.2') + num="4.2.2", +) RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Host', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.Host", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' - 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify [LDAP]\n" + "server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n" + "\n" + ), link=None, level=3, - num='4.2.3') + num="4.2.3", +) RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Port', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.Port", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n" + "\n" + ), link=None, level=3, - num='4.2.4') + num="4.2.4", +) RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.Port.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' - '\n' - ), + "[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n" + "\n" + ), link=None, level=3, - num='4.2.5') + num="4.2.5", +) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify the prefix\n' - 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify the prefix\n" + "of value used to construct the DN to bound to during authentication via [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.6') + num="4.2.6", +) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify the suffix\n' - 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify the suffix\n" + "of value used to construct the DN to bound to during authentication via [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.7') + num="4.2.7", +) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' - '\n' + "[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n" + "\n" "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.2.8') + num="4.2.8", +) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.9') + num="4.2.9", +) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' - 'to enable SSL/TLS `ldaps://` protocol.\n' - '\n' - ), + "[ClickHouse] SHALL use `yes` value as the default for `` parameter\n" + "to enable SSL/TLS `ldaps://` protocol.\n" + "\n" + ), link=None, level=3, - num='4.2.10') + num="4.2.10", +) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' - 'plain text `ldap://` protocol.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n" + "plain text `ldap://` protocol.\n" + "\n" + ), link=None, level=3, - num='4.2.11') + num="4.2.11", +) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' - 'SSL/TLS `ldaps://` protocol.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n" + "SSL/TLS `ldaps://` protocol.\n" + "\n" + ), link=None, level=3, - num='4.2.12') + num="4.2.12", +) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' - 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n" + "legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n" + "\n" + ), link=None, level=3, - num='4.2.13') + num="4.2.13", +) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify\n' - 'the minimum protocol version of SSL/TLS.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify\n" + "the minimum protocol version of SSL/TLS.\n" + "\n" + ), link=None, level=3, - num='4.2.14') + num="4.2.14", +) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' - 'as a value of the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n" + "as a value of the `` parameter.\n" + "\n" + ), link=None, level=3, - num='4.2.15') + num="4.2.15", +) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n" + "\n" + ), link=None, level=3, - num='4.2.16') + num="4.2.16", +) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' - 'certificate verification behavior.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n" + "certificate verification behavior.\n" + "\n" + ), link=None, level=3, - num='4.2.17') + num="4.2.17", +) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n" + "\n" + ), link=None, level=3, - num='4.2.18') + num="4.2.18", +) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' - 'provided, the session SHALL be immediately terminated.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n" + "enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n" + "provided, the session SHALL be immediately terminated.\n" + "\n" + ), link=None, level=3, - num='4.2.19') + num="4.2.19", +) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no\n' - 'certificate is provided, the session SHALL proceed normally.\n' - 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n" + "enable requesting of client certificate. If no\n" + "certificate is provided, the session SHALL proceed normally.\n" + "If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n" + "\n" + ), link=None, level=3, - num='4.2.20') + num="4.2.20", +) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no certificate is provided, the session\n' - 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' - 'immediately terminated.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n" + "enable requesting of client certificate. If no certificate is provided, the session\n" + "SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n" + "immediately terminated.\n" + "\n" + ), link=None, level=3, - num='4.2.21') + num="4.2.21", +) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' - 'disable requesting of client certificate.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n" + "disable requesting of client certificate.\n" + "\n" + ), link=None, level=3, - num='4.2.22') + num="4.2.22", +) RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' - '[ClickHouse] to establish connection with the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` to specify the path to certificate file used by\n" + "[ClickHouse] to establish connection with the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.23') + num="4.2.23", +) RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' - 'specified by the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n" + "specified by the `` parameter.\n" + "\n" + ), link=None, level=3, - num='4.2.24') + num="4.2.24", +) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify to a path to\n' - 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify to a path to\n" + "the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.25') + num="4.2.25", +) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' - '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify a path to a specific\n" + "[CA] certificate file used to verify certificates provided by the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.26') + num="4.2.26", +) RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n' - 'The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' - '```\n' - '\n' - 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' - '[ClickHouse] and therefore might change.\n' - '\n' - ), + "[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n" + "The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n" + "```\n" + "\n" + "The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n" + "[ClickHouse] and therefore might change.\n" + "\n" + ), link=None, level=3, - num='4.2.27') + num="4.2.27", +) RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' - 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' - 'After period of time since the last successful attempt expires then on the authentication attempt\n' - 'SHALL result in contacting the [LDAP] server to verify the username and password. \n' - '\n' - ), + "[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n" + "that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n" + "to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n" + "After period of time since the last successful attempt expires then on the authentication attempt\n" + "SHALL result in contacting the [LDAP] server to verify the username and password. \n" + "\n" + ), link=None, level=3, - num='4.2.28') + num="4.2.28", +) RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'SHALL have a default value of `0` that disables caching and forces contacting\n' - 'the [LDAP] server for each authentication request.\n' - '\n' - ), + "[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n" + "SHALL have a default value of `0` that disables caching and forces contacting\n" + "the [LDAP] server for each authentication request.\n" + "\n" + ), link=None, level=3, - num='4.2.29') + num="4.2.29", +) RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' - '\n' - 'For example:\n' - '\n' - '* negative integer\n' - '* string\n' - '* empty value\n' - '* extremely large positive value (overflow)\n' - '* extremely large negative value (overflow)\n' - '\n' - 'The error SHALL appear in the log and SHALL be similar to the following:\n' - '\n' - '```bash\n' - ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' - '```\n' - '\n' - ), + "[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n" + "\n" + "For example:\n" + "\n" + "* negative integer\n" + "* string\n" + "* empty value\n" + "* extremely large positive value (overflow)\n" + "* extremely large negative value (overflow)\n" + "\n" + "The error SHALL appear in the log and SHALL be similar to the following:\n" + "\n" + "```bash\n" + " Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.30') + num="4.2.30", +) RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( - name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', - version='2.0', + name="RQ.SRS-007.LDAP.Configuration.Server.Syntax", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n' - 'configuration file or of any configuration file inside the `config.d` directory.\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' localhost\n' - ' 636\n' - ' cn=\n' - ' , ou=users, dc=example, dc=com\n' - ' 0\n' - ' yes\n' - ' tls1.2\n' - ' demand\n' - ' /path/to/tls_cert_file\n' - ' /path/to/tls_key_file\n' - ' /path/to/tls_ca_cert_file\n' - ' /path/to/tls_ca_cert_dir\n' - ' ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n" + "configuration file or of any configuration file inside the `config.d` directory.\n" + "\n" + "```xml\n" + "\n" + " \n" + " localhost\n" + " 636\n" + " cn=\n" + " , ou=users, dc=example, dc=com\n" + " 0\n" + " yes\n" + " tls1.2\n" + " demand\n" + " /path/to/tls_cert_file\n" + " /path/to/tls_key_file\n" + " /path/to/tls_ca_cert_file\n" + " /path/to/tls_ca_cert_dir\n" + " ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.31') + num="4.2.31", +) RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.RBAC', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.RBAC", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating users identified using an [LDAP] server using\n' - 'the following RBAC command\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support creating users identified using an [LDAP] server using\n" + "the following RBAC command\n" + "\n" + "```sql\n" "CREATE USER name IDENTIFIED WITH ldap SERVER 'server_name'\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='4.2.32') + num="4.2.32", +) RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Syntax', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using\n' - 'an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory.\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' \n' - ' my_ldap_server\n' - ' \n' - ' \n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using\n" + "an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory.\n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " \n" + " my_ldap_server\n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=3, - num='4.2.33') + num="4.2.33", +) RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.Name.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, - description=( - '[ClickHouse] SHALL not support empty string as a user name.\n' - '\n' - ), + description=("[ClickHouse] SHALL not support empty string as a user name.\n" "\n"), link=None, level=3, - num='4.2.34') + num="4.2.34", +) RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' - 'time user configuration contains any of the `` entries.\n' - '\n' - ), + "[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n" + "time user configuration contains any of the `` entries.\n" + "\n" + ), link=None, level=3, - num='4.2.35') + num="4.2.35", +) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentication attempt\n' - 'if the name of the [LDAP] server used inside the `` entry\n' - 'is not defined in the `` section.\n' - '\n' - ), + "[ClickHouse] SHALL throw an error during any authentication attempt\n" + "if the name of the [LDAP] server used inside the `` entry\n" + "is not defined in the `` section.\n" + "\n" + ), link=None, level=3, - num='4.2.36') + num="4.2.36", +) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentication attempt\n' - 'if the name of the [LDAP] server used inside the `` entry\n' - 'is empty.\n' - '\n' - ), + "[ClickHouse] SHALL throw an error during any authentication attempt\n" + "if the name of the [LDAP] server used inside the `` entry\n" + "is empty.\n" + "\n" + ), link=None, level=3, - num='4.2.37') + num="4.2.37", +) RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n" + "\n" + ), link=None, level=3, - num='4.2.38') + num="4.2.38", +) RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.Name.Long", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support long user names of at least 256 bytes\n' - 'to specify users that can be authenticated using an [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support long user names of at least 256 bytes\n" + "to specify users that can be authenticated using an [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.39') + num="4.2.39", +) RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( - name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', - version='1.0', + name="RQ.SRS-007.LDAP.Configuration.User.Name.UTF8", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' - '\n' - ), + "[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n" "\n" + ), link=None, level=3, - num='4.2.40') + num="4.2.40", +) RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Username.Empty', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Username.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support authenticating users with empty username.\n' - '\n' - ), + "[ClickHouse] SHALL not support authenticating users with empty username.\n" + "\n" + ), link=None, level=3, - num='4.2.41') + num="4.2.41", +) RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Username.Long', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Username.Long", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n" + "\n" + ), link=None, level=3, - num='4.2.42') + num="4.2.42", +) RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Username.UTF8", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' - '\n' - ), + "[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n" + "\n" + ), link=None, level=3, - num='4.2.43') + num="4.2.43", +) RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Password.Empty', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Password.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support authenticating users with empty passwords\n' - 'even if an empty password is valid for the user and\n' - 'is allowed by the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL not support authenticating users with empty passwords\n" + "even if an empty password is valid for the user and\n" + "is allowed by the [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.44') + num="4.2.44", +) RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Password.Long', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Password.Long", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support long password of at least 256 bytes\n' - 'that can be used to authenticate users using an [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support long password of at least 256 bytes\n" + "that can be used to authenticate users using an [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.45') + num="4.2.45", +) RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( - name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.Password.UTF8", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' - 'used to authenticate users using an [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support [UTF-8] characters in passwords\n" + "used to authenticate users using an [LDAP] server.\n" + "\n" + ), link=None, level=3, - num='4.2.46') + num="4.2.46", +) RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance = Requirement( - name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users\n' - 'when `verification_cooldown` parameter is set to a positive value when comparing\n' - 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' - 'making a large number of repeated requests.\n' - '\n' - ), + "[ClickHouse] SHALL provide better login performance of [LDAP] authenticated users\n" + "when `verification_cooldown` parameter is set to a positive value when comparing\n" + "to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n" + "making a large number of repeated requests.\n" + "\n" + ), link=None, level=3, - num='4.2.47') + num="4.2.47", +) RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( - name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' - '`verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' - 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' + "[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n" + "`verification_cooldown` parameter in the [LDAP] server configuration section\n" + "if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n" + "change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n" "to result in contacting the [LDAP] server to verify user's username and password.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.2.48') + num="4.2.48", +) RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( - name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', - version='1.0', + name="RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' - '`verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'for the user if the password provided in the current authentication attempt does not match\n' - 'the valid password provided during the first successful authentication request that was cached\n' - 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' + "[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n" + "`verification_cooldown` parameter in the [LDAP] server configuration section\n" + "for the user if the password provided in the current authentication attempt does not match\n" + "the valid password provided during the first successful authentication request that was cached\n" + "for this exact user. The reset SHALL cause the next authentication attempt for this user\n" "to result in contacting the [LDAP] server to verify user's username and password.\n" - '\n' - ), + "\n" + ), link=None, level=3, - num='4.2.49') + num="4.2.49", +) SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( - name='SRS-007 ClickHouse Authentication of Users via LDAP', + name="SRS-007 ClickHouse Authentication of Users via LDAP", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -1218,85 +1283,297 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='Generic', level=2, num='4.1'), - Heading(name='RQ.SRS-007.LDAP.Authentication', level=3, num='4.1.1'), - Heading(name='RQ.SRS-007.LDAP.Authentication.MultipleServers', level=3, num='4.1.2'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', level=3, num='4.1.3'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', level=3, num='4.1.4'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', level=3, num='4.1.5'), - Heading(name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', level=3, num='4.1.6'), - Heading(name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', level=3, num='4.1.7'), - Heading(name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', level=3, num='4.1.8'), - Heading(name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', level=3, num='4.1.9'), - Heading(name='RQ.SRS-007.LDAP.User.Configuration.Invalid', level=3, num='4.1.10'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', level=3, num='4.1.11'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', level=3, num='4.1.12'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', level=3, num='4.1.13'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Valid', level=3, num='4.1.14'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Invalid', level=3, num='4.1.15'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', level=3, num='4.1.16'), - Heading(name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', level=3, num='4.1.17'), - Heading(name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', level=3, num='4.1.18'), - Heading(name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', level=3, num='4.1.19'), - Heading(name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', level=3, num='4.1.20'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Parallel', level=3, num='4.1.21'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', level=3, num='4.1.22'), - Heading(name='Specific', level=2, num='4.2'), - Heading(name='RQ.SRS-007.LDAP.UnreachableServer', level=3, num='4.2.1'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Name', level=3, num='4.2.2'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Host', level=3, num='4.2.3'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Port', level=3, num='4.2.4'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', level=3, num='4.2.5'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', level=3, num='4.2.6'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', level=3, num='4.2.7'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', level=3, num='4.2.8'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', level=3, num='4.2.9'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', level=3, num='4.2.10'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', level=3, num='4.2.11'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', level=3, num='4.2.12'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', level=3, num='4.2.13'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', level=3, num='4.2.14'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', level=3, num='4.2.15'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', level=3, num='4.2.16'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', level=3, num='4.2.17'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', level=3, num='4.2.18'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', level=3, num='4.2.19'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', level=3, num='4.2.20'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', level=3, num='4.2.21'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', level=3, num='4.2.22'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', level=3, num='4.2.23'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', level=3, num='4.2.24'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', level=3, num='4.2.25'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', level=3, num='4.2.26'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', level=3, num='4.2.27'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown', level=3, num='4.2.28'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default', level=3, num='4.2.29'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid', level=3, num='4.2.30'), - Heading(name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', level=3, num='4.2.31'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.RBAC', level=3, num='4.2.32'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Syntax', level=3, num='4.2.33'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', level=3, num='4.2.34'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', level=3, num='4.2.35'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', level=3, num='4.2.36'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', level=3, num='4.2.37'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', level=3, num='4.2.38'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', level=3, num='4.2.39'), - Heading(name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', level=3, num='4.2.40'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Username.Empty', level=3, num='4.2.41'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Username.Long', level=3, num='4.2.42'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', level=3, num='4.2.43'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Password.Empty', level=3, num='4.2.44'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Password.Long', level=3, num='4.2.45'), - Heading(name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', level=3, num='4.2.46'), - Heading(name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance', level=3, num='4.2.47'), - Heading(name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', level=3, num='4.2.48'), - Heading(name='RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword', level=3, num='4.2.49'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="Generic", level=2, num="4.1"), + Heading(name="RQ.SRS-007.LDAP.Authentication", level=3, num="4.1.1"), + Heading( + name="RQ.SRS-007.LDAP.Authentication.MultipleServers", level=3, num="4.1.2" ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Protocol.PlainText", + level=3, + num="4.1.3", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Protocol.TLS", level=3, num="4.1.4" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS", + level=3, + num="4.1.5", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation", + level=3, + num="4.1.6", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned", + level=3, + num="4.1.7", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority", + level=3, + num="4.1.8", + ), + Heading( + name="RQ.SRS-007.LDAP.Server.Configuration.Invalid", level=3, num="4.1.9" + ), + Heading( + name="RQ.SRS-007.LDAP.User.Configuration.Invalid", level=3, num="4.1.10" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous", + level=3, + num="4.1.11", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated", + level=3, + num="4.1.12", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword", + level=3, + num="4.1.13", + ), + Heading(name="RQ.SRS-007.LDAP.Authentication.Valid", level=3, num="4.1.14"), + Heading(name="RQ.SRS-007.LDAP.Authentication.Invalid", level=3, num="4.1.15"), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser", + level=3, + num="4.1.16", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.UsernameChanged", level=3, num="4.1.17" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.PasswordChanged", level=3, num="4.1.18" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.LDAPServerRestart", + level=3, + num="4.1.19", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart", + level=3, + num="4.1.20", + ), + Heading(name="RQ.SRS-007.LDAP.Authentication.Parallel", level=3, num="4.1.21"), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid", + level=3, + num="4.1.22", + ), + Heading(name="Specific", level=2, num="4.2"), + Heading(name="RQ.SRS-007.LDAP.UnreachableServer", level=3, num="4.2.1"), + Heading(name="RQ.SRS-007.LDAP.Configuration.Server.Name", level=3, num="4.2.2"), + Heading(name="RQ.SRS-007.LDAP.Configuration.Server.Host", level=3, num="4.2.3"), + Heading(name="RQ.SRS-007.LDAP.Configuration.Server.Port", level=3, num="4.2.4"), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.Port.Default", + level=3, + num="4.2.5", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix", + level=3, + num="4.2.6", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix", + level=3, + num="4.2.7", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value", + level=3, + num="4.2.8", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS", level=3, num="4.2.9" + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default", + level=3, + num="4.2.10", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No", + level=3, + num="4.2.11", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes", + level=3, + num="4.2.12", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS", + level=3, + num="4.2.13", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion", + level=3, + num="4.2.14", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values", + level=3, + num="4.2.15", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default", + level=3, + num="4.2.16", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert", + level=3, + num="4.2.17", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default", + level=3, + num="4.2.18", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand", + level=3, + num="4.2.19", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow", + level=3, + num="4.2.20", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try", + level=3, + num="4.2.21", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never", + level=3, + num="4.2.22", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile", + level=3, + num="4.2.23", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile", + level=3, + num="4.2.24", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir", + level=3, + num="4.2.25", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile", + level=3, + num="4.2.26", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite", + level=3, + num="4.2.27", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown", + level=3, + num="4.2.28", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Default", + level=3, + num="4.2.29", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.VerificationCooldown.Invalid", + level=3, + num="4.2.30", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.Server.Syntax", level=3, num="4.2.31" + ), + Heading(name="RQ.SRS-007.LDAP.Configuration.User.RBAC", level=3, num="4.2.32"), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.Syntax", level=3, num="4.2.33" + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.Name.Empty", level=3, num="4.2.34" + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP", + level=3, + num="4.2.35", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined", + level=3, + num="4.2.36", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty", + level=3, + num="4.2.37", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer", + level=3, + num="4.2.38", + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.Name.Long", level=3, num="4.2.39" + ), + Heading( + name="RQ.SRS-007.LDAP.Configuration.User.Name.UTF8", level=3, num="4.2.40" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Username.Empty", level=3, num="4.2.41" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Username.Long", level=3, num="4.2.42" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Username.UTF8", level=3, num="4.2.43" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Password.Empty", level=3, num="4.2.44" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Password.Long", level=3, num="4.2.45" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.Password.UTF8", level=3, num="4.2.46" + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Performance", + level=3, + num="4.2.47", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters", + level=3, + num="4.2.48", + ), + Heading( + name="RQ.SRS-007.LDAP.Authentication.VerificationCooldown.Reset.InvalidPassword", + level=3, + num="4.2.49", + ), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_007_LDAP_Authentication, RQ_SRS_007_LDAP_Authentication_MultipleServers, @@ -1369,8 +1646,8 @@ SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance, RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters, RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_InvalidPassword, - ), - content=''' + ), + content=""" # SRS-007 ClickHouse Authentication of Users via LDAP # Software Requirements Specification @@ -1982,4 +2259,5 @@ to result in contacting the [LDAP] server to verify user's username and password [GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md [Git]: https://git-scm.com/ -''') +""", +) diff --git a/tests/testflows/ldap/authentication/tests/authentications.py b/tests/testflows/ldap/authentication/tests/authentications.py index 1902e0bc2cb..8f98adce746 100644 --- a/tests/testflows/ldap/authentication/tests/authentications.py +++ b/tests/testflows/ldap/authentication/tests/authentications.py @@ -2,7 +2,6 @@ import random import time -from helpers.common import Pool from testflows.core import * from testflows.asserts import error from ldap.authentication.tests.common import * @@ -14,7 +13,7 @@ servers = { "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { "host": "openldap2", @@ -23,24 +22,38 @@ servers = { "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", "tls_require_cert": "never", - } + }, } + @TestStep(When) @Name("I login as {username} and execute query") @Args(format_name=True) -def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True): - """Execute query as some user. - """ - self.context.node.query("SELECT 1", +def login_and_execute_query( + self, username, password, exitcode=None, message=None, steps=True +): + """Execute query as some user.""" + self.context.node.query( + "SELECT 1", settings=[("user", username), ("password", password)], exitcode=exitcode or 0, - message=message, steps=steps) + message=message, + steps=steps, + ) + @TestScenario -def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None, rbac=False): - """Add user to LDAP and ClickHouse and then try to login. - """ +def add_user_to_ldap_and_login( + self, + server, + user=None, + ch_user=None, + login=None, + exitcode=None, + message=None, + rbac=False, +): + """Add user to LDAP and ClickHouse and then try to login.""" self.context.ldap_node = self.context.cluster.node(server) if ch_user is None: @@ -54,75 +67,123 @@ def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None ch_user["username"] = ch_user.get("username", user["cn"]) ch_user["server"] = ch_user.get("server", user["_server"]) - with ldap_authenticated_users(ch_user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): + with ldap_authenticated_users( + ch_user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac + ): username = login.get("username", user["cn"]) password = login.get("password", user["userpassword"]) - login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) + login_and_execute_query( + username=username, password=password, exitcode=exitcode, message=message + ) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Parallel("1.0"), - RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid("1.0"), ) def parallel_login(self, server, user_count=10, timeout=300, rbac=False): - """Check that login of valid and invalid LDAP authenticated users works in parallel. - """ + """Check that login of valid and invalid LDAP authenticated users works in parallel.""" self.context.ldap_node = self.context.cluster.node(server) user = None - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + users = [ + {"cn": f"parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ] with ldap_users(*users): - with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users], rbac=rbac): + with ldap_authenticated_users( + *[{"username": user["cn"], "server": server} for user in users], rbac=rbac + ): def login_with_valid_username_and_password(users, i, iterations=10): with When(f"valid users try to login #{i}"): for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + random_user = users[random.randint(0, len(users) - 1)] + login_and_execute_query( + username=random_user["cn"], + password=random_user["userpassword"], + steps=False, + ) def login_with_valid_username_and_invalid_password(users, i, iterations=10): - with When(f"users try to login with valid username and invalid password #{i}"): + with When( + f"users try to login with valid username and invalid password #{i}" + ): for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], - password=(random_user["userpassword"] + randomword(1)), - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + random_user = users[random.randint(0, len(users) - 1)] + login_and_execute_query( + username=random_user["cn"], + password=(random_user["userpassword"] + randomword(1)), + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False, + ) def login_with_invalid_username_and_valid_password(users, i, iterations=10): - with When(f"users try to login with invalid username and valid password #{i}"): + with When( + f"users try to login with invalid username and valid password #{i}" + ): for i in range(iterations): - random_user = dict(users[random.randint(0, len(users)-1)]) + random_user = dict(users[random.randint(0, len(users) - 1)]) random_user["cn"] += randomword(1) - login_and_execute_query(username=random_user["cn"], - password=random_user["userpassword"], - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + login_and_execute_query( + username=random_user["cn"], + password=random_user["userpassword"], + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False, + ) with When("I login in parallel"): tasks = [] with Pool(4) as pool: try: for i in range(5): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_invalid_username_and_valid_password, (users, i, 50,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_valid_username_and_invalid_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_invalid_username_and_valid_password, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser("1.0") + RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser("1.0"), ) def login_after_user_is_deleted_from_ldap(self, server, rbac=False): - """Check that login fails after user is deleted from LDAP. - """ + """Check that login fails after user is deleted from LDAP.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -131,31 +192,37 @@ def login_after_user_is_deleted_from_ldap(self, server, rbac=False): user = {"cn": "myuser", "userpassword": "myuser"} user = add_user_to_ldap(**user) - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", - restart=True, rbac=rbac): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + restart=True, + rbac=rbac, + ): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I delete this user from LDAP"): delete_user_from_ldap(user) with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name", ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_PasswordChanged("1.0") + RQ_SRS_007_LDAP_Authentication_PasswordChanged("1.0"), ) def login_after_user_password_changed_in_ldap(self, server, rbac=False): - """Check that login fails after user password is changed in LDAP. - """ + """Check that login fails after user password is changed in LDAP.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -164,17 +231,23 @@ def login_after_user_password_changed_in_ldap(self, server, rbac=False): user = {"cn": "myuser", "userpassword": "myuser"} user = add_user_to_ldap(**user) - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml", - restart=True, rbac=rbac): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + restart=True, + rbac=rbac, + ): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name", ) with And("when I try to login with the new password it should work"): @@ -185,14 +258,14 @@ def login_after_user_password_changed_in_ldap(self, server, rbac=False): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_UsernameChanged("1.0") + RQ_SRS_007_LDAP_Authentication_UsernameChanged("1.0"), ) def login_after_user_cn_changed_in_ldap(self, server, rbac=False): - """Check that login fails after user cn is changed in LDAP. - """ + """Check that login fails after user cn is changed in LDAP.""" self.context.ldap_node = self.context.cluster.node(server) user = None new_user = None @@ -202,31 +275,37 @@ def login_after_user_cn_changed_in_ldap(self, server, rbac=False): user = {"cn": "myuser", "userpassword": "myuser"} user = add_user_to_ldap(**user) - with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + restart=True, + rbac=rbac, + ): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I change user password in LDAP"): new_user = change_user_cn_in_ldap(user, "myuser2") with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name", ) finally: with Finally("I make sure LDAP user is deleted"): if new_user is not None: delete_user_from_ldap(new_user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_LDAPServerRestart("1.0") + RQ_SRS_007_LDAP_Authentication_LDAPServerRestart("1.0"), ) def login_after_ldap_server_is_restarted(self, server, timeout=300, rbac=False): - """Check that login succeeds after LDAP server is restarted. - """ + """Check that login succeeds after LDAP server is restarted.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -235,18 +314,27 @@ def login_after_ldap_server_is_restarted(self, server, timeout=300, rbac=False): user = {"cn": "myuser", "userpassword": getuid()} user = add_user_to_ldap(**user) - with ldap_authenticated_users({"username": user["cn"], "server": server}, rbac=rbac): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, rbac=rbac + ): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I restart LDAP server"): self.context.ldap_node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): + with Then( + "I try to login until it works", description=f"timeout {timeout} sec" + ): started = time.time() while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) + r = self.context.node.query( + "SELECT 1", + settings=[ + ("user", user["cn"]), + ("password", user["userpassword"]), + ], + no_checks=True, + ) if r.exitcode == 0: break assert time.time() - started < timeout, error(r.output) @@ -255,14 +343,14 @@ def login_after_ldap_server_is_restarted(self, server, timeout=300, rbac=False): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart("1.0") + RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart("1.0"), ) def login_after_clickhouse_server_is_restarted(self, server, timeout=300, rbac=False): - """Check that login succeeds after ClickHouse server is restarted. - """ + """Check that login succeeds after ClickHouse server is restarted.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -271,18 +359,27 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=300, rbac=F user = {"cn": "myuser", "userpassword": getuid()} user = add_user_to_ldap(**user) - with ldap_authenticated_users({"username": user["cn"], "server": server}, rbac=rbac): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, rbac=rbac + ): login_and_execute_query(username=user["cn"], password=user["userpassword"]) with When("I restart ClickHouse server"): self.context.node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): + with Then( + "I try to login until it works", description=f"timeout {timeout} sec" + ): started = time.time() while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) + r = self.context.node.query( + "SELECT 1", + settings=[ + ("user", user["cn"]), + ("password", user["userpassword"]), + ], + no_checks=True, + ) if r.exitcode == 0: break assert time.time() - started < timeout, error(r.output) @@ -291,28 +388,30 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=300, rbac=F if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0") + RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0"), ) def valid_username_with_valid_empty_password(self, server, rbac=False): - """Check that we can't login using valid username that has empty password. - """ + """Check that we can't login using valid username that has empty password.""" user = {"cn": "empty_password", "userpassword": ""} exitcode = 4 message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server, rbac=rbac) + add_user_to_ldap_and_login( + user=user, exitcode=exitcode, message=message, server=server, rbac=rbac + ) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Invalid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0") + RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0"), ) def valid_username_and_invalid_empty_password(self, server, rbac=False): - """Check that we can't login using valid username but invalid empty password. - """ + """Check that we can't login using valid username but invalid empty password.""" username = "user_non_empty_password" user = {"cn": username, "userpassword": username} login = {"password": ""} @@ -320,25 +419,29 @@ def valid_username_and_invalid_empty_password(self, server, rbac=False): exitcode = 4 message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) + add_user_to_ldap_and_login( + user=user, + login=login, + exitcode=exitcode, + message=message, + server=server, + rbac=rbac, + ) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Valid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_Valid("1.0")) def valid_username_and_password(self, server, rbac=False): - """Check that we can login using valid username and password. - """ + """Check that we can login using valid username and password.""" username = "valid_username_and_password" user = {"cn": username, "userpassword": username} with When(f"I add user {username} to LDAP and try to login"): add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_Invalid("1.0")) def valid_username_and_password_invalid_server(self, server=None, rbac=False): """Check that we can't login using valid username and valid password but for a different server. @@ -350,126 +453,157 @@ def valid_username_and_password_invalid_server(self, server=None, rbac=False): exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - with ldap_authenticated_users(user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac): - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + with ldap_authenticated_users( + user, config_file=f"ldap_users_{getuid()}.xml", restart=True, rbac=rbac + ): + login_and_execute_query( + username="user2", password="user2", exitcode=exitcode, message=message + ) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Valid("1.0"), RQ_SRS_007_LDAP_Authentication_Username_Long("1.0"), - RQ_SRS_007_LDAP_Configuration_User_Name_Long("1.0") + RQ_SRS_007_LDAP_Configuration_User_Name_Long("1.0"), ) def valid_long_username_and_short_password(self, server, rbac=False): - """Check that we can login using valid very long username and short password. - """ + """Check that we can login using valid very long username and short password.""" username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_Invalid("1.0")) def invalid_long_username_and_valid_short_password(self, server, rbac=False): - """Check that we can't login using slightly invalid long username but valid password. - """ + """Check that we can't login using slightly invalid long username but valid password.""" username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} login = {"username": f"{username}?"} exitcode = 4 - message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, + login=login, + exitcode=exitcode, + message=message, + server=server, + rbac=rbac, + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_Long("1.0") + RQ_SRS_007_LDAP_Authentication_Password_Long("1.0"), ) def valid_short_username_and_long_password(self, server, rbac=False): - """Check that we can login using valid short username with very long password. - """ + """Check that we can login using valid short username with very long password.""" username = "long_password" - user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + user = { + "cn": username, + "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890", + } add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_Invalid("1.0")) def valid_short_username_and_invalid_long_password(self, server, rbac=False): - """Check that we can't login using valid short username and invalid long password. - """ + """Check that we can't login using valid short username and invalid long password.""" username = "long_password" - user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + user = { + "cn": username, + "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890", + } login = {"password": user["userpassword"] + "1"} exitcode = 4 - message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, + login=login, + exitcode=exitcode, + message=message, + server=server, + rbac=rbac, + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_Invalid("1.0")) def valid_username_and_invalid_password(self, server, rbac=False): - """Check that we can't login using valid username and invalid password. - """ + """Check that we can't login using valid username and invalid password.""" username = "valid_username_and_invalid_password" user = {"cn": username, "userpassword": username} login = {"password": user["userpassword"] + "1"} exitcode = 4 - message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, + login=login, + exitcode=exitcode, + message=message, + server=server, + rbac=rbac, + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_Invalid("1.0")) def invalid_username_and_valid_password(self, server, rbac=False): - """Check that we can't login using slightly invalid username but valid password. - """ + """Check that we can't login using slightly invalid username but valid password.""" username = "invalid_username_and_valid_password" user = {"cn": username, "userpassword": username} login = {"username": user["cn"] + "1"} exitcode = 4 - message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, + login=login, + exitcode=exitcode, + message=message, + server=server, + rbac=rbac, + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server, rbac=rbac) @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Valid("1.0"), RQ_SRS_007_LDAP_Authentication_Username_UTF8("1.0"), - RQ_SRS_007_LDAP_Configuration_User_Name_UTF8("1.0") + RQ_SRS_007_LDAP_Configuration_User_Name_UTF8("1.0"), ) def valid_utf8_username_and_ascii_password(self, server, rbac=False): - """Check that we can login using valid utf-8 username with ascii password. - """ + """Check that we can login using valid utf-8 username with ascii password.""" username = "utf8_username_Gãńdåłf_Thê_Gręât" user = {"cn": username, "userpassword": "utf8_username"} add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Valid("1.0"), - RQ_SRS_007_LDAP_Authentication_Password_UTF8("1.0") + RQ_SRS_007_LDAP_Authentication_Password_UTF8("1.0"), ) def valid_ascii_username_and_utf8_password(self, server, rbac=False): - """Check that we can login using valid ascii username with utf-8 password. - """ + """Check that we can login using valid ascii username with utf-8 password.""" username = "utf8_password" user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} add_user_to_ldap_and_login(user=user, server=server, rbac=rbac) + @TestScenario def empty_username_and_empty_password(self, server=None, rbac=False): """Check that we can login using empty username and empty password as @@ -477,11 +611,10 @@ def empty_username_and_empty_password(self, server=None, rbac=False): """ login_and_execute_query(username="", password="") + @TestScenario @Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Default("1.0")) def default_verification_cooldown_value(self, server, rbac=False): """Check that the default value (0) for the verification cooldown parameter disables caching and forces contacting the LDAP server for each @@ -492,10 +625,18 @@ def default_verification_cooldown_value(self, server, rbac=False): error_exitcode = 4 user = None - with Given("I have an LDAP configuration that uses the default verification_cooldown value (0)"): - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} + with Given( + "I have an LDAP configuration that uses the default verification_cooldown value (0)" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -505,27 +646,37 @@ def default_verification_cooldown_value(self, server, rbac=False): user = add_user_to_ldap(**user) with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login immediately with the old user password it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) + with Then( + "when I try to login immediately with the old user password it should fail" + ): + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], + exitcode=error_exitcode, + message=error_message, + ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0")) def valid_verification_cooldown_value_cn_change(self, server, rbac=False): """Check that we can perform requests without contacting the LDAP server after successful authentication when the verification_cooldown parameter @@ -534,15 +685,19 @@ def valid_verification_cooldown_value_cn_change(self, server, rbac=False): user = None new_user = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 2 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -552,25 +707,33 @@ def valid_verification_cooldown_value_cn_change(self, server, rbac=False): user = add_user_to_ldap(**user) with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user cn in LDAP"): new_user = change_user_cn_in_ldap(user, "testVCD2") - with Then("when I try to login again with the old user cn it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "when I try to login again with the old user cn it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) finally: with Finally("I make sure LDAP user is deleted"): if new_user is not None: delete_user_from_ldap(new_user, exitcode=None) + @TestScenario @Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0")) def valid_verification_cooldown_value_password_change(self, server, rbac=False): """Check that we can perform requests without contacting the LDAP server after successful authentication when the verification_cooldown parameter @@ -578,15 +741,19 @@ def valid_verification_cooldown_value_password_change(self, server, rbac=False): """ user = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 2 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -596,25 +763,33 @@ def valid_verification_cooldown_value_password_change(self, server, rbac=False): user = add_user_to_ldap(**user) with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login again with the old password it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "when I try to login again with the old password it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown("1.0")) def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False): """Check that we can perform requests without contacting the LDAP server after successful authentication when the verification_cooldown parameter @@ -622,15 +797,19 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False) """ user = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 2 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -640,18 +819,26 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False) user = add_user_to_ldap(**user) with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml"): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) try: with And("then I stop the ldap server"): self.context.ldap_node.stop() - with Then("when I try to login again with the server offline it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "when I try to login again with the server offline it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) finally: with Finally("I start the ldap server back up"): self.context.ldap_node.start() @@ -661,22 +848,26 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False) if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestOutline def repeat_requests(self, server, iterations, vcd_value, rbac=False, timeout=600): - """Run repeated requests from some user to the LDAP server. - """ + """Run repeated requests from some user to the LDAP server.""" user = None - with Given(f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": vcd_value - }} + with Given( + f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": vcd_value, + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -686,10 +877,16 @@ def repeat_requests(self, server, iterations, vcd_value, rbac=False, timeout=600 user = add_user_to_ldap(**user) with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, config_file=f"ldap_users_{getuid()}.xml"): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + ): with When(f"I login and execute some query {iterations} times"): start_time = time.time() - r = self.context.node.command(f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done", timeout=timeout) + r = self.context.node.command( + f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done", + timeout=timeout, + ) end_time = time.time() return end_time - start_time @@ -699,11 +896,10 @@ def repeat_requests(self, server, iterations, vcd_value, rbac=False, timeout=600 if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Tags("verification_cooldown") -@Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Performance("1.0")) def verification_cooldown_performance(self, server, rbac=False, iterations=5000): """Check that login performance is better when the verification cooldown parameter is set to a positive value when comparing to the case when @@ -713,49 +909,67 @@ def verification_cooldown_performance(self, server, rbac=False, iterations=5000) vcd_time = 0 no_vcd_time = 0 - with Example(f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations"): - vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="600", rbac=rbac) + with Example( + f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations" + ): + vcd_time = repeat_requests( + server=server, iterations=iterations, vcd_value="600", rbac=rbac + ) metric("login_with_vcd_value_600", units="seconds", value=vcd_time) - with Example(f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations"): - no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) + with Example( + f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations" + ): + no_vcd_time = repeat_requests( + server=server, iterations=iterations, vcd_value="0", rbac=rbac + ) metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) with Then("Log the performance improvement as a percentage"): - metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) + metric( + "percentage_improvement", + units="%", + value=100 * (no_vcd_time - vcd_time) / vcd_time, + ) + @TestOutline -def check_verification_cooldown_reset_on_core_server_parameter_change(self, server, - parameter_name, parameter_value, rbac=False): +def check_verification_cooldown_reset_on_core_server_parameter_change( + self, server, parameter_name, parameter_value, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after one of the core server parameters is changed in the LDAP server configuration. """ - config_d_dir="/etc/clickhouse-server/config.d" - config_file="ldap_servers.xml" + config_d_dir = "/etc/clickhouse-server/config.d" + config_file = "ldap_servers.xml" error_message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" error_exitcode = 4 user = None - config=None - updated_config=None + config = None + updated_config = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 600 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) with And("LDAP authenticated user"): users = [ {"cn": f"testVCD_0", "userpassword": "testVCD_0"}, - {"cn": f"testVCD_1", "userpassword": "testVCD_1"} + {"cn": f"testVCD_1", "userpassword": "testVCD_1"}, ] with And("I create LDAP servers configuration file"): @@ -763,86 +977,132 @@ def check_verification_cooldown_reset_on_core_server_parameter_change(self, serv with ldap_users(*users) as users: with ldap_servers(servers, restart=True): - with ldap_authenticated_users(*[{"username": user["cn"], "server": server} for user in users]): + with ldap_authenticated_users( + *[{"username": user["cn"], "server": server} for user in users] + ): with When("I login and execute a query"): for user in users: with By(f"as user {user['cn']}"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): for user in users: with By(f"for user {user['cn']}"): change_user_password_in_ldap(user, "newpassword") - with And(f"I change the server {parameter_name} core parameter", description=f"{parameter_value}"): + with And( + f"I change the server {parameter_name} core parameter", + description=f"{parameter_value}", + ): servers["openldap1"][parameter_name] = parameter_value - with And("I create an updated the config file that has a different server host name"): - updated_config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + with And( + "I create an updated the config file that has a different server host name" + ): + updated_config = create_ldap_servers_config_content( + servers, config_d_dir, config_file + ) with modify_config(updated_config, restart=False): - with Then("when I try to log in it should fail as cache should have been reset"): + with Then( + "when I try to log in it should fail as cache should have been reset" + ): for user in users: with By(f"as user {user['cn']}"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message.format(user=user["cn"])) + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], + exitcode=error_exitcode, + message=error_message.format(user=user["cn"]), + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_host_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_host_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server host name is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="host", parameter_value="openldap2", rbac=rbac) + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, parameter_name="host", parameter_value="openldap2", rbac=rbac + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_port_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_port_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server port is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="port", parameter_value="9006", rbac=rbac) + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, parameter_name="port", parameter_value="9006", rbac=rbac + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server auth_dn_prefix is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="auth_dn_prefix", parameter_value="cxx=", rbac=rbac) + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, + parameter_name="auth_dn_prefix", + parameter_value="cxx=", + rbac=rbac, + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_007_LDAP_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server auth_dn_suffix is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, parameter_name="auth_dn_suffix", - parameter_value=",ou=company,dc=users,dc=com", rbac=rbac) + parameter_value=",ou=company,dc=users,dc=com", + rbac=rbac, + ) @TestScenario @@ -860,15 +1120,19 @@ def scenario(self, server, rbac=False): error_exitcode = 4 error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 600 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -878,48 +1142,68 @@ def scenario(self, server, rbac=False): user = add_user_to_ldap(**user) with ldap_servers(servers): - with ldap_authenticated_users({"username": user["cn"], "server": server}, - config_file=f"ldap_users_{getuid()}.xml"): + with ldap_authenticated_users( + {"username": user["cn"], "server": server}, + config_file=f"ldap_users_{getuid()}.xml", + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") - with Then("When I try to log in with the cached password it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "When I try to log in with the cached password it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) - with And("When I try to log in with an incorrect password it should fail"): - login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, - message=error_message) + with And( + "When I try to log in with an incorrect password it should fail" + ): + login_and_execute_query( + username=user["cn"], + password="incorrect", + exitcode=error_exitcode, + message=error_message, + ) - with And("When I try to log in with the cached password it should fail"): - login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, - message=error_message) + with And( + "When I try to log in with the cached password it should fail" + ): + login_and_execute_query( + username=user["cn"], + password="incorrect", + exitcode=error_exitcode, + message=error_message, + ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestFeature def verification_cooldown(self, rbac, servers=None, node="clickhouse1"): - """Check verification cooldown parameter functionality. - """ - for scenario in loads(current_module(), Scenario, filter=has.tag("verification_cooldown")): + """Check verification cooldown parameter functionality.""" + for scenario in loads( + current_module(), Scenario, filter=has.tag("verification_cooldown") + ): scenario(server="openldap1", rbac=rbac) @TestOutline(Feature) @Name("user authentications") -@Requirements( - RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword("1.0") +@Requirements(RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword("1.0")) +@Examples( + "rbac", + [(False,), (True, Requirements(RQ_SRS_007_LDAP_Configuration_User_RBAC("1.0")))], ) -@Examples("rbac", [ - (False,), - (True, Requirements(RQ_SRS_007_LDAP_Configuration_User_RBAC("1.0"))) -]) def feature(self, rbac, servers=None, node="clickhouse1"): """Check that users can be authenticated using an LDAP server when users are configured either using an XML configuration file or RBAC. @@ -930,11 +1214,9 @@ def feature(self, rbac, servers=None, node="clickhouse1"): servers = globals()["servers"] with ldap_servers(servers): - for scenario in loads(current_module(), Scenario, filter=~has.tag("verification_cooldown")): + for scenario in loads( + current_module(), Scenario, filter=~has.tag("verification_cooldown") + ): scenario(server="openldap1", rbac=rbac) Feature(test=verification_cooldown)(rbac=rbac, servers=servers, node=node) - - - - diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index a6bf11fd2cf..17b4fcd3e62 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -14,42 +14,24 @@ import testflows.settings as settings from testflows.core import * from testflows.asserts import error -def getuid(): - return str(uuid.uuid1()).replace('-', '_') +from helpers.common import ( + xml_indent, + xml_with_utf8, + xml_append, + add_config, + getuid, + Config, +) -xml_with_utf8 = '\n' +ASCII_CHARS = string.ascii_lowercase + string.ascii_uppercase + string.digits -def xml_indent(elem, level=0, by=" "): - i = "\n" + level * by - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + by - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: - xml_indent(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i - -def xml_append(root, tag, text): - element = xmltree.Element(tag) - element.text = text - root.append(element) - return element - -Config = namedtuple("Config", "content path name uid preprocessed_name") - -ASCII_CHARS = string.ascii_lowercase + string.ascii_uppercase + string.digits def randomword(length, chars=ASCII_CHARS): - return ''.join(random.choice(chars) for i in range(length)) + return "".join(random.choice(chars) for i in range(length)) + def restart(node=None, safe=False, timeout=300): - """Restart ClickHouse server and wait for config to be reloaded. - """ + """Restart ClickHouse server and wait for config to be reloaded.""" with When("I restart ClickHouse server node"): if node is None: node = current().context.node @@ -61,135 +43,39 @@ def restart(node=None, safe=False, timeout=300): bash.close() with And("getting current log size"): - logsize = \ - node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ - 0].strip() + logsize = ( + node.command( + "stat --format=%s /var/log/clickhouse-server/clickhouse-server.log" + ) + .output.split(" ")[0] + .strip() + ) with And("restarting ClickHouse server"): node.restart(safe=safe) - with Then("tailing the log file from using previous log size as the offset"): + with Then( + "tailing the log file from using previous log size as the offset" + ): bash.prompt = bash.__class__.prompt bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + bash.send( + f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log" + ) with And("waiting for config reload message in the log file"): bash.expect( f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", - timeout=timeout) + timeout=timeout, + ) -def add_config(config, timeout=300, restart=False, modify=False): - """Add dynamic configuration file to ClickHouse. - :param node: node - :param config: configuration file description - :param timeout: timeout, default: 20 sec - """ - node = current().context.node - cluster = current().context.cluster - - def check_preprocessed_config_is_updated(after_removal=False): - """Check that preprocessed config is updated. - """ - started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" - - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if after_removal: - if exitcode == 1: - break - else: - if exitcode == 0: - break - time.sleep(1) - - if settings.debug: - node.command(f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}") - - if after_removal: - assert exitcode == 1, error() - else: - assert exitcode == 0, error() - - def wait_for_config_to_be_loaded(): - """Wait for config to be loaded. - """ - if restart: - with When("I close terminal to the node to be restarted"): - bash.close() - - with And("I stop ClickHouse to apply the config changes"): - node.stop(safe=False) - - with And("I get the current log size"): - cmd = node.cluster.command(None, - f"stat --format=%s {cluster.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log") - logsize = cmd.output.split(" ")[0].strip() - - with And("I start ClickHouse back up"): - node.start() - - with Then("I tail the log file from using previous log size as the offset"): - bash.prompt = bash.__class__.prompt - bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") - - with Then("I wait for config reload message in the log file"): - if restart: - bash.expect( - f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", - timeout=timeout) - else: - bash.expect( - f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", - timeout=timeout) - - try: - with Given(f"{config.name}"): - if settings.debug: - with When("I output the content of the config"): - debug(config.content) - - with node.cluster.shell(node.name) as bash: - bash.expect(bash.prompt) - bash.send("tail -v -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") - # make sure tail process is launched and started to follow the file - bash.expect("<==") - bash.expect("\n") - - with When("I add the config", description=config.path): - command = f"cat < {config.path}\n{config.content}\nHEREDOC" - node.command(command, steps=False, exitcode=0) - - with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - check_preprocessed_config_is_updated() - - with And("I wait for config to be reloaded"): - wait_for_config_to_be_loaded() - yield - finally: - if not modify: - with Finally(f"I remove {config.name}"): - with node.cluster.shell(node.name) as bash: - bash.expect(bash.prompt) - bash.send("tail -v -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") - # make sure tail process is launched and started to follow the file - bash.expect("<==") - bash.expect("\n") - - with By("removing the config file", description=config.path): - node.command(f"rm -rf {config.path}", exitcode=0) - - with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - check_preprocessed_config_is_updated(after_removal=True) - - with And("I wait for config to be reloaded"): - wait_for_config_to_be_loaded() - -def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml"): - """Create LDAP servers configuration content. - """ +def create_ldap_servers_config_content( + servers, + config_d_dir="/etc/clickhouse-server/config.d", + config_file="ldap_servers.xml", +): + """Create LDAP servers configuration content.""" uid = getuid() path = os.path.join(config_d_dir, config_file) name = config_file @@ -205,28 +91,39 @@ def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-se xml_servers.append(xml_server) xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + content = xml_with_utf8 + str( + xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8" + ) return Config(content, path, name, uid, "config.xml") -@contextmanager -def modify_config(config, restart=False): - """Apply updated configuration file. - """ - return add_config(config, restart=restart, modify=True) @contextmanager -def ldap_servers(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", - timeout=300, restart=False, config=None): - """Add LDAP servers configuration. - """ +def modify_config(config, restart=False, node=None): + """Apply updated configuration file.""" + return add_config(config, restart=restart, modify=True, node=node) + + +@contextmanager +def ldap_servers( + servers, + config_d_dir="/etc/clickhouse-server/config.d", + config_file="ldap_servers.xml", + timeout=300, + restart=False, + config=None, + node=None, +): + """Add LDAP servers configuration.""" if config is None: config = create_ldap_servers_config_content(servers, config_d_dir, config_file) - return add_config(config, restart=restart) + return add_config(config, restart=restart, node=node) -def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-server/users.d", config_file="ldap_users.xml"): - """Create LDAP users configuration file content. - """ + +def create_ldap_users_config_content( + *users, config_d_dir="/etc/clickhouse-server/users.d", config_file="ldap_users.xml" +): + """Create LDAP users configuration file content.""" uid = getuid() path = os.path.join(config_d_dir, config_file) name = config_file @@ -236,26 +133,32 @@ def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-serve xml_users.append(xmltree.Comment(text=f"LDAP users {uid}")) for user in users: - xml_user = xmltree.Element(user['username']) + xml_user = xmltree.Element(user["username"]) xml_user_server = xmltree.Element("ldap") xml_append(xml_user_server, "server", user["server"]) xml_user.append(xml_user_server) xml_users.append(xml_user) xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + content = xml_with_utf8 + str( + xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8" + ) return Config(content, path, name, uid, "users.xml") -def add_users_identified_with_ldap(*users): + +def add_users_identified_with_ldap(*users, node=None): """Add one or more users that are identified via an ldap server using RBAC. """ - node = current().context.node + if node is None: + node = current().context.node try: with Given("I create users"): for user in users: - node.query(f"CREATE USER '{user['username']}' IDENTIFIED WITH LDAP SERVER '{user['server']}'") + node.query( + f"CREATE USER '{user['username']}' IDENTIFIED WITH LDAP SERVER '{user['server']}'" + ) yield finally: with Finally("I remove users"): @@ -263,30 +166,46 @@ def add_users_identified_with_ldap(*users): with By(f"dropping user {user['username']}", flags=TE): node.query(f"DROP USER IF EXISTS '{user['username']}'") + @contextmanager -def ldap_authenticated_users(*users, config_d_dir="/etc/clickhouse-server/users.d", - config_file=None, timeout=300, restart=True, config=None, rbac=False): - """Add LDAP authenticated users. - """ +def ldap_authenticated_users( + *users, + config_d_dir="/etc/clickhouse-server/users.d", + config_file=None, + timeout=300, + restart=True, + config=None, + rbac=False, + node=None, +): + """Add LDAP authenticated users.""" + if node is None: + node = current().context.node + if rbac: - return add_users_identified_with_ldap(*users) + return add_users_identified_with_ldap(*users, node=node) else: if config_file is None: config_file = f"ldap_users_{getuid()}.xml" if config is None: - config = create_ldap_users_config_content(*users, config_d_dir=config_d_dir, config_file=config_file) - return add_config(config, timeout=timeout, restart=restart) + config = create_ldap_users_config_content( + *users, config_d_dir=config_d_dir, config_file=config_file + ) + return add_config(config, timeout=timeout, restart=restart, node=node) + def invalid_server_config(servers, message=None, tail=30, timeout=300): - """Check that ClickHouse errors when trying to load invalid LDAP servers configuration file. - """ + """Check that ClickHouse errors when trying to load invalid LDAP servers configuration file.""" node = current().context.node if message is None: message = "Exception: Failed to merge config with '/etc/clickhouse-server/config.d/ldap_servers.xml'" config = create_ldap_servers_config_content(servers) try: - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + node.command( + 'echo -e "%s" > /var/log/clickhouse-server/clickhouse-server.err.log' + % ("-\\n" * tail) + ) with When("I add the config", description=config.path): command = f"cat < {config.path}\n{config.content}\nHEREDOC" @@ -294,7 +213,7 @@ def invalid_server_config(servers, message=None, tail=30, timeout=300): with Then("server shall fail to merge the new config"): started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + command = f'tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep "{message}"' while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode if exitcode == 0: @@ -306,23 +225,26 @@ def invalid_server_config(servers, message=None, tail=30, timeout=300): with By("removing the config file", description=config.path): node.command(f"rm -rf {config.path}", exitcode=0) + def invalid_user_config(servers, config, message=None, tail=30, timeout=300): - """Check that ClickHouse errors when trying to load invalid LDAP users configuration file. - """ + """Check that ClickHouse errors when trying to load invalid LDAP users configuration file.""" node = current().context.node if message is None: message = "Exception: Failed to merge config with '/etc/clickhouse-server/users.d/ldap_users.xml'" with ldap_servers(servers): try: - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("\\n" * tail)) + node.command( + 'echo -e "%s" > /var/log/clickhouse-server/clickhouse-server.err.log' + % ("\\n" * tail) + ) with When("I add the config", description=config.path): command = f"cat < {config.path}\n{config.content}\nHEREDOC" node.command(command, steps=False, exitcode=0) with Then("server shall fail to merge the new config"): started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + command = f'tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep "{message}"' while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode if exitcode == 0: @@ -334,7 +256,17 @@ def invalid_user_config(servers, config, message=None, tail=30, timeout=300): with By("removing the config file", description=config.path): node.command(f"rm -rf {config.path}", exitcode=0) -def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=None, uid=None, uidnumber=None, node=None): + +def add_user_to_ldap( + cn, + userpassword, + givenname=None, + homedirectory=None, + sn=None, + uid=None, + uidnumber=None, + node=None, +): """Add user entry to LDAP.""" if node is None: node = current().context.ldap_node @@ -360,7 +292,7 @@ def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=No "uid": uid, "uidnumber": uidnumber, "userpassword": userpassword, - "_server": node.name + "_server": node.name, } lines = [] @@ -377,73 +309,102 @@ def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=No ldif = "\n".join(lines) r = node.command( - f"echo -e \"{ldif}\" | ldapadd -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + f'echo -e "{ldif}" | ldapadd -x -H ldap://localhost -D "cn=admin,dc=company,dc=com" -w admin' + ) assert r.exitcode == 0, error() return user + def delete_user_from_ldap(user, node=None, exitcode=0): """Delete user entry from LDAP.""" if node is None: node = current().context.ldap_node r = node.command( - f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{user['dn']}\"") + f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{user['dn']}\"" + ) if exitcode is not None: assert r.exitcode == exitcode, error() + def change_user_password_in_ldap(user, new_password, node=None, exitcode=0): """Change user password in LDAP.""" if node is None: node = current().context.ldap_node - ldif = (f"dn: {user['dn']}\n" + ldif = ( + f"dn: {user['dn']}\n" "changetype: modify\n" "replace: userpassword\n" - f"userpassword: {new_password}") + f"userpassword: {new_password}" + ) r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + f'echo -e "{ldif}" | ldapmodify -x -H ldap://localhost -D "cn=admin,dc=company,dc=com" -w admin' + ) if exitcode is not None: assert r.exitcode == exitcode, error() + def change_user_cn_in_ldap(user, new_cn, node=None, exitcode=0): """Change user password in LDAP.""" if node is None: node = current().context.ldap_node new_user = dict(user) - new_user['dn'] = f"cn={new_cn},ou=users,dc=company,dc=com" - new_user['cn'] = new_cn + new_user["dn"] = f"cn={new_cn},ou=users,dc=company,dc=com" + new_user["cn"] = new_cn ldif = ( f"dn: {user['dn']}\n" "changetype: modrdn\n" f"newrdn: cn = {new_user['cn']}\n" f"deleteoldrdn: 1\n" - ) + ) r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + f'echo -e "{ldif}" | ldapmodify -x -H ldap://localhost -D "cn=admin,dc=company,dc=com" -w admin' + ) if exitcode is not None: assert r.exitcode == exitcode, error() return new_user + @contextmanager -def ldap_user(cn, userpassword, givenname=None, homedirectory=None, sn=None, uid=None, uidnumber=None, node=None): +def ldap_user( + cn, + userpassword, + givenname=None, + homedirectory=None, + sn=None, + uid=None, + uidnumber=None, + node=None, +): """Add new user to the LDAP server.""" try: user = None with Given(f"I add user {cn} to LDAP"): - user = add_user_to_ldap(cn, userpassword, givenname, homedirectory, sn, uid, uidnumber, node=node) + user = add_user_to_ldap( + cn, + userpassword, + givenname, + homedirectory, + sn, + uid, + uidnumber, + node=node, + ) yield user finally: with Finally(f"I delete user {cn} from LDAP"): if user is not None: delete_user_from_ldap(user, node=node) + @contextmanager def ldap_users(*users, node=None): """Add multiple new users to the LDAP server.""" @@ -459,6 +420,7 @@ def ldap_users(*users, node=None): for _user in _users: delete_user_from_ldap(_user, node=node) + def login(servers, *users, config=None): """Configure LDAP server and LDAP authenticated users and try to login and execute a query""" @@ -467,7 +429,12 @@ def login(servers, *users, config=None): for user in users: if user.get("login", False): with When(f"I login as {user['username']} and execute query"): - current().context.node.query("SELECT 1", - settings=[("user", user["username"]), ("password", user["password"])], + current().context.node.query( + "SELECT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], exitcode=user.get("exitcode", None), - message=user.get("message", None)) + message=user.get("message", None), + ) diff --git a/tests/testflows/ldap/authentication/tests/connections.py b/tests/testflows/ldap/authentication/tests/connections.py index dfb920181e1..4dbbfb2070a 100644 --- a/tests/testflows/ldap/authentication/tests/connections.py +++ b/tests/testflows/ldap/authentication/tests/connections.py @@ -4,22 +4,22 @@ from testflows.asserts import error from ldap.authentication.tests.common import login from ldap.authentication.requirements import * + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Protocol_PlainText("1.0"), RQ_SRS_007_LDAP_Configuration_Server_EnableTLS("1.0"), RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Port_Default("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Port_Default("1.0"), ) def plain_text(self): - """Check that we can perform LDAP user authentication using `plain text` connection protocol. - """ + """Check that we can perform LDAP user authentication using `plain text` connection protocol.""" servers = { "openldap1": { "host": "openldap1", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -27,10 +27,11 @@ def plain_text(self): ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Protocol_PlainText("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Port("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Port("1.0"), ) def plain_text_with_custom_port(self): """Check that we can perform LDAP user authentication using `plain text` connection protocol @@ -42,7 +43,7 @@ def plain_text_with_custom_port(self): "port": "3089", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -50,10 +51,11 @@ def plain_text_with_custom_port(self): ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Protocol_TLS("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Port("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Port("1.0"), ) def tls_with_custom_port(self): """Check that we can perform LDAP user authentication using `TLS` connection protocol @@ -65,7 +67,7 @@ def tls_with_custom_port(self): "port": "6036", "tls_require_cert": "never", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -73,10 +75,11 @@ def tls_with_custom_port(self): ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Port("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Port("1.0"), ) def starttls_with_custom_port(self): """Check that we can perform LDAP user authentication using `StartTLS` connection protocol @@ -89,7 +92,7 @@ def starttls_with_custom_port(self): "enable_tls": "starttls", "tls_require_cert": "never", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -97,16 +100,16 @@ def starttls_with_custom_port(self): ] login(servers, *users) + def tls_connection(enable_tls, tls_require_cert): - """Try to login using LDAP user authentication over a TLS connection. - """ + """Try to login using LDAP user authentication over a TLS connection.""" servers = { "openldap2": { "host": "openldap2", "enable_tls": enable_tls, "tls_require_cert": tls_require_cert, "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -116,51 +119,57 @@ def tls_connection(enable_tls, tls_require_cert): requirements = [] if tls_require_cert == "never": - requirements = [RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never("1.0")] + requirements = [ + RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never("1.0") + ] elif tls_require_cert == "allow": - requirements = [RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow("1.0")] + requirements = [ + RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow("1.0") + ] elif tls_require_cert == "try": - requirements = [RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try("1.0")] + requirements = [ + RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try("1.0") + ] elif tls_require_cert == "demand": - requirements = [RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand("1.0")] + requirements = [ + RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand("1.0") + ] - with Example(name=f"tls_require_cert='{tls_require_cert}'", requirements=requirements): + with Example( + name=f"tls_require_cert='{tls_require_cert}'", requirements=requirements + ): login(servers, *users) + @TestScenario -@Examples("enable_tls tls_require_cert", [ - ("yes", "never"), - ("yes", "allow"), - ("yes", "try"), - ("yes", "demand") -]) +@Examples( + "enable_tls tls_require_cert", + [("yes", "never"), ("yes", "allow"), ("yes", "try"), ("yes", "demand")], +) @Requirements( RQ_SRS_007_LDAP_Authentication_Protocol_TLS("1.0"), RQ_SRS_007_LDAP_Configuration_Server_EnableTLS("1.0"), RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes("1.0"), RQ_SRS_007_LDAP_Configuration_Server_Port_Default("1.0"), RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default("1.0") + RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default("1.0"), ) def tls(self): - """Check that we can perform LDAP user authentication using `TLS` connection protocol. - """ + """Check that we can perform LDAP user authentication using `TLS` connection protocol.""" for example in self.examples: tls_connection(*example) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default("1.0")) def tls_enable_tls_default_yes(self): - """Check that the default value for the `enable_tls` is set to `yes`. - """ + """Check that the default value for the `enable_tls` is set to `yes`.""" servers = { "openldap2": { "host": "openldap2", "tls_require_cert": "never", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -168,20 +177,20 @@ def tls_enable_tls_default_yes(self): ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default("1.0") ) def tls_require_cert_default_demand(self): - """Check that the default value for the `tls_require_cert` is set to `demand`. - """ + """Check that the default value for the `tls_require_cert` is set to `demand`.""" servers = { "openldap2": { "host": "openldap2", "enable_tls": "yes", "port": "636", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -189,32 +198,33 @@ def tls_require_cert_default_demand(self): ] login(servers, *users) + @TestScenario -@Examples("enable_tls tls_require_cert", [ - ("starttls", "never"), - ("starttls", "allow"), - ("starttls", "try"), - ("starttls", "demand") -]) +@Examples( + "enable_tls tls_require_cert", + [ + ("starttls", "never"), + ("starttls", "allow"), + ("starttls", "try"), + ("starttls", "demand"), + ], +) @Requirements( RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS("1.0"), RQ_SRS_007_LDAP_Configuration_Server_EnableTLS("1.0"), RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Port_Default("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Port_Default("1.0"), ) def starttls(self): - """Check that we can perform LDAP user authentication using legacy `StartTLS` connection protocol. - """ + """Check that we can perform LDAP user authentication using legacy `StartTLS` connection protocol.""" for example in self.examples: tls_connection(*example) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite("1.0")) def tls_cipher_suite(self): - """Check that `tls_cipher_suite` parameter can be used specify allowed cipher suites. - """ + """Check that `tls_cipher_suite` parameter can be used specify allowed cipher suites.""" servers = { "openldap4": { "host": "openldap4", @@ -223,7 +233,7 @@ def tls_cipher_suite(self): "tls_cipher_suite": "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC", "tls_minimum_protocol_version": "tls1.2", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -231,18 +241,22 @@ def tls_cipher_suite(self): ] login(servers, *users) + @TestOutline(Scenario) @Requirements( RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values("1.0") + RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values("1.0"), +) +@Examples( + "version exitcode message", + [ + ("ssl2", None, None), + ("ssl3", None, None), + ("tls1.0", None, None), + ("tls1.1", None, None), + ("tls1.2", None, None), + ], ) -@Examples("version exitcode message", [ - ("ssl2", None, None), - ("ssl3", None, None), - ("tls1.0", None, None), - ("tls1.1", None, None), - ("tls1.2", None, None) -]) def tls_minimum_protocol_version(self, version, exitcode, message): """Check that `tls_minimum_protocol_version` parameter can be used specify to specify the minimum protocol version of SSL/TLS. @@ -255,14 +269,20 @@ def tls_minimum_protocol_version(self, version, exitcode, message): "tls_require_cert": "never", "tls_minimum_protocol_version": version, "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } - users = [{ - "server": "openldap4", "username": "user4", "password": "user4", - "login": True, "exitcode": int(exitcode) if exitcode is not None else None, "message": message - }] + users = [ + { + "server": "openldap4", + "username": "user4", + "password": "user4", + "login": True, + "exitcode": int(exitcode) if exitcode is not None else None, + "message": message, + } + ] # Note: this code was an attempt to produce a negative case but did not work # ldap_node = self.context.cluster.node("openldap4") @@ -280,11 +300,11 @@ def tls_minimum_protocol_version(self, version, exitcode, message): login(servers, *users) + @TestFeature @Name("connection protocols") def feature(self, node="clickhouse1"): - """Check different LDAP connection protocols. - """ + """Check different LDAP connection protocols.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/ldap/authentication/tests/multiple_servers.py b/tests/testflows/ldap/authentication/tests/multiple_servers.py index c4317187b74..4295c428e1c 100644 --- a/tests/testflows/ldap/authentication/tests/multiple_servers.py +++ b/tests/testflows/ldap/authentication/tests/multiple_servers.py @@ -2,13 +2,14 @@ from testflows.core import * from testflows.asserts import error from ldap.authentication.tests.common import login -from ldap.authentication.requirements import RQ_SRS_007_LDAP_Authentication_MultipleServers +from ldap.authentication.requirements import ( + RQ_SRS_007_LDAP_Authentication_MultipleServers, +) + @TestScenario @Name("multiple servers") -@Requirements( - RQ_SRS_007_LDAP_Authentication_MultipleServers("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Authentication_MultipleServers("1.0")) def scenario(self, node="clickhouse1"): """Check that multiple LDAP servers can be used to authenticate users. @@ -21,7 +22,7 @@ def scenario(self, node="clickhouse1"): "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { "host": "openldap2", @@ -33,9 +34,21 @@ def scenario(self, node="clickhouse1"): }, } users = [ - {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, - {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + }, + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + }, ] - with When("I add multiple LDAP servers and users that use different servers and try to login"): + with When( + "I add multiple LDAP servers and users that use different servers and try to login" + ): login(servers, *users) diff --git a/tests/testflows/ldap/authentication/tests/sanity.py b/tests/testflows/ldap/authentication/tests/sanity.py index 542fa2a48b1..cb23c33f3b5 100644 --- a/tests/testflows/ldap/authentication/tests/sanity.py +++ b/tests/testflows/ldap/authentication/tests/sanity.py @@ -3,6 +3,7 @@ from testflows.asserts import error from ldap.authentication.tests.common import add_user_to_ldap, delete_user_from_ldap + @TestScenario @Name("sanity") def scenario(self, server="openldap1"): @@ -13,7 +14,8 @@ def scenario(self, server="openldap1"): with When("I search LDAP database"): r = self.context.ldap_node.command( - "ldapsearch -x -H ldap://localhost -b \"dc=company,dc=com\" -D \"cn=admin,dc=company,dc=com\" -w admin") + 'ldapsearch -x -H ldap://localhost -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin' + ) assert r.exitcode == 0, error() with Then("I should find an entry for user1"): @@ -24,7 +26,8 @@ def scenario(self, server="openldap1"): with And("I search LDAP database again"): r = self.context.ldap_node.command( - "ldapsearch -x -H ldap://localhost -b \"dc=company,dc=com\" -D \"cn=admin,dc=company,dc=com\" -w admin") + 'ldapsearch -x -H ldap://localhost -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin' + ) assert r.exitcode == 0, error() with Then("I should find an entry for the new user"): @@ -35,7 +38,8 @@ def scenario(self, server="openldap1"): with And("I search LDAP database again"): r = self.context.ldap_node.command( - "ldapsearch -x -H ldap://localhost -b \"dc=company,dc=com\" -D \"cn=admin,dc=company,dc=com\" -w admin") + 'ldapsearch -x -H ldap://localhost -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin' + ) assert r.exitcode == 0, error() with Then("I should not find an entry for the deleted user"): diff --git a/tests/testflows/ldap/authentication/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py index 5e0e145d035..af15a1495df 100644 --- a/tests/testflows/ldap/authentication/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -3,232 +3,331 @@ from testflows.core import * from ldap.authentication.tests.common import * from ldap.authentication.requirements import * -@TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Name("1.0") -) -def empty_server_name(self, timeout=300): - """Check that empty string as a server name is not allowed. - """ - servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - invalid_server_config(servers, timeout=timeout) @TestScenario @Requirements( RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_UnreachableServer("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Name("1.0"), +) +def empty_server_name(self, timeout=300): + """Check that empty string as a server name is not allowed.""" + servers = { + "": { + "host": "foo", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + invalid_server_config(servers, timeout=timeout) + + +@TestScenario +@Requirements( + RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), + RQ_SRS_007_LDAP_UnreachableServer("1.0"), ) def invalid_host(self): """Check that server returns an error when LDAP server host name is invalid. """ servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + users = [ + { + "server": "foo", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Host("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Host("1.0"), ) def empty_host(self): """Check that server returns an error when LDAP server host value is empty. """ servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + users = [ + { + "server": "foo", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_Host("1.0") + RQ_SRS_007_LDAP_Configuration_Server_Host("1.0"), ) def missing_host(self): """Check that server returns an error when LDAP server host is missing. """ servers = {"foo": {"port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + users = [ + { + "server": "foo", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def invalid_port(self): """Check that server returns an error when LDAP server port is not valid. """ servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def invalid_auth_dn_prefix(self): """Check that server returns an error when LDAP server port is not valid. """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "foo=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "foo=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def invalid_auth_dn_suffix(self): """Check that server returns an error when LDAP server port is not valid. """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",foo=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",foo=users,dc=company,dc=com", + } + } + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def invalid_enable_tls_value(self): """Check that server returns an error when enable_tls option has invalid value. """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "foo", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def invalid_tls_require_cert_value(self): """Check that server returns an error when tls_require_cert option has invalid value. """ - servers = {"openldap2": { - "host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "foo", - "ca_cert_dir": "/container/service/slapd/assets/certs/", - "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] + servers = { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "foo", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + } + } + users = [ + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def empty_ca_cert_dir(self): - """Check that server returns an error when ca_cert_dir is empty. - """ - servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "demand", - "ca_cert_dir": "", - "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] + """Check that server returns an error when ca_cert_dir is empty.""" + servers = { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + } + } + users = [ + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Server_Configuration_Invalid("1.0")) def empty_ca_cert_file(self): - """Check that server returns an error when ca_cert_file is empty. - """ - servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "demand", - "ca_cert_dir": "/container/service/slapd/assets/certs/", - "ca_cert_file": "" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] + """Check that server returns an error when ca_cert_file is empty.""" + servers = { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "", + } + } + users = [ + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario @Requirements( RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value("1.0"), RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix("1.0"), - RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix("1.0") + RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix("1.0"), ) def auth_dn_value(self): """Check that server configuration can properly define the `dn` value of the user.""" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + user = { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + } login(servers, user) + @TestOutline(Scenario) -@Examples("invalid_value", [ - ("-1", Name("negative int")), - ("foo", Name("string")), - ("", Name("empty string")), - ("36893488147419103232", Name("overflow with extremely large int value")), - ("-36893488147419103232", Name("overflow with extremely large negative int value")), - ("@#", Name("special characters")) -]) -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid("1.0") +@Examples( + "invalid_value", + [ + ("-1", Name("negative int")), + ("foo", Name("string")), + ("", Name("empty string")), + ("36893488147419103232", Name("overflow with extremely large int value")), + ( + "-36893488147419103232", + Name("overflow with extremely large negative int value"), + ), + ("@#", Name("special characters")), + ], ) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_VerificationCooldown_Invalid("1.0")) def invalid_verification_cooldown_value(self, invalid_value, timeout=300): """Check that server returns an error when LDAP server verification cooldown parameter is invalid. @@ -236,19 +335,26 @@ def invalid_verification_cooldown_value(self, invalid_value, timeout=300): error_message = f" Syntax error: Not a valid unsigned integer{': ' + invalid_value if invalid_value else invalid_value}" - with Given("LDAP server configuration that uses a negative integer for the verification_cooldown parameter"): - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": f"{invalid_value}" - }} + with Given( + "LDAP server configuration that uses a negative integer for the verification_cooldown parameter" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": f"{invalid_value}", + } + } with When("I try to use this configuration then it should not work"): invalid_server_config(servers, message=error_message, tail=30, timeout=timeout) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_Server_Syntax("2.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_Server_Syntax("2.0")) def syntax(self): """Check that server configuration with valid syntax can be loaded. ```xml @@ -279,23 +385,23 @@ def syntax(self): "auth_dn_suffix": ",ou=users,dc=company,dc=com", "verification_cooldown": "0", "enable_tls": "yes", - "tls_minimum_protocol_version": "tls1.2" , + "tls_minimum_protocol_version": "tls1.2", "tls_require_cert": "demand", "tls_cert_file": "/container/service/slapd/assets/certs/ldap.crt", "tls_key_file": "/container/service/slapd/assets/certs/ldap.key", "tls_ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", "tls_ca_cert_dir": "/container/service/slapd/assets/certs/", - "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384" + "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384", } } with ldap_servers(servers): pass + @TestFeature @Name("server config") def feature(self, node="clickhouse1"): - """Check that LDAP server configuration. - """ + """Check that LDAP server configuration.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/ldap/authentication/tests/user_config.py b/tests/testflows/ldap/authentication/tests/user_config.py index ebcfb6899c2..e1e2456e381 100644 --- a/tests/testflows/ldap/authentication/tests/user_config.py +++ b/tests/testflows/ldap/authentication/tests/user_config.py @@ -5,91 +5,136 @@ from testflows.core import * from ldap.authentication.tests.common import * from ldap.authentication.requirements import * + @TestScenario @Requirements( RQ_SRS_007_LDAP_User_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_User_Name_Empty("1.0") + RQ_SRS_007_LDAP_Configuration_User_Name_Empty("1.0"), ) def empty_user_name(self, timeout=300): - """Check that empty string as a user name is not allowed. - """ - servers = {"openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{"server": "openldap1", "username": "", "password": "user1", "login": True}] + """Check that empty string as a user name is not allowed.""" + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + {"server": "openldap1", "username": "", "password": "user1", "login": True} + ] config = create_ldap_users_config_content(*users) invalid_user_config(servers, config, timeout=timeout) + @TestScenario @Requirements( RQ_SRS_007_LDAP_User_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty("1.0") + RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty("1.0"), ) def empty_server_name(self, timeout=300): - """Check that if server name is an empty string then login is not allowed. - """ + """Check that if server name is an empty string then login is not allowed.""" message = "Exception: LDAP server name cannot be empty for user" - servers = {"openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{"server": "", "username": "user1", "password": "user1", "login": True, - "errorcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + { + "server": "", + "username": "user1", + "password": "user1", + "login": True, + "errorcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] config = create_ldap_users_config_content(*users) invalid_user_config(servers, config, message=message, tail=30, timeout=timeout) + @TestScenario @Requirements( RQ_SRS_007_LDAP_User_Configuration_Invalid("1.0"), - RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined("1.0") + RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined("1.0"), ) def empty_server_not_defined(self): - """Check that if server is not defined then login is not allowed. - """ - servers = {"openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{"server": "foo", "username": "user1", "password": "user1", "login": True, - "errorcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" - }] + """Check that if server is not defined then login is not allowed.""" + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + { + "server": "foo", + "username": "user1", + "password": "user1", + "login": True, + "errorcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_User_Syntax("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_User_Syntax("1.0")) def valid_user_config(self): """Check syntax of valid user configuration of LDAP authenticated user.""" - servers = {"openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{"server": "openldap1", "username": "user1", "password": "user1", "login": True}] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + ] login(servers, *users) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer("1.0")) def multiple_servers(self): """Check that user configuration allows to specify only one LDAP server for a given user and if multiple servers are specified then the first one is used.""" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { - "host": "openldap2", "enable_tls": "yes", "tls_require_cert": "never", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap2", + "enable_tls": "yes", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + user = { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + } with When("I first create regular user configuration file"): config = create_ldap_users_config_content(user) @@ -101,17 +146,21 @@ def multiple_servers(self): xml_user_ldap = xml_users.find(user["username"]).find("ldap") xml_append(xml_user_ldap, "server", "openldap2") xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + content = xml_with_utf8 + str( + xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), + "utf-8", + ) - new_config = Config(content, config.path, config.name, config.uid, config.preprocessed_name) + new_config = Config( + content, config.path, config.name, config.uid, config.preprocessed_name + ) with Then("I login and expect it to work as the first server shall be used"): login(servers, user, config=new_config) + @TestScenario -@Requirements( - RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP("1.0") -) +@Requirements(RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP("1.0")) def ldap_and_password(self): """Check that user can't be authenticated if both `ldap` and `password` is specified for the same user. We expect an error message to be present in the log @@ -120,14 +169,20 @@ def ldap_and_password(self): node = self.context.node servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } user = { - "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, "errorcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name", } with When("I first create regular user configuration file"): @@ -140,15 +195,24 @@ def ldap_and_password(self): xml_user = xml_users.find(user["username"]) xml_append(xml_user, "password", "hellothere") xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + content = xml_with_utf8 + str( + xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), + "utf-8", + ) - new_config = Config(content, config.path, config.name, config.uid, config.preprocessed_name) + new_config = Config( + content, config.path, config.name, config.uid, config.preprocessed_name + ) error_message = "DB::Exception: More than one field of 'password'" - with Then("I expect an error when I try to load the configuration file", description=error_message): + with Then( + "I expect an error when I try to load the configuration file", + description=error_message, + ): invalid_user_config(servers, new_config, message=error_message, tail=30) + @TestFeature @Name("user config") def feature(self, node="clickhouse1"): diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml deleted file mode 100644 index 53ffa10384e..00000000000 --- a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml +++ /dev/null @@ -1,442 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 7200 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env/clickhouse-service.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env/clickhouse-service.yml new file mode 100644 index 00000000000..74661f6fa04 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env/clickhouse-service.yml @@ -0,0 +1,29 @@ +version: '2.3' + +services: + clickhouse: + image: clickhouse/integration-test + init: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "tail -f /dev/null" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env/docker-compose.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env/docker-compose.yml new file mode 100644 index 00000000000..36e25ef766e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env/docker-compose.yml @@ -0,0 +1,162 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env/openldap-service.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env/openldap-service.yml new file mode 100644 index 00000000000..606ea3f723f --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env/openldap-service.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + environment: + PHPLDAPADMIN_HTTPS=false: + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env/zookeeper-service.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/clickhouse-service.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/clickhouse-service.yml new file mode 100644 index 00000000000..a73d31421c8 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/clickhouse-service.yml @@ -0,0 +1,29 @@ +version: '2.3' + +services: + clickhouse: + image: registry.gitlab.com/altinity-public/container-images/test/clickhouse-integration-test:21.12 + privileged: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 10 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/docker-compose.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/docker-compose.yml new file mode 100644 index 00000000000..36e25ef766e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/docker-compose.yml @@ -0,0 +1,162 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/openldap-service.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/openldap-service.yml new file mode 100644 index 00000000000..606ea3f723f --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/openldap-service.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + environment: + PHPLDAPADMIN_HTTPS=false: + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/zookeeper-service.yml b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/external_user_directory_env_arm64/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py index bae019ceae7..de53bf9128e 100755 --- a/tests/testflows/ldap/external_user_directory/regression.py +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -8,59 +8,98 @@ append_path(sys.path, "..", "..") from helpers.cluster import Cluster from helpers.argparser import argparser from ldap.external_user_directory.requirements import * +from helpers.common import check_clickhouse_version # Cross-outs of known fails xfails = { - "connection protocols/tls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls require cert default demand": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls with custom port": - [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], - "connection protocols/tls cipher suite": - [(Fail, "can't get it to work")] + "connection protocols/tls/tls_require_cert='try'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/tls/tls_require_cert='demand'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/starttls/tls_require_cert='try'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/starttls/tls_require_cert='demand'": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/tls require cert default demand": [ + (Fail, "can't be tested with self-signed certificates") + ], + "connection protocols/starttls with custom port": [ + ( + Fail, + "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server", + ) + ], + "connection protocols/tls cipher suite": [(Fail, "can't get it to work")], } +ffails = { + "user authentications/verification cooldown performance/:": ( + Skip, + "causes timeout on 21.8", + ( + lambda test: check_clickhouse_version(">=21.8")(test) + and check_clickhouse_version("<21.9")(test) + ), + ) +} + + @TestFeature @Name("external user directory") @ArgumentParser(argparser) -@Specifications( - SRS_009_ClickHouse_LDAP_External_User_Directory -) -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") -) +@Specifications(SRS_009_ClickHouse_LDAP_External_User_Directory) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0")) @XFails(xfails) -def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): - """ClickHouse LDAP external user directory regression module. - """ +@FFails(ffails) +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse LDAP external user directory regression module.""" nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), } + self.context.clickhouse_version = clickhouse_version + if stress is not None: self.context.stress = stress - if parallel is not None: - self.context.parallel = parallel - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "ldap_external_user_directory_env")) as cluster: + from platform import processor as current_cpu + + folder_name = os.path.basename(current_dir()) + if current_cpu() == "aarch64": + env = f"{folder_name}_env_arm64" + else: + env = f"{folder_name}_env" + + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), env), + ) as cluster: self.context.cluster = cluster Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.external_user_directory.tests.simple", "scenario")) Feature(run=load("ldap.external_user_directory.tests.restart", "feature")) Feature(run=load("ldap.external_user_directory.tests.server_config", "feature")) - Feature(run=load("ldap.external_user_directory.tests.external_user_directory_config", "feature")) + Feature( + run=load( + "ldap.external_user_directory.tests.external_user_directory_config", + "feature", + ) + ) Feature(run=load("ldap.external_user_directory.tests.connections", "feature")) - Feature(run=load("ldap.external_user_directory.tests.authentications", "feature")) + Feature( + run=load("ldap.external_user_directory.tests.authentications", "feature") + ) Feature(run=load("ldap.external_user_directory.tests.roles", "feature")) + if main(): regression() diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py index 90969725725..e15cc7a034e 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.py +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -9,1574 +9,1665 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.1.1.1') + num="4.1.1.1", +) RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories.\n" + "\n" + ), link=None, level=4, - num='4.1.1.2') + num="4.1.1.2", +) RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories_Lookup = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL attempt to authenticate external [LDAP] user\n' - 'using [LDAP] external user directory in the same order\n' - 'in which user directories are specified in the `config.xml` file.\n' - 'If a user cannot be authenticated using the first [LDAP] external user directory\n' - 'then the next user directory in the list SHALL be used.\n' - '\n' - ), + "[ClickHouse] SHALL attempt to authenticate external [LDAP] user\n" + "using [LDAP] external user directory in the same order\n" + "in which user directories are specified in the `config.xml` file.\n" + "If a user cannot be authenticated using the first [LDAP] external user directory\n" + "then the next user directory in the list SHALL be used.\n" + "\n" + ), link=None, level=4, - num='4.1.1.3') + num="4.1.1.3", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server\n' - 'as soon as they are added to the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server\n" + "as soon as they are added to the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.1.1.4') + num="4.1.1.4", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not allow authentication of users that\n' - 'were previously defined only on the [LDAP] server but were removed\n' - 'from the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL not allow authentication of users that\n" + "were previously defined only on the [LDAP] server but were removed\n" + "from the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.1.1.5') + num="4.1.1.5", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' - 'user name and password match [LDAP] server records for the user\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n" + "user name and password match [LDAP] server records for the user\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.6') + num="4.1.1.6", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' - 'do not match [LDAP] server records for the user\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n" + "do not match [LDAP] server records for the user\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.7') + num="4.1.1.7", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' - 'on the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n" + "on the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.8') + num="4.1.1.8", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' - 'for the user is changed on the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication if the password\n" + "for the user is changed on the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.9') + num="4.1.1.9", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.10') + num="4.1.1.10", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users after server is restarted\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users after server is restarted\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.11') + num="4.1.1.11", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of users using [LDAP] server\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.12') + num="4.1.1.12", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authentication of valid users and\n' - 'prohibit authentication of invalid users using [LDAP] server\n' - 'in parallel without having invalid attempts affecting valid authentications\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support authentication of valid users and\n" + "prohibit authentication of invalid users using [LDAP] server\n" + "in parallel without having invalid attempts affecting valid authentications\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.13') + num="4.1.1.13", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of external [LDAP] users\n' - 'authenticated using multiple [LDAP] external user directories.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of external [LDAP] users\n" + "authenticated using multiple [LDAP] external user directories.\n" + "\n" + ), link=None, level=4, - num='4.1.1.14') + num="4.1.1.14", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of users defined only locally\n' - 'when one or more [LDAP] external user directories are specified in the configuration file.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of users defined only locally\n" + "when one or more [LDAP] external user directories are specified in the configuration file.\n" + "\n" + ), link=None, level=4, - num='4.1.1.15') + num="4.1.1.15", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users\n' - 'authenticated using multiple [LDAP] external user directories.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users\n" + "authenticated using multiple [LDAP] external user directories.\n" + "\n" + ), link=None, level=4, - num='4.1.1.16') + num="4.1.1.16", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user\n' - 'authenticated using the same [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user\n" + "authenticated using the same [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.1.17') + num="4.1.1.17", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_DynamicallyAddedAndRemovedUsers = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.DynamicallyAddedAndRemovedUsers', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.DynamicallyAddedAndRemovedUsers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of users using\n' - '[LDAP] external user directory when [LDAP] users are dynamically added and\n' - 'removed.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of users using\n" + "[LDAP] external user directory when [LDAP] users are dynamically added and\n" + "removed.\n" + "\n" + ), link=None, level=4, - num='4.1.1.18') + num="4.1.1.18", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol\n' - 'while connecting to the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol\n" + "while connecting to the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.1') + num="4.1.2.1", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol\n' - 'while connecting to the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol\n" + "while connecting to the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.2') + num="4.1.2.2", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' - 'plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n" + "plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.3') + num="4.1.2.3", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_Validation = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support certificate validation used for [TLS] connections\n' - 'to the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support certificate validation used for [TLS] connections\n" + "to the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.4') + num="4.1.2.4", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_SelfSigned = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support self-signed certificates for [TLS] connections\n' - 'to the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support self-signed certificates for [TLS] connections\n" + "to the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.5') + num="4.1.2.5", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_SpecificCertificationAuthority = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections\n' - 'to the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections\n" + "to the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.6') + num="4.1.2.6", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_Anonymous = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n" + "authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.7') + num="4.1.2.7", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_Unauthenticated = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n" + "authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.8') + num="4.1.2.8", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' - 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' - '\n' - ), + "[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n" + "authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.9') + num="4.1.2.9", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.1.2.10') + num="4.1.2.10", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority', - version='2.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL lookup user presence in the same order\n' - 'as user directories are defined in the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL lookup user presence in the same order\n" + "as user directories are defined in the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.2.1.1') + num="4.2.1.1", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Restart_Server = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support restarting server when one or more LDAP external directories\n' - 'are configured.\n' - '\n' - ), + "[ClickHouse] SHALL support restarting server when one or more LDAP external directories\n" + "are configured.\n" + "\n" + ), link=None, level=4, - num='4.2.1.2') + num="4.2.1.2", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Restart_Server_ParallelLogins = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server.ParallelLogins', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server.ParallelLogins", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support restarting server when one or more LDAP external directories\n' - 'are configured during parallel [LDAP] user logins.\n' - '\n' - ), + "[ClickHouse] SHALL support restarting server when one or more LDAP external directories\n" + "are configured during parallel [LDAP] user logins.\n" + "\n" + ), link=None, level=4, - num='4.2.1.3') + num="4.2.1.3", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed', - version='2.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL allow authentication even if the roles that are specified in the configuration\n' - 'of the external user directory are not defined at the time of the authentication attempt.\n' - '\n' - ), + "[ClickHouse] SHALL allow authentication even if the roles that are specified in the configuration\n" + "of the external user directory are not defined at the time of the authentication attempt.\n" + "\n" + ), link=None, level=4, - num='4.2.2.1') + num="4.2.2.1", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP\n' - 'users authenticated using external user directory if it is removed\n' - 'including currently cached users that are still able to authenticated where the removed\n' - 'role is specified in the configuration of the external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP\n" + "users authenticated using external user directory if it is removed\n" + "including currently cached users that are still able to authenticated where the removed\n" + "role is specified in the configuration of the external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.2.2') + num="4.2.2.2", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reassign the role and add the privileges provided by the role\n' - 'when it is re-added after removal for all LDAP users authenticated using external user directory\n' - 'including any cached users where the re-added role was specified in the configuration of the external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL reassign the role and add the privileges provided by the role\n" + "when it is re-added after removal for all LDAP users authenticated using external user directory\n" + "including any cached users where the re-added role was specified in the configuration of the external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.2.3') + num="4.2.2.3", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP\n' - 'users authenticated using external user directory unless the role is specified\n' - 'in the configuration of the external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP\n" + "users authenticated using external user directory unless the role is specified\n" + "in the configuration of the external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.2.4') + num="4.2.2.4", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory\n' - 'including cached users when new privilege is added to one of the roles specified\n' - 'in the configuration of the external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory\n" + "including cached users when new privilege is added to one of the roles specified\n" + "in the configuration of the external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.2.5') + num="4.2.2.5", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory\n' - 'including cached users when privilege is removed from all the roles specified\n' - 'in the configuration of the external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory\n" + "including cached users when privilege is removed from all the roles specified\n" + "in the configuration of the external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.2.6') + num="4.2.2.6", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NotPresent_Added = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NotPresent.Added', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NotPresent.Added", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL add a role to the users authenticated using LDAP external user directory\n' - 'that did not exist during the time of authentication but are defined in the \n' - 'configuration file as soon as the role with that name becomes\n' - 'available.\n' - '\n' - ), + "[ClickHouse] SHALL add a role to the users authenticated using LDAP external user directory\n" + "that did not exist during the time of authentication but are defined in the \n" + "configuration file as soon as the role with that name becomes\n" + "available.\n" + "\n" + ), link=None, level=4, - num='4.2.2.7') + num="4.2.2.7", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' - '\n' - ), + "[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n" + "\n" + ), link=None, level=4, - num='4.2.3.1') + num="4.2.3.1", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the [LDAP] servers defined in the\n' - '`ldap_servers` section of the `config.xml` as the server to be used\n' - 'for a external user directory that uses an [LDAP] server as a source of user definitions.\n' - '\n' - ), + "[ClickHouse] SHALL support using the [LDAP] servers defined in the\n" + "`ldap_servers` section of the `config.xml` as the server to be used\n" + "for a external user directory that uses an [LDAP] server as a source of user definitions.\n" + "\n" + ), link=None, level=4, - num='4.2.3.2') + num="4.2.3.2", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support empty string as a server name.\n' - '\n' - ), + "[ClickHouse] SHALL not support empty string as a server name.\n" "\n" + ), link=None, level=4, - num='4.2.3.3') + num="4.2.3.3", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' - 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify [LDAP]\n" + "server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n" + "\n" + ), link=None, level=4, - num='4.2.3.4') + num="4.2.3.4", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n" + "\n" + ), link=None, level=4, - num='4.2.3.5') + num="4.2.3.5", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' - '\n' - ), + "[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n" + "\n" + ), link=None, level=4, - num='4.2.3.6') + num="4.2.3.6", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify the prefix\n' - 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify the prefix\n" + "of value used to construct the DN to bound to during authentication via [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.7') + num="4.2.3.7", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify the suffix\n' - 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify the suffix\n" + "of value used to construct the DN to bound to during authentication via [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.8') + num="4.2.3.8", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Value = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' - '\n' + "[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n" + "\n" "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" - '\n' - ), + "\n" + ), link=None, level=4, - num='4.2.3.9') + num="4.2.3.9", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.10') + num="4.2.3.10", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' - 'to enable SSL/TLS `ldaps://` protocol.\n' - '\n' - ), + "[ClickHouse] SHALL use `yes` value as the default for `` parameter\n" + "to enable SSL/TLS `ldaps://` protocol.\n" + "\n" + ), link=None, level=4, - num='4.2.3.11') + num="4.2.3.11", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' - 'plain text `ldap://` protocol.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n" + "plain text `ldap://` protocol.\n" + "\n" + ), link=None, level=4, - num='4.2.3.12') + num="4.2.3.12", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' - 'SSL/TLS `ldaps://` protocol.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n" + "SSL/TLS `ldaps://` protocol.\n" + "\n" + ), link=None, level=4, - num='4.2.3.13') + num="4.2.3.13", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' - 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n" + "legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n" + "\n" + ), link=None, level=4, - num='4.2.3.14') + num="4.2.3.14", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify\n' - 'the minimum protocol version of SSL/TLS.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify\n" + "the minimum protocol version of SSL/TLS.\n" + "\n" + ), link=None, level=4, - num='4.2.3.15') + num="4.2.3.15", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' - 'as a value of the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n" + "as a value of the `` parameter.\n" + "\n" + ), link=None, level=4, - num='4.2.3.16') + num="4.2.3.16", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n" + "\n" + ), link=None, level=4, - num='4.2.3.17') + num="4.2.3.17", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' - 'certificate verification behavior.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n" + "certificate verification behavior.\n" + "\n" + ), link=None, level=4, - num='4.2.3.18') + num="4.2.3.18", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n" + "\n" + ), link=None, level=4, - num='4.2.3.19') + num="4.2.3.19", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' - 'provided, the session SHALL be immediately terminated.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n" + "enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n" + "provided, the session SHALL be immediately terminated.\n" + "\n" + ), link=None, level=4, - num='4.2.3.20') + num="4.2.3.20", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no\n' - 'certificate is provided, the session SHALL proceed normally.\n' - 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n" + "enable requesting of client certificate. If no\n" + "certificate is provided, the session SHALL proceed normally.\n" + "If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n" + "\n" + ), link=None, level=4, - num='4.2.3.21') + num="4.2.3.21", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n' - 'enable requesting of client certificate. If no certificate is provided, the session\n' - 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' - 'immediately terminated.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n" + "enable requesting of client certificate. If no certificate is provided, the session\n" + "SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n" + "immediately terminated.\n" + "\n" + ), link=None, level=4, - num='4.2.3.22') + num="4.2.3.22", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' - 'disable requesting of client certificate.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n" + "disable requesting of client certificate.\n" + "\n" + ), link=None, level=4, - num='4.2.3.23') + num="4.2.3.23", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' - '[ClickHouse] to establish connection with the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` to specify the path to certificate file used by\n" + "[ClickHouse] to establish connection with the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.24') + num="4.2.3.24", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSKeyFile = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' - 'specified by the `` parameter.\n' - '\n' - ), + "[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n" + "specified by the `` parameter.\n" + "\n" + ), link=None, level=4, - num='4.2.3.25') + num="4.2.3.25", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify to a path to\n' - 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify to a path to\n" + "the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.26') + num="4.2.3.26", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertFile = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' - '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `` parameter to specify a path to a specific\n" + "[CA] certificate file used to verify certificates provided by the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.27') + num="4.2.3.27", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n' - 'The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' - '```\n' - '\n' - 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' - '[ClickHouse] and therefore might change.\n' - '\n' - ), + "[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n" + "The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n" + "```\n" + "\n" + "The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n" + "[ClickHouse] and therefore might change.\n" + "\n" + ), link=None, level=4, - num='4.2.3.28') + num="4.2.3.28", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n' - 'to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n' - 'After period of time since the last successful attempt expires then on the authentication attempt\n' - 'SHALL result in contacting the [LDAP] server to verify the username and password.\n' - '\n' - ), + "[ClickHouse] SHALL support `verification_cooldown` parameter in the [LDAP] server configuration section\n" + "that SHALL define a period of time, in seconds, after a successful bind attempt, during which a user SHALL be assumed\n" + "to be successfully authenticated for all consecutive requests without contacting the [LDAP] server.\n" + "After period of time since the last successful attempt expires then on the authentication attempt\n" + "SHALL result in contacting the [LDAP] server to verify the username and password.\n" + "\n" + ), link=None, level=4, - num='4.2.3.29') + num="4.2.3.29", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Default = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'SHALL have a default value of `0` that disables caching and forces contacting\n' - 'the [LDAP] server for each authentication request.\n' - '\n' - ), + "[ClickHouse] `verification_cooldown` parameter in the [LDAP] server configuration section\n" + "SHALL have a default value of `0` that disables caching and forces contacting\n" + "the [LDAP] server for each authentication request.\n" + "\n" + ), link=None, level=4, - num='4.2.3.30') + num="4.2.3.30", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Invalid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n' - '\n' - 'For example:\n' - '\n' - '* negative integer\n' - '* string\n' - '* empty value\n' - '* extremely large positive value (overflow)\n' - '* extremely large negative value (overflow)\n' - '\n' - 'The error SHALL appear in the log and SHALL be similar to the following:\n' - '\n' - '```bash\n' - ' Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n' - '```\n' - '\n' - ), + "[Clickhouse] SHALL return an error if the value provided for the `verification_cooldown` parameter is not a valid positive integer.\n" + "\n" + "For example:\n" + "\n" + "* negative integer\n" + "* string\n" + "* empty value\n" + "* extremely large positive value (overflow)\n" + "* extremely large negative value (overflow)\n" + "\n" + "The error SHALL appear in the log and SHALL be similar to the following:\n" + "\n" + "```bash\n" + " Access(user directories): Could not parse LDAP server `openldap1`: Poco::Exception. Code: 1000, e.code() = 0, e.displayText() = Syntax error: Not a valid unsigned integer: *input value*\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.2.3.31') + num="4.2.3.31", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax', - version='2.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n' - 'configuration file or of any configuration file inside the `config.d` directory.\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' localhost\n' - ' 636\n' - ' cn=\n' - ' , ou=users, dc=example, dc=com\n' - ' 0\n' - ' yes\n' - ' tls1.2\n' - ' demand\n' - ' /path/to/tls_cert_file\n' - ' /path/to/tls_key_file\n' - ' /path/to/tls_ca_cert_file\n' - ' /path/to/tls_ca_cert_dir\n' - ' ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n" + "configuration file or of any configuration file inside the `config.d` directory.\n" + "\n" + "```xml\n" + "\n" + " \n" + " localhost\n" + " 636\n" + " cn=\n" + " , ou=users, dc=example, dc=com\n" + " 0\n" + " yes\n" + " tls1.2\n" + " demand\n" + " /path/to/tls_cert_file\n" + " /path/to/tls_key_file\n" + " /path/to/tls_ca_cert_file\n" + " /path/to/tls_ca_cert_dir\n" + " ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.2.3.32') + num="4.2.3.32", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml`\n' - 'that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions.\n' - '\n' - ), + "[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml`\n" + "that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions.\n" + "\n" + ), link=None, level=4, - num='4.2.3.33') + num="4.2.3.33", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne', - version='2.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml`\n' - 'that SHALL allow to define more than one external user directory that use an [LDAP] server as a source\n' - 'of user definitions.\n' - '\n' - ), + "[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml`\n" + "that SHALL allow to define more than one external user directory that use an [LDAP] server as a source\n" + "of user definitions.\n" + "\n" + ), link=None, level=4, - num='4.2.3.34') + num="4.2.3.34", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Syntax = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `` section with the following syntax\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' my_ldap_server\n' - ' \n' - ' \n' - ' \n' - ' \n' - ' \n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `` section with the following syntax\n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " my_ldap_server\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.2.3.35') + num="4.2.3.35", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `server` parameter in the `` sub-section in the ``\n' - 'section of the `config.xml` that SHALL specify one of LDAP server names\n' - 'defined in `` section.\n' - '\n' - ), + "[ClickHouse] SHALL support `server` parameter in the `` sub-section in the ``\n" + "section of the `config.xml` that SHALL specify one of LDAP server names\n" + "defined in `` section.\n" + "\n" + ), link=None, level=4, - num='4.2.3.36') + num="4.2.3.36", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n' - 'is empty.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n" + "is empty.\n" + "\n" + ), link=None, level=4, - num='4.2.3.37') + num="4.2.3.37", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n' - 'is missing.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n" + "is missing.\n" + "\n" + ), link=None, level=4, - num='4.2.3.38') + num="4.2.3.38", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the ``\n' - 'if more than one `server` parameter is defined in the configuration.\n' - '\n' - ), + "[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the ``\n" + "if more than one `server` parameter is defined in the configuration.\n" + "\n" + ), link=None, level=4, - num='4.2.3.39') + num="4.2.3.39", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the server specified as the value of the ``\n' - 'parameter is not defined.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the server specified as the value of the ``\n" + "parameter is not defined.\n" + "\n" + ), link=None, level=4, - num='4.2.3.40') + num="4.2.3.40", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the ``\n' - 'section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL\n' - 'be assigned to all users retrieved from the [LDAP] server.\n' - '\n' - ), + "[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the ``\n" + "section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL\n" + "be assigned to all users retrieved from the [LDAP] server.\n" + "\n" + ), link=None, level=4, - num='4.2.3.41') + num="4.2.3.41", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only use the first definitition of the `roles` parameter\n' - 'in the `` sub-section in the ``\n' - 'if more than one `roles` parameter is defined in the configuration.\n' - '\n' - ), + "[ClickHouse] SHALL only use the first definitition of the `roles` parameter\n" + "in the `` sub-section in the ``\n" + "if more than one `roles` parameter is defined in the configuration.\n" + "\n" + ), link=None, level=4, - num='4.2.3.42') + num="4.2.3.42", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid', - version='2.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid", + version="2.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not return an error if the role specified in the ``\n' - 'parameter does not exist locally. \n' - '\n' - ), + "[ClickHouse] SHALL not return an error if the role specified in the ``\n" + "parameter does not exist locally. \n" + "\n" + ), link=None, level=4, - num='4.2.3.43') + num="4.2.3.43", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n' - 'to perform any action if the `roles` parameter in the `` sub-section in the ``\n' - 'section is empty.\n' - '\n' - ), + "[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n" + "to perform any action if the `roles` parameter in the `` sub-section in the ``\n" + "section is empty.\n" + "\n" + ), link=None, level=4, - num='4.2.3.44') + num="4.2.3.44", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n' - 'to perform any action if the `roles` parameter in the `` sub-section in the ``\n' - 'section is missing.\n' - '\n' - ), + "[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n" + "to perform any action if the `roles` parameter in the `` sub-section in the ``\n" + "section is missing.\n" + "\n" + ), link=None, level=4, - num='4.2.3.45') + num="4.2.3.45", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Empty = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support authenticating users with empty username\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL not support authenticating users with empty username\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.4.1') + num="4.2.4.1", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.4.2') + num="4.2.4.2", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8 = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters\n' - 'when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters\n" + "when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.4.3') + num="4.2.4.3", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support authenticating users with empty passwords\n' - 'even if an empty password is valid for the user and\n' - 'is allowed by the [LDAP] server when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL not support authenticating users with empty passwords\n" + "even if an empty password is valid for the user and\n" + "is allowed by the [LDAP] server when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.4.4') + num="4.2.4.4", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support long password of at least 256 bytes\n' - 'that can be used to authenticate users when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support long password of at least 256 bytes\n" + "that can be used to authenticate users when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.4.5') + num="4.2.4.5", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8 = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' - 'used to authenticate users when using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support [UTF-8] characters in passwords\n" + "used to authenticate users when using [LDAP] external user directory.\n" + "\n" + ), link=None, level=4, - num='4.2.4.6') + num="4.2.4.6", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL provide better login performance of users authenticated using [LDAP] external user directory\n' - 'when `verification_cooldown` parameter is set to a positive value when comparing\n' - 'to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n' - 'making a large number of repeated requests.\n' - '\n' - ), + "[ClickHouse] SHALL provide better login performance of users authenticated using [LDAP] external user directory\n" + "when `verification_cooldown` parameter is set to a positive value when comparing\n" + "to the the case when `verification_cooldown` is turned off either for a single user or multiple users\n" + "making a large number of repeated requests.\n" + "\n" + ), link=None, level=4, - num='4.2.4.7') + num="4.2.4.7", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n' - '`verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n' - 'change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n' + "[ClickHouse] SHALL reset any currently cached [LDAP] authentication bind requests enabled by the\n" + "`verification_cooldown` parameter in the [LDAP] server configuration section\n" + "if either `host`, `port`, `auth_dn_prefix`, or `auth_dn_suffix` parameter values\n" + "change in the configuration file. The reset SHALL cause any subsequent authentication attempts for any user\n" "to result in contacting the [LDAP] server to verify user's username and password.\n" - '\n' - ), + "\n" + ), link=None, level=4, - num='4.2.4.8') + num="4.2.4.8", +) RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_InvalidPassword = Requirement( - name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword', - version='1.0', + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n' - '`verification_cooldown` parameter in the [LDAP] server configuration section\n' - 'for the user if the password provided in the current authentication attempt does not match\n' - 'the valid password provided during the first successful authentication request that was cached\n' - 'for this exact user. The reset SHALL cause the next authentication attempt for this user\n' + "[ClickHouse] SHALL reset current cached [LDAP] authentication bind request enabled by the\n" + "`verification_cooldown` parameter in the [LDAP] server configuration section\n" + "for the user if the password provided in the current authentication attempt does not match\n" + "the valid password provided during the first successful authentication request that was cached\n" + "for this exact user. The reset SHALL cause the next authentication attempt for this user\n" "to result in contacting the [LDAP] server to verify user's username and password.\n" - '\n' - ), + "\n" + ), link=None, level=4, - num='4.2.4.9') + num="4.2.4.9", +) SRS_009_ClickHouse_LDAP_External_User_Directory = Specification( - name='SRS-009 ClickHouse LDAP External User Directory', + name="SRS-009 ClickHouse LDAP External User Directory", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -1588,113 +1679,481 @@ SRS_009_ClickHouse_LDAP_External_User_Directory = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='LDAP', level=2, num='3.1'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='Generic', level=2, num='4.1'), - Heading(name='User Authentication', level=3, num='4.1.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication', level=4, num='4.1.1.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories', level=4, num='4.1.1.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup', level=4, num='4.1.1.3'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers', level=4, num='4.1.1.4'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers', level=4, num='4.1.1.5'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid', level=4, num='4.1.1.6'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid', level=4, num='4.1.1.7'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged', level=4, num='4.1.1.8'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged', level=4, num='4.1.1.9'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart', level=4, num='4.1.1.10'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart', level=4, num='4.1.1.11'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel', level=4, num='4.1.1.12'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid', level=4, num='4.1.1.13'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers', level=4, num='4.1.1.14'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly', level=4, num='4.1.1.15'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP', level=4, num='4.1.1.16'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser', level=4, num='4.1.1.17'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.DynamicallyAddedAndRemovedUsers', level=4, num='4.1.1.18'), - Heading(name='Connection', level=3, num='4.1.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText', level=4, num='4.1.2.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS', level=4, num='4.1.2.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS', level=4, num='4.1.2.3'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation', level=4, num='4.1.2.4'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned', level=4, num='4.1.2.5'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority', level=4, num='4.1.2.6'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous', level=4, num='4.1.2.7'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated', level=4, num='4.1.2.8'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword', level=4, num='4.1.2.9'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer', level=4, num='4.1.2.10'), - Heading(name='Specific', level=2, num='4.2'), - Heading(name='User Discovery', level=3, num='4.2.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority', level=4, num='4.2.1.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server', level=4, num='4.2.1.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server.ParallelLogins', level=4, num='4.2.1.3'), - Heading(name='Roles', level=3, num='4.2.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed', level=4, num='4.2.2.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges', level=4, num='4.2.2.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges', level=4, num='4.2.2.3'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New', level=4, num='4.2.2.4'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege', level=4, num='4.2.2.5'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege', level=4, num='4.2.2.6'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NotPresent.Added', level=4, num='4.2.2.7'), - Heading(name='Configuration', level=3, num='4.2.3'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid', level=4, num='4.2.3.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition', level=4, num='4.2.3.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name', level=4, num='4.2.3.3'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host', level=4, num='4.2.3.4'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port', level=4, num='4.2.3.5'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default', level=4, num='4.2.3.6'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix', level=4, num='4.2.3.7'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix', level=4, num='4.2.3.8'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value', level=4, num='4.2.3.9'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS', level=4, num='4.2.3.10'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default', level=4, num='4.2.3.11'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No', level=4, num='4.2.3.12'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes', level=4, num='4.2.3.13'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS', level=4, num='4.2.3.14'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion', level=4, num='4.2.3.15'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values', level=4, num='4.2.3.16'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default', level=4, num='4.2.3.17'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert', level=4, num='4.2.3.18'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default', level=4, num='4.2.3.19'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand', level=4, num='4.2.3.20'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow', level=4, num='4.2.3.21'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try', level=4, num='4.2.3.22'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never', level=4, num='4.2.3.23'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile', level=4, num='4.2.3.24'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile', level=4, num='4.2.3.25'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir', level=4, num='4.2.3.26'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile', level=4, num='4.2.3.27'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite', level=4, num='4.2.3.28'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown', level=4, num='4.2.3.29'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default', level=4, num='4.2.3.30'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid', level=4, num='4.2.3.31'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax', level=4, num='4.2.3.32'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory', level=4, num='4.2.3.33'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne', level=4, num='4.2.3.34'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax', level=4, num='4.2.3.35'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server', level=4, num='4.2.3.36'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty', level=4, num='4.2.3.37'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing', level=4, num='4.2.3.38'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne', level=4, num='4.2.3.39'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid', level=4, num='4.2.3.40'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles', level=4, num='4.2.3.41'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne', level=4, num='4.2.3.42'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid', level=4, num='4.2.3.43'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty', level=4, num='4.2.3.44'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing', level=4, num='4.2.3.45'), - Heading(name='Authentication', level=3, num='4.2.4'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty', level=4, num='4.2.4.1'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long', level=4, num='4.2.4.2'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8', level=4, num='4.2.4.3'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty', level=4, num='4.2.4.4'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long', level=4, num='4.2.4.5'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8', level=4, num='4.2.4.6'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance', level=4, num='4.2.4.7'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters', level=4, num='4.2.4.8'), - Heading(name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword', level=4, num='4.2.4.9'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="LDAP", level=2, num="3.1"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="Generic", level=2, num="4.1"), + Heading(name="User Authentication", level=3, num="4.1.1"), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication", + level=4, + num="4.1.1.1", ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories", + level=4, + num="4.1.1.2", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup", + level=4, + num="4.1.1.3", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers", + level=4, + num="4.1.1.4", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers", + level=4, + num="4.1.1.5", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid", + level=4, + num="4.1.1.6", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid", + level=4, + num="4.1.1.7", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged", + level=4, + num="4.1.1.8", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged", + level=4, + num="4.1.1.9", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart", + level=4, + num="4.1.1.10", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart", + level=4, + num="4.1.1.11", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel", + level=4, + num="4.1.1.12", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid", + level=4, + num="4.1.1.13", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers", + level=4, + num="4.1.1.14", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly", + level=4, + num="4.1.1.15", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP", + level=4, + num="4.1.1.16", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser", + level=4, + num="4.1.1.17", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.DynamicallyAddedAndRemovedUsers", + level=4, + num="4.1.1.18", + ), + Heading(name="Connection", level=3, num="4.1.2"), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText", + level=4, + num="4.1.2.1", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS", + level=4, + num="4.1.2.2", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS", + level=4, + num="4.1.2.3", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation", + level=4, + num="4.1.2.4", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned", + level=4, + num="4.1.2.5", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority", + level=4, + num="4.1.2.6", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous", + level=4, + num="4.1.2.7", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated", + level=4, + num="4.1.2.8", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword", + level=4, + num="4.1.2.9", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer", + level=4, + num="4.1.2.10", + ), + Heading(name="Specific", level=2, num="4.2"), + Heading(name="User Discovery", level=3, num="4.2.1"), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority", + level=4, + num="4.2.1.1", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server", + level=4, + num="4.2.1.2", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Restart.Server.ParallelLogins", + level=4, + num="4.2.1.3", + ), + Heading(name="Roles", level=3, num="4.2.2"), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed", + level=4, + num="4.2.2.1", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges", + level=4, + num="4.2.2.2", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges", + level=4, + num="4.2.2.3", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New", + level=4, + num="4.2.2.4", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege", + level=4, + num="4.2.2.5", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege", + level=4, + num="4.2.2.6", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NotPresent.Added", + level=4, + num="4.2.2.7", + ), + Heading(name="Configuration", level=3, num="4.2.3"), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid", + level=4, + num="4.2.3.1", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition", + level=4, + num="4.2.3.2", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name", + level=4, + num="4.2.3.3", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host", + level=4, + num="4.2.3.4", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port", + level=4, + num="4.2.3.5", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default", + level=4, + num="4.2.3.6", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix", + level=4, + num="4.2.3.7", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix", + level=4, + num="4.2.3.8", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value", + level=4, + num="4.2.3.9", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS", + level=4, + num="4.2.3.10", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default", + level=4, + num="4.2.3.11", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No", + level=4, + num="4.2.3.12", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes", + level=4, + num="4.2.3.13", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS", + level=4, + num="4.2.3.14", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion", + level=4, + num="4.2.3.15", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values", + level=4, + num="4.2.3.16", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default", + level=4, + num="4.2.3.17", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert", + level=4, + num="4.2.3.18", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default", + level=4, + num="4.2.3.19", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand", + level=4, + num="4.2.3.20", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow", + level=4, + num="4.2.3.21", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try", + level=4, + num="4.2.3.22", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never", + level=4, + num="4.2.3.23", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile", + level=4, + num="4.2.3.24", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile", + level=4, + num="4.2.3.25", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir", + level=4, + num="4.2.3.26", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile", + level=4, + num="4.2.3.27", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite", + level=4, + num="4.2.3.28", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown", + level=4, + num="4.2.3.29", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Default", + level=4, + num="4.2.3.30", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.VerificationCooldown.Invalid", + level=4, + num="4.2.3.31", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax", + level=4, + num="4.2.3.32", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory", + level=4, + num="4.2.3.33", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne", + level=4, + num="4.2.3.34", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax", + level=4, + num="4.2.3.35", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server", + level=4, + num="4.2.3.36", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty", + level=4, + num="4.2.3.37", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing", + level=4, + num="4.2.3.38", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne", + level=4, + num="4.2.3.39", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid", + level=4, + num="4.2.3.40", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles", + level=4, + num="4.2.3.41", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne", + level=4, + num="4.2.3.42", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid", + level=4, + num="4.2.3.43", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty", + level=4, + num="4.2.3.44", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing", + level=4, + num="4.2.3.45", + ), + Heading(name="Authentication", level=3, num="4.2.4"), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty", + level=4, + num="4.2.4.1", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long", + level=4, + num="4.2.4.2", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8", + level=4, + num="4.2.4.3", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty", + level=4, + num="4.2.4.4", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long", + level=4, + num="4.2.4.5", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8", + level=4, + num="4.2.4.6", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Performance", + level=4, + num="4.2.4.7", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.ChangeInCoreServerParameters", + level=4, + num="4.2.4.8", + ), + Heading( + name="RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.VerificationCooldown.Reset.InvalidPassword", + level=4, + num="4.2.4.9", + ), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication, RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories, @@ -1788,8 +2247,8 @@ SRS_009_ClickHouse_LDAP_External_User_Directory = Specification( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance, RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters, RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_InvalidPassword, - ), - content=''' + ), + content=""" # SRS-009 ClickHouse LDAP External User Directory # Software Requirements Specification @@ -2600,4 +3059,5 @@ to result in contacting the [LDAP] server to verify user's username and password [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md [Git]: https://git-scm.com/ [GitHub]: https://github.com -''') +""", +) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 830fe01501b..83daa175a24 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import random -from helpers.common import Pool from testflows.core import * from testflows.asserts import error @@ -14,7 +13,7 @@ servers = { "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { "host": "openldap2", @@ -23,13 +22,15 @@ servers = { "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", "tls_require_cert": "never", - } + }, } + @TestOutline -def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None): - """Add user to LDAP and ClickHouse and then try to login. - """ +def add_user_to_ldap_and_login( + self, server, user=None, ch_user=None, login=None, exitcode=None, message=None +): + """Add user to LDAP and ClickHouse and then try to login.""" self.context.ldap_node = self.context.cluster.node(server) if ch_user is None: @@ -43,78 +44,126 @@ def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None username = login.get("username", user["cn"]) password = login.get("password", user["userpassword"]) - login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) + login_and_execute_query( + username=username, password=password, exitcode=exitcode, message=message + ) + def login_with_valid_username_and_password(users, i, iterations=10): - """Login with valid username and password. - """ + """Login with valid username and password.""" with When(f"valid users try to login #{i}"): for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + random_user = users[random.randint(0, len(users) - 1)] + login_and_execute_query( + username=random_user["cn"], + password=random_user["userpassword"], + steps=False, + ) + def login_with_valid_username_and_invalid_password(users, i, iterations=10): - """Login with valid username and invalid password. - """ + """Login with valid username and invalid password.""" with When(f"users try to login with valid username and invalid password #{i}"): for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], + random_user = users[random.randint(0, len(users) - 1)] + login_and_execute_query( + username=random_user["cn"], password=(random_user["userpassword"] + randomword(1)), exitcode=4, message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + steps=False, + ) + def login_with_invalid_username_and_valid_password(users, i, iterations=10): - """Login with invalid username and valid password. - """ + """Login with invalid username and valid password.""" with When(f"users try to login with invalid username and valid password #{i}"): for i in range(iterations): - random_user = dict(users[random.randint(0, len(users)-1)]) + random_user = dict(users[random.randint(0, len(users) - 1)]) random_user["cn"] += randomword(1) - login_and_execute_query(username=random_user["cn"], + login_and_execute_query( + username=random_user["cn"], password=random_user["userpassword"], exitcode=4, message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + steps=False, + ) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid( + "1.0" + ), ) def parallel_login(self, server, user_count=10, timeout=300): - """Check that login of valid and invalid LDAP authenticated users works in parallel. - """ + """Check that login of valid and invalid LDAP authenticated users works in parallel.""" self.context.ldap_node = self.context.cluster.node(server) user = None with Given("a group of LDAP users"): - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + users = [ + {"cn": f"parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ] with ldap_users(*users): tasks = [] with Pool(4) as pool: try: - with When("users try to login in parallel", description=""" + with When( + "users try to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_invalid_username_and_valid_password, (users, i, 50,))) - + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_valid_username_and_invalid_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_invalid_username_and_valid_password, + ( + users, + i, + 50, + ), + ) + ) + finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) - + + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid( + "1.0" + ), ) def parallel_login_with_the_same_user(self, server, timeout=300): """Check that valid and invalid logins of the same @@ -130,20 +179,51 @@ def parallel_login_with_the_same_user(self, server, timeout=300): tasks = [] with Pool(4) as pool: try: - with When("the same user tries to login in parallel", description=""" + with When( + "the same user tries to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_invalid_username_and_valid_password, (users, i, 50,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_valid_username_and_invalid_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_invalid_username_and_valid_password, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Tags("custom config") def login_after_ldap_external_user_directory_is_removed(self, server): @@ -157,13 +237,18 @@ def login_after_ldap_external_user_directory_is_removed(self, server): with And("I attempt to login after LDAP external user directory is removed"): exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + login_and_execute_query( + username="user2", password="user2", exitcode=exitcode, message=message + ) + @TestScenario @Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid( + "1.0" + ), ) def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=300): """Check that valid and invalid logins of the same @@ -171,144 +256,239 @@ def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=300 works in parallel. """ with Given("I have two LDAP servers"): - entries = [ - (["openldap1"], []), - (["openldap2"], []) - ] + entries = [(["openldap1"], []), (["openldap2"], [])] with Given("I define only one LDAP user"): users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] - with And("I create config file to define LDAP external user directory for each LDAP server"): + with And( + "I create config file to define LDAP external user directory for each LDAP server" + ): config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): with ldap_users(*users, node=self.context.cluster.node("openldap1")): with ldap_users(*users, node=self.context.cluster.node("openldap2")): tasks = [] - with Pool(4) as pool: + with Pool(4) as pool: try: - with When("the same user tries to login in parallel", description=""" + with When( + "the same user tries to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_invalid_username_and_valid_password, (users, i, 50,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_valid_username_and_invalid_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_invalid_username_and_valid_password, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Tags("custom config") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers( + "1.0" + ), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid( + "1.0" + ), ) def parallel_login_with_multiple_servers(self, server, user_count=10, timeout=300): """Check that login of valid and invalid LDAP authenticated users works in parallel using multiple LDAP external user directories. """ with Given("I have two LDAP servers"): - entries = [ - (["openldap1"], []), - (["openldap2"], []) - ] + entries = [(["openldap1"], []), (["openldap2"], [])] with And("I define a group of users to be created on each LDAP server"): user_groups = { - "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], - "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + "openldap1_users": [ + {"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], + "openldap2_users": [ + {"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], } with And("I have a list of checks that I want to run for each user group"): checks = [ login_with_valid_username_and_password, login_with_valid_username_and_invalid_password, - login_with_invalid_username_and_valid_password + login_with_invalid_username_and_valid_password, ] - with And("I create config file to define LDAP external user directory for each LDAP server"): + with And( + "I create config file to define LDAP external user directory for each LDAP server" + ): config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): - with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): - with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): + with ldap_users( + *user_groups["openldap1_users"], node=self.context.cluster.node("openldap1") + ): + with ldap_users( + *user_groups["openldap2_users"], + node=self.context.cluster.node("openldap2"), + ): tasks = [] with Pool(4) as pool: try: - with When("users in each group try to login in parallel", description=""" + with When( + "users in each group try to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): for users in user_groups.values(): for check in checks: - tasks.append(pool.submit(check, (users, i, 50,))) + tasks.append( + pool.submit( + check, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Tags("custom config") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP( + "1.0" + ), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid( + "1.0" + ), ) -def parallel_login_with_rbac_and_multiple_servers(self, server, user_count=10, timeout=300): +def parallel_login_with_rbac_and_multiple_servers( + self, server, user_count=10, timeout=300 +): """Check that login of valid and invalid users works in parallel using local users defined using RBAC and LDAP users authenticated using multiple LDAP external user directories. """ with Given("I have two LDAP servers"): - entries = [ - (["openldap1"], []), - (["openldap2"], []) - ] + entries = [(["openldap1"], []), (["openldap2"], [])] with And("I define a group of users to be created on each LDAP server"): user_groups = { - "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], - "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], - "local_users": [{"cn": f"local_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + "openldap1_users": [ + {"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], + "openldap2_users": [ + {"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], + "local_users": [ + {"cn": f"local_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], } with And("I have a list of checks that I want to run for each user group"): checks = [ login_with_valid_username_and_password, login_with_valid_username_and_invalid_password, - login_with_invalid_username_and_valid_password + login_with_invalid_username_and_valid_password, ] - with And("I create config file to define LDAP external user directory for each LDAP server"): + with And( + "I create config file to define LDAP external user directory for each LDAP server" + ): config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): - with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): - with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): + with ldap_users( + *user_groups["openldap1_users"], node=self.context.cluster.node("openldap1") + ): + with ldap_users( + *user_groups["openldap2_users"], + node=self.context.cluster.node("openldap2"), + ): with rbac_users(*user_groups["local_users"]): tasks = [] with Pool(4) as pool: try: - with When("users in each group try to login in parallel", description=""" + with When( + "users in each group try to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): for users in user_groups.values(): for check in checks: - tasks.append(pool.submit(check, (users, i, 50,))) + tasks.append( + pool.submit( + check, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly("1.0") @@ -320,7 +500,10 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=300): self.context.ldap_node = self.context.cluster.node(server) user = None - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + users = [ + {"cn": f"parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ] with rbac_users(*users): tasks = [] @@ -328,34 +511,61 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=300): try: with When("I login in parallel"): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_invalid_username_and_valid_password, (users, i, 50,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_valid_username_and_invalid_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_invalid_username_and_valid_password, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers("1.0") ) def login_after_user_is_added_to_ldap(self, server): - """Check that user can login as soon as it is added to LDAP. - """ + """Check that user can login as soon as it is added to LDAP.""" user = {"cn": "myuser", "userpassword": "myuser"} with When(f"I add user to LDAP and try to login"): add_user_to_ldap_and_login(user=user, server=server) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers("1.0"), ) def login_after_user_is_deleted_from_ldap(self, server): - """Check that login fails after user is deleted from LDAP. - """ + """Check that login fails after user is deleted from LDAP.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -370,23 +580,25 @@ def login_after_user_is_deleted_from_ldap(self, server): delete_user_from_ldap(user) with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name", ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged("1.0"), ) def login_after_user_password_changed_in_ldap(self, server): - """Check that login fails after user password is changed in LDAP. - """ + """Check that login fails after user password is changed in LDAP.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -401,9 +613,11 @@ def login_after_user_password_changed_in_ldap(self, server): change_user_password_in_ldap(user, "newpassword") with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name", ) with And("when I try to login with the new password it should work"): @@ -414,14 +628,14 @@ def login_after_user_password_changed_in_ldap(self, server): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged("1.0"), ) def login_after_user_cn_changed_in_ldap(self, server): - """Check that login fails after user cn is changed in LDAP. - """ + """Check that login fails after user cn is changed in LDAP.""" self.context.ldap_node = self.context.cluster.node(server) user = None new_user = None @@ -437,23 +651,25 @@ def login_after_user_cn_changed_in_ldap(self, server): new_user = change_user_cn_in_ldap(user, "myuser2") with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name", ) finally: with Finally("I make sure LDAP user is deleted"): if new_user is not None: delete_user_from_ldap(new_user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart("1.0"), ) def login_after_ldap_server_is_restarted(self, server, timeout=60): - """Check that login succeeds after LDAP server is restarted. - """ + """Check that login succeeds after LDAP server is restarted.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -467,12 +683,16 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60): with When("I restart LDAP server"): self.context.ldap_node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): + with Then( + "I try to login until it works", description=f"timeout {timeout} sec" + ): started = time.time() while True: - r = self.context.node.query("SELECT 1", + r = self.context.node.query( + "SELECT 1", settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) + no_checks=True, + ) if r.exitcode == 0: break assert time.time() - started < timeout, error(r.output) @@ -481,14 +701,14 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart("1.0"), ) def login_after_clickhouse_server_is_restarted(self, server, timeout=60): - """Check that login succeeds after ClickHouse server is restarted. - """ + """Check that login succeeds after ClickHouse server is restarted.""" self.context.ldap_node = self.context.cluster.node(server) user = None @@ -502,12 +722,16 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60): with When("I restart ClickHouse server"): self.context.node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): + with Then( + "I try to login until it works", description=f"timeout {timeout} sec" + ): started = time.time() while True: - r = self.context.node.query("SELECT 1", + r = self.context.node.query( + "SELECT 1", settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) + no_checks=True, + ) if r.exitcode == 0: break assert time.time() - started < timeout, error(r.output) @@ -516,28 +740,30 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0"), ) def valid_username_with_valid_empty_password(self, server): - """Check that we can't login using valid username that has empty password. - """ + """Check that we can't login using valid username that has empty password.""" user = {"cn": "empty_password", "userpassword": ""} exitcode = 4 message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login( + user=user, exitcode=exitcode, message=message, server=server + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0"), ) def valid_username_and_invalid_empty_password(self, server): - """Check that we can't login using valid username but invalid empty password. - """ + """Check that we can't login using valid username but invalid empty password.""" username = "user_non_empty_password" user = {"cn": username, "userpassword": username} login = {"password": ""} @@ -545,25 +771,24 @@ def valid_username_and_invalid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login( + user=user, login=login, exitcode=exitcode, message=message, server=server + ) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0")) def valid_username_and_password(self, server): - """Check that we can login using valid username and password. - """ + """Check that we can login using valid username and password.""" username = "valid_username_and_password" user = {"cn": username, "userpassword": username} with When(f"I add user {username} to LDAP and try to login"): add_user_to_ldap_and_login(user=user, server=server) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")) def valid_username_and_password_invalid_server(self, server=None): """Check that we can't login using valid username and valid password but for a different server. @@ -573,124 +798,133 @@ def valid_username_and_password_invalid_server(self, server=None): exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + login_and_execute_query( + username="user2", password="user2", exitcode=exitcode, message=message + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long("1.0"), ) def valid_long_username_and_short_password(self, server): - """Check that we can login using valid very long username and short password. - """ + """Check that we can login using valid very long username and short password.""" username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} add_user_to_ldap_and_login(user=user, server=server) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")) def invalid_long_username_and_valid_short_password(self, server): - """Check that we can't login using slightly invalid long username but valid password. - """ + """Check that we can't login using slightly invalid long username but valid password.""" username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} login = {"username": f"{username}?"} exitcode = 4 - message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, login=login, exitcode=exitcode, message=message, server=server + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long("1.0"), ) def valid_short_username_and_long_password(self, server): - """Check that we can login using valid short username with very long password. - """ + """Check that we can login using valid short username with very long password.""" username = "long_password" - user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + user = { + "cn": username, + "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890", + } add_user_to_ldap_and_login(user=user, server=server) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")) def valid_short_username_and_invalid_long_password(self, server): - """Check that we can't login using valid short username and invalid long password. - """ + """Check that we can't login using valid short username and invalid long password.""" username = "long_password" - user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + user = { + "cn": username, + "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890", + } login = {"password": user["userpassword"] + "1"} exitcode = 4 - message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, login=login, exitcode=exitcode, message=message, server=server + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")) def valid_username_and_invalid_password(self, server): - """Check that we can't login using valid username and invalid password. - """ + """Check that we can't login using valid username and invalid password.""" username = "valid_username_and_invalid_password" user = {"cn": username, "userpassword": username} login = {"password": user["userpassword"] + "1"} exitcode = 4 - message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, login=login, exitcode=exitcode, message=message, server=server + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")) def invalid_username_and_valid_password(self, server): - """Check that we can't login using slightly invalid username but valid password. - """ + """Check that we can't login using slightly invalid username but valid password.""" username = "invalid_username_and_valid_password" user = {"cn": username, "userpassword": username} login = {"username": user["cn"] + "1"} exitcode = 4 - message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + message = f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login( + user=user, login=login, exitcode=exitcode, message=message, server=server + ) - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8("1.0"), ) def valid_utf8_username_and_ascii_password(self, server): - """Check that we can login using valid utf-8 username with ascii password. - """ + """Check that we can login using valid utf-8 username with ascii password.""" username = "utf8_username_Gãńdåłf_Thê_Gręât" user = {"cn": username, "userpassword": "utf8_username"} add_user_to_ldap_and_login(user=user, server=server) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8("1.0"), ) def valid_ascii_username_and_utf8_password(self, server): - """Check that we can login using valid ascii username with utf-8 password. - """ + """Check that we can login using valid ascii username with utf-8 password.""" username = "utf8_password" user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} add_user_to_ldap_and_login(user=user, server=server) + @TestScenario def empty_username_and_empty_password(self, server=None): """Check that we can login using empty username and empty password as @@ -698,10 +932,13 @@ def empty_username_and_empty_password(self, server=None): """ login_and_execute_query(username="", password="") + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Default("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Default( + "1.0" + ) ) def default_verification_cooldown_value(self, server, rbac=False): """Check that the default value (0) for the verification cooldown parameter @@ -713,10 +950,18 @@ def default_verification_cooldown_value(self, server, rbac=False): error_exitcode = 4 user = None - with Given("I have an LDAP configuration that uses the default verification_cooldown value (0)"): - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} + with Given( + "I have an LDAP configuration that uses the default verification_cooldown value (0)" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -727,26 +972,39 @@ def default_verification_cooldown_value(self, server, rbac=False): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login immediately with the old user password it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) + with Then( + "when I try to login immediately with the old user password it should fail" + ): + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], + exitcode=error_exitcode, + message=error_message, + ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown( + "1.0" + ) ) def valid_verification_cooldown_value_cn_change(self, server, rbac=False): """Check that we can perform requests without contacting the LDAP server @@ -759,15 +1017,19 @@ def valid_verification_cooldown_value_cn_change(self, server, rbac=False): user = None new_user = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 2 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -778,30 +1040,47 @@ def valid_verification_cooldown_value_cn_change(self, server, rbac=False): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user cn in LDAP"): new_user = change_user_cn_in_ldap(user, "testVCD2") - with Then("when I try to login again with the old user cn it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "when I try to login again with the old user cn it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) - with And("when I sleep for 2 seconds and try to log in, it should fail"): + with And( + "when I sleep for 2 seconds and try to log in, it should fail" + ): time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], + exitcode=error_exitcode, + message=error_message, + ) finally: with Finally("I make sure LDAP user is deleted"): if new_user is not None: delete_user_from_ldap(new_user, exitcode=None) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown( + "1.0" + ) ) def valid_verification_cooldown_value_password_change(self, server, rbac=False): """Check that we can perform requests without contacting the LDAP server @@ -813,15 +1092,19 @@ def valid_verification_cooldown_value_password_change(self, server, rbac=False): error_exitcode = 4 user = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 2 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "2", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -832,30 +1115,47 @@ def valid_verification_cooldown_value_password_change(self, server, rbac=False): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login again with the old password it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "when I try to login again with the old password it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) - with And("when I sleep for 2 seconds and try to log in, it should fail"): + with And( + "when I sleep for 2 seconds and try to log in, it should fail" + ): time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], + exitcode=error_exitcode, + message=error_message, + ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown( + "1.0" + ) ) def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False): """Check that we can perform requests without contacting the LDAP server @@ -867,15 +1167,19 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False) error_exitcode = 4 user = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 2 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "300" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 2 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "300", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -886,17 +1190,25 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False) with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) try: with And("then I stop the ldap server"): self.context.ldap_node.stop() - with Then("when I try to login again with the server offline it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "when I try to login again with the server offline it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) finally: with Finally("I start the ldap server back up"): @@ -907,22 +1219,26 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False) if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestOutline def repeat_requests(self, server, iterations, vcd_value, rbac=False): - """Run repeated requests from some user to the LDAP server. - """ + """Run repeated requests from some user to the LDAP server.""" user = None - with Given(f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": vcd_value - }} + with Given( + f"I have an LDAP configuration that sets verification_cooldown parameter to {vcd_value} sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": vcd_value, + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -933,10 +1249,14 @@ def repeat_requests(self, server, iterations, vcd_value, rbac=False): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When(f"I login and execute some query {iterations} times"): start_time = time.time() - r = self.context.node.command(f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done") + r = self.context.node.command( + f"time for i in {{1..{iterations}}}; do clickhouse client -q \"SELECT 1\" --user {user['cn']} --password {user['userpassword']} > /dev/null; done" + ) end_time = time.time() return end_time - start_time @@ -946,10 +1266,13 @@ def repeat_requests(self, server, iterations, vcd_value, rbac=False): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance( + "1.0" + ) ) def verification_cooldown_performance(self, server, rbac=False, iterations=5000): """Check login performance when the verification cooldown @@ -960,48 +1283,66 @@ def verification_cooldown_performance(self, server, rbac=False, iterations=5000) vcd_time = 0 no_vcd_time = 0 - with Example(f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations"): - vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="600", rbac=rbac) + with Example( + f"Repeated requests with verification cooldown parameter set to 600 seconds, {iterations} iterations" + ): + vcd_time = repeat_requests( + server=server, iterations=iterations, vcd_value="600", rbac=rbac + ) metric("login_with_vcd_value_600", units="seconds", value=vcd_time) - with Example(f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations"): - no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) + with Example( + f"Repeated requests with verification cooldown parameter set to 0 seconds, {iterations} iterations" + ): + no_vcd_time = repeat_requests( + server=server, iterations=iterations, vcd_value="0", rbac=rbac + ) metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) with Then("Log the performance improvement as a percentage"): - metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) + metric( + "percentage_improvement", + units="%", + value=100 * (no_vcd_time - vcd_time) / vcd_time, + ) + @TestOutline -def check_verification_cooldown_reset_on_core_server_parameter_change(self, server, - parameter_name, parameter_value, rbac=False): +def check_verification_cooldown_reset_on_core_server_parameter_change( + self, server, parameter_name, parameter_value, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after one of the core server parameters is changed in the LDAP server configuration. """ - config_d_dir="/etc/clickhouse-server/config.d" - config_file="ldap_servers.xml" + config_d_dir = "/etc/clickhouse-server/config.d" + config_file = "ldap_servers.xml" error_message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" error_exitcode = 4 user = None - config=None - updated_config=None + config = None + updated_config = None - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 600 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) with And("LDAP authenticated user"): users = [ {"cn": f"testVCD_0", "userpassword": "testVCD_0"}, - {"cn": f"testVCD_1", "userpassword": "testVCD_1"} + {"cn": f"testVCD_1", "userpassword": "testVCD_1"}, ] with And("I create LDAP servers configuration file"): @@ -1010,88 +1351,137 @@ def check_verification_cooldown_reset_on_core_server_parameter_change(self, serv with ldap_users(*users) as users: with ldap_servers(servers=None, restart=False, config=config): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When("I login and execute a query"): for user in users: with By(f"as user {user['cn']}"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): for user in users: with By(f"for user {user['cn']}"): change_user_password_in_ldap(user, "newpassword") - with And(f"I change the server {parameter_name} core parameter", description=f"{parameter_value}"): + with And( + f"I change the server {parameter_name} core parameter", + description=f"{parameter_value}", + ): servers["openldap1"][parameter_name] = parameter_value - with And("I create an updated the config file that has a different server host name"): - updated_config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + with And( + "I create an updated the config file that has a different server host name" + ): + updated_config = create_ldap_servers_config_content( + servers, config_d_dir, config_file + ) with modify_config(updated_config, restart=False): - with Then("when I try to log in it should fail as cache should have been reset"): + with Then( + "when I try to log in it should fail as cache should have been reset" + ): for user in users: with By(f"as user {user['cn']}"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message.format(user=user["cn"])) + login_and_execute_query( + username=user["cn"], + password=user["userpassword"], + exitcode=error_exitcode, + message=error_message.format(user=user["cn"]), + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_host_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_host_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server host name is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="host", parameter_value="openldap2", rbac=rbac) + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, parameter_name="host", parameter_value="openldap2", rbac=rbac + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_port_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_port_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server port is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="port", parameter_value="9006", rbac=rbac) + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, parameter_name="port", parameter_value="9006", rbac=rbac + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_auth_dn_prefix_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server auth_dn_prefix is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, - parameter_name="auth_dn_prefix", parameter_value="cxx=", rbac=rbac) + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, + parameter_name="auth_dn_prefix", + parameter_value="cxx=", + rbac=rbac, + ) + @TestScenario @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_ChangeInCoreServerParameters( + "1.0" + ) ) -def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change(self, server, rbac=False): +def verification_cooldown_reset_on_server_auth_dn_suffix_parameter_change( + self, server, rbac=False +): """Check that the LDAP login cache is reset for all the LDAP authentication users when verification_cooldown parameter is set after server auth_dn_suffix is changed in the LDAP server configuration. """ - check_verification_cooldown_reset_on_core_server_parameter_change(server=server, + check_verification_cooldown_reset_on_core_server_parameter_change( + server=server, parameter_name="auth_dn_suffix", - parameter_value=",ou=company,dc=users,dc=com", rbac=rbac) + parameter_value=",ou=company,dc=users,dc=com", + rbac=rbac, + ) + @TestScenario @Name("verification cooldown reset when invalid password is provided") @Tags("verification_cooldown") @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_InvalidPassword("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Reset_InvalidPassword( + "1.0" + ) ) def scenario(self, server, rbac=False): """Check that cached bind requests for the user are discarded when @@ -1101,15 +1491,19 @@ def scenario(self, server, rbac=False): error_exitcode = 4 error_message = "DB::Exception: testVCD: Authentication failed: password is incorrect or there is no user with such name" - with Given("I have an LDAP configuration that sets verification_cooldown parameter to 600 sec"): - servers = { "openldap1": { - "host": "openldap1", - "port": "389", - "enable_tls": "no", - "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "600" - }} + with Given( + "I have an LDAP configuration that sets verification_cooldown parameter to 600 sec" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": "600", + } + } self.context.ldap_node = self.context.cluster.node(server) @@ -1120,34 +1514,53 @@ def scenario(self, server, rbac=False): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_external_user_directory( + server=server, roles=roles, restart=True + ): with When("I login and execute a query"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) with And("I change user password in LDAP"): change_user_password_in_ldap(user, "newpassword") - with Then("When I try to log in with the cached password it should work"): - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + with Then( + "When I try to log in with the cached password it should work" + ): + login_and_execute_query( + username=user["cn"], password=user["userpassword"] + ) - with And("When I try to log in with an incorrect password it should fail"): - login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, - message=error_message) + with And( + "When I try to log in with an incorrect password it should fail" + ): + login_and_execute_query( + username=user["cn"], + password="incorrect", + exitcode=error_exitcode, + message=error_message, + ) - with And("When I try to log in with the cached password it should fail"): - login_and_execute_query(username=user["cn"], password="incorrect", exitcode=error_exitcode, - message=error_message) + with And( + "When I try to log in with the cached password it should fail" + ): + login_and_execute_query( + username=user["cn"], + password="incorrect", + exitcode=error_exitcode, + message=error_message, + ) finally: with Finally("I make sure LDAP user is deleted"): if user is not None: delete_user_from_ldap(user, exitcode=None) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("2.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("2.0")) def user_lookup_priority(self, server): """Check that users are looked up in the same priority as they are defined in the `` section @@ -1160,35 +1573,56 @@ def user_lookup_priority(self, server): """ self.context.ldap_node = self.context.cluster.node(server) - message="DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + message = "DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" exitcode = 4 users = { "default": {"username": "default", "password": "userdefault"}, "local": {"username": "local", "password": "userlocal"}, - "ldap": {"username": "ldap", "password": "userldap"} + "ldap": {"username": "ldap", "password": "userldap"}, } - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users.values() + ] + ): with rbac_users({"cn": "local", "userpassword": "local"}): - with When("I try to login as 'default' user which is also defined in users.xml it should fail"): - login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) + with When( + "I try to login as 'default' user which is also defined in users.xml it should fail" + ): + login_and_execute_query( + **users["default"], + exitcode=exitcode, + message=message.format(username="default"), + ) - with When("I try to login as 'local' user which is also defined in local storage it should fail"): - login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) + with When( + "I try to login as 'local' user which is also defined in local storage it should fail" + ): + login_and_execute_query( + **users["local"], + exitcode=exitcode, + message=message.format(username="local"), + ) - with When("I try to login as 'ldap' user defined only in LDAP it should work"): + with When( + "I try to login as 'ldap' user defined only in LDAP it should work" + ): login_and_execute_query(**users["ldap"]) + @TestOutline(Feature) @Name("user authentications") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword( + "1.0" + ), ) def feature(self, servers=None, server=None, node="clickhouse1"): - """Check that users can be authenticated using an LDAP external user directory. - """ + """Check that users can be authenticated using an LDAP external user directory.""" self.context.node = self.context.cluster.node(node) if servers is None: @@ -1200,11 +1634,20 @@ def feature(self, servers=None, server=None, node="clickhouse1"): with ldap_servers(servers): with rbac_roles("ldap_role") as roles: with ldap_external_user_directory(server=server, roles=roles, restart=True): - for scenario in loads(current_module(), Scenario, filter=~has.tag("custom config") & ~has.tag("verification_cooldown")): + for scenario in loads( + current_module(), + Scenario, + filter=~has.tag("custom config") + & ~has.tag("verification_cooldown"), + ): Scenario(test=scenario)(server=server) - for scenario in loads(current_module(), Scenario, filter=has.tag("custom config")): + for scenario in loads( + current_module(), Scenario, filter=has.tag("custom config") + ): Scenario(test=scenario)(server=server) - for scenario in loads(current_module(), Scenario, filter=has.tag("verification_cooldown")): + for scenario in loads( + current_module(), Scenario, filter=has.tag("verification_cooldown") + ): Scenario(test=scenario)(server=server) diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py index 6256fcd1f39..871be815a35 100644 --- a/tests/testflows/ldap/external_user_directory/tests/common.py +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -5,13 +5,34 @@ from contextlib import contextmanager import testflows.settings as settings from testflows.core import * from testflows.asserts import error -from ldap.authentication.tests.common import getuid, Config, ldap_servers, add_config, modify_config, restart -from ldap.authentication.tests.common import xmltree, xml_indent, xml_append, xml_with_utf8 -from ldap.authentication.tests.common import ldap_user, ldap_users, add_user_to_ldap, delete_user_from_ldap -from ldap.authentication.tests.common import change_user_password_in_ldap, change_user_cn_in_ldap +from ldap.authentication.tests.common import ( + getuid, + Config, + ldap_servers, + add_config, + modify_config, + restart, +) +from ldap.authentication.tests.common import ( + xmltree, + xml_indent, + xml_append, + xml_with_utf8, +) +from ldap.authentication.tests.common import ( + ldap_user, + ldap_users, + add_user_to_ldap, + delete_user_from_ldap, +) +from ldap.authentication.tests.common import ( + change_user_password_in_ldap, + change_user_cn_in_ldap, +) from ldap.authentication.tests.common import create_ldap_servers_config_content from ldap.authentication.tests.common import randomword + @contextmanager def table(name, create_statement, on_cluster=False): node = current().context.node @@ -26,14 +47,18 @@ def table(name, create_statement, on_cluster=False): else: node.query(f"DROP TABLE IF EXISTS {name}") + @contextmanager -def rbac_users(*users): - node = current().context.node +def rbac_users(*users, node=None): + if node is None: + node = current().context.node try: with Given("I have local users"): for user in users: with By(f"creating user {user['cn']}", format_name=False): - node.query(f"CREATE USER OR REPLACE {user['cn']} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user['userpassword']}'") + node.query( + f"CREATE USER OR REPLACE {user['cn']} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user['userpassword']}'" + ) yield users finally: with Finally("I drop local users"): @@ -41,9 +66,11 @@ def rbac_users(*users): with By(f"dropping user {user['cn']}", flags=TE, format_name=False): node.query(f"DROP USER IF EXISTS {user['cn']}") + @contextmanager -def rbac_roles(*roles): - node = current().context.node +def rbac_roles(*roles, node=None): + if node is None: + node = current().context.node try: with Given("I have roles"): for role in roles: @@ -56,22 +83,31 @@ def rbac_roles(*roles): with By(f"dropping role {role}", flags=TE): node.query(f"DROP ROLE IF EXISTS {role}") + def verify_ldap_user_exists(server, username, password): - """Check that LDAP user is defined on the LDAP server. - """ + """Check that LDAP user is defined on the LDAP server.""" with By("searching LDAP database"): ldap_node = current().context.cluster.node(server) r = ldap_node.command( - f"ldapwhoami -H ldap://localhost -D 'cn={user_name},ou=users,dc=company,dc=com' -w {password}") + f"ldapwhoami -H ldap://localhost -D 'cn={user_name},ou=users,dc=company,dc=com' -w {password}" + ) assert r.exitcode == 0, error() -def create_ldap_external_user_directory_config_content(server=None, roles=None, **kwargs): - """Create LDAP external user directory configuration file content. - """ - return create_entries_ldap_external_user_directory_config_content(entries=[([server], [roles])], **kwargs) -def create_entries_ldap_external_user_directory_config_content(entries, config_d_dir="/etc/clickhouse-server/config.d", - config_file="ldap_external_user_directories.xml"): +def create_ldap_external_user_directory_config_content( + server=None, roles=None, **kwargs +): + """Create LDAP external user directory configuration file content.""" + return create_entries_ldap_external_user_directory_config_content( + entries=[([server], [roles])], **kwargs + ) + + +def create_entries_ldap_external_user_directory_config_content( + entries, + config_d_dir="/etc/clickhouse-server/config.d", + config_file="ldap_external_user_directories.xml", +): """Create configurattion file content that contains one or more entries for the LDAP external user directory. @@ -93,9 +129,13 @@ def create_entries_ldap_external_user_directory_config_content(entries, config_d path = os.path.join(config_d_dir, config_file) name = config_file - root = xmltree.fromstring("") + root = xmltree.fromstring( + "" + ) xml_user_directories = root.find("user_directories") - xml_user_directories.append(xmltree.Comment(text=f"LDAP external user directories {uid}")) + xml_user_directories.append( + xmltree.Comment(text=f"LDAP external user directories {uid}") + ) for entry in entries: servers, roles_entries = entry @@ -114,11 +154,16 @@ def create_entries_ldap_external_user_directory_config_content(entries, config_d xml_user_directories.append(xml_directory) xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + content = xml_with_utf8 + str( + xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8" + ) return Config(content, path, name, uid, "config.xml") -def invalid_ldap_external_user_directory_config(server, roles, message, tail=30, timeout=60, config=None): + +def invalid_ldap_external_user_directory_config( + server, roles, message, tail=30, timeout=60, config=None +): """Check that ClickHouse errors when trying to load invalid LDAP external user directory configuration file. """ @@ -126,17 +171,25 @@ def invalid_ldap_external_user_directory_config(server, roles, message, tail=30, node = current().context.node if config is None: - config = create_ldap_external_user_directory_config_content(server=server, roles=roles) + config = create_ldap_external_user_directory_config_content( + server=server, roles=roles + ) try: with Given("I prepare the error log by writting empty lines into it"): - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + node.command( + 'echo -e "%s" > /var/log/clickhouse-server/clickhouse-server.err.log' + % ("-\\n" * tail) + ) with When("I add the config", description=config.path): command = f"cat < {config.path}\n{config.content}\nHEREDOC" node.command(command, steps=False, exitcode=0) - with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + with Then( + f"{config.preprocessed_name} should be updated", + description=f"timeout {timeout}", + ): started = time.time() command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" while time.time() - started < timeout: @@ -152,15 +205,24 @@ def invalid_ldap_external_user_directory_config(server, roles, message, tail=30, finally: with Finally(f"I remove {config.name}"): with By("removing invalid configuration file"): - system_config_path = os.path.join(current_dir(), "..", "configs", node.name, "config.d", config.path.split("config.d/")[-1]) - cluster.command(None, f'rm -rf {system_config_path}', timeout=timeout, exitcode=0) + system_config_path = os.path.join( + current_dir(), + "..", + "configs", + node.name, + "config.d", + config.path.split("config.d/")[-1], + ) + cluster.command( + None, f"rm -rf {system_config_path}", timeout=timeout, exitcode=0 + ) with And("restarting the node"): node.restart(safe=False) with Then("error log should contain the expected error message"): started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + command = f'tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep "{message}"' while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode if exitcode == 0: @@ -168,34 +230,67 @@ def invalid_ldap_external_user_directory_config(server, roles, message, tail=30, time.sleep(1) assert exitcode == 0, error() + @contextmanager -def ldap_external_user_directory(server, roles, config_d_dir="/etc/clickhouse-server/config.d", - config_file=None, timeout=60, restart=True, config=None): - """Add LDAP external user directory. - """ +def ldap_external_user_directory( + server, + roles, + config_d_dir="/etc/clickhouse-server/config.d", + config_file=None, + timeout=60, + restart=True, + config=None, +): + """Add LDAP external user directory.""" if config_file is None: config_file = f"ldap_external_user_directory_{getuid()}.xml" if config is None: - config = create_ldap_external_user_directory_config_content(server=server, roles=roles, config_d_dir=config_d_dir, config_file=config_file) + config = create_ldap_external_user_directory_config_content( + server=server, + roles=roles, + config_d_dir=config_d_dir, + config_file=config_file, + ) return add_config(config, restart=restart) + def login(servers, directory_server, *users, config=None): """Configure LDAP server and LDAP external user directory and try to login and execute a query""" with ldap_servers(servers): with rbac_roles(f"role_{getuid()}") as roles: - with ldap_external_user_directory(server=servers[directory_server]["host"], roles=roles, restart=True, config=config): + with ldap_external_user_directory( + server=servers[directory_server]["host"], + roles=roles, + restart=True, + config=config, + ): for user in users: if user.get("login", False): with When(f"I login as {user['username']} and execute query"): - current().context.node.query("SELECT 1", - settings=[("user", user["username"]), ("password", user["password"])], + current().context.node.query( + "SELECT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], exitcode=user.get("exitcode", None), - message=user.get("message", None)) + message=user.get("message", None), + ) + @TestStep(When) @Name("I login as {username} and execute query") -def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True, timeout=60, poll=False): +def login_and_execute_query( + self, + username, + password, + exitcode=None, + message=None, + steps=True, + timeout=60, + poll=False, +): if poll: start_time = time.time() attempt = 0 @@ -203,10 +298,17 @@ def login_and_execute_query(self, username, password, exitcode=None, message=Non with By("repeatedly trying to login until successful or timeout"): while True: with When(f"attempt #{attempt}"): - r = self.context.node.query("SELECT 1", settings=[("user", username), ("password", password)], - no_checks=True, steps=False, timeout=timeout) + r = self.context.node.query( + "SELECT 1", + settings=[("user", username), ("password", password)], + no_checks=True, + steps=False, + timeout=timeout, + ) - if r.exitcode == (0 if exitcode is None else exitcode) and (message in r.output if message is not None else True): + if r.exitcode == (0 if exitcode is None else exitcode) and ( + message in r.output if message is not None else True + ): break if time.time() - start_time > timeout: @@ -214,7 +316,11 @@ def login_and_execute_query(self, username, password, exitcode=None, message=Non attempt += 1 else: - self.context.node.query("SELECT 1", + self.context.node.query( + "SELECT 1", settings=[("user", username), ("password", password)], exitcode=(0 if exitcode is None else exitcode), - message=message, steps=steps, timeout=timeout) + message=message, + steps=steps, + timeout=timeout, + ) diff --git a/tests/testflows/ldap/external_user_directory/tests/connections.py b/tests/testflows/ldap/external_user_directory/tests/connections.py index ba734bb6c71..d2c3c15c3d9 100644 --- a/tests/testflows/ldap/external_user_directory/tests/connections.py +++ b/tests/testflows/ldap/external_user_directory/tests/connections.py @@ -4,22 +4,24 @@ from testflows.asserts import error from ldap.external_user_directory.tests.common import login from ldap.external_user_directory.requirements import * + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No( + "1.0" + ), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0"), ) def plain_text(self): - """Check that we can perform LDAP user authentication using `plain text` connection protocol. - """ + """Check that we can perform LDAP user authentication using `plain text` connection protocol.""" servers = { "openldap1": { "host": "openldap1", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -27,10 +29,11 @@ def plain_text(self): ] login(servers, "openldap1", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0"), ) def plain_text_with_custom_port(self): """Check that we can perform LDAP user authentication using `plain text` connection protocol @@ -42,7 +45,7 @@ def plain_text_with_custom_port(self): "port": "3089", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -50,10 +53,11 @@ def plain_text_with_custom_port(self): ] login(servers, "openldap3", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0"), ) def tls_with_custom_port(self): """Check that we can perform LDAP user authentication using `TLS` connection protocol @@ -65,7 +69,7 @@ def tls_with_custom_port(self): "port": "6036", "tls_require_cert": "never", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -73,10 +77,11 @@ def tls_with_custom_port(self): ] login(servers, "openldap4", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0"), ) def starttls_with_custom_port(self): """Check that we can perform LDAP user authentication using `StartTLS` connection protocol @@ -89,7 +94,7 @@ def starttls_with_custom_port(self): "enable_tls": "starttls", "tls_require_cert": "never", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -97,6 +102,7 @@ def starttls_with_custom_port(self): ] login(servers, "openldap4", *users) + def tls_connection(enable_tls, tls_require_cert): """Try to login using LDAP user authentication over a TLS connection.""" servers = { @@ -105,7 +111,7 @@ def tls_connection(enable_tls, tls_require_cert): "enable_tls": enable_tls, "tls_require_cert": tls_require_cert, "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -115,41 +121,64 @@ def tls_connection(enable_tls, tls_require_cert): requirements = [] if tls_require_cert == "never": - requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never("1.0")] + requirements = [ + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never( + "1.0" + ) + ] elif tls_require_cert == "allow": - requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow("1.0")] + requirements = [ + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow( + "1.0" + ) + ] elif tls_require_cert == "try": - requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try("1.0")] + requirements = [ + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try( + "1.0" + ) + ] elif tls_require_cert == "demand": - requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand("1.0")] + requirements = [ + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand( + "1.0" + ) + ] - with Example(name=f"tls_require_cert='{tls_require_cert}'", requirements=requirements): + with Example( + name=f"tls_require_cert='{tls_require_cert}'", requirements=requirements + ): login(servers, "openldap2", *users) + @TestScenario -@Examples("enable_tls tls_require_cert", [ - ("yes", "never"), - ("yes", "allow"), - ("yes", "try"), - ("yes", "demand") -]) +@Examples( + "enable_tls tls_require_cert", + [("yes", "never"), ("yes", "allow"), ("yes", "try"), ("yes", "demand")], +) @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes( + "1.0" + ), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default( + "1.0" + ), ) def tls(self): - """Check that we can perform LDAP user authentication using `TLS` connection protocol. - """ + """Check that we can perform LDAP user authentication using `TLS` connection protocol.""" for example in self.examples: tls_connection(*example) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default( + "1.0" + ) ) def tls_enable_tls_default_yes(self): """Check that the default value for the `enable_tls` is set to `yes`.""" @@ -158,7 +187,7 @@ def tls_enable_tls_default_yes(self): "host": "openldap2", "tls_require_cert": "never", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -166,9 +195,12 @@ def tls_enable_tls_default_yes(self): ] login(servers, "openldap2", *users) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default( + "1.0" + ) ) def tls_require_cert_default_demand(self): """Check that the default value for the `tls_require_cert` is set to `demand`.""" @@ -178,7 +210,7 @@ def tls_require_cert_default_demand(self): "enable_tls": "yes", "port": "636", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -186,25 +218,31 @@ def tls_require_cert_default_demand(self): ] login(servers, "openldap2", *users) + @TestScenario -@Examples("enable_tls tls_require_cert", [ - ("starttls", "never"), - ("starttls", "allow"), - ("starttls", "try"), - ("starttls", "demand") -]) +@Examples( + "enable_tls tls_require_cert", + [ + ("starttls", "never"), + ("starttls", "allow"), + ("starttls", "try"), + ("starttls", "demand"), + ], +) @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS( + "1.0" + ), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0"), ) def starttls(self): - """Check that we can perform LDAP user authentication using legacy `StartTLS` connection protocol. - """ + """Check that we can perform LDAP user authentication using legacy `StartTLS` connection protocol.""" for example in self.examples: tls_connection(*example) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite("1.0") @@ -219,7 +257,7 @@ def tls_cipher_suite(self): "tls_cipher_suite": "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC", "tls_minimum_protocol_version": "tls1.2", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } users = [ @@ -227,18 +265,26 @@ def tls_cipher_suite(self): ] login(servers, "openldap4", *users) + @TestOutline(Scenario) @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion( + "1.0" + ), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values( + "1.0" + ), +) +@Examples( + "version exitcode message", + [ + ("ssl2", None, None), + ("ssl3", None, None), + ("tls1.0", None, None), + ("tls1.1", None, None), + ("tls1.2", None, None), + ], ) -@Examples("version exitcode message", [ - ("ssl2", None, None), - ("ssl3", None, None), - ("tls1.0", None, None), - ("tls1.1", None, None), - ("tls1.2", None, None) -]) def tls_minimum_protocol_version(self, version, exitcode, message): """Check that `tls_minimum_protocol_version` parameter can be used specify to specify the minimum protocol version of SSL/TLS.""" @@ -250,16 +296,23 @@ def tls_minimum_protocol_version(self, version, exitcode, message): "tls_require_cert": "never", "tls_minimum_protocol_version": version, "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } - users = [{ - "server": "openldap4", "username": "user4", "password": "user4", - "login": True, "exitcode": int(exitcode) if exitcode is not None else None, "message": message - }] + users = [ + { + "server": "openldap4", + "username": "user4", + "password": "user4", + "login": True, + "exitcode": int(exitcode) if exitcode is not None else None, + "message": message, + } + ] + + login(servers, "openldap4", *users) - login(servers,"openldap4", *users) @TestFeature @Name("connection protocols") diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py index 3f1b8076ffa..f1fd956825e 100644 --- a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -3,9 +3,12 @@ from testflows.core import * from ldap.external_user_directory.tests.common import * from ldap.external_user_directory.requirements import * + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("2.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne( + "2.0" + ) ) def more_than_one_user_directory(self): """Check when more than one LDAP user directory is @@ -14,81 +17,125 @@ def more_than_one_user_directory(self): message = "DB::Exception: Duplicate storage type 'ldap' at user_directories" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { - "host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "never" - } + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "never", + }, } users = [ - {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, - {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + }, + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + }, ] role = f"role_{getuid()}" - entries = [ - (["openldap1"], [(role,)]), - (["openldap2"], [(role,)]) - ] + entries = [(["openldap1"], [(role,)]), (["openldap2"], [(role,)])] with ldap_servers(servers): with rbac_roles(role) as roles: config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): - with When(f"I login as {users[0]['username']} authenticated using openldap1"): - current().context.node.query(f"SELECT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) - - with And(f"I login as {users[1]['username']} authenticated using openldap2"): - current().context.node.query(f"SELECT 1", - settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): + with When( + f"I login as {users[0]['username']} authenticated using openldap1" + ): + current().context.node.query( + f"SELECT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) + with And( + f"I login as {users[1]['username']} authenticated using openldap2" + ): + current().context.node.query( + f"SELECT 1", + settings=[ + ("user", users[1]["username"]), + ("password", users[1]["password"]), + ], + ) @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty( + "1.0" + ) ) def empty_server(self, timeout=300): - """Check that empty string in a `server` field is not allowed. - """ + """Check that empty string in a `server` field is not allowed.""" message = "DB::Exception: Empty 'server' field for LDAP user directory" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } with ldap_servers(servers): with rbac_roles(f"role_{getuid()}") as roles: - invalid_ldap_external_user_directory_config(server="", roles=roles, message=message, timeout=timeout) + invalid_ldap_external_user_directory_config( + server="", roles=roles, message=message, timeout=timeout + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing( + "1.0" + ) ) def missing_server(self, timeout=300): - """Check that missing `server` field is not allowed. - """ + """Check that missing `server` field is not allowed.""" message = "DB::Exception: Missing 'server' field for LDAP user directory" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } with ldap_servers(servers): with rbac_roles(f"role_{getuid()}") as roles: - invalid_ldap_external_user_directory_config(server=None, roles=roles, message=message, timeout=timeout) + invalid_ldap_external_user_directory_config( + server=None, roles=roles, message=message, timeout=timeout + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne( + "1.0" + ) ) def defined_twice_server(self): """Check that when `server` field is defined twice that only the first @@ -96,131 +143,210 @@ def defined_twice_server(self): """ servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + user = { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + } role = f"role_{getuid()}" - entries = [ - (["openldap1", "openldap2"], [(role,)]) - ] + entries = [(["openldap1", "openldap2"], [(role,)])] with ldap_servers(servers): with rbac_roles(role) as roles: config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): with When(f"I login as {user['username']} and execute query"): - current().context.node.query("SELECT 1", - settings=[("user", user["username"]), ("password", user["password"])]) + current().context.node.query( + "SELECT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid( + "1.0" + ) ) def invalid_server(self): - """Check when `server` field value is invalid. - """ + """Check when `server` field value is invalid.""" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + user = { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + } role = f"role_{getuid()}" - entries = [ - (["openldap2"], [(role,)]) - ] + entries = [(["openldap2"], [(role,)])] with ldap_servers(servers): with rbac_roles(role) as roles: config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): with When(f"I login as {user['username']} and execute query"): - current().context.node.query("SELECT 1", - settings=[("user", user["username"]), ("password", user["password"])], - exitcode=4, message="DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.") + current().context.node.query( + "SELECT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + exitcode=4, + message="DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.", + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty( + "1.0" + ) ) def empty_roles(self): - """Check when `roles` parameter is empty then user can't read any tables. - """ + """Check when `roles` parameter is empty then user can't read any tables.""" message = "DB::Exception: user1: Not enough privileges." exitcode = 241 servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } user = {"server": "openldap1", "username": "user1", "password": "user1"} - entries = [ - (["openldap1"], [[]]) - ] + entries = [(["openldap1"], [[]])] with ldap_servers(servers): - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): with When(f"I login as {user['username']} and execute query"): - current().context.node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", user["username"]), ("password", user["password"])], - exitcode=exitcode, message=message) + current().context.node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne( + "1.0" + ) ) def defined_twice_roles(self): - """Check that when `roles` is defined twice then only the first entry is used. - """ + """Check that when `roles` is defined twice then only the first entry is used.""" node = self.context.node - create_statement = "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()" + create_statement = ( + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()" + ) servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + user = { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + } roles = [f"role0_{getuid()}", f"role1_{getuid()}"] - entries = [ - (["openldap1"], [[roles[0]],[roles[1]]]) - ] + entries = [(["openldap1"], [[roles[0]], [roles[1]]])] with ldap_servers(servers): with rbac_roles(*roles): - with table(f"table0_{getuid()}", create_statement) as table0_name, \ - table(f"table1_{getuid()}", create_statement) as table1_name: + with table(f"table0_{getuid()}", create_statement) as table0_name, table( + f"table1_{getuid()}", create_statement + ) as table1_name: - with Given("I grant select privilege for the first table to the first role"): + with Given( + "I grant select privilege for the first table to the first role" + ): node.query(f"GRANT SELECT ON {table0_name} TO {roles[0]}") - with And("I grant select privilege for the second table to the second role"): + with And( + "I grant select privilege for the second table to the second role" + ): node.query(f"GRANT SELECT ON {table1_name} TO {roles[1]}") - config = create_entries_ldap_external_user_directory_config_content(entries) + config = create_entries_ldap_external_user_directory_config_content( + entries + ) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): - with When(f"I login as {user['username']} and try to read from the first table"): - current().context.node.query(f"SELECT * FROM {table0_name} LIMIT 1", - settings=[("user", user["username"]), ("password", user["password"])]) + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): + with When( + f"I login as {user['username']} and try to read from the first table" + ): + current().context.node.query( + f"SELECT * FROM {table0_name} LIMIT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + ) + + with And( + f"I login as {user['username']} again and try to read from the second table" + ): + current().context.node.query( + f"SELECT * FROM {table0_name} LIMIT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + ) - with And(f"I login as {user['username']} again and try to read from the second table"): - current().context.node.query(f"SELECT * FROM {table0_name} LIMIT 1", - settings=[("user", user["username"]), ("password", user["password"])]) @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid("2.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid( + "2.0" + ) ) def invalid_role_in_roles(self): """Check that no error is returned when LDAP users try to authenticate @@ -228,8 +354,11 @@ def invalid_role_in_roles(self): """ servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } user = {"server": "openldap1", "username": "user1", "password": "user1"} @@ -237,12 +366,20 @@ def invalid_role_in_roles(self): with ldap_servers(servers): with ldap_external_user_directory("openldap1", roles=["foo"], restart=True): with When(f"I login as {user['username']} and execute query"): - current().context.node.query("SELECT 1", - settings=[("user", user["username"]), ("password", user["password"])]) + current().context.node.query( + "SELECT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + ) + @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing( + "1.0" + ) ) def missing_roles(self): """Check that when the `roles` are missing then @@ -252,25 +389,38 @@ def missing_roles(self): exitcode = 241 servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } user = {"server": "openldap1", "username": "user1", "password": "user1"} - entries = [ - (["openldap1"], None) - ] + entries = [(["openldap1"], None)] with ldap_servers(servers): - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: config = create_entries_ldap_external_user_directory_config_content(entries) - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): with When(f"I login as {user['username']} and execute query"): - current().context.node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", user["username"]), ("password", user["password"])], - exitcode=exitcode, message=message) + current().context.node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", user["username"]), + ("password", user["password"]), + ], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("external user directory config") @@ -278,11 +428,10 @@ def missing_roles(self): RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Syntax("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition("1.0"), ) def feature(self, node="clickhouse1"): - """Check LDAP external user directory configuration. - """ + """Check LDAP external user directory configuration.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): diff --git a/tests/testflows/ldap/external_user_directory/tests/restart.py b/tests/testflows/ldap/external_user_directory/tests/restart.py index cfcf63d932f..96290d33e62 100644 --- a/tests/testflows/ldap/external_user_directory/tests/restart.py +++ b/tests/testflows/ldap/external_user_directory/tests/restart.py @@ -1,12 +1,12 @@ import random -from helpers.common import Pool from testflows.core import * from testflows.asserts import error from ldap.external_user_directory.tests.common import * from ldap.external_user_directory.requirements import * + @TestScenario def one_external_user_directory(self, node="clickhouse1"): """Check that we can restart ClickHouse server when one @@ -20,13 +20,15 @@ def one_external_user_directory(self, node="clickhouse1"): "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server="openldap1", roles=roles, restart=True): + with ldap_external_user_directory( + server="openldap1", roles=roles, restart=True + ): with Given("I login and execute query"): login_and_execute_query(username="user1", password="user1") @@ -36,6 +38,7 @@ def one_external_user_directory(self, node="clickhouse1"): with Then("I should be able to login and execute query after restart"): login_and_execute_query(username="user1", password="user1") + @TestScenario def multiple_external_user_directories(self, node="clickhouse1"): """Check that we can restart ClickHouse server when two @@ -49,7 +52,7 @@ def multiple_external_user_directories(self, node="clickhouse1"): "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { "host": "openldap2", @@ -58,35 +61,45 @@ def multiple_external_user_directories(self, node="clickhouse1"): "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", "tls_require_cert": "never", - } + }, } with Given("I have two LDAP servers"): - entries = [ - (["openldap1"], []), - (["openldap2"], []) - ] + entries = [(["openldap1"], []), (["openldap2"], [])] - with And("I create config file to define LDAP external user directory for each LDAP server"): + with And( + "I create config file to define LDAP external user directory for each LDAP server" + ): config = create_entries_ldap_external_user_directory_config_content(entries) with ldap_servers(servers): - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): - with Given("I login and execute query using a user defined in the first LDAP server"): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): + with Given( + "I login and execute query using a user defined in the first LDAP server" + ): login_and_execute_query(username="user1", password="user1") - with And("I login and execute query using a user defined the second LDAP server"): + with And( + "I login and execute query using a user defined the second LDAP server" + ): login_and_execute_query(username="user2", password="user2") with When("I restart the server"): restart() - with Then("I should be able to login and execute query again using a user defined in the first LDAP server"): + with Then( + "I should be able to login and execute query again using a user defined in the first LDAP server" + ): login_and_execute_query(username="user1", password="user1") - with And("I should be able to login and execute query again using a user defined in the second LDAP server"): + with And( + "I should be able to login and execute query again using a user defined in the second LDAP server" + ): login_and_execute_query(username="user2", password="user2") + @TestScenario def dynamically_added_users(self, node="clickhouse1", count=10): """Check that we can restart ClickHouse server when one @@ -101,20 +114,24 @@ def dynamically_added_users(self, node="clickhouse1", count=10): "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } with ldap_servers(servers): with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server="openldap1", roles=roles, restart=True): + with ldap_external_user_directory( + server="openldap1", roles=roles, restart=True + ): with Given("I login and execute query using existing LDAP user"): login_and_execute_query(username="user1", password="user1") with When("I then restart the server"): restart() - with Then("after restart I should be able to login and execute query using existing LDAP user"): + with Then( + "after restart I should be able to login and execute query using existing LDAP user" + ): login_and_execute_query(username="user1", password="user1") dynamic_users = [] @@ -124,11 +141,21 @@ def dynamically_added_users(self, node="clickhouse1", count=10): {"cn": f"dynamic_user{i}", "userpassword": randomword(20)} ) - with ldap_users(*dynamic_users, node=self.context.cluster.node("openldap1")): - with Then("I should be able to login and execute queries using dynamically added users"): + with ldap_users( + *dynamic_users, node=self.context.cluster.node("openldap1") + ): + with Then( + "I should be able to login and execute queries using dynamically added users" + ): for dynamic_user in dynamic_users: - with When(f"using dynamically added user {dynamic_user['cn']}"): - login_and_execute_query(username=dynamic_user["cn"], password=dynamic_user["userpassword"]) + with When( + f"using dynamically added user {dynamic_user['cn']}" + ): + login_and_execute_query( + username=dynamic_user["cn"], + password=dynamic_user["userpassword"], + ) + @TestScenario @Requirements( @@ -147,7 +174,7 @@ def parallel_login(self, server=None, user_count=10, timeout=300): "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, "openldap2": { "host": "openldap2", @@ -156,50 +183,67 @@ def parallel_login(self, server=None, user_count=10, timeout=300): "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", "tls_require_cert": "never", - } + }, } with Given("I have two LDAP servers"): - entries = [ - (["openldap1"], []), - (["openldap2"], []) - ] + entries = [(["openldap1"], []), (["openldap2"], [])] with And("I define a group of users to be created on each LDAP server"): user_groups = { - "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in - range(user_count)], - "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in - range(user_count)], - "local_users": [{"cn": f"local_parallel_user{i}", "userpassword": randomword(20)} for i in - range(user_count)] + "openldap1_users": [ + {"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], + "openldap2_users": [ + {"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], + "local_users": [ + {"cn": f"local_parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ], } @TestStep(When) @Name("I login as {username} and execute query") - def login_and_execute_query_during_restart(self, username, password, exitcode, message, steps=True, timeout=60): + def login_and_execute_query_during_restart( + self, username, password, exitcode, message, steps=True, timeout=60 + ): """Execute a query and ignore exitcode and message as during restart exit codes and messages vary based on the state of the restarted container and the ClickHouse server and there are too many cases and complete list is not fully known therefore trying to list all possible cases produces random fails. """ - r = self.context.cluster.command(None, f"{self.context.cluster.docker_compose} exec {self.context.node.name} " + - f"clickhouse client -q \"SELECT 1\" --user {username} --password {password}", steps=steps, timeout=timeout) + r = self.context.cluster.command( + None, + f"{self.context.cluster.docker_compose} exec {self.context.node.name} " + + f'clickhouse client -q "SELECT 1" --user {username} --password {password}', + steps=steps, + timeout=timeout, + ) return r @TestStep(When) @Name("I login as {username} and execute query") - def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True, timeout=60): - self.context.node.query("SELECT 1", + def login_and_execute_query( + self, username, password, exitcode=None, message=None, steps=True, timeout=60 + ): + self.context.node.query( + "SELECT 1", settings=[("user", username), ("password", password)], exitcode=exitcode or 0, - message=message, steps=steps, timeout=timeout) + message=message, + steps=steps, + timeout=timeout, + ) - def login_with_valid_username_and_password(users, i, iterations=10, during_restart=False): - """Login with valid username and password. - """ + def login_with_valid_username_and_password( + users, i, iterations=10, during_restart=False + ): + """Login with valid username and password.""" query = login_and_execute_query if during_restart: query = login_and_execute_query_during_restart @@ -208,12 +252,18 @@ def parallel_login(self, server=None, user_count=10, timeout=300): for i in range(iterations): random_user = users[random.randint(0, len(users) - 1)] - query(username=random_user["cn"], password=random_user["userpassword"], - exitcode=0, message="1", steps=False) + query( + username=random_user["cn"], + password=random_user["userpassword"], + exitcode=0, + message="1", + steps=False, + ) - def login_with_valid_username_and_invalid_password(users, i, iterations=10, during_restart=False): - """Login with valid username and invalid password. - """ + def login_with_valid_username_and_invalid_password( + users, i, iterations=10, during_restart=False + ): + """Login with valid username and invalid password.""" query = login_and_execute_query if during_restart: query = login_and_execute_query_during_restart @@ -222,15 +272,18 @@ def parallel_login(self, server=None, user_count=10, timeout=300): for i in range(iterations): random_user = users[random.randint(0, len(users) - 1)] - query(username=random_user["cn"], + query( + username=random_user["cn"], password=(random_user["userpassword"] + randomword(1)), exitcode=4, message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + steps=False, + ) - def login_with_invalid_username_and_valid_password(users, i, iterations=10, during_restart=False): - """Login with invalid username and valid password. - """ + def login_with_invalid_username_and_valid_password( + users, i, iterations=10, during_restart=False + ): + """Login with invalid username and valid password.""" query = login_and_execute_query if during_restart: query = login_and_execute_query_during_restart @@ -240,35 +293,51 @@ def parallel_login(self, server=None, user_count=10, timeout=300): random_user = dict(users[random.randint(0, len(users) - 1)]) random_user["cn"] += randomword(1) - query(username=random_user["cn"], + query( + username=random_user["cn"], password=random_user["userpassword"], exitcode=4, message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + steps=False, + ) with And("I have a list of checks that I want to run for each user group"): checks = [ login_with_valid_username_and_password, login_with_valid_username_and_invalid_password, - login_with_invalid_username_and_valid_password + login_with_invalid_username_and_valid_password, ] - with And("I create config file to define LDAP external user directory for each LDAP server"): + with And( + "I create config file to define LDAP external user directory for each LDAP server" + ): config = create_entries_ldap_external_user_directory_config_content(entries) with ldap_servers(servers): - with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): - with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): - with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + with ldap_external_user_directory( + server=None, roles=None, restart=True, config=config + ): + with ldap_users( + *user_groups["openldap1_users"], + node=self.context.cluster.node("openldap1"), + ): + with ldap_users( + *user_groups["openldap2_users"], + node=self.context.cluster.node("openldap2"), + ): with rbac_users(*user_groups["local_users"]): tasks = [] with Pool(4) as pool: try: - with When("I restart the server during parallel login of users in each group"): + with When( + "I restart the server during parallel login of users in each group" + ): for users in user_groups.values(): for check in checks: - tasks.append(pool.submit(check, (users, 0, 25, True))) - + tasks.append( + pool.submit(check, (users, 0, 25, True)) + ) + tasks.append(pool.submit(restart)) finally: with Then("logins during restart should work"): @@ -278,20 +347,25 @@ def parallel_login(self, server=None, user_count=10, timeout=300): tasks = [] with Pool(4) as pool: try: - with When("I perform parallel login of users in each group after restart"): + with When( + "I perform parallel login of users in each group after restart" + ): for users in user_groups.values(): for check in checks: - tasks.append(pool.submit(check, (users, 0, 10, False))) + tasks.append( + pool.submit( + check, (users, 0, 10, False) + ) + ) finally: with Then("logins after restart should work"): for task in tasks: task.result(timeout=timeout) + @TestOutline(Feature) @Name("restart") -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Restart_Server("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Restart_Server("1.0")) def feature(self, servers=None, server=None, node="clickhouse1"): """Check that we can restart ClickHouse server when one or more external user directories are configured. diff --git a/tests/testflows/ldap/external_user_directory/tests/roles.py b/tests/testflows/ldap/external_user_directory/tests/roles.py index 364ee219e48..266abd12eaa 100644 --- a/tests/testflows/ldap/external_user_directory/tests/roles.py +++ b/tests/testflows/ldap/external_user_directory/tests/roles.py @@ -3,10 +3,9 @@ from testflows.core import * from ldap.external_user_directory.tests.common import * from ldap.external_user_directory.requirements import * + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New("1.0")) def new_role(self, server): """Check that new roles can't be assigned to any LDAP user authenticated using external user directory. @@ -18,17 +17,32 @@ def new_role(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): - with When(f"I login and execute query simple query to cache the LDAP user"): - node.query(f"SELECT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with When( + f"I login and execute query simple query to cache the LDAP user" + ): + node.query( + f"SELECT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) with rbac_roles(f"new_role0_{uid}") as new_roles: @@ -36,20 +50,25 @@ def new_role(self, server): exitcode = 239 with And("I try to grant new role to the cached LDAP user"): - node.query(f"GRANT {new_roles[0]} TO {users[0]['username']}", - exitcode=exitcode, message=message.format(user=users[0]["username"])) + node.query( + f"GRANT {new_roles[0]} TO {users[0]['username']}", + exitcode=exitcode, + message=message.format(user=users[0]["username"]), + ) message = "DB::Exception: There is no role `{user}` in user directories" exitcode = 255 with And("I try to grant new role to the non-cached LDAP user"): - node.query(f"GRANT {new_roles[0]} TO {users[1]['username']}", - exitcode=exitcode, message=message.format(user=users[1]["username"])) + node.query( + f"GRANT {new_roles[0]} TO {users[1]['username']}", + exitcode=exitcode, + message=message.format(user=users[1]["username"]), + ) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege("1.0")) def add_privilege(self, server): """Check that we can add privilege to a role used in the external user directory configuration. @@ -63,38 +82,74 @@ def add_privilege(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): with When(f"I login and execute query that requires no privileges"): - node.query(f"SELECT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + node.query( + f"SELECT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) - with And(f"I login and try to read from the table without having select privilege"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])], - exitcode=exitcode, message=message.format(user=users[0]["username"])) + with And( + f"I login and try to read from the table without having select privilege" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + exitcode=exitcode, + message=message.format(user=users[0]["username"]), + ) - with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + with When( + f"I grant select privilege to one of the two roles assigned to LDAP users" + ): node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") - with And(f"I login again and expect that cached LDAP user can successfully read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with And( + f"I login again and expect that cached LDAP user can successfully read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) + + with And( + f"I login again and expect that non-cached LDAP user can successfully read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[1]["username"]), + ("password", users[1]["password"]), + ], + ) - with And(f"I login again and expect that non-cached LDAP user can successfully read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege("1.0")) def remove_privilege(self, server): """Check that we can remove privilege from a role used in the external user directory configuration. @@ -108,39 +163,73 @@ def remove_privilege(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: - with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + with When( + f"I grant select privilege to one of the two roles assigned to LDAP users" + ): node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): - with When(f"I login then LDAP user should be able to read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with When( + f"I login then LDAP user should be able to read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) - with When(f"I revoke select privilege from all the roles assigned to LDAP users"): + with When( + f"I revoke select privilege from all the roles assigned to LDAP users" + ): node.query(f"REVOKE SELECT ON {table_name} FROM {roles[0]}") - with When(f"I login again then cached LDAP user should not be able to read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])], - exitcode=exitcode, message=message.format(user=users[0]["username"])) + with When( + f"I login again then cached LDAP user should not be able to read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + exitcode=exitcode, + message=message.format(user=users[0]["username"]), + ) + + with When( + f"I login with non-cached LDAP user then the user should also not be able to read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[1]["username"]), + ("password", users[1]["password"]), + ], + exitcode=exitcode, + message=message.format(user=users[1]["username"]), + ) - with When(f"I login with non-cached LDAP user then the user should also not be able to read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[1]["username"]), ("password", users[1]["password"])], - exitcode=exitcode, message=message.format(user=users[1]["username"])) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed("2.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed("2.0")) def remove_role(self, server): """Check that when a role used in the external user directory configuration is dynamically removed then any LDAP users should still be authenticated using @@ -153,31 +242,50 @@ def remove_role(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): - with When(f"I login and execute query that requires no privileges"): - node.query(f"SELECT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): + with When(f"I login and execute query that requires no privileges"): + node.query( + f"SELECT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) - with And("I remove one of the roles"): - node.query(f"DROP ROLE {roles[1]}") + with And("I remove one of the roles"): + node.query(f"DROP ROLE {roles[1]}") - with And(f"I try to login using cached LDAP user"): - node.query(f"SELECT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with And(f"I try to login using cached LDAP user"): + node.query( + f"SELECT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) + + with And(f"I try to login again using non-cached LDAP user"): + node.query( + f"SELECT 1", + settings=[ + ("user", users[1]["username"]), + ("password", users[1]["password"]), + ], + ) - with And(f"I try to login again using non-cached LDAP user"): - node.query(f"SELECT 1", - settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges("1.0")) def remove_privilege_by_removing_role(self, server): """Check that when the role used in the external user directory configuration is dynamically removed then privileges are removed from all @@ -192,35 +300,59 @@ def remove_privilege_by_removing_role(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: - with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): - node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + with When( + f"I grant select privilege to one of the two roles assigned to LDAP users" + ): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): - with When(f"I login and expect that LDAP user can read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with When( + f"I login and expect that LDAP user can read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) - with And("I remove the role that grants the privilege"): - node.query(f"DROP ROLE {roles[0]}") + with And("I remove the role that grants the privilege"): + node.query(f"DROP ROLE {roles[0]}") + + with And( + f"I try to relogin and expect that cached LDAP user can login " + "but does not have privilege that was provided by the removed role" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + exitcode=exitcode, + message=message.format(user=users[0]["username"]), + ) - with And(f"I try to relogin and expect that cached LDAP user can login " - "but does not have privilege that was provided by the removed role"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])], - exitcode=exitcode, message=message.format(user=users[0]["username"])) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges("1.0")) def readd_privilege_by_readding_role(self, server): """Check that when the role used in the external user directory configuration is dynamically removed then all the privileges are removed from any @@ -234,58 +366,103 @@ def readd_privilege_by_readding_role(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: - with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): - node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + with When( + f"I grant select privilege to one of the two roles assigned to LDAP users" + ): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): - with When(f"I login and expect that LDAP user can read from the table"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with When( + f"I login and expect that LDAP user can read from the table" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) - with And("I remove the role that grants the privilege"): - node.query(f"DROP ROLE {roles[0]}") + with And("I remove the role that grants the privilege"): + node.query(f"DROP ROLE {roles[0]}") - message = "DB::Exception: {user}: Not enough privileges." - exitcode = 241 + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 - with And(f"I try to relogin and expect that cached LDAP user can login " - "but does not have privilege that was provided by the removed role"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])], - exitcode=exitcode, message=message.format(user=users[0]["username"])) + with And( + f"I try to relogin and expect that cached LDAP user can login " + "but does not have privilege that was provided by the removed role" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + exitcode=exitcode, + message=message.format(user=users[0]["username"]), + ) - with And(f"I try to login using non-cached LDAP user and expect it to succeed"): - node.query(f"SELECT 1", - settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + with And( + f"I try to login using non-cached LDAP user and expect it to succeed" + ): + node.query( + f"SELECT 1", + settings=[ + ("user", users[1]["username"]), + ("password", users[1]["password"]), + ], + ) - with When("I re-add the role"): - node.query(f"CREATE ROLE {roles[0]}") + with When("I re-add the role"): + node.query(f"CREATE ROLE {roles[0]}") - with And(f"I grant select privilege to the re-added role"): - node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + with And(f"I grant select privilege to the re-added role"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") - with And(f"I try to relogin and expect that cached LDAP user can login " - "and again has the privilege that is provided by the role"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + with And( + f"I try to relogin and expect that cached LDAP user can login " + "and again has the privilege that is provided by the role" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[0]["username"]), + ("password", users[0]["password"]), + ], + ) + + with And( + "I try to login using non-cached LDAP expect it to also work again and expect" + "for the user also to have privilege provided by the role" + ): + node.query( + f"SELECT * FROM {table_name} LIMIT 1", + settings=[ + ("user", users[1]["username"]), + ("password", users[1]["password"]), + ], + ) - with And("I try to login using non-cached LDAP expect it to also work again and expect" - "for the user also to have privilege provided by the role"): - node.query(f"SELECT * FROM {table_name} LIMIT 1", - settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NotPresent_Added("1.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NotPresent_Added("1.0")) def not_present_role_added(self, server): """Check that when the role used in the external user directory configuration which was not present during LDAP user authentication @@ -299,18 +476,29 @@ def not_present_role_added(self, server): users = [ {"username": f"user0_{uid}", "password": "user0_password"}, - {"username": f"user1_{uid}", "password": "user1_password"} + {"username": f"user1_{uid}", "password": "user1_password"}, ] roles = [f"role0_{uid}", f"role1_{uid}"] - with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with table( + f"table_{getuid()}", + "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) as table_name: with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with ldap_users( + *[ + {"cn": user["username"], "userpassword": user["password"]} + for user in users + ] + ): with When(f"I login using clickhouse-client"): with self.context.cluster.shell(node=node.name) as shell: - with shell(f"TERM=dumb clickhouse client --user {users[0]['username']} --password {users[0]['password']} | tee", - asynchronous=True, name="client") as client: + with shell( + f"TERM=dumb clickhouse client --user {users[0]['username']} --password {users[0]['password']} | tee", + asynchronous=True, + name="client", + ) as client: client.app.expect("clickhouse1 :\) ") with When("I execute select on the table"): @@ -321,12 +509,18 @@ def not_present_role_added(self, server): client.app.expect("clickhouse1 :\) ") try: - with Given("I add the role and grant the select privilege to it for the table"): + with Given( + "I add the role and grant the select privilege to it for the table" + ): node.query(f"CREATE ROLE {roles[0]}") - node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + node.query( + f"GRANT SELECT ON {table_name} TO {roles[0]}" + ) with When("I re-execute select on the table"): - client.app.send(f"SELECT * FROM {table_name} LIMIT 1") + client.app.send( + f"SELECT * FROM {table_name} LIMIT 1" + ) with Then("I expect to get no errors"): client.app.expect("Ok\.") @@ -336,6 +530,7 @@ def not_present_role_added(self, server): with Finally("I delete the role"): node.query(f"DROP ROLE IF EXISTS {roles[0]}") + @TestFeature @Name("roles") @Requirements( @@ -350,8 +545,11 @@ def feature(self, node="clickhouse1"): servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } user = {"server": "openldap1", "username": "user1", "password": "user1"} diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py index 31e1c42da94..a26713e28cf 100644 --- a/tests/testflows/ldap/external_user_directory/tests/server_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -7,39 +7,55 @@ from ldap.external_user_directory.requirements import * from ldap.authentication.tests.common import invalid_server_config -@TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name("1.0") -) -def empty_server_name(self, timeout=60): - """Check that empty string as a server name is not allowed. - """ - servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - invalid_server_config(servers, timeout=timeout) @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name("1.0"), +) +def empty_server_name(self, timeout=60): + """Check that empty string as a server name is not allowed.""" + servers = { + "": { + "host": "foo", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + invalid_server_config(servers, timeout=timeout) + + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer( + "1.0" + ), ) def invalid_host(self): """Check that server returns an error when LDAP server host name is invalid. """ servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." - }] + users = [ + { + "server": "foo", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.", + } + ] login(servers, "foo", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0"), ) def empty_host(self, tail=30, timeout=300): """Check that server returns an error when LDAP server @@ -52,10 +68,11 @@ def empty_host(self, tail=30, timeout=300): invalid_server_config(servers, message=message, tail=30, timeout=timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0"), ) def missing_host(self, tail=30, timeout=300): """Check that server returns an error when LDAP server @@ -65,18 +82,27 @@ def missing_host(self, tail=30, timeout=300): message = "DB::Exception: Missing 'host' entry" servers = {"foo": {"port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." - }] + users = [ + { + "server": "foo", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 36, + "message": "DB::Exception: LDAP server 'foo' is not configured.", + } + ] with Given("I prepare the error log by writting empty lines into it"): - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + node.command( + 'echo -e "%s" > /var/log/clickhouse-server/clickhouse-server.err.log' + % ("-\\n" * tail) + ) with ldap_servers(servers): with Then("server shall fail to merge the new config"): started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + command = f'tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep "{message}"' while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode if exitcode == 0: @@ -84,6 +110,7 @@ def missing_host(self, tail=30, timeout=300): time.sleep(1) assert exitcode == 0, error() + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), @@ -93,155 +120,247 @@ def invalid_port(self): port is not valid. """ servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." - }] + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.", + } + ] login(servers, "openldap1", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0"), ) def invalid_auth_dn_prefix(self): """Check that server returns an error when LDAP server definition has invalid auth_dn_prefix. """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "foo=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." - }] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "foo=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.", + } + ] login(servers, "openldap1", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0"), ) def invalid_auth_dn_suffix(self): """Check that server returns an error when LDAP server definition has invalid auth_dn_suffix. """ - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",foo=users,dc=company,dc=com" - }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." - }] + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",foo=users,dc=company,dc=com", + } + } + users = [ + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.", + } + ] login(servers, "openldap1", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), ) def invalid_enable_tls_value(self, timeout=60): """Check that server returns an error when enable_tls option has invalid value. """ message = "Syntax error: Cannot convert to boolean: foo" - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "foo", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } invalid_server_config(servers, message=message, tail=30, timeout=timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0"), ) def invalid_tls_require_cert_value(self): """Check that server returns an error when tls_require_cert option has invalid value. """ - servers = {"openldap2": { - "host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "foo", - "ca_cert_dir": "/container/service/slapd/assets/certs/", - "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name." - }] + servers = { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "foo", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + } + } + users = [ + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name.", + } + ] login(servers, "openldap2", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir("1.0"), ) def empty_ca_cert_dir(self): - """Check that server returns an error when ca_cert_dir is empty. - """ - servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "demand", - "ca_cert_dir": "", - "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - }] + """Check that server returns an error when ca_cert_dir is empty.""" + servers = { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + } + } + users = [ + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name", + } + ] login(servers, "openldap2", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile("1.0"), ) def empty_ca_cert_file(self): - """Check that server returns an error when ca_cert_file is empty. - """ - servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "tls_require_cert": "demand", - "ca_cert_dir": "/container/service/slapd/assets/certs/", - "ca_cert_file": "" - }} - users = [{ - "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 4, - "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name." - }] + """Check that server returns an error when ca_cert_file is empty.""" + servers = { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "", + } + } + users = [ + { + "server": "openldap2", + "username": "user2", + "password": "user2", + "login": True, + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name.", + } + ] login(servers, "openldap2", *users) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Value("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0"), - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0"), ) def auth_dn_value(self): """Check that server configuration can properly define the `dn` value of the user.""" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" - }} - user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + } + } + user = { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + } login(servers, "openldap1", user) + @TestOutline(Scenario) -@Examples("invalid_value", [ - ("-1", Name("negative int")), - ("foo", Name("string")), - ("", Name("empty string")), - ("36893488147419103232", Name("overflow with extremely large int value")), - ("-36893488147419103232", Name("overflow with extremely large negative int value")), - ("@#", Name("special characters")) -]) +@Examples( + "invalid_value", + [ + ("-1", Name("negative int")), + ("foo", Name("string")), + ("", Name("empty string")), + ("36893488147419103232", Name("overflow with extremely large int value")), + ( + "-36893488147419103232", + Name("overflow with extremely large negative int value"), + ), + ("@#", Name("special characters")), + ], +) @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Invalid("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_VerificationCooldown_Invalid( + "1.0" + ) ) def invalid_verification_cooldown_value(self, invalid_value, timeout=300): """Check that server returns an error when LDAP server @@ -250,19 +369,26 @@ def invalid_verification_cooldown_value(self, invalid_value, timeout=300): error_message = f" Syntax error: Not a valid unsigned integer{': ' + invalid_value if invalid_value else invalid_value}" - with Given("LDAP server configuration that uses a negative integer for the verification_cooldown parameter"): - servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", - "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": f"{invalid_value}" - }} + with Given( + "LDAP server configuration that uses a negative integer for the verification_cooldown parameter" + ): + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "verification_cooldown": f"{invalid_value}", + } + } with When("I try to use this configuration then it should not work"): invalid_server_config(servers, message=error_message, tail=30, timeout=timeout) + @TestScenario -@Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("2.0") -) +@Requirements(RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("2.0")) def syntax(self): """Check that server configuration with valid syntax can be loaded. ```xml @@ -293,23 +419,23 @@ def syntax(self): "auth_dn_suffix": ",ou=users,dc=company,dc=com", "verification_cooldown": "0", "enable_tls": "yes", - "tls_minimum_protocol_version": "tls1.2" , + "tls_minimum_protocol_version": "tls1.2", "tls_require_cert": "demand", "tls_cert_file": "/container/service/slapd/assets/certs/ldap.crt", "tls_key_file": "/container/service/slapd/assets/certs/ldap.key", "tls_ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", "tls_ca_cert_dir": "/container/service/slapd/assets/certs/", - "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384" + "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384", } } with ldap_servers(servers): pass + @TestFeature @Name("server config") def feature(self, node="clickhouse1"): - """Check LDAP server configuration. - """ + """Check LDAP server configuration.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): scenario() diff --git a/tests/testflows/ldap/external_user_directory/tests/simple.py b/tests/testflows/ldap/external_user_directory/tests/simple.py index c48048833c7..3c2ecc2cce6 100644 --- a/tests/testflows/ldap/external_user_directory/tests/simple.py +++ b/tests/testflows/ldap/external_user_directory/tests/simple.py @@ -3,11 +3,11 @@ from testflows.asserts import error from ldap.external_user_directory.tests.common import login + @TestScenario @Name("simple") def scenario(self, node="clickhouse1"): - """Check that an LDAP external user directory can be used to authenticate a user. - """ + """Check that an LDAP external user directory can be used to authenticate a user.""" self.context.node = self.context.cluster.node(node) servers = { "openldap1": { @@ -15,10 +15,15 @@ def scenario(self, node="clickhouse1"): "port": "389", "enable_tls": "no", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", }, } users = [ - {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + }, ] login(servers, "openldap1", *users) diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index c35f34c971d..5b3ea30ef73 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -4,27 +4,47 @@ from testflows.core import * append_path(sys.path, "..") -from helpers.common import Pool, join from helpers.argparser import argparser + @TestModule @Name("ldap") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path, parallel=None, stress=None): - """ClickHouse LDAP integration regression module. - """ - args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse LDAP integration regression module.""" + args = { + "local": local, + "clickhouse_binary_path": clickhouse_binary_path, + "clickhouse_version": clickhouse_version, + } + + self.context.clickhouse_version = clickhouse_version if stress is not None: self.context.stress = stress with Pool(3) as pool: try: - Feature(test=load("ldap.authentication.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("ldap.external_user_directory.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("ldap.role_mapping.regression", "regression"), parallel=True, executor=pool)(**args) + Feature( + test=load("ldap.authentication.regression", "regression"), + parallel=True, + executor=pool, + )(**args) + Feature( + test=load("ldap.external_user_directory.regression", "regression"), + parallel=True, + executor=pool, + )(**args) + Feature( + test=load("ldap.role_mapping.regression", "regression"), + parallel=True, + executor=pool, + )(**args) finally: join() + if main(): regression() diff --git a/tests/testflows/ldap/role_mapping/configs/clickhouse/common.xml b/tests/testflows/ldap/role_mapping/configs/clickhouse/common.xml deleted file mode 100644 index 31fa972199f..00000000000 --- a/tests/testflows/ldap/role_mapping/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - 0.0.0.0 - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/ldap/role_mapping/configs/clickhouse/config.d/ssl.xml b/tests/testflows/ldap/role_mapping/configs/clickhouse/config.d/ssl.xml index 77e03e9cf0f..c90b60f98b6 100644 --- a/tests/testflows/ldap/role_mapping/configs/clickhouse/config.d/ssl.xml +++ b/tests/testflows/ldap/role_mapping/configs/clickhouse/config.d/ssl.xml @@ -2,7 +2,8 @@ /etc/clickhouse-server/ssl/server.crt - /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem none true diff --git a/tests/testflows/ldap/role_mapping/configs/clickhouse/config.xml b/tests/testflows/ldap/role_mapping/configs/clickhouse/config.xml deleted file mode 100644 index 53ffa10384e..00000000000 --- a/tests/testflows/ldap/role_mapping/configs/clickhouse/config.xml +++ /dev/null @@ -1,442 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 7200 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/ldap/role_mapping/configs/clickhouse/users.d/common.xml b/tests/testflows/ldap/role_mapping/configs/clickhouse/users.d/common.xml new file mode 100644 index 00000000000..f6e0553ca33 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/configs/clickhouse/users.d/common.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/tests/testflows/ldap/role_mapping/configs/clickhouse/users.xml b/tests/testflows/ldap/role_mapping/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/ldap/role_mapping/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/ldap/role_mapping/regression.py b/tests/testflows/ldap/role_mapping/regression.py index a2c70d8bd41..fc2b85dba6f 100755 --- a/tests/testflows/ldap/role_mapping/regression.py +++ b/tests/testflows/ldap/role_mapping/regression.py @@ -8,45 +8,73 @@ append_path(sys.path, "..", "..") from helpers.cluster import Cluster from helpers.argparser import argparser from ldap.role_mapping.requirements import * +from helpers.common import check_clickhouse_version # Cross-outs of known fails xfails = { - "mapping/roles removed and added in parallel": - [(Fail, "known bug")], - "user dn detection/mapping/roles removed and added in parallel": - [(Fail, "known bug")] + "mapping/roles removed and added in parallel": [(Fail, "known bug")], + "user dn detection/mapping/roles removed and added in parallel": [ + (Fail, "known bug") + ], + "cluster secret/external user directory/:/:/cluster with secret/ldap user/:mapped True/select using mapped role/with privilege on source and distributed": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/34130") + ], } +# Force results without running the test +ffails = { + "cluster secret": ( + Skip, + "feature available on 20.10+", + check_clickhouse_version("<20.10"), + ) +} + + @TestFeature @Name("role mapping") @ArgumentParser(argparser) -@Specifications( - SRS_014_ClickHouse_LDAP_Role_Mapping -) -@Requirements( - RQ_SRS_014_LDAP_RoleMapping("1.0") -) +@Specifications(SRS_014_ClickHouse_LDAP_Role_Mapping) +@Requirements(RQ_SRS_014_LDAP_RoleMapping("1.0")) @XFails(xfails) -def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): - """ClickHouse LDAP role mapping regression module. - """ +@FFails(ffails) +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse LDAP role mapping regression module.""" nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), } + self.context.clickhouse_version = clickhouse_version + if stress is not None: self.context.stress = stress - if parallel is not None: - self.context.parallel = parallel - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "ldap_role_mapping_env")) as cluster: + from platform import processor as current_cpu + + folder_name = os.path.basename(current_dir()) + if current_cpu() == "aarch64": + env = f"{folder_name}_env_arm64" + else: + env = f"{folder_name}_env" + + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), env), + ) as cluster: self.context.cluster = cluster - Scenario(run=load("ldap.authentication.tests.sanity", "scenario"), name="ldap sanity") + Scenario( + run=load("ldap.authentication.tests.sanity", "scenario"), name="ldap sanity" + ) Feature(run=load("ldap.role_mapping.tests.server_config", "feature")) Feature(run=load("ldap.role_mapping.tests.mapping", "feature")) - #Feature(run=load("ldap.role_mapping.tests.user_dn_detection", "feature")) + Feature(run=load("ldap.role_mapping.tests.user_dn_detection", "feature")) + Feature(run=load("ldap.role_mapping.tests.cluster_secret", "feature")) + if main(): regression() diff --git a/tests/testflows/ldap/role_mapping/requirements/requirements.md b/tests/testflows/ldap/role_mapping/requirements/requirements.md index e857694f520..a9414749be3 100644 --- a/tests/testflows/ldap/role_mapping/requirements/requirements.md +++ b/tests/testflows/ldap/role_mapping/requirements/requirements.md @@ -76,6 +76,8 @@ * 4.8.8.3 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithUTF8Characters](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingprefixwithutf8characters) * 4.8.8.4 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialXMLCharacters](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingprefixwithspecialxmlcharacters) * 4.8.8.5 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialRegexCharacters](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingprefixwithspecialregexcharacters) + * 4.9 [Cluster With And Without Secret](#cluster-with-and-without-secret) + * 4.9.8.1 [RQ.SRS-014.LDAP.ClusterWithAndWithoutSecret.DistributedTable](#rqsrs-014ldapclusterwithandwithoutsecretdistributedtable) * 5 [References](#references) ## Revision History @@ -548,6 +550,67 @@ version: 1.0 [ClickHouse] SHALL support regex special characters as the value of the `` parameter in the `` section of the `config.xml`. +### Cluster With And Without Secret + +##### RQ.SRS-014.LDAP.ClusterWithAndWithoutSecret.DistributedTable +version: 1.0 + +[ClickHouse] SHALL support propagating query user roles and their corresponding privileges +when using `Distributed` table to the remote servers for the users that are authenticated +using LDAP either via external user directory or defined in `users.xml` when +cluster is configured with and without ``. + +For example, + +```xml + + + + qwerty123 + + true + + dwh + host1 + + + + true + + dwh + host2 + + + + + +``` + +or + +```xml + + + + + true + + dwh + host1 + + + + true + + dwh + host2 + + + + + +``` + ## References * **Access Control and Account Management**: https://clickhouse.com/docs/en/operations/access-rights/ diff --git a/tests/testflows/ldap/role_mapping/requirements/requirements.py b/tests/testflows/ldap/role_mapping/requirements/requirements.py index 5fb8c646483..e63e8593e99 100644 --- a/tests/testflows/ldap/role_mapping/requirements/requirements.py +++ b/tests/testflows/ldap/role_mapping/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.210505.1133630. +# document by TestFlows v1.7.220210.1155232. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -9,916 +9,1036 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_014_LDAP_RoleMapping = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping of [LDAP] groups to [RBAC] roles\n' - 'for users authenticated using [LDAP] external user directory.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping of [LDAP] groups to [RBAC] roles\n" + "for users authenticated using [LDAP] external user directory.\n" + "\n" + ), link=None, level=3, - num='4.1.1') + num="4.1.1", +) RQ_SRS_014_LDAP_RoleMapping_WithFixedRoles = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.WithFixedRoles', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.WithFixedRoles", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping of [LDAP] groups to [RBAC] roles\n' - 'for users authenticated using [LDAP] external user directory when\n' - 'one or more roles are specified in the `` section.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping of [LDAP] groups to [RBAC] roles\n" + "for users authenticated using [LDAP] external user directory when\n" + "one or more roles are specified in the `` section.\n" + "\n" + ), link=None, level=3, - num='4.1.2') + num="4.1.2", +) RQ_SRS_014_LDAP_RoleMapping_Search = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Search', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Search", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL perform search on the [LDAP] server and map the results to [RBAC] role names \n' - 'when authenticating users using the [LDAP] external user directory if the `` section is configured\n' - 'as part of the [LDAP] external user directory. The matched roles SHALL be assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL perform search on the [LDAP] server and map the results to [RBAC] role names \n" + "when authenticating users using the [LDAP] external user directory if the `` section is configured\n" + "as part of the [LDAP] external user directory. The matched roles SHALL be assigned to the user.\n" + "\n" + ), link=None, level=3, - num='4.1.3') + num="4.1.3", +) RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithUTF8Characters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithUTF8Characters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithUTF8Characters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n' - 'to an [RBAC] role that contains UTF-8 characters.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n" + "to an [RBAC] role that contains UTF-8 characters.\n" + "\n" + ), link=None, level=3, - num='4.2.1') + num="4.2.1", +) RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_Long = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.Long', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.Long", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n' - 'to an [RBAC] role that has a name with more than 128 characters.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n" + "to an [RBAC] role that has a name with more than 128 characters.\n" + "\n" + ), link=None, level=3, - num='4.2.2') + num="4.2.2", +) RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithSpecialXMLCharacters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialXMLCharacters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialXMLCharacters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n' - 'to an [RBAC] role that has a name that contains special characters that need to be escaped in XML.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n" + "to an [RBAC] role that has a name that contains special characters that need to be escaped in XML.\n" + "\n" + ), link=None, level=3, - num='4.2.3') + num="4.2.3", +) RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithSpecialRegexCharacters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialRegexCharacters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialRegexCharacters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n' - 'to an [RBAC] role that has a name that contains special characters that need to be escaped in regex.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping [LDAP] search results for users authenticated using [LDAP] external user directory\n" + "to an [RBAC] role that has a name that contains special characters that need to be escaped in regex.\n" + "\n" + ), link=None, level=3, - num='4.2.4') + num="4.2.4", +) RQ_SRS_014_LDAP_RoleMapping_Map_MultipleRoles = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Map.MultipleRoles', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Map.MultipleRoles", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support mapping one or more [LDAP] search results for users authenticated using \n' - '[LDAP] external user directory to one or more [RBAC] role.\n' - '\n' - ), + "[ClickHouse] SHALL support mapping one or more [LDAP] search results for users authenticated using \n" + "[LDAP] external user directory to one or more [RBAC] role.\n" + "\n" + ), link=None, level=3, - num='4.3.1') + num="4.3.1", +) RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_Removed = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.Removed', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.Removed", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not assign [RBAC] role(s) for any users authenticated using [LDAP] external user directory\n' - 'if the corresponding [LDAP] group(s) that map those role(s) are removed. Any users that have active sessions SHALL still\n' - 'have privileges provided by the role(s) until the next time they are authenticated.\n' - '\n' - ), + "[ClickHouse] SHALL not assign [RBAC] role(s) for any users authenticated using [LDAP] external user directory\n" + "if the corresponding [LDAP] group(s) that map those role(s) are removed. Any users that have active sessions SHALL still\n" + "have privileges provided by the role(s) until the next time they are authenticated.\n" + "\n" + ), link=None, level=3, - num='4.4.1') + num="4.4.1", +) RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_RemovedAndAdded_Parallel = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.RemovedAndAdded.Parallel', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.RemovedAndAdded.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users using [LDAP] external user directory \n' - 'when [LDAP] groups are removed and added \n' - 'at the same time as [LDAP] user authentications are performed in parallel.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users using [LDAP] external user directory \n" + "when [LDAP] groups are removed and added \n" + "at the same time as [LDAP] user authentications are performed in parallel.\n" + "\n" + ), link=None, level=3, - num='4.4.2') + num="4.4.2", +) RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_UserRemoved = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemoved', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemoved", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not assign [RBAC] role(s) for the user authenticated using [LDAP] external user directory\n' - 'if the user has been removed from the corresponding [LDAP] group(s) that map those role(s). \n' - 'Any active user sessions SHALL have privileges provided by the role(s) until the next time the user is authenticated.\n' - '\n' - ), + "[ClickHouse] SHALL not assign [RBAC] role(s) for the user authenticated using [LDAP] external user directory\n" + "if the user has been removed from the corresponding [LDAP] group(s) that map those role(s). \n" + "Any active user sessions SHALL have privileges provided by the role(s) until the next time the user is authenticated.\n" + "\n" + ), link=None, level=3, - num='4.4.3') + num="4.4.3", +) RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_UserRemovedAndAdded_Parallel = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemovedAndAdded.Parallel', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemovedAndAdded.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users using [LDAP] external user directory\n' - 'when [LDAP] users are added and removed from [LDAP] groups used to map to [RBAC] roles\n' - 'at the same time as [LDAP] user authentications are performed in parallel.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users using [LDAP] external user directory\n" + "when [LDAP] users are added and removed from [LDAP] groups used to map to [RBAC] roles\n" + "at the same time as [LDAP] user authentications are performed in parallel.\n" + "\n" + ), link=None, level=3, - num='4.4.4') + num="4.4.4", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NotPresent = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NotPresent', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NotPresent", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not reject authentication attempt using [LDAP] external user directory if any of the roles that are \n' - 'are mapped from [LDAP] but are not present locally.\n' - '\n' - ), + "[ClickHouse] SHALL not reject authentication attempt using [LDAP] external user directory if any of the roles that are \n" + "are mapped from [LDAP] but are not present locally.\n" + "\n" + ), link=None, level=3, - num='4.5.1') + num="4.5.1", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Added = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Added', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Added", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL add the privileges provided by the [LDAP] mapped role when the\n' - 'role is not present during user authentication using [LDAP] external user directory\n' - 'as soon as the role is added.\n' - '\n' - ), + "[ClickHouse] SHALL add the privileges provided by the [LDAP] mapped role when the\n" + "role is not present during user authentication using [LDAP] external user directory\n" + "as soon as the role is added.\n" + "\n" + ), link=None, level=3, - num='4.5.2') + num="4.5.2", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Removed = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Removed', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Removed", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL remove the privileges provided by the role from all the\n' - 'users authenticated using [LDAP] external user directory if the [RBAC] role that was mapped\n' - 'as a result of [LDAP] search is removed.\n' - '\n' - ), + "[ClickHouse] SHALL remove the privileges provided by the role from all the\n" + "users authenticated using [LDAP] external user directory if the [RBAC] role that was mapped\n" + "as a result of [LDAP] search is removed.\n" + "\n" + ), link=None, level=3, - num='4.5.3') + num="4.5.3", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Readded = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Readded', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Readded", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL reassign the [RBAC] role and add all the privileges provided by the role\n' - 'when it is re-added after removal for all [LDAP] users authenticated using external user directory\n' - 'for any role that was mapped as a result of [LDAP] search.\n' - '\n' - ), + "[ClickHouse] SHALL reassign the [RBAC] role and add all the privileges provided by the role\n" + "when it is re-added after removal for all [LDAP] users authenticated using external user directory\n" + "for any role that was mapped as a result of [LDAP] search.\n" + "\n" + ), link=None, level=3, - num='4.5.4') + num="4.5.4", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_RemovedAndAdded_Parallel = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedAndAdded.Parallel', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedAndAdded.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authenticating users using [LDAP] external user directory\n' - 'when [RBAC] roles that are mapped by [LDAP] groups\n' - 'are added and removed at the same time as [LDAP] user authentications are performed in parallel.\n' - '\n' - ), + "[ClickHouse] SHALL support authenticating users using [LDAP] external user directory\n" + "when [RBAC] roles that are mapped by [LDAP] groups\n" + "are added and removed at the same time as [LDAP] user authentications are performed in parallel.\n" + "\n" + ), link=None, level=3, - num='4.5.5') + num="4.5.5", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_New = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.New', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.New", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not allow any new roles to be assigned to any\n' - 'users authenticated using [LDAP] external user directory unless the role is specified\n' - 'in the configuration of the external user directory or was mapped as a result of [LDAP] search.\n' - '\n' - ), + "[ClickHouse] SHALL not allow any new roles to be assigned to any\n" + "users authenticated using [LDAP] external user directory unless the role is specified\n" + "in the configuration of the external user directory or was mapped as a result of [LDAP] search.\n" + "\n" + ), link=None, level=3, - num='4.5.6') + num="4.5.6", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NewPrivilege = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NewPrivilege', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NewPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL add new privilege to all the users authenticated using [LDAP] external user directory\n' - 'when new privilege is added to one of the roles that were mapped as a result of [LDAP] search.\n' - '\n' - ), + "[ClickHouse] SHALL add new privilege to all the users authenticated using [LDAP] external user directory\n" + "when new privilege is added to one of the roles that were mapped as a result of [LDAP] search.\n" + "\n" + ), link=None, level=3, - num='4.5.7') + num="4.5.7", +) RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_RemovedPrivilege = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedPrivilege', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL remove privilege from all the users authenticated using [LDAP] external user directory\n' - 'when the privilege that was provided by the mapped role is removed from all the roles \n' - 'that were mapped as a result of [LDAP] search.\n' - '\n' - ), + "[ClickHouse] SHALL remove privilege from all the users authenticated using [LDAP] external user directory\n" + "when the privilege that was provided by the mapped role is removed from all the roles \n" + "that were mapped as a result of [LDAP] search.\n" + "\n" + ), link=None, level=3, - num='4.5.8') + num="4.5.8", +) RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server\n' - 'when using [LDAP] external user directory that has role mapping enabled.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of users using [LDAP] server\n" + "when using [LDAP] external user directory that has role mapping enabled.\n" + "\n" + ), link=None, level=3, - num='4.6.1') + num="4.6.1", +) RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.ValidAndInvalid', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.ValidAndInvalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support authentication of valid users and\n' - 'prohibit authentication of invalid users using [LDAP] server\n' - 'in parallel without having invalid attempts affecting valid authentications\n' - 'when using [LDAP] external user directory that has role mapping enabled.\n' - '\n' - ), + "[ClickHouse] SHALL support authentication of valid users and\n" + "prohibit authentication of invalid users using [LDAP] server\n" + "in parallel without having invalid attempts affecting valid authentications\n" + "when using [LDAP] external user directory that has role mapping enabled.\n" + "\n" + ), link=None, level=3, - num='4.6.2') + num="4.6.2", +) RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_MultipleServers = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.MultipleServers', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.MultipleServers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of external [LDAP] users\n' - 'authenticated using multiple [LDAP] external user directories that have\n' - 'role mapping enabled.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of external [LDAP] users\n" + "authenticated using multiple [LDAP] external user directories that have\n" + "role mapping enabled.\n" + "\n" + ), link=None, level=3, - num='4.6.3') + num="4.6.3", +) RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_LocalOnly = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalOnly', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalOnly", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of users defined only locally\n' - 'when one or more [LDAP] external user directories with role mapping\n' - 'are specified in the configuration file.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of users defined only locally\n" + "when one or more [LDAP] external user directories with role mapping\n" + "are specified in the configuration file.\n" + "\n" + ), link=None, level=3, - num='4.6.4') + num="4.6.4", +) RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_LocalAndMultipleLDAP = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalAndMultipleLDAP', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalAndMultipleLDAP", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users\n' - 'authenticated using multiple [LDAP] external user directories with role mapping enabled.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users\n" + "authenticated using multiple [LDAP] external user directories with role mapping enabled.\n" + "\n" + ), link=None, level=3, - num='4.6.5') + num="4.6.5", +) RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_SameUser = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.SameUser', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.SameUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user\n' - 'authenticated using the same [LDAP] external user directory with role mapping enabled.\n' - '\n' - ), + "[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user\n" + "authenticated using the same [LDAP] external user directory with role mapping enabled.\n" + "\n" + ), link=None, level=3, - num='4.6.6') + num="4.6.6", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `` parameter in the `` section\n' - 'of the `config.xml` that SHALL be used to construct the `DN` to bind to.\n' - 'The resulting `DN` SHALL be constructed by replacing all `{user_name}` substrings of the template \n' - 'with the actual user name during each authentication attempt.\n' - '\n' - 'For example, \n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' \n' - ' uid={user_name},ou=users,dc=example,dc=com\n' - ' \n' - ' \n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the `` parameter in the `` section\n" + "of the `config.xml` that SHALL be used to construct the `DN` to bind to.\n" + "The resulting `DN` SHALL be constructed by replacing all `{user_name}` substrings of the template \n" + "with the actual user name during each authentication attempt.\n" + "\n" + "For example, \n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " \n" + " uid={user_name},ou=users,dc=example,dc=com\n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.7.1.1') + num="4.7.1.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN_ConflictWith_AuthDN = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if both `` and `` or `` parameters\n' - 'are specified as part of [LDAP] server description in the `` section of the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if both `` and `` or `` parameters\n" + "are specified as part of [LDAP] server description in the `` section of the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.7.1.2') + num="4.7.1.2", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section\n' - 'of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. \n' - '\n' - ), + "[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section\n" + "of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. \n" + "\n" + ), link=None, level=4, - num='4.7.2.1') + num="4.7.2.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the \n' - '`` section of the `config.xml` that SHALL specify how \n' - 'to construct the base DN for the LDAP search to detect the actual user DN.\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - '\n' - ' ...\n' - ' CN=Users,DC=example,DC=com\n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the \n" + "`` section of the `config.xml` that SHALL specify how \n" + "to construct the base DN for the LDAP search to detect the actual user DN.\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "\n" + " ...\n" + " CN=Users,DC=example,DC=com\n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.7.2.2') + num="4.7.2.2", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the \n' - '`` section of the `config.xml` that SHALL the scope of the \n' - 'LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values\n' - '\n' - '* `base`\n' - '* `one_level`\n' - '* `children`\n' - '* `subtree`\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - '\n' - ' ...\n' - ' one_level\n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the \n" + "`` section of the `config.xml` that SHALL the scope of the \n" + "LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values\n" + "\n" + "* `base`\n" + "* `one_level`\n" + "* `children`\n" + "* `subtree`\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "\n" + " ...\n" + " one_level\n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.7.2.3') + num="4.7.2.3", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the \n' - '`` section of the `config.xml` that SHALL specify the LDAP search\n' - 'filter used to detect the actual user DN.\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - '\n' - ' ...\n' - ' (&(objectClass=user)(sAMAccountName={user_name}))\n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the \n" + "`` section of the `config.xml` that SHALL specify the LDAP search\n" + "filter used to detect the actual user DN.\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "\n" + " ...\n" + " (&(objectClass=user)(sAMAccountName={user_name}))\n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.7.2.4') + num="4.7.2.4", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Syntax = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `role_mapping` sub-section in the `` section\n' - 'of the `config.xml`.\n' - '\n' - 'For example,\n' - '\n' - '```xml\n' - '\n' - ' \n' - ' \n' - ' \n' - ' \n' - ' ou=groups,dc=example,dc=com\n' - ' cn\n' - ' subtree\n' - ' (&(objectClass=groupOfNames)(member={bind_dn}))\n' - ' clickhouse_\n' - ' \n' - ' \n' - ' \n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the `role_mapping` sub-section in the `` section\n" + "of the `config.xml`.\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " ou=groups,dc=example,dc=com\n" + " cn\n" + " subtree\n" + " (&(objectClass=groupOfNames)(member={bind_dn}))\n" + " clickhouse_\n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + ), link=None, level=4, - num='4.8.1.1') + num="4.8.1.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SpecialCharactersEscaping = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SpecialCharactersEscaping', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SpecialCharactersEscaping", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support properly escaped special XML characters that can be present\n' - 'as part of the values for different configuration parameters inside the\n' - '`` section of the `config.xml` such as\n' - '\n' - '* `` parameter\n' - '* `` parameter\n' - '\n' - ), + "[ClickHouse] SHALL support properly escaped special XML characters that can be present\n" + "as part of the values for different configuration parameters inside the\n" + "`` section of the `config.xml` such as\n" + "\n" + "* `` parameter\n" + "* `` parameter\n" + "\n" + ), link=None, level=4, - num='4.8.2.1') + num="4.8.2.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support multiple `` sections defined inside the same `` section \n' - 'of the `config.xml` and all of the `` sections SHALL be applied.\n' - '\n' - ), + "[ClickHouse] SHALL support multiple `` sections defined inside the same `` section \n" + "of the `config.xml` and all of the `` sections SHALL be applied.\n" + "\n" + ), link=None, level=4, - num='4.8.3.1') + num="4.8.3.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections_IdenticalParameters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections.IdenticalParameters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections.IdenticalParameters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not duplicate mapped roles when multiple `` sections \n' - 'with identical parameters are defined inside the `` section \n' - 'of the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL not duplicate mapped roles when multiple `` sections \n" + "with identical parameters are defined inside the `` section \n" + "of the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.8.3.2') + num="4.8.3.2", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_BaseDN = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.BaseDN', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.BaseDN", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `` parameter in the `` section \n' - 'of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search.\n' - '\n' - 'The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of \n' - 'the template with the actual user name and bind `DN` during each [LDAP] search.\n' - '\n' - ), + "[ClickHouse] SHALL support the `` parameter in the `` section \n" + "of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search.\n" + "\n" + "The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of \n" + "the template with the actual user name and bind `DN` during each [LDAP] search.\n" + "\n" + ), link=None, level=4, - num='4.8.4.1') + num="4.8.4.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Attribute = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Attribute', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Attribute", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `` parameter in the `` section of \n' - 'the `config.xml` that SHALL specify the name of the attribute whose values SHALL be returned by the [LDAP] search.\n' - '\n' - ), + "[ClickHouse] SHALL support the `` parameter in the `` section of \n" + "the `config.xml` that SHALL specify the name of the attribute whose values SHALL be returned by the [LDAP] search.\n" + "\n" + ), link=None, level=4, - num='4.8.5.1') + num="4.8.5.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Scope = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `` parameter in the `` section of \n' - 'the `config.xml` that SHALL define the scope of the LDAP search as defined \n' - 'by the https://ldapwiki.com/wiki/LDAP%20Search%20Scopes.\n' - '\n' - ), + "[ClickHouse] SHALL support the `` parameter in the `` section of \n" + "the `config.xml` that SHALL define the scope of the LDAP search as defined \n" + "by the https://ldapwiki.com/wiki/LDAP%20Search%20Scopes.\n" + "\n" + ), link=None, level=4, - num='4.8.6.1') + num="4.8.6.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Scope_Value_Base = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Base', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Base", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `base` value for the the `` parameter in the \n' - '`` section of the `config.xml` that SHALL\n' - 'limit the scope as specified by the https://ldapwiki.com/wiki/BaseObject.\n' - '\n' - ), + "[ClickHouse] SHALL support the `base` value for the the `` parameter in the \n" + "`` section of the `config.xml` that SHALL\n" + "limit the scope as specified by the https://ldapwiki.com/wiki/BaseObject.\n" + "\n" + ), link=None, level=4, - num='4.8.6.2') + num="4.8.6.2", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Scope_Value_OneLevel = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.OneLevel', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.OneLevel", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `one_level` value for the the `` parameter in the \n' - '`` section of the `config.xml` that SHALL\n' - 'limit the scope as specified by the https://ldapwiki.com/wiki/SingleLevel.\n' - '\n' - ), + "[ClickHouse] SHALL support the `one_level` value for the the `` parameter in the \n" + "`` section of the `config.xml` that SHALL\n" + "limit the scope as specified by the https://ldapwiki.com/wiki/SingleLevel.\n" + "\n" + ), link=None, level=4, - num='4.8.6.3') + num="4.8.6.3", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Scope_Value_Children = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Children', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Children", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `children` value for the the `` parameter in the \n' - '`` section of the `config.xml` that SHALL\n' - 'limit the scope as specified by the https://ldapwiki.com/wiki/SubordinateSubtree.\n' - '\n' - ), + "[ClickHouse] SHALL support the `children` value for the the `` parameter in the \n" + "`` section of the `config.xml` that SHALL\n" + "limit the scope as specified by the https://ldapwiki.com/wiki/SubordinateSubtree.\n" + "\n" + ), link=None, level=4, - num='4.8.6.4') + num="4.8.6.4", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Scope_Value_Subtree = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Subtree', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Subtree", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `children` value for the the `` parameter in the \n' - '`` section of the `config.xml` that SHALL\n' - 'limit the scope as specified by the https://ldapwiki.com/wiki/WholeSubtree.\n' - '\n' - ), + "[ClickHouse] SHALL support the `children` value for the the `` parameter in the \n" + "`` section of the `config.xml` that SHALL\n" + "limit the scope as specified by the https://ldapwiki.com/wiki/WholeSubtree.\n" + "\n" + ), link=None, level=4, - num='4.8.6.5') + num="4.8.6.5", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Scope_Value_Default = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Default', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `subtree` as the default value for the the `` parameter in the \n' - '`` section of the `config.xml` when the `` parameter is not specified.\n' - '\n' - ), + "[ClickHouse] SHALL support the `subtree` as the default value for the the `` parameter in the \n" + "`` section of the `config.xml` when the `` parameter is not specified.\n" + "\n" + ), link=None, level=4, - num='4.8.6.6') + num="4.8.6.6", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SearchFilter = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SearchFilter', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SearchFilter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `` parameter in the ``\n' - 'section of the `config.xml` that SHALL specify the template used to construct \n' - 'the [LDAP filter](https://ldap.com/ldap-filters/) for the search.\n' - '\n' - 'The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings \n' - 'of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search.\n' - ' \n' - ), + "[ClickHouse] SHALL support the `` parameter in the ``\n" + "section of the `config.xml` that SHALL specify the template used to construct \n" + "the [LDAP filter](https://ldap.com/ldap-filters/) for the search.\n" + "\n" + "The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings \n" + "of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search.\n" + " \n" + ), link=None, level=4, - num='4.8.7.1') + num="4.8.7.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `` parameter in the ``\n' - 'section of the `config.xml` that SHALL be expected to be in front of each string in \n' - 'the original list of strings returned by the [LDAP] search. \n' - 'Prefix SHALL be removed from the original strings and resulting strings SHALL be treated as [RBAC] role names. \n' - '\n' - ), + "[ClickHouse] SHALL support the `` parameter in the ``\n" + "section of the `config.xml` that SHALL be expected to be in front of each string in \n" + "the original list of strings returned by the [LDAP] search. \n" + "Prefix SHALL be removed from the original strings and resulting strings SHALL be treated as [RBAC] role names. \n" + "\n" + ), link=None, level=4, - num='4.8.8.1') + num="4.8.8.1", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_Default = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.Default', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support empty string as the default value of the `` parameter in \n' - 'the `` section of the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL support empty string as the default value of the `` parameter in \n" + "the `` section of the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.8.8.2') + num="4.8.8.2", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithUTF8Characters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithUTF8Characters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithUTF8Characters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support UTF8 characters as the value of the `` parameter in\n' - 'the `` section of the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL support UTF8 characters as the value of the `` parameter in\n" + "the `` section of the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.8.8.3') + num="4.8.8.3", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialXMLCharacters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialXMLCharacters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialXMLCharacters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support XML special characters as the value of the `` parameter in\n' - 'the `` section of the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL support XML special characters as the value of the `` parameter in\n" + "the `` section of the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.8.8.4') + num="4.8.8.4", +) RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialRegexCharacters = Requirement( - name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialRegexCharacters', - version='1.0', + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialRegexCharacters", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support regex special characters as the value of the `` parameter in\n' - 'the `` section of the `config.xml`.\n' - '\n' - ), + "[ClickHouse] SHALL support regex special characters as the value of the `` parameter in\n" + "the `` section of the `config.xml`.\n" + "\n" + ), link=None, level=4, - num='4.8.8.5') + num="4.8.8.5", +) + +RQ_SRS_014_LDAP_ClusterWithAndWithoutSecret_DistributedTable = Requirement( + name="RQ.SRS-014.LDAP.ClusterWithAndWithoutSecret.DistributedTable", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse] SHALL support propagating query user roles and their corresponding privileges\n" + "when using `Distributed` table to the remote servers for the users that are authenticated\n" + "using LDAP either via external user directory or defined in `users.xml` when\n" + "cluster is configured with and without ``.\n" + "\n" + "For example,\n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " qwerty123\n" + " \n" + " true\n" + " \n" + " dwh\n" + " host1\n" + " \n" + " \n" + " \n" + " true\n" + " \n" + " dwh\n" + " host2\n" + " \n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + "or \n" + "\n" + "```xml\n" + "\n" + " \n" + " \n" + " \n" + " true\n" + " \n" + " dwh\n" + " host1\n" + " \n" + " \n" + " \n" + " true\n" + " \n" + " dwh\n" + " host2\n" + " \n" + " \n" + " \n" + " \n" + "\n" + "```\n" + "\n" + ), + link=None, + level=4, + num="4.9.8.1", +) SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( - name='SRS-014 ClickHouse LDAP Role Mapping', + name="SRS-014 ClickHouse LDAP Role Mapping", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -930,81 +1050,261 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='LDAP', level=2, num='3.1'), - Heading(name='Requirements', level=1, num='4'), - Heading(name='General', level=2, num='4.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping', level=3, num='4.1.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.WithFixedRoles', level=3, num='4.1.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Search', level=3, num='4.1.3'), - Heading(name='Mapped Role Names', level=2, num='4.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithUTF8Characters', level=3, num='4.2.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.Long', level=3, num='4.2.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialXMLCharacters', level=3, num='4.2.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialRegexCharacters', level=3, num='4.2.4'), - Heading(name='Multiple Roles', level=2, num='4.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Map.MultipleRoles', level=3, num='4.3.1'), - Heading(name='LDAP Groups', level=2, num='4.4'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.Removed', level=3, num='4.4.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.RemovedAndAdded.Parallel', level=3, num='4.4.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemoved', level=3, num='4.4.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemovedAndAdded.Parallel', level=3, num='4.4.4'), - Heading(name='RBAC Roles', level=2, num='4.5'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NotPresent', level=3, num='4.5.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Added', level=3, num='4.5.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Removed', level=3, num='4.5.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Readded', level=3, num='4.5.4'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedAndAdded.Parallel', level=3, num='4.5.5'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.New', level=3, num='4.5.6'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NewPrivilege', level=3, num='4.5.7'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedPrivilege', level=3, num='4.5.8'), - Heading(name='Authentication', level=2, num='4.6'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel', level=3, num='4.6.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.ValidAndInvalid', level=3, num='4.6.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.MultipleServers', level=3, num='4.6.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalOnly', level=3, num='4.6.4'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalAndMultipleLDAP', level=3, num='4.6.5'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.SameUser', level=3, num='4.6.6'), - Heading(name='Server Configuration', level=2, num='4.7'), - Heading(name='BindDN Parameter', level=3, num='4.7.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN', level=4, num='4.7.1.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN', level=4, num='4.7.1.2'), - Heading(name='User DN Detection', level=3, num='4.7.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection', level=4, num='4.7.2.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN', level=4, num='4.7.2.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope', level=4, num='4.7.2.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter', level=4, num='4.7.2.4'), - Heading(name='External User Directory Configuration', level=2, num='4.8'), - Heading(name='Syntax', level=3, num='4.8.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax', level=4, num='4.8.1.1'), - Heading(name='Special Characters Escaping', level=3, num='4.8.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SpecialCharactersEscaping', level=4, num='4.8.2.1'), - Heading(name='Multiple Sections', level=3, num='4.8.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections', level=4, num='4.8.3.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections.IdenticalParameters', level=4, num='4.8.3.2'), - Heading(name='BaseDN Parameter', level=3, num='4.8.4'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.BaseDN', level=4, num='4.8.4.1'), - Heading(name='Attribute Parameter', level=3, num='4.8.5'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Attribute', level=4, num='4.8.5.1'), - Heading(name='Scope Parameter', level=3, num='4.8.6'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope', level=4, num='4.8.6.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Base', level=4, num='4.8.6.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.OneLevel', level=4, num='4.8.6.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Children', level=4, num='4.8.6.4'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Subtree', level=4, num='4.8.6.5'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Default', level=4, num='4.8.6.6'), - Heading(name='Search Filter Parameter', level=3, num='4.8.7'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SearchFilter', level=4, num='4.8.7.1'), - Heading(name='Prefix Parameter', level=3, num='4.8.8'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix', level=4, num='4.8.8.1'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.Default', level=4, num='4.8.8.2'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithUTF8Characters', level=4, num='4.8.8.3'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialXMLCharacters', level=4, num='4.8.8.4'), - Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialRegexCharacters', level=4, num='4.8.8.5'), - Heading(name='References', level=1, num='5'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="LDAP", level=2, num="3.1"), + Heading(name="Requirements", level=1, num="4"), + Heading(name="General", level=2, num="4.1"), + Heading(name="RQ.SRS-014.LDAP.RoleMapping", level=3, num="4.1.1"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.WithFixedRoles", level=3, num="4.1.2" ), + Heading(name="RQ.SRS-014.LDAP.RoleMapping.Search", level=3, num="4.1.3"), + Heading(name="Mapped Role Names", level=2, num="4.2"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithUTF8Characters", + level=3, + num="4.2.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.Long", level=3, num="4.2.2" + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialXMLCharacters", + level=3, + num="4.2.3", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Map.Role.Name.WithSpecialRegexCharacters", + level=3, + num="4.2.4", + ), + Heading(name="Multiple Roles", level=2, num="4.3"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Map.MultipleRoles", level=3, num="4.3.1" + ), + Heading(name="LDAP Groups", level=2, num="4.4"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.Removed", level=3, num="4.4.1" + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.RemovedAndAdded.Parallel", + level=3, + num="4.4.2", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemoved", + level=3, + num="4.4.3", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.LDAP.Group.UserRemovedAndAdded.Parallel", + level=3, + num="4.4.4", + ), + Heading(name="RBAC Roles", level=2, num="4.5"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NotPresent", + level=3, + num="4.5.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Added", level=3, num="4.5.2" + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Removed", level=3, num="4.5.3" + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.Readded", level=3, num="4.5.4" + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedAndAdded.Parallel", + level=3, + num="4.5.5", + ), + Heading(name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.New", level=3, num="4.5.6"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.NewPrivilege", + level=3, + num="4.5.7", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.RBAC.Role.RemovedPrivilege", + level=3, + num="4.5.8", + ), + Heading(name="Authentication", level=2, num="4.6"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel", + level=3, + num="4.6.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.ValidAndInvalid", + level=3, + num="4.6.2", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.MultipleServers", + level=3, + num="4.6.3", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalOnly", + level=3, + num="4.6.4", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.LocalAndMultipleLDAP", + level=3, + num="4.6.5", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Authentication.Parallel.SameUser", + level=3, + num="4.6.6", + ), + Heading(name="Server Configuration", level=2, num="4.7"), + Heading(name="BindDN Parameter", level=3, num="4.7.1"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN", + level=4, + num="4.7.1.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN", + level=4, + num="4.7.1.2", + ), + Heading(name="User DN Detection", level=3, num="4.7.2"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection", + level=4, + num="4.7.2.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN", + level=4, + num="4.7.2.2", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope", + level=4, + num="4.7.2.3", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter", + level=4, + num="4.7.2.4", + ), + Heading(name="External User Directory Configuration", level=2, num="4.8"), + Heading(name="Syntax", level=3, num="4.8.1"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax", + level=4, + num="4.8.1.1", + ), + Heading(name="Special Characters Escaping", level=3, num="4.8.2"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SpecialCharactersEscaping", + level=4, + num="4.8.2.1", + ), + Heading(name="Multiple Sections", level=3, num="4.8.3"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections", + level=4, + num="4.8.3.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.MultipleSections.IdenticalParameters", + level=4, + num="4.8.3.2", + ), + Heading(name="BaseDN Parameter", level=3, num="4.8.4"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.BaseDN", + level=4, + num="4.8.4.1", + ), + Heading(name="Attribute Parameter", level=3, num="4.8.5"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Attribute", + level=4, + num="4.8.5.1", + ), + Heading(name="Scope Parameter", level=3, num="4.8.6"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope", + level=4, + num="4.8.6.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Base", + level=4, + num="4.8.6.2", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.OneLevel", + level=4, + num="4.8.6.3", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Children", + level=4, + num="4.8.6.4", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Subtree", + level=4, + num="4.8.6.5", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Scope.Value.Default", + level=4, + num="4.8.6.6", + ), + Heading(name="Search Filter Parameter", level=3, num="4.8.7"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.SearchFilter", + level=4, + num="4.8.7.1", + ), + Heading(name="Prefix Parameter", level=3, num="4.8.8"), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix", + level=4, + num="4.8.8.1", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.Default", + level=4, + num="4.8.8.2", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithUTF8Characters", + level=4, + num="4.8.8.3", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialXMLCharacters", + level=4, + num="4.8.8.4", + ), + Heading( + name="RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialRegexCharacters", + level=4, + num="4.8.8.5", + ), + Heading(name="Cluster With And Without Secret", level=2, num="4.9"), + Heading( + name="RQ.SRS-014.LDAP.ClusterWithAndWithoutSecret.DistributedTable", + level=4, + num="4.9.8.1", + ), + Heading(name="References", level=1, num="5"), + ), requirements=( RQ_SRS_014_LDAP_RoleMapping, RQ_SRS_014_LDAP_RoleMapping_WithFixedRoles, @@ -1056,8 +1356,9 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithUTF8Characters, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialXMLCharacters, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialRegexCharacters, - ), - content=''' + RQ_SRS_014_LDAP_ClusterWithAndWithoutSecret_DistributedTable, + ), + content=""" # SRS-014 ClickHouse LDAP Role Mapping # Software Requirements Specification @@ -1136,6 +1437,8 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( * 4.8.8.3 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithUTF8Characters](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingprefixwithutf8characters) * 4.8.8.4 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialXMLCharacters](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingprefixwithspecialxmlcharacters) * 4.8.8.5 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Prefix.WithSpecialRegexCharacters](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingprefixwithspecialregexcharacters) + * 4.9 [Cluster With And Without Secret](#cluster-with-and-without-secret) + * 4.9.8.1 [RQ.SRS-014.LDAP.ClusterWithAndWithoutSecret.DistributedTable](#rqsrs-014ldapclusterwithandwithoutsecretdistributedtable) * 5 [References](#references) ## Revision History @@ -1608,6 +1911,67 @@ version: 1.0 [ClickHouse] SHALL support regex special characters as the value of the `` parameter in the `` section of the `config.xml`. +### Cluster With And Without Secret + +##### RQ.SRS-014.LDAP.ClusterWithAndWithoutSecret.DistributedTable +version: 1.0 + +[ClickHouse] SHALL support propagating query user roles and their corresponding privileges +when using `Distributed` table to the remote servers for the users that are authenticated +using LDAP either via external user directory or defined in `users.xml` when +cluster is configured with and without ``. + +For example, + +```xml + + + + qwerty123 + + true + + dwh + host1 + + + + true + + dwh + host2 + + + + + +``` + +or + +```xml + + + + + true + + dwh + host1 + + + + true + + dwh + host2 + + + + + +``` + ## References * **Access Control and Account Management**: https://clickhouse.com/docs/en/operations/access-rights/ @@ -1628,4 +1992,5 @@ the `` section of the `config.xml`. [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/role_mapping/requirements/requirements.md [Git]: https://git-scm.com/ [GitHub]: https://github.com -''') +""", +) diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env/clickhouse-service.yml b/tests/testflows/ldap/role_mapping/role_mapping_env/clickhouse-service.yml new file mode 100644 index 00000000000..7ff0139ab9b --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env/clickhouse-service.yml @@ -0,0 +1,37 @@ +version: '2.3' + +services: + clickhouse: + image: clickhouse/integration-test + init: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/common.xml:/etc/clickhouse-server/users.d/common.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/common.xml:/etc/clickhouse-server/config.d/common.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl/dhparam.pem:/etc/clickhouse-server/ssl/dhparam.pem" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl/server.crt:/etc/clickhouse-server/ssl/server.crt" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl/server.key:/etc/clickhouse-server/ssl/server.key" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "tail -f /dev/null" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env/docker-compose.yml b/tests/testflows/ldap/role_mapping/role_mapping_env/docker-compose.yml new file mode 100644 index 00000000000..624a5a18498 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env/docker-compose.yml @@ -0,0 +1,159 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env/openldap-service.yml b/tests/testflows/ldap/role_mapping/role_mapping_env/openldap-service.yml new file mode 100644 index 00000000000..606ea3f723f --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env/openldap-service.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + environment: + PHPLDAPADMIN_HTTPS=false: + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env/zookeeper-service.yml b/tests/testflows/ldap/role_mapping/role_mapping_env/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/clickhouse-service.yml b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/clickhouse-service.yml new file mode 100644 index 00000000000..c96cb512837 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/clickhouse-service.yml @@ -0,0 +1,37 @@ +version: '2.3' + +services: + clickhouse: + image: registry.gitlab.com/altinity-public/container-images/test/clickhouse-integration-test:21.12 + init: true + privileged: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/common.xml:/etc/clickhouse-server/users.d/common.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ports.xml:/etc/clickhouse-server/config.d/ports.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/ssl.xml:/etc/clickhouse-server/config.d/ssl.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/storage.xml:/etc/clickhouse-server/config.d/storage.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl/dhparam.pem:/etc/clickhouse-server/ssl/dhparam.pem" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl/server.crt:/etc/clickhouse-server/ssl/server.crt" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl/server.key:/etc/clickhouse-server/ssl/server.key" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "tail -f /dev/null" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/docker-compose.yml b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/docker-compose.yml new file mode 100644 index 00000000000..624a5a18498 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/docker-compose.yml @@ -0,0 +1,159 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/openldap-service.yml b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/openldap-service.yml new file mode 100644 index 00000000000..606ea3f723f --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/openldap-service.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + environment: + PHPLDAPADMIN_HTTPS=false: + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/zookeeper-service.yml b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/role_mapping/role_mapping_env_arm64/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/role_mapping/tests/cluster_secret.py b/tests/testflows/ldap/role_mapping/tests/cluster_secret.py new file mode 100644 index 00000000000..1309230d345 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/tests/cluster_secret.py @@ -0,0 +1,595 @@ +from testflows.core import * +from testflows.asserts import error + +from ldap.role_mapping.requirements import * +from ldap.role_mapping.tests.common import * + + +def cluster_node(name): + """Get cluster node instance.""" + return current().context.cluster.node(name) + + +@TestStep(Given) +def add_sharded_cluster( + self, node, name="sharded_cluster_with_secret", with_secret=True, restart=False +): + """Add configuration of sharded cluster that uses secret.""" + entries = {"remote_servers": {name: []}} + + if with_secret: + entries["remote_servers"][name].append({"secret": "qwerty123"}) + + for node_name in self.context.cluster.nodes["clickhouse"]: + entries["remote_servers"][name].append( + { + "shard": {"replica": {"host": node_name, "port": "9000"}}, + }, + ) + + config = create_xml_config_content(entries=entries, config_file=f"{name}.xml") + return add_config(config, node=node, restart=restart) + + +@TestStep(Given) +def create_table(self, on_cluster, name=None, node=None): + """Create table on cluster.""" + if node is None: + node = self.context.node + if name is None: + name = getuid() + + try: + node.query( + f"CREATE TABLE {name} ON CLUSTER {on_cluster} (d Date, a String, b UInt8, x String, y Int8) " + f"ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') " + "PARTITION BY y ORDER BY (d, b)" + ) + yield name + finally: + with Finally(f"I drop table {name} on cluster {on_cluster} on {node.name}"): + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster} SYNC") + + +@TestStep(Given) +def create_distributed_table(self, on_cluster, over, name=None, node=None): + """Create distributed table on cluster over some underlying table.""" + if node is None: + node = self.context.node + if name is None: + name = getuid() + + try: + node.query( + f"CREATE TABLE {name} ON CLUSTER {on_cluster} AS {over} " + f"ENGINE = Distributed({on_cluster}, default, {over}, rand())" + ) + yield name + finally: + with Finally(f"I drop table {name} on cluster {on_cluster} on {node.name}"): + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster} SYNC") + + +@TestStep(Given) +def grant_select(self, cluster, privilege, role_or_user, node=None): + """Grant select privilege on a table on a given cluster + to a role or a user. + """ + if node is None: + node = self.context.node + + try: + node.query(f"GRANT ON CLUSTER {cluster} {privilege} TO {role_or_user}") + yield + finally: + with Finally( + f"I remove privilege '{privilege}' on {cluster} from {role_or_user}" + ): + node.query(f"REVOKE ON CLUSTER {cluster} {privilege} FROM {role_or_user}") + + +@TestScenario +def select_using_mapped_role(self, cluster, role_name, role_mapped, user): + """Check accessing normal and distributed table using + a user and the specified role that is either granted + rights to access the tables or not and is or is not assigned to the user + from all cluster nodes. + """ + # default cluster node + node = cluster_node("clickhouse1") + + query_settings = [("user", user["username"]), ("password", user["password"])] + + with Given(f"I create base table on cluster {cluster}"): + src_table = create_table(on_cluster=cluster, node=node) + + with And(f"I create distristibuted table over base table on cluster {cluster}"): + dist_table = create_distributed_table( + on_cluster=cluster, over=src_table, node=node + ) + + with And("I check that grants for the user"): + for name in self.context.cluster.nodes["clickhouse"]: + for attempt in retries(timeout=10): + with attempt: + with By(f"executing query on node {name}", flags=TE): + r = self.context.cluster.node(name).query( + f"SHOW GRANTS", settings=query_settings + ) + if role_mapped: + with Then("check that role is mapped"): + assert role_name in r.output, error() + + with Example("no privilege on source table"): + with When("user tries to read from the source table without privilege"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"executing query on node {name}", flags=TE): + self.context.cluster.node(name).query( + f"SELECT * FROM {src_table}", + settings=query_settings, + exitcode=241, + message=f"DB::Exception:", + ) + + with Example("with privilege on source table"): + with Given("I grant SELECT on source table to the mapped role"): + grant_select( + cluster=cluster, + privilege=f"SELECT ON {src_table}", + role_or_user=role_name, + node=node, + ) + + with Then("user should be able to read from the source table"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"executing query on node {name}", flags=TE): + self.context.cluster.node(name).query( + f"SELECT * FROM {src_table}", + settings=query_settings, + exitcode=0 if role_mapped else 241, + message="" if role_mapped else "DB::Exception:", + ) + + with Example("with privilege only on distributed table"): + with Given("I grant SELECT on distributed table to the mapped role"): + grant_select( + cluster=cluster, + privilege=f"SELECT ON {dist_table}", + role_or_user=role_name, + node=node, + ) + + with Then("user should still not be able to read from distributed table"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"executing query on node {name}", flags=TE): + self.context.cluster.node(name).query( + f"SELECT * FROM {dist_table}", + settings=query_settings, + exitcode=241, + message=f"DB::Exception:", + ) + + with Example("with privilege only on source but not on distributed table"): + with Given("I grant SELECT on source table to the mapped role"): + grant_select( + cluster=cluster, + privilege=f"SELECT ON {src_table}", + role_or_user=role_name, + node=node, + ) + + with Then("user should still not be able to read from distributed table"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"executing query on node {name}", flags=TE): + self.context.cluster.node(name).query( + f"SELECT * FROM {dist_table}", + settings=query_settings, + exitcode=241, + message=f"DB::Exception:", + ) + + with Example("with privilege on source and distributed"): + with Given("I grant SELECT on source table to the mapped role"): + grant_select( + cluster=cluster, + privilege=f"SELECT ON {src_table}", + role_or_user=role_name, + node=node, + ) + + with And("I grant SELECT on distributed table to the mapped role"): + grant_select( + cluster=cluster, + privilege=f"SELECT ON {dist_table}", + role_or_user=role_name, + node=node, + ) + + with Then("user should be able to read from the distributed table"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"executing query on node {name}", flags=TE): + self.context.cluster.node(name).query( + f"SELECT * FROM {dist_table}", + settings=query_settings, + exitcode=0 if role_mapped else 241, + message="" if role_mapped else "DB::Exception:", + ) + + +@TestFeature +def execute_tests(self, role_name, role_mapped, ldap_user, local_user): + """Execute all scenarios on cluster with or without secret + for LDAP and local users, using a role that might be + mapped or not. + """ + for cluster_type in ["with secret", "without secret"]: + with Feature("cluster " + cluster_type): + for user in [ldap_user, local_user]: + with Feature(user["type"]): + with Feature(f"role {role_name} mapped {role_mapped}"): + if role_mapped and user["type"] == "local user": + with Given(f"I grant role {role_name} to local RBAC user"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + cluster_node(name).query( + f"GRANT {role_name} TO {local_user['username']}" + ) + + for scenario in ordered(loads(current_module(), Scenario)): + scenario( + cluster="sharded_cluster_" + + cluster_type.replace(" ", "_"), + role_name=role_name, + role_mapped=role_mapped, + user=user, + ) + + +@TestOutline(Feature) +def outline_using_external_user_directory( + self, ldap_servers, mapping, ldap_roles_or_groups, rbac_roles, mapped_roles +): + """Check using simple and distributed table access when using + LDAP external user directory or LDAP authenticated existing RBAC users + with and without cluster secret. + + Where mapping can be one of the following: + 'static' or 'dynamic' or 'dynamic and static' + """ + ldap_user = { + "type": "ldap user", + "server": "openldap1", + "username": "user1", + "password": "user1", + "dn": "cn=user1,ou=users,dc=company,dc=com", + } + + local_user = { + "type": "local user", + "username": "local_user1", + "password": "local_user1", + } + + role_mappings = [ + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", + "prefix": "clickhouse_", + } + ] + + if mapping in ["dynamic", "dynamic and static"]: + with Given("I add LDAP groups"): + for name in ldap_servers: + for group_name in ldap_roles_or_groups: + with By(f"adding {group_name}"): + ldap_groups = add_ldap_groups( + groups=({"cn": group_name},), node=cluster_node(name) + ) + + with And("I add LDAP user to the group"): + add_user_to_group_in_ldap( + user=ldap_user, + group=ldap_groups[0], + node=cluster_node(name), + ) + + with Given( + f"I add LDAP external user directory configuration with {mapping} role mapping" + ): + for name in self.context.cluster.nodes["clickhouse"]: + if mapping == "dynamic": + By( + f"on node {name}", + test=add_ldap_external_user_directory, + parallel=True, + )( + server="openldap1", + role_mappings=role_mappings, + restart=True, + node=cluster_node(name), + ) + elif mapping == "dynamic and static": + By( + f"on node {name}", + test=add_ldap_external_user_directory, + parallel=True, + )( + server="openldap1", + role_mappings=role_mappings, + roles=ldap_roles_or_groups, + restart=True, + node=cluster_node(name), + ) + else: + By( + f"on node {name}", + test=add_ldap_external_user_directory, + parallel=True, + )( + server="openldap1", + roles=ldap_roles_or_groups, + restart=True, + node=cluster_node(name), + ) + + with And("I add local RBAC user"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + add_rbac_users(users=[local_user], node=cluster_node(name)) + + with And("I add RBAC roles on cluster"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + add_rbac_roles(roles=rbac_roles, node=cluster_node(name)) + + for role_name in rbac_roles: + execute_tests( + role_name=role_name, + role_mapped=(role_name in mapped_roles), + ldap_user=ldap_user, + local_user=local_user, + ) + + +@TestFeature +def using_authenticated_users(self, ldap_servers): + """Check using simple and distributed table access when using + LDAP authenticated existing users with and without cluster secret. + """ + role_name = f"role_{getuid()}" + + ldap_user = { + "type": "ldap authenticated user", + "cn": "myuser", + "username": "myuser", + "userpassword": "myuser", + "password": "myuser", + "server": "openldap1", + } + + local_user = { + "type": "local user", + "username": "local_user2", + "password": "local_user2", + } + + with Given("I add LDAP user"): + add_user = { + "cn": ldap_user["cn"], + "userpassword": ldap_user["userpassword"], + } + for name in ldap_servers: + add_ldap_users(users=[add_user], node=cluster_node(name)) + + with And("I add LDAP authenticated users configuration"): + for name in self.context.cluster.nodes["clickhouse"]: + By(f"on node {name}", test=add_ldap_authenticated_users, parallel=True)( + users=[ldap_user], rbac=True, node=cluster_node(name) + ) + + with And("I add local RBAC user"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + add_rbac_users(users=[local_user], node=cluster_node(name)) + + with And("I add RBAC role on cluster that user will use"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + add_rbac_roles(roles=(f"{role_name}",), node=cluster_node(name)) + + with And("I grant role to LDAP authenticated user"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + cluster_node(name).query( + f"GRANT {role_name} TO {ldap_user['username']}" + ) + + with And("I grant role to local RBAC user"): + for name in self.context.cluster.nodes["clickhouse"]: + with By(f"on node {name}"): + cluster_node(name).query( + f"GRANT {role_name} TO {local_user['username']}" + ) + + execute_tests( + role_name=role_name, + role_mapped=role_name, + ldap_user=ldap_user, + local_user=local_user, + ) + + +@TestFeature +def using_external_user_directory(self, ldap_servers): + """Check using LDAP external user directory with different + role mapping mode and different cases of role existens. + """ + uid = getuid() + + for mapping in ["dynamic", "static", "dynamic and static"]: + with Example(f"{mapping}"): + with Example("all mapped roles exist"): + if mapping == "dynamic": + ldap_roles_or_groups = [ + f"clickhouse_role0_{uid}", + f"clickhouse_role1_{uid}", + ] + elif mapping == "dynamic and static": + ldap_roles_or_groups = [ + f"clickhouse_role0_{uid}", + f"clickhouse_role1_{uid}", + f"role2_{uid}", + f"role3_{uid}", + ] + else: + ldap_roles_or_groups = [ + f"role0_{uid}", + f"role1_{uid}", + f"role2_{uid}", + f"role3_{uid}", + ] + + rbac_roles = [f"role0_{uid}", f"role1_{uid}"] + mapped_roles = [f"role0_{uid}", f"role1_{uid}"] + + outline_using_external_user_directory( + ldap_servers=ldap_servers, + mapping=mapping, + ldap_roles_or_groups=ldap_roles_or_groups, + rbac_roles=rbac_roles, + mapped_roles=mapped_roles, + ) + + with Example("some mapped roles exist"): + if mapping == "dynamic": + ldap_roles_or_groups = [ + f"clickhouse_role0_{uid}", + f"clickhouse_role1_{uid}", + ] + elif mapping == "dynamic and static": + ldap_roles_or_groups = [ + f"clickhouse_role0_{uid}", + f"clickhouse_role1_{uid}", + f"role2_{uid}", + f"role3_{uid}", + ] + else: + ldap_roles_or_groups = [f"role0_{uid}", f"role1_{uid}"] + + rbac_roles = [f"role0_{uid}", f"role_not_mapped_{uid}", f"role2_{uid}"] + + if mapping == "dynamic and static": + mapped_roles = [f"role0_{uid}", f"role2_{uid}"] + else: + mapped_roles = [f"role0_{uid}"] + + outline_using_external_user_directory( + ldap_servers=ldap_servers, + mapping=mapping, + ldap_roles_or_groups=ldap_roles_or_groups, + rbac_roles=rbac_roles, + mapped_roles=mapped_roles, + ) + + with Example("no mapped roles exist"): + if mapping == "dynamic": + ldap_roles_or_groups = [ + f"clickhouse_role0_{uid}", + f"clickhouse_role1_{uid}", + ] + elif mapping == "dynamic and static": + ldap_roles_or_groups = [ + f"clickhouse_role0_{uid}", + f"clickhouse_role1_{uid}", + f"role2_{uid}", + f"role3_{uid}", + ] + else: + ldap_roles_or_groups = [f"role0_{uid}", f"role1_{uid}"] + + rbac_roles = [f"role_not_mapped0_{uid}", f"role_not_mapped1_{uid}"] + mapped_roles = [] + + outline_using_external_user_directory( + ldap_servers=ldap_servers, + mapping=mapping, + ldap_roles_or_groups=ldap_roles_or_groups, + rbac_roles=rbac_roles, + mapped_roles=mapped_roles, + ) + + with Example("empty roles"): + ldap_roles_or_groups = [] + rbac_roles = [f"role0_{uid}", f"role1_{uid}"] + mapped_roles = [] + + outline_using_external_user_directory( + ldap_servers=ldap_servers, + mapping=mapping, + ldap_roles_or_groups=ldap_roles_or_groups, + rbac_roles=rbac_roles, + mapped_roles=mapped_roles, + ) + + +@TestFeature +@Name("cluster secret") +@Requirements(RQ_SRS_014_LDAP_ClusterWithAndWithoutSecret_DistributedTable("1.0")) +def feature(self): + """Check using Distributed table when cluster is configured with and without secret + using users authenticated via LDAP either through external user directory + or defined using RBAC with LDAP server authentication. + """ + ldap_servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + }, + } + + with Given("I fix LDAP access permissions"): + for name in ldap_servers: + fix_ldap_permissions(node=cluster_node(name)) + + with And( + "I add LDAP servers configuration on all nodes", description=f"{ldap_servers}" + ): + for name in self.context.cluster.nodes["clickhouse"]: + By(f"on node {name}", test=add_ldap_servers_configuration, parallel=True)( + servers=ldap_servers, node=cluster_node(name) + ) + + with And("I add sharded cluster that uses secrets on all the nodes"): + for name in self.context.cluster.nodes["clickhouse"]: + By( + f"adding configuration on {name}", + test=add_sharded_cluster, + parallel=True, + )( + node=cluster_node(name), + name="sharded_cluster_with_secret", + with_secret=True, + ) + + with And("I add sharded cluster that does not use secrets on all the nodes"): + for name in self.context.cluster.nodes["clickhouse"]: + By( + f"adding configuration on {name}", + test=add_sharded_cluster, + parallel=True, + )( + node=cluster_node(name), + name="sharded_cluster_without_secret", + with_secret=False, + ) + + Feature("external user directory", test=using_external_user_directory)( + ldap_servers=ldap_servers + ) + Feature("authenticated users", test=using_authenticated_users)( + ldap_servers=ldap_servers + ) diff --git a/tests/testflows/ldap/role_mapping/tests/common.py b/tests/testflows/ldap/role_mapping/tests/common.py index 565503296e3..ec7cd6b0144 100644 --- a/tests/testflows/ldap/role_mapping/tests/common.py +++ b/tests/testflows/ldap/role_mapping/tests/common.py @@ -3,15 +3,20 @@ import os from testflows.core import * from testflows.asserts import error -from ldap.authentication.tests.common import getuid, create_ldap_servers_config_content, add_config, Config +from helpers.common import create_xml_config_content, add_config +from ldap.authentication.tests.common import ( + getuid, + create_ldap_servers_config_content, + ldap_authenticated_users, +) from ldap.external_user_directory.tests.common import rbac_roles, rbac_users, ldap_users -from ldap.authentication.tests.common import xmltree, xml_indent, xml_append, xml_with_utf8 + @TestStep(Given) -def create_table(self, name, create_statement, on_cluster=False): - """Create table. - """ - node = current().context.node +def create_table(self, name, create_statement, on_cluster=False, node=None): + """Create table.""" + if node is None: + node = current().context.node try: with Given(f"I have a {name} table"): node.query(create_statement.format(name=name)) @@ -23,19 +28,27 @@ def create_table(self, name, create_statement, on_cluster=False): else: node.query(f"DROP TABLE IF EXISTS {name}") + @TestStep(Given) -def add_ldap_servers_configuration(self, servers, config=None, config_d_dir="/etc/clickhouse-server/config.d", - config_file="ldap_servers.xml", timeout=60, restart=False): - """Add LDAP servers configuration to config.xml. - """ +def add_ldap_servers_configuration( + self, + servers, + config=None, + config_d_dir="/etc/clickhouse-server/config.d", + config_file="ldap_servers.xml", + timeout=60, + restart=False, + node=None, +): + """Add LDAP servers configuration to config.xml.""" if config is None: config = create_ldap_servers_config_content(servers, config_d_dir, config_file) - return add_config(config, restart=restart) + return add_config(config, restart=restart, node=node) + @TestStep(Given) def add_ldap_groups(self, groups, node=None): - """Add multiple new groups to the LDAP server. - """ + """Add multiple new groups to the LDAP server.""" try: _groups = [] for group in groups: @@ -47,45 +60,89 @@ def add_ldap_groups(self, groups, node=None): for _group in _groups: delete_group_from_ldap(_group, node=node) + @TestStep(Given) -def add_ldap_external_user_directory(self, server, roles=None, role_mappings=None, - config_d_dir="/etc/clickhouse-server/config.d", - config_file=None, timeout=60, restart=True, config=None): - """Add LDAP external user directory. - """ +def add_ldap_external_user_directory( + self, + server, + roles=None, + role_mappings=None, + config_d_dir="/etc/clickhouse-server/config.d", + config_file=None, + timeout=60, + restart=True, + config=None, + node=None, +): + """Add LDAP external user directory.""" if config_file is None: config_file = f"ldap_external_user_directory_with_role_mapping_{getuid()}.xml" if config is None: - config = create_ldap_external_user_directory_config_content(server=server, roles=roles, - role_mappings=role_mappings, config_d_dir=config_d_dir, config_file=config_file) + config = create_ldap_external_user_directory_config_content( + server=server, + roles=roles, + role_mappings=role_mappings, + config_d_dir=config_d_dir, + config_file=config_file, + ) + + return add_config(config, restart=restart, node=node) - return add_config(config, restart=restart) @TestStep(Given) -def add_rbac_roles(self, roles): - """Add RBAC roles. - """ - with rbac_roles(*roles) as _roles: +def add_rbac_roles(self, roles, node=None): + """Add RBAC roles.""" + with rbac_roles(*roles, node=node) as _roles: yield _roles + @TestStep(Given) -def add_rbac_users(self, users): - """Add RBAC users. - """ - with rbac_users(*users) as _users: - yield _users +def add_rbac_users(self, users, node=None): + """Add RBAC users.""" + if node is None: + node = self.context.node + try: + with Given(f"I create local users on {node}"): + for user in users: + username = user.get("username", None) or user["cn"] + password = user.get("password", None) or user["userpassword"] + with By(f"creating user {username}"): + node.query( + f"CREATE USER OR REPLACE {username} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{password}'" + ) + yield users + finally: + with Finally(f"I drop local users on {node}"): + for user in users: + username = user.get("username", None) or user["cn"] + with By(f"dropping user {username}", flags=TE): + node.query(f"DROP USER IF EXISTS {username}") + @TestStep(Given) def add_ldap_users(self, users, node=None): - """Add LDAP users. - """ + """Add LDAP users.""" with ldap_users(*users, node=node) as _users: yield _users + +@TestStep(Given) +def add_ldap_authenticated_users( + self, users, config_file=None, rbac=False, node=None, restart=True +): + """Add LDAP authenticated users.""" + if config_file is None: + config_file = f"ldap_users_{getuid()}.xml" + + with ldap_authenticated_users( + *users, config_file=config_file, restart=restart, rbac=rbac, node=node + ): + yield users + + def add_group_to_ldap(cn, gidnumber=None, node=None, _gidnumber=[600], exitcode=0): - """Add new group entry to LDAP. - """ + """Add new group entry to LDAP.""" _gidnumber[0] += 1 if node is None: @@ -98,7 +155,7 @@ def add_group_to_ldap(cn, gidnumber=None, node=None, _gidnumber=[600], exitcode= "dn": f"cn={cn},ou=groups,dc=company,dc=com", "objectclass": ["top", "groupOfUniqueNames"], "uniquemember": "cn=admin,dc=company,dc=com", - "_server": node.name + "_server": node.name, } lines = [] @@ -115,29 +172,31 @@ def add_group_to_ldap(cn, gidnumber=None, node=None, _gidnumber=[600], exitcode= ldif = "\n".join(lines) r = node.command( - f"echo -e \"{ldif}\" | ldapadd -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + f'echo -e "{ldif}" | ldapadd -x -H ldap://localhost -D "cn=admin,dc=company,dc=com" -w admin' + ) if exitcode is not None: assert r.exitcode == exitcode, error() return group + def delete_group_from_ldap(group, node=None, exitcode=0): - """Delete group entry from LDAP. - """ + """Delete group entry from LDAP.""" if node is None: node = current().context.ldap_node with By(f"deleting group {group['dn']}"): r = node.command( - f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{group['dn']}\"") + f"ldapdelete -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin \"{group['dn']}\"" + ) if exitcode is not None: assert r.exitcode == exitcode, error() + def fix_ldap_permissions(node=None, exitcode=0): - """Fix LDAP access permissions. - """ + """Fix LDAP access permissions.""" if node is None: node = current().context.ldap_node @@ -147,91 +206,67 @@ def fix_ldap_permissions(node=None, exitcode=0): "delete: olcAccess\n" "-\n" "add: olcAccess\n" - "olcAccess: to attrs=userPassword,shadowLastChange by self write by dn=\\\"cn=admin,dc=company,dc=com\\\" write by anonymous auth by * none\n" - "olcAccess: to * by self write by dn=\\\"cn=admin,dc=company,dc=com\\\" read by users read by * none" + 'olcAccess: to attrs=userPassword,shadowLastChange by self write by dn=\\"cn=admin,dc=company,dc=com\\" write by anonymous auth by * none\n' + 'olcAccess: to * by self write by dn=\\"cn=admin,dc=company,dc=com\\" read by users read by * none' ) - r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -Y EXTERNAL -Q -H ldapi:///") + r = node.command(f'echo -e "{ldif}" | ldapmodify -Y EXTERNAL -Q -H ldapi:///') if exitcode is not None: assert r.exitcode == exitcode, error() + def add_user_to_group_in_ldap(user, group, node=None, exitcode=0): - """Add user to a group in LDAP. - """ + """Add user to a group in LDAP.""" if node is None: node = current().context.ldap_node - ldif = (f"dn: {group['dn']}\n" + ldif = ( + f"dn: {group['dn']}\n" "changetype: modify\n" "add: uniquemember\n" - f"uniquemember: {user['dn']}") + f"uniquemember: {user['dn']}" + ) with By(f"adding user {user['dn']} to group {group['dn']}"): r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + f'echo -e "{ldif}" | ldapmodify -x -H ldap://localhost -D "cn=admin,dc=company,dc=com" -w admin' + ) if exitcode is not None: assert r.exitcode == exitcode, error() + def delete_user_from_group_in_ldap(user, group, node=None, exitcode=0): - """Delete user from a group in LDAP. - """ + """Delete user from a group in LDAP.""" if node is None: node = current().context.ldap_node - ldif = (f"dn: {group['dn']}\n" + ldif = ( + f"dn: {group['dn']}\n" "changetype: modify\n" "delete: uniquemember\n" - f"uniquemember: {user['dn']}") + f"uniquemember: {user['dn']}" + ) with By(f"deleting user {user['dn']} from group {group['dn']}"): r = node.command( - f"echo -e \"{ldif}\" | ldapmodify -x -H ldap://localhost -D \"cn=admin,dc=company,dc=com\" -w admin") + f'echo -e "{ldif}" | ldapmodify -x -H ldap://localhost -D "cn=admin,dc=company,dc=com" -w admin' + ) if exitcode is not None: assert r.exitcode == exitcode, error() -def create_xml_config_content(entries, config_d_dir="/etc/clickhouse-server/config.d", - config_file="ldap_external_user_directories.xml"): - """Create XML configuration file from a dictionary. - """ - uid = getuid() - path = os.path.join(config_d_dir, config_file) - name = config_file - root = xmltree.Element("yandex") - root.append(xmltree.Comment(text=f"config uid: {uid}")) - def create_xml_tree(entries, root): - for k,v in entries.items(): - if type(v) is dict: - xml_element = xmltree.Element(k) - create_xml_tree(v, xml_element) - root.append(xml_element) - elif type(v) in (list, tuple): - xml_element = xmltree.Element(k) - for e in v: - create_xml_tree(e, xml_element) - root.append(xml_element) - else: - xml_append(root, k, v) +def create_ldap_external_user_directory_config_content( + server=None, roles=None, role_mappings=None, **kwargs +): + """Create LDAP external user directory configuration file content.""" + kwargs["config_file"] = kwargs.pop( + "config_file", "external_ldap_user_directory.xml" + ) - create_xml_tree(entries, root) - xml_indent(root) - content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") - - return Config(content, path, name, uid, "config.xml") - -def create_ldap_external_user_directory_config_content(server=None, roles=None, role_mappings=None, **kwargs): - """Create LDAP external user directory configuration file content. - """ - entries = { - "user_directories": { - "ldap": { - } - } - } + entries = {"user_directories": {"ldap": {}}} entries["user_directories"]["ldap"] = [] @@ -239,7 +274,9 @@ def create_ldap_external_user_directory_config_content(server=None, roles=None, entries["user_directories"]["ldap"].append({"server": server}) if roles: - entries["user_directories"]["ldap"].append({"roles": [{r: None} for r in roles]}) + entries["user_directories"]["ldap"].append( + {"roles": [{r: None} for r in roles]} + ) if role_mappings: for role_mapping in role_mappings: @@ -247,7 +284,10 @@ def create_ldap_external_user_directory_config_content(server=None, roles=None, return create_xml_config_content(entries, **kwargs) + def create_entries_ldap_external_user_directory_config_content(entries, **kwargs): - """Create LDAP external user directory configuration file content. - """ + """Create LDAP external user directory configuration file content.""" + kwargs["config_file"] = kwargs.pop( + "config_file", "external_ldap_user_directory.xml" + ) return create_xml_config_content(entries, **kwargs) diff --git a/tests/testflows/ldap/role_mapping/tests/mapping.py b/tests/testflows/ldap/role_mapping/tests/mapping.py index 4f018d05aff..b74e3a073fe 100644 --- a/tests/testflows/ldap/role_mapping/tests/mapping.py +++ b/tests/testflows/ldap/role_mapping/tests/mapping.py @@ -2,28 +2,32 @@ from testflows.core import * from testflows.asserts import error -from helpers.common import Pool - from ldap.role_mapping.requirements import * from ldap.role_mapping.tests.common import * from ldap.external_user_directory.tests.common import randomword -from ldap.external_user_directory.tests.authentications import login_with_valid_username_and_password -from ldap.external_user_directory.tests.authentications import login_with_invalid_username_and_valid_password -from ldap.external_user_directory.tests.authentications import login_with_valid_username_and_invalid_password +from ldap.external_user_directory.tests.authentications import ( + login_with_valid_username_and_password, +) +from ldap.external_user_directory.tests.authentications import ( + login_with_invalid_username_and_valid_password, +) +from ldap.external_user_directory.tests.authentications import ( + login_with_valid_username_and_invalid_password, +) + def remove_ldap_groups_in_parallel(groups, i, iterations=10): - """Remove LDAP groups. - """ + """Remove LDAP groups.""" with When(f"LDAP groups are removed #{i}"): for j in range(iterations): for group in groups: with When(f"I delete group #{j}", description=f"{group}"): delete_group_from_ldap(group, exitcode=None) + def add_ldap_groups_in_parallel(ldap_user, names, i, iterations=10): - """Add LDAP groups. - """ + """Add LDAP groups.""" with When(f"LDAP groups are added #{i}"): for j in range(iterations): for name in names: @@ -31,48 +35,53 @@ def add_ldap_groups_in_parallel(ldap_user, names, i, iterations=10): group = add_group_to_ldap(cn=name, exitcode=None) with When(f"I add user to the group"): - add_user_to_group_in_ldap(user=ldap_user, group=group, exitcode=None) + add_user_to_group_in_ldap( + user=ldap_user, group=group, exitcode=None + ) + def add_user_to_ldap_groups_in_parallel(ldap_user, groups, i, iterations=10): - """Add user to LDAP groups. - """ + """Add user to LDAP groups.""" with When(f"user is added to LDAP groups #{i}"): for j in range(iterations): for group in groups: with When(f"I add user to the group {group['dn']} #{j}"): - add_user_to_group_in_ldap(user=ldap_user, group=group, exitcode=None) + add_user_to_group_in_ldap( + user=ldap_user, group=group, exitcode=None + ) + def remove_user_from_ldap_groups_in_parallel(ldap_user, groups, i, iterations=10): - """Remove user from LDAP groups. - """ + """Remove user from LDAP groups.""" with When(f"user is removed from LDAP groups #{i}"): for j in range(iterations): for group in groups: with When(f"I remove user from the group {group['dn']} #{j}"): - delete_user_from_group_in_ldap(user=ldap_user, group=group, exitcode=None) + delete_user_from_group_in_ldap( + user=ldap_user, group=group, exitcode=None + ) + def add_roles_in_parallel(role_names, i, iterations=10): - """Add roles. - """ + """Add roles.""" with When(f"roles are added #{i}"): for j in range(iterations): for role_name in role_names: with When(f"I add role {role_name} #{j}"): current().context.node.query(f"CREATE ROLE OR REPLACE {role_name}") + def remove_roles_in_parallel(role_names, i, iterations=10): - """Remove roles. - """ + """Remove roles.""" with When(f"roles are removed #{i}"): for j in range(iterations): for role_name in role_names: with When(f"I remove role {role_name} #{j}"): current().context.node.query(f"DROP ROLE IF EXISTS {role_name}") + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Map_MultipleRoles("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Map_MultipleRoles("1.0")) def multiple_roles(self, ldap_server, ldap_user): """Check that users authenticated using LDAP external user directory can be assigned multiple LDAP mapped roles. @@ -84,12 +93,14 @@ def multiple_roles(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix":"" + "prefix": "", } ] with Given("I add LDAP groups"): - groups = add_ldap_groups(groups=({"cn": f"role0_{uid}"}, {"cn": f"role1_{uid}"})) + groups = add_ldap_groups( + groups=({"cn": f"role0_{uid}"}, {"cn": f"role1_{uid}"}) + ) with And("I add LDAP user to each LDAP group"): add_user_to_group_in_ldap(user=ldap_user, group=groups[0]) @@ -99,23 +110,30 @@ def multiple_roles(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"role0_{uid}", f"role1_{uid}")) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped LDAP roles"): with By(f"checking that first role is assigned", description=f"{roles[0]}"): assert roles[0] in r.output, error() - with And(f"checking that second role is also assigned", description=f"{roles[1]}"): + with And( + f"checking that second role is also assigned", description=f"{roles[1]}" + ): assert roles[1] in r.output, error() + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_WithFixedRoles("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_WithFixedRoles("1.0")) def with_fixed_roles(self, ldap_server, ldap_user): """Check that LDAP users can be assigned roles dynamically and statically using the `` section. @@ -129,7 +147,7 @@ def with_fixed_roles(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -146,12 +164,18 @@ def with_fixed_roles(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{fixed_role_name}",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, roles=roles, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, roles=roles, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped and fixed roles"): with By("checking that mapped role is assigned"): @@ -159,17 +183,19 @@ def with_fixed_roles(self, ldap_server, ldap_user): with And("checking that fixed role is assigned"): assert roles[0] in r.output, error() + @TestOutline -def map_role(self, role_name, ldap_server, ldap_user, rbac_role_name=None, role_mappings=None): - """Check that we can map a role with a given name. - """ +def map_role( + self, role_name, ldap_server, ldap_user, rbac_role_name=None, role_mappings=None +): + """Check that we can map a role with a given name.""" if role_mappings is None: role_mappings = [ { "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -186,45 +212,46 @@ def map_role(self, role_name, ldap_server, ldap_user, rbac_role_name=None, role_ roles = add_rbac_roles(roles=(f"'{rbac_role_name}'",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped LDAP role"): with By(f"checking that the role is assigned", description=f"{role_name}"): assert roles[0].strip("'") in r.output, error() + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithUTF8Characters("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithUTF8Characters("1.0")) def role_name_with_utf8_characters(self, ldap_server, ldap_user): - """Check that we can map a role that contains UTF8 characters. - """ + """Check that we can map a role that contains UTF8 characters.""" uid = getuid() role_name = f"role_{uid}_Gãńdåłf_Thê_Gręât" map_role(role_name=role_name, ldap_server=ldap_server, ldap_user=ldap_user) + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_Long("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_Long("1.0")) def role_name_with_more_than_128_characters(self, ldap_server, ldap_user): - """Check that we can map a role that contains more than 128 characters. - """ + """Check that we can map a role that contains more than 128 characters.""" uid = getuid() role_name = f"role_{uid}_{'r'*128}" map_role(role_name=role_name, ldap_server=ldap_server, ldap_user=ldap_user) + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithSpecialXMLCharacters("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Map_Role_Name_WithSpecialXMLCharacters("1.0")) def role_name_with_special_xml_characters(self, ldap_server, ldap_user): """Check that we can map a role that contains special XML characters that must be escaped. @@ -233,7 +260,13 @@ def role_name_with_special_xml_characters(self, ldap_server, ldap_user): role_name = f"role_{uid}_\\<\\>" rbac_role_name = f"role_{uid}_<>" - map_role(role_name=role_name, ldap_server=ldap_server, ldap_user=ldap_user, rbac_role_name=rbac_role_name) + map_role( + role_name=role_name, + ldap_server=ldap_server, + ldap_user=ldap_user, + rbac_role_name=rbac_role_name, + ) + @TestScenario @Requirements( @@ -247,22 +280,37 @@ def role_name_with_special_regex_characters(self, ldap_server, ldap_user): role_name = f"role_{uid}_\\+.?$" rbac_role_name = f"role_{uid}_+.?$" - map_role(role_name=role_name, ldap_server=ldap_server, ldap_user=ldap_user, rbac_role_name=rbac_role_name) + map_role( + role_name=role_name, + ldap_server=ldap_server, + ldap_user=ldap_user, + rbac_role_name=rbac_role_name, + ) + @TestOutline -def map_groups_with_prefixes(self, prefixes, group_names, role_names, - expected, not_expected, ldap_server, ldap_user): - """Check that we can map multiple groups to roles whith one or more prefixes. - """ +def map_groups_with_prefixes( + self, + prefixes, + group_names, + role_names, + expected, + not_expected, + ldap_server, + ldap_user, +): + """Check that we can map multiple groups to roles whith one or more prefixes.""" role_mappings = [] for prefix in prefixes: - role_mappings.append({ - "base_dn": "ou=groups,dc=company,dc=com", - "attribute": "cn", - "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": prefix - }) + role_mappings.append( + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", + "prefix": prefix, + } + ) with Given("I add LDAP group"): groups = add_ldap_groups(groups=({"cn": name} for name in group_names)) @@ -275,27 +323,40 @@ def map_groups_with_prefixes(self, prefixes, group_names, role_names, roles = add_rbac_roles(roles=(f"'{name}'" for name in role_names)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped roles"): - with By(f"checking that the roles are assigned", description=f"{', '.join(expected)}"): + with By( + f"checking that the roles are assigned", + description=f"{', '.join(expected)}", + ): for name in expected: assert name in r.output, error() with And("I expect the user not to have mapped roles"): - with By(f"checking that the roles are not assigned", description=f"{', '.join(not_expected)}"): + with By( + f"checking that the roles are not assigned", + description=f"{', '.join(not_expected)}", + ): for name in not_expected: assert name not in r.output, error() + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Syntax("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix("1.0"), ) def prefix_non_empty(self, ldap_server, ldap_user): """Check that only group names with specified prefix are mapped to roles @@ -304,31 +365,34 @@ def prefix_non_empty(self, ldap_server, ldap_user): uid = getuid() with Given("I define group names"): - group_names=[ - f"clickhouse_role_{uid}", - f"role0_{uid}" - ] + group_names = [f"clickhouse_role_{uid}", f"role0_{uid}"] with And("I define role names"): - role_names=[ - f"role_{uid}", - f"role0_{uid}" - ] + role_names = [f"role_{uid}", f"role0_{uid}"] with And("I define group prefixes to be mapped"): prefixes = ["clickhouse_"] with And("I define the expected mapped and not mapped roles"): - expected=[f"role_{uid}"] - not_expected=[f"role0_{uid}"] + expected = [f"role_{uid}"] + not_expected = [f"role0_{uid}"] + + map_groups_with_prefixes( + ldap_server=ldap_server, + ldap_user=ldap_user, + prefixes=prefixes, + group_names=group_names, + role_names=role_names, + expected=expected, + not_expected=not_expected, + ) - map_groups_with_prefixes(ldap_server=ldap_server, ldap_user=ldap_user, - prefixes=prefixes, group_names=group_names, role_names=role_names, - expected=expected, not_expected=not_expected) @TestScenario @Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_Default("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_Default( + "1.0" + ) ) def prefix_default_value(self, ldap_server, ldap_user): """Check that when prefix is not specified the default value of prefix @@ -345,108 +409,124 @@ def prefix_default_value(self, ldap_server, ldap_user): } ] - map_role(role_name=role_name, ldap_server=ldap_server, ldap_user=ldap_user, role_mappings=role_mappings) + map_role( + role_name=role_name, + ldap_server=ldap_server, + ldap_user=ldap_user, + role_mappings=role_mappings, + ) + @TestScenario @Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithUTF8Characters("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithUTF8Characters( + "1.0" + ) ) def prefix_with_utf8_characters(self, ldap_server, ldap_user): - """Check that we can map a role when prefix contains UTF8 characters. - """ + """Check that we can map a role when prefix contains UTF8 characters.""" uid = getuid() with Given("I define group names"): - group_names=[ - f"Gãńdåłf_Thê_Gręât_role_{uid}", - f"role0_{uid}" - ] + group_names = [f"Gãńdåłf_Thê_Gręât_role_{uid}", f"role0_{uid}"] with And("I define role names"): - role_names=[ - f"role_{uid}", - f"role0_{uid}" - ] + role_names = [f"role_{uid}", f"role0_{uid}"] with And("I define group prefixes to be mapped"): prefixes = ["Gãńdåłf_Thê_Gręât_"] with And("I define the expected mapped and not mapped roles"): - expected=[f"role_{uid}"] - not_expected=[f"role0_{uid}"] + expected = [f"role_{uid}"] + not_expected = [f"role0_{uid}"] + + map_groups_with_prefixes( + ldap_server=ldap_server, + ldap_user=ldap_user, + prefixes=prefixes, + group_names=group_names, + role_names=role_names, + expected=expected, + not_expected=not_expected, + ) - map_groups_with_prefixes(ldap_server=ldap_server, ldap_user=ldap_user, - prefixes=prefixes, group_names=group_names, role_names=role_names, - expected=expected, not_expected=not_expected) @TestScenario @Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SpecialCharactersEscaping("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialXMLCharacters("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SpecialCharactersEscaping( + "1.0" + ), + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialXMLCharacters( + "1.0" + ), ) def prefix_with_special_xml_characters(self, ldap_server, ldap_user): - """Check that we can map a role when prefix contains special XML characters. - """ + """Check that we can map a role when prefix contains special XML characters.""" uid = getuid() with Given("I define group names"): - group_names=[ - f"clickhouse\\<\\>_role_{uid}", - f"role0_{uid}" - ] + group_names = [f"clickhouse\\<\\>_role_{uid}", f"role0_{uid}"] with And("I define role names"): - role_names=[ - f"role_{uid}", - f"role0_{uid}" - ] + role_names = [f"role_{uid}", f"role0_{uid}"] with And("I define group prefixes to be mapped"): prefixes = ["clickhouse<>_"] with And("I define the expected mapped and not mapped roles"): - expected=[f"role_{uid}"] - not_expected=[f"role0_{uid}"] + expected = [f"role_{uid}"] + not_expected = [f"role0_{uid}"] + + map_groups_with_prefixes( + ldap_server=ldap_server, + ldap_user=ldap_user, + prefixes=prefixes, + group_names=group_names, + role_names=role_names, + expected=expected, + not_expected=not_expected, + ) - map_groups_with_prefixes(ldap_server=ldap_server, ldap_user=ldap_user, - prefixes=prefixes, group_names=group_names, role_names=role_names, - expected=expected, not_expected=not_expected) @TestScenario @Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialRegexCharacters("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Prefix_WithSpecialRegexCharacters( + "1.0" + ) ) def prefix_with_special_regex_characters(self, ldap_server, ldap_user): - """Check that we can map a role when prefix contains special regex characters. - """ + """Check that we can map a role when prefix contains special regex characters.""" uid = getuid() with Given("I define group names"): - group_names=[ - f"clickhouse\\+.?\\$_role_{uid}", - f"role0_{uid}" - ] + group_names = [f"clickhouse\\+.?\\$_role_{uid}", f"role0_{uid}"] with And("I define role names"): - role_names=[ - f"role_{uid}", - f"role0_{uid}" - ] + role_names = [f"role_{uid}", f"role0_{uid}"] with And("I define group prefixes to be mapped"): prefixes = ["clickhouse+.?\\$_"] with And("I define the expected mapped and not mapped roles"): - expected=[f"role_{uid}"] - not_expected=[f"role0_{uid}"] + expected = [f"role_{uid}"] + not_expected = [f"role0_{uid}"] + + map_groups_with_prefixes( + ldap_server=ldap_server, + ldap_user=ldap_user, + prefixes=prefixes, + group_names=group_names, + role_names=role_names, + expected=expected, + not_expected=not_expected, + ) - map_groups_with_prefixes(ldap_server=ldap_server, ldap_user=ldap_user, - prefixes=prefixes, group_names=group_names, role_names=role_names, - expected=expected, not_expected=not_expected) @TestScenario @Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections( + "1.0" + ) ) def multiple_sections_with_different_prefixes(self, ldap_server, ldap_user): """Check that we can map multiple roles with multiple role mapping sections @@ -455,34 +535,35 @@ def multiple_sections_with_different_prefixes(self, ldap_server, ldap_user): uid = getuid() with Given("I define group names"): - group_names=[ + group_names = [ f"clickhouse0_role0_{uid}", f"clickhouse1_role1_{uid}", - f"role2_{uid}" + f"role2_{uid}", ] with And("I define role names"): - role_names=[ - f"role0_{uid}", - f"role1_{uid}", - f"role2_{uid}" - ] + role_names = [f"role0_{uid}", f"role1_{uid}", f"role2_{uid}"] with And("I define group prefixes to be mapped"): prefixes = ["clickhouse0_", "clickhouse1_"] with And("I define the expected mapped and not mapped roles"): - expected=[f"role0_{uid}", f"role1_{uid}"] - not_expected=[f"role2_{uid}"] + expected = [f"role0_{uid}", f"role1_{uid}"] + not_expected = [f"role2_{uid}"] + + map_groups_with_prefixes( + ldap_server=ldap_server, + ldap_user=ldap_user, + prefixes=prefixes, + group_names=group_names, + role_names=role_names, + expected=expected, + not_expected=not_expected, + ) - map_groups_with_prefixes(ldap_server=ldap_server, ldap_user=ldap_user, - prefixes=prefixes, group_names=group_names, role_names=role_names, - expected=expected, not_expected=not_expected) @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_Removed("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_Removed("1.0")) def group_removed(self, ldap_server, ldap_user): """Check that roles are not mapped after the corresponding LDAP group is removed. @@ -495,7 +576,7 @@ def group_removed(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -510,12 +591,18 @@ def group_removed(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped LDAP role"): with By(f"checking that the role is assigned", description=f"{role_name}"): @@ -525,17 +612,21 @@ def group_removed(self, ldap_server, ldap_user): delete_group_from_ldap(group) with When(f"I login as an LDAP user after LDAP group is removed"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user not to have mapped LDAP role"): with By(f"checking that the role is not assigned", description=f"{role_name}"): assert role_name not in r.output, error() + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_UserRemoved("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_UserRemoved("1.0")) def user_removed_from_group(self, ldap_server, ldap_user): """Check that roles are not mapped after the user has been removed from the corresponding LDAP group. @@ -548,7 +639,7 @@ def user_removed_from_group(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -562,12 +653,18 @@ def user_removed_from_group(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped LDAP role"): with By(f"checking that the role is assigned", description=f"{role_name}"): @@ -577,17 +674,21 @@ def user_removed_from_group(self, ldap_server, ldap_user): delete_user_from_group_in_ldap(user=ldap_user, group=groups[0]) with And(f"I login as an LDAP user after user has been removed from the group"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user not to have mapped LDAP role"): with By(f"checking that the role is not assigned", description=f"{role_name}"): assert role_name not in r.output, error() + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NotPresent("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NotPresent("1.0")) def role_not_present(self, ldap_server, ldap_user): """Check that LDAP users can still be authenticated even if the mapped role is not present. @@ -600,7 +701,7 @@ def role_not_present(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -611,12 +712,19 @@ def role_not_present(self, ldap_server, ldap_user): add_user_to_group_in_ldap(user=ldap_user, group=groups[0]) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + no_checks=True, + ) with Then("I expect the login to succeed"): assert r.exitcode == 0, error() @@ -624,10 +732,9 @@ def role_not_present(self, ldap_server, ldap_user): with And("the user not to have any mapped LDAP role"): assert r.output == "", error() + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NotPresent("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NotPresent("1.0")) def add_new_role_not_present(self, ldap_server, ldap_user): """Check that LDAP user can still authenticate when the LDAP user is added to a new LDAP group that does not match any existing @@ -641,7 +748,7 @@ def add_new_role_not_present(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "clickhouse_" + "prefix": "clickhouse_", } ] @@ -655,12 +762,19 @@ def add_new_role_not_present(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + no_checks=True, + ) with Then("I expect the login to succeed"): assert r.exitcode == 0, error() @@ -669,14 +783,22 @@ def add_new_role_not_present(self, ldap_server, ldap_user): assert f"{role_name}" in r.output, error() with When("I add LDAP group that maps to unknown role"): - unknown_groups = add_ldap_groups(groups=({"cn": "clickhouse_" + role_name + "_unknown"},)) + unknown_groups = add_ldap_groups( + groups=({"cn": "clickhouse_" + role_name + "_unknown"},) + ) with And("I add LDAP user to the group that maps to unknown role"): add_user_to_group_in_ldap(user=ldap_user, group=unknown_groups[0]) with And(f"I again login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + no_checks=True, + ) with Then("I expect the login to succeed"): assert r.exitcode == 0, error() @@ -687,9 +809,17 @@ def add_new_role_not_present(self, ldap_server, ldap_user): with When("I add matching previously unknown RBAC role"): unknown_roles = add_rbac_roles(roles=(f"{role_name}_unknown",)) - with And(f"I again login as an LDAP user after previously unknown RBAC role has been added"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])], no_checks=True) + with And( + f"I again login as an LDAP user after previously unknown RBAC role has been added" + ): + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + no_checks=True, + ) with Then("I expect the login to succeed"): assert r.exitcode == 0, error() @@ -700,10 +830,11 @@ def add_new_role_not_present(self, ldap_server, ldap_user): with And("the user should have the previously unknown mapped LDAP role"): assert f"{role_name}_unknown" in r.output, error() + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Removed("1.0"), - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Readded("1.0") + RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Readded("1.0"), ) def role_removed_and_readded(self, ldap_server, ldap_user): """Check that when a mapped role is removed the privileges provided by the role @@ -718,7 +849,7 @@ def role_removed_and_readded(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] with Given("I add LDAP group"): @@ -731,21 +862,26 @@ def role_removed_and_readded(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I create a table for which the role will provide privilege"): - table_name = create_table(name=f"table_{uid}", - create_statement="CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") + table_name = create_table( + name=f"table_{uid}", + create_statement="CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) with And("I grant select privilege on the table to the role"): self.context.node.query(f"GRANT SELECT ON {table_name} TO {role_name}") with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as LDAP user using clickhouse-client"): with self.context.cluster.shell(node=self.context.node.name) as shell: with shell( - f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", - asynchronous=True, name="client") as client: + f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", + asynchronous=True, + name="client", + ) as client: client.app.expect("clickhouse1 :\) ") with When("I execute SHOW GRANTS"): @@ -769,12 +905,16 @@ def role_removed_and_readded(self, ldap_server, ldap_user): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") with Then("I expect to get not enough privileges error"): - client.app.expect(f"DB::Exception: {ldap_user['username']}: Not enough privileges.") + client.app.expect( + f"DB::Exception: {ldap_user['username']}: Not enough privileges." + ) client.app.expect("clickhouse1 :\) ") with When("I add the role that grant the privilege back"): self.context.node.query(f"CREATE ROLE {role_name}") - self.context.node.query(f"GRANT SELECT ON {table_name} TO {role_name}") + self.context.node.query( + f"GRANT SELECT ON {table_name} TO {role_name}" + ) with And("I execute select on the table after role is added back"): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") @@ -783,10 +923,11 @@ def role_removed_and_readded(self, ldap_server, ldap_user): client.app.expect("Ok\.") client.app.expect("clickhouse1 :\) ") + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_NewPrivilege("1.0"), - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_RemovedPrivilege("1.0") + RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_RemovedPrivilege("1.0"), ) def privilege_new_and_removed(self, ldap_server, ldap_user): """Check that when a new privilege is added to the mapped role @@ -802,7 +943,7 @@ def privilege_new_and_removed(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] with Given("I add LDAP group"): @@ -815,18 +956,23 @@ def privilege_new_and_removed(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I create a table for which the role will provide privilege"): - table_name = create_table(name=f"table_{uid}", - create_statement="CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") + table_name = create_table( + name=f"table_{uid}", + create_statement="CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as LDAP user using clickhouse-client"): with self.context.cluster.shell(node=self.context.node.name) as shell: with shell( - f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", - asynchronous=True, name="client") as client: + f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", + asynchronous=True, + name="client", + ) as client: client.app.expect("clickhouse1 :\) ") with When("I execute SHOW GRANTS"): @@ -836,15 +982,21 @@ def privilege_new_and_removed(self, ldap_server, ldap_user): client.app.expect(f"{role_name}") client.app.expect("clickhouse1 :\) ") - with And("I execute select on the table when the mapped role does not provide this privilege"): + with And( + "I execute select on the table when the mapped role does not provide this privilege" + ): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") with Then("I expect to get not enough privileges error"): - client.app.expect(f"DB::Exception: {ldap_user['username']}: Not enough privileges.") + client.app.expect( + f"DB::Exception: {ldap_user['username']}: Not enough privileges." + ) client.app.expect("clickhouse1 :\) ") with When("I grant select privilege on the table to the mapped role"): - self.context.node.query(f"GRANT SELECT ON {table_name} TO {role_name}") + self.context.node.query( + f"GRANT SELECT ON {table_name} TO {role_name}" + ) with And("I execute select on the table"): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") @@ -854,19 +1006,22 @@ def privilege_new_and_removed(self, ldap_server, ldap_user): client.app.expect("clickhouse1 :\) ") with When("I remove the privilege from the mapped role"): - self.context.node.query(f"REVOKE SELECT ON {table_name} FROM {role_name}") + self.context.node.query( + f"REVOKE SELECT ON {table_name} FROM {role_name}" + ) with And("I re-execute select on the table"): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") with Then("I expect to get not enough privileges error"): - client.app.expect(f"DB::Exception: {ldap_user['username']}: Not enough privileges.") + client.app.expect( + f"DB::Exception: {ldap_user['username']}: Not enough privileges." + ) client.app.expect("clickhouse1 :\) ") + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Added("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_Added("1.0")) def role_added(self, ldap_server, ldap_user): """Check that when the mapped role is not present during LDAP user authentication but is later added then the authenticated LDAP users is granted the privileges provided @@ -880,7 +1035,7 @@ def role_added(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] with Given("I add LDAP group"): @@ -890,18 +1045,23 @@ def role_added(self, ldap_server, ldap_user): add_user_to_group_in_ldap(user=ldap_user, group=groups[0]) with And("I create a table for which the role will provide privilege"): - table_name = create_table(name=f"table_{uid}", - create_statement="CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") + table_name = create_table( + name=f"table_{uid}", + create_statement="CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()", + ) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as LDAP user using clickhouse-client"): with self.context.cluster.shell(node=self.context.node.name) as shell: with shell( - f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", - asynchronous=True, name="client") as client: + f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", + asynchronous=True, + name="client", + ) as client: client.app.expect("clickhouse1 :\) ") with When("I execute SHOW GRANTS"): @@ -915,12 +1075,16 @@ def role_added(self, ldap_server, ldap_user): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") with Then("I expect to get not enough privileges error"): - client.app.expect(f"DB::Exception: {ldap_user['username']}: Not enough privileges.") + client.app.expect( + f"DB::Exception: {ldap_user['username']}: Not enough privileges." + ) client.app.expect("clickhouse1 :\) ") with When("I add the role that grant the privilege"): self.context.node.query(f"CREATE ROLE {role_name}") - self.context.node.query(f"GRANT SELECT ON {table_name} TO {role_name}") + self.context.node.query( + f"GRANT SELECT ON {table_name} TO {role_name}" + ) with And("I execute select on the table after role is added"): client.app.send(f"SELECT * FROM {table_name} LIMIT 1") @@ -929,13 +1093,11 @@ def role_added(self, ldap_server, ldap_user): client.app.expect("Ok\.") client.app.expect("clickhouse1 :\) ") + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_New("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_New("1.0")) def role_new(self, ldap_server, ldap_user): - """Check that no new roles can be granted to LDAP authenticated users. - """ + """Check that no new roles can be granted to LDAP authenticated users.""" uid = getuid() role_name = f"role_{uid}" @@ -944,7 +1106,7 @@ def role_new(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -955,23 +1117,32 @@ def role_new(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as LDAP user using clickhouse-client"): with self.context.cluster.shell(node=self.context.node.name) as shell: with shell( - f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", - asynchronous=True, name="client") as client: + f"TERM=dumb clickhouse client --user {ldap_user['username']} --password {ldap_user['password']}", + asynchronous=True, + name="client", + ) as client: client.app.expect("clickhouse1 :\) ") with When("I try to grant new role to user"): - self.context.node.query(f"GRANT {role_name} TO {ldap_user['username']}", - message=message, exitcode=exitcode) + self.context.node.query( + f"GRANT {role_name} TO {ldap_user['username']}", + message=message, + exitcode=exitcode, + ) + @TestScenario @Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections_IdenticalParameters("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections_IdenticalParameters( + "1.0" + ) ) def multiple_sections_with_identical_parameters(self, ldap_server, ldap_user): """Check behaviour when multiple role mapping sections @@ -985,7 +1156,7 @@ def multiple_sections_with_identical_parameters(self, ldap_server, ldap_user): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] * 4 @@ -999,22 +1170,29 @@ def multiple_sections_with_identical_parameters(self, ldap_server, ldap_user): roles = add_rbac_roles(roles=(f"{role_name}",)) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) with When(f"I login as an LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", ldap_user["username"]), ("password", ldap_user["password"])]) + r = self.context.node.query( + f"SHOW GRANTS", + settings=[ + ("user", ldap_user["username"]), + ("password", ldap_user["password"]), + ], + ) with Then("I expect the user to have mapped LDAP role"): with By(f"checking that the role is assigned", description=f"{role_name}"): assert roles[0].strip("'") in r.output, error() + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_RemovedAndAdded_Parallel("1.0") -) -def group_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, timeout=200): +@Requirements(RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_RemovedAndAdded_Parallel("1.0")) +def group_removed_and_added_in_parallel( + self, ldap_server, ldap_user, count=20, timeout=200 +): """Check that user can be authenticated successfully when LDAP groups are removed and added in parallel. """ @@ -1028,7 +1206,7 @@ def group_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -1045,17 +1223,48 @@ def group_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, add_rbac_roles(roles=role_names) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) tasks = [] with Pool(4) as pool: try: - with When("user try to login while LDAP groups are added and removed in parallel"): + with When( + "user try to login while LDAP groups are added and removed in parallel" + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(remove_ldap_groups_in_parallel, (groups, i, 10,))) - tasks.append(pool.submit(add_ldap_groups_in_parallel,(ldap_user, role_names, i, 10,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + remove_ldap_groups_in_parallel, + ( + groups, + i, + 10, + ), + ) + ) + tasks.append( + pool.submit( + add_ldap_groups_in_parallel, + ( + ldap_user, + role_names, + i, + 10, + ), + ) + ) finally: with Finally("it should work", flags=TE): for task in tasks: @@ -1065,11 +1274,14 @@ def group_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, for group in groups: delete_group_from_ldap(group, exitcode=None) + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_LDAP_Group_UserRemovedAndAdded_Parallel("1.0") ) -def user_removed_and_added_in_ldap_groups_in_parallel(self, ldap_server, ldap_user, count=20, timeout=200): +def user_removed_and_added_in_ldap_groups_in_parallel( + self, ldap_server, ldap_user, count=20, timeout=200 +): """Check that user can be authenticated successfully when it is removed and added from mapping LDAP groups in parallel. """ @@ -1083,7 +1295,7 @@ def user_removed_and_added_in_ldap_groups_in_parallel(self, ldap_server, ldap_us "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -1098,27 +1310,60 @@ def user_removed_and_added_in_ldap_groups_in_parallel(self, ldap_server, ldap_us add_rbac_roles(roles=role_names) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) tasks = [] with Pool(4) as pool: try: - with When("user try to login while user is added and removed from LDAP groups in parallel"): + with When( + "user try to login while user is added and removed from LDAP groups in parallel" + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(remove_user_from_ldap_groups_in_parallel, (ldap_user, groups, i, 1,))) - tasks.append(pool.submit(add_user_to_ldap_groups_in_parallel, (ldap_user, groups, i, 1,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + remove_user_from_ldap_groups_in_parallel, + ( + ldap_user, + groups, + i, + 1, + ), + ) + ) + tasks.append( + pool.submit( + add_user_to_ldap_groups_in_parallel, + ( + ldap_user, + groups, + i, + 1, + ), + ) + ) finally: with Finally("it should work", flags=TE): for task in tasks: task.result(timeout=timeout) + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_RemovedAndAdded_Parallel("1.0") -) -def roles_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, timeout=200): +@Requirements(RQ_SRS_014_LDAP_RoleMapping_RBAC_Role_RemovedAndAdded_Parallel("1.0")) +def roles_removed_and_added_in_parallel( + self, ldap_server, ldap_user, count=20, timeout=200 +): """Check that user can be authenticated successfully when roles that are mapped by the LDAP groups are removed and added in parallel. """ @@ -1132,7 +1377,7 @@ def roles_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "" + "prefix": "", } ] @@ -1148,17 +1393,47 @@ def roles_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, add_rbac_roles(roles=role_names) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) tasks = [] with Pool(4) as pool: try: - with When("user try to login while mapped roles are added and removed in parallel"): + with When( + "user try to login while mapped roles are added and removed in parallel" + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(remove_roles_in_parallel, (role_names, i, 10,))) - tasks.append(pool.submit(add_roles_in_parallel, (role_names, i, 10,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + remove_roles_in_parallel, + ( + role_names, + i, + 10, + ), + ) + ) + tasks.append( + pool.submit( + add_roles_in_parallel, + ( + role_names, + i, + 10, + ), + ) + ) finally: with Finally("it should work", flags=TE): for task in tasks: @@ -1169,15 +1444,21 @@ def roles_removed_and_added_in_parallel(self, ldap_server, ldap_user, count=20, with By(f"dropping role {role_name}", flags=TE): self.context.node.query(f"DROP ROLE IF EXISTS {role_name}") + @TestOutline -def parallel_login(self, ldap_server, ldap_user, user_count=10, timeout=200, role_count=10): +def parallel_login( + self, ldap_server, ldap_user, user_count=10, timeout=200, role_count=10 +): """Check that login of valid and invalid LDAP authenticated users with mapped roles works in parallel. """ uid = getuid() role_names = [f"role{i}_{uid}" for i in range(role_count)] - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + users = [ + {"cn": f"parallel_user{i}", "userpassword": randomword(20)} + for i in range(user_count) + ] groups = [{"cn": f"clickhouse_{role_name}"} for role_name in role_names] role_mappings = [ @@ -1185,7 +1466,7 @@ def parallel_login(self, ldap_server, ldap_user, user_count=10, timeout=200, rol "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "clickhouse_" + "prefix": "clickhouse_", } ] @@ -1204,89 +1485,171 @@ def parallel_login(self, ldap_server, ldap_user, user_count=10, timeout=200, rol add_rbac_roles(roles=role_names) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=role_mappings, restart=True + ) tasks = [] with Pool(4) as pool: try: - with When("users try to login in parallel", description=""" + with When( + "users try to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): - tasks.append(pool.submit(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(pool.submit(login_with_invalid_username_and_valid_password, (users, i, 50,))) + tasks.append( + pool.submit( + login_with_valid_username_and_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_valid_username_and_invalid_password, + ( + users, + i, + 50, + ), + ) + ) + tasks.append( + pool.submit( + login_with_invalid_username_and_valid_password, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0"), ) -def parallel_login_of_multiple_users(self, ldap_server, ldap_user, timeout=200, role_count=10): +def parallel_login_of_multiple_users( + self, ldap_server, ldap_user, timeout=200, role_count=10 +): """Check that valid and invalid logins of multiple LDAP authenticated users with mapped roles works in parallel. """ - parallel_login(user_count=10, ldap_user=ldap_user,ldap_server=ldap_server, - timeout=timeout, role_count=role_count) + parallel_login( + user_count=10, + ldap_user=ldap_user, + ldap_server=ldap_server, + timeout=timeout, + role_count=role_count, + ) + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_SameUser("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0"), ) -def parallel_login_of_the_same_user(self, ldap_server, ldap_user, timeout=200, role_count=10): +def parallel_login_of_the_same_user( + self, ldap_server, ldap_user, timeout=200, role_count=10 +): """Check that valid and invalid logins of the same LDAP authenticated user with mapped roles works in parallel. """ - parallel_login(user_count=10, ldap_user=ldap_user,ldap_server=ldap_server, - timeout=timeout, role_count=role_count) + parallel_login( + user_count=10, + ldap_user=ldap_user, + ldap_server=ldap_server, + timeout=timeout, + role_count=role_count, + ) + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_MultipleServers("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0"), ) -def parallel_login_of_ldap_users_with_multiple_servers(self, ldap_server, ldap_user, timeout=200): +def parallel_login_of_ldap_users_with_multiple_servers( + self, ldap_server, ldap_user, timeout=200 +): """Check that valid and invalid logins of multiple LDAP users that have mapped roles works in parallel using multiple LDAP external user directories. """ - parallel_login_with_multiple_servers(ldap_server=ldap_server, ldap_user=ldap_user, - user_count=10, role_count=10,timeout=timeout, with_ldap_users=True, with_local_users=False) + parallel_login_with_multiple_servers( + ldap_server=ldap_server, + ldap_user=ldap_user, + user_count=10, + role_count=10, + timeout=timeout, + with_ldap_users=True, + with_local_users=False, + ) + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0") + RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_ValidAndInvalid("1.0"), ) -def parallel_login_of_local_and_ldap_users_with_multiple_servers(self, ldap_server, ldap_user, timeout=200): +def parallel_login_of_local_and_ldap_users_with_multiple_servers( + self, ldap_server, ldap_user, timeout=200 +): """Check that valid and invalid logins of local users and LDAP users that have mapped roles works in parallel using multiple LDAP external user directories. """ - parallel_login_with_multiple_servers(ldap_server=ldap_server, ldap_user=ldap_user, - user_count=10, role_count=10, timeout=timeout, with_local_users=True, with_ldap_users=True) + parallel_login_with_multiple_servers( + ldap_server=ldap_server, + ldap_user=ldap_user, + user_count=10, + role_count=10, + timeout=timeout, + with_local_users=True, + with_ldap_users=True, + ) + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_LocalOnly("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_LocalOnly("1.0")) def parallel_login_of_local_users(self, ldap_server, ldap_user, timeout=200): """Check that valid and invalid logins of local users works in parallel when multiple LDAP external user directories with role mapping are configured. """ - parallel_login_with_multiple_servers(ldap_server=ldap_server, ldap_user=ldap_user, - user_count=10, role_count=10, timeout=timeout, with_local_users=True, with_ldap_users=False) + parallel_login_with_multiple_servers( + ldap_server=ldap_server, + ldap_user=ldap_user, + user_count=10, + role_count=10, + timeout=timeout, + with_local_users=True, + with_ldap_users=False, + ) + @TestOutline -def parallel_login_with_multiple_servers(self, ldap_server, ldap_user, user_count=10, - role_count=10, timeout=200, with_ldap_users=True, with_local_users=False): +def parallel_login_with_multiple_servers( + self, + ldap_server, + ldap_user, + user_count=10, + role_count=10, + timeout=200, + with_ldap_users=True, + with_local_users=False, +): """Check that login of valid and invalid local users or LDAP users that have mapped roles works in parallel using multiple LDAP external user directories. """ @@ -1304,50 +1667,69 @@ def parallel_login_with_multiple_servers(self, ldap_server, ldap_user, user_coun if with_ldap_users: with And("I define a group of users to be created on each LDAP server"): user_groups["openldap1_users"] = [ - {"cn": f"openldap1_parallel_user{i}_{uid}", "userpassword": randomword(20)} for i in range(user_count) + { + "cn": f"openldap1_parallel_user{i}_{uid}", + "userpassword": randomword(20), + } + for i in range(user_count) ] user_groups["openldap2_users"] = [ - {"cn": f"openldap2_parallel_user{i}_{uid}", "userpassword": randomword(20)} for i in range(user_count) + { + "cn": f"openldap2_parallel_user{i}_{uid}", + "userpassword": randomword(20), + } + for i in range(user_count) ] if with_local_users: with And("I define a group of local users to be created"): user_groups["local_users"] = [ - {"cn": f"local_parallel_user{i}_{uid}", "userpassword": randomword(20)} for i in range(user_count) + {"cn": f"local_parallel_user{i}_{uid}", "userpassword": randomword(20)} + for i in range(user_count) ] with And("I have a list of checks that I want to run for each user group"): checks = [ login_with_valid_username_and_password, login_with_valid_username_and_invalid_password, - login_with_invalid_username_and_valid_password + login_with_invalid_username_and_valid_password, ] - with And("I create config file to define LDAP external user directory for each LDAP server"): + with And( + "I create config file to define LDAP external user directory for each LDAP server" + ): entries = { "user_directories": [ - {"ldap": [ - {"server": "openldap1"}, - {"role_mappings" : [ + { + "ldap": [ + {"server": "openldap1"}, { - "base_dn": "ou=groups,dc=company,dc=com", - "attribute": "cn", - "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "clickhouse_" - } - ]} - ]}, - {"ldap": [ - {"server": "openldap2"}, - {"role_mappings": [ + "role_mappings": [ + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", + "prefix": "clickhouse_", + } + ] + }, + ] + }, + { + "ldap": [ + {"server": "openldap2"}, { - "base_dn": "ou=groups,dc=company,dc=com", - "attribute": "cn", - "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", - "prefix": "clickhouse_" - } - ]} - ]} + "role_mappings": [ + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": "(&(objectClass=groupOfUniqueNames)(uniquemember={bind_dn}))", + "prefix": "clickhouse_", + } + ] + }, + ] + }, ] } config = create_entries_ldap_external_user_directory_config_content(entries) @@ -1357,24 +1739,40 @@ def parallel_login_with_multiple_servers(self, ldap_server, ldap_user, user_coun if with_ldap_users: with And("I add LDAP users to each LDAP server"): - openldap1_users = add_ldap_users(users=user_groups["openldap1_users"], node=cluster.node("openldap1")) - openldap2_users = add_ldap_users(users=user_groups["openldap2_users"], node=cluster.node("openldap2")) + openldap1_users = add_ldap_users( + users=user_groups["openldap1_users"], node=cluster.node("openldap1") + ) + openldap2_users = add_ldap_users( + users=user_groups["openldap2_users"], node=cluster.node("openldap2") + ) with And("I add all LDAP groups to each LDAP server"): - openldap1_groups = add_ldap_groups(groups=groups, node=cluster.node("openldap1")) - openldap2_groups = add_ldap_groups(groups=groups, node=cluster.node("openldap2")) + openldap1_groups = add_ldap_groups( + groups=groups, node=cluster.node("openldap1") + ) + openldap2_groups = add_ldap_groups( + groups=groups, node=cluster.node("openldap2") + ) with And("I add all users to LDAP groups on the first LDAP server"): for group in openldap1_groups: for user in openldap1_users: - with By(f"adding LDAP user {user['dn']} to the group {group['dn']}"): - add_user_to_group_in_ldap(user=user, group=group, node=cluster.node("openldap1")) + with By( + f"adding LDAP user {user['dn']} to the group {group['dn']}" + ): + add_user_to_group_in_ldap( + user=user, group=group, node=cluster.node("openldap1") + ) with And("I add all users to LDAP groups on the second LDAP server"): for group in openldap2_groups: for user in openldap2_users: - with By(f"adding LDAP user {user['dn']} to the group {group['dn']}"): - add_user_to_group_in_ldap(user=user, group=group, node=cluster.node("openldap2")) + with By( + f"adding LDAP user {user['dn']} to the group {group['dn']}" + ): + add_user_to_group_in_ldap( + user=user, group=group, node=cluster.node("openldap2") + ) with And("I add RBAC roles"): add_rbac_roles(roles=role_names) @@ -1391,28 +1789,38 @@ def parallel_login_with_multiple_servers(self, ldap_server, ldap_user, user_coun tasks = [] with Pool(4) as pool: try: - with When("users in each group try to login in parallel", description=""" + with When( + "users in each group try to login in parallel", + description=""" * with valid username and password * with invalid username and valid password * with valid username and invalid password - """): + """, + ): for i in range(10): for users in user_groups.values(): for check in checks: - tasks.append(pool.submit(check, (users, i, 50,))) + tasks.append( + pool.submit( + check, + ( + users, + i, + 50, + ), + ) + ) finally: with Then("it should work"): for task in tasks: task.result(timeout=timeout) + @TestFeature @Name("mapping") -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Search("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Search("1.0")) def feature(self): - """Check role LDAP role mapping. - """ + """Check role LDAP role mapping.""" self.context.node = self.context.cluster.node("clickhouse1") self.context.ldap_node = self.context.cluster.node("openldap1") @@ -1421,7 +1829,7 @@ def feature(self): "host": "openldap1", "port": "389", "enable_tls": "no", - "bind_dn": "cn={user_name},ou=users,dc=company,dc=com" + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", }, "openldap2": { "host": "openldap2", @@ -1429,12 +1837,17 @@ def feature(self): "enable_tls": "yes", "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "tls_require_cert": "never", - } + }, } users = [ - {"server": "openldap1", "username": "user1", "password": "user1", "login": True, - "dn": "cn=user1,ou=users,dc=company,dc=com"}, + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "dn": "cn=user1,ou=users,dc=company,dc=com", + }, ] with Given("I fix LDAP access permissions"): diff --git a/tests/testflows/ldap/role_mapping/tests/server_config.py b/tests/testflows/ldap/role_mapping/tests/server_config.py index 8008d9003d7..b9d308d3833 100644 --- a/tests/testflows/ldap/role_mapping/tests/server_config.py +++ b/tests/testflows/ldap/role_mapping/tests/server_config.py @@ -6,47 +6,55 @@ from ldap.role_mapping.requirements import * from ldap.authentication.tests.common import invalid_server_config from ldap.external_user_directory.tests.common import login + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN("1.0")) def valid_bind_dn(self): - """Check that LDAP users can login when `bind_dn` is valid. - """ + """Check that LDAP users can login when `bind_dn` is valid.""" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "bind_dn": "cn={user_name},ou=users,dc=company,dc=com" + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", } } user = { - "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, } login(servers, "openldap1", user) + @TestScenario -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN("1.0")) def invalid_bind_dn(self): - """Check that LDAP users can't login when `bind_dn` is invalid. - """ + """Check that LDAP users can't login when `bind_dn` is invalid.""" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", - "bind_dn": "cn={user_name},ou=users,dc=company2,dc=com" - }} + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company2,dc=com", + } + } user = { - "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, "exitcode": 4, - "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.", } login(servers, "openldap1", user) + @TestScenario @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN_ConflictWith_AuthDN("1.0") @@ -58,10 +66,12 @@ def bind_dn_conflict_with_auth_dn(self, timeout=60): message = "DB::Exception: Deprecated 'auth_dn_prefix' and 'auth_dn_suffix' entries cannot be used with 'bind_dn' entry" servers = { "openldap1": { - "host": "openldap1", "port": "389", "enable_tls": "no", + "host": "openldap1", + "port": "389", + "enable_tls": "no", "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "auth_dn_prefix": "cn=", - "auth_dn_suffix": ",ou=users,dc=company,dc=com" + "auth_dn_suffix": ",ou=users,dc=company,dc=com", } } @@ -71,8 +81,7 @@ def bind_dn_conflict_with_auth_dn(self, timeout=60): @TestFeature @Name("server config") def feature(self, node="clickhouse1"): - """Check LDAP server configuration. - """ + """Check LDAP server configuration.""" self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): scenario() diff --git a/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py index 147da8a5dcc..aa81f235108 100644 --- a/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py +++ b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py @@ -7,10 +7,12 @@ from testflows.asserts import error from ldap.role_mapping.requirements import * from ldap.role_mapping.tests.common import * + @TestOutline -def check_config(self, entries, valid=True, ldap_server="openldap1", user="user1", password="user1"): - """Apply LDAP server configuration and check login. - """ +def check_config( + self, entries, valid=True, ldap_server="openldap1", user="user1", password="user1" +): + """Apply LDAP server configuration and check login.""" if valid: exitcode = 0 message = "1" @@ -19,16 +21,24 @@ def check_config(self, entries, valid=True, ldap_server="openldap1", user="user1 message = "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" with Given("I add LDAP server configuration"): - config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + config = create_xml_config_content( + entries=entries, config_file="ldap_servers.xml" + ) add_ldap_servers_configuration(servers=None, config=config) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=ldap_server, - role_mappings=None, restart=True) + add_ldap_external_user_directory( + server=ldap_server, role_mappings=None, restart=True + ) with When(f"I login I try to login as an LDAP user"): - r = self.context.node.query(f"SELECT 1", settings=[ - ("user", user), ("password", password)], exitcode=exitcode, message=message) + r = self.context.node.query( + f"SELECT 1", + settings=[("user", user), ("password", password)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Tags("config") @@ -36,8 +46,7 @@ def check_config(self, entries, valid=True, ldap_server="openldap1", user="user1 RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") ) def config_invalid_base_dn(self): - """Check when invalid `base_dn` is specified in the user_dn_detection section. - """ + """Check when invalid `base_dn` is specified in the user_dn_detection section.""" with Given("I define LDAP server configuration with invalid base_dn"): entries = { @@ -50,8 +59,8 @@ def config_invalid_base_dn(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "ou=user,dc=company,dc=com", - "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + }, } } ] @@ -59,14 +68,14 @@ def config_invalid_base_dn(self): check_config(entries=entries, valid=False) + @TestScenario @Tags("config") @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") ) def config_empty_base_dn(self): - """Check when empty `base_dn` is specified in the user_dn_detection section. - """ + """Check when empty `base_dn` is specified in the user_dn_detection section.""" with Given("I define LDAP server configuration with invalid base_dn"): entries = { "ldap_servers": [ @@ -78,8 +87,8 @@ def config_empty_base_dn(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "", - "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + }, } } ] @@ -87,14 +96,14 @@ def config_empty_base_dn(self): check_config(entries=entries, valid=False) + @TestScenario @Tags("config") @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") ) def config_missing_base_dn(self): - """Check when missing `base_dn` is specified in the user_dn_detection section. - """ + """Check when missing `base_dn` is specified in the user_dn_detection section.""" with Given("I define LDAP server configuration with invalid base_dn"): entries = { "ldap_servers": [ @@ -106,7 +115,7 @@ def config_missing_base_dn(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } + }, } } ] @@ -114,14 +123,14 @@ def config_missing_base_dn(self): check_config(entries=entries, valid=False) + @TestScenario @Tags("config") @Requirements( # FIXME ) def config_invalid_search_filter(self): - """Check when invalid `search_filter` is specified in the user_dn_detection section. - """ + """Check when invalid `search_filter` is specified in the user_dn_detection section.""" with Given("I define LDAP server configuration with invalid search_filter"): entries = { "ldap_servers": [ @@ -133,8 +142,8 @@ def config_invalid_search_filter(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - "search_filter": "(&(objectClass=inetOrgPersons)(uid={user_name}))" - } + "search_filter": "(&(objectClass=inetOrgPersons)(uid={user_name}))", + }, } } ] @@ -142,14 +151,14 @@ def config_invalid_search_filter(self): check_config(entries=entries, valid=False) + @TestScenario @Tags("config") @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") ) def config_missing_search_filter(self): - """Check when missing `search_filter` is specified in the user_dn_detection section. - """ + """Check when missing `search_filter` is specified in the user_dn_detection section.""" with Given("I define LDAP server configuration with invalid search_filter"): entries = { "ldap_servers": [ @@ -161,7 +170,7 @@ def config_missing_search_filter(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - } + }, } } ] @@ -169,14 +178,14 @@ def config_missing_search_filter(self): check_config(entries=entries, valid=False) + @TestScenario @Tags("config") @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") ) def config_empty_search_filter(self): - """Check when empty `search_filter` is specified in the user_dn_detection section. - """ + """Check when empty `search_filter` is specified in the user_dn_detection section.""" with Given("I define LDAP server configuration with invalid search_filter"): entries = { "ldap_servers": [ @@ -188,8 +197,8 @@ def config_empty_search_filter(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - "search_filter": "" - } + "search_filter": "", + }, } } ] @@ -197,15 +206,17 @@ def config_empty_search_filter(self): check_config(entries=entries, valid=False) + @TestScenario @Tags("config") @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter( + "1.0" + ), ) def config_valid(self): - """Check valid config with valid user_dn_detection section. - """ + """Check valid config with valid user_dn_detection section.""" with Given("I define LDAP server configuration"): entries = { "ldap_servers": [ @@ -217,8 +228,8 @@ def config_valid(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + }, } } ] @@ -226,11 +237,14 @@ def config_valid(self): check_config(entries=entries, valid=True) + @TestScenario @Tags("config") @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"), - RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter( + "1.0" + ), ) def config_valid_tls_connection(self): """Check valid config with valid user_dn_detection section when @@ -248,28 +262,37 @@ def config_valid_tls_connection(self): "tls_require_cert": "never", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + }, } } ] } - check_config(entries=entries, valid=True, ldap_server="openldap2", user="user2", password="user2") + check_config( + entries=entries, + valid=True, + ldap_server="openldap2", + user="user2", + password="user2", + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope("1.0") ) -@Examples("scope base_dn", [ - ("base", "cn=user1,ou=users,dc=company,dc=com"), - ("one_level","ou=users,dc=company,dc=com"), - ("children","ou=users,dc=company,dc=com"), - ("subtree","ou=users,dc=company,dc=com") # default value -]) +@Examples( + "scope base_dn", + [ + ("base", "cn=user1,ou=users,dc=company,dc=com"), + ("one_level", "ou=users,dc=company,dc=com"), + ("children", "ou=users,dc=company,dc=com"), + ("subtree", "ou=users,dc=company,dc=com"), # default value + ], +) def check_valid_scope_values(self, scope, base_dn): - """Check configuration with valid scope values. - """ + """Check configuration with valid scope values.""" with Given("I define LDAP server configuration"): entries = { "ldap_servers": [ @@ -282,8 +305,8 @@ def check_valid_scope_values(self, scope, base_dn): "user_dn_detection": { "base_dn": base_dn, "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", - "scope": scope - } + "scope": scope, + }, } } ] @@ -291,6 +314,7 @@ def check_valid_scope_values(self, scope, base_dn): check_config(entries=entries, valid=True) + @TestSuite def mapping(self): """Run all role mapping tests with both @@ -298,8 +322,13 @@ def mapping(self): user DN detection. """ users = [ - {"server": "openldap1", "username": "user1", "password": "user1", "login": True, - "dn": "cn=user1,ou=users,dc=company,dc=com"}, + { + "server": "openldap1", + "username": "user1", + "password": "user1", + "login": True, + "dn": "cn=user1,ou=users,dc=company,dc=com", + }, ] entries = { @@ -312,8 +341,8 @@ def mapping(self): "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + }, }, "openldap2": { "host": "openldap2", @@ -323,20 +352,25 @@ def mapping(self): "tls_require_cert": "never", "user_dn_detection": { "base_dn": "ou=users,dc=company,dc=com", - "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" - } - } + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + }, + }, }, ] } with Given("I add LDAP servers configuration"): - config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + config = create_xml_config_content( + entries=entries, config_file="ldap_servers.xml" + ) add_ldap_servers_configuration(servers=None, config=config) - for scenario in loads(importlib.import_module("ldap.role_mapping.tests.mapping"), Scenario): + for scenario in loads( + importlib.import_module("ldap.role_mapping.tests.mapping"), Scenario + ): scenario(ldap_server="openldap1", ldap_user=users[0]) + @TestOutline def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection): """Check that roles get mapped properly when bind_dn and user_dn are different @@ -361,7 +395,7 @@ def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection): entries["ldap_servers"][0]["openldap1"]["user_dn_detection"] = { "base_dn": "ou=users,dc=company,dc=com", "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", - "scope": "subtree" + "scope": "subtree", } with And("I define role mappings"): @@ -370,21 +404,23 @@ def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection): "base_dn": "ou=groups,dc=company,dc=com", "attribute": "cn", "search_filter": f"(&(objectClass=groupOfUniqueNames)(uniquemember={{{map_by}}}))", - "prefix":"" + "prefix": "", } ] with Given("I add LDAP users"): - first_user = add_ldap_users(users=[ - {"cn": f"first_user", "userpassword": "user", "uid": "second_user"} - ])[0] + first_user = add_ldap_users( + users=[{"cn": f"first_user", "userpassword": "user", "uid": "second_user"}] + )[0] - second_user = add_ldap_users(users=[ - {"cn": f"second_user", "userpassword": "user", "uid": "first_user"} - ])[0] + second_user = add_ldap_users( + users=[{"cn": f"second_user", "userpassword": "user", "uid": "first_user"}] + )[0] with Given("I add LDAP groups"): - groups = add_ldap_groups(groups=({"cn": f"role0_{uid}"}, {"cn": f"role1_{uid}"})) + groups = add_ldap_groups( + groups=({"cn": f"role0_{uid}"}, {"cn": f"role1_{uid}"}) + ) with And("I add LDAP user to each LDAP group"): with By("adding first group to first user"): @@ -396,12 +432,18 @@ def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection): roles = add_rbac_roles(roles=(f"role0_{uid}", f"role1_{uid}")) with Given("I add LDAP server configuration"): - config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + config = create_xml_config_content( + entries=entries, config_file="ldap_servers.xml" + ) add_ldap_servers_configuration(servers=None, config=config) with And("I add LDAP external user directory configuration"): - add_ldap_external_user_directory(server=self.context.ldap_node.name, - role_mappings=role_mappings, restart=True) + add_ldap_external_user_directory( + server=self.context.ldap_node.name, + role_mappings=role_mappings, + restart=True, + ) + @TestScenario def map_roles_by_user_dn_when_base_dn_and_user_dn_are_different(self): @@ -414,22 +456,27 @@ def map_roles_by_user_dn_when_base_dn_and_user_dn_are_different(self): """ uid = getuid() - setup_different_bind_dn_and_user_dn(uid=uid, map_by="user_dn", user_dn_detection=True) + setup_different_bind_dn_and_user_dn( + uid=uid, map_by="user_dn", user_dn_detection=True + ) with When(f"I login as first LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", "first_user"), ("password", "user")]) + r = self.context.node.query( + f"SHOW GRANTS", settings=[("user", "first_user"), ("password", "user")] + ) with Then("I expect the first user to have mapped LDAP roles from second user"): assert f"GRANT role1_{uid} TO first_user" in r.output, error() with When(f"I login as second LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", "second_user"), ("password", "user")]) + r = self.context.node.query( + f"SHOW GRANTS", settings=[("user", "second_user"), ("password", "user")] + ) with Then("I expect the second user to have mapped LDAP roles from first user"): assert f"GRANT role0_{uid} TO second_user" in r.output, error() + @TestScenario def map_roles_by_bind_dn_when_base_dn_and_user_dn_are_different(self): """Check the case when we map roles by bind_dn when bind_dn and user_dn @@ -437,30 +484,32 @@ def map_roles_by_bind_dn_when_base_dn_and_user_dn_are_different(self): """ uid = getuid() - setup_different_bind_dn_and_user_dn(uid=uid, map_by="bind_dn", user_dn_detection=True) + setup_different_bind_dn_and_user_dn( + uid=uid, map_by="bind_dn", user_dn_detection=True + ) with When(f"I login as first LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", "first_user"), ("password", "user")]) + r = self.context.node.query( + f"SHOW GRANTS", settings=[("user", "first_user"), ("password", "user")] + ) with Then("I expect the first user to have no mapped LDAP roles"): assert f"GRANT role0_{uid} TO first_user" == r.output, error() with When(f"I login as second LDAP user"): - r = self.context.node.query(f"SHOW GRANTS", settings=[ - ("user", "second_user"), ("password", "user")]) + r = self.context.node.query( + f"SHOW GRANTS", settings=[("user", "second_user"), ("password", "user")] + ) with Then("I expect the second user to have no mapped LDAP roles"): assert f"GRANT role1_{uid} TO second_user" in r.output, error() + @TestFeature @Name("user dn detection") -@Requirements( - RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection("1.0") -) +@Requirements(RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection("1.0")) def feature(self): - """Check LDAP user DN detection. - """ + """Check LDAP user DN detection.""" self.context.node = self.context.cluster.node("clickhouse1") self.context.ldap_node = self.context.cluster.node("openldap1") diff --git a/tests/testflows/map_type/configs/clickhouse/config.xml b/tests/testflows/map_type/configs/clickhouse/config.xml deleted file mode 100644 index 842a0573d49..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/config.xml +++ /dev/null @@ -1,448 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - 0.0.0.0 - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/map_type/configs/clickhouse/users.xml b/tests/testflows/map_type/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/map_type/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py index 049585dea81..321a6944b2b 100755 --- a/tests/testflows/map_type/regression.py +++ b/tests/testflows/map_type/regression.py @@ -11,120 +11,162 @@ from helpers.argparser import argparser from map_type.requirements import SRS018_ClickHouse_Map_Data_Type xfails = { - "tests/table map with key integer/Int:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], - "tests/table map with value integer/Int:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], - "tests/table map with key integer/UInt256": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], - "tests/table map with value integer/UInt256": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], - "tests/select map with key integer/Int64": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], - "tests/select map with value integer/Int64": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030")], - "tests/cast tuple of two arrays to map/string -> int": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21029")], - "tests/mapcontains/null key in map": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], - "tests/mapcontains/null key not in map": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], - "tests/mapkeys/null key not in map": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], - "tests/mapkeys/null key in map": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028")], - "tests/mapcontains/select nullable key": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], - "tests/mapkeys/select keys from column": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], - "tests/table map select key with value string/LowCardinality:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], - "tests/table map select key with key string/FixedString": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], - "tests/table map select key with key string/Nullable": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], - "tests/table map select key with key string/Nullable(NULL)": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], - "tests/table map select key with key string/LowCardinality:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], - "tests/table map select key with key integer/Int:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032")], - "tests/table map select key with key integer/UInt256": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031")], - "tests/table map select key with key integer/toNullable": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406")], - "tests/table map select key with key integer/toNullable(NULL)": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026")], - "tests/select map with key integer/Int128": - [(Fail, "large Int128 as key not supported")], - "tests/select map with key integer/Int256": - [(Fail, "large Int256 as key not supported")], - "tests/select map with key integer/UInt256": - [(Fail, "large UInt256 as key not supported")], - "tests/select map with key integer/toNullable": - [(Fail, "Nullable type as key not supported")], - "tests/select map with key integer/toNullable(NULL)": - [(Fail, "Nullable type as key not supported")], - "tests/select map with key string/Nullable": - [(Fail, "Nullable type as key not supported")], - "tests/select map with key string/Nullable(NULL)": - [(Fail, "Nullable type as key not supported")], - "tests/table map queries/select map with nullable value": - [(Fail, "Nullable value not supported")], - "tests/table map with key integer/toNullable": - [(Fail, "Nullable type as key not supported")], - "tests/table map with key integer/toNullable(NULL)": - [(Fail, "Nullable type as key not supported")], - "tests/table map with key string/Nullable": - [(Fail, "Nullable type as key not supported")], - "tests/table map with key string/Nullable(NULL)": - [(Fail, "Nullable type as key not supported")], - "tests/table map with key string/LowCardinality(String)": - [(Fail, "LowCardinality(String) as key not supported")], - "tests/table map with key string/LowCardinality(String) cast from String": - [(Fail, "LowCardinality(String) as key not supported")], - "tests/table map with key string/LowCardinality(String) for key and value": - [(Fail, "LowCardinality(String) as key not supported")], - "tests/table map with key string/LowCardinality(FixedString)": - [(Fail, "LowCardinality(FixedString) as key not supported")], - "tests/table map with value string/LowCardinality(String) for key and value": - [(Fail, "LowCardinality(String) as key not supported")], + "tests/table map with key integer/Int:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032") + ], + "tests/table map with value integer/Int:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032") + ], + "tests/table map with key integer/UInt256": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031") + ], + "tests/table map with value integer/UInt256": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031") + ], + "tests/select map with key integer/Int64": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030") + ], + "tests/select map with value integer/Int64": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21030") + ], + "tests/cast tuple of two arrays to map/string -> int": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21029") + ], + "tests/mapcontains/null key in map": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028") + ], + "tests/mapcontains/null key not in map": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028") + ], + "tests/mapkeys/null key not in map": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028") + ], + "tests/mapkeys/null key in map": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21028") + ], + "tests/mapcontains/select nullable key": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026") + ], + "tests/mapkeys/select keys from column": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026") + ], + "tests/table map select key with value string/LowCardinality:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406") + ], + "tests/table map select key with key string/FixedString": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406") + ], + "tests/table map select key with key string/Nullable": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406") + ], + "tests/table map select key with key string/Nullable(NULL)": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026") + ], + "tests/table map select key with key string/LowCardinality:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406") + ], + "tests/table map select key with key integer/Int:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21032") + ], + "tests/table map select key with key integer/UInt256": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21031") + ], + "tests/table map select key with key integer/toNullable": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21406") + ], + "tests/table map select key with key integer/toNullable(NULL)": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/21026") + ], + "tests/select map with key integer/Int128": [ + (Fail, "large Int128 as key not supported") + ], + "tests/select map with key integer/Int256": [ + (Fail, "large Int256 as key not supported") + ], + "tests/select map with key integer/UInt256": [ + (Fail, "large UInt256 as key not supported") + ], + "tests/select map with key integer/toNullable": [ + (Fail, "Nullable type as key not supported") + ], + "tests/select map with key integer/toNullable(NULL)": [ + (Fail, "Nullable type as key not supported") + ], + "tests/select map with key string/Nullable": [ + (Fail, "Nullable type as key not supported") + ], + "tests/select map with key string/Nullable(NULL)": [ + (Fail, "Nullable type as key not supported") + ], + "tests/table map queries/select map with nullable value": [ + (Fail, "Nullable value not supported") + ], + "tests/table map with key integer/toNullable": [ + (Fail, "Nullable type as key not supported") + ], + "tests/table map with key integer/toNullable(NULL)": [ + (Fail, "Nullable type as key not supported") + ], + "tests/table map with key string/Nullable": [ + (Fail, "Nullable type as key not supported") + ], + "tests/table map with key string/Nullable(NULL)": [ + (Fail, "Nullable type as key not supported") + ], + "tests/table map with key string/LowCardinality(String)": [ + (Fail, "LowCardinality(String) as key not supported") + ], + "tests/table map with key string/LowCardinality(String) cast from String": [ + (Fail, "LowCardinality(String) as key not supported") + ], + "tests/table map with key string/LowCardinality(String) for key and value": [ + (Fail, "LowCardinality(String) as key not supported") + ], + "tests/table map with key string/LowCardinality(FixedString)": [ + (Fail, "LowCardinality(FixedString) as key not supported") + ], + "tests/table map with value string/LowCardinality(String) for key and value": [ + (Fail, "LowCardinality(String) as key not supported") + ], # JSON related - "tests/table map with duplicated keys/Map(Int64, String))": - [(Fail, "new bug due to JSON changes")], - "tests/table map with key integer/UInt64": - [(Fail, "new bug due to JSON changes")], - "tests/table map with value integer/UInt64": - [(Fail, "new bug due to JSON changes")] + "tests/table map with duplicated keys/Map(Int64, String))": [ + (Fail, "new bug due to JSON changes") + ], + "tests/table map with key integer/UInt64": [(Fail, "new bug due to JSON changes")], + "tests/table map with value integer/UInt64": [ + (Fail, "new bug due to JSON changes") + ], } -xflags = { -} +xflags = {} + @TestModule @ArgumentParser(argparser) @XFails(xfails) @XFlags(xflags) @Name("map type") -@Specifications( - SRS018_ClickHouse_Map_Data_Type -) -def regression(self, local, clickhouse_binary_path, stress=None): - """Map type regression. - """ - nodes = { - "clickhouse": - ("clickhouse1", "clickhouse2", "clickhouse3") - } +@Specifications(SRS018_ClickHouse_Map_Data_Type) +def regression( + self, local, clickhouse_binary_path, clickhouser_version=None, stress=None +): + """Map type regression.""" + nodes = {"clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3")} if stress is not None: self.context.stress = stress + self.context.clickhouse_version = clickhouse_version - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "map_type_env")) as cluster: + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), "map_type_env"), + ) as cluster: self.context.cluster = cluster Feature(run=load("map_type.tests.feature", "feature")) + if main(): regression() diff --git a/tests/testflows/map_type/requirements/requirements.py b/tests/testflows/map_type/requirements/requirements.py index 7569f7cc177..d25c6149658 100644 --- a/tests/testflows/map_type/requirements/requirements.py +++ b/tests/testflows/map_type/requirements/requirements.py @@ -9,793 +9,831 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_018_ClickHouse_Map_DataType = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs.\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type that stores `key:value` pairs.\n" + "\n" + ), link=None, level=3, - num='3.1.1') + num="3.1.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' - 'compared to `Array(Tuple(K,V))` data type.\n' - '\n' - ), + "[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n" + "compared to `Array(Tuple(K,V))` data type.\n" + "\n" + ), link=None, level=3, - num='3.2.1') + num="3.2.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n' - 'compared to `Tuple(Array(String), Array(String))` data type where the first\n' - 'array defines an array of keys and the second array defines an array of values.\n' - '\n' - ), + "[ClickHouse] SHALL provide comparable performance for `Map(key, value)` data type as\n" + "compared to `Tuple(Array(String), Array(String))` data type where the first\n" + "array defines an array of keys and the second array defines an array of values.\n" + "\n" + ), link=None, level=3, - num='3.2.2') + num="3.2.2", +) RQ_SRS_018_ClickHouse_Map_DataType_Key_String = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Key.String", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type.\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type where key is of a [String] type.\n" + "\n" + ), link=None, level=3, - num='3.3.1') + num="3.3.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type.\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type where key is of an [Integer] type.\n" + "\n" + ), link=None, level=3, - num='3.3.2') + num="3.3.2", +) RQ_SRS_018_ClickHouse_Map_DataType_Value_String = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.String", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type.\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [String] type.\n" + "\n" + ), link=None, level=3, - num='3.4.1') + num="3.4.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type.\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Integer] type.\n" + "\n" + ), link=None, level=3, - num='3.4.2') + num="3.4.2", +) RQ_SRS_018_ClickHouse_Map_DataType_Value_Array = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Array", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type.\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type where value is of a [Array] type.\n" + "\n" + ), link=None, level=3, - num='3.4.3') + num="3.4.3", +) RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type.\n' - '\n' - ), + "[ClickHouse] SHALL not support creating table columns that have `Nullable(Map(key, value))` data type.\n" + "\n" + ), link=None, level=3, - num='3.5.1') + num="3.5.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type.\n' - '\n' - ), + "[ClickHouse] SHALL not support creating table columns that have `Map(Nothing, Nothing))` data type.\n" + "\n" + ), link=None, level=3, - num='3.5.2') + num="3.5.2", +) RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys.\n' - '\n' - ), + "[ClickHouse] MAY support `Map(key, value)` data type with duplicated keys.\n" + "\n" + ), link=None, level=3, - num='3.6.1') + num="3.6.1", +) RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Array(Map(key, value))` data type.\n' - '\n' - ), + "[ClickHouse] SHALL support `Array(Map(key, value))` data type.\n" "\n" + ), link=None, level=3, - num='3.7.1') + num="3.7.1", +) RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type.\n' - '\n' - ), + "[ClickHouse] SHALL support defining `Map(key, value)` data type inside the [Nested] data type.\n" + "\n" + ), link=None, level=3, - num='3.8.1') + num="3.8.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax.\n' - 'If `key` has duplicates then the first `key:value` pair MAY be returned. \n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support getting the value from a `Map(key, value)` data type using `map[key]` syntax.\n" + "If `key` has duplicates then the first `key:value` pair MAY be returned. \n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT a['key2'] FROM table_map;\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.9.1') + num="3.9.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when key does not match the key type.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT map(1,2) AS m, m[1024]\n' - '```\n' - '\n' - 'Exceptions:\n' - '\n' - '* when key is `NULL` the return value MAY be `NULL`\n' - '* when key value is not valid for the key type, for example it is out of range for [Integer] type, \n' - ' when reading from a table column it MAY return the default value for key data type\n' - '\n' - ), + "[ClickHouse] SHALL return an error when key does not match the key type.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT map(1,2) AS m, m[1024]\n" + "```\n" + "\n" + "Exceptions:\n" + "\n" + "* when key is `NULL` the return value MAY be `NULL`\n" + "* when key value is not valid for the key type, for example it is out of range for [Integer] type, \n" + " when reading from a table column it MAY return the default value for key data type\n" + "\n" + ), link=None, level=3, - num='3.9.2') + num="3.9.2", +) RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return default value for the data type of the value\n' + "[ClickHouse] SHALL return default value for the data type of the value\n" "when there's no corresponding `key` defined in the `Map(key, value)` data type. \n" - '\n' - '\n' - ), + "\n" + "\n" + ), link=None, level=3, - num='3.9.3') + num="3.9.3", +) RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function.\n' - '\n' - '``` sql\n' + "[ClickHouse] SHALL support converting [Tuple(Array, Array)] to `Map(key, value)` using the [CAST] function.\n" + "\n" + "``` sql\n" "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map;\n" - '```\n' - '\n' - '``` text\n' - '┌─map───────────────────────────┐\n' + "```\n" + "\n" + "``` text\n" + "┌─map───────────────────────────┐\n" "│ {1:'Ready',2:'Steady',3:'Go'} │\n" - '└───────────────────────────────┘\n' - '```\n' - '\n' - ), + "└───────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.10.1') + num="3.10.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)`\n' - '\n' - '* when arrays are not of equal size\n' - '\n' - ' For example,\n' - '\n' - ' ```sql\n' + "[ClickHouse] MAY return an error when casting [Tuple(Array, Array)] to `Map(key, value)`\n" + "\n" + "* when arrays are not of equal size\n" + "\n" + " For example,\n" + "\n" + " ```sql\n" " SELECT CAST(([2, 1, 1023], ['', '']), 'Map(UInt8, String)') AS map, map[10]\n" - ' ```\n' - '\n' - ), + " ```\n" + "\n" + ), link=None, level=3, - num='3.10.2') + num="3.10.2", +) RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support converting [Array(Tuple(K,V))] to `Map(key, value)` using the [CAST] function.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.11.1') + num="3.11.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n' - '\n' - '* when element is not a [Tuple]\n' - '\n' - ' ```sql\n' + "[ClickHouse] MAY return an error when casting [Array(Tuple(K, V))] to `Map(key, value)`\n" + "\n" + "* when element is not a [Tuple]\n" + "\n" + " ```sql\n" " SELECT CAST(([(1,2),(3)]), 'Map(UInt8, UInt8)') AS map\n" - ' ```\n' - '\n' - '* when [Tuple] does not contain two elements\n' - '\n' - ' ```sql\n' + " ```\n" + "\n" + "* when [Tuple] does not contain two elements\n" + "\n" + " ```sql\n" " SELECT CAST(([(1,2),(3,)]), 'Map(UInt8, UInt8)') AS map\n" - ' ```\n' - '\n' - ), + " ```\n" + "\n" + ), link=None, level=3, - num='3.11.2') + num="3.11.2", +) RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used \n' - 'to retrieve an [Array] of map keys.\n' - '\n' - '```sql\n' - 'SELECT m.keys FROM t_map;\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `keys` subcolumn in the `Map(key, value)` type that can be used \n" + "to retrieve an [Array] of map keys.\n" + "\n" + "```sql\n" + "SELECT m.keys FROM t_map;\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.12.1') + num="3.12.1", +) RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support applying [Array] functions to the `keys` subcolumn in the `Map(key, value)` type.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT * FROM t_map WHERE has(m.keys, 'a');\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.12.2') + num="3.12.2", +) RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_InlineDefinedMap = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] MAY not support using inline defined map to get `keys` subcolumn.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT map( 'aa', 4, '44' , 5) as c, c.keys\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.12.3') + num="3.12.3", +) RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used \n' - 'to retrieve an [Array] of map values.\n' - '\n' - '```sql\n' - 'SELECT m.values FROM t_map;\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `values` subcolumn in the `Map(key, value)` type that can be used \n" + "to retrieve an [Array] of map values.\n" + "\n" + "```sql\n" + "SELECT m.values FROM t_map;\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.12.4') + num="3.12.4", +) RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support applying [Array] functions to the `values` subcolumn in the `Map(key, value)` type.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT * FROM t_map WHERE has(m.values, 'a');\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.12.5') + num="3.12.5", +) RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_InlineDefinedMap = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] MAY not support using inline defined map to get `values` subcolumn.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] MAY not support using inline defined map to get `values` subcolumn.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT map( 'aa', 4, '44' , 5) as c, c.values\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.12.6') + num="3.12.6", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using inline defined maps as an argument to map functions.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support using inline defined maps as an argument to map functions.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT map( 'aa', 4, '44' , 5) as c, mapKeys(c)\n" "SELECT map( 'aa', 4, '44' , 5) as c, mapValues(c)\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=3, - num='3.13.1') + num="3.13.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function\n' - 'that SHALL return number of keys in the map.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT length(map(1,2,3,4))\n' - 'SELECT length(map())\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [length] function\n" + "that SHALL return number of keys in the map.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT length(map(1,2,3,4))\n" + "SELECT length(map())\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.13.2.1') + num="3.13.2.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function\n' - 'that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is \n' - 'greater or equal to 1 it SHALL return 0.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT empty(map(1,2,3,4))\n' - 'SELECT empty(map())\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [empty] function\n" + "that SHALL return 1 if number of keys in the map is 0 otherwise if the number of keys is \n" + "greater or equal to 1 it SHALL return 0.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT empty(map(1,2,3,4))\n" + "SELECT empty(map())\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.13.3.1') + num="3.13.3.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function\n' - 'that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is\n' - 'greater or equal to 1 it SHALL return 1.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT notEmpty(map(1,2,3,4))\n' - 'SELECT notEmpty(map())\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `Map(key, value)` data type as an argument to the [notEmpty] function\n" + "that SHALL return 0 if number if keys in the map is 0 otherwise if the number of keys is\n" + "greater or equal to 1 it SHALL return 1.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT notEmpty(map(1,2,3,4))\n" + "SELECT notEmpty(map())\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.13.4.1') + num="3.13.4.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type\n' - 'using `map` function.\n' - '\n' - '**Syntax** \n' - '\n' - '``` sql\n' - 'map(key1, value1[, key2, value2, ...])\n' - '```\n' - '\n' - 'For example,\n' - '\n' - '``` sql\n' + "[ClickHouse] SHALL support arranging `key, value` pairs into `Map(key, value)` data type\n" + "using `map` function.\n" + "\n" + "**Syntax** \n" + "\n" + "``` sql\n" + "map(key1, value1[, key2, value2, ...])\n" + "```\n" + "\n" + "For example,\n" + "\n" + "``` sql\n" "SELECT map('key1', number, 'key2', number * 2) FROM numbers(3);\n" - '\n' + "\n" "┌─map('key1', number, 'key2', multiply(number, 2))─┐\n" "│ {'key1':0,'key2':0} │\n" "│ {'key1':1,'key2':2} │\n" "│ {'key1':2,'key2':4} │\n" - '└──────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "└──────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.13.5.1') + num="3.13.5.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `map` function is called with non even number of arguments.\n" + "\n" + ), link=None, level=4, - num='3.13.5.2') + num="3.13.5.2", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types.\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `map` function is called with mixed key or value types.\n" + "\n" + "\n" + ), link=None, level=4, - num='3.13.5.3') + num="3.13.5.3", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type.\n' - '\n' - 'For example,\n' - '\n' - '``` sql\n' + "[ClickHouse] SHALL support converting the results of `mapAdd` function to a `Map(key, value)` data type.\n" + "\n" + "For example,\n" + "\n" + "``` sql\n" 'SELECT CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), "Map(Int8,Int8)")\n' - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='3.13.5.4') + num="3.13.5.4", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support converting the results of `mapSubstract` function to a `Map(key, value)` data type.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" 'SELECT CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), "Map(Int8,Int8)")\n' - '```\n' - ), + "```\n" + ), link=None, level=4, - num='3.13.5.5') + num="3.13.5.5", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support converting the results of `mapPopulateSeries` function to a `Map(key, value)` data type.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" 'SELECT CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), "Map(Int8,Int8)")\n' - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='3.13.5.6') + num="3.13.5.6", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support `mapContains(map, key)` function to check weather `map.keys` contains the `key`.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT mapContains(a, 'abc') from table_map;\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='3.13.6.1') + num="3.13.6.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT mapKeys(a) from table_map;\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `mapKeys(map)` function to return all the map keys in the [Array] format.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT mapKeys(a) from table_map;\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.13.7.1') + num="3.13.7.1", +) RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues = Requirement( - name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', - version='1.0', + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT mapValues(a) from table_map;\n' - '```\n' - '\n' - '[Nested]: https://clickhouse.com/docs/en/sql-reference/data-types/nested-data-structures/nested/\n' - '[length]: https://clickhouse.com/docs/en/sql-reference/functions/array-functions/#array_functions-length\n' - '[empty]: https://clickhouse.com/docs/en/sql-reference/functions/array-functions/#function-empty\n' - '[notEmpty]: https://clickhouse.com/docs/en/sql-reference/functions/array-functions/#function-notempty\n' - '[CAST]: https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast\n' - '[Tuple]: https://clickhouse.com/docs/en/sql-reference/data-types/tuple/\n' - '[Tuple(Array,Array)]: https://clickhouse.com/docs/en/sql-reference/data-types/tuple/\n' - '[Array]: https://clickhouse.com/docs/en/sql-reference/data-types/array/ \n' - '[String]: https://clickhouse.com/docs/en/sql-reference/data-types/string/\n' - '[Integer]: https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/\n' - '[ClickHouse]: https://clickhouse.com\n' - '[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md \n' - '[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md\n' - '[Git]: https://git-scm.com/\n' - '[GitHub]: https://github.com\n' - ), + "[ClickHouse] SHALL support `mapValues(map)` function to return all the map values in the [Array] format.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT mapValues(a) from table_map;\n" + "```\n" + "\n" + "[Nested]: https://clickhouse.com/docs/en/sql-reference/data-types/nested-data-structures/nested/\n" + "[length]: https://clickhouse.com/docs/en/sql-reference/functions/array-functions/#array_functions-length\n" + "[empty]: https://clickhouse.com/docs/en/sql-reference/functions/array-functions/#function-empty\n" + "[notEmpty]: https://clickhouse.com/docs/en/sql-reference/functions/array-functions/#function-notempty\n" + "[CAST]: https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions/#type_conversion_function-cast\n" + "[Tuple]: https://clickhouse.com/docs/en/sql-reference/data-types/tuple/\n" + "[Tuple(Array,Array)]: https://clickhouse.com/docs/en/sql-reference/data-types/tuple/\n" + "[Array]: https://clickhouse.com/docs/en/sql-reference/data-types/array/ \n" + "[String]: https://clickhouse.com/docs/en/sql-reference/data-types/string/\n" + "[Integer]: https://clickhouse.com/docs/en/sql-reference/data-types/int-uint/\n" + "[ClickHouse]: https://clickhouse.com\n" + "[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/map_type/requirements/requirements.md \n" + "[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md\n" + "[Git]: https://git-scm.com/\n" + "[GitHub]: https://github.com\n" + ), link=None, level=4, - num='3.13.8.1') + num="3.13.8.1", +) SRS018_ClickHouse_Map_Data_Type = Specification( - name='SRS018 ClickHouse Map Data Type', + name="SRS018 ClickHouse Map Data Type", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -807,69 +845,211 @@ SRS018_ClickHouse_Map_Data_Type = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Requirements', level=1, num='3'), - Heading(name='General', level=2, num='3.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType', level=3, num='3.1.1'), - Heading(name='Performance', level=2, num='3.2'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples', level=3, num='3.2.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays', level=3, num='3.2.2'), - Heading(name='Key Types', level=2, num='3.3'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.String', level=3, num='3.3.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer', level=3, num='3.3.2'), - Heading(name='Value Types', level=2, num='3.4'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.String', level=3, num='3.4.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer', level=3, num='3.4.2'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Array', level=3, num='3.4.3'), - Heading(name='Invalid Types', level=2, num='3.5'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable', level=3, num='3.5.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing', level=3, num='3.5.2'), - Heading(name='Duplicated Keys', level=2, num='3.6'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys', level=3, num='3.6.1'), - Heading(name='Array of Maps', level=2, num='3.7'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps', level=3, num='3.7.1'), - Heading(name='Nested With Maps', level=2, num='3.8'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps', level=3, num='3.8.1'), - Heading(name='Value Retrieval', level=2, num='3.9'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval', level=3, num='3.9.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid', level=3, num='3.9.2'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound', level=3, num='3.9.3'), - Heading(name='Converting Tuple(Array, Array) to Map', level=2, num='3.10'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap', level=3, num='3.10.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid', level=3, num='3.10.2'), - Heading(name='Converting Array(Tuple(K,V)) to Map', level=2, num='3.11'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap', level=3, num='3.11.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid', level=3, num='3.11.2'), - Heading(name='Keys and Values Subcolumns', level=2, num='3.12'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys', level=3, num='3.12.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions', level=3, num='3.12.2'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap', level=3, num='3.12.3'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values', level=3, num='3.12.4'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions', level=3, num='3.12.5'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap', level=3, num='3.12.6'), - Heading(name='Functions', level=2, num='3.13'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap', level=3, num='3.13.1'), - Heading(name='`length`', level=3, num='3.13.2'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length', level=4, num='3.13.2.1'), - Heading(name='`empty`', level=3, num='3.13.3'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty', level=4, num='3.13.3.1'), - Heading(name='`notEmpty`', level=3, num='3.13.4'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty', level=4, num='3.13.4.1'), - Heading(name='`map`', level=3, num='3.13.5'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map', level=4, num='3.13.5.1'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments', level=4, num='3.13.5.2'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes', level=4, num='3.13.5.3'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd', level=4, num='3.13.5.4'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract', level=4, num='3.13.5.5'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries', level=4, num='3.13.5.6'), - Heading(name='`mapContains`', level=3, num='3.13.6'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains', level=4, num='3.13.6.1'), - Heading(name='`mapKeys`', level=3, num='3.13.7'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys', level=4, num='3.13.7.1'), - Heading(name='`mapValues`', level=3, num='3.13.8'), - Heading(name='RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues', level=4, num='3.13.8.1'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Requirements", level=1, num="3"), + Heading(name="General", level=2, num="3.1"), + Heading(name="RQ.SRS-018.ClickHouse.Map.DataType", level=3, num="3.1.1"), + Heading(name="Performance", level=2, num="3.2"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.ArrayOfTuples", + level=3, + num="3.2.1", ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Performance.Vs.TupleOfArrays", + level=3, + num="3.2.2", + ), + Heading(name="Key Types", level=2, num="3.3"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Key.String", level=3, num="3.3.1" + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Key.Integer", level=3, num="3.3.2" + ), + Heading(name="Value Types", level=2, num="3.4"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.String", level=3, num="3.4.1" + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Integer", + level=3, + num="3.4.2", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Array", level=3, num="3.4.3" + ), + Heading(name="Invalid Types", level=2, num="3.5"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Invalid.Nullable", + level=3, + num="3.5.1", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Invalid.NothingNothing", + level=3, + num="3.5.2", + ), + Heading(name="Duplicated Keys", level=2, num="3.6"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.DuplicatedKeys", + level=3, + num="3.6.1", + ), + Heading(name="Array of Maps", level=2, num="3.7"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.ArrayOfMaps", level=3, num="3.7.1" + ), + Heading(name="Nested With Maps", level=2, num="3.8"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.NestedWithMaps", + level=3, + num="3.8.1", + ), + Heading(name="Value Retrieval", level=2, num="3.9"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval", + level=3, + num="3.9.1", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyInvalid", + level=3, + num="3.9.2", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Value.Retrieval.KeyNotFound", + level=3, + num="3.9.3", + ), + Heading(name="Converting Tuple(Array, Array) to Map", level=2, num="3.10"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysToMap", + level=3, + num="3.10.1", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.TupleOfArraysMap.Invalid", + level=3, + num="3.10.2", + ), + Heading(name="Converting Array(Tuple(K,V)) to Map", level=2, num="3.11"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap", + level=3, + num="3.11.1", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Conversion.From.ArrayOfTuplesToMap.Invalid", + level=3, + num="3.11.2", + ), + Heading(name="Keys and Values Subcolumns", level=2, num="3.12"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys", + level=3, + num="3.12.1", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.ArrayFunctions", + level=3, + num="3.12.2", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Keys.InlineDefinedMap", + level=3, + num="3.12.3", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values", + level=3, + num="3.12.4", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.ArrayFunctions", + level=3, + num="3.12.5", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.SubColumns.Values.InlineDefinedMap", + level=3, + num="3.12.6", + ), + Heading(name="Functions", level=2, num="3.13"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.InlineDefinedMap", + level=3, + num="3.13.1", + ), + Heading(name="`length`", level=3, num="3.13.2"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Length", + level=4, + num="3.13.2.1", + ), + Heading(name="`empty`", level=3, num="3.13.3"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Empty", + level=4, + num="3.13.3.1", + ), + Heading(name="`notEmpty`", level=3, num="3.13.4"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.NotEmpty", + level=4, + num="3.13.4.1", + ), + Heading(name="`map`", level=3, num="3.13.5"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map", + level=4, + num="3.13.5.1", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.InvalidNumberOfArguments", + level=4, + num="3.13.5.2", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MixedKeyOrValueTypes", + level=4, + num="3.13.5.3", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapAdd", + level=4, + num="3.13.5.4", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapSubstract", + level=4, + num="3.13.5.5", + ), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.Map.MapPopulateSeries", + level=4, + num="3.13.5.6", + ), + Heading(name="`mapContains`", level=3, num="3.13.6"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapContains", + level=4, + num="3.13.6.1", + ), + Heading(name="`mapKeys`", level=3, num="3.13.7"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapKeys", + level=4, + num="3.13.7.1", + ), + Heading(name="`mapValues`", level=3, num="3.13.8"), + Heading( + name="RQ.SRS-018.ClickHouse.Map.DataType.Functions.MapValues", + level=4, + num="3.13.8.1", + ), + ), requirements=( RQ_SRS_018_ClickHouse_Map_DataType, RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples, @@ -910,8 +1090,8 @@ SRS018_ClickHouse_Map_Data_Type = Specification( RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains, RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys, RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues, - ), - content=''' + ), + content=""" # SRS018 ClickHouse Map Data Type # Software Requirements Specification @@ -1424,4 +1604,5 @@ SELECT mapValues(a) from table_map; [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/map_type/requirements/requirements.md [Git]: https://git-scm.com/ [GitHub]: https://github.com -''') +""", +) diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py index 6ce1b6ab8a6..754d5b75a38 100644 --- a/tests/testflows/map_type/tests/common.py +++ b/tests/testflows/map_type/tests/common.py @@ -5,17 +5,20 @@ from testflows.core import * from testflows.core.name import basename, parentname from testflows._core.testtype import TestSubType + def getuid(): if current().subtype == TestSubType.Example: - testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + testname = ( + f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + ) else: testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" - return testname + "_" + str(uuid.uuid1()).replace('-', '_') + return testname + "_" + str(uuid.uuid1()).replace("-", "_") + @TestStep(Given) def create_table(self, name, statement, on_cluster=False): - """Create table. - """ + """Create table.""" node = current().context.node try: with Given(f"I have a {name} table"): diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py index 5d7c900d591..0aee235c1ed 100755 --- a/tests/testflows/map_type/tests/feature.py +++ b/tests/testflows/map_type/tests/feature.py @@ -7,10 +7,10 @@ from testflows.asserts import error from map_type.requirements import * from map_type.tests.common import * + @TestOutline def select_map(self, map, output, exitcode=0, message=None): - """Create a map using select statement. - """ + """Create a map using select statement.""" node = self.context.node with When("I create a map using select", description=map): @@ -19,10 +19,20 @@ def select_map(self, map, output, exitcode=0, message=None): with Then("I expect output to match", description=output): assert r.output == output, error() + @TestOutline -def table_map(self, type, data, select, filter, exitcode, message, check_insert=False, order_by=None): - """Check using a map column in a table. - """ +def table_map( + self, + type, + data, + select, + filter, + exitcode, + message, + check_insert=False, + order_by=None, +): + """Check using a map column in a table.""" uid = getuid() node = self.context.node @@ -30,344 +40,973 @@ def table_map(self, type, data, select, filter, exitcode, message, check_insert= order_by = "m" with Given(f"table definition with {type}"): - sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + sql = ( + "CREATE TABLE {name} (m " + + type + + ") ENGINE = MergeTree() ORDER BY " + + order_by + ) with And(f"I create a table", description=sql): table = create_table(name=uid, statement=sql) with When("I insert data into the map column"): if check_insert: - node.query(f"INSERT INTO {table} VALUES {data}", exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table} VALUES {data}", exitcode=exitcode, message=message + ) else: node.query(f"INSERT INTO {table} VALUES {data}") if not check_insert: with And("I try to read from the table"): - node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + node.query( + f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0")) +@Examples( + "map output", + [ + ("map('',1)", "{'':1}", Name("empty string")), + ("map('hello',1)", "{'hello':1}", Name("non-empty string")), + ("map('Gãńdåłf_Thê_Gręât',1)", "{'Gãńdåłf_Thê_Gręât':1}", Name("utf-8 string")), + ("map('hello there',1)", "{'hello there':1}", Name("multi word string")), + ("map('hello',1,'there',2)", "{'hello':1,'there':2}", Name("multiple keys")), + ("map(toString(1),1)", "{'1':1}", Name("toString")), + ("map(toFixedString('1',1),1)", "{'1':1}", Name("toFixedString")), + ("map(toNullable('1'),1)", "{'1':1}", Name("Nullable")), + ("map(toNullable(NULL),1)", "{NULL:1}", Name("Nullable(NULL)")), + ("map(toLowCardinality('1'),1)", "{'1':1}", Name("LowCardinality(String)")), + ( + "map(toLowCardinality(toFixedString('1',1)),1)", + "{'1':1}", + Name("LowCardinality(FixedString)"), + ), + ], + row_format="%20s,%20s", ) -@Examples("map output", [ - ("map('',1)", "{'':1}", Name("empty string")), - ("map('hello',1)", "{'hello':1}", Name("non-empty string")), - ("map('Gãńdåłf_Thê_Gręât',1)", "{'Gãńdåłf_Thê_Gręât':1}", Name("utf-8 string")), - ("map('hello there',1)", "{'hello there':1}", Name("multi word string")), - ("map('hello',1,'there',2)", "{'hello':1,'there':2}", Name("multiple keys")), - ("map(toString(1),1)", "{'1':1}", Name("toString")), - ("map(toFixedString('1',1),1)", "{'1':1}", Name("toFixedString")), - ("map(toNullable('1'),1)", "{'1':1}", Name("Nullable")), - ("map(toNullable(NULL),1)", "{NULL:1}", Name("Nullable(NULL)")), - ("map(toLowCardinality('1'),1)", "{'1':1}", Name("LowCardinality(String)")), - ("map(toLowCardinality(toFixedString('1',1)),1)", "{'1':1}", Name("LowCardinality(FixedString)")), -], row_format="%20s,%20s") def select_map_with_key_string(self, map, output): - """Create a map using select that has key string type. - """ + """Create a map using select that has key string type.""" select_map(map=map, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0")) +@Examples( + "map output", + [ + ("map('key','')", "{'key':''}", Name("empty string")), + ("map('key','hello')", "{'key':'hello'}", Name("non-empty string")), + ( + "map('key','Gãńdåłf_Thê_Gręât')", + "{'key':'Gãńdåłf_Thê_Gręât'}", + Name("utf-8 string"), + ), + ( + "map('key','hello there')", + "{'key':'hello there'}", + Name("multi word string"), + ), + ( + "map('key','hello','key2','there')", + "{'key':'hello','key2':'there'}", + Name("multiple keys"), + ), + ("map('key',toString(1))", "{'key':'1'}", Name("toString")), + ("map('key',toFixedString('1',1))", "{'key':'1'}", Name("toFixedString")), + ("map('key',toNullable('1'))", "{'key':'1'}", Name("Nullable")), + ("map('key',toNullable(NULL))", "{'key':NULL}", Name("Nullable(NULL)")), + ( + "map('key',toLowCardinality('1'))", + "{'key':'1'}", + Name("LowCardinality(String)"), + ), + ( + "map('key',toLowCardinality(toFixedString('1',1)))", + "{'key':'1'}", + Name("LowCardinality(FixedString)"), + ), + ], + row_format="%20s,%20s", ) -@Examples("map output", [ - ("map('key','')", "{'key':''}", Name("empty string")), - ("map('key','hello')", "{'key':'hello'}", Name("non-empty string")), - ("map('key','Gãńdåłf_Thê_Gręât')", "{'key':'Gãńdåłf_Thê_Gręât'}", Name("utf-8 string")), - ("map('key','hello there')", "{'key':'hello there'}", Name("multi word string")), - ("map('key','hello','key2','there')", "{'key':'hello','key2':'there'}", Name("multiple keys")), - ("map('key',toString(1))", "{'key':'1'}", Name("toString")), - ("map('key',toFixedString('1',1))", "{'key':'1'}", Name("toFixedString")), - ("map('key',toNullable('1'))", "{'key':'1'}", Name("Nullable")), - ("map('key',toNullable(NULL))", "{'key':NULL}", Name("Nullable(NULL)")), - ("map('key',toLowCardinality('1'))", "{'key':'1'}", Name("LowCardinality(String)")), - ("map('key',toLowCardinality(toFixedString('1',1)))", "{'key':'1'}", Name("LowCardinality(FixedString)")), -], row_format="%20s,%20s") def select_map_with_value_string(self, map, output): - """Create a map using select that has value string type. - """ + """Create a map using select that has value string type.""" select_map(map=map, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0")) +@Examples( + "map output", + [ + ("map('key',[])", "{'key':[]}", Name("empty Array")), + ("map('key',[1,2,3])", "{'key':[1,2,3]}", Name("non-empty array of ints")), + ( + "map('key',['1','2','3'])", + "{'key':['1','2','3']}", + Name("non-empty array of strings"), + ), + ( + "map('key',[map(1,2),map(2,3)])", + "{'key':[{1:2},{2:3}]}", + Name("non-empty array of maps"), + ), + ( + "map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])])", + "{'key':[{1:[{1:[1]}]},{2:[{2:[3]}]}]}", + Name("non-empty array of maps of array of maps"), + ), + ], ) -@Examples("map output", [ - ("map('key',[])", "{'key':[]}", Name("empty Array")), - ("map('key',[1,2,3])", "{'key':[1,2,3]}", Name("non-empty array of ints")), - ("map('key',['1','2','3'])", "{'key':['1','2','3']}", Name("non-empty array of strings")), - ("map('key',[map(1,2),map(2,3)])", "{'key':[{1:2},{2:3}]}", Name("non-empty array of maps")), - ("map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])])", "{'key':[{1:[{1:[1]}]},{2:[{2:[3]}]}]}", Name("non-empty array of maps of array of maps")), -]) def select_map_with_value_array(self, map, output): - """Create a map using select that has value array type. - """ + """Create a map using select that has value array type.""" select_map(map=map, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0")) +@Examples( + "map output", + [ + ("(map(1,127,2,0,3,-128))", "{1:127,2:0,3:-128}", Name("Int8")), + ("(map(1,0,2,255))", "{1:0,2:255}", Name("UInt8")), + ("(map(1,32767,2,0,3,-32768))", "{1:32767,2:0,3:-32768}", Name("Int16")), + ("(map(1,0,2,65535))", "{1:0,2:65535}", Name("UInt16")), + ( + "(map(1,2147483647,2,0,3,-2147483648))", + "{1:2147483647,2:0,3:-2147483648}", + Name("Int32"), + ), + ("(map(1,0,2,4294967295))", "{1:0,2:4294967295}", Name("UInt32")), + ( + "(map(1,9223372036854775807,2,0,3,-9223372036854775808))", + '{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}', + Name("Int64"), + ), + ( + "(map(1,0,2,18446744073709551615))", + "{1:0,2:18446744073709551615}", + Name("UInt64"), + ), + ( + "(map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", + "{1:1.7014118346046923e38,2:0,3:-1.7014118346046923e38}", + Name("Int128"), + ), + ( + "(map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", + "{1:5.78960446186581e76,2:0,3:-5.78960446186581e76}", + Name("Int256"), + ), + ( + "(map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", + "{1:0,2:1.157920892373162e77}", + Name("UInt256"), + ), + ("(map(1,toNullable(1)))", "{1:1}", Name("toNullable")), + ("(map(1,toNullable(NULL)))", "{1:NULL}", Name("toNullable(NULL)")), + ], ) -@Examples("map output", [ - ("(map(1,127,2,0,3,-128))", '{1:127,2:0,3:-128}', Name("Int8")), - ("(map(1,0,2,255))", '{1:0,2:255}', Name("UInt8")), - ("(map(1,32767,2,0,3,-32768))", '{1:32767,2:0,3:-32768}', Name("Int16")), - ("(map(1,0,2,65535))", '{1:0,2:65535}', Name("UInt16")), - ("(map(1,2147483647,2,0,3,-2147483648))", '{1:2147483647,2:0,3:-2147483648}', Name("Int32")), - ("(map(1,0,2,4294967295))", '{1:0,2:4294967295}', Name("UInt32")), - ("(map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}', Name("Int64")), - ("(map(1,0,2,18446744073709551615))", '{1:0,2:18446744073709551615}', Name("UInt64")), - ("(map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{1:1.7014118346046923e38,2:0,3:-1.7014118346046923e38}', Name("Int128")), - ("(map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{1:5.78960446186581e76,2:0,3:-5.78960446186581e76}', Name("Int256")), - ("(map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{1:0,2:1.157920892373162e77}', Name("UInt256")), - ("(map(1,toNullable(1)))", '{1:1}', Name("toNullable")), - ("(map(1,toNullable(NULL)))", '{1:NULL}', Name("toNullable(NULL)")), -]) def select_map_with_value_integer(self, map, output): - """Create a map using select that has value integer type. - """ + """Create a map using select that has value integer type.""" select_map(map=map, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0")) +@Examples( + "map output", + [ + ("(map(127,1,0,1,-128,1))", "{127:1,0:1,-128:1}", Name("Int8")), + ("(map(0,1,255,1))", "{0:1,255:1}", Name("UInt8")), + ("(map(32767,1,0,1,-32768,1))", "{32767:1,0:1,-32768:1}", Name("Int16")), + ("(map(0,1,65535,1))", "{0:1,65535:1}", Name("UInt16")), + ( + "(map(2147483647,1,0,1,-2147483648,1))", + "{2147483647:1,0:1,-2147483648:1}", + Name("Int32"), + ), + ("(map(0,1,4294967295,1))", "{0:1,4294967295:1}", Name("UInt32")), + ( + "(map(9223372036854775807,1,0,1,-9223372036854775808,1))", + '{"9223372036854775807":1,"0":1,"-9223372036854775808":1}', + Name("Int64"), + ), + ( + "(map(0,1,18446744073709551615,1))", + "{0:1,18446744073709551615:1}", + Name("UInt64"), + ), + ( + "(map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", + "{1.7014118346046923e38:1,0:1,-1.7014118346046923e38:1}", + Name("Int128"), + ), + ( + "(map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", + "{5.78960446186581e76:1,0:1,-5.78960446186581e76:1}", + Name("Int256"), + ), + ( + "(map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", + "{0:1,1.157920892373162e77:1}", + Name("UInt256"), + ), + ("(map(toNullable(1),1))", "{1:1}", Name("toNullable")), + ("(map(toNullable(NULL),1))", "{NULL:1}", Name("toNullable(NULL)")), + ], ) -@Examples("map output", [ - ("(map(127,1,0,1,-128,1))", '{127:1,0:1,-128:1}', Name("Int8")), - ("(map(0,1,255,1))", '{0:1,255:1}', Name("UInt8")), - ("(map(32767,1,0,1,-32768,1))", '{32767:1,0:1,-32768:1}', Name("Int16")), - ("(map(0,1,65535,1))", '{0:1,65535:1}', Name("UInt16")), - ("(map(2147483647,1,0,1,-2147483648,1))", '{2147483647:1,0:1,-2147483648:1}', Name("Int32")), - ("(map(0,1,4294967295,1))", '{0:1,4294967295:1}', Name("UInt32")), - ("(map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"9223372036854775807":1,"0":1,"-9223372036854775808":1}', Name("Int64")), - ("(map(0,1,18446744073709551615,1))", '{0:1,18446744073709551615:1}', Name("UInt64")), - ("(map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{1.7014118346046923e38:1,0:1,-1.7014118346046923e38:1}', Name("Int128")), - ("(map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{5.78960446186581e76:1,0:1,-5.78960446186581e76:1}', Name("Int256")), - ("(map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{0:1,1.157920892373162e77:1}', Name("UInt256")), - ("(map(toNullable(1),1))", '{1:1}', Name("toNullable")), - ("(map(toNullable(NULL),1))", '{NULL:1}', Name("toNullable(NULL)")), -]) def select_map_with_key_integer(self, map, output): - """Create a map using select that has key integer type. - """ + """Create a map using select that has key integer type.""" select_map(map=map, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0")) +@Examples( + "type data output", + [ + ( + "Map(String, Int8)", + "('2020-01-01', map('',1))", + '{"d":"2020-01-01","m":{"":1}}', + Name("empty string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('hello',1))", + '{"d":"2020-01-01","m":{"hello":1}}', + Name("non-empty string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", + '{"d":"2020-01-01","m":{"Gãńdåłf_Thê_Gręât":1}}', + Name("utf-8 string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('hello there',1))", + '{"d":"2020-01-01","m":{"hello there":1}}', + Name("multi word string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('hello',1,'there',2))", + '{"d":"2020-01-01","m":{"hello":1,"there":2}}', + Name("multiple keys"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map(toString(1),1))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("toString"), + ), + ( + "Map(FixedString(1), Int8)", + "('2020-01-01', map(toFixedString('1',1),1))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("FixedString"), + ), + ( + "Map(Nullable(String), Int8)", + "('2020-01-01', map(toNullable('1'),1))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("Nullable"), + ), + ( + "Map(Nullable(String), Int8)", + "('2020-01-01', map(toNullable(NULL),1))", + '{"d":"2020-01-01","m":{null:1}}', + Name("Nullable(NULL)"), + ), + ( + "Map(LowCardinality(String), Int8)", + "('2020-01-01', map(toLowCardinality('1'),1))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("LowCardinality(String)"), + ), + ( + "Map(LowCardinality(String), Int8)", + "('2020-01-01', map('1',1))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("LowCardinality(String) cast from String"), + ), + ( + "Map(LowCardinality(String), LowCardinality(String))", + "('2020-01-01', map('1','1'))", + '{"d":"2020-01-01","m":{"1":"1"}}', + Name("LowCardinality(String) for key and value"), + ), + ( + "Map(LowCardinality(FixedString(1)), Int8)", + "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("LowCardinality(FixedString)"), + ), + ], ) -@Examples("type data output", [ - ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"d":"2020-01-01","m":{"":1}}', Name("empty string")), - ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"d":"2020-01-01","m":{"hello":1}}', Name("non-empty string")), - ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"d":"2020-01-01","m":{"Gãńdåłf_Thê_Gręât":1}}', Name("utf-8 string")), - ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"d":"2020-01-01","m":{"hello there":1}}', Name("multi word string")), - ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"d":"2020-01-01","m":{"hello":1,"there":2}}', Name("multiple keys")), - ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("toString")), - ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("FixedString")), - ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("Nullable")), - ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("Nullable(NULL)")), - ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String)")), - ("Map(LowCardinality(String), Int8)", "('2020-01-01', map('1',1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(String) cast from String")), - ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"d":"2020-01-01","m":{"1":"1"}}', Name("LowCardinality(String) for key and value")), - ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"d":"2020-01-01","m":{"1":1}}', Name("LowCardinality(FixedString)")), -]) def table_map_with_key_string(self, type, data, output): - """Check what values we can insert into map type column with key string. - """ + """Check what values we can insert into map type column with key string.""" insert_into_table(type=type, data=data, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Key_String("1.0")) +@Examples( + "type data output select", + [ + ( + "Map(String, Int8)", + "('2020-01-01', map('',1))", + '{"m":1}', + "m[''] AS m", + Name("empty string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('hello',1))", + '{"m":1}', + "m['hello'] AS m", + Name("non-empty string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", + '{"m":1}', + "m['Gãńdåłf_Thê_Gręât'] AS m", + Name("utf-8 string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('hello there',1))", + '{"m":1}', + "m['hello there'] AS m", + Name("multi word string"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map('hello',1,'there',2))", + '{"m":1}', + "m['hello'] AS m", + Name("multiple keys"), + ), + ( + "Map(String, Int8)", + "('2020-01-01', map(toString(1),1))", + '{"m":1}', + "m['1'] AS m", + Name("toString"), + ), + ( + "Map(FixedString(1), Int8)", + "('2020-01-01', map(toFixedString('1',1),1))", + '{"m":1}', + "m['1'] AS m", + Name("FixedString"), + ), + ( + "Map(Nullable(String), Int8)", + "('2020-01-01', map(toNullable('1'),1))", + '{"m":1}}', + "m['1'] AS m", + Name("Nullable"), + ), + ( + "Map(Nullable(String), Int8)", + "('2020-01-01', map(toNullable(NULL),1))", + '{"m":1}', + "m[null] AS m", + Name("Nullable(NULL)"), + ), + ( + "Map(LowCardinality(String), Int8)", + "('2020-01-01', map(toLowCardinality('1'),1))", + '{"m":1}}', + "m['1'] AS m", + Name("LowCardinality(String)"), + ), + ( + "Map(LowCardinality(String), Int8)", + "('2020-01-01', map('1',1))", + '{"m":1}', + "m['1'] AS m", + Name("LowCardinality(String) cast from String"), + ), + ( + "Map(LowCardinality(String), LowCardinality(String))", + "('2020-01-01', map('1','1'))", + '{"m":"1"}', + "m['1'] AS m", + Name("LowCardinality(String) for key and value"), + ), + ( + "Map(LowCardinality(FixedString(1)), Int8)", + "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", + '{"m":1}', + "m['1'] AS m", + Name("LowCardinality(FixedString)"), + ), + ], ) -@Examples("type data output select", [ - ("Map(String, Int8)", "('2020-01-01', map('',1))", '{"m":1}', "m[''] AS m", Name("empty string")), - ("Map(String, Int8)", "('2020-01-01', map('hello',1))", '{"m":1}', "m['hello'] AS m", Name("non-empty string")), - ("Map(String, Int8)", "('2020-01-01', map('Gãńdåłf_Thê_Gręât',1))", '{"m":1}', "m['Gãńdåłf_Thê_Gręât'] AS m", Name("utf-8 string")), - ("Map(String, Int8)", "('2020-01-01', map('hello there',1))", '{"m":1}', "m['hello there'] AS m", Name("multi word string")), - ("Map(String, Int8)", "('2020-01-01', map('hello',1,'there',2))", '{"m":1}', "m['hello'] AS m", Name("multiple keys")), - ("Map(String, Int8)", "('2020-01-01', map(toString(1),1))", '{"m":1}', "m['1'] AS m", Name("toString")), - ("Map(FixedString(1), Int8)", "('2020-01-01', map(toFixedString('1',1),1))", '{"m":1}', "m['1'] AS m", Name("FixedString")), - ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable('1'),1))", '{"m":1}}', "m['1'] AS m", Name("Nullable")), - ("Map(Nullable(String), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"m":1}', "m[null] AS m", Name("Nullable(NULL)")), - ("Map(LowCardinality(String), Int8)", "('2020-01-01', map(toLowCardinality('1'),1))", '{"m":1}}', "m['1'] AS m", Name("LowCardinality(String)")), - ("Map(LowCardinality(String), Int8)", "('2020-01-01', map('1',1))", '{"m":1}', "m['1'] AS m", Name("LowCardinality(String) cast from String")), - ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"m":"1"}', "m['1'] AS m", Name("LowCardinality(String) for key and value")), - ("Map(LowCardinality(FixedString(1)), Int8)", "('2020-01-01', map(toLowCardinality(toFixedString('1',1)),1))", '{"m":1}', "m['1'] AS m", Name("LowCardinality(FixedString)")), -]) def table_map_select_key_with_key_string(self, type, data, output, select): - """Check what values we can insert into map type column with key string and if key can be selected. - """ + """Check what values we can insert into map type column with key string and if key can be selected.""" insert_into_table(type=type, data=data, output=output, select=select) -@TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") -) -@Examples("type data output", [ - ("Map(String, String)", "('2020-01-01', map('key',''))", '{"d":"2020-01-01","m":{"key":""}}', Name("empty string")), - ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"d":"2020-01-01","m":{"key":"hello"}}', Name("non-empty string")), - ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"d":"2020-01-01","m":{"key":"Gãńdåłf_Thê_Gręât"}}', Name("utf-8 string")), - ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"d":"2020-01-01","m":{"key":"hello there"}}', Name("multi word string")), - ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"d":"2020-01-01","m":{"key":"hello","key2":"there"}}', Name("multiple keys")), - ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("toString")), - ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("FixedString")), - ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("Nullable")), - ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"d":"2020-01-01","m":{"key":null}}', Name("Nullable(NULL)")), - ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String)")), - ("Map(String, LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(String) cast from String")), - ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('1','1'))", '{"d":"2020-01-01","m":{"1":"1"}}', Name("LowCardinality(String) for key and value")), - ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"d":"2020-01-01","m":{"key":"1"}}', Name("LowCardinality(FixedString)")) -]) -def table_map_with_value_string(self, type, data, output): - """Check what values we can insert into map type column with value string. - """ - insert_into_table(type=type, data=data, output=output) @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0")) +@Examples( + "type data output", + [ + ( + "Map(String, String)", + "('2020-01-01', map('key',''))", + '{"d":"2020-01-01","m":{"key":""}}', + Name("empty string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key','hello'))", + '{"d":"2020-01-01","m":{"key":"hello"}}', + Name("non-empty string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", + '{"d":"2020-01-01","m":{"key":"Gãńdåłf_Thê_Gręât"}}', + Name("utf-8 string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key', 'hello there'))", + '{"d":"2020-01-01","m":{"key":"hello there"}}', + Name("multi word string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key','hello','key2','there'))", + '{"d":"2020-01-01","m":{"key":"hello","key2":"there"}}', + Name("multiple keys"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key', toString(1)))", + '{"d":"2020-01-01","m":{"key":"1"}}', + Name("toString"), + ), + ( + "Map(String, FixedString(1))", + "('2020-01-01', map('key',toFixedString('1',1)))", + '{"d":"2020-01-01","m":{"key":"1"}}', + Name("FixedString"), + ), + ( + "Map(String, Nullable(String))", + "('2020-01-01', map('key',toNullable('1')))", + '{"d":"2020-01-01","m":{"key":"1"}}', + Name("Nullable"), + ), + ( + "Map(String, Nullable(String))", + "('2020-01-01', map('key',toNullable(NULL)))", + '{"d":"2020-01-01","m":{"key":null}}', + Name("Nullable(NULL)"), + ), + ( + "Map(String, LowCardinality(String))", + "('2020-01-01', map('key',toLowCardinality('1')))", + '{"d":"2020-01-01","m":{"key":"1"}}', + Name("LowCardinality(String)"), + ), + ( + "Map(String, LowCardinality(String))", + "('2020-01-01', map('key','1'))", + '{"d":"2020-01-01","m":{"key":"1"}}', + Name("LowCardinality(String) cast from String"), + ), + ( + "Map(LowCardinality(String), LowCardinality(String))", + "('2020-01-01', map('1','1'))", + '{"d":"2020-01-01","m":{"1":"1"}}', + Name("LowCardinality(String) for key and value"), + ), + ( + "Map(String, LowCardinality(FixedString(1)))", + "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", + '{"d":"2020-01-01","m":{"key":"1"}}', + Name("LowCardinality(FixedString)"), + ), + ], +) +def table_map_with_value_string(self, type, data, output): + """Check what values we can insert into map type column with value string.""" + insert_into_table(type=type, data=data, output=output) + + +@TestOutline(Scenario) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_String("1.0")) +@Examples( + "type data output", + [ + ( + "Map(String, String)", + "('2020-01-01', map('key',''))", + '{"m":""}', + Name("empty string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key','hello'))", + '{"m":"hello"}', + Name("non-empty string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", + '{"m":"Gãńdåłf_Thê_Gręât"}', + Name("utf-8 string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key', 'hello there'))", + '{"m":"hello there"}', + Name("multi word string"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key','hello','key2','there'))", + '{"m":"hello"}', + Name("multiple keys"), + ), + ( + "Map(String, String)", + "('2020-01-01', map('key', toString(1)))", + '{"m":"1"}', + Name("toString"), + ), + ( + "Map(String, FixedString(1))", + "('2020-01-01', map('key',toFixedString('1',1)))", + '{"m":"1"}', + Name("FixedString"), + ), + ( + "Map(String, Nullable(String))", + "('2020-01-01', map('key',toNullable('1')))", + '{"m":"1"}', + Name("Nullable"), + ), + ( + "Map(String, Nullable(String))", + "('2020-01-01', map('key',toNullable(NULL)))", + '{"m":null}', + Name("Nullable(NULL)"), + ), + ( + "Map(String, LowCardinality(String))", + "('2020-01-01', map('key',toLowCardinality('1')))", + '{"m":"1"}', + Name("LowCardinality(String)"), + ), + ( + "Map(String, LowCardinality(String))", + "('2020-01-01', map('key','1'))", + '{"m":"1"}', + Name("LowCardinality(String) cast from String"), + ), + ( + "Map(LowCardinality(String), LowCardinality(String))", + "('2020-01-01', map('key','1'))", + '{"m":"1"}', + Name("LowCardinality(String) for key and value"), + ), + ( + "Map(String, LowCardinality(FixedString(1)))", + "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", + '{"m":"1"}', + Name("LowCardinality(FixedString)"), + ), + ], ) -@Examples("type data output", [ - ("Map(String, String)", "('2020-01-01', map('key',''))", '{"m":""}', Name("empty string")), - ("Map(String, String)", "('2020-01-01', map('key','hello'))", '{"m":"hello"}', Name("non-empty string")), - ("Map(String, String)", "('2020-01-01', map('key','Gãńdåłf_Thê_Gręât'))", '{"m":"Gãńdåłf_Thê_Gręât"}', Name("utf-8 string")), - ("Map(String, String)", "('2020-01-01', map('key', 'hello there'))", '{"m":"hello there"}', Name("multi word string")), - ("Map(String, String)", "('2020-01-01', map('key','hello','key2','there'))", '{"m":"hello"}', Name("multiple keys")), - ("Map(String, String)", "('2020-01-01', map('key', toString(1)))", '{"m":"1"}', Name("toString")), - ("Map(String, FixedString(1))", "('2020-01-01', map('key',toFixedString('1',1)))", '{"m":"1"}', Name("FixedString")), - ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable('1')))", '{"m":"1"}', Name("Nullable")), - ("Map(String, Nullable(String))", "('2020-01-01', map('key',toNullable(NULL)))", '{"m":null}', Name("Nullable(NULL)")), - ("Map(String, LowCardinality(String))", "('2020-01-01', map('key',toLowCardinality('1')))", '{"m":"1"}', Name("LowCardinality(String)")), - ("Map(String, LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"m":"1"}', Name("LowCardinality(String) cast from String")), - ("Map(LowCardinality(String), LowCardinality(String))", "('2020-01-01', map('key','1'))", '{"m":"1"}', Name("LowCardinality(String) for key and value")), - ("Map(String, LowCardinality(FixedString(1)))", "('2020-01-01', map('key',toLowCardinality(toFixedString('1',1))))", '{"m":"1"}', Name("LowCardinality(FixedString)")) -]) def table_map_select_key_with_value_string(self, type, data, output): - """Check what values we can insert into map type column with value string and if it can be selected by key. - """ + """Check what values we can insert into map type column with value string and if it can be selected by key.""" insert_into_table(type=type, data=data, output=output, select="m['key'] AS m") + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0")) +@Examples( + "type data output", + [ + ( + "Map(Int8, Int8)", + "('2020-01-01', map(1,127,2,0,3,-128))", + '{"d":"2020-01-01","m":{"1":127,"2":0,"3":-128}}', + Name("Int8"), + ), + ( + "Map(Int8, UInt8)", + "('2020-01-01', map(1,0,2,255))", + '{"d":"2020-01-01","m":{"1":0,"2":255}}', + Name("UInt8"), + ), + ( + "Map(Int8, Int16)", + "('2020-01-01', map(1,127,2,0,3,-128))", + '{"d":"2020-01-01","m":{"1":32767,"2":0,"3":-32768}}', + Name("Int16"), + ), + ( + "Map(Int8, UInt16)", + "('2020-01-01', map(1,0,2,65535))", + '{"d":"2020-01-01","m":{"1":0,"2":65535}}', + Name("UInt16"), + ), + ( + "Map(Int8, Int32)", + "('2020-01-01', map(1,127,2,0,3,-128))", + '{"d":"2020-01-01","m":{"1":2147483647,"2":0,"3":-2147483648}}', + Name("Int32"), + ), + ( + "Map(Int8, UInt32)", + "('2020-01-01', map(1,0,2,4294967295))", + '{"d":"2020-01-01","m":{"1":0,"2":4294967295}}', + Name("UInt32"), + ), + ( + "Map(Int8, Int64)", + "('2020-01-01', map(1,9223372036854775807,2,0,3,-9223372036854775808))", + '{"d":"2020-01-01","m":{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}}', + Name("Int64"), + ), + ( + "Map(Int8, UInt64)", + "('2020-01-01', map(1,0,2,18446744073709551615))", + '{"d":"2020-01-01","m":{1:"0",2:"18446744073709551615"}}', + Name("UInt64"), + ), + ( + "Map(Int8, Int128)", + "('2020-01-01', map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", + '{"d":"2020-01-01","m":{1:"170141183460469231731687303715884105727",2:"0",3:"-170141183460469231731687303715884105728"}}', + Name("Int128"), + ), + ( + "Map(Int8, Int256)", + "('2020-01-01', map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", + '{"d":"2020-01-01","m":{1:"57896044618658097711785492504343953926634992332820282019728792003956564819967",2:"0",3:"-57896044618658097711785492504343953926634992332820282019728792003956564819968"}}', + Name("Int256"), + ), + ( + "Map(Int8, UInt256)", + "('2020-01-01', map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", + '{"d":"2020-01-01","m":{1:"0",2:"115792089237316195423570985008687907853269984665640564039457584007913129639935"}}', + Name("UInt256"), + ), + ( + "Map(Int8, Nullable(Int8))", + "('2020-01-01', map(1,toNullable(1)))", + '{"d":"2020-01-01","m":{"1":1}}', + Name("toNullable"), + ), + ( + "Map(Int8, Nullable(Int8))", + "('2020-01-01', map(1,toNullable(NULL)))", + '{"d":"2020-01-01","m":{"1":null}}', + Name("toNullable(NULL)"), + ), + ], ) -@Examples("type data output", [ - ("Map(Int8, Int8)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{"1":127,"2":0,"3":-128}}', Name("Int8")), - ("Map(Int8, UInt8)", "('2020-01-01', map(1,0,2,255))", '{"d":"2020-01-01","m":{"1":0,"2":255}}', Name("UInt8")), - ("Map(Int8, Int16)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{"1":32767,"2":0,"3":-32768}}', Name("Int16")), - ("Map(Int8, UInt16)", "('2020-01-01', map(1,0,2,65535))", '{"d":"2020-01-01","m":{"1":0,"2":65535}}', Name("UInt16")), - ("Map(Int8, Int32)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{"1":2147483647,"2":0,"3":-2147483648}}', Name("Int32")), - ("Map(Int8, UInt32)", "('2020-01-01', map(1,0,2,4294967295))", '{"d":"2020-01-01","m":{"1":0,"2":4294967295}}', Name("UInt32")), - ("Map(Int8, Int64)", "('2020-01-01', map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{"d":"2020-01-01","m":{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}}', Name("Int64")), - ("Map(Int8, UInt64)", "('2020-01-01', map(1,0,2,18446744073709551615))", '{"d":"2020-01-01","m":{1:"0",2:"18446744073709551615"}}', Name("UInt64")), - ("Map(Int8, Int128)", "('2020-01-01', map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{"d":"2020-01-01","m":{1:"170141183460469231731687303715884105727",2:"0",3:"-170141183460469231731687303715884105728"}}', Name("Int128")), - ("Map(Int8, Int256)", "('2020-01-01', map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{"d":"2020-01-01","m":{1:"57896044618658097711785492504343953926634992332820282019728792003956564819967",2:"0",3:"-57896044618658097711785492504343953926634992332820282019728792003956564819968"}}', Name("Int256")), - ("Map(Int8, UInt256)", "('2020-01-01', map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{"d":"2020-01-01","m":{1:"0",2:"115792089237316195423570985008687907853269984665640564039457584007913129639935"}}', Name("UInt256")), - ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(1)))", '{"d":"2020-01-01","m":{"1":1}}', Name("toNullable")), - ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(NULL)))", '{"d":"2020-01-01","m":{"1":null}}', Name("toNullable(NULL)")), -]) def table_map_with_value_integer(self, type, data, output): - """Check what values we can insert into map type column with value integer. - """ + """Check what values we can insert into map type column with value integer.""" insert_into_table(type=type, data=data, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Array("1.0")) +@Examples( + "type data output", + [ + ( + "Map(String, Array(Int8))", + "('2020-01-01', map('key',[]))", + '{"d":"2020-01-01","m":{"key":[]}}', + Name("empty array"), + ), + ( + "Map(String, Array(Int8))", + "('2020-01-01', map('key',[1,2,3]))", + '{"d":"2020-01-01","m":{"key":[1,2,3]}}', + Name("non-empty array of ints"), + ), + ( + "Map(String, Array(String))", + "('2020-01-01', map('key',['1','2','3']))", + '{"d":"2020-01-01","m":{"key":["1","2","3"]}}', + Name("non-empty array of strings"), + ), + ( + "Map(String, Array(Map(Int8, Int8)))", + "('2020-01-01', map('key',[map(1,2),map(2,3)]))", + '{"d":"2020-01-01","m":{"key":[{"1":2},{"2":3}]}}', + Name("non-empty array of maps"), + ), + ( + "Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", + "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", + '{"d":"2020-01-01","m":{"key":[{"1":[{"1":[1]}]},{"2":[{"2":[3]}]}]}}', + Name("non-empty array of maps of array of maps"), + ), + ], ) -@Examples("type data output", [ - ("Map(String, Array(Int8))", "('2020-01-01', map('key',[]))", '{"d":"2020-01-01","m":{"key":[]}}', Name("empty array")), - ("Map(String, Array(Int8))", "('2020-01-01', map('key',[1,2,3]))", '{"d":"2020-01-01","m":{"key":[1,2,3]}}', Name("non-empty array of ints")), - ("Map(String, Array(String))", "('2020-01-01', map('key',['1','2','3']))", '{"d":"2020-01-01","m":{"key":["1","2","3"]}}', Name("non-empty array of strings")), - ("Map(String, Array(Map(Int8, Int8)))", "('2020-01-01', map('key',[map(1,2),map(2,3)]))", '{"d":"2020-01-01","m":{"key":[{"1":2},{"2":3}]}}', Name("non-empty array of maps")), - ("Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", '{"d":"2020-01-01","m":{"key":[{"1":[{"1":[1]}]},{"2":[{"2":[3]}]}]}}', Name("non-empty array of maps of array of maps")), -]) def table_map_with_value_array(self, type, data, output): - """Check what values we can insert into map type column with value Array. - """ + """Check what values we can insert into map type column with value Array.""" insert_into_table(type=type, data=data, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0")) +@Examples( + "type data output", + [ + ( + "Map(Int8, Int8)", + "('2020-01-01', map(127,1,0,1,-128,1))", + '{"d":"2020-01-01","m":{"127":1,"0":1,"-128":1}}', + Name("Int8"), + ), + ( + "Map(UInt8, Int8)", + "('2020-01-01', map(0,1,255,1))", + '{"d":"2020-01-01","m":{"0":1,"255":1}}', + Name("UInt8"), + ), + ( + "Map(Int16, Int8)", + "('2020-01-01', map(127,1,0,1,-128,1))", + '{"d":"2020-01-01","m":{"32767":1,"0":1,"-32768":1}}', + Name("Int16"), + ), + ( + "Map(UInt16, Int8)", + "('2020-01-01', map(0,1,65535,1))", + '{"d":"2020-01-01","m":{"0":1,"65535":1}}', + Name("UInt16"), + ), + ( + "Map(Int32, Int8)", + "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", + '{"d":"2020-01-01","m":{"2147483647":1,"0":1,"-2147483648":1}}', + Name("Int32"), + ), + ( + "Map(UInt32, Int8)", + "('2020-01-01', map(0,1,4294967295,1))", + '{"d":"2020-01-01","m":{"0":1,"4294967295":1}}', + Name("UInt32"), + ), + ( + "Map(Int64, Int8)", + "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", + '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', + Name("Int64"), + ), + ( + "Map(UInt64, Int8)", + "('2020-01-01', map(0,1,18446744073709551615,1))", + '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', + Name("UInt64"), + ), + ( + "Map(Int128, Int8)", + "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", + '{"d":"2020-01-01","m":{170141183460469231731687303715884105727:1,0:1,"-170141183460469231731687303715884105728":1}}', + Name("Int128"), + ), + ( + "Map(Int256, Int8)", + "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", + '{"d":"2020-01-01","m":{"57896044618658097711785492504343953926634992332820282019728792003956564819967":1,"0":1,"-57896044618658097711785492504343953926634992332820282019728792003956564819968":1}}', + Name("Int256"), + ), + ( + "Map(UInt256, Int8)", + "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", + '{"d":"2020-01-01","m":{"0":1,"115792089237316195423570985008687907853269984665640564039457584007913129639935":1}}', + Name("UInt256"), + ), + ( + "Map(Nullable(Int8), Int8)", + "('2020-01-01', map(toNullable(1),1))", + '{"d":"2020-01-01","m":{1:1}}', + Name("toNullable"), + ), + ( + "Map(Nullable(Int8), Int8)", + "('2020-01-01', map(toNullable(NULL),1))", + '{"d":"2020-01-01","m":{null:1}}', + Name("toNullable(NULL)"), + ), + ], ) -@Examples("type data output", [ - ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{"127":1,"0":1,"-128":1}}', Name("Int8")), - ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{"0":1,"255":1}}', Name("UInt8")), - ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{"32767":1,"0":1,"-32768":1}}', Name("Int16")), - ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{"0":1,"65535":1}}', Name("UInt16")), - ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"d":"2020-01-01","m":{"2147483647":1,"0":1,"-2147483648":1}}', Name("Int32")), - ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{"0":1,"4294967295":1}}', Name("UInt32")), - ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', Name("Int64")), - ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', Name("UInt64")), - ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"d":"2020-01-01","m":{170141183460469231731687303715884105727:1,0:1,"-170141183460469231731687303715884105728":1}}', Name("Int128")), - ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"d":"2020-01-01","m":{"57896044618658097711785492504343953926634992332820282019728792003956564819967":1,"0":1,"-57896044618658097711785492504343953926634992332820282019728792003956564819968":1}}', Name("Int256")), - ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"d":"2020-01-01","m":{"0":1,"115792089237316195423570985008687907853269984665640564039457584007913129639935":1}}', Name("UInt256")), - ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), - ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"d":"2020-01-01","m":{null:1}}', Name("toNullable(NULL)")), -]) def table_map_with_key_integer(self, type, data, output): - """Check what values we can insert into map type column with key integer. - """ + """Check what values we can insert into map type column with key integer.""" insert_into_table(type=type, data=data, output=output) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0")) +@Examples( + "type data output select", + [ + ( + "Map(Int8, Int8)", + "('2020-01-01', map(127,1,0,1,-128,1))", + '{"m":1}', + "m[127] AS m", + Name("Int8"), + ), + ( + "Map(UInt8, Int8)", + "('2020-01-01', map(0,1,255,1))", + '{"m":2}', + "(m[255] + m[0]) AS m", + Name("UInt8"), + ), + ( + "Map(Int16, Int8)", + "('2020-01-01', map(127,1,0,1,-128,1))", + '{"m":3}', + "(m[-128] + m[0] + m[-128]) AS m", + Name("Int16"), + ), + ( + "Map(UInt16, Int8)", + "('2020-01-01', map(0,1,65535,1))", + '{"m":2}', + "(m[0] + m[65535]) AS m", + Name("UInt16"), + ), + ( + "Map(Int32, Int8)", + "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", + '{"m":3}', + "(m[2147483647] + m[0] + m[-2147483648]) AS m", + Name("Int32"), + ), + ( + "Map(UInt32, Int8)", + "('2020-01-01', map(0,1,4294967295,1))", + '{"m":2}', + "(m[0] + m[4294967295]) AS m", + Name("UInt32"), + ), + ( + "Map(Int64, Int8)", + "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", + '{"m":3}', + "(m[9223372036854775807] + m[0] + m[-9223372036854775808]) AS m", + Name("Int64"), + ), + ( + "Map(UInt64, Int8)", + "('2020-01-01', map(0,1,18446744073709551615,1))", + '{"m":2}', + "(m[0] + m[18446744073709551615]) AS m", + Name("UInt64"), + ), + ( + "Map(Int128, Int8)", + "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", + '{"m":3}', + "(m[170141183460469231731687303715884105727] + m[0] + m[-170141183460469231731687303715884105728]) AS m", + Name("Int128"), + ), + ( + "Map(Int256, Int8)", + "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", + '{"m":3}', + "(m[57896044618658097711785492504343953926634992332820282019728792003956564819967] + m[0] + m[-57896044618658097711785492504343953926634992332820282019728792003956564819968]) AS m", + Name("Int256"), + ), + ( + "Map(UInt256, Int8)", + "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", + '{"m":2}', + "(m[0] + m[115792089237316195423570985008687907853269984665640564039457584007913129639935]) AS m", + Name("UInt256"), + ), + ( + "Map(Nullable(Int8), Int8)", + "('2020-01-01', map(toNullable(1),1))", + '{"m":1}', + "m[1] AS m", + Name("toNullable"), + ), + ( + "Map(Nullable(Int8), Int8)", + "('2020-01-01', map(toNullable(NULL),1))", + '{"m":1}', + "m[null] AS m", + Name("toNullable(NULL)"), + ), + ], ) -@Examples("type data output select", [ - ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"m":1}', "m[127] AS m", Name("Int8")), - ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"m":2}', "(m[255] + m[0]) AS m", Name("UInt8")), - ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"m":3}', "(m[-128] + m[0] + m[-128]) AS m", Name("Int16")), - ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"m":2}', "(m[0] + m[65535]) AS m", Name("UInt16")), - ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"m":3}', "(m[2147483647] + m[0] + m[-2147483648]) AS m", Name("Int32")), - ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"m":2}', "(m[0] + m[4294967295]) AS m", Name("UInt32")), - ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"m":3}', "(m[9223372036854775807] + m[0] + m[-9223372036854775808]) AS m", Name("Int64")), - ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"m":2}', "(m[0] + m[18446744073709551615]) AS m", Name("UInt64")), - ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"m":3}', "(m[170141183460469231731687303715884105727] + m[0] + m[-170141183460469231731687303715884105728]) AS m", Name("Int128")), - ("Map(Int256, Int8)", "('2020-01-01', map(57896044618658097711785492504343953926634992332820282019728792003956564819967,1,0,1,-57896044618658097711785492504343953926634992332820282019728792003956564819968,1))", '{"m":3}', "(m[57896044618658097711785492504343953926634992332820282019728792003956564819967] + m[0] + m[-57896044618658097711785492504343953926634992332820282019728792003956564819968]) AS m", Name("Int256")), - ("Map(UInt256, Int8)", "('2020-01-01', map(0,1,115792089237316195423570985008687907853269984665640564039457584007913129639935,1))", '{"m":2}', "(m[0] + m[115792089237316195423570985008687907853269984665640564039457584007913129639935]) AS m", Name("UInt256")), - ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(1),1))", '{"m":1}', "m[1] AS m", Name("toNullable")), - ("Map(Nullable(Int8), Int8)", "('2020-01-01', map(toNullable(NULL),1))", '{"m":1}', "m[null] AS m", Name("toNullable(NULL)")), -]) def table_map_select_key_with_key_integer(self, type, data, output, select): - """Check what values we can insert into map type column with key integer and if we can use the key to select the value. - """ + """Check what values we can insert into map type column with key integer and if we can use the key to select the value.""" insert_into_table(type=type, data=data, output=output, select=select) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_ArrayOfMaps("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_NestedWithMaps("1.0"), +) +@Examples( + "type data output partition_by", + [ + ( + "Array(Map(String, Int8))", + "('2020-01-01', [map('hello',1),map('hello',1,'there',2)])", + '{"d":"2020-01-01","m":[{"hello":1},{"hello":1,"there":2}]}', + "m", + Name("Array(Map(String, Int8))"), + ), + ( + "Nested(x Map(String, Int8))", + "('2020-01-01', [map('hello',1)])", + '{"d":"2020-01-01","m.x":[{"hello":1}]}', + "m.x", + Name("Nested(x Map(String, Int8)"), + ), + ], ) -@Examples("type data output partition_by", [ - ("Array(Map(String, Int8))", - "('2020-01-01', [map('hello',1),map('hello',1,'there',2)])", - '{"d":"2020-01-01","m":[{"hello":1},{"hello":1,"there":2}]}', - "m", - Name("Array(Map(String, Int8))")), - ("Nested(x Map(String, Int8))", - "('2020-01-01', [map('hello',1)])", - '{"d":"2020-01-01","m.x":[{"hello":1}]}', - "m.x", - Name("Nested(x Map(String, Int8)")) -]) def table_with_map_inside_another_type(self, type, data, output, partition_by): - """Check what values we can insert into a type that has map type. - """ + """Check what values we can insert into a type that has map type.""" insert_into_table(type=type, data=data, output=output, partition_by=partition_by) + @TestOutline def insert_into_table(self, type, data, output, partition_by="m", select="*"): - """Check we can insert data into a table. - """ + """Check we can insert data into a table.""" uid = getuid() node = self.context.node with Given(f"table definition with {type}"): - sql = "CREATE TABLE {name} (d DATE, m " + type + ") ENGINE = MergeTree() PARTITION BY " + partition_by + " ORDER BY d" + sql = ( + "CREATE TABLE {name} (d DATE, m " + + type + + ") ENGINE = MergeTree() PARTITION BY " + + partition_by + + " ORDER BY d" + ) with Given(f"I create a table", description=sql): table = create_table(name=uid, statement=sql) @@ -382,30 +1021,34 @@ def insert_into_table(self, type, data, output, partition_by="m", select="*"): with Then("I expect output to match", description=output): assert r.output == output, error() + @TestScenario @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MixedKeyOrValueTypes("1.0") ) def select_map_with_invalid_mixed_key_and_value_types(self): - """Check that creating a map with mixed key types fails. - """ + """Check that creating a map with mixed key types fails.""" node = self.context.node exitcode = 130 message = "DB::Exception: There is no supertype for types String, UInt8 because some of them are String/FixedString and some of them are not" - with Check("attempt to create a map using SELECT with mixed key types then it fails"): + with Check( + "attempt to create a map using SELECT with mixed key types then it fails" + ): node.query("SELECT map('hello',1,2,3)", exitcode=exitcode, message=message) - with Check("attempt to create a map using SELECT with mixed value types then it fails"): + with Check( + "attempt to create a map using SELECT with mixed value types then it fails" + ): node.query("SELECT map(1,'hello',2,2)", exitcode=exitcode, message=message) + @TestScenario @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_InvalidNumberOfArguments("1.0") ) def select_map_with_invalid_number_of_arguments(self): - """Check that creating a map with invalid number of arguments fails. - """ + """Check that creating a map with invalid number of arguments fails.""" node = self.context.node exitcode = 42 message = "DB::Exception: Function map requires even number of arguments" @@ -413,10 +1056,10 @@ def select_map_with_invalid_number_of_arguments(self): with When("I create a map using SELECT with invalid number of arguments"): node.query("SELECT map(1,2,3)", exitcode=exitcode, message=message) + @TestScenario def select_map_empty(self): - """Check that we can can create a empty map by not passing any arguments. - """ + """Check that we can can create a empty map by not passing any arguments.""" node = self.context.node with When("I create a map using SELECT with no arguments"): @@ -425,10 +1068,10 @@ def select_map_empty(self): with Then("it should create an empty map"): assert r.output == "{}", error() + @TestScenario def insert_invalid_mixed_key_and_value_types(self): - """Check that inserting a map with mixed key or value types fails. - """ + """Check that inserting a map with mixed key or value types fails.""" uid = getuid() node = self.context.node exitcode = 130 @@ -448,47 +1091,64 @@ def insert_invalid_mixed_key_and_value_types(self): sql = f"INSERT INTO {table} VALUES ('2020-01-01', map(1,'hello',2,2))" node.query(sql, exitcode=exitcode, message=message) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0")) +@Examples( + "type data output", + [ + ( + "Map(String, String)", + "('2020-01-01', map('hello','there','hello','over there'))", + '{"d":"2020-01-01","m":{"hello":"there","hello":"over there"}}', + Name("Map(String, String))"), + ), + ( + "Map(Int64, String)", + "('2020-01-01', map(12345,'there',12345,'over there'))", + '{"d":"2020-01-01","m":{"12345":"there","12345":"over there"}}', + Name("Map(Int64, String))"), + ), + ], ) -@Examples("type data output", [ - ("Map(String, String)", - "('2020-01-01', map('hello','there','hello','over there'))", - '{"d":"2020-01-01","m":{"hello":"there","hello":"over there"}}', - Name("Map(String, String))")), - ("Map(Int64, String)", - "('2020-01-01', map(12345,'there',12345,'over there'))", - '{"d":"2020-01-01","m":{"12345":"there","12345":"over there"}}', - Name("Map(Int64, String))")), -]) def table_map_with_duplicated_keys(self, type, data, output): - """Check that map supports duplicated keys. - """ + """Check that map supports duplicated keys.""" insert_into_table(type=type, data=data, output=output) -@TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0") -) -@Examples("map output", [ - ("map('hello','there','hello','over there')", "{'hello':'there','hello':'over there'}", Name("String")), - ("map(12345,'there',12345,'over there')", "{12345:'there',12345:'over there'}", Name("Integer")) -]) -def select_map_with_duplicated_keys(self, map, output): - """Check creating a map with duplicated keys. - """ - select_map(map=map, output=output) @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_DuplicatedKeys("1.0")) +@Examples( + "map output", + [ + ( + "map('hello','there','hello','over there')", + "{'hello':'there','hello':'over there'}", + Name("String"), + ), + ( + "map(12345,'there',12345,'over there')", + "{12345:'there',12345:'over there'}", + Name("Integer"), + ), + ], ) +def select_map_with_duplicated_keys(self, map, output): + """Check creating a map with duplicated keys.""" + select_map(map=map, output=output) + + +@TestOutline(Scenario) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0")) def select_map_key_not_found(self): node = self.context.node with When("map is empty"): - node.query("SELECT map() AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + node.query( + "SELECT map() AS m, m[1]", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) with When("map has integer values"): r = node.query("SELECT map(1,2) AS m, m[2] FORMAT Values") @@ -505,19 +1165,48 @@ def select_map_key_not_found(self): with Then("empty array be returned for key that is not found"): assert r.output == "({1:[2]},[])", error() + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyNotFound("1.0")) +@Examples( + "type data select exitcode message", + [ + ( + "Map(UInt8, UInt8), y Int8", + "(y) VALUES (1)", + "m[1] AS v", + 0, + '{"v":0}', + Name("empty map"), + ), + ( + "Map(UInt8, UInt8)", + "VALUES (map(1,2))", + "m[2] AS v", + 0, + '{"v":0}', + Name("map has integer values"), + ), + ( + "Map(UInt8, String)", + "VALUES (map(1,'2'))", + "m[2] AS v", + 0, + '{"v":""}', + Name("map has string values"), + ), + ( + "Map(UInt8, Array(Int8))", + "VALUES (map(1,[2]))", + "m[2] AS v", + 0, + '{"v":[]}', + Name("map has array values"), + ), + ], ) -@Examples("type data select exitcode message", [ - ("Map(UInt8, UInt8), y Int8", "(y) VALUES (1)", "m[1] AS v", 0, '{"v":0}', Name("empty map")), - ("Map(UInt8, UInt8)", "VALUES (map(1,2))", "m[2] AS v", 0, '{"v":0}', Name("map has integer values")), - ("Map(UInt8, String)", "VALUES (map(1,'2'))", "m[2] AS v", 0, '{"v":""}', Name("map has string values")), - ("Map(UInt8, Array(Int8))", "VALUES (map(1,[2]))", "m[2] AS v", 0, '{"v":[]}', Name("map has array values")), -]) def table_map_key_not_found(self, type, data, select, exitcode, message, order_by=None): - """Check values returned from a map column when key is not found. - """ + """Check values returned from a map column when key is not found.""" uid = getuid() node = self.context.node @@ -525,7 +1214,12 @@ def table_map_key_not_found(self, type, data, select, exitcode, message, order_b order_by = "m" with Given(f"table definition with {type}"): - sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + sql = ( + "CREATE TABLE {name} (m " + + type + + ") ENGINE = MergeTree() ORDER BY " + + order_by + ) with And(f"I create a table", description=sql): table = create_table(name=uid, statement=sql) @@ -534,67 +1228,185 @@ def table_map_key_not_found(self, type, data, select, exitcode, message, order_b node.query(f"INSERT INTO {table} {data}") with And("I try to read from the table"): - node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + node.query( + f"SELECT {select} FROM {table} FORMAT JSONEachRow", + exitcode=exitcode, + message=message, + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0")) def invalid_key(self): - """Check when key is not valid. - """ + """Check when key is not valid.""" node = self.context.node with When("I try to use an integer key that is too large"): - node.query("SELECT map(1,2) AS m, m[256]", exitcode=43, message="DB::Exception: Illegal types of arguments") + node.query( + "SELECT map(1,2) AS m, m[256]", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) with When("I try to use an integer key that is negative when key is unsigned"): - node.query("SELECT map(1,2) AS m, m[-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + node.query( + "SELECT map(1,2) AS m, m[-1]", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) with When("I try to use a string key when key is an integer"): - node.query("SELECT map(1,2) AS m, m['1']", exitcode=43, message="DB::Exception: Illegal types of arguments") + node.query( + "SELECT map(1,2) AS m, m['1']", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) with When("I try to use an integer key when key is a string"): - r = node.query("SELECT map('1',2) AS m, m[1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + r = node.query( + "SELECT map('1',2) AS m, m[1]", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) with When("I try to use an empty key when key is a string"): - r = node.query("SELECT map('1',2) AS m, m[]", exitcode=62, message="DB::Exception: Syntax error: failed at position") + r = node.query( + "SELECT map('1',2) AS m, m[]", + exitcode=62, + message="DB::Exception: Syntax error: failed at position", + ) with When("I try to use wrong type conversion in key"): - r = node.query("SELECT map(1,2) AS m, m[toInt8('1')]", exitcode=43, message="DB::Exception: Illegal types of arguments") + r = node.query( + "SELECT map(1,2) AS m, m[toInt8('1')]", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) - with When("in array of maps I try to use an integer key that is negative when key is unsigned"): - node.query("SELECT [map(1,2)] AS m, m[1][-1]", exitcode=43, message="DB::Exception: Illegal types of arguments") + with When( + "in array of maps I try to use an integer key that is negative when key is unsigned" + ): + node.query( + "SELECT [map(1,2)] AS m, m[1][-1]", + exitcode=43, + message="DB::Exception: Illegal types of arguments", + ) with When("I try to use a NULL key when key is not nullable"): r = node.query("SELECT map(1,2) AS m, m[NULL] FORMAT Values") with Then("it should return NULL"): assert r.output == "({1:2},NULL)", error() + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval_KeyInvalid("1.0")) +@Examples( + "type data select exitcode message order_by", + [ + ( + "Map(UInt8, UInt8)", + "(map(1,2))", + "m[256] AS v", + 0, + '{"v":0}', + "m", + Name("key too large)"), + ), + ( + "Map(UInt8, UInt8)", + "(map(1,2))", + "m[-1] AS v", + 0, + '{"v":0}', + "m", + Name("key is negative"), + ), + ( + "Map(UInt8, UInt8)", + "(map(1,2))", + "m['1'] AS v", + 43, + "DB::Exception: Illegal types of arguments", + "m", + Name("string when key is integer"), + ), + ( + "Map(String, UInt8)", + "(map('1',2))", + "m[1] AS v", + 43, + "DB::Exception: Illegal types of arguments", + "m", + Name("integer when key is string"), + ), + ( + "Map(String, UInt8)", + "(map('1',2))", + "m[] AS v", + 62, + "DB::Exception: Syntax error: failed at position", + "m", + Name("empty when key is string"), + ), + ( + "Map(UInt8, UInt8)", + "(map(1,2))", + "m[toInt8('1')] AS v", + 0, + '{"v":2}', + "m", + Name("wrong type conversion when key is integer"), + ), + ( + "Map(String, UInt8)", + "(map('1',2))", + "m[toFixedString('1',1)] AS v", + 0, + '{"v":2}', + "m", + Name("wrong type conversion when key is string"), + ), + ( + "Map(UInt8, UInt8)", + "(map(1,2))", + "m[NULL] AS v", + 0, + '{"v":null}', + "m", + Name("NULL key when key is not nullable"), + ), + ( + "Array(Map(UInt8, UInt8))", + "([map(1,2)])", + "m[1]['1'] AS v", + 43, + "DB::Exception: Illegal types of arguments", + "m", + Name("string when key is integer in array of maps"), + ), + ( + "Nested(x Map(UInt8, UInt8))", + "([map(1,2)])", + "m.x[1]['1'] AS v", + 43, + "DB::Exception: Illegal types of arguments", + "m.x", + Name("string when key is integer in nested map"), + ), + ], ) -@Examples("type data select exitcode message order_by", [ - ("Map(UInt8, UInt8)", "(map(1,2))", "m[256] AS v", 0, '{"v":0}', "m", Name("key too large)")), - ("Map(UInt8, UInt8)", "(map(1,2))", "m[-1] AS v", 0, '{"v":0}', "m", Name("key is negative")), - ("Map(UInt8, UInt8)", "(map(1,2))", "m['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer")), - ("Map(String, UInt8)", "(map('1',2))", "m[1] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("integer when key is string")), - ("Map(String, UInt8)", "(map('1',2))", "m[] AS v", 62, "DB::Exception: Syntax error: failed at position", "m", Name("empty when key is string")), - ("Map(UInt8, UInt8)", "(map(1,2))", "m[toInt8('1')] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is integer")), - ("Map(String, UInt8)", "(map('1',2))", "m[toFixedString('1',1)] AS v", 0, '{"v":2}', "m", Name("wrong type conversion when key is string")), - ("Map(UInt8, UInt8)", "(map(1,2))", "m[NULL] AS v", 0, '{"v":null}', "m", Name("NULL key when key is not nullable")), - ("Array(Map(UInt8, UInt8))", "([map(1,2)])", "m[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m", Name("string when key is integer in array of maps")), - ("Nested(x Map(UInt8, UInt8))", "([map(1,2)])", "m.x[1]['1'] AS v", 43, "DB::Exception: Illegal types of arguments", "m.x", Name("string when key is integer in nested map")), -]) def table_map_invalid_key(self, type, data, select, exitcode, message, order_by="m"): - """Check selecting values from a map column using an invalid key. - """ + """Check selecting values from a map column using an invalid key.""" uid = getuid() node = self.context.node with Given(f"table definition with {type}"): - sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + sql = ( + "CREATE TABLE {name} (m " + + type + + ") ENGINE = MergeTree() ORDER BY " + + order_by + ) with And(f"I create a table", description=sql): table = create_table(name=uid, statement=sql) @@ -603,35 +1415,114 @@ def table_map_invalid_key(self, type, data, select, exitcode, message, order_by= node.query(f"INSERT INTO {table} VALUES {data}") with And("I try to read from the table"): - node.query(f"SELECT {select} FROM {table} FORMAT JSONEachRow", exitcode=exitcode, message=message) + node.query( + f"SELECT {select} FROM {table} FORMAT JSONEachRow", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval("1.0") +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Value_Retrieval("1.0")) +@Examples( + "type data select filter exitcode message order_by", + [ + ( + "Map(UInt8, UInt8)", + "(map(1,1)),(map(1,2)),(map(2,3))", + "m[1] AS v", + "1=1 ORDER BY m[1]", + 0, + '{"v":0}\n{"v":1}\n{"v":2}', + None, + Name("select the same key from all the rows"), + ), + ( + "Map(String, String)", + "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", + "m", + "m = map('e','f','c','d')", + 0, + "", + None, + Name("filter rows by map having different pair order"), + ), + ( + "Map(String, String)", + "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", + "m", + "m = map('c','d','e','f')", + 0, + '{"m":{"c":"d","e":"f"}}', + None, + Name("filter rows by map having the same pair order"), + ), + ( + "Map(String, String)", + "(map('a','b')),(map('e','f'))", + "m", + "m = map()", + 0, + "", + None, + Name("filter rows by empty map"), + ), + ( + "Map(String, Int8)", + "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", + "m", + "m['a'] = 1", + 0, + '{"m":{"a":1,"b":2}}', + None, + Name("filter rows by map key value"), + ), + ( + "Map(String, Int8)", + "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", + "m", + "m['a'] = 1 AND m['b'] = 2", + 0, + '{"m":{"a":1,"b":2}}', + None, + Name("filter rows by map multiple key value combined with AND"), + ), + ( + "Map(String, Int8)", + "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", + "m", + "m['a'] = 1 OR m['b'] = 3", + 0, + '{"m":{"a":1,"b":2}}\n{"m":{"b":3}}', + None, + Name("filter rows by map multiple key value combined with OR"), + ), + ( + "Map(String, Array(Int8))", + "(map('a',[])),(map('b',[1])),(map('c',[2]))", + "m['b'] AS v", + "m['b'] IN ([1],[2])", + 0, + '{"v":[1]}', + None, + Name("filter rows by map array value using IN"), + ), + ( + "Map(String, Nullable(String))", + "(map('a',NULL)),(map('a',1))", + "m", + "isNull(m['a']) = 1", + 0, + '{"m":{"a":null}}', + None, + Name("select map with nullable value"), + ), + ], ) -@Examples("type data select filter exitcode message order_by", [ - ("Map(UInt8, UInt8)", "(map(1,1)),(map(1,2)),(map(2,3))", "m[1] AS v", "1=1 ORDER BY m[1]", 0, '{"v":0}\n{"v":1}\n{"v":2}', None, - Name("select the same key from all the rows")), - ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('e','f','c','d')", 0, '', None, - Name("filter rows by map having different pair order")), - ("Map(String, String)", "(map('a','b')),(map('c','d','e','f')),(map('e','f'))", "m", "m = map('c','d','e','f')", 0, '{"m":{"c":"d","e":"f"}}', None, - Name("filter rows by map having the same pair order")), - ("Map(String, String)", "(map('a','b')),(map('e','f'))", "m", "m = map()", 0, '', None, - Name("filter rows by empty map")), - ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1", 0, '{"m":{"a":1,"b":2}}', None, - Name("filter rows by map key value")), - ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 AND m['b'] = 2", 0, '{"m":{"a":1,"b":2}}', None, - Name("filter rows by map multiple key value combined with AND")), - ("Map(String, Int8)", "(map('a',1,'b',2)),(map('a',2)),(map('b',3))", "m", "m['a'] = 1 OR m['b'] = 3", 0, '{"m":{"a":1,"b":2}}\n{"m":{"b":3}}', None, - Name("filter rows by map multiple key value combined with OR")), - ("Map(String, Array(Int8))", "(map('a',[])),(map('b',[1])),(map('c',[2]))", "m['b'] AS v", "m['b'] IN ([1],[2])", 0, '{"v":[1]}', None, - Name("filter rows by map array value using IN")), - ("Map(String, Nullable(String))", "(map('a',NULL)),(map('a',1))", "m", "isNull(m['a']) = 1", 0, '{"m":{"a":null}}', None, - Name("select map with nullable value")) -]) -def table_map_queries(self, type, data, select, filter, exitcode, message, order_by=None): - """Check retrieving map values and using maps in queries. - """ +def table_map_queries( + self, type, data, select, filter, exitcode, message, order_by=None +): + """Check retrieving map values and using maps in queries.""" uid = getuid() node = self.context.node @@ -639,7 +1530,12 @@ def table_map_queries(self, type, data, select, filter, exitcode, message, order order_by = "m" with Given(f"table definition with {type}"): - sql = "CREATE TABLE {name} (m " + type + ") ENGINE = MergeTree() ORDER BY " + order_by + sql = ( + "CREATE TABLE {name} (m " + + type + + ") ENGINE = MergeTree() ORDER BY " + + order_by + ) with And(f"I create a table", description=sql): table = create_table(name=uid, statement=sql) @@ -648,24 +1544,37 @@ def table_map_queries(self, type, data, select, filter, exitcode, message, order node.query(f"INSERT INTO {table} VALUES {data}") with And("I try to read from the table"): - node.query(f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", exitcode=exitcode, message=message) + node.query( + f"SELECT {select} FROM {table} WHERE {filter} FORMAT JSONEachRow", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Invalid_Nullable("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_Invalid_NothingNothing("1.0"), +) +@Examples( + "type exitcode message", + [ + ( + "Nullable(Map(String, String))", + 43, + "DB::Exception: Nested type Map(String,String) cannot be inside Nullable type", + Name("nullable map"), + ), + ( + "Map(Nothing, Nothing)", + 37, + "DB::Exception: Column `m` with type Map(Nothing,Nothing) is not allowed in key expression, it's not comparable", + Name("map with nothing type for key and value"), + ), + ], ) -@Examples("type exitcode message", [ - ("Nullable(Map(String, String))", - 43, "DB::Exception: Nested type Map(String,String) cannot be inside Nullable type", - Name("nullable map")), - ("Map(Nothing, Nothing)", - 37, "DB::Exception: Column `m` with type Map(Nothing,Nothing) is not allowed in key expression, it's not comparable", - Name("map with nothing type for key and value")) -]) def table_map_unsupported_types(self, type, exitcode, message): - """Check creating a table with unsupported map column types. - """ + """Check creating a table with unsupported map column types.""" uid = getuid() node = self.context.node @@ -677,109 +1586,265 @@ def table_map_unsupported_types(self, type, exitcode, message): with Finally("drop table if any"): node.query(f"DROP TABLE IF EXISTS {uid}") + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0"), +) +@Examples( + "tuple type exitcode message", + [ + ( + "([1, 2, 3], ['Ready', 'Steady', 'Go'])", + "Map(UInt8, String)", + 0, + "{1:'Ready',2:'Steady',3:'Go'}", + Name("int -> int"), + ), + ( + "([1, 2, 3], ['Ready', 'Steady', 'Go'])", + "Map(String, String)", + 0, + "{'1':'Ready','2':'Steady','3':'Go'}", + Name("int -> string"), + ), + ( + "(['1', '2', '3'], ['Ready', 'Steady', 'Go'])", + "Map(UInt8, String)", + 0, + "{1:'Ready',187:'Steady',143:'Go'}", + Name("string -> int"), + ), + ( + "([],[])", + "Map(String, String)", + 0, + "{}", + Name("empty arrays to map str:str"), + ), + ( + "([],[])", + "Map(UInt8, Array(Int8))", + 0, + "{}", + Name("empty arrays to map uint8:array"), + ), + ( + "([[1]],['hello'])", + "Map(String, String)", + 0, + "{'[1]':'hello'}", + Name("array -> string"), + ), + ( + "([(1,2),(3,4)])", + "Map(UInt8, UInt8)", + 0, + "{1:2,3:4}", + Name("array of two tuples"), + ), + ( + "([1, 2], ['Ready', 'Steady', 'Go'])", + "Map(UInt8, String)", + 53, + "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", + Name("unequal array sizes"), + ), + ], ) -@Examples("tuple type exitcode message", [ - ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", - 0, "{1:'Ready',2:'Steady',3:'Go'}", Name("int -> int")), - ("([1, 2, 3], ['Ready', 'Steady', 'Go'])", "Map(String, String)", - 0, "{'1':'Ready','2':'Steady','3':'Go'}", Name("int -> string")), - ("(['1', '2', '3'], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", - 0, "{1:'Ready',187:'Steady',143:'Go'}", Name("string -> int")), - ("([],[])", "Map(String, String)", - 0, "{}", Name("empty arrays to map str:str")), - ("([],[])", "Map(UInt8, Array(Int8))", - 0, "{}", Name("empty arrays to map uint8:array")), - ("([[1]],['hello'])", "Map(String, String)", - 0, "{'[1]':'hello'}", Name("array -> string")), - ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", - 0, "{1:2,3:4}", Name("array of two tuples")), - ("([1, 2], ['Ready', 'Steady', 'Go'])", "Map(UInt8, String)", - 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", - Name("unequal array sizes")), -]) def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): - """Check casting Tuple(Array, Array) to a map type. - """ + """Check casting Tuple(Array, Array) to a map type.""" node = self.context.node with When("I try to cast tuple", description=tuple): - node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + node.query( + f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysToMap("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_TupleOfArraysMap_Invalid("1.0"), ) -@Examples("tuple type exitcode message check_insert", [ - ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", - 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> int")), - ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(String, String)", - 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> string")), - ("((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", - 0, '', True, Name("string -> int")), - ("(([],[]))", "Map(String, String)", - 0, '{"m":{}}', False, Name("empty arrays to map str:str")), - ("(([],[]))", "Map(UInt8, Array(Int8))", - 0, '{"m":{}}', False, Name("empty arrays to map uint8:array")), - ("(([[1]],['hello']))", "Map(String, String)", - 53, 'DB::Exception: Type mismatch in IN or VALUES section', True, Name("array -> string")), - ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", - 0, '{"m":{"1":2,"3":4}}', False, Name("array of two tuples")), - ("(([1, 2], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", - 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", True, - Name("unequal array sizes")), -]) -def table_map_cast_tuple_of_arrays_to_map(self, tuple, type, exitcode, message, check_insert): - """Check converting Tuple(Array, Array) into map on insert into a map type column. - """ - table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) +@Examples( + "tuple type exitcode message check_insert", + [ + ( + "(([1, 2, 3], ['Ready', 'Steady', 'Go']))", + "Map(UInt8, String)", + 0, + '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', + False, + Name("int -> int"), + ), + ( + "(([1, 2, 3], ['Ready', 'Steady', 'Go']))", + "Map(String, String)", + 0, + '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', + False, + Name("int -> string"), + ), + ( + "((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", + "Map(UInt8, String)", + 0, + "", + True, + Name("string -> int"), + ), + ( + "(([],[]))", + "Map(String, String)", + 0, + '{"m":{}}', + False, + Name("empty arrays to map str:str"), + ), + ( + "(([],[]))", + "Map(UInt8, Array(Int8))", + 0, + '{"m":{}}', + False, + Name("empty arrays to map uint8:array"), + ), + ( + "(([[1]],['hello']))", + "Map(String, String)", + 53, + "DB::Exception: Type mismatch in IN or VALUES section", + True, + Name("array -> string"), + ), + ( + "(([(1,2),(3,4)]))", + "Map(UInt8, UInt8)", + 0, + '{"m":{"1":2,"3":4}}', + False, + Name("array of two tuples"), + ), + ( + "(([1, 2], ['Ready', 'Steady', 'Go']))", + "Map(UInt8, String)", + 53, + "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", + True, + Name("unequal array sizes"), + ), + ], +) +def table_map_cast_tuple_of_arrays_to_map( + self, tuple, type, exitcode, message, check_insert +): + """Check converting Tuple(Array, Array) into map on insert into a map type column.""" + table_map( + type=type, + data=tuple, + select="*", + filter="1=1", + exitcode=exitcode, + message=message, + check_insert=check_insert, + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid( + "1.0" + ), +) +@Examples( + "tuple type exitcode message", + [ + ( + "([(1,2),(3,4)])", + "Map(UInt8, UInt8)", + 0, + "{1:2,3:4}", + Name("array of two tuples"), + ), + ( + "([(1,2),(3)])", + "Map(UInt8, UInt8)", + 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", + Name("not a tuple"), + ), + ( + "([(1,2),(3,)])", + "Map(UInt8, UInt8)", + 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", + Name("invalid tuple"), + ), + ], ) -@Examples("tuple type exitcode message", [ - ("([(1,2),(3,4)])", "Map(UInt8, UInt8)", 0, "{1:2,3:4}", - Name("array of two tuples")), - ("([(1,2),(3)])", "Map(UInt8, UInt8)", 130, - "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", - Name("not a tuple")), - ("([(1,2),(3,)])", "Map(UInt8, UInt8)", 130, - "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", - Name("invalid tuple")), -]) def cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message): - """Check casting Array(Tuple(K,V)) to a map type. - """ + """Check casting Array(Tuple(K,V)) to a map type.""" node = self.context.node with When("I try to cast tuple", description=tuple): - node.query(f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message) + node.query( + f"SELECT CAST({tuple}, '{type}') AS map", exitcode=exitcode, message=message + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid( + "1.0" + ), ) -@Examples("tuple type exitcode message check_insert", [ - ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", 0, '{"m":{"1":2,"3":4}}', False, - Name("array of two tuples")), - ("(([(1,2),(3)]))", "Map(UInt8, UInt8)", 130, - "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", True, - Name("not a tuple")), - ("(([(1,2),(3,)]))", "Map(UInt8, UInt8)", 130, - "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", True, - Name("invalid tuple")), -]) -def table_map_cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message, check_insert): - """Check converting Array(Tuple(K,V),...) into map on insert into a map type column. - """ - table_map(type=type, data=tuple, select="*", filter="1=1", exitcode=exitcode, message=message, check_insert=check_insert) +@Examples( + "tuple type exitcode message check_insert", + [ + ( + "(([(1,2),(3,4)]))", + "Map(UInt8, UInt8)", + 0, + '{"m":{"1":2,"3":4}}', + False, + Name("array of two tuples"), + ), + ( + "(([(1,2),(3)]))", + "Map(UInt8, UInt8)", + 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", + True, + Name("not a tuple"), + ), + ( + "(([(1,2),(3,)]))", + "Map(UInt8, UInt8)", + 130, + "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), Tuple(UInt8) because Tuples have different sizes", + True, + Name("invalid tuple"), + ), + ], +) +def table_map_cast_array_of_two_tuples_to_map( + self, tuple, type, exitcode, message, check_insert +): + """Check converting Array(Tuple(K,V),...) into map on insert into a map type column.""" + table_map( + type=type, + data=tuple, + select="*", + filter="1=1", + exitcode=exitcode, + message=message, + check_insert=check_insert, + ) + @TestScenario @Requirements( @@ -791,7 +1856,12 @@ def subcolumns_keys_using_inline_defined_map(self): message = "DB::Exception: Missing columns: 'c.keys'" with When("I try to access keys sub-column using an inline defined map"): - node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.keys", exitcode=exitcode, message=message) + node.query( + "SELECT map( 'aa', 4, '44' , 5) as c, c.keys", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( @@ -803,261 +1873,373 @@ def subcolumns_values_using_inline_defined_map(self): message = "DB::Exception: Missing columns: 'c.values'" with When("I try to access values sub-column using an inline defined map"): - node.query("SELECT map( 'aa', 4, '44' , 5) as c, c.values", exitcode=exitcode, message=message) + node.query( + "SELECT map( 'aa', 4, '44' , 5) as c, c.values", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys("1.0"), RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Keys_ArrayFunctions("1.0"), RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_SubColumns_Values_ArrayFunctions("1.0"), +) +@Examples( + "type data select filter exitcode message", + [ + # keys + ( + "Map(String, String)", + "(map('a','b','c','d')),(map('e','f'))", + "m.keys AS keys", + "1=1", + 0, + '{"keys":["a","c"]}\n{"keys":["e"]}', + Name("select keys"), + ), + ( + "Map(String, String)", + "(map('a','b','c','d')),(map('e','f'))", + "m.keys AS keys", + "has(m.keys, 'e')", + 0, + '{"keys":["e"]}', + Name("filter by using keys in an array function"), + ), + ( + "Map(String, String)", + "(map('a','b','c','d')),(map('e','f'))", + "has(m.keys, 'e') AS r", + "1=1", + 0, + '{"r":0}\n{"r":1}', + Name("column that uses keys in an array function"), + ), + # values + ( + "Map(String, String)", + "(map('a','b','c','d')),(map('e','f'))", + "m.values AS values", + "1=1", + 0, + '{"values":["b","d"]}\n{"values":["f"]}', + Name("select values"), + ), + ( + "Map(String, String)", + "(map('a','b','c','d')),(map('e','f'))", + "m.values AS values", + "has(m.values, 'f')", + 0, + '{"values":["f"]}', + Name("filter by using values in an array function"), + ), + ( + "Map(String, String)", + "(map('a','b','c','d')),(map('e','f'))", + "has(m.values, 'f') AS r", + "1=1", + 0, + '{"r":0}\n{"r":1}', + Name("column that uses values in an array function"), + ), + ], ) -@Examples("type data select filter exitcode message", [ - # keys - ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "1=1", - 0, '{"keys":["a","c"]}\n{"keys":["e"]}', Name("select keys")), - ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.keys AS keys", "has(m.keys, 'e')", - 0, '{"keys":["e"]}', Name("filter by using keys in an array function")), - ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.keys, 'e') AS r", "1=1", - 0, '{"r":0}\n{"r":1}', Name("column that uses keys in an array function")), - # values - ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "1=1", - 0, '{"values":["b","d"]}\n{"values":["f"]}', Name("select values")), - ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "m.values AS values", "has(m.values, 'f')", - 0, '{"values":["f"]}', Name("filter by using values in an array function")), - ("Map(String, String)", "(map('a','b','c','d')),(map('e','f'))", "has(m.values, 'f') AS r", "1=1", - 0, '{"r":0}\n{"r":1}', Name("column that uses values in an array function")) -]) def subcolumns(self, type, data, select, filter, exitcode, message, order_by=None): - """Check usage of sub-columns in queries. - """ - table_map(type=type, data=data, select=select, filter=filter, exitcode=exitcode, message=message, order_by=order_by) + """Check usage of sub-columns in queries.""" + table_map( + type=type, + data=data, + select=select, + filter=filter, + exitcode=exitcode, + message=message, + order_by=order_by, + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_Length("1.0")) def length(self): - """Check usage of length function with map data type. - """ - table_map(type="Map(String, String)", + """Check usage of length function with map data type.""" + table_map( + type="Map(String, String)", data="(map('a','b','c','d')),(map('e','f'))", select="length(m) AS len, m", filter="length(m) = 1", - exitcode=0, message='{"len":"1","m":{"e":"f"}}') + exitcode=0, + message='{"len":"1","m":{"e":"f"}}', + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_Empty("1.0")) def empty(self): - """Check usage of empty function with map data type. - """ - table_map(type="Map(String, String)", + """Check usage of empty function with map data type.""" + table_map( + type="Map(String, String)", data="(map('e','f'))", select="empty(m) AS em, m", filter="empty(m) <> 1", - exitcode=0, message='{"em":0,"m":{"e":"f"}}') + exitcode=0, + message='{"em":0,"m":{"e":"f"}}', + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_NotEmpty("1.0")) def notempty(self): - """Check usage of notEmpty function with map data type. - """ - table_map(type="Map(String, String)", + """Check usage of notEmpty function with map data type.""" + table_map( + type="Map(String, String)", data="(map('e','f'))", select="notEmpty(m) AS em, m", filter="notEmpty(m) = 1", - exitcode=0, message='{"em":1,"m":{"e":"f"}}') + exitcode=0, + message='{"em":1,"m":{"e":"f"}}', + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapAdd("1.0")) def cast_from_mapadd(self): - """Check converting the result of mapAdd function to a map data type. - """ - select_map(map="CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), 'Map(Int8, Int8)')", output="{1:2,2:2}") + """Check converting the result of mapAdd function to a map data type.""" + select_map( + map="CAST(mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])), 'Map(Int8, Int8)')", + output="{1:2,2:2}", + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapSubstract("1.0")) def cast_from_mapsubstract(self): - """Check converting the result of mapSubstract function to a map data type. - """ - select_map(map="CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), 'Map(Int8, Int8)')", output="{1:-1,2:0}") + """Check converting the result of mapSubstract function to a map data type.""" + select_map( + map="CAST(mapSubtract(([toUInt8(1), 2], [toInt32(1), 1]), ([toUInt8(1), 2], [toInt32(2), 1])), 'Map(Int8, Int8)')", + output="{1:-1,2:0}", + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map_MapPopulateSeries("1.0")) def cast_from_mappopulateseries(self): - """Check converting the result of mapPopulateSeries function to a map data type. - """ - select_map(map="CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), 'Map(Int8, Int8)')", output="{1:11,2:22,3:0,4:44,5:0}") + """Check converting the result of mapPopulateSeries function to a map data type.""" + select_map( + map="CAST(mapPopulateSeries([1,2,4], [11,22,44], 5), 'Map(Int8, Int8)')", + output="{1:11,2:22,3:0,4:44,5:0}", + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapContains("1.0")) def mapcontains(self): - """Check usages of mapContains function with map data type. - """ + """Check usages of mapContains function with map data type.""" node = self.context.node with Example("key in map"): - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map('a','b'))", select="m", filter="mapContains(m, 'a')", - exitcode=0, message='{"m":{"a":"b"}}') + exitcode=0, + message='{"m":{"a":"b"}}', + ) with Example("key not in map"): - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map('a','b'))", select="m", filter="NOT mapContains(m, 'a')", - exitcode=0, message='{"m":{"e":"f"}}') + exitcode=0, + message='{"m":{"e":"f"}}', + ) with Example("null key not in map"): - table_map(type="Map(Nullable(String), String)", + table_map( + type="Map(Nullable(String), String)", data="(map('e','f')),(map('a','b'))", select="m", filter="mapContains(m, NULL)", - exitcode=0, message='') + exitcode=0, + message="", + ) with Example("null key in map"): - table_map(type="Map(Nullable(String), String)", + table_map( + type="Map(Nullable(String), String)", data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", select="m", filter="mapContains(m, NULL)", - exitcode=0, message='{null:"c"}') + exitcode=0, + message='{null:"c"}', + ) with Example("select nullable key"): - node.query("SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", exitcode=0, message="{2:3}") + node.query( + "SELECT map(NULL, 1, 2, 3) AS m, mapContains(m, toNullable(toUInt8(2)))", + exitcode=0, + message="{2:3}", + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapKeys("1.0")) def mapkeys(self): - """Check usages of mapKeys function with map data type. - """ + """Check usages of mapKeys function with map data type.""" with Example("key in map"): - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map('a','b'))", select="m", filter="has(mapKeys(m), 'a')", - exitcode=0, message='{"m":{"a":"b"}}') + exitcode=0, + message='{"m":{"a":"b"}}', + ) with Example("key not in map"): - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map('a','b'))", select="m", filter="NOT has(mapKeys(m), 'a')", - exitcode=0, message='{"m":{"e":"f"}}') + exitcode=0, + message='{"m":{"e":"f"}}', + ) with Example("null key not in map"): - table_map(type="Map(Nullable(String), String)", + table_map( + type="Map(Nullable(String), String)", data="(map('e','f')),(map('a','b'))", select="m", filter="has(mapKeys(m), NULL)", - exitcode=0, message='') + exitcode=0, + message="", + ) with Example("null key in map"): - table_map(type="Map(Nullable(String), String)", + table_map( + type="Map(Nullable(String), String)", data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", select="m", filter="has(mapKeys(m), NULL)", - exitcode=0, message='{"m":{null:"c"}}') + exitcode=0, + message='{"m":{null:"c"}}', + ) with Example("select keys from column"): - table_map(type="Map(Nullable(String), String)", + table_map( + type="Map(Nullable(String), String)", data="(map('e','f')),(map('a','b')),(map(NULL,'c'))", select="mapKeys(m) AS keys", filter="1 = 1", - exitcode=0, message='{"keys":["a"]}\n{"keys":["e"]}\n{"keys":[null]}') + exitcode=0, + message='{"keys":["a"]}\n{"keys":["e"]}\n{"keys":[null]}', + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_MapValues("1.0")) def mapvalues(self): - """Check usages of mapValues function with map data type. - """ + """Check usages of mapValues function with map data type.""" with Example("value in map"): - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map('a','b'))", select="m", filter="has(mapValues(m), 'b')", - exitcode=0, message='{"m":{"a":"b"}}') + exitcode=0, + message='{"m":{"a":"b"}}', + ) with Example("value not in map"): - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map('a','b'))", select="m", filter="NOT has(mapValues(m), 'b')", - exitcode=0, message='{"m":{"e":"f"}}') + exitcode=0, + message='{"m":{"e":"f"}}', + ) with Example("null value not in map"): - table_map(type="Map(String, Nullable(String))", + table_map( + type="Map(String, Nullable(String))", data="(map('e','f')),(map('a','b'))", select="m", filter="has(mapValues(m), NULL)", - exitcode=0, message='') + exitcode=0, + message="", + ) with Example("null value in map"): - table_map(type="Map(String, Nullable(String))", + table_map( + type="Map(String, Nullable(String))", data="(map('e','f')),(map('a','b')),(map('c',NULL))", select="m", filter="has(mapValues(m), NULL)", - exitcode=0, message='{"m":{"c":null}}') + exitcode=0, + message='{"m":{"c":null}}', + ) with Example("select values from column"): - table_map(type="Map(String, Nullable(String))", + table_map( + type="Map(String, Nullable(String))", data="(map('e','f')),(map('a','b')),(map('c',NULL))", select="mapValues(m) AS values", filter="1 = 1", - exitcode=0, message='{"values":["b"]}\n{"values":[null]}\n{"values":["f"]}') + exitcode=0, + message='{"values":["b"]}\n{"values":[null]}\n{"values":["f"]}', + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Functions_InlineDefinedMap("1.0")) def functions_with_inline_defined_map(self): """Check that a map defined inline inside the select statement can be used with functions that work with maps. """ with Example("mapKeys"): - select_map(map="map(1,2,3,4) as map, mapKeys(map) AS keys", output="{1:2,3:4}\t[1,3]") + select_map( + map="map(1,2,3,4) as map, mapKeys(map) AS keys", output="{1:2,3:4}\t[1,3]" + ) with Example("mapValyes"): - select_map(map="map(1,2,3,4) as map, mapValues(map) AS values", output="{1:2,3:4}\t[2,4]") + select_map( + map="map(1,2,3,4) as map, mapValues(map) AS values", + output="{1:2,3:4}\t[2,4]", + ) with Example("mapContains"): - select_map(map="map(1,2,3,4) as map, mapContains(map, 1) AS contains", output="{1:2,3:4}\t1") + select_map( + map="map(1,2,3,4) as map, mapContains(map, 1) AS contains", + output="{1:2,3:4}\t1", + ) + @TestScenario def empty_map(self): """Check creating of an empty map `{}` using the map() function when inserting data into a map type table column. """ - table_map(type="Map(String, String)", + table_map( + type="Map(String, String)", data="(map('e','f')),(map())", select="m", filter="1=1", - exitcode=0, message='{"m":{}}\n{"m":{"e":"f"}}') + exitcode=0, + message='{"m":{}}\n{"m":{"e":"f"}}', + ) + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_TupleOfArrays("1.0")) def performance_vs_two_tuple_of_arrays(self, len=10, rows=6000000): - """Check performance of using map data type vs Tuple(Array, Array). - """ + """Check performance of using map data type vs Tuple(Array, Array).""" uid = getuid() node = self.context.node @@ -1073,7 +2255,9 @@ def performance_vs_two_tuple_of_arrays(self, len=10, rows=6000000): keys = range(len) values = range(len) start_time = time.time() - node.query(f"INSERT INTO {tuple_table} SELECT ({keys},{values}) FROM numbers({rows})") + node.query( + f"INSERT INTO {tuple_table} SELECT ({keys},{values}) FROM numbers({rows})" + ) tuple_insert_time = time.time() - start_time metric("tuple insert time", tuple_insert_time, "sec") @@ -1081,34 +2265,40 @@ def performance_vs_two_tuple_of_arrays(self, len=10, rows=6000000): keys = range(len) values = range(len) start_time = time.time() - node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + node.query( + f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})" + ) map_insert_time = time.time() - start_time metric("map insert time", map_insert_time, "sec") with And("I retrieve particular key value from table with tuples"): start_time = time.time() - node.query(f"SELECT sum(arrayFirst((v, k) -> k = {len-1}, tupleElement(pairs, 2), tupleElement(pairs, 1))) AS sum FROM {tuple_table}", - exitcode=0, message=f"{rows*(len-1)}") + node.query( + f"SELECT sum(arrayFirst((v, k) -> k = {len-1}, tupleElement(pairs, 2), tupleElement(pairs, 1))) AS sum FROM {tuple_table}", + exitcode=0, + message=f"{rows*(len-1)}", + ) tuple_select_time = time.time() - start_time metric("tuple(array, array) select time", tuple_select_time, "sec") with And("I retrieve particular key value from table with map"): start_time = time.time() - node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", - exitcode=0, message=f"{rows*(len-1)}") + node.query( + f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, + message=f"{rows*(len-1)}", + ) map_select_time = time.time() - start_time metric("map select time", map_select_time, "sec") - metric("insert difference", (1 - map_insert_time/tuple_insert_time) * 100, "%") - metric("select difference", (1 - map_select_time/tuple_select_time) * 100, "%") + metric("insert difference", (1 - map_insert_time / tuple_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time / tuple_select_time) * 100, "%") + @TestScenario -@Requirements( - RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples("1.0") -) +@Requirements(RQ_SRS_018_ClickHouse_Map_DataType_Performance_Vs_ArrayOfTuples("1.0")) def performance_vs_array_of_tuples(self, len=10, rows=6000000): - """Check performance of using map data type vs Array(Tuple(K,V)). - """ + """Check performance of using map data type vs Array(Tuple(K,V)).""" uid = getuid() node = self.context.node @@ -1121,7 +2311,7 @@ def performance_vs_array_of_tuples(self, len=10, rows=6000000): map_table = create_table(name=f"map_{uid}", statement=sql) with When("I insert data into table with an array of tuples"): - pairs = list(zip(range(len),range(len))) + pairs = list(zip(range(len), range(len))) start_time = time.time() node.query(f"INSERT INTO {array_table} SELECT ({pairs}) FROM numbers({rows})") array_insert_time = time.time() - start_time @@ -1131,31 +2321,39 @@ def performance_vs_array_of_tuples(self, len=10, rows=6000000): keys = range(len) values = range(len) start_time = time.time() - node.query(f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})") + node.query( + f"INSERT INTO {map_table} SELECT ({keys},{values}) FROM numbers({rows})" + ) map_insert_time = time.time() - start_time metric("map insert time", map_insert_time, "sec") with And("I retrieve particular key value from table with an array of tuples"): start_time = time.time() - node.query(f"SELECT sum(arrayFirst((v) -> v.1 = {len-1}, pairs).2) AS sum FROM {array_table}", - exitcode=0, message=f"{rows*(len-1)}") + node.query( + f"SELECT sum(arrayFirst((v) -> v.1 = {len-1}, pairs).2) AS sum FROM {array_table}", + exitcode=0, + message=f"{rows*(len-1)}", + ) array_select_time = time.time() - start_time metric("array(tuple(k,v)) select time", array_select_time, "sec") with And("I retrieve particular key value from table with map"): start_time = time.time() - node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", - exitcode=0, message=f"{rows*(len-1)}") + node.query( + f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, + message=f"{rows*(len-1)}", + ) map_select_time = time.time() - start_time metric("map select time", map_select_time, "sec") - metric("insert difference", (1 - map_insert_time/array_insert_time) * 100, "%") - metric("select difference", (1 - map_select_time/array_select_time) * 100, "%") + metric("insert difference", (1 - map_insert_time / array_insert_time) * 100, "%") + metric("select difference", (1 - map_select_time / array_select_time) * 100, "%") + @TestScenario def performance(self, len=10, rows=6000000): - """Check insert and select performance of using map data type. - """ + """Check insert and select performance of using map data type.""" uid = getuid() node = self.context.node @@ -1164,26 +2362,33 @@ def performance(self, len=10, rows=6000000): map_table = create_table(name=f"map_{uid}", statement=sql) with When("I insert data into table with a map"): - values = [x for pair in zip(range(len),range(len)) for x in pair] + values = [x for pair in zip(range(len), range(len)) for x in pair] start_time = time.time() - node.query(f"INSERT INTO {map_table} SELECT (map({','.join([str(v) for v in values])})) FROM numbers({rows})") + node.query( + f"INSERT INTO {map_table} SELECT (map({','.join([str(v) for v in values])})) FROM numbers({rows})" + ) map_insert_time = time.time() - start_time metric("map insert time", map_insert_time, "sec") with And("I retrieve particular key value from table with map"): start_time = time.time() - node.query(f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", - exitcode=0, message=f"{rows*(len-1)}") + node.query( + f"SELECT sum(pairs[{len-1}]) AS sum FROM {map_table}", + exitcode=0, + message=f"{rows*(len-1)}", + ) map_select_time = time.time() - start_time metric("map select time", map_select_time, "sec") + # FIXME: add tests for different table engines + @TestFeature @Name("tests") @Requirements( RQ_SRS_018_ClickHouse_Map_DataType("1.0"), - RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map("1.0") + RQ_SRS_018_ClickHouse_Map_DataType_Functions_Map("1.0"), ) def feature(self, node="clickhouse1"): self.context.node = self.context.cluster.node(node) diff --git a/tests/testflows/rbac/configs/clickhouse/common.xml b/tests/testflows/rbac/configs/clickhouse/common.xml deleted file mode 100644 index 0ba01589b90..00000000000 --- a/tests/testflows/rbac/configs/clickhouse/common.xml +++ /dev/null @@ -1,6 +0,0 @@ - - Europe/Moscow - :: - /var/lib/clickhouse/ - /var/lib/clickhouse/tmp/ - diff --git a/tests/testflows/rbac/configs/clickhouse/config.xml b/tests/testflows/rbac/configs/clickhouse/config.xml deleted file mode 100644 index f71f14f4733..00000000000 --- a/tests/testflows/rbac/configs/clickhouse/config.xml +++ /dev/null @@ -1,456 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - 0.0.0.0 - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - query_views_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 7200 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/rbac/configs/clickhouse/users.xml b/tests/testflows/rbac/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/rbac/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/rbac/helper/common.py b/tests/testflows/rbac/helper/common.py index b1d4da536dd..352ce6cb77e 100755 --- a/tests/testflows/rbac/helper/common.py +++ b/tests/testflows/rbac/helper/common.py @@ -11,15 +11,20 @@ from testflows.core import * from helpers.common import instrument_clickhouse_server_log from rbac.helper.tables import table_types + def permutations(table_count=1): - return [*range((1 << table_count)-1)] + return [*range((1 << table_count) - 1)] + def getuid(): if current().subtype == TestSubType.Example: - testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + testname = ( + f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + ) else: testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" - return testname + "_" + str(uuid.uuid1()).replace('-', '_') + return testname + "_" + str(uuid.uuid1()).replace("-", "_") + @contextmanager def table(node, name, table_type_name="MergeTree"): @@ -35,10 +40,13 @@ def table(node, name, table_type_name="MergeTree"): for name in names: with Finally(f"I drop the table {name}"): if table_type.cluster: - node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {table_type.cluster}") + node.query( + f"DROP TABLE IF EXISTS {name} ON CLUSTER {table_type.cluster}" + ) else: node.query(f"DROP TABLE IF EXISTS {name}") + @contextmanager def user(node, name): try: @@ -52,6 +60,7 @@ def user(node, name): with Finally("I drop the user"): node.query(f"DROP USER IF EXISTS {name}") + @contextmanager def role(node, role): try: @@ -65,10 +74,10 @@ def role(node, role): with Finally("I drop the role"): node.query(f"DROP ROLE IF EXISTS {role}") + @TestStep(Given) def row_policy(self, name, table, node=None): - """Create a row policy with a given name on a given table. - """ + """Create a row policy with a given name on a given table.""" if node is None: node = self.context.node @@ -81,33 +90,41 @@ def row_policy(self, name, table, node=None): with Finally(f"I delete row policy {name}"): node.query(f"DROP ROW POLICY IF EXISTS {name} ON {table}") + tables = { - "table0" : 1 << 0, - "table1" : 1 << 1, - "table2" : 1 << 2, - "table3" : 1 << 3, - "table4" : 1 << 4, - "table5" : 1 << 5, - "table6" : 1 << 6, - "table7" : 1 << 7, + "table0": 1 << 0, + "table1": 1 << 1, + "table2": 1 << 2, + "table3": 1 << 3, + "table4": 1 << 4, + "table5": 1 << 5, + "table6": 1 << 6, + "table7": 1 << 7, } + @contextmanager def grant_select_on_table(node, grants, target_name, *table_names): try: tables_granted = [] for table_number in range(len(table_names)): - if(grants & tables[f"table{table_number}"]): + if grants & tables[f"table{table_number}"]: with When(f"I grant select privilege on {table_names[table_number]}"): - node.query(f"GRANT SELECT ON {table_names[table_number]} TO {target_name}") + node.query( + f"GRANT SELECT ON {table_names[table_number]} TO {target_name}" + ) - tables_granted.append(f'{table_names[table_number]}') + tables_granted.append(f"{table_names[table_number]}") - yield (', ').join(tables_granted) + yield (", ").join(tables_granted) finally: for table_number in range(len(table_names)): - with Finally(f"I revoke the select privilege on {table_names[table_number]}"): - node.query(f"REVOKE SELECT ON {table_names[table_number]} FROM {target_name}") + with Finally( + f"I revoke the select privilege on {table_names[table_number]}" + ): + node.query( + f"REVOKE SELECT ON {table_names[table_number]} FROM {target_name}" + ) diff --git a/tests/testflows/rbac/helper/errors.py b/tests/testflows/rbac/helper/errors.py index 65fdd3a8e42..fc8c88dbfc7 100755 --- a/tests/testflows/rbac/helper/errors.py +++ b/tests/testflows/rbac/helper/errors.py @@ -4,120 +4,183 @@ not_found = "Exception: There is no {type} `{name}`" + def user_not_found_in_disk(name): - return (192,not_found.format(type="user",name=name)) + return (192, not_found.format(type="user", name=name)) + def role_not_found_in_disk(name): - return (255,not_found.format(type="role",name=name)) + return (255, not_found.format(type="role", name=name)) + def settings_profile_not_found_in_disk(name): - return (180,not_found.format(type="settings profile",name=name)) + return (180, not_found.format(type="settings profile", name=name)) + def quota_not_found_in_disk(name): - return (199,not_found.format(type="quota",name=name)) + return (199, not_found.format(type="quota", name=name)) + def row_policy_not_found_in_disk(name): - return (11,not_found.format(type="row policy",name=name)) + return (11, not_found.format(type="row policy", name=name)) + def table_does_not_exist(name): - return(60,"Exception: Table {name} doesn't exist".format(name=name)) + return (60, "Exception: Table {name} doesn't exist".format(name=name)) + # Errors: cannot_rename cannot_rename = "Exception: {type} `{name}`: cannot rename to `{name_new}` because {type} `{name_new}` already exists" cannot_rename_exitcode = 237 -def cannot_rename_user(name,name_new): - return (cannot_rename_exitcode, cannot_rename.format(type="user", name=name, name_new=name_new)) -def cannot_rename_role(name,name_new): - return (cannot_rename_exitcode, cannot_rename.format(type="role", name=name, name_new=name_new)) +def cannot_rename_user(name, name_new): + return ( + cannot_rename_exitcode, + cannot_rename.format(type="user", name=name, name_new=name_new), + ) -def cannot_rename_settings_profile(name,name_new): - return (cannot_rename_exitcode, cannot_rename.format(type="settings profile", name=name, name_new=name_new)) -def cannot_rename_quota(name,name_new): - return (cannot_rename_exitcode, cannot_rename.format(type="quota", name=name, name_new=name_new)) +def cannot_rename_role(name, name_new): + return ( + cannot_rename_exitcode, + cannot_rename.format(type="role", name=name, name_new=name_new), + ) + + +def cannot_rename_settings_profile(name, name_new): + return ( + cannot_rename_exitcode, + cannot_rename.format(type="settings profile", name=name, name_new=name_new), + ) + + +def cannot_rename_quota(name, name_new): + return ( + cannot_rename_exitcode, + cannot_rename.format(type="quota", name=name, name_new=name_new), + ) + + +def cannot_rename_row_policy(name, name_new): + return ( + cannot_rename_exitcode, + cannot_rename.format(type="row policy", name=name, name_new=name_new), + ) -def cannot_rename_row_policy(name,name_new): - return (cannot_rename_exitcode, cannot_rename.format(type="row policy", name=name, name_new=name_new)) # Errors: cannot insert -cannot_insert = "Exception: {type} `{name}`: cannot insert because {type} `{name}` already exists" +cannot_insert = ( + "Exception: {type} `{name}`: cannot insert because {type} `{name}` already exists" +) cannot_insert_exitcode = 237 + def cannot_insert_user(name): - return (cannot_insert_exitcode, cannot_insert.format(type="user",name=name)) + return (cannot_insert_exitcode, cannot_insert.format(type="user", name=name)) + def cannot_insert_role(name): - return (cannot_insert_exitcode, cannot_insert.format(type="role",name=name)) + return (cannot_insert_exitcode, cannot_insert.format(type="role", name=name)) + def cannot_insert_settings_profile(name): - return (cannot_insert_exitcode, cannot_insert.format(type="settings profile",name=name)) + return ( + cannot_insert_exitcode, + cannot_insert.format(type="settings profile", name=name), + ) + def cannot_insert_quota(name): - return (cannot_insert_exitcode, cannot_insert.format(type="quota",name=name)) + return (cannot_insert_exitcode, cannot_insert.format(type="quota", name=name)) + def cannot_insert_row_policy(name): - return (cannot_insert_exitcode, cannot_insert.format(type="row policy",name=name)) + return (cannot_insert_exitcode, cannot_insert.format(type="row policy", name=name)) + # Error: default is readonly cannot_remove_default = "Exception: Cannot remove {type} `default` from users.xml because this storage is readonly" cannot_remove_default_exitcode = 239 + def cannot_update_default(): - return (cannot_remove_default_exitcode, "Exception: Cannot update user `default` in users.xml because this storage is readonly") + return ( + cannot_remove_default_exitcode, + "Exception: Cannot update user `default` in users.xml because this storage is readonly", + ) + def cannot_remove_user_default(): return (cannot_remove_default_exitcode, cannot_remove_default.format(type="user")) + def cannot_remove_settings_profile_default(): - return (cannot_remove_default_exitcode, cannot_remove_default.format(type="settings profile")) + return ( + cannot_remove_default_exitcode, + cannot_remove_default.format(type="settings profile"), + ) + def cannot_remove_quota_default(): return (cannot_remove_default_exitcode, cannot_remove_default.format(type="quota")) + # Other syntax errors + def unknown_setting(setting): return (115, f"Exception: Unknown setting {setting}.") + def cluster_not_found(cluster): return (170, f"Exception: Requested cluster '{cluster}' not found.") + ## Privileges + def not_enough_privileges(name): return (241, f"Exception: {name}: Not enough privileges.") + def cannot_parse_string_as_float(string): return (6, f"Exception: Cannot parse string '{string}' as Float64") + def missing_columns(name): return (47, f"Exception: Missing columns: '{name}' while processing query") + # Errors: wrong name wrong_name = "Exception: Wrong {type} name. Cannot find {type} `{name}` to drop" + def wrong_column_name(name): - return (10, wrong_name.format(type="column",name=name)) + return (10, wrong_name.format(type="column", name=name)) + def wrong_index_name(name): - return (36, wrong_name.format(type="index",name=name)) + return (36, wrong_name.format(type="index", name=name)) + def wrong_constraint_name(name): - return (36, wrong_name.format(type="constraint",name=name)) + return (36, wrong_name.format(type="constraint", name=name)) + # Errors: cannot add cannot_add = "Exception: Cannot add index {name}: index with this name already exists" cannot_add_exitcode = 44 + def cannot_add_index(name): return (cannot_add_exitcode, cannot_add.format(name=name)) + def cannot_add_constraint(name): return (cannot_add_exitcode, cannot_add.format(name=name)) diff --git a/tests/testflows/rbac/helper/tables.py b/tests/testflows/rbac/helper/tables.py index ee6289bcbb5..fc8242c0303 100755 --- a/tests/testflows/rbac/helper/tables.py +++ b/tests/testflows/rbac/helper/tables.py @@ -3,39 +3,102 @@ from collections import namedtuple table_tuple = namedtuple("table_tuple", "create_statement cluster") table_types = { - "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = MergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), - "ReplicatedMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), - "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), - "ReplicatedSummingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), - "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), - "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), - "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), - "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), - "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), + "MergeTree": table_tuple( + "CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = MergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "ReplacingMergeTree": table_tuple( + "CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "SummingMergeTree": table_tuple( + "CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "AggregatingMergeTree": table_tuple( + "CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "CollapsingMergeTree": table_tuple( + "CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "VersionedCollapsingMergeTree": table_tuple( + "CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "GraphiteMergeTree": table_tuple( + "CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + None, + ), + "ReplicatedMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), + "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), + "ReplicatedSummingMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), + "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ + ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), + "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ + ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ + ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), + "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ + ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ + ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), + "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ + ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "sharded_cluster", + ), + "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple( + "CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ + ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", + "one_shard_cluster", + ), } diff --git a/tests/testflows/rbac/rbac_env/clickhouse-service.yml b/tests/testflows/rbac/rbac_env/clickhouse-service.yml index ac52e3b83eb..c808372d7e9 100755 --- a/tests/testflows/rbac/rbac_env/clickhouse-service.yml +++ b/tests/testflows/rbac/rbac_env/clickhouse-service.yml @@ -3,6 +3,7 @@ version: '2.3' services: clickhouse: image: clickhouse/integration-test + init: true expose: - "9000" - "9009" @@ -15,9 +16,9 @@ services: - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" - entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + entrypoint: bash -c "tail -f /dev/null" healthcheck: - test: clickhouse client --query='select 1' + test: echo 1 interval: 10s timeout: 10s retries: 3 diff --git a/tests/testflows/rbac/rbac_env_arm64/clickhouse-service.yml b/tests/testflows/rbac/rbac_env_arm64/clickhouse-service.yml new file mode 100755 index 00000000000..a7d6c7053d2 --- /dev/null +++ b/tests/testflows/rbac/rbac_env_arm64/clickhouse-service.yml @@ -0,0 +1,29 @@ +version: '2.3' + +services: + clickhouse: + image: registry.gitlab.com/altinity-public/container-images/test/clickhouse-integration-test:21.12 + privileged: true + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/rbac/rbac_env_arm64/docker-compose.yml b/tests/testflows/rbac/rbac_env_arm64/docker-compose.yml new file mode 100755 index 00000000000..29f2ef52470 --- /dev/null +++ b/tests/testflows/rbac/rbac_env_arm64/docker-compose.yml @@ -0,0 +1,60 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy diff --git a/tests/testflows/rbac/rbac_env_arm64/zookeeper-service.yml b/tests/testflows/rbac/rbac_env_arm64/zookeeper-service.yml new file mode 100755 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/rbac/rbac_env_arm64/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/rbac/regression.py b/tests/testflows/rbac/regression.py index 4c133bd232e..eb1d6c9acf7 100755 --- a/tests/testflows/rbac/regression.py +++ b/tests/testflows/rbac/regression.py @@ -9,6 +9,7 @@ append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser from rbac.requirements import SRS_006_ClickHouse_Role_Based_Access_Control +from helpers.common import check_clickhouse_version issue_14091 = "https://github.com/ClickHouse/ClickHouse/issues/14091" issue_14149 = "https://github.com/ClickHouse/ClickHouse/issues/14149" @@ -33,151 +34,223 @@ issue_25413 = "https://github.com/ClickHouse/ClickHouse/issues/25413" issue_26746 = "https://github.com/ClickHouse/ClickHouse/issues/26746" xfails = { - "syntax/show create quota/I show create quota current": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/12495")], - "views/:/create with subquery privilege granted directly or via role/:": - [(Fail, issue_14091)], - "views/:/create with join query privilege granted directly or via role/:": - [(Fail, issue_14091)], - "views/:/create with union query privilege granted directly or via role/:": - [(Fail, issue_14091)], - "views/:/create with join union subquery privilege granted directly or via role/:": - [(Fail, issue_14091)], - "views/:/create with nested views privilege granted directly or via role/:": - [(Fail, issue_14091)], - "views/view/select with join query privilege granted directly or via role/:": - [(Fail, issue_14149)], - "views/view/select with join union subquery privilege granted directly or via role/:": - [(Fail, issue_14149)], - "views/view/select with nested views privilege granted directly or via role/:": - [(Fail, issue_14149)], - "views/live view/refresh with privilege granted directly or via role/:": - [(Fail, issue_14224)], - "views/live view/refresh with privilege revoked directly or from role/:": - [(Fail, issue_14224)], - "views/live view/select:": - [(Fail, issue_14418)], - "views/live view/select:/:": - [(Fail, issue_14418)], - "views/materialized view/select with:": - [(Fail, issue_14451)], - "views/materialized view/select with:/:": - [(Fail, issue_14451)], - "views/materialized view/modify query:": - [(Fail, issue_14674)], - "views/materialized view/modify query:/:": - [(Fail, issue_14674)], - "views/materialized view/insert on source table privilege granted directly or via role/:": - [(Fail, issue_14810)], - "privileges/alter ttl/table_type=:/user with some privileges": - [(Fail, issue_14566)], - "privileges/alter ttl/table_type=:/role with some privileges": - [(Fail, issue_14566)], - "privileges/alter ttl/table_type=:/user with privileges on cluster": - [(Fail, issue_14566)], - "privileges/alter ttl/table_type=:/user with privileges from user with grant option": - [(Fail, issue_14566)], - "privileges/alter ttl/table_type=:/user with privileges from role with grant option": - [(Fail, issue_14566)], - "privileges/alter ttl/table_type=:/role with privileges from user with grant option": - [(Fail, issue_14566)], - "privileges/alter ttl/table_type=:/role with privileges from role with grant option": - [(Fail, issue_14566)], - "privileges/distributed table/:/special cases/insert with table on source table of materialized view:": - [(Fail, issue_14810)], - "privileges/distributed table/cluster tests/cluster='sharded*": - [(Fail, issue_15165)], - "privileges/distributed table/cluster tests/cluster=:/special cases/insert with table on source table of materialized view privilege granted directly or via role/:": - [(Fail, issue_14810)], - "views/materialized view/select from implicit target table privilege granted directly or via role/select from implicit target table, privilege granted directly": - [(Fail, ".inner table is not created as expected")], - "views/materialized view/insert on target table privilege granted directly or via role/insert on target table, privilege granted through a role": - [(Fail, ".inner table is not created as expected")], - "views/materialized view/select from implicit target table privilege granted directly or via role/select from implicit target table, privilege granted through a role": - [(Fail, ".inner table is not created as expected")], - "views/materialized view/insert on target table privilege granted directly or via role/insert on target table, privilege granted directly": - [(Fail, ".inner table is not created as expected")], - "views/materialized view/select from source table privilege granted directly or via role/select from implicit target table, privilege granted directly": - [(Fail, ".inner table is not created as expected")], - "views/materialized view/select from source table privilege granted directly or via role/select from implicit target table, privilege granted through a role": - [(Fail, ".inner table is not created as expected")], - "privileges/alter move/:/:/:/:/move partition to implicit target table of a materialized view": - [(Fail, ".inner table is not created as expected")], - "privileges/alter move/:/:/:/:/user without ALTER MOVE PARTITION privilege/": - [(Fail, issue_16403)], - "privileges/alter move/:/:/:/:/user with revoked ALTER MOVE PARTITION privilege/": - [(Fail, issue_16403)], - "privileges/create table/create with join query privilege granted directly or via role/:": - [(Fail, issue_17653)], - "privileges/create table/create with join union subquery privilege granted directly or via role/:": - [(Fail, issue_17653)], - "privileges/create table/create with nested tables privilege granted directly or via role/:": - [(Fail, issue_17653)], - "privileges/kill mutation/no privilege/kill mutation on cluster": - [(Fail, issue_17146)], - "privileges/kill query/privilege granted directly or via role/:/": - [(Fail, issue_17147)], - "privileges/show dictionaries/:/check privilege/:/exists/EXISTS with privilege": - [(Fail, issue_17655)], - "privileges/public tables/sensitive tables": - [(Fail, issue_18110)], - "privileges/: row policy/nested live:": - [(Fail, issue_21083)], - "privileges/: row policy/nested mat:": - [(Fail, issue_21084)], - "privileges/show dictionaries/:/check privilege/check privilege=SHOW DICTIONARIES/show dict/SHOW DICTIONARIES with privilege": - [(Fail, "new bug")], - "privileges/show dictionaries/:/check privilege/check privilege=CREATE DICTIONARY/show dict/SHOW DICTIONARIES with privilege": - [(Fail, "new bug")], - "privileges/show dictionaries/:/check privilege/check privilege=DROP DICTIONARY/show dict/SHOW DICTIONARIES with privilege": - [(Fail, "new bug")], - "privileges/kill mutation/:/:/KILL ALTER : without privilege": - [(Fail, issue_25413)], - "privileges/kill mutation/:/:/KILL ALTER : with revoked privilege": - [(Fail, issue_25413)], - "privileges/kill mutation/:/:/KILL ALTER : with revoked ALL privilege": - [(Fail, issue_25413)], - "privileges/create table/create with subquery privilege granted directly or via role/create with subquery, privilege granted directly": - [(Fail, issue_26746)], - "privileges/create table/create with subquery privilege granted directly or via role/create with subquery, privilege granted through a role": - [(Fail, issue_26746)], - "views/live view/create with join subquery privilege granted directly or via role/create with join subquery, privilege granted directly": - [(Fail, issue_26746)], - "views/live view/create with join subquery privilege granted directly or via role/create with join subquery, privilege granted through a role": - [(Fail, issue_26746)] + "syntax/show create quota/I show create quota current": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/12495") + ], + "views/:/create with subquery privilege granted directly or via role/:": [ + (Fail, issue_14091) + ], + "views/:/create with join query privilege granted directly or via role/:": [ + (Fail, issue_14091) + ], + "views/:/create with union query privilege granted directly or via role/:": [ + (Fail, issue_14091) + ], + "views/:/create with join union subquery privilege granted directly or via role/:": [ + (Fail, issue_14091) + ], + "views/:/create with nested views privilege granted directly or via role/:": [ + (Fail, issue_14091) + ], + "views/view/select with join query privilege granted directly or via role/:": [ + (Fail, issue_14149) + ], + "views/view/select with join union subquery privilege granted directly or via role/:": [ + (Fail, issue_14149) + ], + "views/view/select with nested views privilege granted directly or via role/:": [ + (Fail, issue_14149) + ], + "views/live view/refresh with privilege granted directly or via role/:": [ + (Fail, issue_14224) + ], + "views/live view/refresh with privilege revoked directly or from role/:": [ + (Fail, issue_14224) + ], + "views/live view/select:": [(Fail, issue_14418)], + "views/live view/select:/:": [(Fail, issue_14418)], + "views/materialized view/select with:": [(Fail, issue_14451)], + "views/materialized view/select with:/:": [(Fail, issue_14451)], + "views/materialized view/modify query:": [(Fail, issue_14674)], + "views/materialized view/modify query:/:": [(Fail, issue_14674)], + "views/materialized view/insert on source table privilege granted directly or via role/:": [ + (Fail, issue_14810) + ], + "privileges/alter ttl/table_type=:/user with some privileges": [ + (Fail, issue_14566) + ], + "privileges/alter ttl/table_type=:/role with some privileges": [ + (Fail, issue_14566) + ], + "privileges/alter ttl/table_type=:/user with privileges on cluster": [ + (Fail, issue_14566) + ], + "privileges/alter ttl/table_type=:/user with privileges from user with grant option": [ + (Fail, issue_14566) + ], + "privileges/alter ttl/table_type=:/user with privileges from role with grant option": [ + (Fail, issue_14566) + ], + "privileges/alter ttl/table_type=:/role with privileges from user with grant option": [ + (Fail, issue_14566) + ], + "privileges/alter ttl/table_type=:/role with privileges from role with grant option": [ + (Fail, issue_14566) + ], + "privileges/distributed table/:/special cases/insert with table on source table of materialized view:": [ + (Fail, issue_14810) + ], + "privileges/distributed table/cluster tests/cluster='sharded*": [ + (Fail, issue_15165) + ], + "privileges/distributed table/cluster tests/cluster=:/special cases/insert with table on source table of materialized view privilege granted directly or via role/:": [ + (Fail, issue_14810) + ], + "views/materialized view/select from implicit target table privilege granted directly or via role/select from implicit target table, privilege granted directly": [ + (Fail, ".inner table is not created as expected") + ], + "views/materialized view/insert on target table privilege granted directly or via role/insert on target table, privilege granted through a role": [ + (Fail, ".inner table is not created as expected") + ], + "views/materialized view/select from implicit target table privilege granted directly or via role/select from implicit target table, privilege granted through a role": [ + (Fail, ".inner table is not created as expected") + ], + "views/materialized view/insert on target table privilege granted directly or via role/insert on target table, privilege granted directly": [ + (Fail, ".inner table is not created as expected") + ], + "views/materialized view/select from source table privilege granted directly or via role/select from implicit target table, privilege granted directly": [ + (Fail, ".inner table is not created as expected") + ], + "views/materialized view/select from source table privilege granted directly or via role/select from implicit target table, privilege granted through a role": [ + (Fail, ".inner table is not created as expected") + ], + "privileges/alter move/:/:/:/:/move partition to implicit target table of a materialized view": [ + (Fail, ".inner table is not created as expected") + ], + "privileges/alter move/:/:/:/:/user without ALTER MOVE PARTITION privilege/": [ + (Fail, issue_16403) + ], + "privileges/alter move/:/:/:/:/user with revoked ALTER MOVE PARTITION privilege/": [ + (Fail, issue_16403) + ], + "privileges/create table/create with join query privilege granted directly or via role/:": [ + (Fail, issue_17653) + ], + "privileges/create table/create with join union subquery privilege granted directly or via role/:": [ + (Fail, issue_17653) + ], + "privileges/create table/create with nested tables privilege granted directly or via role/:": [ + (Fail, issue_17653) + ], + "privileges/kill mutation/no privilege/kill mutation on cluster": [ + (Fail, issue_17146) + ], + "privileges/kill query/privilege granted directly or via role/:/": [ + (Fail, issue_17147) + ], + "privileges/show dictionaries/:/check privilege/:/exists/EXISTS with privilege": [ + (Fail, issue_17655) + ], + "privileges/public tables/sensitive tables": [(Fail, issue_18110)], + "privileges/: row policy/nested live:": [(Fail, issue_21083)], + "privileges/: row policy/nested mat:": [(Fail, issue_21084)], + "privileges/show dictionaries/:/check privilege/check privilege=SHOW DICTIONARIES/show dict/SHOW DICTIONARIES with privilege": [ + (Fail, "new bug") + ], + "privileges/show dictionaries/:/check privilege/check privilege=CREATE DICTIONARY/show dict/SHOW DICTIONARIES with privilege": [ + (Fail, "new bug") + ], + "privileges/show dictionaries/:/check privilege/check privilege=DROP DICTIONARY/show dict/SHOW DICTIONARIES with privilege": [ + (Fail, "new bug") + ], + "privileges/kill mutation/:/:/KILL ALTER : without privilege": [ + (Fail, issue_25413) + ], + "privileges/kill mutation/:/:/KILL ALTER : with revoked privilege": [ + (Fail, issue_25413) + ], + "privileges/kill mutation/:/:/KILL ALTER : with revoked ALL privilege": [ + (Fail, issue_25413) + ], + "privileges/create table/create with subquery privilege granted directly or via role/create with subquery, privilege granted directly": [ + (Fail, issue_26746) + ], + "privileges/create table/create with subquery privilege granted directly or via role/create with subquery, privilege granted through a role": [ + (Fail, issue_26746) + ], + "views/live view/create with join subquery privilege granted directly or via role/create with join subquery, privilege granted directly": [ + (Fail, issue_26746) + ], + "views/live view/create with join subquery privilege granted directly or via role/create with join subquery, privilege granted through a role": [ + (Fail, issue_26746) + ], } xflags = { - "privileges/alter index/table_type='ReplicatedVersionedCollapsingMergeTree-sharded_cluster'/role with privileges from role with grant option/granted=:/I try to ALTER INDEX with given privileges/I check order by when privilege is granted": - (SKIP, 0) + "privileges/alter index/table_type='ReplicatedVersionedCollapsingMergeTree-sharded_cluster'/role with privileges from role with grant option/granted=:/I try to ALTER INDEX with given privileges/I check order by when privilege is granted": ( + SKIP, + 0, + ) } +ffails = { + "/clickhouse/rbac/privileges/:/table_type='ReplicatedReplacingMergeTree-sharded_cluster": ( + Skip, + "Causes clickhouse timeout on 21.10", + ( + lambda test: check_clickhouse_version(">=21.10")(test) + and check_clickhouse_version("<21.11")(test) + ), + ), + "/clickhouse/rbac/views": ( + Skip, + "Does not work on clickhouse 21.09", + ( + lambda test: check_clickhouse_version(">=21.9")(test) + and check_clickhouse_version("<21.10")(test) + ), + ), +} + + @TestModule @ArgumentParser(argparser) @XFails(xfails) @XFlags(xflags) +@FFails(ffails) @Name("rbac") -@Specifications( - SRS_006_ClickHouse_Role_Based_Access_Control -) -def regression(self, local, clickhouse_binary_path, stress=None): - """RBAC regression. - """ - nodes = { - "clickhouse": - ("clickhouse1", "clickhouse2", "clickhouse3") - } +@Specifications(SRS_006_ClickHouse_Role_Based_Access_Control) +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """RBAC regression.""" + nodes = {"clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3")} + + self.context.clickhouse_version = clickhouse_version if stress is not None: self.context.stress = stress - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "rbac_env")) as cluster: + from platform import processor as current_cpu + + folder_name = os.path.basename(current_dir()) + if current_cpu() == "aarch64": + env = f"{folder_name}_env_arm64" + else: + env = f"{folder_name}_env" + + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), env), + ) as cluster: self.context.cluster = cluster Feature(run=load("rbac.tests.syntax.feature", "feature")) Feature(run=load("rbac.tests.privileges.feature", "feature")) Feature(run=load("rbac.tests.views.feature", "feature")) + if main(): regression() diff --git a/tests/testflows/rbac/requirements/__init__.py b/tests/testflows/rbac/requirements/__init__.py index 75e9d5b4bb8..02f7d430154 100644 --- a/tests/testflows/rbac/requirements/__init__.py +++ b/tests/testflows/rbac/requirements/__init__.py @@ -1 +1 @@ -from .requirements import * \ No newline at end of file +from .requirements import * diff --git a/tests/testflows/rbac/requirements/requirements.py b/tests/testflows/rbac/requirements/requirements.py index d970ff629da..552588e49b9 100755 --- a/tests/testflows/rbac/requirements/requirements.py +++ b/tests/testflows/rbac/requirements/requirements.py @@ -9,8853 +9,9362 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_006_RBAC = Requirement( - name='RQ.SRS-006.RBAC', - version='1.0', + name="RQ.SRS-006.RBAC", + version="1.0", priority=None, group=None, type=None, uid=None, - description=( - '[ClickHouse] SHALL support role based access control.\n' - '\n' - ), + description=("[ClickHouse] SHALL support role based access control.\n" "\n"), link=None, level=3, - num='5.1.1') + num="5.1.1", +) RQ_SRS_006_RBAC_Login = Requirement( - name='RQ.SRS-006.RBAC.Login', - version='1.0', + name="RQ.SRS-006.RBAC.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only allow access to the server for a given\n' - 'user only when correct username and password are used during\n' - 'the connection to the server.\n' - '\n' - ), + "[ClickHouse] SHALL only allow access to the server for a given\n" + "user only when correct username and password are used during\n" + "the connection to the server.\n" + "\n" + ), link=None, level=3, - num='5.2.1') + num="5.2.1", +) RQ_SRS_006_RBAC_Login_DefaultUser = Requirement( - name='RQ.SRS-006.RBAC.Login.DefaultUser', - version='1.0', + name="RQ.SRS-006.RBAC.Login.DefaultUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use the **default user** when no username and password\n' - 'are specified during the connection to the server.\n' - '\n' - ), + "[ClickHouse] SHALL use the **default user** when no username and password\n" + "are specified during the connection to the server.\n" + "\n" + ), link=None, level=3, - num='5.2.2') + num="5.2.2", +) RQ_SRS_006_RBAC_User = Requirement( - name='RQ.SRS-006.RBAC.User', - version='1.0', + name="RQ.SRS-006.RBAC.User", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creation and manipulation of\n' - 'one or more **user** accounts to which roles, privileges,\n' - 'settings profile, quotas and row policies can be assigned.\n' - '\n' - ), + "[ClickHouse] SHALL support creation and manipulation of\n" + "one or more **user** accounts to which roles, privileges,\n" + "settings profile, quotas and row policies can be assigned.\n" + "\n" + ), link=None, level=3, - num='5.3.1') + num="5.3.1", +) RQ_SRS_006_RBAC_User_Roles = Requirement( - name='RQ.SRS-006.RBAC.User.Roles', - version='1.0', + name="RQ.SRS-006.RBAC.User.Roles", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **roles**\n' - 'to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **roles**\n" + "to a **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.2') + num="5.3.2", +) RQ_SRS_006_RBAC_User_Privileges = Requirement( - name='RQ.SRS-006.RBAC.User.Privileges', - version='1.0', + name="RQ.SRS-006.RBAC.User.Privileges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more privileges to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more privileges to a **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.3') + num="5.3.3", +) RQ_SRS_006_RBAC_User_Variables = Requirement( - name='RQ.SRS-006.RBAC.User.Variables', - version='1.0', + name="RQ.SRS-006.RBAC.User.Variables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more variables to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more variables to a **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.4') + num="5.3.4", +) RQ_SRS_006_RBAC_User_Variables_Constraints = Requirement( - name='RQ.SRS-006.RBAC.User.Variables.Constraints', - version='1.0', + name="RQ.SRS-006.RBAC.User.Variables.Constraints", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning min, max and read-only constraints\n' - 'for the variables that can be set and read by the **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning min, max and read-only constraints\n" + "for the variables that can be set and read by the **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.5') + num="5.3.5", +) RQ_SRS_006_RBAC_User_SettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.User.SettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.User.SettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **settings profiles**\n' - 'to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **settings profiles**\n" + "to a **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.6') + num="5.3.6", +) RQ_SRS_006_RBAC_User_Quotas = Requirement( - name='RQ.SRS-006.RBAC.User.Quotas', - version='1.0', + name="RQ.SRS-006.RBAC.User.Quotas", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **quotas** to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **quotas** to a **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.7') + num="5.3.7", +) RQ_SRS_006_RBAC_User_RowPolicies = Requirement( - name='RQ.SRS-006.RBAC.User.RowPolicies', - version='1.0', + name="RQ.SRS-006.RBAC.User.RowPolicies", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **row policies** to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **row policies** to a **user**.\n" + "\n" + ), link=None, level=3, - num='5.3.8') + num="5.3.8", +) RQ_SRS_006_RBAC_User_DefaultRole = Requirement( - name='RQ.SRS-006.RBAC.User.DefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.User.DefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning a default role to a **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning a default role to a **user**.\n" "\n" + ), link=None, level=3, - num='5.3.9') + num="5.3.9", +) RQ_SRS_006_RBAC_User_RoleSelection = Requirement( - name='RQ.SRS-006.RBAC.User.RoleSelection', - version='1.0', + name="RQ.SRS-006.RBAC.User.RoleSelection", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support selection of one or more **roles** from the available roles\n' - 'that are assigned to a **user** using `SET ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support selection of one or more **roles** from the available roles\n" + "that are assigned to a **user** using `SET ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.3.10') + num="5.3.10", +) RQ_SRS_006_RBAC_User_ShowCreate = Requirement( - name='RQ.SRS-006.RBAC.User.ShowCreate', - version='1.0', + name="RQ.SRS-006.RBAC.User.ShowCreate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the command of how **user** account was created.\n' - '\n' - ), + "[ClickHouse] SHALL support showing the command of how **user** account was created.\n" + "\n" + ), link=None, level=3, - num='5.3.11') + num="5.3.11", +) RQ_SRS_006_RBAC_User_ShowPrivileges = Requirement( - name='RQ.SRS-006.RBAC.User.ShowPrivileges', - version='1.0', + name="RQ.SRS-006.RBAC.User.ShowPrivileges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support listing the privileges of the **user**.\n' - '\n' - ), + "[ClickHouse] SHALL support listing the privileges of the **user**.\n" "\n" + ), link=None, level=3, - num='5.3.12') + num="5.3.12", +) RQ_SRS_006_RBAC_User_Use_DefaultRole = Requirement( - name='RQ.SRS-006.RBAC.User.Use.DefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.User.Use.DefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL by default use default role or roles assigned\n' - 'to the user if specified.\n' - '\n' - ), + "[ClickHouse] SHALL by default use default role or roles assigned\n" + "to the user if specified.\n" + "\n" + ), link=None, level=3, - num='5.3.13') + num="5.3.13", +) RQ_SRS_006_RBAC_User_Use_AllRolesWhenNoDefaultRole = Requirement( - name='RQ.SRS-006.RBAC.User.Use.AllRolesWhenNoDefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.User.Use.AllRolesWhenNoDefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL by default use all the roles assigned to the user\n' - 'if no default role or roles are specified for the user.\n' - '\n' - ), + "[ClickHouse] SHALL by default use all the roles assigned to the user\n" + "if no default role or roles are specified for the user.\n" + "\n" + ), link=None, level=3, - num='5.3.14') + num="5.3.14", +) RQ_SRS_006_RBAC_User_Create = Requirement( - name='RQ.SRS-006.RBAC.User.Create', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating **user** accounts using `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support creating **user** accounts using `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.1') + num="5.3.15.1", +) RQ_SRS_006_RBAC_User_Create_IfNotExists = Requirement( - name='RQ.SRS-006.RBAC.User.Create.IfNotExists', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.IfNotExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE USER` statement\n' - 'to skip raising an exception if a user with the same **name** already exists.\n' - 'If the `IF NOT EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if a user with the same **name** already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE USER` statement\n" + "to skip raising an exception if a user with the same **name** already exists.\n" + "If the `IF NOT EXISTS` clause is not specified then an exception SHALL be\n" + "raised if a user with the same **name** already exists.\n" + "\n" + ), link=None, level=4, - num='5.3.15.2') + num="5.3.15.2", +) RQ_SRS_006_RBAC_User_Create_Replace = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Replace', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Replace", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE USER` statement\n' - 'to replace existing user account if already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE USER` statement\n" + "to replace existing user account if already exists.\n" + "\n" + ), link=None, level=4, - num='5.3.15.3') + num="5.3.15.3", +) RQ_SRS_006_RBAC_User_Create_Password_NoPassword = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.NoPassword', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.NoPassword", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying no password when creating\n' - 'user account using `IDENTIFIED WITH NO_PASSWORD` clause .\n' - '\n' - ), + "[ClickHouse] SHALL support specifying no password when creating\n" + "user account using `IDENTIFIED WITH NO_PASSWORD` clause .\n" + "\n" + ), link=None, level=4, - num='5.3.15.4') + num="5.3.15.4", +) RQ_SRS_006_RBAC_User_Create_Password_NoPassword_Login = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.NoPassword.Login', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.NoPassword.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use no password for the user when connecting to the server\n' - 'when an account was created with `IDENTIFIED WITH NO_PASSWORD` clause.\n' - '\n' - ), + "[ClickHouse] SHALL use no password for the user when connecting to the server\n" + "when an account was created with `IDENTIFIED WITH NO_PASSWORD` clause.\n" + "\n" + ), link=None, level=4, - num='5.3.15.5') + num="5.3.15.5", +) RQ_SRS_006_RBAC_User_Create_Password_PlainText = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.PlainText', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.PlainText", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying plaintext password when creating\n' - 'user account using `IDENTIFIED WITH PLAINTEXT_PASSWORD BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying plaintext password when creating\n" + "user account using `IDENTIFIED WITH PLAINTEXT_PASSWORD BY` clause.\n" + "\n" + ), link=None, level=4, - num='5.3.15.6') + num="5.3.15.6", +) RQ_SRS_006_RBAC_User_Create_Password_PlainText_Login = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.PlainText.Login', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.PlainText.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL use the plaintext password passed by the user when connecting to the server\n' - 'when an account was created with `IDENTIFIED WITH PLAINTEXT_PASSWORD` clause\n' - 'and compare the password with the one used in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL use the plaintext password passed by the user when connecting to the server\n" + "when an account was created with `IDENTIFIED WITH PLAINTEXT_PASSWORD` clause\n" + "and compare the password with the one used in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.7') + num="5.3.15.7", +) RQ_SRS_006_RBAC_User_Create_Password_Sha256Password = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Password', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Password", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the result of applying SHA256\n' - 'to some password when creating user account using `IDENTIFIED WITH SHA256_PASSWORD BY` or `IDENTIFIED BY`\n' - 'clause.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the result of applying SHA256\n" + "to some password when creating user account using `IDENTIFIED WITH SHA256_PASSWORD BY` or `IDENTIFIED BY`\n" + "clause.\n" + "\n" + ), link=None, level=4, - num='5.3.15.8') + num="5.3.15.8", +) RQ_SRS_006_RBAC_User_Create_Password_Sha256Password_Login = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Password.Login', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Password.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL calculate `SHA256` of the password passed by the user when connecting to the server\n' + "[ClickHouse] SHALL calculate `SHA256` of the password passed by the user when connecting to the server\n" "when an account was created with `IDENTIFIED WITH SHA256_PASSWORD` or with 'IDENTIFIED BY' clause\n" - 'and compare the calculated hash to the one used in the `CREATE USER` statement.\n' - '\n' - ), + "and compare the calculated hash to the one used in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.9') + num="5.3.15.9", +) RQ_SRS_006_RBAC_User_Create_Password_Sha256Hash = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the result of applying SHA256\n' - 'to some already calculated hash when creating user account using `IDENTIFIED WITH SHA256_HASH`\n' - 'clause.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the result of applying SHA256\n" + "to some already calculated hash when creating user account using `IDENTIFIED WITH SHA256_HASH`\n" + "clause.\n" + "\n" + ), link=None, level=4, - num='5.3.15.10') + num="5.3.15.10", +) RQ_SRS_006_RBAC_User_Create_Password_Sha256Hash_Login = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash.Login', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL calculate `SHA256` of the already calculated hash passed by\n' - 'the user when connecting to the server\n' - 'when an account was created with `IDENTIFIED WITH SHA256_HASH` clause\n' - 'and compare the calculated hash to the one used in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL calculate `SHA256` of the already calculated hash passed by\n" + "the user when connecting to the server\n" + "when an account was created with `IDENTIFIED WITH SHA256_HASH` clause\n" + "and compare the calculated hash to the one used in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.11') + num="5.3.15.11", +) RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Password = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the result of applying SHA1 two times\n' - 'to a password when creating user account using `IDENTIFIED WITH DOUBLE_SHA1_PASSWORD`\n' - 'clause.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the result of applying SHA1 two times\n" + "to a password when creating user account using `IDENTIFIED WITH DOUBLE_SHA1_PASSWORD`\n" + "clause.\n" + "\n" + ), link=None, level=4, - num='5.3.15.12') + num="5.3.15.12", +) RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Password_Login = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password.Login', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL calculate `SHA1` two times over the password passed by\n' - 'the user when connecting to the server\n' - 'when an account was created with `IDENTIFIED WITH DOUBLE_SHA1_PASSWORD` clause\n' - 'and compare the calculated value to the one used in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL calculate `SHA1` two times over the password passed by\n" + "the user when connecting to the server\n" + "when an account was created with `IDENTIFIED WITH DOUBLE_SHA1_PASSWORD` clause\n" + "and compare the calculated value to the one used in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.13') + num="5.3.15.13", +) RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Hash = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the result of applying SHA1 two times\n' - 'to a hash when creating user account using `IDENTIFIED WITH DOUBLE_SHA1_HASH`\n' - 'clause.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the result of applying SHA1 two times\n" + "to a hash when creating user account using `IDENTIFIED WITH DOUBLE_SHA1_HASH`\n" + "clause.\n" + "\n" + ), link=None, level=4, - num='5.3.15.14') + num="5.3.15.14", +) RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Hash_Login = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash.Login', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash.Login", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL calculate `SHA1` two times over the hash passed by\n' - 'the user when connecting to the server\n' - 'when an account was created with `IDENTIFIED WITH DOUBLE_SHA1_HASH` clause\n' - 'and compare the calculated value to the one used in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL calculate `SHA1` two times over the hash passed by\n" + "the user when connecting to the server\n" + "when an account was created with `IDENTIFIED WITH DOUBLE_SHA1_HASH` clause\n" + "and compare the calculated value to the one used in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.15') + num="5.3.15.15", +) RQ_SRS_006_RBAC_User_Create_Host_Name = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.Name', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.Name", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more hostnames from\n' - 'which user can access the server using the `HOST NAME` clause\n' - 'in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more hostnames from\n" + "which user can access the server using the `HOST NAME` clause\n" + "in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.16') + num="5.3.15.16", +) RQ_SRS_006_RBAC_User_Create_Host_Regexp = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.Regexp', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.Regexp", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more regular expressions\n' - 'to match hostnames from which user can access the server\n' - 'using the `HOST REGEXP` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more regular expressions\n" + "to match hostnames from which user can access the server\n" + "using the `HOST REGEXP` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.17') + num="5.3.15.17", +) RQ_SRS_006_RBAC_User_Create_Host_IP = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.IP', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.IP", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more IP address or subnet from\n' - 'which user can access the server using the `HOST IP` clause in the\n' - '`CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more IP address or subnet from\n" + "which user can access the server using the `HOST IP` clause in the\n" + "`CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.18') + num="5.3.15.18", +) RQ_SRS_006_RBAC_User_Create_Host_Any = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.Any', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.Any", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `HOST ANY` clause in the `CREATE USER` statement\n' - 'to indicate that user can access the server from any host.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `HOST ANY` clause in the `CREATE USER` statement\n" + "to indicate that user can access the server from any host.\n" + "\n" + ), link=None, level=4, - num='5.3.15.19') + num="5.3.15.19", +) RQ_SRS_006_RBAC_User_Create_Host_None = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.None', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support fobidding access from any host using `HOST NONE` clause in the\n' - '`CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support fobidding access from any host using `HOST NONE` clause in the\n" + "`CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.20') + num="5.3.15.20", +) RQ_SRS_006_RBAC_User_Create_Host_Local = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.Local', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.Local", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting user access to local only using `HOST LOCAL` clause in the\n' - '`CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting user access to local only using `HOST LOCAL` clause in the\n" + "`CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.21') + num="5.3.15.21", +) RQ_SRS_006_RBAC_User_Create_Host_Like = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.Like', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.Like", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying host using `LIKE` command syntax using the\n' - '`HOST LIKE` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying host using `LIKE` command syntax using the\n" + "`HOST LIKE` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.22') + num="5.3.15.22", +) RQ_SRS_006_RBAC_User_Create_Host_Default = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Host.Default', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Host.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support user access to server from any host\n' - 'if no `HOST` clause is specified in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support user access to server from any host\n" + "if no `HOST` clause is specified in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.23') + num="5.3.15.23", +) RQ_SRS_006_RBAC_User_Create_DefaultRole = Requirement( - name='RQ.SRS-006.RBAC.User.Create.DefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.DefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more default roles\n' - 'using `DEFAULT ROLE` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more default roles\n" + "using `DEFAULT ROLE` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.24') + num="5.3.15.24", +) RQ_SRS_006_RBAC_User_Create_DefaultRole_None = Requirement( - name='RQ.SRS-006.RBAC.User.Create.DefaultRole.None', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.DefaultRole.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying no default roles\n' - 'using `DEFAULT ROLE NONE` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying no default roles\n" + "using `DEFAULT ROLE NONE` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.25') + num="5.3.15.25", +) RQ_SRS_006_RBAC_User_Create_DefaultRole_All = Requirement( - name='RQ.SRS-006.RBAC.User.Create.DefaultRole.All', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.DefaultRole.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying all roles to be used as default\n' - 'using `DEFAULT ROLE ALL` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying all roles to be used as default\n" + "using `DEFAULT ROLE ALL` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.26') + num="5.3.15.26", +) RQ_SRS_006_RBAC_User_Create_Settings = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Settings', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Settings", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying settings and profile\n' - 'using `SETTINGS` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying settings and profile\n" + "using `SETTINGS` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.27') + num="5.3.15.27", +) RQ_SRS_006_RBAC_User_Create_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.User.Create.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying cluster on which the user\n' - 'will be created using `ON CLUSTER` clause in the `CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying cluster on which the user\n" + "will be created using `ON CLUSTER` clause in the `CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.15.28') + num="5.3.15.28", +) RQ_SRS_006_RBAC_User_Create_Syntax = Requirement( - name='RQ.SRS-006.RBAC.User.Create.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.User.Create.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for `CREATE USER` statement.\n' - '\n' - '```sql\n' - 'CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]\n' + "[ClickHouse] SHALL support the following syntax for `CREATE USER` statement.\n" + "\n" + "```sql\n" + "CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]\n" " [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]\n" " [HOST {LOCAL | NAME 'name' | NAME REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]\n" - ' [DEFAULT ROLE role [,...]]\n' + " [DEFAULT ROLE role [,...]]\n" " [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='5.3.15.29') + num="5.3.15.29", +) RQ_SRS_006_RBAC_User_Alter = Requirement( - name='RQ.SRS-006.RBAC.User.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering **user** accounts using `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering **user** accounts using `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.1') + num="5.3.16.1", +) RQ_SRS_006_RBAC_User_Alter_OrderOfEvaluation = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.OrderOfEvaluation', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.OrderOfEvaluation", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support evaluating `ALTER USER` statement from left to right\n' - 'where things defined on the right override anything that was previously defined on\n' - 'the left.\n' - '\n' - ), + "[ClickHouse] SHALL support evaluating `ALTER USER` statement from left to right\n" + "where things defined on the right override anything that was previously defined on\n" + "the left.\n" + "\n" + ), link=None, level=4, - num='5.3.16.2') + num="5.3.16.2", +) RQ_SRS_006_RBAC_User_Alter_IfExists = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF EXISTS` clause in the `ALTER USER` statement\n' - 'to skip raising an exception (producing a warning instead) if a user with the specified **name** does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a user with the **name** does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF EXISTS` clause in the `ALTER USER` statement\n" + "to skip raising an exception (producing a warning instead) if a user with the specified **name** does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be raised if a user with the **name** does not exist.\n" + "\n" + ), link=None, level=4, - num='5.3.16.3') + num="5.3.16.3", +) RQ_SRS_006_RBAC_User_Alter_Cluster = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the cluster the user is on\n' - 'when altering user account using `ON CLUSTER` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the cluster the user is on\n" + "when altering user account using `ON CLUSTER` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.4') + num="5.3.16.4", +) RQ_SRS_006_RBAC_User_Alter_Rename = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Rename', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Rename", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying a new name for the user when\n' - 'altering user account using `RENAME` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying a new name for the user when\n" + "altering user account using `RENAME` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.5') + num="5.3.16.5", +) RQ_SRS_006_RBAC_User_Alter_Password_PlainText = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Password.PlainText', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Password.PlainText", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying plaintext password when altering\n' - 'user account using `IDENTIFIED WITH PLAINTEXT_PASSWORD BY` or\n' - 'using shorthand `IDENTIFIED BY` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying plaintext password when altering\n" + "user account using `IDENTIFIED WITH PLAINTEXT_PASSWORD BY` or\n" + "using shorthand `IDENTIFIED BY` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.6') + num="5.3.16.6", +) RQ_SRS_006_RBAC_User_Alter_Password_Sha256Password = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Password.Sha256Password', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Password.Sha256Password", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the result of applying SHA256\n' - 'to some password as identification when altering user account using\n' - '`IDENTIFIED WITH SHA256_PASSWORD` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the result of applying SHA256\n" + "to some password as identification when altering user account using\n" + "`IDENTIFIED WITH SHA256_PASSWORD` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.7') + num="5.3.16.7", +) RQ_SRS_006_RBAC_User_Alter_Password_DoubleSha1Password = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Password.DoubleSha1Password', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Password.DoubleSha1Password", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the result of applying Double SHA1\n' - 'to some password as identification when altering user account using\n' - '`IDENTIFIED WITH DOUBLE_SHA1_PASSWORD` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the result of applying Double SHA1\n" + "to some password as identification when altering user account using\n" + "`IDENTIFIED WITH DOUBLE_SHA1_PASSWORD` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.8') + num="5.3.16.8", +) RQ_SRS_006_RBAC_User_Alter_Host_AddDrop = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.AddDrop', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.AddDrop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering user by adding and dropping access to hosts\n' - 'with the `ADD HOST` or the `DROP HOST` in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering user by adding and dropping access to hosts\n" + "with the `ADD HOST` or the `DROP HOST` in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.9') + num="5.3.16.9", +) RQ_SRS_006_RBAC_User_Alter_Host_Local = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.Local', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.Local", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting user access to local only using `HOST LOCAL` clause in the\n' - '`ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting user access to local only using `HOST LOCAL` clause in the\n" + "`ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.10') + num="5.3.16.10", +) RQ_SRS_006_RBAC_User_Alter_Host_Name = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.Name', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.Name", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more hostnames from\n' - 'which user can access the server using the `HOST NAME` clause\n' - 'in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more hostnames from\n" + "which user can access the server using the `HOST NAME` clause\n" + "in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.11') + num="5.3.16.11", +) RQ_SRS_006_RBAC_User_Alter_Host_Regexp = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.Regexp', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.Regexp", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more regular expressions\n' - 'to match hostnames from which user can access the server\n' - 'using the `HOST REGEXP` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more regular expressions\n" + "to match hostnames from which user can access the server\n" + "using the `HOST REGEXP` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.12') + num="5.3.16.12", +) RQ_SRS_006_RBAC_User_Alter_Host_IP = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.IP', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.IP", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more IP address or subnet from\n' - 'which user can access the server using the `HOST IP` clause in the\n' - '`ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more IP address or subnet from\n" + "which user can access the server using the `HOST IP` clause in the\n" + "`ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.13') + num="5.3.16.13", +) RQ_SRS_006_RBAC_User_Alter_Host_Like = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.Like', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.Like", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more similar hosts using `LIKE` command syntax\n' - 'using the `HOST LIKE` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more similar hosts using `LIKE` command syntax\n" + "using the `HOST LIKE` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.14') + num="5.3.16.14", +) RQ_SRS_006_RBAC_User_Alter_Host_Any = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.Any', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.Any", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying `HOST ANY` clause in the `ALTER USER` statement\n' - 'to indicate that user can access the server from any host.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying `HOST ANY` clause in the `ALTER USER` statement\n" + "to indicate that user can access the server from any host.\n" + "\n" + ), link=None, level=4, - num='5.3.16.15') + num="5.3.16.15", +) RQ_SRS_006_RBAC_User_Alter_Host_None = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Host.None', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Host.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support fobidding access from any host using `HOST NONE` clause in the\n' - '`ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support fobidding access from any host using `HOST NONE` clause in the\n" + "`ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.16') + num="5.3.16.16", +) RQ_SRS_006_RBAC_User_Alter_DefaultRole = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.DefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.DefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more default roles\n' - 'using `DEFAULT ROLE` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more default roles\n" + "using `DEFAULT ROLE` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.17') + num="5.3.16.17", +) RQ_SRS_006_RBAC_User_Alter_DefaultRole_All = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.DefaultRole.All', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.DefaultRole.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying all roles to be used as default\n' - 'using `DEFAULT ROLE ALL` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying all roles to be used as default\n" + "using `DEFAULT ROLE ALL` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.18') + num="5.3.16.18", +) RQ_SRS_006_RBAC_User_Alter_DefaultRole_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.DefaultRole.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.DefaultRole.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more roles which will not be used as default\n' - 'using `DEFAULT ROLE ALL EXCEPT` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more roles which will not be used as default\n" + "using `DEFAULT ROLE ALL EXCEPT` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.19') + num="5.3.16.19", +) RQ_SRS_006_RBAC_User_Alter_Settings = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Settings', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Settings", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying one or more variables\n' - 'using `SETTINGS` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying one or more variables\n" + "using `SETTINGS` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.20') + num="5.3.16.20", +) RQ_SRS_006_RBAC_User_Alter_Settings_Min = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Settings.Min', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Settings.Min", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying a minimum value for the variable specifed using `SETTINGS` with `MIN` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying a minimum value for the variable specifed using `SETTINGS` with `MIN` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.21') + num="5.3.16.21", +) RQ_SRS_006_RBAC_User_Alter_Settings_Max = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Settings.Max', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Settings.Max", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying a maximum value for the variable specifed using `SETTINGS` with `MAX` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying a maximum value for the variable specifed using `SETTINGS` with `MAX` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.22') + num="5.3.16.22", +) RQ_SRS_006_RBAC_User_Alter_Settings_Profile = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Settings.Profile', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Settings.Profile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying the name of a profile for the variable specifed using `SETTINGS` with `PROFILE` clause in the `ALTER USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying the name of a profile for the variable specifed using `SETTINGS` with `PROFILE` clause in the `ALTER USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.16.23') + num="5.3.16.23", +) RQ_SRS_006_RBAC_User_Alter_Syntax = Requirement( - name='RQ.SRS-006.RBAC.User.Alter.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.User.Alter.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `ALTER USER` statement.\n' - '\n' - '```sql\n' - 'ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]\n' - ' [RENAME TO new_name]\n' + "[ClickHouse] SHALL support the following syntax for the `ALTER USER` statement.\n" + "\n" + "```sql\n" + "ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name]\n" + " [RENAME TO new_name]\n" " [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]\n" " [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]\n" - ' [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]\n' + " [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]\n" " [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='5.3.16.24') + num="5.3.16.24", +) RQ_SRS_006_RBAC_User_ShowCreateUser = Requirement( - name='RQ.SRS-006.RBAC.User.ShowCreateUser', - version='1.0', + name="RQ.SRS-006.RBAC.User.ShowCreateUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the `CREATE USER` statement used to create the current user object\n' - 'using the `SHOW CREATE USER` statement with `CURRENT_USER` or no argument.\n' - '\n' - ), + "[ClickHouse] SHALL support showing the `CREATE USER` statement used to create the current user object\n" + "using the `SHOW CREATE USER` statement with `CURRENT_USER` or no argument.\n" + "\n" + ), link=None, level=4, - num='5.3.17.1') + num="5.3.17.1", +) RQ_SRS_006_RBAC_User_ShowCreateUser_For = Requirement( - name='RQ.SRS-006.RBAC.User.ShowCreateUser.For', - version='1.0', + name="RQ.SRS-006.RBAC.User.ShowCreateUser.For", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the `CREATE USER` statement used to create the specified user object\n' - 'using the `FOR` clause in the `SHOW CREATE USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support showing the `CREATE USER` statement used to create the specified user object\n" + "using the `FOR` clause in the `SHOW CREATE USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.17.2') + num="5.3.17.2", +) RQ_SRS_006_RBAC_User_ShowCreateUser_Syntax = Requirement( - name='RQ.SRS-006.RBAC.User.ShowCreateUser.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.User.ShowCreateUser.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the following syntax for `SHOW CREATE USER` statement.\n' - '\n' - '```sql\n' - 'SHOW CREATE USER [name | CURRENT_USER]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support showing the following syntax for `SHOW CREATE USER` statement.\n" + "\n" + "```sql\n" + "SHOW CREATE USER [name | CURRENT_USER]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.3.17.3') + num="5.3.17.3", +) RQ_SRS_006_RBAC_User_Drop = Requirement( - name='RQ.SRS-006.RBAC.User.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.User.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing a user account using `DROP USER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing a user account using `DROP USER` statement.\n" + "\n" + ), link=None, level=4, - num='5.3.18.1') + num="5.3.18.1", +) RQ_SRS_006_RBAC_User_Drop_IfExists = Requirement( - name='RQ.SRS-006.RBAC.User.Drop.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.User.Drop.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP USER` statement\n' - 'to skip raising an exception if the user account does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if a user does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP USER` statement\n" + "to skip raising an exception if the user account does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be\n" + "raised if a user does not exist.\n" + "\n" + ), link=None, level=4, - num='5.3.18.2') + num="5.3.18.2", +) RQ_SRS_006_RBAC_User_Drop_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.User.Drop.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.User.Drop.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP USER` statement\n' - 'to specify the name of the cluster the user should be dropped from.\n' - '\n' - ), + "[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP USER` statement\n" + "to specify the name of the cluster the user should be dropped from.\n" + "\n" + ), link=None, level=4, - num='5.3.18.3') + num="5.3.18.3", +) RQ_SRS_006_RBAC_User_Drop_Syntax = Requirement( - name='RQ.SRS-006.RBAC.User.Drop.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.User.Drop.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for `DROP USER` statement\n' - '\n' - '```sql\n' - 'DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for `DROP USER` statement\n" + "\n" + "```sql\n" + "DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.3.18.4') + num="5.3.18.4", +) RQ_SRS_006_RBAC_Role = Requirement( - name='RQ.SRS-006.RBAC.Role', - version='1.0', + name="RQ.SRS-006.RBAC.Role", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClikHouse] SHALL support creation and manipulation of **roles**\n' - 'to which privileges, settings profile, quotas and row policies can be\n' - 'assigned.\n' - '\n' - ), + "[ClikHouse] SHALL support creation and manipulation of **roles**\n" + "to which privileges, settings profile, quotas and row policies can be\n" + "assigned.\n" + "\n" + ), link=None, level=3, - num='5.4.1') + num="5.4.1", +) RQ_SRS_006_RBAC_Role_Privileges = Requirement( - name='RQ.SRS-006.RBAC.Role.Privileges', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Privileges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more privileges to a **role**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more privileges to a **role**.\n" + "\n" + ), link=None, level=3, - num='5.4.2') + num="5.4.2", +) RQ_SRS_006_RBAC_Role_Variables = Requirement( - name='RQ.SRS-006.RBAC.Role.Variables', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Variables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more variables to a **role**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more variables to a **role**.\n" + "\n" + ), link=None, level=3, - num='5.4.3') + num="5.4.3", +) RQ_SRS_006_RBAC_Role_SettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.Role.SettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.Role.SettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **settings profiles**\n' - 'to a **role**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **settings profiles**\n" + "to a **role**.\n" + "\n" + ), link=None, level=3, - num='5.4.4') + num="5.4.4", +) RQ_SRS_006_RBAC_Role_Quotas = Requirement( - name='RQ.SRS-006.RBAC.Role.Quotas', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Quotas", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **quotas** to a **role**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **quotas** to a **role**.\n" + "\n" + ), link=None, level=3, - num='5.4.5') + num="5.4.5", +) RQ_SRS_006_RBAC_Role_RowPolicies = Requirement( - name='RQ.SRS-006.RBAC.Role.RowPolicies', - version='1.0', + name="RQ.SRS-006.RBAC.Role.RowPolicies", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning one or more **row policies** to a **role**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning one or more **row policies** to a **role**.\n" + "\n" + ), link=None, level=3, - num='5.4.6') + num="5.4.6", +) RQ_SRS_006_RBAC_Role_Create = Requirement( - name='RQ.SRS-006.RBAC.Role.Create', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating a **role** using `CREATE ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support creating a **role** using `CREATE ROLE` statement.\n" + "\n" + ), link=None, level=4, - num='5.4.7.1') + num="5.4.7.1", +) RQ_SRS_006_RBAC_Role_Create_IfNotExists = Requirement( - name='RQ.SRS-006.RBAC.Role.Create.IfNotExists', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Create.IfNotExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE ROLE` statement\n' - 'to raising an exception if a role with the same **name** already exists.\n' - 'If the `IF NOT EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if a role with the same **name** already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE ROLE` statement\n" + "to raising an exception if a role with the same **name** already exists.\n" + "If the `IF NOT EXISTS` clause is not specified then an exception SHALL be\n" + "raised if a role with the same **name** already exists.\n" + "\n" + ), link=None, level=4, - num='5.4.7.2') + num="5.4.7.2", +) RQ_SRS_006_RBAC_Role_Create_Replace = Requirement( - name='RQ.SRS-006.RBAC.Role.Create.Replace', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Create.Replace", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE ROLE` statement\n' - 'to replace existing role if it already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE ROLE` statement\n" + "to replace existing role if it already exists.\n" + "\n" + ), link=None, level=4, - num='5.4.7.3') + num="5.4.7.3", +) RQ_SRS_006_RBAC_Role_Create_Settings = Requirement( - name='RQ.SRS-006.RBAC.Role.Create.Settings', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Create.Settings", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying settings and profile using `SETTINGS`\n' - 'clause in the `CREATE ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying settings and profile using `SETTINGS`\n" + "clause in the `CREATE ROLE` statement.\n" + "\n" + ), link=None, level=4, - num='5.4.7.4') + num="5.4.7.4", +) RQ_SRS_006_RBAC_Role_Create_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Role.Create.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Create.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `CREATE ROLE` statement\n' - '\n' - '``` sql\n' - 'CREATE ROLE [IF NOT EXISTS | OR REPLACE] name\n' + "[ClickHouse] SHALL support the following syntax for the `CREATE ROLE` statement\n" + "\n" + "``` sql\n" + "CREATE ROLE [IF NOT EXISTS | OR REPLACE] name\n" " [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='5.4.7.5') + num="5.4.7.5", +) RQ_SRS_006_RBAC_Role_Alter = Requirement( - name='RQ.SRS-006.RBAC.Role.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering one **role** using `ALTER ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering one **role** using `ALTER ROLE` statement.\n" + "\n" + ), link=None, level=4, - num='5.4.8.1') + num="5.4.8.1", +) RQ_SRS_006_RBAC_Role_Alter_IfExists = Requirement( - name='RQ.SRS-006.RBAC.Role.Alter.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Alter.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering one **role** using `ALTER ROLE IF EXISTS` statement, where no exception\n' - 'will be thrown if the role does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support altering one **role** using `ALTER ROLE IF EXISTS` statement, where no exception\n" + "will be thrown if the role does not exist.\n" + "\n" + ), link=None, level=4, - num='5.4.8.2') + num="5.4.8.2", +) RQ_SRS_006_RBAC_Role_Alter_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Role.Alter.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Alter.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering one **role** using `ALTER ROLE role ON CLUSTER` statement to specify the\n' - 'cluster location of the specified role.\n' - '\n' - ), + "[ClickHouse] SHALL support altering one **role** using `ALTER ROLE role ON CLUSTER` statement to specify the\n" + "cluster location of the specified role.\n" + "\n" + ), link=None, level=4, - num='5.4.8.3') + num="5.4.8.3", +) RQ_SRS_006_RBAC_Role_Alter_Rename = Requirement( - name='RQ.SRS-006.RBAC.Role.Alter.Rename', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Alter.Rename", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering one **role** using `ALTER ROLE role RENAME TO` statement which renames the\n' - 'role to a specified new name. If the new name already exists, that an exception SHALL be raised unless the\n' - '`IF EXISTS` clause is specified, by which no exception will be raised and nothing will change.\n' - '\n' - ), + "[ClickHouse] SHALL support altering one **role** using `ALTER ROLE role RENAME TO` statement which renames the\n" + "role to a specified new name. If the new name already exists, that an exception SHALL be raised unless the\n" + "`IF EXISTS` clause is specified, by which no exception will be raised and nothing will change.\n" + "\n" + ), link=None, level=4, - num='5.4.8.4') + num="5.4.8.4", +) RQ_SRS_006_RBAC_Role_Alter_Settings = Requirement( - name='RQ.SRS-006.RBAC.Role.Alter.Settings', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Alter.Settings", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the settings of one **role** using `ALTER ROLE role SETTINGS ...` statement.\n' - 'Altering variable values, creating max and min values, specifying readonly or writable, and specifying the\n' - 'profiles for which this alter change shall be applied to, are all supported, using the following syntax.\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support altering the settings of one **role** using `ALTER ROLE role SETTINGS ...` statement.\n" + "Altering variable values, creating max and min values, specifying readonly or writable, and specifying the\n" + "profiles for which this alter change shall be applied to, are all supported, using the following syntax.\n" + "\n" + "```sql\n" "[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]\n" - '```\n' - '\n' - 'One or more variables and profiles may be specified as shown above.\n' - '\n' - ), + "```\n" + "\n" + "One or more variables and profiles may be specified as shown above.\n" + "\n" + ), link=None, level=4, - num='5.4.8.5') + num="5.4.8.5", +) RQ_SRS_006_RBAC_Role_Alter_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Role.Alter.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Alter.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '```sql\n' - 'ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name]\n' - ' [RENAME TO new_name]\n' + "```sql\n" + "ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name]\n" + " [RENAME TO new_name]\n" " [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='5.4.8.6') + num="5.4.8.6", +) RQ_SRS_006_RBAC_Role_Drop = Requirement( - name='RQ.SRS-006.RBAC.Role.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more roles using `DROP ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing one or more roles using `DROP ROLE` statement.\n" + "\n" + ), link=None, level=4, - num='5.4.9.1') + num="5.4.9.1", +) RQ_SRS_006_RBAC_Role_Drop_IfExists = Requirement( - name='RQ.SRS-006.RBAC.Role.Drop.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Drop.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP ROLE` statement\n' - 'to skip raising an exception if the role does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if a role does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP ROLE` statement\n" + "to skip raising an exception if the role does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be\n" + "raised if a role does not exist.\n" + "\n" + ), link=None, level=4, - num='5.4.9.2') + num="5.4.9.2", +) RQ_SRS_006_RBAC_Role_Drop_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Role.Drop.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Drop.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP ROLE` statement to specify the cluster from which to drop the specified role.\n' - '\n' - ), + "[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP ROLE` statement to specify the cluster from which to drop the specified role.\n" + "\n" + ), link=None, level=4, - num='5.4.9.3') + num="5.4.9.3", +) RQ_SRS_006_RBAC_Role_Drop_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Role.Drop.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Role.Drop.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP ROLE` statement\n' - '\n' - '``` sql\n' - 'DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `DROP ROLE` statement\n" + "\n" + "``` sql\n" + "DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.4.9.4') + num="5.4.9.4", +) RQ_SRS_006_RBAC_Role_ShowCreate = Requirement( - name='RQ.SRS-006.RBAC.Role.ShowCreate', - version='1.0', + name="RQ.SRS-006.RBAC.Role.ShowCreate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support viewing the settings for a role upon creation with the `SHOW CREATE ROLE`\n' - 'statement.\n' - '\n' - ), + "[ClickHouse] SHALL support viewing the settings for a role upon creation with the `SHOW CREATE ROLE`\n" + "statement.\n" + "\n" + ), link=None, level=4, - num='5.4.10.1') + num="5.4.10.1", +) RQ_SRS_006_RBAC_Role_ShowCreate_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Role.ShowCreate.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Role.ShowCreate.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `SHOW CREATE ROLE` command.\n' - '\n' - '```sql\n' - 'SHOW CREATE ROLE name\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `SHOW CREATE ROLE` command.\n" + "\n" + "```sql\n" + "SHOW CREATE ROLE name\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.4.10.2') + num="5.4.10.2", +) RQ_SRS_006_RBAC_PartialRevokes = Requirement( - name='RQ.SRS-006.RBAC.PartialRevokes', - version='1.0', + name="RQ.SRS-006.RBAC.PartialRevokes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support partial revoking of privileges granted\n' - 'to a **user** or a **role**.\n' - '\n' - ), + "[ClickHouse] SHALL support partial revoking of privileges granted\n" + "to a **user** or a **role**.\n" + "\n" + ), link=None, level=3, - num='5.5.1') + num="5.5.1", +) RQ_SRS_006_RBAC_PartialRevoke_Syntax = Requirement( - name='RQ.SRS-006.RBAC.PartialRevoke.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.PartialRevoke.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support partial revokes by using `partial_revokes` variable\n' - 'that can be set or unset using the following syntax.\n' - '\n' - 'To disable partial revokes the `partial_revokes` variable SHALL be set to `0`\n' - '\n' - '```sql\n' - 'SET partial_revokes = 0\n' - '```\n' - '\n' - 'To enable partial revokes the `partial revokes` variable SHALL be set to `1`\n' - '\n' - '```sql\n' - 'SET partial_revokes = 1\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support partial revokes by using `partial_revokes` variable\n" + "that can be set or unset using the following syntax.\n" + "\n" + "To disable partial revokes the `partial_revokes` variable SHALL be set to `0`\n" + "\n" + "```sql\n" + "SET partial_revokes = 0\n" + "```\n" + "\n" + "To enable partial revokes the `partial revokes` variable SHALL be set to `1`\n" + "\n" + "```sql\n" + "SET partial_revokes = 1\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.5.2') + num="5.5.2", +) RQ_SRS_006_RBAC_SettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creation and manipulation of **settings profiles**\n' - 'that can include value definition for one or more variables and can\n' - 'can be assigned to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support creation and manipulation of **settings profiles**\n" + "that can include value definition for one or more variables and can\n" + "can be assigned to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=3, - num='5.6.1') + num="5.6.1", +) RQ_SRS_006_RBAC_SettingsProfile_Constraints = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Constraints', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Constraints", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning min, max and read-only constraints\n' - 'for the variables specified in the **settings profile**.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning min, max and read-only constraints\n" + "for the variables specified in the **settings profile**.\n" + "\n" + ), link=None, level=3, - num='5.6.2') + num="5.6.2", +) RQ_SRS_006_RBAC_SettingsProfile_Create = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating settings profile using the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support creating settings profile using the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.1') + num="5.6.3.1", +) RQ_SRS_006_RBAC_SettingsProfile_Create_IfNotExists = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.IfNotExists', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.IfNotExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE SETTINGS PROFILE` statement\n' - 'to skip raising an exception if a settings profile with the same **name** already exists.\n' - 'If `IF NOT EXISTS` clause is not specified then an exception SHALL be raised if\n' - 'a settings profile with the same **name** already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE SETTINGS PROFILE` statement\n" + "to skip raising an exception if a settings profile with the same **name** already exists.\n" + "If `IF NOT EXISTS` clause is not specified then an exception SHALL be raised if\n" + "a settings profile with the same **name** already exists.\n" + "\n" + ), link=None, level=4, - num='5.6.3.2') + num="5.6.3.2", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Replace = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Replace', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Replace", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE SETTINGS PROFILE` statement\n' - 'to replace existing settings profile if it already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE SETTINGS PROFILE` statement\n" + "to replace existing settings profile if it already exists.\n" + "\n" + ), link=None, level=4, - num='5.6.3.3') + num="5.6.3.3", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Variables = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Variables', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Variables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning values and constraints to one or more\n' - 'variables in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning values and constraints to one or more\n" + "variables in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.4') + num="5.6.3.4", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Value = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Value', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Value", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning variable value in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning variable value in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.5') + num="5.6.3.5", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Constraints', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Constraints", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting `MIN`, `MAX`, `READONLY`, and `WRITABLE`\n' - 'constraints for the variables in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support setting `MIN`, `MAX`, `READONLY`, and `WRITABLE`\n" + "constraints for the variables in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.6') + num="5.6.3.6", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning settings profile to one or more users\n' - 'or roles in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning settings profile to one or more users\n" + "or roles in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.7') + num="5.6.3.7", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_None = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.None', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning settings profile to no users or roles using\n' - '`TO NONE` clause in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning settings profile to no users or roles using\n" + "`TO NONE` clause in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.8') + num="5.6.3.8", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_All = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.All', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning settings profile to all current users and roles\n' - 'using `TO ALL` clause in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning settings profile to all current users and roles\n" + "using `TO ALL` clause in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.9') + num="5.6.3.9", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support excluding assignment to one or more users or roles using\n' - 'the `ALL EXCEPT` clause in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support excluding assignment to one or more users or roles using\n" + "the `ALL EXCEPT` clause in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.10') + num="5.6.3.10", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Inherit', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Inherit", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support inheriting profile settings from indicated profile using\n' - 'the `INHERIT` clause in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support inheriting profile settings from indicated profile using\n" + "the `INHERIT` clause in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.11') + num="5.6.3.11", +) RQ_SRS_006_RBAC_SettingsProfile_Create_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying what cluster to create settings profile on\n' - 'using `ON CLUSTER` clause in the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying what cluster to create settings profile on\n" + "using `ON CLUSTER` clause in the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.3.12') + num="5.6.3.12", +) RQ_SRS_006_RBAC_SettingsProfile_Create_Syntax = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Create.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `CREATE SETTINGS PROFILE` statement.\n' - '\n' - '``` sql\n' - 'CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name\n' - ' [ON CLUSTER cluster_name]\n' + "[ClickHouse] SHALL support the following syntax for the `CREATE SETTINGS PROFILE` statement.\n" + "\n" + "``` sql\n" + "CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name\n" + " [ON CLUSTER cluster_name]\n" " [SET varname [= value] [MIN min] [MAX max] [READONLY|WRITABLE] | [INHERIT 'profile_name'] [,...]]\n" - ' [TO {user_or_role [,...] | NONE | ALL | ALL EXCEPT user_or_role [,...]}]\n' - '```\n' - '\n' - ), + " [TO {user_or_role [,...] | NONE | ALL | ALL EXCEPT user_or_role [,...]}]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.6.3.13') + num="5.6.3.13", +) RQ_SRS_006_RBAC_SettingsProfile_Alter = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering settings profile using the `ALTER STETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering settings profile using the `ALTER STETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.1') + num="5.6.4.1", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_IfExists = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF EXISTS` clause in the `ALTER SETTINGS PROFILE` statement\n' - 'to not raise exception if a settings profile does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if a settings profile does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF EXISTS` clause in the `ALTER SETTINGS PROFILE` statement\n" + "to not raise exception if a settings profile does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be\n" + "raised if a settings profile does not exist.\n" + "\n" + ), link=None, level=4, - num='5.6.4.2') + num="5.6.4.2", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Rename = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Rename', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Rename", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support renaming settings profile using the `RANAME TO` clause\n' - 'in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support renaming settings profile using the `RANAME TO` clause\n" + "in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.3') + num="5.6.4.3", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering values and constraints of one or more\n' - 'variables in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering values and constraints of one or more\n" + "variables in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.4') + num="5.6.4.4", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Value', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Value", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering value of the variable in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering value of the variable in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.5') + num="5.6.4.5", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Constraints', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Constraints", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering `MIN`, `MAX`, `READONLY`, and `WRITABLE`\n' - 'constraints for the variables in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering `MIN`, `MAX`, `READONLY`, and `WRITABLE`\n" + "constraints for the variables in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.6') + num="5.6.4.6", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning settings profile to one or more users\n' - 'or roles using the `TO` clause in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning settings profile to one or more users\n" + "or roles using the `TO` clause in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.7') + num="5.6.4.7", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_None = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.None', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning settings profile to no users or roles using the\n' - '`TO NONE` clause in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning settings profile to no users or roles using the\n" + "`TO NONE` clause in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.8') + num="5.6.4.8", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_All = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.All', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning settings profile to all current users and roles\n' - 'using the `TO ALL` clause in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning settings profile to all current users and roles\n" + "using the `TO ALL` clause in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.9') + num="5.6.4.9", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support excluding assignment to one or more users or roles using\n' - 'the `TO ALL EXCEPT` clause in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support excluding assignment to one or more users or roles using\n" + "the `TO ALL EXCEPT` clause in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.10') + num="5.6.4.10", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_Inherit = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.Inherit', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.Inherit", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the settings profile by inheriting settings from\n' - 'specified profile using `INHERIT` clause in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the settings profile by inheriting settings from\n" + "specified profile using `INHERIT` clause in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.11') + num="5.6.4.11", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the settings profile on a specified cluster using\n' - '`ON CLUSTER` clause in the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the settings profile on a specified cluster using\n" + "`ON CLUSTER` clause in the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.4.12') + num="5.6.4.12", +) RQ_SRS_006_RBAC_SettingsProfile_Alter_Syntax = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `ALTER SETTINGS PROFILE` statement.\n' - '\n' - '``` sql\n' - 'ALTER SETTINGS PROFILE [IF EXISTS] name\n' - ' [ON CLUSTER cluster_name]\n' - ' [RENAME TO new_name]\n' + "[ClickHouse] SHALL support the following syntax for the `ALTER SETTINGS PROFILE` statement.\n" + "\n" + "``` sql\n" + "ALTER SETTINGS PROFILE [IF EXISTS] name\n" + " [ON CLUSTER cluster_name]\n" + " [RENAME TO new_name]\n" " [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...]\n" - ' [TO {user_or_role [,...] | NONE | ALL | ALL EXCEPT user_or_role [,...]]}\n' - '```\n' - '\n' - ), + " [TO {user_or_role [,...] | NONE | ALL | ALL EXCEPT user_or_role [,...]]}\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.6.4.13') + num="5.6.4.13", +) RQ_SRS_006_RBAC_SettingsProfile_Drop = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more settings profiles using the `DROP SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing one or more settings profiles using the `DROP SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.5.1') + num="5.6.5.1", +) RQ_SRS_006_RBAC_SettingsProfile_Drop_IfExists = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Drop.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Drop.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP SETTINGS PROFILE` statement\n' - 'to skip raising an exception if the settings profile does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if a settings profile does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP SETTINGS PROFILE` statement\n" + "to skip raising an exception if the settings profile does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be\n" + "raised if a settings profile does not exist.\n" + "\n" + ), link=None, level=4, - num='5.6.5.2') + num="5.6.5.2", +) RQ_SRS_006_RBAC_SettingsProfile_Drop_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Drop.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Drop.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support dropping one or more settings profiles on specified cluster using\n' - '`ON CLUSTER` clause in the `DROP SETTINGS PROFILE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support dropping one or more settings profiles on specified cluster using\n" + "`ON CLUSTER` clause in the `DROP SETTINGS PROFILE` statement.\n" + "\n" + ), link=None, level=4, - num='5.6.5.3') + num="5.6.5.3", +) RQ_SRS_006_RBAC_SettingsProfile_Drop_Syntax = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.Drop.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.Drop.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP SETTINGS PROFILE` statement\n' - '\n' - '``` sql\n' - 'DROP SETTINGS PROFILE [IF EXISTS] name [,name,...]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `DROP SETTINGS PROFILE` statement\n" + "\n" + "``` sql\n" + "DROP SETTINGS PROFILE [IF EXISTS] name [,name,...]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.6.5.4') + num="5.6.5.4", +) RQ_SRS_006_RBAC_SettingsProfile_ShowCreateSettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.SettingsProfile.ShowCreateSettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.SettingsProfile.ShowCreateSettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the `CREATE SETTINGS PROFILE` statement used to create the settings profile\n' - 'using the `SHOW CREATE SETTINGS PROFILE` statement with the following syntax\n' - '\n' - '``` sql\n' - 'SHOW CREATE SETTINGS PROFILE name\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support showing the `CREATE SETTINGS PROFILE` statement used to create the settings profile\n" + "using the `SHOW CREATE SETTINGS PROFILE` statement with the following syntax\n" + "\n" + "``` sql\n" + "SHOW CREATE SETTINGS PROFILE name\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.6.6.1') + num="5.6.6.1", +) RQ_SRS_006_RBAC_Quotas = Requirement( - name='RQ.SRS-006.RBAC.Quotas', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creation and manipulation of **quotas**\n' - 'that can be used to limit resource usage by a **user** or a **role**\n' - 'over a period of time.\n' - '\n' - ), + "[ClickHouse] SHALL support creation and manipulation of **quotas**\n" + "that can be used to limit resource usage by a **user** or a **role**\n" + "over a period of time.\n" + "\n" + ), link=None, level=3, - num='5.7.1') + num="5.7.1", +) RQ_SRS_006_RBAC_Quotas_Keyed = Requirement( - name='RQ.SRS-006.RBAC.Quotas.Keyed', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.Keyed", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating **quotas** that are keyed\n' - 'so that a quota is tracked separately for each key value.\n' - '\n' - ), + "[ClickHouse] SHALL support creating **quotas** that are keyed\n" + "so that a quota is tracked separately for each key value.\n" + "\n" + ), link=None, level=3, - num='5.7.2') + num="5.7.2", +) RQ_SRS_006_RBAC_Quotas_Queries = Requirement( - name='RQ.SRS-006.RBAC.Quotas.Queries', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.Queries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **queries** quota to limit the total number of requests.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **queries** quota to limit the total number of requests.\n" + "\n" + ), link=None, level=3, - num='5.7.3') + num="5.7.3", +) RQ_SRS_006_RBAC_Quotas_Errors = Requirement( - name='RQ.SRS-006.RBAC.Quotas.Errors', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.Errors", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **errors** quota to limit the number of queries that threw an exception.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **errors** quota to limit the number of queries that threw an exception.\n" + "\n" + ), link=None, level=3, - num='5.7.4') + num="5.7.4", +) RQ_SRS_006_RBAC_Quotas_ResultRows = Requirement( - name='RQ.SRS-006.RBAC.Quotas.ResultRows', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.ResultRows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **result rows** quota to limit the\n' - 'the total number of rows given as the result.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **result rows** quota to limit the\n" + "the total number of rows given as the result.\n" + "\n" + ), link=None, level=3, - num='5.7.5') + num="5.7.5", +) RQ_SRS_006_RBAC_Quotas_ReadRows = Requirement( - name='RQ.SRS-006.RBAC.Quotas.ReadRows', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.ReadRows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **read rows** quota to limit the total\n' - 'number of source rows read from tables for running the query on all remote servers.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **read rows** quota to limit the total\n" + "number of source rows read from tables for running the query on all remote servers.\n" + "\n" + ), link=None, level=3, - num='5.7.6') + num="5.7.6", +) RQ_SRS_006_RBAC_Quotas_ResultBytes = Requirement( - name='RQ.SRS-006.RBAC.Quotas.ResultBytes', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.ResultBytes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **result bytes** quota to limit the total number\n' - 'of bytes that can be returned as the result.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **result bytes** quota to limit the total number\n" + "of bytes that can be returned as the result.\n" + "\n" + ), link=None, level=3, - num='5.7.7') + num="5.7.7", +) RQ_SRS_006_RBAC_Quotas_ReadBytes = Requirement( - name='RQ.SRS-006.RBAC.Quotas.ReadBytes', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.ReadBytes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **read bytes** quota to limit the total number\n' - 'of source bytes read from tables for running the query on all remote servers.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **read bytes** quota to limit the total number\n" + "of source bytes read from tables for running the query on all remote servers.\n" + "\n" + ), link=None, level=3, - num='5.7.8') + num="5.7.8", +) RQ_SRS_006_RBAC_Quotas_ExecutionTime = Requirement( - name='RQ.SRS-006.RBAC.Quotas.ExecutionTime', - version='1.0', + name="RQ.SRS-006.RBAC.Quotas.ExecutionTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting **execution time** quota to limit the maximum\n' - 'query execution time.\n' - '\n' - ), + "[ClickHouse] SHALL support setting **execution time** quota to limit the maximum\n" + "query execution time.\n" + "\n" + ), link=None, level=3, - num='5.7.9') + num="5.7.9", +) RQ_SRS_006_RBAC_Quota_Create = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating quotas using the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support creating quotas using the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.1') + num="5.7.10.1", +) RQ_SRS_006_RBAC_Quota_Create_IfNotExists = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.IfNotExists', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.IfNotExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE QUOTA` statement\n' - 'to skip raising an exception if a quota with the same **name** already exists.\n' - 'If `IF NOT EXISTS` clause is not specified then an exception SHALL be raised if\n' - 'a quota with the same **name** already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE QUOTA` statement\n" + "to skip raising an exception if a quota with the same **name** already exists.\n" + "If `IF NOT EXISTS` clause is not specified then an exception SHALL be raised if\n" + "a quota with the same **name** already exists.\n" + "\n" + ), link=None, level=4, - num='5.7.10.2') + num="5.7.10.2", +) RQ_SRS_006_RBAC_Quota_Create_Replace = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Replace', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Replace", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE QUOTA` statement\n' - 'to replace existing quota if it already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE QUOTA` statement\n" + "to replace existing quota if it already exists.\n" + "\n" + ), link=None, level=4, - num='5.7.10.3') + num="5.7.10.3", +) RQ_SRS_006_RBAC_Quota_Create_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating quotas on a specific cluster with the\n' - '`ON CLUSTER` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support creating quotas on a specific cluster with the\n" + "`ON CLUSTER` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.4') + num="5.7.10.4", +) RQ_SRS_006_RBAC_Quota_Create_Interval = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Interval', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Interval", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support defining the quota interval that specifies\n' - 'a period of time over for which the quota SHALL apply using the\n' - '`FOR INTERVAL` clause in the `CREATE QUOTA` statement.\n' - '\n' - 'This statement SHALL also support a number and a time period which will be one\n' - 'of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n' - '\n' - '`FOR INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some real number\n' - 'to define the interval.\n' - '\n' - ), + "[ClickHouse] SHALL support defining the quota interval that specifies\n" + "a period of time over for which the quota SHALL apply using the\n" + "`FOR INTERVAL` clause in the `CREATE QUOTA` statement.\n" + "\n" + "This statement SHALL also support a number and a time period which will be one\n" + "of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n" + "\n" + "`FOR INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some real number\n" + "to define the interval.\n" + "\n" + ), link=None, level=4, - num='5.7.10.5') + num="5.7.10.5", +) RQ_SRS_006_RBAC_Quota_Create_Interval_Randomized = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Interval.Randomized', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Interval.Randomized", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support defining the quota randomized interval that specifies\n' - 'a period of time over for which the quota SHALL apply using the\n' - '`FOR RANDOMIZED INTERVAL` clause in the `CREATE QUOTA` statement.\n' - '\n' - 'This statement SHALL also support a number and a time period which will be one\n' - 'of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n' - '\n' - '`FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some\n' - 'real number to define the interval.\n' - '\n' - ), + "[ClickHouse] SHALL support defining the quota randomized interval that specifies\n" + "a period of time over for which the quota SHALL apply using the\n" + "`FOR RANDOMIZED INTERVAL` clause in the `CREATE QUOTA` statement.\n" + "\n" + "This statement SHALL also support a number and a time period which will be one\n" + "of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n" + "\n" + "`FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some\n" + "real number to define the interval.\n" + "\n" + ), link=None, level=4, - num='5.7.10.6') + num="5.7.10.6", +) RQ_SRS_006_RBAC_Quota_Create_Queries = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Queries', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Queries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting number of requests over a period of time\n' - 'using the `QUERIES` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting number of requests over a period of time\n" + "using the `QUERIES` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.7') + num="5.7.10.7", +) RQ_SRS_006_RBAC_Quota_Create_Errors = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Errors', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Errors", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting number of queries that threw an exception\n' - 'using the `ERRORS` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting number of queries that threw an exception\n" + "using the `ERRORS` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.8') + num="5.7.10.8", +) RQ_SRS_006_RBAC_Quota_Create_ResultRows = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.ResultRows', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.ResultRows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the total number of rows given as the result\n' - 'using the `RESULT ROWS` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the total number of rows given as the result\n" + "using the `RESULT ROWS` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.9') + num="5.7.10.9", +) RQ_SRS_006_RBAC_Quota_Create_ReadRows = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.ReadRows', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.ReadRows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the total number of source rows read from tables\n' - 'for running the query on all remote servers\n' - 'using the `READ ROWS` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the total number of source rows read from tables\n" + "for running the query on all remote servers\n" + "using the `READ ROWS` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.10') + num="5.7.10.10", +) RQ_SRS_006_RBAC_Quota_Create_ResultBytes = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.ResultBytes', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.ResultBytes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the total number of bytes that can be returned as the result\n' - 'using the `RESULT BYTES` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the total number of bytes that can be returned as the result\n" + "using the `RESULT BYTES` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.11') + num="5.7.10.11", +) RQ_SRS_006_RBAC_Quota_Create_ReadBytes = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.ReadBytes', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.ReadBytes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the total number of source bytes read from tables\n' - 'for running the query on all remote servers\n' - 'using the `READ BYTES` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the total number of source bytes read from tables\n" + "for running the query on all remote servers\n" + "using the `READ BYTES` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.12') + num="5.7.10.12", +) RQ_SRS_006_RBAC_Quota_Create_ExecutionTime = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.ExecutionTime', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.ExecutionTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the maximum query execution time\n' - 'using the `EXECUTION TIME` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the maximum query execution time\n" + "using the `EXECUTION TIME` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.13') + num="5.7.10.13", +) RQ_SRS_006_RBAC_Quota_Create_NoLimits = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.NoLimits', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.NoLimits", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the maximum query execution time\n' - 'using the `NO LIMITS` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the maximum query execution time\n" + "using the `NO LIMITS` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.14') + num="5.7.10.14", +) RQ_SRS_006_RBAC_Quota_Create_TrackingOnly = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.TrackingOnly', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.TrackingOnly", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the maximum query execution time\n' - 'using the `TRACKING ONLY` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the maximum query execution time\n" + "using the `TRACKING ONLY` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.15') + num="5.7.10.15", +) RQ_SRS_006_RBAC_Quota_Create_KeyedBy = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.KeyedBy', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.KeyedBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support to track quota for some key\n' - 'following the `KEYED BY` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support to track quota for some key\n" + "following the `KEYED BY` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.16') + num="5.7.10.16", +) RQ_SRS_006_RBAC_Quota_Create_KeyedByOptions = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.KeyedByOptions', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.KeyedByOptions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support to track quota separately for some parameter\n' + "[ClickHouse] SHALL support to track quota separately for some parameter\n" "using the `KEYED BY 'parameter'` clause in the `CREATE QUOTA` statement.\n" - '\n' + "\n" "'parameter' can be one of:\n" "`{'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}`\n" - '\n' - ), + "\n" + ), link=None, level=4, - num='5.7.10.17') + num="5.7.10.17", +) RQ_SRS_006_RBAC_Quota_Create_Assignment = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Assignment', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Assignment", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning quota to one or more users\n' - 'or roles using the `TO` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning quota to one or more users\n" + "or roles using the `TO` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.18') + num="5.7.10.18", +) RQ_SRS_006_RBAC_Quota_Create_Assignment_None = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Assignment.None', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Assignment.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning quota to no users or roles using\n' - '`TO NONE` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning quota to no users or roles using\n" + "`TO NONE` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.19') + num="5.7.10.19", +) RQ_SRS_006_RBAC_Quota_Create_Assignment_All = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Assignment.All', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Assignment.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning quota to all current users and roles\n' - 'using `TO ALL` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning quota to all current users and roles\n" + "using `TO ALL` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.20') + num="5.7.10.20", +) RQ_SRS_006_RBAC_Quota_Create_Assignment_Except = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Assignment.Except', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Assignment.Except", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support excluding assignment of quota to one or more users or roles using\n' - 'the `EXCEPT` clause in the `CREATE QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support excluding assignment of quota to one or more users or roles using\n" + "the `EXCEPT` clause in the `CREATE QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.10.21') + num="5.7.10.21", +) RQ_SRS_006_RBAC_Quota_Create_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Quota.Create.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Create.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `CREATE QUOTA` statement\n' - '\n' - '```sql\n' - 'CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]\n' + "[ClickHouse] SHALL support the following syntax for the `CREATE QUOTA` statement\n" + "\n" + "```sql\n" + "CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name]\n" " [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}]\n" - ' [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}\n' - ' {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] |\n' - ' NO LIMITS | TRACKING ONLY} [,...]]\n' - ' [TO {role [,...] | ALL | ALL EXCEPT role [,...]}]\n' - '```\n' - '\n' - ), + " [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}\n" + " {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] |\n" + " NO LIMITS | TRACKING ONLY} [,...]]\n" + " [TO {role [,...] | ALL | ALL EXCEPT role [,...]}]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.7.10.22') + num="5.7.10.22", +) RQ_SRS_006_RBAC_Quota_Alter = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering quotas using the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering quotas using the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.1') + num="5.7.11.1", +) RQ_SRS_006_RBAC_Quota_Alter_IfExists = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF EXISTS` clause in the `ALTER QUOTA` statement\n' - 'to skip raising an exception if a quota does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be raised if\n' - 'a quota does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF EXISTS` clause in the `ALTER QUOTA` statement\n" + "to skip raising an exception if a quota does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be raised if\n" + "a quota does not exist.\n" + "\n" + ), link=None, level=4, - num='5.7.11.2') + num="5.7.11.2", +) RQ_SRS_006_RBAC_Quota_Alter_Rename = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Rename', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Rename", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `RENAME TO` clause in the `ALTER QUOTA` statement\n' - 'to rename the quota to the specified name.\n' - '\n' - ), + "[ClickHouse] SHALL support `RENAME TO` clause in the `ALTER QUOTA` statement\n" + "to rename the quota to the specified name.\n" + "\n" + ), link=None, level=4, - num='5.7.11.3') + num="5.7.11.3", +) RQ_SRS_006_RBAC_Quota_Alter_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering quotas on a specific cluster with the\n' - '`ON CLUSTER` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering quotas on a specific cluster with the\n" + "`ON CLUSTER` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.4') + num="5.7.11.4", +) RQ_SRS_006_RBAC_Quota_Alter_Interval = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Interval', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Interval", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support redefining the quota interval that specifies\n' - 'a period of time over for which the quota SHALL apply using the\n' - '`FOR INTERVAL` clause in the `ALTER QUOTA` statement.\n' - '\n' - 'This statement SHALL also support a number and a time period which will be one\n' - 'of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n' - '\n' - '`FOR INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some real number\n' - 'to define the interval.\n' - '\n' - ), + "[ClickHouse] SHALL support redefining the quota interval that specifies\n" + "a period of time over for which the quota SHALL apply using the\n" + "`FOR INTERVAL` clause in the `ALTER QUOTA` statement.\n" + "\n" + "This statement SHALL also support a number and a time period which will be one\n" + "of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n" + "\n" + "`FOR INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some real number\n" + "to define the interval.\n" + "\n" + ), link=None, level=4, - num='5.7.11.5') + num="5.7.11.5", +) RQ_SRS_006_RBAC_Quota_Alter_Interval_Randomized = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Interval.Randomized', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Interval.Randomized", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support redefining the quota randomized interval that specifies\n' - 'a period of time over for which the quota SHALL apply using the\n' - '`FOR RANDOMIZED INTERVAL` clause in the `ALTER QUOTA` statement.\n' - '\n' - 'This statement SHALL also support a number and a time period which will be one\n' - 'of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n' - '\n' - '`FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some\n' - 'real number to define the interval.\n' - '\n' - ), + "[ClickHouse] SHALL support redefining the quota randomized interval that specifies\n" + "a period of time over for which the quota SHALL apply using the\n" + "`FOR RANDOMIZED INTERVAL` clause in the `ALTER QUOTA` statement.\n" + "\n" + "This statement SHALL also support a number and a time period which will be one\n" + "of `{SECOND | MINUTE | HOUR | DAY | MONTH}`. Thus, the complete syntax SHALL be:\n" + "\n" + "`FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY}` where number is some\n" + "real number to define the interval.\n" + "\n" + ), link=None, level=4, - num='5.7.11.6') + num="5.7.11.6", +) RQ_SRS_006_RBAC_Quota_Alter_Queries = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Queries', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Queries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of number of requests over a period of time\n' - 'using the `QUERIES` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of number of requests over a period of time\n" + "using the `QUERIES` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.7') + num="5.7.11.7", +) RQ_SRS_006_RBAC_Quota_Alter_Errors = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Errors', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Errors", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of number of queries that threw an exception\n' - 'using the `ERRORS` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of number of queries that threw an exception\n" + "using the `ERRORS` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.8') + num="5.7.11.8", +) RQ_SRS_006_RBAC_Quota_Alter_ResultRows = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.ResultRows', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.ResultRows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of the total number of rows given as the result\n' - 'using the `RESULT ROWS` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of the total number of rows given as the result\n" + "using the `RESULT ROWS` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.9') + num="5.7.11.9", +) RQ_SRS_006_RBAC_Quota_Alter_ReadRows = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.ReadRows', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.ReadRows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of the total number of source rows read from tables\n' - 'for running the query on all remote servers\n' - 'using the `READ ROWS` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of the total number of source rows read from tables\n" + "for running the query on all remote servers\n" + "using the `READ ROWS` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.10') + num="5.7.11.10", +) RQ_SRS_006_RBAC_Quota_ALter_ResultBytes = Requirement( - name='RQ.SRS-006.RBAC.Quota.ALter.ResultBytes', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ALter.ResultBytes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of the total number of bytes that can be returned as the result\n' - 'using the `RESULT BYTES` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of the total number of bytes that can be returned as the result\n" + "using the `RESULT BYTES` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.11') + num="5.7.11.11", +) RQ_SRS_006_RBAC_Quota_Alter_ReadBytes = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.ReadBytes', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.ReadBytes", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of the total number of source bytes read from tables\n' - 'for running the query on all remote servers\n' - 'using the `READ BYTES` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of the total number of source bytes read from tables\n" + "for running the query on all remote servers\n" + "using the `READ BYTES` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.12') + num="5.7.11.12", +) RQ_SRS_006_RBAC_Quota_Alter_ExecutionTime = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.ExecutionTime', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.ExecutionTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering the limit of the maximum query execution time\n' - 'using the `EXECUTION TIME` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering the limit of the maximum query execution time\n" + "using the `EXECUTION TIME` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.13') + num="5.7.11.13", +) RQ_SRS_006_RBAC_Quota_Alter_NoLimits = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.NoLimits', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.NoLimits", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the maximum query execution time\n' - 'using the `NO LIMITS` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the maximum query execution time\n" + "using the `NO LIMITS` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.14') + num="5.7.11.14", +) RQ_SRS_006_RBAC_Quota_Alter_TrackingOnly = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.TrackingOnly', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.TrackingOnly", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support limiting the maximum query execution time\n' - 'using the `TRACKING ONLY` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support limiting the maximum query execution time\n" + "using the `TRACKING ONLY` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.15') + num="5.7.11.15", +) RQ_SRS_006_RBAC_Quota_Alter_KeyedBy = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.KeyedBy', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.KeyedBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering quota to track quota separately for some key\n' - 'following the `KEYED BY` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering quota to track quota separately for some key\n" + "following the `KEYED BY` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.16') + num="5.7.11.16", +) RQ_SRS_006_RBAC_Quota_Alter_KeyedByOptions = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.KeyedByOptions', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.KeyedByOptions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering quota to track quota separately for some parameter\n' + "[ClickHouse] SHALL support altering quota to track quota separately for some parameter\n" "using the `KEYED BY 'parameter'` clause in the `ALTER QUOTA` statement.\n" - '\n' + "\n" "'parameter' can be one of:\n" "`{'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}`\n" - '\n' - ), + "\n" + ), link=None, level=4, - num='5.7.11.17') + num="5.7.11.17", +) RQ_SRS_006_RBAC_Quota_Alter_Assignment = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Assignment', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning quota to one or more users\n' - 'or roles using the `TO` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning quota to one or more users\n" + "or roles using the `TO` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.18') + num="5.7.11.18", +) RQ_SRS_006_RBAC_Quota_Alter_Assignment_None = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Assignment.None', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning quota to no users or roles using\n' - '`TO NONE` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning quota to no users or roles using\n" + "`TO NONE` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.19') + num="5.7.11.19", +) RQ_SRS_006_RBAC_Quota_Alter_Assignment_All = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Assignment.All', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning quota to all current users and roles\n' - 'using `TO ALL` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning quota to all current users and roles\n" + "using `TO ALL` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.20') + num="5.7.11.20", +) RQ_SRS_006_RBAC_Quota_Alter_Assignment_Except = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Assignment.Except', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment.Except", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support excluding assignment of quota to one or more users or roles using\n' - 'the `EXCEPT` clause in the `ALTER QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support excluding assignment of quota to one or more users or roles using\n" + "the `EXCEPT` clause in the `ALTER QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.11.21') + num="5.7.11.21", +) RQ_SRS_006_RBAC_Quota_Alter_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Quota.Alter.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Alter.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `ALTER QUOTA` statement\n' - '\n' - '``` sql\n' - 'ALTER QUOTA [IF EXIST] name\n' - ' {{{QUERIES | ERRORS | RESULT ROWS | READ ROWS | RESULT BYTES | READ BYTES | EXECUTION TIME} number} [, ...] FOR INTERVAL number time_unit} [, ...]\n' - ' [KEYED BY USERNAME | KEYED BY IP | NOT KEYED] [ALLOW CUSTOM KEY | DISALLOW CUSTOM KEY]\n' - ' [TO {user_or_role [,...] | NONE | ALL} [EXCEPT user_or_role [,...]]]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `ALTER QUOTA` statement\n" + "\n" + "``` sql\n" + "ALTER QUOTA [IF EXIST] name\n" + " {{{QUERIES | ERRORS | RESULT ROWS | READ ROWS | RESULT BYTES | READ BYTES | EXECUTION TIME} number} [, ...] FOR INTERVAL number time_unit} [, ...]\n" + " [KEYED BY USERNAME | KEYED BY IP | NOT KEYED] [ALLOW CUSTOM KEY | DISALLOW CUSTOM KEY]\n" + " [TO {user_or_role [,...] | NONE | ALL} [EXCEPT user_or_role [,...]]]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.7.11.22') + num="5.7.11.22", +) RQ_SRS_006_RBAC_Quota_Drop = Requirement( - name='RQ.SRS-006.RBAC.Quota.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more quotas using the `DROP QUOTA` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing one or more quotas using the `DROP QUOTA` statement.\n" + "\n" + ), link=None, level=4, - num='5.7.12.1') + num="5.7.12.1", +) RQ_SRS_006_RBAC_Quota_Drop_IfExists = Requirement( - name='RQ.SRS-006.RBAC.Quota.Drop.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Drop.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP QUOTA` statement\n' - 'to skip raising an exception when the quota does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if the quota does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support using `IF EXISTS` clause in the `DROP QUOTA` statement\n" + "to skip raising an exception when the quota does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be\n" + "raised if the quota does not exist.\n" + "\n" + ), link=None, level=4, - num='5.7.12.2') + num="5.7.12.2", +) RQ_SRS_006_RBAC_Quota_Drop_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Quota.Drop.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Drop.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP QUOTA` statement\n' - 'to indicate the cluster the quota to be dropped is located on.\n' - '\n' - ), + "[ClickHouse] SHALL support using `ON CLUSTER` clause in the `DROP QUOTA` statement\n" + "to indicate the cluster the quota to be dropped is located on.\n" + "\n" + ), link=None, level=4, - num='5.7.12.3') + num="5.7.12.3", +) RQ_SRS_006_RBAC_Quota_Drop_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Quota.Drop.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.Drop.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP QUOTA` statement\n' - '\n' - '``` sql\n' - 'DROP QUOTA [IF EXISTS] name [,name...]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `DROP QUOTA` statement\n" + "\n" + "``` sql\n" + "DROP QUOTA [IF EXISTS] name [,name...]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.7.12.4') + num="5.7.12.4", +) RQ_SRS_006_RBAC_Quota_ShowQuotas = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowQuotas', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowQuotas", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing all of the current quotas\n' - 'using the `SHOW QUOTAS` statement with the following syntax\n' - '\n' - ), + "[ClickHouse] SHALL support showing all of the current quotas\n" + "using the `SHOW QUOTAS` statement with the following syntax\n" + "\n" + ), link=None, level=4, - num='5.7.13.1') + num="5.7.13.1", +) RQ_SRS_006_RBAC_Quota_ShowQuotas_IntoOutfile = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowQuotas.IntoOutfile', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.IntoOutfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `INTO OUTFILE` clause in the `SHOW QUOTAS` statement to define an outfile by some given string literal.\n' - '\n' - ), + "[ClickHouse] SHALL support the `INTO OUTFILE` clause in the `SHOW QUOTAS` statement to define an outfile by some given string literal.\n" + "\n" + ), link=None, level=4, - num='5.7.13.2') + num="5.7.13.2", +) RQ_SRS_006_RBAC_Quota_ShowQuotas_Format = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowQuotas.Format', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.Format", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `FORMAT` clause in the `SHOW QUOTAS` statement to define a format for the output quota list.\n' - '\n' - 'The types of valid formats are many, listed in output column:\n' - 'https://clickhouse.com/docs/en/interfaces/formats/\n' - '\n' - ), + "[ClickHouse] SHALL support the `FORMAT` clause in the `SHOW QUOTAS` statement to define a format for the output quota list.\n" + "\n" + "The types of valid formats are many, listed in output column:\n" + "https://clickhouse.com/docs/en/interfaces/formats/\n" + "\n" + ), link=None, level=4, - num='5.7.13.3') + num="5.7.13.3", +) RQ_SRS_006_RBAC_Quota_ShowQuotas_Settings = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowQuotas.Settings', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.Settings", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `SETTINGS` clause in the `SHOW QUOTAS` statement to define settings in the showing of all quotas.\n' - '\n' - ), + "[ClickHouse] SHALL support the `SETTINGS` clause in the `SHOW QUOTAS` statement to define settings in the showing of all quotas.\n" + "\n" + ), link=None, level=4, - num='5.7.13.4') + num="5.7.13.4", +) RQ_SRS_006_RBAC_Quota_ShowQuotas_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowQuotas.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the `SHOW QUOTAS` statement\n' - 'with the following syntax\n' - '``` sql\n' - 'SHOW QUOTAS\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support using the `SHOW QUOTAS` statement\n" + "with the following syntax\n" + "``` sql\n" + "SHOW QUOTAS\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.7.13.5') + num="5.7.13.5", +) RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Name = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Name', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Name", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the `CREATE QUOTA` statement used to create the quota with some given name\n' - 'using the `SHOW CREATE QUOTA` statement with the following syntax\n' - '\n' - '``` sql\n' - 'SHOW CREATE QUOTA name\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support showing the `CREATE QUOTA` statement used to create the quota with some given name\n" + "using the `SHOW CREATE QUOTA` statement with the following syntax\n" + "\n" + "``` sql\n" + "SHOW CREATE QUOTA name\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.7.14.1') + num="5.7.14.1", +) RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Current = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Current', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Current", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the `CREATE QUOTA` statement used to create the CURRENT quota\n' - 'using the `SHOW CREATE QUOTA CURRENT` statement or the shorthand form\n' - '`SHOW CREATE QUOTA`\n' - '\n' - ), + "[ClickHouse] SHALL support showing the `CREATE QUOTA` statement used to create the CURRENT quota\n" + "using the `SHOW CREATE QUOTA CURRENT` statement or the shorthand form\n" + "`SHOW CREATE QUOTA`\n" + "\n" + ), link=None, level=4, - num='5.7.14.2') + num="5.7.14.2", +) RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax when\n' - 'using the `SHOW CREATE QUOTA` statement.\n' - '\n' - '```sql\n' - 'SHOW CREATE QUOTA [name | CURRENT]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax when\n" + "using the `SHOW CREATE QUOTA` statement.\n" + "\n" + "```sql\n" + "SHOW CREATE QUOTA [name | CURRENT]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.7.14.3') + num="5.7.14.3", +) RQ_SRS_006_RBAC_RowPolicy = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creation and manipulation of table **row policies**\n' - 'that can be used to limit access to the table contents for a **user** or a **role**\n' - 'using a specified **condition**.\n' - '\n' - ), + "[ClickHouse] SHALL support creation and manipulation of table **row policies**\n" + "that can be used to limit access to the table contents for a **user** or a **role**\n" + "using a specified **condition**.\n" + "\n" + ), link=None, level=3, - num='5.8.1') + num="5.8.1", +) RQ_SRS_006_RBAC_RowPolicy_Condition = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Condition', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Condition", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support row policy **conditions** that can be any SQL\n' - 'expression that returns a boolean.\n' - '\n' - ), + "[ClickHouse] SHALL support row policy **conditions** that can be any SQL\n" + "expression that returns a boolean.\n" + "\n" + ), link=None, level=3, - num='5.8.2') + num="5.8.2", +) RQ_SRS_006_RBAC_RowPolicy_Restriction = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Restriction', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Restriction", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL restrict all access to a table when a row policy with a condition is created on that table.\n' - 'All users require a permissive row policy in order to view the table.\n' - '\n' - ), + "[ClickHouse] SHALL restrict all access to a table when a row policy with a condition is created on that table.\n" + "All users require a permissive row policy in order to view the table.\n" + "\n" + ), link=None, level=3, - num='5.8.3') + num="5.8.3", +) RQ_SRS_006_RBAC_RowPolicy_Nesting = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Nesting', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Nesting", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL restrict rows of tables or views created on top of a table with row policies according to those policies.\n' - '\n' - ), + "[ClickHouse] SHALL restrict rows of tables or views created on top of a table with row policies according to those policies.\n" + "\n" + ), link=None, level=3, - num='5.8.4') + num="5.8.4", +) RQ_SRS_006_RBAC_RowPolicy_Create = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support creating row policy using the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support creating row policy using the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.1') + num="5.8.5.1", +) RQ_SRS_006_RBAC_RowPolicy_Create_IfNotExists = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.IfNotExists', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.IfNotExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE ROW POLICY` statement\n' - 'to skip raising an exception if a row policy with the same **name** already exists.\n' - 'If the `IF NOT EXISTS` clause is not specified then an exception SHALL be raised if\n' - 'a row policy with the same **name** already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `IF NOT EXISTS` clause in the `CREATE ROW POLICY` statement\n" + "to skip raising an exception if a row policy with the same **name** already exists.\n" + "If the `IF NOT EXISTS` clause is not specified then an exception SHALL be raised if\n" + "a row policy with the same **name** already exists.\n" + "\n" + ), link=None, level=4, - num='5.8.5.2') + num="5.8.5.2", +) RQ_SRS_006_RBAC_RowPolicy_Create_Replace = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Replace', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Replace", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE ROW POLICY` statement\n' - 'to replace existing row policy if it already exists.\n' - '\n' - ), + "[ClickHouse] SHALL support `OR REPLACE` clause in the `CREATE ROW POLICY` statement\n" + "to replace existing row policy if it already exists.\n" + "\n" + ), link=None, level=4, - num='5.8.5.3') + num="5.8.5.3", +) RQ_SRS_006_RBAC_RowPolicy_Create_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying cluster on which to create the role policy\n' - 'using the `ON CLUSTER` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying cluster on which to create the role policy\n" + "using the `ON CLUSTER` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.4') + num="5.8.5.4", +) RQ_SRS_006_RBAC_RowPolicy_Create_On = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.On', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying table on which to create the role policy\n' - 'using the `ON` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying table on which to create the role policy\n" + "using the `ON` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.5') + num="5.8.5.5", +) RQ_SRS_006_RBAC_RowPolicy_Create_Access = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Access', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Access", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support allowing or restricting access to rows using the\n' - '`AS` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support allowing or restricting access to rows using the\n" + "`AS` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.6') + num="5.8.5.6", +) RQ_SRS_006_RBAC_RowPolicy_Create_Access_Permissive = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Access.Permissive', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Access.Permissive", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support allowing access to rows using the\n' - '`AS PERMISSIVE` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support allowing access to rows using the\n" + "`AS PERMISSIVE` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.7') + num="5.8.5.7", +) RQ_SRS_006_RBAC_RowPolicy_Create_Access_Restrictive = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Access.Restrictive', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Access.Restrictive", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support restricting access to rows using the\n' - '`AS RESTRICTIVE` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support restricting access to rows using the\n" + "`AS RESTRICTIVE` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.8') + num="5.8.5.8", +) RQ_SRS_006_RBAC_RowPolicy_Create_ForSelect = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.ForSelect', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.ForSelect", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying which rows are affected\n' - 'using the `FOR SELECT` clause in the `CREATE ROW POLICY` statement.\n' - 'REQUIRES CONDITION.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying which rows are affected\n" + "using the `FOR SELECT` clause in the `CREATE ROW POLICY` statement.\n" + "REQUIRES CONDITION.\n" + "\n" + ), link=None, level=4, - num='5.8.5.9') + num="5.8.5.9", +) RQ_SRS_006_RBAC_RowPolicy_Create_Condition = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Condition', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Condition", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying a condition that\n' - 'that can be any SQL expression which returns a boolean using the `USING`\n' - 'clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying a condition that\n" + "that can be any SQL expression which returns a boolean using the `USING`\n" + "clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.10') + num="5.8.5.10", +) RQ_SRS_006_RBAC_RowPolicy_Create_Assignment = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning row policy to one or more users\n' - 'or roles using the `TO` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning row policy to one or more users\n" + "or roles using the `TO` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.11') + num="5.8.5.11", +) RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_None = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.None', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning row policy to no users or roles using\n' - 'the `TO NONE` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning row policy to no users or roles using\n" + "the `TO NONE` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.12') + num="5.8.5.12", +) RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_All = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.All', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support assigning row policy to all current users and roles\n' - 'using `TO ALL` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support assigning row policy to all current users and roles\n" + "using `TO ALL` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.13') + num="5.8.5.13", +) RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support excluding assignment of row policy to one or more users or roles using\n' - 'the `ALL EXCEPT` clause in the `CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support excluding assignment of row policy to one or more users or roles using\n" + "the `ALL EXCEPT` clause in the `CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.5.14') + num="5.8.5.14", +) RQ_SRS_006_RBAC_RowPolicy_Create_Syntax = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Create.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Create.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `CRETE ROW POLICY` statement\n' - '\n' - '``` sql\n' - 'CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table\n' - ' [AS {PERMISSIVE | RESTRICTIVE}]\n' - ' [FOR SELECT]\n' - ' [USING condition]\n' - ' [TO {role [,...] | ALL | ALL EXCEPT role [,...]}]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `CRETE ROW POLICY` statement\n" + "\n" + "``` sql\n" + "CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table\n" + " [AS {PERMISSIVE | RESTRICTIVE}]\n" + " [FOR SELECT]\n" + " [USING condition]\n" + " [TO {role [,...] | ALL | ALL EXCEPT role [,...]}]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.8.5.15') + num="5.8.5.15", +) RQ_SRS_006_RBAC_RowPolicy_Alter = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering row policy using the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering row policy using the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.1') + num="5.8.6.1", +) RQ_SRS_006_RBAC_RowPolicy_Alter_IfExists = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `IF EXISTS` clause in the `ALTER ROW POLICY` statement\n' - 'to skip raising an exception if a row policy does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be raised if\n' - 'a row policy does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support the `IF EXISTS` clause in the `ALTER ROW POLICY` statement\n" + "to skip raising an exception if a row policy does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be raised if\n" + "a row policy does not exist.\n" + "\n" + ), link=None, level=4, - num='5.8.6.2') + num="5.8.6.2", +) RQ_SRS_006_RBAC_RowPolicy_Alter_ForSelect = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.ForSelect', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.ForSelect", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support modifying rows on which to apply the row policy\n' - 'using the `FOR SELECT` clause in the `ALTER ROW POLICY` statement.\n' - 'REQUIRES FUNCTION CONFIRMATION.\n' - '\n' - ), + "[ClickHouse] SHALL support modifying rows on which to apply the row policy\n" + "using the `FOR SELECT` clause in the `ALTER ROW POLICY` statement.\n" + "REQUIRES FUNCTION CONFIRMATION.\n" + "\n" + ), link=None, level=4, - num='5.8.6.3') + num="5.8.6.3", +) RQ_SRS_006_RBAC_RowPolicy_Alter_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying cluster on which to alter the row policy\n' - 'using the `ON CLUSTER` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying cluster on which to alter the row policy\n" + "using the `ON CLUSTER` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.4') + num="5.8.6.4", +) RQ_SRS_006_RBAC_RowPolicy_Alter_On = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.On', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying table on which to alter the row policy\n' - 'using the `ON` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying table on which to alter the row policy\n" + "using the `ON` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.5') + num="5.8.6.5", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Rename = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Rename', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Rename", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support renaming the row policy using the `RENAME` clause\n' - 'in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support renaming the row policy using the `RENAME` clause\n" + "in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.6') + num="5.8.6.6", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Access = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Access', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Access", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support altering access to rows using the\n' - '`AS` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support altering access to rows using the\n" + "`AS` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.7') + num="5.8.6.7", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Permissive = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Permissive', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Permissive", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support permitting access to rows using the\n' - '`AS PERMISSIVE` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support permitting access to rows using the\n" + "`AS PERMISSIVE` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.8') + num="5.8.6.8", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Restrictive = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Restrictive', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Restrictive", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support restricting access to rows using the\n' - '`AS RESTRICTIVE` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support restricting access to rows using the\n" + "`AS RESTRICTIVE` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.9') + num="5.8.6.9", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Condition = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Condition', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Condition", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support re-specifying the row policy condition\n' - 'using the `USING` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support re-specifying the row policy condition\n" + "using the `USING` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.10') + num="5.8.6.10", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Condition_None = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Condition.None', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Condition.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing the row policy condition\n' - 'using the `USING NONE` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing the row policy condition\n" + "using the `USING NONE` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.11') + num="5.8.6.11", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning row policy to one or more users\n' - 'or roles using the `TO` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning row policy to one or more users\n" + "or roles using the `TO` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.12') + num="5.8.6.12", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_None = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.None', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning row policy to no users or roles using\n' - 'the `TO NONE` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning row policy to no users or roles using\n" + "the `TO NONE` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.13') + num="5.8.6.13", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_All = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.All', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support reassigning row policy to all current users and roles\n' - 'using the `TO ALL` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support reassigning row policy to all current users and roles\n" + "using the `TO ALL` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.14') + num="5.8.6.14", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support excluding assignment of row policy to one or more users or roles using\n' - 'the `ALL EXCEPT` clause in the `ALTER ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support excluding assignment of row policy to one or more users or roles using\n" + "the `ALL EXCEPT` clause in the `ALTER ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.6.15') + num="5.8.6.15", +) RQ_SRS_006_RBAC_RowPolicy_Alter_Syntax = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Alter.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `ALTER ROW POLICY` statement\n' - '\n' - '``` sql\n' - 'ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table\n' - ' [RENAME TO new_name]\n' - ' [AS {PERMISSIVE | RESTRICTIVE}]\n' - ' [FOR SELECT]\n' - ' [USING {condition | NONE}][,...]\n' - ' [TO {role [,...] | ALL | ALL EXCEPT role [,...]}]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `ALTER ROW POLICY` statement\n" + "\n" + "``` sql\n" + "ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table\n" + " [RENAME TO new_name]\n" + " [AS {PERMISSIVE | RESTRICTIVE}]\n" + " [FOR SELECT]\n" + " [USING {condition | NONE}][,...]\n" + " [TO {role [,...] | ALL | ALL EXCEPT role [,...]}]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.8.6.16') + num="5.8.6.16", +) RQ_SRS_006_RBAC_RowPolicy_Drop = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing one or more row policies using the `DROP ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing one or more row policies using the `DROP ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.7.1') + num="5.8.7.1", +) RQ_SRS_006_RBAC_RowPolicy_Drop_IfExists = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Drop.IfExists', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Drop.IfExists", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using the `IF EXISTS` clause in the `DROP ROW POLICY` statement\n' - 'to skip raising an exception when the row policy does not exist.\n' - 'If the `IF EXISTS` clause is not specified then an exception SHALL be\n' - 'raised if the row policy does not exist.\n' - '\n' - ), + "[ClickHouse] SHALL support using the `IF EXISTS` clause in the `DROP ROW POLICY` statement\n" + "to skip raising an exception when the row policy does not exist.\n" + "If the `IF EXISTS` clause is not specified then an exception SHALL be\n" + "raised if the row policy does not exist.\n" + "\n" + ), link=None, level=4, - num='5.8.7.2') + num="5.8.7.2", +) RQ_SRS_006_RBAC_RowPolicy_Drop_On = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Drop.On', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Drop.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing row policy from one or more specified tables\n' - 'using the `ON` clause in the `DROP ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing row policy from one or more specified tables\n" + "using the `ON` clause in the `DROP ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.7.3') + num="5.8.7.3", +) RQ_SRS_006_RBAC_RowPolicy_Drop_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Drop.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Drop.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing row policy from specified cluster\n' - 'using the `ON CLUSTER` clause in the `DROP ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing row policy from specified cluster\n" + "using the `ON CLUSTER` clause in the `DROP ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.7.4') + num="5.8.7.4", +) RQ_SRS_006_RBAC_RowPolicy_Drop_Syntax = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.Drop.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.Drop.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `DROP ROW POLICY` statement.\n' - '\n' - '``` sql\n' - 'DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `DROP ROW POLICY` statement.\n" + "\n" + "``` sql\n" + "DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.8.7.5') + num="5.8.7.5", +) RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing the `CREATE ROW POLICY` statement used to create the row policy\n' - 'using the `SHOW CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support showing the `CREATE ROW POLICY` statement used to create the row policy\n" + "using the `SHOW CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.8.1') + num="5.8.8.1", +) RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy_On = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.On', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing statement used to create row policy on specific table\n' - 'using the `ON` in the `SHOW CREATE ROW POLICY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support showing statement used to create row policy on specific table\n" + "using the `ON` in the `SHOW CREATE ROW POLICY` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.8.2') + num="5.8.8.2", +) RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy_Syntax = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for `SHOW CREATE ROW POLICY`.\n' - '\n' - '``` sql\n' - 'SHOW CREATE [ROW] POLICY name ON [database.]table\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for `SHOW CREATE ROW POLICY`.\n" + "\n" + "``` sql\n" + "SHOW CREATE [ROW] POLICY name ON [database.]table\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.8.8.3') + num="5.8.8.3", +) RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing row policies using the `SHOW ROW POLICIES` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support showing row policies using the `SHOW ROW POLICIES` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.8.4') + num="5.8.8.4", +) RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies_On = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.On', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support showing row policies on a specific table\n' - 'using the `ON` clause in the `SHOW ROW POLICIES` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support showing row policies on a specific table\n" + "using the `ON` clause in the `SHOW ROW POLICIES` statement.\n" + "\n" + ), link=None, level=4, - num='5.8.8.5') + num="5.8.8.5", +) RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies_Syntax = Requirement( - name='RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for `SHOW ROW POLICIES`.\n' - '\n' - '```sql\n' - 'SHOW [ROW] POLICIES [ON [database.]table]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for `SHOW ROW POLICIES`.\n" + "\n" + "```sql\n" + "SHOW [ROW] POLICIES [ON [database.]table]\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.8.8.6') + num="5.8.8.6", +) RQ_SRS_006_RBAC_SetDefaultRole = Requirement( - name='RQ.SRS-006.RBAC.SetDefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.SetDefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting or changing granted roles to default for one or more\n' - 'users using `SET DEFAULT ROLE` statement which\n' - 'SHALL permanently change the default roles for the user or users if successful.\n' - '\n' - ), + "[ClickHouse] SHALL support setting or changing granted roles to default for one or more\n" + "users using `SET DEFAULT ROLE` statement which\n" + "SHALL permanently change the default roles for the user or users if successful.\n" + "\n" + ), link=None, level=3, - num='5.9.1') + num="5.9.1", +) RQ_SRS_006_RBAC_SetDefaultRole_CurrentUser = Requirement( - name='RQ.SRS-006.RBAC.SetDefaultRole.CurrentUser', - version='1.0', + name="RQ.SRS-006.RBAC.SetDefaultRole.CurrentUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting or changing granted roles to default for\n' - 'the current user using `CURRENT_USER` clause in the `SET DEFAULT ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support setting or changing granted roles to default for\n" + "the current user using `CURRENT_USER` clause in the `SET DEFAULT ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.9.2') + num="5.9.2", +) RQ_SRS_006_RBAC_SetDefaultRole_All = Requirement( - name='RQ.SRS-006.RBAC.SetDefaultRole.All', - version='1.0', + name="RQ.SRS-006.RBAC.SetDefaultRole.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting or changing all granted roles to default\n' - 'for one or more users using `ALL` clause in the `SET DEFAULT ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support setting or changing all granted roles to default\n" + "for one or more users using `ALL` clause in the `SET DEFAULT ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.9.3') + num="5.9.3", +) RQ_SRS_006_RBAC_SetDefaultRole_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.SetDefaultRole.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.SetDefaultRole.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support setting or changing all granted roles except those specified\n' - 'to default for one or more users using `ALL EXCEPT` clause in the `SET DEFAULT ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support setting or changing all granted roles except those specified\n" + "to default for one or more users using `ALL EXCEPT` clause in the `SET DEFAULT ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.9.4') + num="5.9.4", +) RQ_SRS_006_RBAC_SetDefaultRole_None = Requirement( - name='RQ.SRS-006.RBAC.SetDefaultRole.None', - version='1.0', + name="RQ.SRS-006.RBAC.SetDefaultRole.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support removing all granted roles from default\n' - 'for one or more users using `NONE` clause in the `SET DEFAULT ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support removing all granted roles from default\n" + "for one or more users using `NONE` clause in the `SET DEFAULT ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.9.5') + num="5.9.5", +) RQ_SRS_006_RBAC_SetDefaultRole_Syntax = Requirement( - name='RQ.SRS-006.RBAC.SetDefaultRole.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.SetDefaultRole.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `SET DEFAULT ROLE` statement.\n' - '\n' - '```sql\n' - 'SET DEFAULT ROLE\n' - ' {NONE | role [,...] | ALL | ALL EXCEPT role [,...]}\n' - ' TO {user|CURRENT_USER} [,...]\n' - '\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `SET DEFAULT ROLE` statement.\n" + "\n" + "```sql\n" + "SET DEFAULT ROLE\n" + " {NONE | role [,...] | ALL | ALL EXCEPT role [,...]}\n" + " TO {user|CURRENT_USER} [,...]\n" + "\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.9.6') + num="5.9.6", +) RQ_SRS_006_RBAC_SetRole = Requirement( - name='RQ.SRS-006.RBAC.SetRole', - version='1.0', + name="RQ.SRS-006.RBAC.SetRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support activating role or roles for the current user\n' - 'using `SET ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support activating role or roles for the current user\n" + "using `SET ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.10.1') + num="5.10.1", +) RQ_SRS_006_RBAC_SetRole_Default = Requirement( - name='RQ.SRS-006.RBAC.SetRole.Default', - version='1.0', + name="RQ.SRS-006.RBAC.SetRole.Default", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support activating default roles for the current user\n' - 'using `DEFAULT` clause in the `SET ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support activating default roles for the current user\n" + "using `DEFAULT` clause in the `SET ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.10.2') + num="5.10.2", +) RQ_SRS_006_RBAC_SetRole_None = Requirement( - name='RQ.SRS-006.RBAC.SetRole.None', - version='1.0', + name="RQ.SRS-006.RBAC.SetRole.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support activating no roles for the current user\n' - 'using `NONE` clause in the `SET ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support activating no roles for the current user\n" + "using `NONE` clause in the `SET ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.10.3') + num="5.10.3", +) RQ_SRS_006_RBAC_SetRole_All = Requirement( - name='RQ.SRS-006.RBAC.SetRole.All', - version='1.0', + name="RQ.SRS-006.RBAC.SetRole.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support activating all roles for the current user\n' - 'using `ALL` clause in the `SET ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support activating all roles for the current user\n" + "using `ALL` clause in the `SET ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.10.4') + num="5.10.4", +) RQ_SRS_006_RBAC_SetRole_AllExcept = Requirement( - name='RQ.SRS-006.RBAC.SetRole.AllExcept', - version='1.0', + name="RQ.SRS-006.RBAC.SetRole.AllExcept", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support activating all roles except those specified\n' - 'for the current user using `ALL EXCEPT` clause in the `SET ROLE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support activating all roles except those specified\n" + "for the current user using `ALL EXCEPT` clause in the `SET ROLE` statement.\n" + "\n" + ), link=None, level=3, - num='5.10.5') + num="5.10.5", +) RQ_SRS_006_RBAC_SetRole_Syntax = Requirement( - name='RQ.SRS-006.RBAC.SetRole.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.SetRole.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '```sql\n' - 'SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]}\n' - '```\n' - '\n' - ), + "```sql\n" + "SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]}\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.10.6') + num="5.10.6", +) RQ_SRS_006_RBAC_Grant_Privilege_To = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.To', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.To", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting privileges to one or more users or roles using `TO` clause\n' - 'in the `GRANT PRIVILEGE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting privileges to one or more users or roles using `TO` clause\n" + "in the `GRANT PRIVILEGE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.1') + num="5.11.1", +) RQ_SRS_006_RBAC_Grant_Privilege_ToCurrentUser = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.ToCurrentUser', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.ToCurrentUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting privileges to current user using `TO CURRENT_USER` clause\n' - 'in the `GRANT PRIVILEGE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting privileges to current user using `TO CURRENT_USER` clause\n" + "in the `GRANT PRIVILEGE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.2') + num="5.11.2", +) RQ_SRS_006_RBAC_Grant_Privilege_Select = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Select', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **select** privilege to one or more users or roles\n' - 'for a database or a table using the `GRANT SELECT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **select** privilege to one or more users or roles\n" + "for a database or a table using the `GRANT SELECT` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.3') + num="5.11.3", +) RQ_SRS_006_RBAC_Grant_Privilege_Insert = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Insert', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Insert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **insert** privilege to one or more users or roles\n' - 'for a database or a table using the `GRANT INSERT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **insert** privilege to one or more users or roles\n" + "for a database or a table using the `GRANT INSERT` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.4') + num="5.11.4", +) RQ_SRS_006_RBAC_Grant_Privilege_Alter = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **alter** privilege to one or more users or roles\n' - 'for a database or a table using the `GRANT ALTER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **alter** privilege to one or more users or roles\n" + "for a database or a table using the `GRANT ALTER` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.5') + num="5.11.5", +) RQ_SRS_006_RBAC_Grant_Privilege_Create = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Create', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **create** privilege to one or more users or roles\n' - 'using the `GRANT CREATE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **create** privilege to one or more users or roles\n" + "using the `GRANT CREATE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.6') + num="5.11.6", +) RQ_SRS_006_RBAC_Grant_Privilege_Drop = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **drop** privilege to one or more users or roles\n' - 'using the `GRANT DROP` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **drop** privilege to one or more users or roles\n" + "using the `GRANT DROP` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.7') + num="5.11.7", +) RQ_SRS_006_RBAC_Grant_Privilege_Truncate = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Truncate', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Truncate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **truncate** privilege to one or more users or roles\n' - 'for a database or a table using `GRANT TRUNCATE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **truncate** privilege to one or more users or roles\n" + "for a database or a table using `GRANT TRUNCATE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.8') + num="5.11.8", +) RQ_SRS_006_RBAC_Grant_Privilege_Optimize = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Optimize', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Optimize", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **optimize** privilege to one or more users or roles\n' - 'for a database or a table using `GRANT OPTIMIZE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **optimize** privilege to one or more users or roles\n" + "for a database or a table using `GRANT OPTIMIZE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.9') + num="5.11.9", +) RQ_SRS_006_RBAC_Grant_Privilege_Show = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Show', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Show", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **show** privilege to one or more users or roles\n' - 'for a database or a table using `GRANT SHOW` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **show** privilege to one or more users or roles\n" + "for a database or a table using `GRANT SHOW` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.10') + num="5.11.10", +) RQ_SRS_006_RBAC_Grant_Privilege_KillQuery = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.KillQuery', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.KillQuery", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **kill query** privilege to one or more users or roles\n' - 'for a database or a table using `GRANT KILL QUERY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **kill query** privilege to one or more users or roles\n" + "for a database or a table using `GRANT KILL QUERY` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.11') + num="5.11.11", +) RQ_SRS_006_RBAC_Grant_Privilege_AccessManagement = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.AccessManagement', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.AccessManagement", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **access management** privileges to one or more users or roles\n' - 'for a database or a table using `GRANT ACCESS MANAGEMENT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **access management** privileges to one or more users or roles\n" + "for a database or a table using `GRANT ACCESS MANAGEMENT` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.12') + num="5.11.12", +) RQ_SRS_006_RBAC_Grant_Privilege_System = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.System', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.System", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **system** privileges to one or more users or roles\n' - 'for a database or a table using `GRANT SYSTEM` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **system** privileges to one or more users or roles\n" + "for a database or a table using `GRANT SYSTEM` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.13') + num="5.11.13", +) RQ_SRS_006_RBAC_Grant_Privilege_Introspection = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Introspection', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Introspection", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **introspection** privileges to one or more users or roles\n' - 'for a database or a table using `GRANT INTROSPECTION` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **introspection** privileges to one or more users or roles\n" + "for a database or a table using `GRANT INTROSPECTION` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.14') + num="5.11.14", +) RQ_SRS_006_RBAC_Grant_Privilege_Sources = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Sources', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Sources", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **sources** privileges to one or more users or roles\n' - 'for a database or a table using `GRANT SOURCES` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **sources** privileges to one or more users or roles\n" + "for a database or a table using `GRANT SOURCES` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.15') + num="5.11.15", +) RQ_SRS_006_RBAC_Grant_Privilege_DictGet = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.DictGet', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.DictGet", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **dictGet** privilege to one or more users or roles\n' - 'for a database or a table using `GRANT dictGet` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **dictGet** privilege to one or more users or roles\n" + "for a database or a table using `GRANT dictGet` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.16') + num="5.11.16", +) RQ_SRS_006_RBAC_Grant_Privilege_None = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.None', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting no privileges to one or more users or roles\n' - 'for a database or a table using `GRANT NONE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting no privileges to one or more users or roles\n" + "for a database or a table using `GRANT NONE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.17') + num="5.11.17", +) RQ_SRS_006_RBAC_Grant_Privilege_All = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.All', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **all** privileges to one or more users or roles\n' - 'using the `GRANT ALL` or `GRANT ALL PRIVILEGES` statements.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **all** privileges to one or more users or roles\n" + "using the `GRANT ALL` or `GRANT ALL PRIVILEGES` statements.\n" + "\n" + ), link=None, level=3, - num='5.11.18') + num="5.11.18", +) RQ_SRS_006_RBAC_Grant_Privilege_GrantOption = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.GrantOption', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.GrantOption", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the **grant option** privilege to one or more users or roles\n' - 'for a database or a table using the `WITH GRANT OPTION` clause in the `GRANT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the **grant option** privilege to one or more users or roles\n" + "for a database or a table using the `WITH GRANT OPTION` clause in the `GRANT` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.19') + num="5.11.19", +) RQ_SRS_006_RBAC_Grant_Privilege_On = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.On', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `ON` clause in the `GRANT` privilege statement\n' - 'which SHALL allow to specify one or more tables to which the privilege SHALL\n' - 'be granted using the following patterns\n' - '\n' - '* `*.*` any table in any database\n' - '* `database.*` any table in the specified database\n' - '* `database.table` specific table in the specified database\n' - '* `*` any table in the current database\n' - '* `table` specific table in the current database\n' - '\n' - ), + "[ClickHouse] SHALL support the `ON` clause in the `GRANT` privilege statement\n" + "which SHALL allow to specify one or more tables to which the privilege SHALL\n" + "be granted using the following patterns\n" + "\n" + "* `*.*` any table in any database\n" + "* `database.*` any table in the specified database\n" + "* `database.table` specific table in the specified database\n" + "* `*` any table in the current database\n" + "* `table` specific table in the current database\n" + "\n" + ), link=None, level=3, - num='5.11.20') + num="5.11.20", +) RQ_SRS_006_RBAC_Grant_Privilege_PrivilegeColumns = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.PrivilegeColumns', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.PrivilegeColumns", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting the privilege **some_privilege** to one or more users or roles\n' - 'for a database or a table using the `GRANT some_privilege(column)` statement for one column.\n' - 'Multiple columns will be supported with `GRANT some_privilege(column1, column2...)` statement.\n' - 'The privileges will be granted for only the specified columns.\n' - '\n' - ), + "[ClickHouse] SHALL support granting the privilege **some_privilege** to one or more users or roles\n" + "for a database or a table using the `GRANT some_privilege(column)` statement for one column.\n" + "Multiple columns will be supported with `GRANT some_privilege(column1, column2...)` statement.\n" + "The privileges will be granted for only the specified columns.\n" + "\n" + ), link=None, level=3, - num='5.11.21') + num="5.11.21", +) RQ_SRS_006_RBAC_Grant_Privilege_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying cluster on which to grant privileges using the `ON CLUSTER`\n' - 'clause in the `GRANT PRIVILEGE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying cluster on which to grant privileges using the `ON CLUSTER`\n" + "clause in the `GRANT PRIVILEGE` statement.\n" + "\n" + ), link=None, level=3, - num='5.11.22') + num="5.11.22", +) RQ_SRS_006_RBAC_Grant_Privilege_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Grant.Privilege.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Privilege.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `GRANT` statement that\n' - 'grants explicit privileges to a user or a role.\n' - '\n' - '```sql\n' - 'GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...]\n' - ' ON {db.table|db.*|*.*|table|*}\n' - ' TO {user | role | CURRENT_USER} [,...]\n' - ' [WITH GRANT OPTION]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `GRANT` statement that\n" + "grants explicit privileges to a user or a role.\n" + "\n" + "```sql\n" + "GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...]\n" + " ON {db.table|db.*|*.*|table|*}\n" + " TO {user | role | CURRENT_USER} [,...]\n" + " [WITH GRANT OPTION]\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.11.23') + num="5.11.23", +) RQ_SRS_006_RBAC_Revoke_Privilege_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking privileges to one or more users or roles\n' - 'for a database or a table on some specific cluster using the `REVOKE ON CLUSTER cluster_name` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking privileges to one or more users or roles\n" + "for a database or a table on some specific cluster using the `REVOKE ON CLUSTER cluster_name` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.1') + num="5.12.1", +) RQ_SRS_006_RBAC_Revoke_Privilege_Select = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Select', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **select** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE SELECT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **select** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE SELECT` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.2') + num="5.12.2", +) RQ_SRS_006_RBAC_Revoke_Privilege_Insert = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Insert', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Insert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **insert** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE INSERT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **insert** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE INSERT` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.3') + num="5.12.3", +) RQ_SRS_006_RBAC_Revoke_Privilege_Alter = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Alter', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Alter", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **alter** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE ALTER` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **alter** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE ALTER` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.4') + num="5.12.4", +) RQ_SRS_006_RBAC_Revoke_Privilege_Create = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Create', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **create** privilege to one or more users or roles\n' - 'using the `REVOKE CREATE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **create** privilege to one or more users or roles\n" + "using the `REVOKE CREATE` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.5') + num="5.12.5", +) RQ_SRS_006_RBAC_Revoke_Privilege_Drop = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **drop** privilege to one or more users or roles\n' - 'using the `REVOKE DROP` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **drop** privilege to one or more users or roles\n" + "using the `REVOKE DROP` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.6') + num="5.12.6", +) RQ_SRS_006_RBAC_Revoke_Privilege_Truncate = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Truncate', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Truncate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **truncate** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE TRUNCATE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **truncate** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE TRUNCATE` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.7') + num="5.12.7", +) RQ_SRS_006_RBAC_Revoke_Privilege_Optimize = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Optimize', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Optimize", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **optimize** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE OPTIMIZE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **optimize** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE OPTIMIZE` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.8') + num="5.12.8", +) RQ_SRS_006_RBAC_Revoke_Privilege_Show = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Show', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Show", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **show** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE SHOW` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **show** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE SHOW` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.9') + num="5.12.9", +) RQ_SRS_006_RBAC_Revoke_Privilege_KillQuery = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.KillQuery', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.KillQuery", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **kill query** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE KILL QUERY` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **kill query** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE KILL QUERY` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.10') + num="5.12.10", +) RQ_SRS_006_RBAC_Revoke_Privilege_AccessManagement = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.AccessManagement', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.AccessManagement", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **access management** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE ACCESS MANAGEMENT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **access management** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE ACCESS MANAGEMENT` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.11') + num="5.12.11", +) RQ_SRS_006_RBAC_Revoke_Privilege_System = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.System', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.System", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **system** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE SYSTEM` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **system** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE SYSTEM` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.12') + num="5.12.12", +) RQ_SRS_006_RBAC_Revoke_Privilege_Introspection = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Introspection', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Introspection", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **introspection** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE INTROSPECTION` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **introspection** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE INTROSPECTION` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.13') + num="5.12.13", +) RQ_SRS_006_RBAC_Revoke_Privilege_Sources = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Sources', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Sources", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **sources** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE SOURCES` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **sources** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE SOURCES` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.14') + num="5.12.14", +) RQ_SRS_006_RBAC_Revoke_Privilege_DictGet = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.DictGet', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.DictGet", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the **dictGet** privilege to one or more users or roles\n' - 'for a database or a table using the `REVOKE dictGet` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the **dictGet** privilege to one or more users or roles\n" + "for a database or a table using the `REVOKE dictGet` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.15') + num="5.12.15", +) RQ_SRS_006_RBAC_Revoke_Privilege_PrivilegeColumns = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.PrivilegeColumns', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.PrivilegeColumns", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking the privilege **some_privilege** to one or more users or roles\n' - 'for a database or a table using the `REVOKE some_privilege(column)` statement for one column.\n' - 'Multiple columns will be supported with `REVOKE some_privilege(column1, column2...)` statement.\n' - 'The privileges will be revoked for only the specified columns.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking the privilege **some_privilege** to one or more users or roles\n" + "for a database or a table using the `REVOKE some_privilege(column)` statement for one column.\n" + "Multiple columns will be supported with `REVOKE some_privilege(column1, column2...)` statement.\n" + "The privileges will be revoked for only the specified columns.\n" + "\n" + ), link=None, level=3, - num='5.12.16') + num="5.12.16", +) RQ_SRS_006_RBAC_Revoke_Privilege_Multiple = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Multiple', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Multiple", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking MULTIPLE **privileges** to one or more users or roles\n' - 'for a database or a table using the `REVOKE privilege1, privilege2...` statement.\n' - '**privileges** refers to any set of Clickhouse defined privilege, whose hierarchy includes\n' - 'SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT,\n' - 'SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking MULTIPLE **privileges** to one or more users or roles\n" + "for a database or a table using the `REVOKE privilege1, privilege2...` statement.\n" + "**privileges** refers to any set of Clickhouse defined privilege, whose hierarchy includes\n" + "SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, SHOW, KILL QUERY, ACCESS MANAGEMENT,\n" + "SYSTEM, INTROSPECTION, SOURCES, dictGet and all of their sub-privileges.\n" + "\n" + ), link=None, level=3, - num='5.12.17') + num="5.12.17", +) RQ_SRS_006_RBAC_Revoke_Privilege_All = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.All', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **all** privileges to one or more users or roles\n' - 'for a database or a table using the `REVOKE ALL` or `REVOKE ALL PRIVILEGES` statements.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **all** privileges to one or more users or roles\n" + "for a database or a table using the `REVOKE ALL` or `REVOKE ALL PRIVILEGES` statements.\n" + "\n" + ), link=None, level=3, - num='5.12.18') + num="5.12.18", +) RQ_SRS_006_RBAC_Revoke_Privilege_None = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.None', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **no** privileges to one or more users or roles\n' - 'for a database or a table using the `REVOKE NONE` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **no** privileges to one or more users or roles\n" + "for a database or a table using the `REVOKE NONE` statement.\n" + "\n" + ), link=None, level=3, - num='5.12.19') + num="5.12.19", +) RQ_SRS_006_RBAC_Revoke_Privilege_On = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.On', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.On", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `ON` clause in the `REVOKE` privilege statement\n' - 'which SHALL allow to specify one or more tables to which the privilege SHALL\n' - 'be revoked using the following patterns\n' - '\n' - '* `db.table` specific table in the specified database\n' - '* `db.*` any table in the specified database\n' - '* `*.*` any table in any database\n' - '* `table` specific table in the current database\n' - '* `*` any table in the current database\n' - '\n' - ), + "[ClickHouse] SHALL support the `ON` clause in the `REVOKE` privilege statement\n" + "which SHALL allow to specify one or more tables to which the privilege SHALL\n" + "be revoked using the following patterns\n" + "\n" + "* `db.table` specific table in the specified database\n" + "* `db.*` any table in the specified database\n" + "* `*.*` any table in any database\n" + "* `table` specific table in the current database\n" + "* `*` any table in the current database\n" + "\n" + ), link=None, level=3, - num='5.12.20') + num="5.12.20", +) RQ_SRS_006_RBAC_Revoke_Privilege_From = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.From', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.From", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `FROM` clause in the `REVOKE` privilege statement\n' - 'which SHALL allow to specify one or more users to which the privilege SHALL\n' - 'be revoked using the following patterns\n' - '\n' - '* `{user | CURRENT_USER} [,...]` some combination of users by name, which may include the current user\n' - '* `ALL` all users\n' - '* `ALL EXCEPT {user | CURRENT_USER} [,...]` the logical reverse of the first pattern\n' - '\n' - ), + "[ClickHouse] SHALL support the `FROM` clause in the `REVOKE` privilege statement\n" + "which SHALL allow to specify one or more users to which the privilege SHALL\n" + "be revoked using the following patterns\n" + "\n" + "* `{user | CURRENT_USER} [,...]` some combination of users by name, which may include the current user\n" + "* `ALL` all users\n" + "* `ALL EXCEPT {user | CURRENT_USER} [,...]` the logical reverse of the first pattern\n" + "\n" + ), link=None, level=3, - num='5.12.21') + num="5.12.21", +) RQ_SRS_006_RBAC_Revoke_Privilege_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Privilege.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Privilege.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `REVOKE` statement that\n' - 'revokes explicit privileges of a user or a role.\n' - '\n' - '```sql\n' - 'REVOKE [ON CLUSTER cluster_name] privilege\n' - ' [(column_name [,...])] [,...]\n' - ' ON {db.table|db.*|*.*|table|*}\n' - ' FROM {user | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user | CURRENT_USER} [,...]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `REVOKE` statement that\n" + "revokes explicit privileges of a user or a role.\n" + "\n" + "```sql\n" + "REVOKE [ON CLUSTER cluster_name] privilege\n" + " [(column_name [,...])] [,...]\n" + " ON {db.table|db.*|*.*|table|*}\n" + " FROM {user | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user | CURRENT_USER} [,...]\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.12.22') + num="5.12.22", +) RQ_SRS_006_RBAC_Grant_Role = Requirement( - name='RQ.SRS-006.RBAC.Grant.Role', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Role", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting one or more roles to\n' - 'one or more users or roles using the `GRANT` role statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting one or more roles to\n" + "one or more users or roles using the `GRANT` role statement.\n" + "\n" + ), link=None, level=3, - num='5.13.1') + num="5.13.1", +) RQ_SRS_006_RBAC_Grant_Role_CurrentUser = Requirement( - name='RQ.SRS-006.RBAC.Grant.Role.CurrentUser', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Role.CurrentUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting one or more roles to current user using\n' - '`TO CURRENT_USER` clause in the `GRANT` role statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting one or more roles to current user using\n" + "`TO CURRENT_USER` clause in the `GRANT` role statement.\n" + "\n" + ), link=None, level=3, - num='5.13.2') + num="5.13.2", +) RQ_SRS_006_RBAC_Grant_Role_AdminOption = Requirement( - name='RQ.SRS-006.RBAC.Grant.Role.AdminOption', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Role.AdminOption", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting `admin option` privilege\n' - 'to one or more users or roles using the `WITH ADMIN OPTION` clause\n' - 'in the `GRANT` role statement.\n' - '\n' - ), + "[ClickHouse] SHALL support granting `admin option` privilege\n" + "to one or more users or roles using the `WITH ADMIN OPTION` clause\n" + "in the `GRANT` role statement.\n" + "\n" + ), link=None, level=3, - num='5.13.3') + num="5.13.3", +) RQ_SRS_006_RBAC_Grant_Role_OnCluster = Requirement( - name='RQ.SRS-006.RBAC.Grant.Role.OnCluster', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Role.OnCluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support specifying cluster on which the user is to be granted one or more roles\n' - 'using `ON CLUSTER` clause in the `GRANT` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support specifying cluster on which the user is to be granted one or more roles\n" + "using `ON CLUSTER` clause in the `GRANT` statement.\n" + "\n" + ), link=None, level=3, - num='5.13.4') + num="5.13.4", +) RQ_SRS_006_RBAC_Grant_Role_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Grant.Role.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Grant.Role.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for `GRANT` role statement\n' - '\n' - '``` sql\n' - 'GRANT\n' - ' ON CLUSTER cluster_name\n' - ' role [, role ...]\n' - ' TO {user | role | CURRENT_USER} [,...]\n' - ' [WITH ADMIN OPTION]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for `GRANT` role statement\n" + "\n" + "``` sql\n" + "GRANT\n" + " ON CLUSTER cluster_name\n" + " role [, role ...]\n" + " TO {user | role | CURRENT_USER} [,...]\n" + " [WITH ADMIN OPTION]\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.13.5') + num="5.13.5", +) RQ_SRS_006_RBAC_Revoke_Role = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Role', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Role", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking one or more roles from\n' - 'one or more users or roles using the `REVOKE` role statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking one or more roles from\n" + "one or more users or roles using the `REVOKE` role statement.\n" + "\n" + ), link=None, level=3, - num='5.14.1') + num="5.14.1", +) RQ_SRS_006_RBAC_Revoke_Role_Keywords = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Role.Keywords', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Role.Keywords", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking one or more roles from\n' - 'special groupings of one or more users or roles with the `ALL`, `ALL EXCEPT`,\n' - 'and `CURRENT_USER` keywords.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking one or more roles from\n" + "special groupings of one or more users or roles with the `ALL`, `ALL EXCEPT`,\n" + "and `CURRENT_USER` keywords.\n" + "\n" + ), link=None, level=3, - num='5.14.2') + num="5.14.2", +) RQ_SRS_006_RBAC_Revoke_Role_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Role.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Role.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking one or more roles from\n' - 'one or more users or roles from one or more clusters\n' - 'using the `REVOKE ON CLUSTER` role statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking one or more roles from\n" + "one or more users or roles from one or more clusters\n" + "using the `REVOKE ON CLUSTER` role statement.\n" + "\n" + ), link=None, level=3, - num='5.14.3') + num="5.14.3", +) RQ_SRS_006_RBAC_Revoke_AdminOption = Requirement( - name='RQ.SRS-006.RBAC.Revoke.AdminOption', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.AdminOption", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking `admin option` privilege\n' - 'in one or more users or roles using the `ADMIN OPTION FOR` clause\n' - 'in the `REVOKE` role statement.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking `admin option` privilege\n" + "in one or more users or roles using the `ADMIN OPTION FOR` clause\n" + "in the `REVOKE` role statement.\n" + "\n" + ), link=None, level=3, - num='5.14.4') + num="5.14.4", +) RQ_SRS_006_RBAC_Revoke_Role_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Revoke.Role.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Revoke.Role.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the following syntax for the `REVOKE` role statement\n' - '\n' - '```sql\n' - 'REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR]\n' - ' role [,...]\n' - ' FROM {user | role | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...]\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support the following syntax for the `REVOKE` role statement\n" + "\n" + "```sql\n" + "REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR]\n" + " role [,...]\n" + " FROM {user | role | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...]\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.14.5') + num="5.14.5", +) RQ_SRS_006_RBAC_Show_Grants = Requirement( - name='RQ.SRS-006.RBAC.Show.Grants', - version='1.0', + name="RQ.SRS-006.RBAC.Show.Grants", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support listing all the privileges granted to current user and role\n' - 'using the `SHOW GRANTS` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support listing all the privileges granted to current user and role\n" + "using the `SHOW GRANTS` statement.\n" + "\n" + ), link=None, level=3, - num='5.15.1') + num="5.15.1", +) RQ_SRS_006_RBAC_Show_Grants_For = Requirement( - name='RQ.SRS-006.RBAC.Show.Grants.For', - version='1.0', + name="RQ.SRS-006.RBAC.Show.Grants.For", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support listing all the privileges granted to a user or a role\n' - 'using the `FOR` clause in the `SHOW GRANTS` statement.\n' - '\n' - ), + "[ClickHouse] SHALL support listing all the privileges granted to a user or a role\n" + "using the `FOR` clause in the `SHOW GRANTS` statement.\n" + "\n" + ), link=None, level=3, - num='5.15.2') + num="5.15.2", +) RQ_SRS_006_RBAC_Show_Grants_Syntax = Requirement( - name='RQ.SRS-006.RBAC.Show.Grants.Syntax', - version='1.0', + name="RQ.SRS-006.RBAC.Show.Grants.Syntax", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[Clickhouse] SHALL use the following syntax for the `SHOW GRANTS` statement\n' - '\n' - '``` sql\n' - 'SHOW GRANTS [FOR user_or_role]\n' - '```\n' - '\n' - ), + "[Clickhouse] SHALL use the following syntax for the `SHOW GRANTS` statement\n" + "\n" + "``` sql\n" + "SHOW GRANTS [FOR user_or_role]\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.15.3') + num="5.15.3", +) RQ_SRS_006_RBAC_Table_PublicTables = Requirement( - name='RQ.SRS-006.RBAC.Table.PublicTables', - version='1.0', + name="RQ.SRS-006.RBAC.Table.PublicTables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support that a user without any privileges will be able to access the following tables\n' - '\n' - '* system.one\n' - '* system.numbers\n' - '* system.contributors\n' - '* system.functions\n' - '\n' - ), + "[ClickHouse] SHALL support that a user without any privileges will be able to access the following tables\n" + "\n" + "* system.one\n" + "* system.numbers\n" + "* system.contributors\n" + "* system.functions\n" + "\n" + ), link=None, level=3, - num='5.16.1') + num="5.16.1", +) RQ_SRS_006_RBAC_Table_SensitiveTables = Requirement( - name='RQ.SRS-006.RBAC.Table.SensitiveTables', - version='1.0', + name="RQ.SRS-006.RBAC.Table.SensitiveTables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL not support a user with no privileges accessing the following `system` tables:\n' - '\n' - '* processes\n' - '* query_log\n' - '* query_thread_log\n' - '* query_views_log\n' - '* clusters\n' - '* events\n' - '* graphite_retentions\n' - '* stack_trace\n' - '* trace_log\n' - '* user_directories\n' - '* zookeeper\n' - '* macros\n' - '\n' - ), + "[ClickHouse] SHALL not support a user with no privileges accessing the following `system` tables:\n" + "\n" + "* processes\n" + "* query_log\n" + "* query_thread_log\n" + "* query_views_log\n" + "* clusters\n" + "* events\n" + "* graphite_retentions\n" + "* stack_trace\n" + "* trace_log\n" + "* user_directories\n" + "* zookeeper\n" + "* macros\n" + "\n" + ), link=None, level=3, - num='5.16.2') + num="5.16.2", +) RQ_SRS_006_RBAC_DistributedTable_Create = Requirement( - name='RQ.SRS-006.RBAC.DistributedTable.Create', - version='1.0', + name="RQ.SRS-006.RBAC.DistributedTable.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully `CREATE` a distributed table if and only if\n' - 'the user has **create table** privilege on the table and **remote** privilege on *.*\n' - '\n' - ), + "[ClickHouse] SHALL successfully `CREATE` a distributed table if and only if\n" + "the user has **create table** privilege on the table and **remote** privilege on *.*\n" + "\n" + ), link=None, level=3, - num='5.17.1') + num="5.17.1", +) RQ_SRS_006_RBAC_DistributedTable_Select = Requirement( - name='RQ.SRS-006.RBAC.DistributedTable.Select', - version='1.0', + name="RQ.SRS-006.RBAC.DistributedTable.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully `SELECT` from a distributed table if and only if\n' - 'the user has **select** privilege on the table and on the remote table specified in the `CREATE` query of the distributed table.\n' - '\n' - 'Does not require **select** privilege for the remote table if the remote table does not exist on the same server as the user.\n' - '\n' - ), + "[ClickHouse] SHALL successfully `SELECT` from a distributed table if and only if\n" + "the user has **select** privilege on the table and on the remote table specified in the `CREATE` query of the distributed table.\n" + "\n" + "Does not require **select** privilege for the remote table if the remote table does not exist on the same server as the user.\n" + "\n" + ), link=None, level=3, - num='5.17.2') + num="5.17.2", +) RQ_SRS_006_RBAC_DistributedTable_Insert = Requirement( - name='RQ.SRS-006.RBAC.DistributedTable.Insert', - version='1.0', + name="RQ.SRS-006.RBAC.DistributedTable.Insert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully `INSERT` into a distributed table if and only if\n' - 'the user has **insert** privilege on the table and on the remote table specified in the `CREATE` query of the distributed table.\n' - '\n' - 'Does not require **insert** privilege for the remote table if the remote table does not exist on the same server as the user,\n' - 'insert executes into the remote table on a different server.\n' - '\n' - ), + "[ClickHouse] SHALL successfully `INSERT` into a distributed table if and only if\n" + "the user has **insert** privilege on the table and on the remote table specified in the `CREATE` query of the distributed table.\n" + "\n" + "Does not require **insert** privilege for the remote table if the remote table does not exist on the same server as the user,\n" + "insert executes into the remote table on a different server.\n" + "\n" + ), link=None, level=3, - num='5.17.3') + num="5.17.3", +) RQ_SRS_006_RBAC_DistributedTable_SpecialTables = Requirement( - name='RQ.SRS-006.RBAC.DistributedTable.SpecialTables', - version='1.0', + name="RQ.SRS-006.RBAC.DistributedTable.SpecialTables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute a query using a distributed table that uses one of the special tables if and only if\n' - 'the user has the necessary privileges to interact with that special table, either granted directly or through a role.\n' - 'Special tables include:\n' - '* materialized view\n' - '* distributed table\n' - '* source table of a materialized view\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute a query using a distributed table that uses one of the special tables if and only if\n" + "the user has the necessary privileges to interact with that special table, either granted directly or through a role.\n" + "Special tables include:\n" + "* materialized view\n" + "* distributed table\n" + "* source table of a materialized view\n" + "\n" + ), link=None, level=3, - num='5.17.4') + num="5.17.4", +) RQ_SRS_006_RBAC_DistributedTable_LocalUser = Requirement( - name='RQ.SRS-006.RBAC.DistributedTable.LocalUser', - version='1.0', + name="RQ.SRS-006.RBAC.DistributedTable.LocalUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute a query using a distributed table from\n' - 'a user present locally, but not remotely.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute a query using a distributed table from\n" + "a user present locally, but not remotely.\n" + "\n" + ), link=None, level=3, - num='5.17.5') + num="5.17.5", +) RQ_SRS_006_RBAC_DistributedTable_SameUserDifferentNodesDifferentPrivileges = Requirement( - name='RQ.SRS-006.RBAC.DistributedTable.SameUserDifferentNodesDifferentPrivileges', - version='1.0', + name="RQ.SRS-006.RBAC.DistributedTable.SameUserDifferentNodesDifferentPrivileges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute a query using a distributed table by a user that exists on multiple nodes\n' - 'if and only if the user has the required privileges on the node the query is being executed from.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute a query using a distributed table by a user that exists on multiple nodes\n" + "if and only if the user has the required privileges on the node the query is being executed from.\n" + "\n" + ), link=None, level=3, - num='5.17.6') + num="5.17.6", +) RQ_SRS_006_RBAC_View = Requirement( - name='RQ.SRS-006.RBAC.View', - version='1.0', + name="RQ.SRS-006.RBAC.View", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to **create**, **select** and **drop**\n' - 'privileges for a view for users or roles.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to **create**, **select** and **drop**\n" + "privileges for a view for users or roles.\n" + "\n" + ), link=None, level=4, - num='5.18.1.1') + num="5.18.1.1", +) RQ_SRS_006_RBAC_View_Create = Requirement( - name='RQ.SRS-006.RBAC.View.Create', - version='1.0', + name="RQ.SRS-006.RBAC.View.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `CREATE VIEW` command if and only if\n' - 'the user has **create view** privilege either explicitly or through roles.\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'CREATE VIEW view AS SELECT * FROM source_table\n' - 'CREATE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `CREATE VIEW` command if and only if\n" + "the user has **create view** privilege either explicitly or through roles.\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "CREATE VIEW view AS SELECT * FROM source_table\n" + "CREATE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.1.2') + num="5.18.1.2", +) RQ_SRS_006_RBAC_View_Select = Requirement( - name='RQ.SRS-006.RBAC.View.Select', - version='1.0', + name="RQ.SRS-006.RBAC.View.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully `SELECT` from a view if and only if\n' - 'the user has **select** privilege for that view either explicitly or through a role.\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'CREATE VIEW view AS SELECT * FROM source_table\n' - 'CREATE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n' - '\n' - 'SELECT * FROM view\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully `SELECT` from a view if and only if\n" + "the user has **select** privilege for that view either explicitly or through a role.\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "CREATE VIEW view AS SELECT * FROM source_table\n" + "CREATE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n" + "\n" + "SELECT * FROM view\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.1.3') + num="5.18.1.3", +) RQ_SRS_006_RBAC_View_Drop = Requirement( - name='RQ.SRS-006.RBAC.View.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.View.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `DROP VIEW` command if and only if\n' - 'the user has **drop view** privilege on that view either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `DROP VIEW` command if and only if\n" + "the user has **drop view** privilege on that view either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.1.4') + num="5.18.1.4", +) RQ_SRS_006_RBAC_MaterializedView = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to **create**, **select**, **alter** and **drop**\n' - 'privileges for a materialized view for users or roles.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to **create**, **select**, **alter** and **drop**\n" + "privileges for a materialized view for users or roles.\n" + "\n" + ), link=None, level=4, - num='5.18.2.1') + num="5.18.2.1", +) RQ_SRS_006_RBAC_MaterializedView_Create = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Create', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `CREATE MATERIALIZED VIEW` command if and only if\n' - 'the user has **create view** privilege either explicitly or through roles.\n' - '\n' - 'If `POPULATE` is specified, the user must have `INSERT` privilege on the view,\n' - 'either explicitly or through roles.\n' - 'For example,\n' - '```sql\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory POPULATE AS SELECT * FROM source_table\n' - '```\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM source_table\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE MATERIALIZED VIEW view0 ENGINE = Memory AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n' - '```\n' - '\n' - 'If the materialized view has a target table explicitly declared in the `TO` clause, the user must have\n' - '**insert** and **select** privilege on the target table.\n' - 'For example,\n' - '```sql\n' - 'CREATE MATERIALIZED VIEW view TO target_table AS SELECT * FROM source_table\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `CREATE MATERIALIZED VIEW` command if and only if\n" + "the user has **create view** privilege either explicitly or through roles.\n" + "\n" + "If `POPULATE` is specified, the user must have `INSERT` privilege on the view,\n" + "either explicitly or through roles.\n" + "For example,\n" + "```sql\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory POPULATE AS SELECT * FROM source_table\n" + "```\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM source_table\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE MATERIALIZED VIEW view0 ENGINE = Memory AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n" + "```\n" + "\n" + "If the materialized view has a target table explicitly declared in the `TO` clause, the user must have\n" + "**insert** and **select** privilege on the target table.\n" + "For example,\n" + "```sql\n" + "CREATE MATERIALIZED VIEW view TO target_table AS SELECT * FROM source_table\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.2.2') + num="5.18.2.2", +) RQ_SRS_006_RBAC_MaterializedView_Select = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Select', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully `SELECT` from a materialized view if and only if\n' - 'the user has **select** privilege for that view either explicitly or through a role.\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM source_table\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE MATERIALIZED VIEW view0 ENGINE = Memory AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n' - '\n' - 'SELECT * FROM view\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully `SELECT` from a materialized view if and only if\n" + "the user has **select** privilege for that view either explicitly or through a role.\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM source_table\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE MATERIALIZED VIEW view ENGINE = Memory AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE MATERIALIZED VIEW view0 ENGINE = Memory AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n" + "\n" + "SELECT * FROM view\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.2.3') + num="5.18.2.3", +) RQ_SRS_006_RBAC_MaterializedView_Select_TargetTable = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Select.TargetTable', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Select.TargetTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully `SELECT` from the target table, implicit or explicit, of a materialized view if and only if\n' - 'the user has `SELECT` privilege for the table, either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only successfully `SELECT` from the target table, implicit or explicit, of a materialized view if and only if\n" + "the user has `SELECT` privilege for the table, either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.2.4') + num="5.18.2.4", +) RQ_SRS_006_RBAC_MaterializedView_Select_SourceTable = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Select.SourceTable', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Select.SourceTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully `SELECT` from the source table of a materialized view if and only if\n' - 'the user has `SELECT` privilege for the table, either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only successfully `SELECT` from the source table of a materialized view if and only if\n" + "the user has `SELECT` privilege for the table, either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.2.5') + num="5.18.2.5", +) RQ_SRS_006_RBAC_MaterializedView_Drop = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `DROP VIEW` command if and only if\n' - 'the user has **drop view** privilege on that view either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `DROP VIEW` command if and only if\n" + "the user has **drop view** privilege on that view either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.2.6') + num="5.18.2.6", +) RQ_SRS_006_RBAC_MaterializedView_ModifyQuery = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.ModifyQuery', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.ModifyQuery", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `MODIFY QUERY` command if and only if\n' - 'the user has **modify query** privilege on that view either explicitly or through a role.\n' - '\n' - 'If the new query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'ALTER TABLE view MODIFY QUERY SELECT * FROM source_table\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `MODIFY QUERY` command if and only if\n" + "the user has **modify query** privilege on that view either explicitly or through a role.\n" + "\n" + "If the new query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "ALTER TABLE view MODIFY QUERY SELECT * FROM source_table\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.2.7') + num="5.18.2.7", +) RQ_SRS_006_RBAC_MaterializedView_Insert = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Insert', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Insert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only succesfully `INSERT` into a materialized view if and only if\n' - 'the user has `INSERT` privilege on the view, either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only succesfully `INSERT` into a materialized view if and only if\n" + "the user has `INSERT` privilege on the view, either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.2.8') + num="5.18.2.8", +) RQ_SRS_006_RBAC_MaterializedView_Insert_SourceTable = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Insert.SourceTable', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Insert.SourceTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only succesfully `INSERT` into a source table of a materialized view if and only if\n' - 'the user has `INSERT` privilege on the source table, either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only succesfully `INSERT` into a source table of a materialized view if and only if\n" + "the user has `INSERT` privilege on the source table, either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.2.9') + num="5.18.2.9", +) RQ_SRS_006_RBAC_MaterializedView_Insert_TargetTable = Requirement( - name='RQ.SRS-006.RBAC.MaterializedView.Insert.TargetTable', - version='1.0', + name="RQ.SRS-006.RBAC.MaterializedView.Insert.TargetTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only succesfully `INSERT` into a target table of a materialized view if and only if\n' - 'the user has `INSERT` privelege on the target table, either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only succesfully `INSERT` into a target table of a materialized view if and only if\n" + "the user has `INSERT` privelege on the target table, either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.2.10') + num="5.18.2.10", +) RQ_SRS_006_RBAC_LiveView = Requirement( - name='RQ.SRS-006.RBAC.LiveView', - version='1.0', + name="RQ.SRS-006.RBAC.LiveView", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to **create**, **select**, **alter** and **drop**\n' - 'privileges for a live view for users or roles.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to **create**, **select**, **alter** and **drop**\n" + "privileges for a live view for users or roles.\n" + "\n" + ), link=None, level=4, - num='5.18.3.1') + num="5.18.3.1", +) RQ_SRS_006_RBAC_LiveView_Create = Requirement( - name='RQ.SRS-006.RBAC.LiveView.Create', - version='1.0', + name="RQ.SRS-006.RBAC.LiveView.Create", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `CREATE LIVE VIEW` command if and only if\n' - 'the user has **create view** privilege either explicitly or through roles.\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'CREATE LIVE VIEW view AS SELECT * FROM source_table\n' - 'CREATE LIVE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE LIVE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE LIVE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE LIVE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE LIVE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `CREATE LIVE VIEW` command if and only if\n" + "the user has **create view** privilege either explicitly or through roles.\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "CREATE LIVE VIEW view AS SELECT * FROM source_table\n" + "CREATE LIVE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE LIVE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE LIVE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE LIVE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE LIVE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.3.2') + num="5.18.3.2", +) RQ_SRS_006_RBAC_LiveView_Select = Requirement( - name='RQ.SRS-006.RBAC.LiveView.Select', - version='1.0', + name="RQ.SRS-006.RBAC.LiveView.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully `SELECT` from a live view if and only if\n' - 'the user has **select** privilege for that view either explicitly or through a role.\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' - 'on all the source tables either explicitly or through a role.\n' - 'For example,\n' - '```sql\n' - 'CREATE LIVE VIEW view AS SELECT * FROM source_table\n' - 'CREATE LIVE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE LIVE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE LIVE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE LIVE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE LIVE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n' - '\n' - 'SELECT * FROM view\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL only successfully `SELECT` from a live view if and only if\n" + "the user has **select** privilege for that view either explicitly or through a role.\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" + "on all the source tables either explicitly or through a role.\n" + "For example,\n" + "```sql\n" + "CREATE LIVE VIEW view AS SELECT * FROM source_table\n" + "CREATE LIVE VIEW view AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE LIVE VIEW view AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE LIVE VIEW view AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE LIVE VIEW view AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE LIVE VIEW view0 AS SELECT column FROM view1 UNION ALL SELECT column FROM view2\n" + "\n" + "SELECT * FROM view\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.18.3.3') + num="5.18.3.3", +) RQ_SRS_006_RBAC_LiveView_Drop = Requirement( - name='RQ.SRS-006.RBAC.LiveView.Drop', - version='1.0', + name="RQ.SRS-006.RBAC.LiveView.Drop", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `DROP VIEW` command if and only if\n' - 'the user has **drop view** privilege on that view either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute a `DROP VIEW` command if and only if\n" + "the user has **drop view** privilege on that view either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.3.4') + num="5.18.3.4", +) RQ_SRS_006_RBAC_LiveView_Refresh = Requirement( - name='RQ.SRS-006.RBAC.LiveView.Refresh', - version='1.0', + name="RQ.SRS-006.RBAC.LiveView.Refresh", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute an `ALTER LIVE VIEW REFRESH` command if and only if\n' - 'the user has **refresh** privilege on that view either explicitly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL only successfully execute an `ALTER LIVE VIEW REFRESH` command if and only if\n" + "the user has **refresh** privilege on that view either explicitly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.18.3.5') + num="5.18.3.5", +) RQ_SRS_006_RBAC_Select = Requirement( - name='RQ.SRS-006.RBAC.Select', - version='1.0', + name="RQ.SRS-006.RBAC.Select", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL execute `SELECT` if and only if the user\n' - 'has the **select** privilege for the destination table\n' - 'either because of the explicit grant or through one of the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL execute `SELECT` if and only if the user\n" + "has the **select** privilege for the destination table\n" + "either because of the explicit grant or through one of the roles assigned to the user.\n" + "\n" + ), link=None, level=3, - num='5.19.1') + num="5.19.1", +) RQ_SRS_006_RBAC_Select_Column = Requirement( - name='RQ.SRS-006.RBAC.Select.Column', - version='1.0', + name="RQ.SRS-006.RBAC.Select.Column", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **select** privilege\n' - 'for one or more specified columns in a table to one or more **users** or **roles**.\n' - 'Any `SELECT` statements SHALL not to be executed, unless the user\n' - 'has the **select** privilege for the destination column\n' - 'either because of the explicit grant or through one of the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **select** privilege\n" + "for one or more specified columns in a table to one or more **users** or **roles**.\n" + "Any `SELECT` statements SHALL not to be executed, unless the user\n" + "has the **select** privilege for the destination column\n" + "either because of the explicit grant or through one of the roles assigned to the user.\n" + "\n" + ), link=None, level=3, - num='5.19.2') + num="5.19.2", +) RQ_SRS_006_RBAC_Select_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Select.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Select.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **select** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `SELECT` statements SHALL succeed only on nodes where\n' - 'the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **select** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `SELECT` statements SHALL succeed only on nodes where\n" + "the table exists and privilege was granted.\n" + "\n" + ), link=None, level=3, - num='5.19.3') + num="5.19.3", +) RQ_SRS_006_RBAC_Select_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Select.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Select.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **select** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **select** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=3, - num='5.19.4') + num="5.19.4", +) RQ_SRS_006_RBAC_Insert = Requirement( - name='RQ.SRS-006.RBAC.Insert', - version='1.0', + name="RQ.SRS-006.RBAC.Insert", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL execute `INSERT INTO` if and only if the user\n' - 'has the **insert** privilege for the destination table\n' - 'either because of the explicit grant or through one of the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL execute `INSERT INTO` if and only if the user\n" + "has the **insert** privilege for the destination table\n" + "either because of the explicit grant or through one of the roles assigned to the user.\n" + "\n" + ), link=None, level=3, - num='5.20.1') + num="5.20.1", +) RQ_SRS_006_RBAC_Insert_Column = Requirement( - name='RQ.SRS-006.RBAC.Insert.Column', - version='1.0', + name="RQ.SRS-006.RBAC.Insert.Column", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **insert** privilege\n' - 'for one or more specified columns in a table to one or more **users** or **roles**.\n' - 'Any `INSERT INTO` statements SHALL not to be executed, unless the user\n' - 'has the **insert** privilege for the destination column\n' - 'either because of the explicit grant or through one of the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **insert** privilege\n" + "for one or more specified columns in a table to one or more **users** or **roles**.\n" + "Any `INSERT INTO` statements SHALL not to be executed, unless the user\n" + "has the **insert** privilege for the destination column\n" + "either because of the explicit grant or through one of the roles assigned to the user.\n" + "\n" + ), link=None, level=3, - num='5.20.2') + num="5.20.2", +) RQ_SRS_006_RBAC_Insert_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Insert.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Insert.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **insert** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `INSERT INTO` statements SHALL succeed only on nodes where\n' - 'the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **insert** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `INSERT INTO` statements SHALL succeed only on nodes where\n" + "the table exists and privilege was granted.\n" + "\n" + ), link=None, level=3, - num='5.20.3') + num="5.20.3", +) RQ_SRS_006_RBAC_Insert_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Insert.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Insert.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **insert** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **insert** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=3, - num='5.20.4') + num="5.20.4", +) RQ_SRS_006_RBAC_Privileges_AlterColumn = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterColumn', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterColumn", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter column** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL\n' - 'return an error, unless the user has the **alter column** privilege for\n' - 'the destination table either because of the explicit grant or through one of\n' - 'the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter column** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL\n" + "return an error, unless the user has the **alter column** privilege for\n" + "the destination table either because of the explicit grant or through one of\n" + "the roles assigned to the user.\n" + "\n" + ), link=None, level=4, - num='5.21.1.1') + num="5.21.1.1", +) RQ_SRS_006_RBAC_Privileges_AlterColumn_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter column** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter column** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.1.2') + num="5.21.1.2", +) RQ_SRS_006_RBAC_Privileges_AlterColumn_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter column** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter column** privilege\n" + "for a database or a specific table to one or more **users** or **roles**\n" + "\n" + ), link=None, level=4, - num='5.21.1.3') + num="5.21.1.3", +) RQ_SRS_006_RBAC_Privileges_AlterColumn_Column = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Column', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Column", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **alter column** privilege\n' - 'for one or more specified columns in a table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error,\n' - 'unless the user has the **alter column** privilege for the destination column\n' - 'either because of the explicit grant or through one of the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **alter column** privilege\n" + "for one or more specified columns in a table to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN` statements SHALL return an error,\n" + "unless the user has the **alter column** privilege for the destination column\n" + "either because of the explicit grant or through one of the roles assigned to the user.\n" + "\n" + ), link=None, level=4, - num='5.21.1.4') + num="5.21.1.4", +) RQ_SRS_006_RBAC_Privileges_AlterColumn_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **alter column** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN`\n' - 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **alter column** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN`\n" + "statements SHALL succeed only on nodes where the table exists and privilege was granted.\n" + "\n" + ), link=None, level=4, - num='5.21.1.5') + num="5.21.1.5", +) RQ_SRS_006_RBAC_Privileges_AlterColumn_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter column** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter column** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.1.6') + num="5.21.1.6", +) RQ_SRS_006_RBAC_Privileges_AlterIndex = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterIndex', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterIndex", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter index** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` statements SHALL\n' - 'return an error, unless the user has the **alter index** privilege for\n' - 'the destination table either because of the explicit grant or through one of\n' - 'the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter index** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX` statements SHALL\n" + "return an error, unless the user has the **alter index** privilege for\n" + "the destination table either because of the explicit grant or through one of\n" + "the roles assigned to the user.\n" + "\n" + ), link=None, level=4, - num='5.21.2.1') + num="5.21.2.1", +) RQ_SRS_006_RBAC_Privileges_AlterIndex_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterIndex.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter index** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter index** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.2.2') + num="5.21.2.2", +) RQ_SRS_006_RBAC_Privileges_AlterIndex_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterIndex.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter index** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter index** privilege\n" + "for a database or a specific table to one or more **users** or **roles**\n" + "\n" + ), link=None, level=4, - num='5.21.2.3') + num="5.21.2.3", +) RQ_SRS_006_RBAC_Privileges_AlterIndex_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterIndex.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **alter index** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX`\n' - 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **alter index** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ORDER BY | ADD|DROP|MATERIALIZE|CLEAR INDEX`\n" + "statements SHALL succeed only on nodes where the table exists and privilege was granted.\n" + "\n" + ), link=None, level=4, - num='5.21.2.4') + num="5.21.2.4", +) RQ_SRS_006_RBAC_Privileges_AlterIndex_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter index** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter index** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.2.5') + num="5.21.2.5", +) RQ_SRS_006_RBAC_Privileges_AlterConstraint = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterConstraint', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter constraint** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL\n' - 'return an error, unless the user has the **alter constraint** privilege for\n' - 'the destination table either because of the explicit grant or through one of\n' - 'the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter constraint** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ADD|CREATE CONSTRAINT` statements SHALL\n" + "return an error, unless the user has the **alter constraint** privilege for\n" + "the destination table either because of the explicit grant or through one of\n" + "the roles assigned to the user.\n" + "\n" + ), link=None, level=4, - num='5.21.3.1') + num="5.21.3.1", +) RQ_SRS_006_RBAC_Privileges_AlterConstraint_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter constraint** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter constraint** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.3.2') + num="5.21.3.2", +) RQ_SRS_006_RBAC_Privileges_AlterConstraint_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter constraint** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter constraint** privilege\n" + "for a database or a specific table to one or more **users** or **roles**\n" + "\n" + ), link=None, level=4, - num='5.21.3.3') + num="5.21.3.3", +) RQ_SRS_006_RBAC_Privileges_AlterConstraint_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **alter constraint** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ADD|DROP CONSTRAINT`\n' - 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **alter constraint** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ADD|DROP CONSTRAINT`\n" + "statements SHALL succeed only on nodes where the table exists and privilege was granted.\n" + "\n" + ), link=None, level=4, - num='5.21.3.4') + num="5.21.3.4", +) RQ_SRS_006_RBAC_Privileges_AlterConstraint_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter constraint** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter constraint** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.3.5') + num="5.21.3.5", +) RQ_SRS_006_RBAC_Privileges_AlterTTL = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterTTL', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterTTL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter ttl** or **alter materialize ttl** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ALTER TTL | ALTER MATERIALIZE TTL` statements SHALL\n' - 'return an error, unless the user has the **alter ttl** or **alter materialize ttl** privilege for\n' - 'the destination table either because of the explicit grant or through one of\n' - 'the roles assigned to the user.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter ttl** or **alter materialize ttl** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ALTER TTL | ALTER MATERIALIZE TTL` statements SHALL\n" + "return an error, unless the user has the **alter ttl** or **alter materialize ttl** privilege for\n" + "the destination table either because of the explicit grant or through one of\n" + "the roles assigned to the user.\n" + "\n" + ), link=None, level=4, - num='5.21.4.1') + num="5.21.4.1", +) RQ_SRS_006_RBAC_Privileges_AlterTTL_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterTTL.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter ttl** or **alter materialize ttl** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter ttl** or **alter materialize ttl** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.4.2') + num="5.21.4.2", +) RQ_SRS_006_RBAC_Privileges_AlterTTL_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterTTL.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter ttl** or **alter materialize ttl** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter ttl** or **alter materialize ttl** privilege\n" + "for a database or a specific table to one or more **users** or **roles**\n" + "\n" + ), link=None, level=4, - num='5.21.4.3') + num="5.21.4.3", +) RQ_SRS_006_RBAC_Privileges_AlterTTL_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterTTL.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **alter ttl** or **alter materialize ttl** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... ALTER TTL | ALTER MATERIALIZE TTL`\n' - 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **alter ttl** or **alter materialize ttl** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... ALTER TTL | ALTER MATERIALIZE TTL`\n" + "statements SHALL succeed only on nodes where the table exists and privilege was granted.\n" + "\n" + ), link=None, level=4, - num='5.21.4.4') + num="5.21.4.4", +) RQ_SRS_006_RBAC_Privileges_AlterTTL_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterTTL.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter ttl** or **alter materialize ttl** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter ttl** or **alter materialize ttl** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.4.5') + num="5.21.4.5", +) RQ_SRS_006_RBAC_Privileges_AlterSettings = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterSettings', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterSettings", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter settings** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... MODIFY SETTING setting` statements SHALL\n' - 'return an error, unless the user has the **alter settings** privilege for\n' - 'the destination table either because of the explicit grant or through one of\n' - 'the roles assigned to the user. The **alter settings** privilege allows\n' - 'modifying table engine settings. It doesn’t affect settings or server configuration parameters.\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter settings** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... MODIFY SETTING setting` statements SHALL\n" + "return an error, unless the user has the **alter settings** privilege for\n" + "the destination table either because of the explicit grant or through one of\n" + "the roles assigned to the user. The **alter settings** privilege allows\n" + "modifying table engine settings. It doesn’t affect settings or server configuration parameters.\n" + "\n" + ), link=None, level=4, - num='5.21.5.1') + num="5.21.5.1", +) RQ_SRS_006_RBAC_Privileges_AlterSettings_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterSettings.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter settings** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter settings** privilege\n" + "for a database or a specific table to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.5.2') + num="5.21.5.2", +) RQ_SRS_006_RBAC_Privileges_AlterSettings_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterSettings.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter settings** privilege\n' - 'for a database or a specific table to one or more **users** or **roles**\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter settings** privilege\n" + "for a database or a specific table to one or more **users** or **roles**\n" + "\n" + ), link=None, level=4, - num='5.21.5.3') + num="5.21.5.3", +) RQ_SRS_006_RBAC_Privileges_AlterSettings_Cluster = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterSettings.Cluster', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.Cluster", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking **alter settings** privilege\n' - 'on a specified cluster to one or more **users** or **roles**.\n' - 'Any `ALTER TABLE ... MODIFY SETTING setting`\n' - 'statements SHALL succeed only on nodes where the table exists and privilege was granted.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking **alter settings** privilege\n" + "on a specified cluster to one or more **users** or **roles**.\n" + "Any `ALTER TABLE ... MODIFY SETTING setting`\n" + "statements SHALL succeed only on nodes where the table exists and privilege was granted.\n" + "\n" + ), link=None, level=4, - num='5.21.5.4') + num="5.21.5.4", +) RQ_SRS_006_RBAC_Privileges_AlterSettings_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterSettings.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter settings** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter settings** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.5.5') + num="5.21.5.5", +) RQ_SRS_006_RBAC_Privileges_AlterUpdate = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterUpdate', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER UPDATE` statement if and only if the user has **alter update** privilege for that column,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER UPDATE` statement if and only if the user has **alter update** privilege for that column,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.21.6.1') + num="5.21.6.1", +) RQ_SRS_006_RBAC_Privileges_AlterUpdate_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterUpdate.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter update** privilege on a column level\n' - 'to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter update** privilege on a column level\n" + "to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.6.2') + num="5.21.6.2", +) RQ_SRS_006_RBAC_Privileges_AlterUpdate_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterUpdate.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter update** privilege on a column level\n' - 'from one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter update** privilege on a column level\n" + "from one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.6.3') + num="5.21.6.3", +) RQ_SRS_006_RBAC_Privileges_AlterUpdate_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterUpdate.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter update** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter update** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.6.4') + num="5.21.6.4", +) RQ_SRS_006_RBAC_Privileges_AlterDelete = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterDelete', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterDelete", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER DELETE` statement if and only if the user has **alter delete** privilege for that table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER DELETE` statement if and only if the user has **alter delete** privilege for that table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.21.7.1') + num="5.21.7.1", +) RQ_SRS_006_RBAC_Privileges_AlterDelete_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterDelete.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterDelete.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter delete** privilege on a column level\n' - 'to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter delete** privilege on a column level\n" + "to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.7.2') + num="5.21.7.2", +) RQ_SRS_006_RBAC_Privileges_AlterDelete_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterDelete.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterDelete.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter delete** privilege on a column level\n' - 'from one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter delete** privilege on a column level\n" + "from one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.7.3') + num="5.21.7.3", +) RQ_SRS_006_RBAC_Privileges_AlterDelete_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterDelete.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterDelete.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter delete** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter delete** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.7.4') + num="5.21.7.4", +) RQ_SRS_006_RBAC_Privileges_AlterFreeze = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFreeze', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER FREEZE` statement if and only if the user has **alter freeze** privilege for that table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER FREEZE` statement if and only if the user has **alter freeze** privilege for that table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.21.8.1') + num="5.21.8.1", +) RQ_SRS_006_RBAC_Privileges_AlterFreeze_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFreeze.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter freeze** privilege on a column level\n' - 'to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter freeze** privilege on a column level\n" + "to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.8.2') + num="5.21.8.2", +) RQ_SRS_006_RBAC_Privileges_AlterFreeze_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFreeze.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter freeze** privilege on a column level\n' - 'from one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter freeze** privilege on a column level\n" + "from one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.8.3') + num="5.21.8.3", +) RQ_SRS_006_RBAC_Privileges_AlterFreeze_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFreeze.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter freeze** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter freeze** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.8.4') + num="5.21.8.4", +) RQ_SRS_006_RBAC_Privileges_AlterFetch = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFetch', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFetch", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER FETCH` statement if and only if the user has **alter fetch** privilege for that table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER FETCH` statement if and only if the user has **alter fetch** privilege for that table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.21.9.1') + num="5.21.9.1", +) RQ_SRS_006_RBAC_Privileges_AlterFetch_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFetch.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFetch.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter fetch** privilege on a column level\n' - 'to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter fetch** privilege on a column level\n" + "to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.9.2') + num="5.21.9.2", +) RQ_SRS_006_RBAC_Privileges_AlterFetch_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFetch.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFetch.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter fetch** privilege on a column level\n' - 'from one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter fetch** privilege on a column level\n" + "from one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.9.3') + num="5.21.9.3", +) RQ_SRS_006_RBAC_Privileges_AlterFetch_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterFetch.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterFetch.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter fetch** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter fetch** privilege\n" + "on tables created using the following engines\n" + "\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.9.4') + num="5.21.9.4", +) RQ_SRS_006_RBAC_Privileges_AlterMove = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterMove', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterMove", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER MOVE` statement if and only if the user has **alter move**, **select**, and **alter delete** privilege on the source table\n' - 'and **insert** privilege on the target table, either directly or through a role.\n' - 'For example,\n' - '```sql\n' - 'ALTER TABLE source_table MOVE PARTITION 1 TO target_table\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER MOVE` statement if and only if the user has **alter move**, **select**, and **alter delete** privilege on the source table\n" + "and **insert** privilege on the target table, either directly or through a role.\n" + "For example,\n" + "```sql\n" + "ALTER TABLE source_table MOVE PARTITION 1 TO target_table\n" + "```\n" + "\n" + ), link=None, level=4, - num='5.21.10.1') + num="5.21.10.1", +) RQ_SRS_006_RBAC_Privileges_AlterMove_Grant = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterMove.Grant', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterMove.Grant", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting **alter move** privilege on a column level\n' - 'to one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support granting **alter move** privilege on a column level\n" + "to one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.10.2') + num="5.21.10.2", +) RQ_SRS_006_RBAC_Privileges_AlterMove_Revoke = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterMove.Revoke', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterMove.Revoke", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support revoking **alter move** privilege on a column level\n' - 'from one or more **users** or **roles**.\n' - '\n' - ), + "[ClickHouse] SHALL support revoking **alter move** privilege on a column level\n" + "from one or more **users** or **roles**.\n" + "\n" + ), link=None, level=4, - num='5.21.10.3') + num="5.21.10.3", +) RQ_SRS_006_RBAC_Privileges_AlterMove_TableEngines = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterMove.TableEngines', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterMove.TableEngines", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support controlling access to the **alter move** privilege\n' - 'on tables created using the following engines\n' - '\n' - '* MergeTree\n' - '* ReplacingMergeTree\n' - '* SummingMergeTree\n' - '* AggregatingMergeTree\n' - '* CollapsingMergeTree\n' - '* VersionedCollapsingMergeTree\n' - '* GraphiteMergeTree\n' - '* ReplicatedMergeTree\n' - '* ReplicatedSummingMergeTree\n' - '* ReplicatedReplacingMergeTree\n' - '* ReplicatedAggregatingMergeTree\n' - '* ReplicatedCollapsingMergeTree\n' - '* ReplicatedVersionedCollapsingMergeTree\n' - '* ReplicatedGraphiteMergeTree\n' - '\n' - ), + "[ClickHouse] SHALL support controlling access to the **alter move** privilege\n" + "on tables created using the following engines\n" + "\n" + "* MergeTree\n" + "* ReplacingMergeTree\n" + "* SummingMergeTree\n" + "* AggregatingMergeTree\n" + "* CollapsingMergeTree\n" + "* VersionedCollapsingMergeTree\n" + "* GraphiteMergeTree\n" + "* ReplicatedMergeTree\n" + "* ReplicatedSummingMergeTree\n" + "* ReplicatedReplacingMergeTree\n" + "* ReplicatedAggregatingMergeTree\n" + "* ReplicatedCollapsingMergeTree\n" + "* ReplicatedVersionedCollapsingMergeTree\n" + "* ReplicatedGraphiteMergeTree\n" + "\n" + ), link=None, level=4, - num='5.21.10.4') + num="5.21.10.4", +) RQ_SRS_006_RBAC_Privileges_CreateTable = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateTable', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only successfully execute a `CREATE TABLE` command if and only if\n' - 'the user has **create table** privilege either explicitly or through roles.\n' - '\n' - 'If the stored query includes one or more source tables, the user must have **select** privilege\n' + "[ClickHouse] SHALL only successfully execute a `CREATE TABLE` command if and only if\n" + "the user has **create table** privilege either explicitly or through roles.\n" + "\n" + "If the stored query includes one or more source tables, the user must have **select** privilege\n" "on all the source tables and **insert** for the table they're trying to create either explicitly or through a role.\n" - 'For example,\n' - '```sql\n' - 'CREATE TABLE table AS SELECT * FROM source_table\n' - 'CREATE TABLE table AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n' - 'CREATE TABLE table AS SELECT * FROM table0 JOIN table1 USING column\n' - 'CREATE TABLE table AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n' - 'CREATE TABLE table AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n' - 'CREATE TABLE table0 AS SELECT column FROM table1 UNION ALL SELECT column FROM table2\n' - '```\n' - '\n' - ), + "For example,\n" + "```sql\n" + "CREATE TABLE table AS SELECT * FROM source_table\n" + "CREATE TABLE table AS SELECT * FROM table0 WHERE column IN (SELECT column FROM table1 WHERE column IN (SELECT column FROM table2 WHERE expression))\n" + "CREATE TABLE table AS SELECT * FROM table0 JOIN table1 USING column\n" + "CREATE TABLE table AS SELECT * FROM table0 UNION ALL SELECT * FROM table1 UNION ALL SELECT * FROM table2\n" + "CREATE TABLE table AS SELECT column FROM table0 JOIN table1 USING column UNION ALL SELECT column FROM table2 WHERE column IN (SELECT column FROM table3 WHERE column IN (SELECT column FROM table4 WHERE expression))\n" + "CREATE TABLE table0 AS SELECT column FROM table1 UNION ALL SELECT column FROM table2\n" + "```\n" + "\n" + ), link=None, level=3, - num='5.22.1') + num="5.22.1", +) RQ_SRS_006_RBAC_Privileges_CreateDatabase = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateDatabase', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateDatabase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE DATABASE` statement if and only if the user has **create database** privilege on the database,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE DATABASE` statement if and only if the user has **create database** privilege on the database,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.22.2') + num="5.22.2", +) RQ_SRS_006_RBAC_Privileges_CreateDictionary = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateDictionary', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateDictionary", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE DICTIONARY` statement if and only if the user has **create dictionary** privilege on the dictionary,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE DICTIONARY` statement if and only if the user has **create dictionary** privilege on the dictionary,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.22.3') + num="5.22.3", +) RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.22.4') + num="5.22.4", +) RQ_SRS_006_RBAC_Privileges_AttachDatabase = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AttachDatabase', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AttachDatabase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ATTACH DATABASE` statement if and only if the user has **create database** privilege on the database,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ATTACH DATABASE` statement if and only if the user has **create database** privilege on the database,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.23.1') + num="5.23.1", +) RQ_SRS_006_RBAC_Privileges_AttachDictionary = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AttachDictionary', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AttachDictionary", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ATTACH DICTIONARY` statement if and only if the user has **create dictionary** privilege on the dictionary,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ATTACH DICTIONARY` statement if and only if the user has **create dictionary** privilege on the dictionary,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.23.2') + num="5.23.2", +) RQ_SRS_006_RBAC_Privileges_AttachTemporaryTable = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ATTACH TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ATTACH TEMPORARY TABLE` statement if and only if the user has **create temporary table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.23.3') + num="5.23.3", +) RQ_SRS_006_RBAC_Privileges_AttachTable = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AttachTable', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AttachTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ATTACH TABLE` statement if and only if the user has **create table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ATTACH TABLE` statement if and only if the user has **create table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.23.4') + num="5.23.4", +) RQ_SRS_006_RBAC_Privileges_DropTable = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropTable', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP TABLE` statement if and only if the user has **drop table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP TABLE` statement if and only if the user has **drop table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.24.1') + num="5.24.1", +) RQ_SRS_006_RBAC_Privileges_DropDatabase = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropDatabase', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropDatabase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP DATABASE` statement if and only if the user has **drop database** privilege on the database,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP DATABASE` statement if and only if the user has **drop database** privilege on the database,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.24.2') + num="5.24.2", +) RQ_SRS_006_RBAC_Privileges_DropDictionary = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropDictionary', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropDictionary", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the dictionary,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the dictionary,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.24.3') + num="5.24.3", +) RQ_SRS_006_RBAC_Privileges_DetachTable = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DetachTable', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DetachTable", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DETACH TABLE` statement if and only if the user has **drop table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DETACH TABLE` statement if and only if the user has **drop table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.25.1') + num="5.25.1", +) RQ_SRS_006_RBAC_Privileges_DetachView = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DetachView', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DetachView", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DETACH VIEW` statement if and only if the user has **drop view** privilege on the view,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DETACH VIEW` statement if and only if the user has **drop view** privilege on the view,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.25.2') + num="5.25.2", +) RQ_SRS_006_RBAC_Privileges_DetachDatabase = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DetachDatabase', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DetachDatabase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DETACH DATABASE` statement if and only if the user has **drop database** privilege on the database,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DETACH DATABASE` statement if and only if the user has **drop database** privilege on the database,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.25.3') + num="5.25.3", +) RQ_SRS_006_RBAC_Privileges_DetachDictionary = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DetachDictionary', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DetachDictionary", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DETACH DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the dictionary,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DETACH DICTIONARY` statement if and only if the user has **drop dictionary** privilege on the dictionary,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.25.4') + num="5.25.4", +) RQ_SRS_006_RBAC_Privileges_Truncate = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Truncate', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Truncate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `TRUNCATE TABLE` statement if and only if the user has **truncate table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `TRUNCATE TABLE` statement if and only if the user has **truncate table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.26.1') + num="5.26.1", +) RQ_SRS_006_RBAC_Privileges_Optimize = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Optimize', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Optimize", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `OPTIMIZE TABLE` statement if and only if the user has **optimize table** privilege on the table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `OPTIMIZE TABLE` statement if and only if the user has **optimize table** privilege on the table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.27.1') + num="5.27.1", +) RQ_SRS_006_RBAC_Privileges_KillQuery = Requirement( - name='RQ.SRS-006.RBAC.Privileges.KillQuery', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.KillQuery", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `KILL QUERY` statement if and only if the user has **kill query** privilege,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `KILL QUERY` statement if and only if the user has **kill query** privilege,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.28.1') + num="5.28.1", +) RQ_SRS_006_RBAC_Privileges_KillMutation = Requirement( - name='RQ.SRS-006.RBAC.Privileges.KillMutation', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.KillMutation", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `KILL MUTATION` statement if and only if\n' - 'the user has the privilege that created the mutation, either directly or through a role.\n' - 'For example, to `KILL MUTATION` after `ALTER UPDATE` query, the user needs `ALTER UPDATE` privilege.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `KILL MUTATION` statement if and only if\n" + "the user has the privilege that created the mutation, either directly or through a role.\n" + "For example, to `KILL MUTATION` after `ALTER UPDATE` query, the user needs `ALTER UPDATE` privilege.\n" + "\n" + ), link=None, level=3, - num='5.29.1') + num="5.29.1", +) RQ_SRS_006_RBAC_Privileges_KillMutation_AlterUpdate = Requirement( - name='RQ.SRS-006.RBAC.Privileges.KillMutation.AlterUpdate', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.KillMutation.AlterUpdate", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `KILL MUTATION` query on an `ALTER UPDATE` mutation if and only if\n' - 'the user has `ALTER UPDATE` privilege on the table where the mutation was created, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `KILL MUTATION` query on an `ALTER UPDATE` mutation if and only if\n" + "the user has `ALTER UPDATE` privilege on the table where the mutation was created, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.29.2') + num="5.29.2", +) RQ_SRS_006_RBAC_Privileges_KillMutation_AlterDelete = Requirement( - name='RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDelete', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDelete", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `KILL MUTATION` query on an `ALTER DELETE` mutation if and only if\n' - 'the user has `ALTER DELETE` privilege on the table where the mutation was created, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `KILL MUTATION` query on an `ALTER DELETE` mutation if and only if\n" + "the user has `ALTER DELETE` privilege on the table where the mutation was created, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.29.3') + num="5.29.3", +) RQ_SRS_006_RBAC_Privileges_KillMutation_AlterDropColumn = Requirement( - name='RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDropColumn', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDropColumn", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `KILL MUTATION` query on an `ALTER DROP COLUMN` mutation if and only if\n' - 'the user has `ALTER DROP COLUMN` privilege on the table where the mutation was created, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `KILL MUTATION` query on an `ALTER DROP COLUMN` mutation if and only if\n" + "the user has `ALTER DROP COLUMN` privilege on the table where the mutation was created, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.29.4') + num="5.29.4", +) RQ_SRS_006_RBAC_ShowTables_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowTables.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowTables.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL grant **show tables** privilege on a table to a user if that user has recieved any grant,\n' - 'including `SHOW TABLES`, on that table, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL grant **show tables** privilege on a table to a user if that user has recieved any grant,\n" + "including `SHOW TABLES`, on that table, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.1') + num="5.30.1", +) RQ_SRS_006_RBAC_ShowTables_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowTables.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowTables.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW TABLES` statement if and only if the user has **show tables** privilege,\n' - 'or any privilege on the table either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW TABLES` statement if and only if the user has **show tables** privilege,\n" + "or any privilege on the table either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.2') + num="5.30.2", +) RQ_SRS_006_RBAC_ExistsTable_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ExistsTable.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ExistsTable.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `EXISTS table` statement if and only if the user has **show tables** privilege,\n' - 'or any privilege on the table either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `EXISTS table` statement if and only if the user has **show tables** privilege,\n" + "or any privilege on the table either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.3') + num="5.30.3", +) RQ_SRS_006_RBAC_CheckTable_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.CheckTable.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.CheckTable.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CHECK table` statement if and only if the user has **show tables** privilege,\n' - 'or any privilege on the table either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CHECK table` statement if and only if the user has **show tables** privilege,\n" + "or any privilege on the table either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.4') + num="5.30.4", +) RQ_SRS_006_RBAC_ShowDatabases_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowDatabases.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowDatabases.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL grant **show databases** privilege on a database to a user if that user has recieved any grant,\n' - 'including `SHOW DATABASES`, on that table, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL grant **show databases** privilege on a database to a user if that user has recieved any grant,\n" + "including `SHOW DATABASES`, on that table, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.5') + num="5.30.5", +) RQ_SRS_006_RBAC_ShowDatabases_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowDatabases.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowDatabases.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW DATABASES` statement if and only if the user has **show databases** privilege,\n' - 'or any privilege on the database either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW DATABASES` statement if and only if the user has **show databases** privilege,\n" + "or any privilege on the database either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.6') + num="5.30.6", +) RQ_SRS_006_RBAC_ShowCreateDatabase_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateDatabase.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateDatabase.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE DATABASE` statement if and only if the user has **show databases** privilege,\n' - 'or any privilege on the database either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE DATABASE` statement if and only if the user has **show databases** privilege,\n" + "or any privilege on the database either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.7') + num="5.30.7", +) RQ_SRS_006_RBAC_UseDatabase_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.UseDatabase.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.UseDatabase.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `USE database` statement if and only if the user has **show databases** privilege,\n' - 'or any privilege on the database either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `USE database` statement if and only if the user has **show databases** privilege,\n" + "or any privilege on the database either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.8') + num="5.30.8", +) RQ_SRS_006_RBAC_ShowColumns_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowColumns.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowColumns.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking the `SHOW COLUMNS` privilege.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking the `SHOW COLUMNS` privilege.\n" + "\n" + ), link=None, level=3, - num='5.30.9') + num="5.30.9", +) RQ_SRS_006_RBAC_ShowCreateTable_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateTable.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateTable.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE TABLE` statement if and only if the user has **show columns** privilege on that table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE TABLE` statement if and only if the user has **show columns** privilege on that table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.10') + num="5.30.10", +) RQ_SRS_006_RBAC_DescribeTable_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.DescribeTable.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.DescribeTable.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DESCRIBE table` statement if and only if the user has **show columns** privilege on that table,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DESCRIBE table` statement if and only if the user has **show columns** privilege on that table,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.11') + num="5.30.11", +) RQ_SRS_006_RBAC_ShowDictionaries_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowDictionaries.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowDictionaries.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL grant **show dictionaries** privilege on a dictionary to a user if that user has recieved any grant,\n' - 'including `SHOW DICTIONARIES`, on that dictionary, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL grant **show dictionaries** privilege on a dictionary to a user if that user has recieved any grant,\n" + "including `SHOW DICTIONARIES`, on that dictionary, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.12') + num="5.30.12", +) RQ_SRS_006_RBAC_ShowDictionaries_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowDictionaries.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowDictionaries.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW DICTIONARIES` statement if and only if the user has **show dictionaries** privilege,\n' - 'or any privilege on the dictionary either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW DICTIONARIES` statement if and only if the user has **show dictionaries** privilege,\n" + "or any privilege on the dictionary either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.13') + num="5.30.13", +) RQ_SRS_006_RBAC_ShowCreateDictionary_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateDictionary.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateDictionary.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE DICTIONARY` statement if and only if the user has **show dictionaries** privilege,\n' - 'or any privilege on the dictionary either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE DICTIONARY` statement if and only if the user has **show dictionaries** privilege,\n" + "or any privilege on the dictionary either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.14') + num="5.30.14", +) RQ_SRS_006_RBAC_ExistsDictionary_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ExistsDictionary.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ExistsDictionary.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `EXISTS dictionary` statement if and only if the user has **show dictionaries** privilege,\n' - 'or any privilege on the dictionary either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `EXISTS dictionary` statement if and only if the user has **show dictionaries** privilege,\n" + "or any privilege on the dictionary either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.30.15') + num="5.30.15", +) RQ_SRS_006_RBAC_Privileges_CreateUser = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateUser', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE USER` statement if and only if the user has **create user** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE USER` statement if and only if the user has **create user** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.1') + num="5.31.1", +) RQ_SRS_006_RBAC_Privileges_CreateUser_DefaultRole = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateUser.DefaultRole', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateUser.DefaultRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE USER` statement with `DEFAULT ROLE ` clause if and only if\n' - 'the user has **create user** privilege and the role with **admin option**, or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE USER` statement with `DEFAULT ROLE ` clause if and only if\n" + "the user has **create user** privilege and the role with **admin option**, or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.2') + num="5.31.2", +) RQ_SRS_006_RBAC_Privileges_AlterUser = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterUser', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER USER` statement if and only if the user has **alter user** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER USER` statement if and only if the user has **alter user** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.3') + num="5.31.3", +) RQ_SRS_006_RBAC_Privileges_DropUser = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropUser', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropUser", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP USER` statement if and only if the user has **drop user** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP USER` statement if and only if the user has **drop user** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.4') + num="5.31.4", +) RQ_SRS_006_RBAC_Privileges_CreateRole = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateRole', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE ROLE` statement if and only if the user has **create role** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE ROLE` statement if and only if the user has **create role** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.5') + num="5.31.5", +) RQ_SRS_006_RBAC_Privileges_AlterRole = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterRole', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER ROLE` statement if and only if the user has **alter role** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER ROLE` statement if and only if the user has **alter role** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.6') + num="5.31.6", +) RQ_SRS_006_RBAC_Privileges_DropRole = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropRole', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropRole", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP ROLE` statement if and only if the user has **drop role** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP ROLE` statement if and only if the user has **drop role** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.7') + num="5.31.7", +) RQ_SRS_006_RBAC_Privileges_CreateRowPolicy = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateRowPolicy', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateRowPolicy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE ROW POLICY` statement if and only if the user has **create row policy** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE ROW POLICY` statement if and only if the user has **create row policy** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.8') + num="5.31.8", +) RQ_SRS_006_RBAC_Privileges_AlterRowPolicy = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterRowPolicy', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterRowPolicy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER ROW POLICY` statement if and only if the user has **alter row policy** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER ROW POLICY` statement if and only if the user has **alter row policy** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.9') + num="5.31.9", +) RQ_SRS_006_RBAC_Privileges_DropRowPolicy = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropRowPolicy', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropRowPolicy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP ROW POLICY` statement if and only if the user has **drop row policy** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP ROW POLICY` statement if and only if the user has **drop row policy** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.10') + num="5.31.10", +) RQ_SRS_006_RBAC_Privileges_CreateQuota = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateQuota', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateQuota", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE QUOTA` statement if and only if the user has **create quota** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE QUOTA` statement if and only if the user has **create quota** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.11') + num="5.31.11", +) RQ_SRS_006_RBAC_Privileges_AlterQuota = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterQuota', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterQuota", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER QUOTA` statement if and only if the user has **alter quota** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER QUOTA` statement if and only if the user has **alter quota** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.12') + num="5.31.12", +) RQ_SRS_006_RBAC_Privileges_DropQuota = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropQuota', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropQuota", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP QUOTA` statement if and only if the user has **drop quota** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP QUOTA` statement if and only if the user has **drop quota** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.13') + num="5.31.13", +) RQ_SRS_006_RBAC_Privileges_CreateSettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.Privileges.CreateSettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.CreateSettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `CREATE SETTINGS PROFILE` statement if and only if the user has **create settings profile** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `CREATE SETTINGS PROFILE` statement if and only if the user has **create settings profile** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.14') + num="5.31.14", +) RQ_SRS_006_RBAC_Privileges_AlterSettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AlterSettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AlterSettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `ALTER SETTINGS PROFILE` statement if and only if the user has **alter settings profile** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `ALTER SETTINGS PROFILE` statement if and only if the user has **alter settings profile** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.15') + num="5.31.15", +) RQ_SRS_006_RBAC_Privileges_DropSettingsProfile = Requirement( - name='RQ.SRS-006.RBAC.Privileges.DropSettingsProfile', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.DropSettingsProfile", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `DROP SETTINGS PROFILE` statement if and only if the user has **drop settings profile** privilege,\n' - 'or either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `DROP SETTINGS PROFILE` statement if and only if the user has **drop settings profile** privilege,\n" + "or either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.31.16') + num="5.31.16", +) RQ_SRS_006_RBAC_Privileges_RoleAdmin = Requirement( - name='RQ.SRS-006.RBAC.Privileges.RoleAdmin', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.RoleAdmin", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute any role grant or revoke by a user with `ROLE ADMIN` privilege.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute any role grant or revoke by a user with `ROLE ADMIN` privilege.\n" + "\n" + ), link=None, level=3, - num='5.31.17') + num="5.31.17", +) RQ_SRS_006_RBAC_ShowUsers_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowUsers.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowUsers.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SHOW USERS` privilege when\n' - 'the user is granted `SHOW USERS`, `SHOW CREATE USER`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SHOW USERS` privilege when\n" + "the user is granted `SHOW USERS`, `SHOW CREATE USER`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n" + "\n" + ), link=None, level=4, - num='5.31.18.1') + num="5.31.18.1", +) RQ_SRS_006_RBAC_ShowUsers_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowUsers.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowUsers.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW USERS` statement if and only if the user has **show users** privilege,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW USERS` statement if and only if the user has **show users** privilege,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.2') + num="5.31.18.2", +) RQ_SRS_006_RBAC_ShowCreateUser_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateUser.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateUser.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE USER` statement if and only if the user has **show users** privilege,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE USER` statement if and only if the user has **show users** privilege,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.3') + num="5.31.18.3", +) RQ_SRS_006_RBAC_ShowRoles_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowRoles.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowRoles.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SHOW ROLES` privilege when\n' - 'the user is granted `SHOW ROLES`, `SHOW CREATE ROLE`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SHOW ROLES` privilege when\n" + "the user is granted `SHOW ROLES`, `SHOW CREATE ROLE`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n" + "\n" + ), link=None, level=4, - num='5.31.18.4') + num="5.31.18.4", +) RQ_SRS_006_RBAC_ShowRoles_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowRoles.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowRoles.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW ROLES` statement if and only if the user has **show roles** privilege,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW ROLES` statement if and only if the user has **show roles** privilege,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.5') + num="5.31.18.5", +) RQ_SRS_006_RBAC_ShowCreateRole_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateRole.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateRole.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE ROLE` statement if and only if the user has **show roles** privilege,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE ROLE` statement if and only if the user has **show roles** privilege,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.6') + num="5.31.18.6", +) RQ_SRS_006_RBAC_ShowRowPolicies_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowRowPolicies.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowRowPolicies.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SHOW ROW POLICIES` privilege when\n' - 'the user is granted `SHOW ROW POLICIES`, `SHOW POLICIES`, `SHOW CREATE ROW POLICY`,\n' - '`SHOW CREATE POLICY`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SHOW ROW POLICIES` privilege when\n" + "the user is granted `SHOW ROW POLICIES`, `SHOW POLICIES`, `SHOW CREATE ROW POLICY`,\n" + "`SHOW CREATE POLICY`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n" + "\n" + ), link=None, level=4, - num='5.31.18.7') + num="5.31.18.7", +) RQ_SRS_006_RBAC_ShowRowPolicies_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowRowPolicies.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowRowPolicies.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW ROW POLICIES` or `SHOW POLICIES` statement if and only if\n' - 'the user has **show row policies** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW ROW POLICIES` or `SHOW POLICIES` statement if and only if\n" + "the user has **show row policies** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.8') + num="5.31.18.8", +) RQ_SRS_006_RBAC_ShowCreateRowPolicy_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateRowPolicy.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateRowPolicy.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE ROW POLICY` or `SHOW CREATE POLICY` statement\n' - 'if and only if the user has **show row policies** privilege,either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE ROW POLICY` or `SHOW CREATE POLICY` statement\n" + "if and only if the user has **show row policies** privilege,either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.9') + num="5.31.18.9", +) RQ_SRS_006_RBAC_ShowQuotas_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowQuotas.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowQuotas.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SHOW QUOTAS` privilege when\n' - 'the user is granted `SHOW QUOTAS`, `SHOW CREATE QUOTA`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SHOW QUOTAS` privilege when\n" + "the user is granted `SHOW QUOTAS`, `SHOW CREATE QUOTA`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n" + "\n" + ), link=None, level=4, - num='5.31.18.10') + num="5.31.18.10", +) RQ_SRS_006_RBAC_ShowQuotas_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowQuotas.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowQuotas.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW QUOTAS` statement if and only if the user has **show quotas** privilege,\n' - 'either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW QUOTAS` statement if and only if the user has **show quotas** privilege,\n" + "either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.11') + num="5.31.18.11", +) RQ_SRS_006_RBAC_ShowCreateQuota_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateQuota.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateQuota.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE QUOTA` statement if and only if\n' - 'the user has **show quotas** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE QUOTA` statement if and only if\n" + "the user has **show quotas** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.12') + num="5.31.18.12", +) RQ_SRS_006_RBAC_ShowSettingsProfiles_Privilege = Requirement( - name='RQ.SRS-006.RBAC.ShowSettingsProfiles.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowSettingsProfiles.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SHOW SETTINGS PROFILES` privilege when\n' - 'the user is granted `SHOW SETTINGS PROFILES`, `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`,\n' - '`SHOW SETTINGS PROFILE`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SHOW SETTINGS PROFILES` privilege when\n" + "the user is granted `SHOW SETTINGS PROFILES`, `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`,\n" + "`SHOW SETTINGS PROFILE`, `SHOW ACCESS`, or `ACCESS MANAGEMENT`.\n" + "\n" + ), link=None, level=4, - num='5.31.18.13') + num="5.31.18.13", +) RQ_SRS_006_RBAC_ShowSettingsProfiles_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowSettingsProfiles.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowSettingsProfiles.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW SETTINGS PROFILES` or `SHOW PROFILES` statement\n' - 'if and only if the user has **show settings profiles** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW SETTINGS PROFILES` or `SHOW PROFILES` statement\n" + "if and only if the user has **show settings profiles** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.14') + num="5.31.18.14", +) RQ_SRS_006_RBAC_ShowCreateSettingsProfile_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.ShowCreateSettingsProfile.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.ShowCreateSettingsProfile.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `SHOW CREATE SETTINGS PROFILE` or `SHOW CREATE PROFILE` statement\n' - 'if and only if the user has **show settings profiles** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `SHOW CREATE SETTINGS PROFILE` or `SHOW CREATE PROFILE` statement\n" + "if and only if the user has **show settings profiles** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=4, - num='5.31.18.15') + num="5.31.18.15", +) RQ_SRS_006_RBAC_dictGet_Privilege = Requirement( - name='RQ.SRS-006.RBAC.dictGet.Privilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictGet.Privilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `dictGet` privilege when\n' - 'the user is granted `dictGet`, `dictHas`, `dictGetHierarchy`, or `dictIsIn`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `dictGet` privilege when\n" + "the user is granted `dictGet`, `dictHas`, `dictGetHierarchy`, or `dictIsIn`.\n" + "\n" + ), link=None, level=3, - num='5.32.1') + num="5.32.1", +) RQ_SRS_006_RBAC_dictGet_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.dictGet.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictGet.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `dictGet` statement\n' - 'if and only if the user has **dictGet** privilege on that dictionary, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `dictGet` statement\n" + "if and only if the user has **dictGet** privilege on that dictionary, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.32.2') + num="5.32.2", +) RQ_SRS_006_RBAC_dictGet_Type_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.dictGet.Type.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictGet.Type.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `dictGet[TYPE]` statement\n' - 'if and only if the user has **dictGet** privilege on that dictionary, either directly or through a role.\n' - 'Available types:\n' - '\n' - '* Int8\n' - '* Int16\n' - '* Int32\n' - '* Int64\n' - '* UInt8\n' - '* UInt16\n' - '* UInt32\n' - '* UInt64\n' - '* Float32\n' - '* Float64\n' - '* Date\n' - '* DateTime\n' - '* UUID\n' - '* String\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `dictGet[TYPE]` statement\n" + "if and only if the user has **dictGet** privilege on that dictionary, either directly or through a role.\n" + "Available types:\n" + "\n" + "* Int8\n" + "* Int16\n" + "* Int32\n" + "* Int64\n" + "* UInt8\n" + "* UInt16\n" + "* UInt32\n" + "* UInt64\n" + "* Float32\n" + "* Float64\n" + "* Date\n" + "* DateTime\n" + "* UUID\n" + "* String\n" + "\n" + ), link=None, level=3, - num='5.32.3') + num="5.32.3", +) RQ_SRS_006_RBAC_dictGet_OrDefault_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.dictGet.OrDefault.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictGet.OrDefault.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `dictGetOrDefault` statement\n' - 'if and only if the user has **dictGet** privilege on that dictionary, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `dictGetOrDefault` statement\n" + "if and only if the user has **dictGet** privilege on that dictionary, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.32.4') + num="5.32.4", +) RQ_SRS_006_RBAC_dictHas_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.dictHas.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictHas.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `dictHas` statement\n' - 'if and only if the user has **dictGet** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `dictHas` statement\n" + "if and only if the user has **dictGet** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.32.5') + num="5.32.5", +) RQ_SRS_006_RBAC_dictGetHierarchy_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.dictGetHierarchy.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictGetHierarchy.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `dictGetHierarchy` statement\n' - 'if and only if the user has **dictGet** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `dictGetHierarchy` statement\n" + "if and only if the user has **dictGet** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.32.6') + num="5.32.6", +) RQ_SRS_006_RBAC_dictIsIn_RequiredPrivilege = Requirement( - name='RQ.SRS-006.RBAC.dictIsIn.RequiredPrivilege', - version='1.0', + name="RQ.SRS-006.RBAC.dictIsIn.RequiredPrivilege", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `dictIsIn` statement\n' - 'if and only if the user has **dictGet** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `dictIsIn` statement\n" + "if and only if the user has **dictGet** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.32.7') + num="5.32.7", +) RQ_SRS_006_RBAC_Privileges_Introspection = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Introspection', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Introspection", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `INTROSPECTION` privilege when\n' - 'the user is granted `INTROSPECTION` or `INTROSPECTION FUNCTIONS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `INTROSPECTION` privilege when\n" + "the user is granted `INTROSPECTION` or `INTROSPECTION FUNCTIONS`.\n" + "\n" + ), link=None, level=3, - num='5.33.1') + num="5.33.1", +) RQ_SRS_006_RBAC_Privileges_Introspection_addressToLine = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Introspection.addressToLine', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Introspection.addressToLine", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `addressToLine` statement if and only if\n' - 'the user has **introspection** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `addressToLine` statement if and only if\n" + "the user has **introspection** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.33.2') + num="5.33.2", +) RQ_SRS_006_RBAC_Privileges_Introspection_addressToSymbol = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Introspection.addressToSymbol', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Introspection.addressToSymbol", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `addressToSymbol` statement if and only if\n' - 'the user has **introspection** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `addressToSymbol` statement if and only if\n" + "the user has **introspection** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.33.3') + num="5.33.3", +) RQ_SRS_006_RBAC_Privileges_Introspection_demangle = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Introspection.demangle', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Introspection.demangle", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `demangle` statement if and only if\n' - 'the user has **introspection** privilege, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `demangle` statement if and only if\n" + "the user has **introspection** privilege, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.33.4') + num="5.33.4", +) RQ_SRS_006_RBAC_Privileges_System_Shutdown = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Shutdown', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Shutdown", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM SHUTDOWN` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM SHUTDOWN`, `SHUTDOWN`,or `SYSTEM KILL`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM SHUTDOWN` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM SHUTDOWN`, `SHUTDOWN`,or `SYSTEM KILL`.\n" + "\n" + ), link=None, level=3, - num='5.34.1') + num="5.34.1", +) RQ_SRS_006_RBAC_Privileges_System_DropCache = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.DropCache', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.DropCache", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM DROP CACHE` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, or `DROP CACHE`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM DROP CACHE` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, or `DROP CACHE`.\n" + "\n" + ), link=None, level=3, - num='5.34.2') + num="5.34.2", +) RQ_SRS_006_RBAC_Privileges_System_DropCache_DNS = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.DropCache.DNS', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.DropCache.DNS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM DROP DNS CACHE` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, `DROP CACHE`, `SYSTEM DROP DNS CACHE`,\n' - '`SYSTEM DROP DNS`, `DROP DNS CACHE`, or `DROP DNS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM DROP DNS CACHE` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, `DROP CACHE`, `SYSTEM DROP DNS CACHE`,\n" + "`SYSTEM DROP DNS`, `DROP DNS CACHE`, or `DROP DNS`.\n" + "\n" + ), link=None, level=3, - num='5.34.3') + num="5.34.3", +) RQ_SRS_006_RBAC_Privileges_System_DropCache_Mark = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.DropCache.Mark', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.DropCache.Mark", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM DROP MARK CACHE` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, `DROP CACHE`, `SYSTEM DROP MARK CACHE`,\n' - '`SYSTEM DROP MARK`, `DROP MARK CACHE`, or `DROP MARKS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM DROP MARK CACHE` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, `DROP CACHE`, `SYSTEM DROP MARK CACHE`,\n" + "`SYSTEM DROP MARK`, `DROP MARK CACHE`, or `DROP MARKS`.\n" + "\n" + ), link=None, level=3, - num='5.34.4') + num="5.34.4", +) RQ_SRS_006_RBAC_Privileges_System_DropCache_Uncompressed = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.DropCache.Uncompressed', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.DropCache.Uncompressed", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM DROP UNCOMPRESSED CACHE` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, `DROP CACHE`, `SYSTEM DROP UNCOMPRESSED CACHE`,\n' - '`SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, or `DROP UNCOMPRESSED`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM DROP UNCOMPRESSED CACHE` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM DROP CACHE`, `DROP CACHE`, `SYSTEM DROP UNCOMPRESSED CACHE`,\n" + "`SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, or `DROP UNCOMPRESSED`.\n" + "\n" + ), link=None, level=3, - num='5.34.5') + num="5.34.5", +) RQ_SRS_006_RBAC_Privileges_System_Reload = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Reload', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Reload", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM RELOAD` privilege when\n' - 'the user is granted `SYSTEM` or `SYSTEM RELOAD`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM RELOAD` privilege when\n" + "the user is granted `SYSTEM` or `SYSTEM RELOAD`.\n" + "\n" + ), link=None, level=3, - num='5.34.6') + num="5.34.6", +) RQ_SRS_006_RBAC_Privileges_System_Reload_Config = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Reload.Config', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Reload.Config", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM RELOAD CONFIG` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD CONFIG`, or `RELOAD CONFIG`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM RELOAD CONFIG` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD CONFIG`, or `RELOAD CONFIG`.\n" + "\n" + ), link=None, level=3, - num='5.34.7') + num="5.34.7", +) RQ_SRS_006_RBAC_Privileges_System_Reload_Dictionary = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionary', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionary", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM RELOAD DICTIONARY` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARIES`, or `RELOAD DICTIONARY`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM RELOAD DICTIONARY` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARIES`, or `RELOAD DICTIONARY`.\n" + "\n" + ), link=None, level=3, - num='5.34.8') + num="5.34.8", +) RQ_SRS_006_RBAC_Privileges_System_Reload_Dictionaries = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionaries', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionaries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM RELOAD DICTIONARIES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARIES`, or `RELOAD DICTIONARY`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM RELOAD DICTIONARIES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARIES`, or `RELOAD DICTIONARY`.\n" + "\n" + ), link=None, level=3, - num='5.34.9') + num="5.34.9", +) RQ_SRS_006_RBAC_Privileges_System_Reload_EmbeddedDictionaries = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Reload.EmbeddedDictionaries', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Reload.EmbeddedDictionaries", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM RELOAD EMBEDDED DICTIONARIES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD DICTIONARY ON *.*`, or `SYSTEM RELOAD EMBEDDED DICTIONARIES`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM RELOAD EMBEDDED DICTIONARIES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM RELOAD`, `SYSTEM RELOAD DICTIONARY ON *.*`, or `SYSTEM RELOAD EMBEDDED DICTIONARIES`.\n" + "\n" + ), link=None, level=3, - num='5.34.10') + num="5.34.10", +) RQ_SRS_006_RBAC_Privileges_System_Merges = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Merges', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Merges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM MERGES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM MERGES`, `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, or `START MERGES`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM MERGES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM MERGES`, `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, or `START MERGES`.\n" + "\n" + ), link=None, level=3, - num='5.34.11') + num="5.34.11", +) RQ_SRS_006_RBAC_Privileges_System_TTLMerges = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.TTLMerges', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.TTLMerges", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM TTL MERGES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM TTL MERGES`, `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, or `START TTL MERGES`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM TTL MERGES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM TTL MERGES`, `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, or `START TTL MERGES`.\n" + "\n" + ), link=None, level=3, - num='5.34.12') + num="5.34.12", +) RQ_SRS_006_RBAC_Privileges_System_Fetches = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Fetches', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Fetches", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM FETCHES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM FETCHES`, `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, or `START FETCHES`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM FETCHES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM FETCHES`, `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, or `START FETCHES`.\n" + "\n" + ), link=None, level=3, - num='5.34.13') + num="5.34.13", +) RQ_SRS_006_RBAC_Privileges_System_Moves = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Moves', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Moves", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM MOVES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM MOVES`, `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, or `START MOVES`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM MOVES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM MOVES`, `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, or `START MOVES`.\n" + "\n" + ), link=None, level=3, - num='5.34.14') + num="5.34.14", +) RQ_SRS_006_RBAC_Privileges_System_Sends = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Sends', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Sends", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM SENDS` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM SENDS`, `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, or `START SENDS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM SENDS` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM SENDS`, `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, or `START SENDS`.\n" + "\n" + ), link=None, level=3, - num='5.34.15') + num="5.34.15", +) RQ_SRS_006_RBAC_Privileges_System_Sends_Distributed = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Sends.Distributed', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Sends.Distributed", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM DISTRIBUTED SENDS` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM DISTRIBUTED SENDS`, `SYSTEM STOP DISTRIBUTED SENDS`,\n' - '`SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, or `START DISTRIBUTED SENDS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM DISTRIBUTED SENDS` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM DISTRIBUTED SENDS`, `SYSTEM STOP DISTRIBUTED SENDS`,\n" + "`SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, or `START DISTRIBUTED SENDS`.\n" + "\n" + ), link=None, level=3, - num='5.34.16') + num="5.34.16", +) RQ_SRS_006_RBAC_Privileges_System_Sends_Replicated = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Sends.Replicated', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Sends.Replicated", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM REPLICATED SENDS` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM REPLICATED SENDS`, `SYSTEM STOP REPLICATED SENDS`,\n' - '`SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, or `START REPLICATED SENDS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM REPLICATED SENDS` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM REPLICATED SENDS`, `SYSTEM STOP REPLICATED SENDS`,\n" + "`SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, or `START REPLICATED SENDS`.\n" + "\n" + ), link=None, level=3, - num='5.34.17') + num="5.34.17", +) RQ_SRS_006_RBAC_Privileges_System_ReplicationQueues = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.ReplicationQueues', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.ReplicationQueues", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM REPLICATION QUEUES` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM REPLICATION QUEUES`, `SYSTEM STOP REPLICATION QUEUES`,\n' - '`SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, or `START REPLICATION QUEUES`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM REPLICATION QUEUES` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM REPLICATION QUEUES`, `SYSTEM STOP REPLICATION QUEUES`,\n" + "`SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, or `START REPLICATION QUEUES`.\n" + "\n" + ), link=None, level=3, - num='5.34.18') + num="5.34.18", +) RQ_SRS_006_RBAC_Privileges_System_SyncReplica = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.SyncReplica', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.SyncReplica", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM SYNC REPLICA` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM SYNC REPLICA`, or `SYNC REPLICA`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM SYNC REPLICA` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM SYNC REPLICA`, or `SYNC REPLICA`.\n" + "\n" + ), link=None, level=3, - num='5.34.19') + num="5.34.19", +) RQ_SRS_006_RBAC_Privileges_System_RestartReplica = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.RestartReplica', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.RestartReplica", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM RESTART REPLICA` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM RESTART REPLICA`, or `RESTART REPLICA`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM RESTART REPLICA` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM RESTART REPLICA`, or `RESTART REPLICA`.\n" + "\n" + ), link=None, level=3, - num='5.34.20') + num="5.34.20", +) RQ_SRS_006_RBAC_Privileges_System_Flush = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Flush', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Flush", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM FLUSH` privilege when\n' - 'the user is granted `SYSTEM` or `SYSTEM FLUSH`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM FLUSH` privilege when\n" + "the user is granted `SYSTEM` or `SYSTEM FLUSH`.\n" + "\n" + ), link=None, level=3, - num='5.34.21') + num="5.34.21", +) RQ_SRS_006_RBAC_Privileges_System_Flush_Distributed = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Flush.Distributed', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Flush.Distributed", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM FLUSH DISTRIBUTED` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM FLUSH DISTRIBUTED`, or `FLUSH DISTRIBUTED`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM FLUSH DISTRIBUTED` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM FLUSH DISTRIBUTED`, or `FLUSH DISTRIBUTED`.\n" + "\n" + ), link=None, level=3, - num='5.34.22') + num="5.34.22", +) RQ_SRS_006_RBAC_Privileges_System_Flush_Logs = Requirement( - name='RQ.SRS-006.RBAC.Privileges.System.Flush.Logs', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.System.Flush.Logs", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully grant `SYSTEM FLUSH LOGS` privilege when\n' - 'the user is granted `SYSTEM`, `SYSTEM FLUSH LOGS`, or `FLUSH LOGS`.\n' - '\n' - ), + "[ClickHouse] SHALL successfully grant `SYSTEM FLUSH LOGS` privilege when\n" + "the user is granted `SYSTEM`, `SYSTEM FLUSH LOGS`, or `FLUSH LOGS`.\n" + "\n" + ), link=None, level=3, - num='5.34.23') + num="5.34.23", +) RQ_SRS_006_RBAC_Privileges_Sources = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking `SOURCES` privilege from\n' - 'the user, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking `SOURCES` privilege from\n" + "the user, either directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.1') + num="5.35.1", +) RQ_SRS_006_RBAC_Privileges_Sources_File = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.File', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.File", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `FILE` source by a user if and only if\n' - 'the user has `FILE` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `FILE` source by a user if and only if\n" + "the user has `FILE` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.2') + num="5.35.2", +) RQ_SRS_006_RBAC_Privileges_Sources_URL = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.URL', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.URL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `URL` source by a user if and only if\n' - 'the user has `URL` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `URL` source by a user if and only if\n" + "the user has `URL` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.3') + num="5.35.3", +) RQ_SRS_006_RBAC_Privileges_Sources_Remote = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.Remote', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.Remote", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `REMOTE` source by a user if and only if\n' - 'the user has `REMOTE` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `REMOTE` source by a user if and only if\n" + "the user has `REMOTE` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.4') + num="5.35.4", +) RQ_SRS_006_RBAC_Privileges_Sources_MySQL = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.MySQL', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.MySQL", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `MySQL` source by a user if and only if\n' - 'the user has `MySQL` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `MySQL` source by a user if and only if\n" + "the user has `MySQL` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.5') + num="5.35.5", +) RQ_SRS_006_RBAC_Privileges_Sources_ODBC = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.ODBC', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.ODBC", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `ODBC` source by a user if and only if\n' - 'the user has `ODBC` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `ODBC` source by a user if and only if\n" + "the user has `ODBC` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.6') + num="5.35.6", +) RQ_SRS_006_RBAC_Privileges_Sources_JDBC = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.JDBC', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.JDBC", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `JDBC` source by a user if and only if\n' - 'the user has `JDBC` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `JDBC` source by a user if and only if\n" + "the user has `JDBC` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.7') + num="5.35.7", +) RQ_SRS_006_RBAC_Privileges_Sources_HDFS = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.HDFS', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.HDFS", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `HDFS` source by a user if and only if\n' - 'the user has `HDFS` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `HDFS` source by a user if and only if\n" + "the user has `HDFS` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.8') + num="5.35.8", +) RQ_SRS_006_RBAC_Privileges_Sources_S3 = Requirement( - name='RQ.SRS-006.RBAC.Privileges.Sources.S3', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.Sources.S3", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the use of `S3` source by a user if and only if\n' - 'the user has `S3` or `SOURCES` privileges granted to them directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL support the use of `S3` source by a user if and only if\n" + "the user has `S3` or `SOURCES` privileges granted to them directly or through a role.\n" + "\n" + ), link=None, level=3, - num='5.35.9') + num="5.35.9", +) RQ_SRS_006_RBAC_Privileges_GrantOption = Requirement( - name='RQ.SRS-006.RBAC.Privileges.GrantOption', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.GrantOption", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL successfully execute `GRANT` or `REVOKE` privilege statements by a user if and only if\n' - 'the user has that privilege with `GRANT OPTION`, either directly or through a role.\n' - '\n' - ), + "[ClickHouse] SHALL successfully execute `GRANT` or `REVOKE` privilege statements by a user if and only if\n" + "the user has that privilege with `GRANT OPTION`, either directly or through a role.\n" + "\n" + ), link=None, level=2, - num='5.36') + num="5.36", +) RQ_SRS_006_RBAC_Privileges_All = Requirement( - name='RQ.SRS-006.RBAC.Privileges.All', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.All", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking `ALL` privilege\n' - 'using `GRANT ALL ON *.* TO user`.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking `ALL` privilege\n" + "using `GRANT ALL ON *.* TO user`.\n" + "\n" + ), link=None, level=2, - num='5.37') + num="5.37", +) RQ_SRS_006_RBAC_Privileges_RoleAll = Requirement( - name='RQ.SRS-006.RBAC.Privileges.RoleAll', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.RoleAll", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting a role named `ALL` using `GRANT ALL TO user`.\n' - 'This shall only grant the user the privileges that have been granted to the role.\n' - '\n' - ), + "[ClickHouse] SHALL support granting a role named `ALL` using `GRANT ALL TO user`.\n" + "This shall only grant the user the privileges that have been granted to the role.\n" + "\n" + ), link=None, level=2, - num='5.38') + num="5.38", +) RQ_SRS_006_RBAC_Privileges_None = Requirement( - name='RQ.SRS-006.RBAC.Privileges.None', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.None", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support granting or revoking `NONE` privilege\n' - 'using `GRANT NONE TO user` or `GRANT USAGE ON *.* TO user`.\n' - '\n' - ), + "[ClickHouse] SHALL support granting or revoking `NONE` privilege\n" + "using `GRANT NONE TO user` or `GRANT USAGE ON *.* TO user`.\n" + "\n" + ), link=None, level=2, - num='5.39') + num="5.39", +) RQ_SRS_006_RBAC_Privileges_AdminOption = Requirement( - name='RQ.SRS-006.RBAC.Privileges.AdminOption', - version='1.0', + name="RQ.SRS-006.RBAC.Privileges.AdminOption", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support a user granting or revoking a role if and only if\n' - 'the user has that role with `ADMIN OPTION` privilege.\n' - '\n' - ), + "[ClickHouse] SHALL support a user granting or revoking a role if and only if\n" + "the user has that role with `ADMIN OPTION` privilege.\n" + "\n" + ), link=None, level=2, - num='5.40') + num="5.40", +) SRS_006_ClickHouse_Role_Based_Access_Control = Specification( - name='SRS-006 ClickHouse Role Based Access Control', + name="SRS-006 ClickHouse Role Based Access Control", description=None, author=None, date=None, @@ -8871,597 +9380,1391 @@ SRS_006_ClickHouse_Role_Based_Access_Control = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Terminology', level=1, num='3'), - Heading(name='Privilege Definitions', level=1, num='4'), - Heading(name='Requirements', level=1, num='5'), - Heading(name='Generic', level=2, num='5.1'), - Heading(name='RQ.SRS-006.RBAC', level=3, num='5.1.1'), - Heading(name='Login', level=2, num='5.2'), - Heading(name='RQ.SRS-006.RBAC.Login', level=3, num='5.2.1'), - Heading(name='RQ.SRS-006.RBAC.Login.DefaultUser', level=3, num='5.2.2'), - Heading(name='User', level=2, num='5.3'), - Heading(name='RQ.SRS-006.RBAC.User', level=3, num='5.3.1'), - Heading(name='RQ.SRS-006.RBAC.User.Roles', level=3, num='5.3.2'), - Heading(name='RQ.SRS-006.RBAC.User.Privileges', level=3, num='5.3.3'), - Heading(name='RQ.SRS-006.RBAC.User.Variables', level=3, num='5.3.4'), - Heading(name='RQ.SRS-006.RBAC.User.Variables.Constraints', level=3, num='5.3.5'), - Heading(name='RQ.SRS-006.RBAC.User.SettingsProfile', level=3, num='5.3.6'), - Heading(name='RQ.SRS-006.RBAC.User.Quotas', level=3, num='5.3.7'), - Heading(name='RQ.SRS-006.RBAC.User.RowPolicies', level=3, num='5.3.8'), - Heading(name='RQ.SRS-006.RBAC.User.DefaultRole', level=3, num='5.3.9'), - Heading(name='RQ.SRS-006.RBAC.User.RoleSelection', level=3, num='5.3.10'), - Heading(name='RQ.SRS-006.RBAC.User.ShowCreate', level=3, num='5.3.11'), - Heading(name='RQ.SRS-006.RBAC.User.ShowPrivileges', level=3, num='5.3.12'), - Heading(name='RQ.SRS-006.RBAC.User.Use.DefaultRole', level=3, num='5.3.13'), - Heading(name='RQ.SRS-006.RBAC.User.Use.AllRolesWhenNoDefaultRole', level=3, num='5.3.14'), - Heading(name='Create User', level=3, num='5.3.15'), - Heading(name='RQ.SRS-006.RBAC.User.Create', level=4, num='5.3.15.1'), - Heading(name='RQ.SRS-006.RBAC.User.Create.IfNotExists', level=4, num='5.3.15.2'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Replace', level=4, num='5.3.15.3'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.NoPassword', level=4, num='5.3.15.4'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.NoPassword.Login', level=4, num='5.3.15.5'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.PlainText', level=4, num='5.3.15.6'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.PlainText.Login', level=4, num='5.3.15.7'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Password', level=4, num='5.3.15.8'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Password.Login', level=4, num='5.3.15.9'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash', level=4, num='5.3.15.10'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash.Login', level=4, num='5.3.15.11'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password', level=4, num='5.3.15.12'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password.Login', level=4, num='5.3.15.13'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash', level=4, num='5.3.15.14'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash.Login', level=4, num='5.3.15.15'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.Name', level=4, num='5.3.15.16'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.Regexp', level=4, num='5.3.15.17'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.IP', level=4, num='5.3.15.18'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.Any', level=4, num='5.3.15.19'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.None', level=4, num='5.3.15.20'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.Local', level=4, num='5.3.15.21'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.Like', level=4, num='5.3.15.22'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Host.Default', level=4, num='5.3.15.23'), - Heading(name='RQ.SRS-006.RBAC.User.Create.DefaultRole', level=4, num='5.3.15.24'), - Heading(name='RQ.SRS-006.RBAC.User.Create.DefaultRole.None', level=4, num='5.3.15.25'), - Heading(name='RQ.SRS-006.RBAC.User.Create.DefaultRole.All', level=4, num='5.3.15.26'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Settings', level=4, num='5.3.15.27'), - Heading(name='RQ.SRS-006.RBAC.User.Create.OnCluster', level=4, num='5.3.15.28'), - Heading(name='RQ.SRS-006.RBAC.User.Create.Syntax', level=4, num='5.3.15.29'), - Heading(name='Alter User', level=3, num='5.3.16'), - Heading(name='RQ.SRS-006.RBAC.User.Alter', level=4, num='5.3.16.1'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.OrderOfEvaluation', level=4, num='5.3.16.2'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.IfExists', level=4, num='5.3.16.3'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Cluster', level=4, num='5.3.16.4'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Rename', level=4, num='5.3.16.5'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Password.PlainText', level=4, num='5.3.16.6'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Password.Sha256Password', level=4, num='5.3.16.7'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Password.DoubleSha1Password', level=4, num='5.3.16.8'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.AddDrop', level=4, num='5.3.16.9'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.Local', level=4, num='5.3.16.10'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.Name', level=4, num='5.3.16.11'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.Regexp', level=4, num='5.3.16.12'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.IP', level=4, num='5.3.16.13'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.Like', level=4, num='5.3.16.14'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.Any', level=4, num='5.3.16.15'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Host.None', level=4, num='5.3.16.16'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.DefaultRole', level=4, num='5.3.16.17'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.DefaultRole.All', level=4, num='5.3.16.18'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.DefaultRole.AllExcept', level=4, num='5.3.16.19'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Settings', level=4, num='5.3.16.20'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Settings.Min', level=4, num='5.3.16.21'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Settings.Max', level=4, num='5.3.16.22'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Settings.Profile', level=4, num='5.3.16.23'), - Heading(name='RQ.SRS-006.RBAC.User.Alter.Syntax', level=4, num='5.3.16.24'), - Heading(name='Show Create User', level=3, num='5.3.17'), - Heading(name='RQ.SRS-006.RBAC.User.ShowCreateUser', level=4, num='5.3.17.1'), - Heading(name='RQ.SRS-006.RBAC.User.ShowCreateUser.For', level=4, num='5.3.17.2'), - Heading(name='RQ.SRS-006.RBAC.User.ShowCreateUser.Syntax', level=4, num='5.3.17.3'), - Heading(name='Drop User', level=3, num='5.3.18'), - Heading(name='RQ.SRS-006.RBAC.User.Drop', level=4, num='5.3.18.1'), - Heading(name='RQ.SRS-006.RBAC.User.Drop.IfExists', level=4, num='5.3.18.2'), - Heading(name='RQ.SRS-006.RBAC.User.Drop.OnCluster', level=4, num='5.3.18.3'), - Heading(name='RQ.SRS-006.RBAC.User.Drop.Syntax', level=4, num='5.3.18.4'), - Heading(name='Role', level=2, num='5.4'), - Heading(name='RQ.SRS-006.RBAC.Role', level=3, num='5.4.1'), - Heading(name='RQ.SRS-006.RBAC.Role.Privileges', level=3, num='5.4.2'), - Heading(name='RQ.SRS-006.RBAC.Role.Variables', level=3, num='5.4.3'), - Heading(name='RQ.SRS-006.RBAC.Role.SettingsProfile', level=3, num='5.4.4'), - Heading(name='RQ.SRS-006.RBAC.Role.Quotas', level=3, num='5.4.5'), - Heading(name='RQ.SRS-006.RBAC.Role.RowPolicies', level=3, num='5.4.6'), - Heading(name='Create Role', level=3, num='5.4.7'), - Heading(name='RQ.SRS-006.RBAC.Role.Create', level=4, num='5.4.7.1'), - Heading(name='RQ.SRS-006.RBAC.Role.Create.IfNotExists', level=4, num='5.4.7.2'), - Heading(name='RQ.SRS-006.RBAC.Role.Create.Replace', level=4, num='5.4.7.3'), - Heading(name='RQ.SRS-006.RBAC.Role.Create.Settings', level=4, num='5.4.7.4'), - Heading(name='RQ.SRS-006.RBAC.Role.Create.Syntax', level=4, num='5.4.7.5'), - Heading(name='Alter Role', level=3, num='5.4.8'), - Heading(name='RQ.SRS-006.RBAC.Role.Alter', level=4, num='5.4.8.1'), - Heading(name='RQ.SRS-006.RBAC.Role.Alter.IfExists', level=4, num='5.4.8.2'), - Heading(name='RQ.SRS-006.RBAC.Role.Alter.Cluster', level=4, num='5.4.8.3'), - Heading(name='RQ.SRS-006.RBAC.Role.Alter.Rename', level=4, num='5.4.8.4'), - Heading(name='RQ.SRS-006.RBAC.Role.Alter.Settings', level=4, num='5.4.8.5'), - Heading(name='RQ.SRS-006.RBAC.Role.Alter.Syntax', level=4, num='5.4.8.6'), - Heading(name='Drop Role', level=3, num='5.4.9'), - Heading(name='RQ.SRS-006.RBAC.Role.Drop', level=4, num='5.4.9.1'), - Heading(name='RQ.SRS-006.RBAC.Role.Drop.IfExists', level=4, num='5.4.9.2'), - Heading(name='RQ.SRS-006.RBAC.Role.Drop.Cluster', level=4, num='5.4.9.3'), - Heading(name='RQ.SRS-006.RBAC.Role.Drop.Syntax', level=4, num='5.4.9.4'), - Heading(name='Show Create Role', level=3, num='5.4.10'), - Heading(name='RQ.SRS-006.RBAC.Role.ShowCreate', level=4, num='5.4.10.1'), - Heading(name='RQ.SRS-006.RBAC.Role.ShowCreate.Syntax', level=4, num='5.4.10.2'), - Heading(name='Partial Revokes', level=2, num='5.5'), - Heading(name='RQ.SRS-006.RBAC.PartialRevokes', level=3, num='5.5.1'), - Heading(name='RQ.SRS-006.RBAC.PartialRevoke.Syntax', level=3, num='5.5.2'), - Heading(name='Settings Profile', level=2, num='5.6'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile', level=3, num='5.6.1'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Constraints', level=3, num='5.6.2'), - Heading(name='Create Settings Profile', level=3, num='5.6.3'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create', level=4, num='5.6.3.1'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.IfNotExists', level=4, num='5.6.3.2'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Replace', level=4, num='5.6.3.3'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Variables', level=4, num='5.6.3.4'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Value', level=4, num='5.6.3.5'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Constraints', level=4, num='5.6.3.6'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment', level=4, num='5.6.3.7'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.None', level=4, num='5.6.3.8'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.All', level=4, num='5.6.3.9'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.AllExcept', level=4, num='5.6.3.10'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Inherit', level=4, num='5.6.3.11'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.OnCluster', level=4, num='5.6.3.12'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Create.Syntax', level=4, num='5.6.3.13'), - Heading(name='Alter Settings Profile', level=3, num='5.6.4'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter', level=4, num='5.6.4.1'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.IfExists', level=4, num='5.6.4.2'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Rename', level=4, num='5.6.4.3'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables', level=4, num='5.6.4.4'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Value', level=4, num='5.6.4.5'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Constraints', level=4, num='5.6.4.6'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment', level=4, num='5.6.4.7'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.None', level=4, num='5.6.4.8'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.All', level=4, num='5.6.4.9'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.AllExcept', level=4, num='5.6.4.10'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.Inherit', level=4, num='5.6.4.11'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.OnCluster', level=4, num='5.6.4.12'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Alter.Syntax', level=4, num='5.6.4.13'), - Heading(name='Drop Settings Profile', level=3, num='5.6.5'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Drop', level=4, num='5.6.5.1'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Drop.IfExists', level=4, num='5.6.5.2'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Drop.OnCluster', level=4, num='5.6.5.3'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.Drop.Syntax', level=4, num='5.6.5.4'), - Heading(name='Show Create Settings Profile', level=3, num='5.6.6'), - Heading(name='RQ.SRS-006.RBAC.SettingsProfile.ShowCreateSettingsProfile', level=4, num='5.6.6.1'), - Heading(name='Quotas', level=2, num='5.7'), - Heading(name='RQ.SRS-006.RBAC.Quotas', level=3, num='5.7.1'), - Heading(name='RQ.SRS-006.RBAC.Quotas.Keyed', level=3, num='5.7.2'), - Heading(name='RQ.SRS-006.RBAC.Quotas.Queries', level=3, num='5.7.3'), - Heading(name='RQ.SRS-006.RBAC.Quotas.Errors', level=3, num='5.7.4'), - Heading(name='RQ.SRS-006.RBAC.Quotas.ResultRows', level=3, num='5.7.5'), - Heading(name='RQ.SRS-006.RBAC.Quotas.ReadRows', level=3, num='5.7.6'), - Heading(name='RQ.SRS-006.RBAC.Quotas.ResultBytes', level=3, num='5.7.7'), - Heading(name='RQ.SRS-006.RBAC.Quotas.ReadBytes', level=3, num='5.7.8'), - Heading(name='RQ.SRS-006.RBAC.Quotas.ExecutionTime', level=3, num='5.7.9'), - Heading(name='Create Quotas', level=3, num='5.7.10'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create', level=4, num='5.7.10.1'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.IfNotExists', level=4, num='5.7.10.2'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Replace', level=4, num='5.7.10.3'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Cluster', level=4, num='5.7.10.4'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Interval', level=4, num='5.7.10.5'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Interval.Randomized', level=4, num='5.7.10.6'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Queries', level=4, num='5.7.10.7'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Errors', level=4, num='5.7.10.8'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.ResultRows', level=4, num='5.7.10.9'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.ReadRows', level=4, num='5.7.10.10'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.ResultBytes', level=4, num='5.7.10.11'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.ReadBytes', level=4, num='5.7.10.12'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.ExecutionTime', level=4, num='5.7.10.13'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.NoLimits', level=4, num='5.7.10.14'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.TrackingOnly', level=4, num='5.7.10.15'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.KeyedBy', level=4, num='5.7.10.16'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.KeyedByOptions', level=4, num='5.7.10.17'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Assignment', level=4, num='5.7.10.18'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Assignment.None', level=4, num='5.7.10.19'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Assignment.All', level=4, num='5.7.10.20'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Assignment.Except', level=4, num='5.7.10.21'), - Heading(name='RQ.SRS-006.RBAC.Quota.Create.Syntax', level=4, num='5.7.10.22'), - Heading(name='Alter Quota', level=3, num='5.7.11'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter', level=4, num='5.7.11.1'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.IfExists', level=4, num='5.7.11.2'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Rename', level=4, num='5.7.11.3'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Cluster', level=4, num='5.7.11.4'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Interval', level=4, num='5.7.11.5'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Interval.Randomized', level=4, num='5.7.11.6'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Queries', level=4, num='5.7.11.7'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Errors', level=4, num='5.7.11.8'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.ResultRows', level=4, num='5.7.11.9'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.ReadRows', level=4, num='5.7.11.10'), - Heading(name='RQ.SRS-006.RBAC.Quota.ALter.ResultBytes', level=4, num='5.7.11.11'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.ReadBytes', level=4, num='5.7.11.12'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.ExecutionTime', level=4, num='5.7.11.13'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.NoLimits', level=4, num='5.7.11.14'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.TrackingOnly', level=4, num='5.7.11.15'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.KeyedBy', level=4, num='5.7.11.16'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.KeyedByOptions', level=4, num='5.7.11.17'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Assignment', level=4, num='5.7.11.18'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Assignment.None', level=4, num='5.7.11.19'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Assignment.All', level=4, num='5.7.11.20'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Assignment.Except', level=4, num='5.7.11.21'), - Heading(name='RQ.SRS-006.RBAC.Quota.Alter.Syntax', level=4, num='5.7.11.22'), - Heading(name='Drop Quota', level=3, num='5.7.12'), - Heading(name='RQ.SRS-006.RBAC.Quota.Drop', level=4, num='5.7.12.1'), - Heading(name='RQ.SRS-006.RBAC.Quota.Drop.IfExists', level=4, num='5.7.12.2'), - Heading(name='RQ.SRS-006.RBAC.Quota.Drop.Cluster', level=4, num='5.7.12.3'), - Heading(name='RQ.SRS-006.RBAC.Quota.Drop.Syntax', level=4, num='5.7.12.4'), - Heading(name='Show Quotas', level=3, num='5.7.13'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowQuotas', level=4, num='5.7.13.1'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowQuotas.IntoOutfile', level=4, num='5.7.13.2'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowQuotas.Format', level=4, num='5.7.13.3'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowQuotas.Settings', level=4, num='5.7.13.4'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowQuotas.Syntax', level=4, num='5.7.13.5'), - Heading(name='Show Create Quota', level=3, num='5.7.14'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Name', level=4, num='5.7.14.1'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Current', level=4, num='5.7.14.2'), - Heading(name='RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Syntax', level=4, num='5.7.14.3'), - Heading(name='Row Policy', level=2, num='5.8'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy', level=3, num='5.8.1'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Condition', level=3, num='5.8.2'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Restriction', level=3, num='5.8.3'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Nesting', level=3, num='5.8.4'), - Heading(name='Create Row Policy', level=3, num='5.8.5'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create', level=4, num='5.8.5.1'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.IfNotExists', level=4, num='5.8.5.2'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Replace', level=4, num='5.8.5.3'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.OnCluster', level=4, num='5.8.5.4'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.On', level=4, num='5.8.5.5'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Access', level=4, num='5.8.5.6'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Access.Permissive', level=4, num='5.8.5.7'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Access.Restrictive', level=4, num='5.8.5.8'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.ForSelect', level=4, num='5.8.5.9'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Condition', level=4, num='5.8.5.10'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment', level=4, num='5.8.5.11'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.None', level=4, num='5.8.5.12'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.All', level=4, num='5.8.5.13'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.AllExcept', level=4, num='5.8.5.14'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Create.Syntax', level=4, num='5.8.5.15'), - Heading(name='Alter Row Policy', level=3, num='5.8.6'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter', level=4, num='5.8.6.1'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.IfExists', level=4, num='5.8.6.2'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.ForSelect', level=4, num='5.8.6.3'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.OnCluster', level=4, num='5.8.6.4'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.On', level=4, num='5.8.6.5'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Rename', level=4, num='5.8.6.6'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Access', level=4, num='5.8.6.7'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Permissive', level=4, num='5.8.6.8'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Restrictive', level=4, num='5.8.6.9'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Condition', level=4, num='5.8.6.10'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Condition.None', level=4, num='5.8.6.11'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment', level=4, num='5.8.6.12'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.None', level=4, num='5.8.6.13'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.All', level=4, num='5.8.6.14'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.AllExcept', level=4, num='5.8.6.15'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Alter.Syntax', level=4, num='5.8.6.16'), - Heading(name='Drop Row Policy', level=3, num='5.8.7'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Drop', level=4, num='5.8.7.1'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Drop.IfExists', level=4, num='5.8.7.2'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Drop.On', level=4, num='5.8.7.3'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Drop.OnCluster', level=4, num='5.8.7.4'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.Drop.Syntax', level=4, num='5.8.7.5'), - Heading(name='Show Create Row Policy', level=3, num='5.8.8'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy', level=4, num='5.8.8.1'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.On', level=4, num='5.8.8.2'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.Syntax', level=4, num='5.8.8.3'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies', level=4, num='5.8.8.4'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.On', level=4, num='5.8.8.5'), - Heading(name='RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.Syntax', level=4, num='5.8.8.6'), - Heading(name='Set Default Role', level=2, num='5.9'), - Heading(name='RQ.SRS-006.RBAC.SetDefaultRole', level=3, num='5.9.1'), - Heading(name='RQ.SRS-006.RBAC.SetDefaultRole.CurrentUser', level=3, num='5.9.2'), - Heading(name='RQ.SRS-006.RBAC.SetDefaultRole.All', level=3, num='5.9.3'), - Heading(name='RQ.SRS-006.RBAC.SetDefaultRole.AllExcept', level=3, num='5.9.4'), - Heading(name='RQ.SRS-006.RBAC.SetDefaultRole.None', level=3, num='5.9.5'), - Heading(name='RQ.SRS-006.RBAC.SetDefaultRole.Syntax', level=3, num='5.9.6'), - Heading(name='Set Role', level=2, num='5.10'), - Heading(name='RQ.SRS-006.RBAC.SetRole', level=3, num='5.10.1'), - Heading(name='RQ.SRS-006.RBAC.SetRole.Default', level=3, num='5.10.2'), - Heading(name='RQ.SRS-006.RBAC.SetRole.None', level=3, num='5.10.3'), - Heading(name='RQ.SRS-006.RBAC.SetRole.All', level=3, num='5.10.4'), - Heading(name='RQ.SRS-006.RBAC.SetRole.AllExcept', level=3, num='5.10.5'), - Heading(name='RQ.SRS-006.RBAC.SetRole.Syntax', level=3, num='5.10.6'), - Heading(name='Grant', level=2, num='5.11'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.To', level=3, num='5.11.1'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.ToCurrentUser', level=3, num='5.11.2'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Select', level=3, num='5.11.3'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Insert', level=3, num='5.11.4'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Alter', level=3, num='5.11.5'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Create', level=3, num='5.11.6'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Drop', level=3, num='5.11.7'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Truncate', level=3, num='5.11.8'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Optimize', level=3, num='5.11.9'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Show', level=3, num='5.11.10'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.KillQuery', level=3, num='5.11.11'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.AccessManagement', level=3, num='5.11.12'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.System', level=3, num='5.11.13'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Introspection', level=3, num='5.11.14'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Sources', level=3, num='5.11.15'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.DictGet', level=3, num='5.11.16'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.None', level=3, num='5.11.17'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.All', level=3, num='5.11.18'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.GrantOption', level=3, num='5.11.19'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.On', level=3, num='5.11.20'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.PrivilegeColumns', level=3, num='5.11.21'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.OnCluster', level=3, num='5.11.22'), - Heading(name='RQ.SRS-006.RBAC.Grant.Privilege.Syntax', level=3, num='5.11.23'), - Heading(name='Revoke', level=2, num='5.12'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Cluster', level=3, num='5.12.1'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Select', level=3, num='5.12.2'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Insert', level=3, num='5.12.3'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Alter', level=3, num='5.12.4'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Create', level=3, num='5.12.5'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Drop', level=3, num='5.12.6'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Truncate', level=3, num='5.12.7'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Optimize', level=3, num='5.12.8'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Show', level=3, num='5.12.9'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.KillQuery', level=3, num='5.12.10'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.AccessManagement', level=3, num='5.12.11'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.System', level=3, num='5.12.12'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Introspection', level=3, num='5.12.13'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Sources', level=3, num='5.12.14'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.DictGet', level=3, num='5.12.15'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.PrivilegeColumns', level=3, num='5.12.16'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Multiple', level=3, num='5.12.17'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.All', level=3, num='5.12.18'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.None', level=3, num='5.12.19'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.On', level=3, num='5.12.20'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.From', level=3, num='5.12.21'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Privilege.Syntax', level=3, num='5.12.22'), - Heading(name='Grant Role', level=2, num='5.13'), - Heading(name='RQ.SRS-006.RBAC.Grant.Role', level=3, num='5.13.1'), - Heading(name='RQ.SRS-006.RBAC.Grant.Role.CurrentUser', level=3, num='5.13.2'), - Heading(name='RQ.SRS-006.RBAC.Grant.Role.AdminOption', level=3, num='5.13.3'), - Heading(name='RQ.SRS-006.RBAC.Grant.Role.OnCluster', level=3, num='5.13.4'), - Heading(name='RQ.SRS-006.RBAC.Grant.Role.Syntax', level=3, num='5.13.5'), - Heading(name='Revoke Role', level=2, num='5.14'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Role', level=3, num='5.14.1'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Role.Keywords', level=3, num='5.14.2'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Role.Cluster', level=3, num='5.14.3'), - Heading(name='RQ.SRS-006.RBAC.Revoke.AdminOption', level=3, num='5.14.4'), - Heading(name='RQ.SRS-006.RBAC.Revoke.Role.Syntax', level=3, num='5.14.5'), - Heading(name='Show Grants', level=2, num='5.15'), - Heading(name='RQ.SRS-006.RBAC.Show.Grants', level=3, num='5.15.1'), - Heading(name='RQ.SRS-006.RBAC.Show.Grants.For', level=3, num='5.15.2'), - Heading(name='RQ.SRS-006.RBAC.Show.Grants.Syntax', level=3, num='5.15.3'), - Heading(name='Table Privileges', level=2, num='5.16'), - Heading(name='RQ.SRS-006.RBAC.Table.PublicTables', level=3, num='5.16.1'), - Heading(name='RQ.SRS-006.RBAC.Table.SensitiveTables', level=3, num='5.16.2'), - Heading(name='Distributed Tables', level=2, num='5.17'), - Heading(name='RQ.SRS-006.RBAC.DistributedTable.Create', level=3, num='5.17.1'), - Heading(name='RQ.SRS-006.RBAC.DistributedTable.Select', level=3, num='5.17.2'), - Heading(name='RQ.SRS-006.RBAC.DistributedTable.Insert', level=3, num='5.17.3'), - Heading(name='RQ.SRS-006.RBAC.DistributedTable.SpecialTables', level=3, num='5.17.4'), - Heading(name='RQ.SRS-006.RBAC.DistributedTable.LocalUser', level=3, num='5.17.5'), - Heading(name='RQ.SRS-006.RBAC.DistributedTable.SameUserDifferentNodesDifferentPrivileges', level=3, num='5.17.6'), - Heading(name='Views', level=2, num='5.18'), - Heading(name='View', level=3, num='5.18.1'), - Heading(name='RQ.SRS-006.RBAC.View', level=4, num='5.18.1.1'), - Heading(name='RQ.SRS-006.RBAC.View.Create', level=4, num='5.18.1.2'), - Heading(name='RQ.SRS-006.RBAC.View.Select', level=4, num='5.18.1.3'), - Heading(name='RQ.SRS-006.RBAC.View.Drop', level=4, num='5.18.1.4'), - Heading(name='Materialized View', level=3, num='5.18.2'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView', level=4, num='5.18.2.1'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Create', level=4, num='5.18.2.2'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Select', level=4, num='5.18.2.3'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Select.TargetTable', level=4, num='5.18.2.4'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Select.SourceTable', level=4, num='5.18.2.5'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Drop', level=4, num='5.18.2.6'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.ModifyQuery', level=4, num='5.18.2.7'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Insert', level=4, num='5.18.2.8'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Insert.SourceTable', level=4, num='5.18.2.9'), - Heading(name='RQ.SRS-006.RBAC.MaterializedView.Insert.TargetTable', level=4, num='5.18.2.10'), - Heading(name='Live View', level=3, num='5.18.3'), - Heading(name='RQ.SRS-006.RBAC.LiveView', level=4, num='5.18.3.1'), - Heading(name='RQ.SRS-006.RBAC.LiveView.Create', level=4, num='5.18.3.2'), - Heading(name='RQ.SRS-006.RBAC.LiveView.Select', level=4, num='5.18.3.3'), - Heading(name='RQ.SRS-006.RBAC.LiveView.Drop', level=4, num='5.18.3.4'), - Heading(name='RQ.SRS-006.RBAC.LiveView.Refresh', level=4, num='5.18.3.5'), - Heading(name='Select', level=2, num='5.19'), - Heading(name='RQ.SRS-006.RBAC.Select', level=3, num='5.19.1'), - Heading(name='RQ.SRS-006.RBAC.Select.Column', level=3, num='5.19.2'), - Heading(name='RQ.SRS-006.RBAC.Select.Cluster', level=3, num='5.19.3'), - Heading(name='RQ.SRS-006.RBAC.Select.TableEngines', level=3, num='5.19.4'), - Heading(name='Insert', level=2, num='5.20'), - Heading(name='RQ.SRS-006.RBAC.Insert', level=3, num='5.20.1'), - Heading(name='RQ.SRS-006.RBAC.Insert.Column', level=3, num='5.20.2'), - Heading(name='RQ.SRS-006.RBAC.Insert.Cluster', level=3, num='5.20.3'), - Heading(name='RQ.SRS-006.RBAC.Insert.TableEngines', level=3, num='5.20.4'), - Heading(name='Alter', level=2, num='5.21'), - Heading(name='Alter Column', level=3, num='5.21.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterColumn', level=4, num='5.21.1.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Grant', level=4, num='5.21.1.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Revoke', level=4, num='5.21.1.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Column', level=4, num='5.21.1.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterColumn.Cluster', level=4, num='5.21.1.5'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines', level=4, num='5.21.1.6'), - Heading(name='Alter Index', level=3, num='5.21.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterIndex', level=4, num='5.21.2.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterIndex.Grant', level=4, num='5.21.2.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterIndex.Revoke', level=4, num='5.21.2.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterIndex.Cluster', level=4, num='5.21.2.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines', level=4, num='5.21.2.5'), - Heading(name='Alter Constraint', level=3, num='5.21.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterConstraint', level=4, num='5.21.3.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.Grant', level=4, num='5.21.3.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.Revoke', level=4, num='5.21.3.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.Cluster', level=4, num='5.21.3.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines', level=4, num='5.21.3.5'), - Heading(name='Alter TTL', level=3, num='5.21.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterTTL', level=4, num='5.21.4.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterTTL.Grant', level=4, num='5.21.4.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterTTL.Revoke', level=4, num='5.21.4.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterTTL.Cluster', level=4, num='5.21.4.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterTTL.TableEngines', level=4, num='5.21.4.5'), - Heading(name='Alter Settings', level=3, num='5.21.5'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterSettings', level=4, num='5.21.5.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterSettings.Grant', level=4, num='5.21.5.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterSettings.Revoke', level=4, num='5.21.5.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterSettings.Cluster', level=4, num='5.21.5.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterSettings.TableEngines', level=4, num='5.21.5.5'), - Heading(name='Alter Update', level=3, num='5.21.6'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterUpdate', level=4, num='5.21.6.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterUpdate.Grant', level=4, num='5.21.6.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterUpdate.Revoke', level=4, num='5.21.6.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterUpdate.TableEngines', level=4, num='5.21.6.4'), - Heading(name='Alter Delete', level=3, num='5.21.7'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterDelete', level=4, num='5.21.7.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterDelete.Grant', level=4, num='5.21.7.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterDelete.Revoke', level=4, num='5.21.7.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterDelete.TableEngines', level=4, num='5.21.7.4'), - Heading(name='Alter Freeze Partition', level=3, num='5.21.8'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFreeze', level=4, num='5.21.8.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFreeze.Grant', level=4, num='5.21.8.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFreeze.Revoke', level=4, num='5.21.8.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFreeze.TableEngines', level=4, num='5.21.8.4'), - Heading(name='Alter Fetch Partition', level=3, num='5.21.9'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFetch', level=4, num='5.21.9.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFetch.Grant', level=4, num='5.21.9.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFetch.Revoke', level=4, num='5.21.9.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterFetch.TableEngines', level=4, num='5.21.9.4'), - Heading(name='Alter Move Partition', level=3, num='5.21.10'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterMove', level=4, num='5.21.10.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterMove.Grant', level=4, num='5.21.10.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterMove.Revoke', level=4, num='5.21.10.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterMove.TableEngines', level=4, num='5.21.10.4'), - Heading(name='Create', level=2, num='5.22'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateTable', level=3, num='5.22.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateDatabase', level=3, num='5.22.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateDictionary', level=3, num='5.22.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable', level=3, num='5.22.4'), - Heading(name='Attach', level=2, num='5.23'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AttachDatabase', level=3, num='5.23.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AttachDictionary', level=3, num='5.23.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable', level=3, num='5.23.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AttachTable', level=3, num='5.23.4'), - Heading(name='Drop', level=2, num='5.24'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropTable', level=3, num='5.24.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropDatabase', level=3, num='5.24.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropDictionary', level=3, num='5.24.3'), - Heading(name='Detach', level=2, num='5.25'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DetachTable', level=3, num='5.25.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DetachView', level=3, num='5.25.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DetachDatabase', level=3, num='5.25.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DetachDictionary', level=3, num='5.25.4'), - Heading(name='Truncate', level=2, num='5.26'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Truncate', level=3, num='5.26.1'), - Heading(name='Optimize', level=2, num='5.27'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Optimize', level=3, num='5.27.1'), - Heading(name='Kill Query', level=2, num='5.28'), - Heading(name='RQ.SRS-006.RBAC.Privileges.KillQuery', level=3, num='5.28.1'), - Heading(name='Kill Mutation', level=2, num='5.29'), - Heading(name='RQ.SRS-006.RBAC.Privileges.KillMutation', level=3, num='5.29.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.KillMutation.AlterUpdate', level=3, num='5.29.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDelete', level=3, num='5.29.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDropColumn', level=3, num='5.29.4'), - Heading(name='Show', level=2, num='5.30'), - Heading(name='RQ.SRS-006.RBAC.ShowTables.Privilege', level=3, num='5.30.1'), - Heading(name='RQ.SRS-006.RBAC.ShowTables.RequiredPrivilege', level=3, num='5.30.2'), - Heading(name='RQ.SRS-006.RBAC.ExistsTable.RequiredPrivilege', level=3, num='5.30.3'), - Heading(name='RQ.SRS-006.RBAC.CheckTable.RequiredPrivilege', level=3, num='5.30.4'), - Heading(name='RQ.SRS-006.RBAC.ShowDatabases.Privilege', level=3, num='5.30.5'), - Heading(name='RQ.SRS-006.RBAC.ShowDatabases.RequiredPrivilege', level=3, num='5.30.6'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateDatabase.RequiredPrivilege', level=3, num='5.30.7'), - Heading(name='RQ.SRS-006.RBAC.UseDatabase.RequiredPrivilege', level=3, num='5.30.8'), - Heading(name='RQ.SRS-006.RBAC.ShowColumns.Privilege', level=3, num='5.30.9'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateTable.RequiredPrivilege', level=3, num='5.30.10'), - Heading(name='RQ.SRS-006.RBAC.DescribeTable.RequiredPrivilege', level=3, num='5.30.11'), - Heading(name='RQ.SRS-006.RBAC.ShowDictionaries.Privilege', level=3, num='5.30.12'), - Heading(name='RQ.SRS-006.RBAC.ShowDictionaries.RequiredPrivilege', level=3, num='5.30.13'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateDictionary.RequiredPrivilege', level=3, num='5.30.14'), - Heading(name='RQ.SRS-006.RBAC.ExistsDictionary.RequiredPrivilege', level=3, num='5.30.15'), - Heading(name='Access Management', level=2, num='5.31'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateUser', level=3, num='5.31.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateUser.DefaultRole', level=3, num='5.31.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterUser', level=3, num='5.31.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropUser', level=3, num='5.31.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateRole', level=3, num='5.31.5'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterRole', level=3, num='5.31.6'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropRole', level=3, num='5.31.7'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateRowPolicy', level=3, num='5.31.8'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterRowPolicy', level=3, num='5.31.9'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropRowPolicy', level=3, num='5.31.10'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateQuota', level=3, num='5.31.11'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterQuota', level=3, num='5.31.12'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropQuota', level=3, num='5.31.13'), - Heading(name='RQ.SRS-006.RBAC.Privileges.CreateSettingsProfile', level=3, num='5.31.14'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AlterSettingsProfile', level=3, num='5.31.15'), - Heading(name='RQ.SRS-006.RBAC.Privileges.DropSettingsProfile', level=3, num='5.31.16'), - Heading(name='RQ.SRS-006.RBAC.Privileges.RoleAdmin', level=3, num='5.31.17'), - Heading(name='Show Access', level=3, num='5.31.18'), - Heading(name='RQ.SRS-006.RBAC.ShowUsers.Privilege', level=4, num='5.31.18.1'), - Heading(name='RQ.SRS-006.RBAC.ShowUsers.RequiredPrivilege', level=4, num='5.31.18.2'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateUser.RequiredPrivilege', level=4, num='5.31.18.3'), - Heading(name='RQ.SRS-006.RBAC.ShowRoles.Privilege', level=4, num='5.31.18.4'), - Heading(name='RQ.SRS-006.RBAC.ShowRoles.RequiredPrivilege', level=4, num='5.31.18.5'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateRole.RequiredPrivilege', level=4, num='5.31.18.6'), - Heading(name='RQ.SRS-006.RBAC.ShowRowPolicies.Privilege', level=4, num='5.31.18.7'), - Heading(name='RQ.SRS-006.RBAC.ShowRowPolicies.RequiredPrivilege', level=4, num='5.31.18.8'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateRowPolicy.RequiredPrivilege', level=4, num='5.31.18.9'), - Heading(name='RQ.SRS-006.RBAC.ShowQuotas.Privilege', level=4, num='5.31.18.10'), - Heading(name='RQ.SRS-006.RBAC.ShowQuotas.RequiredPrivilege', level=4, num='5.31.18.11'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateQuota.RequiredPrivilege', level=4, num='5.31.18.12'), - Heading(name='RQ.SRS-006.RBAC.ShowSettingsProfiles.Privilege', level=4, num='5.31.18.13'), - Heading(name='RQ.SRS-006.RBAC.ShowSettingsProfiles.RequiredPrivilege', level=4, num='5.31.18.14'), - Heading(name='RQ.SRS-006.RBAC.ShowCreateSettingsProfile.RequiredPrivilege', level=4, num='5.31.18.15'), - Heading(name='dictGet', level=2, num='5.32'), - Heading(name='RQ.SRS-006.RBAC.dictGet.Privilege', level=3, num='5.32.1'), - Heading(name='RQ.SRS-006.RBAC.dictGet.RequiredPrivilege', level=3, num='5.32.2'), - Heading(name='RQ.SRS-006.RBAC.dictGet.Type.RequiredPrivilege', level=3, num='5.32.3'), - Heading(name='RQ.SRS-006.RBAC.dictGet.OrDefault.RequiredPrivilege', level=3, num='5.32.4'), - Heading(name='RQ.SRS-006.RBAC.dictHas.RequiredPrivilege', level=3, num='5.32.5'), - Heading(name='RQ.SRS-006.RBAC.dictGetHierarchy.RequiredPrivilege', level=3, num='5.32.6'), - Heading(name='RQ.SRS-006.RBAC.dictIsIn.RequiredPrivilege', level=3, num='5.32.7'), - Heading(name='Introspection', level=2, num='5.33'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Introspection', level=3, num='5.33.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Introspection.addressToLine', level=3, num='5.33.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Introspection.addressToSymbol', level=3, num='5.33.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Introspection.demangle', level=3, num='5.33.4'), - Heading(name='System', level=2, num='5.34'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Shutdown', level=3, num='5.34.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.DropCache', level=3, num='5.34.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.DropCache.DNS', level=3, num='5.34.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.DropCache.Mark', level=3, num='5.34.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.DropCache.Uncompressed', level=3, num='5.34.5'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Reload', level=3, num='5.34.6'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Reload.Config', level=3, num='5.34.7'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionary', level=3, num='5.34.8'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionaries', level=3, num='5.34.9'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Reload.EmbeddedDictionaries', level=3, num='5.34.10'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Merges', level=3, num='5.34.11'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.TTLMerges', level=3, num='5.34.12'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Fetches', level=3, num='5.34.13'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Moves', level=3, num='5.34.14'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Sends', level=3, num='5.34.15'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Sends.Distributed', level=3, num='5.34.16'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Sends.Replicated', level=3, num='5.34.17'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.ReplicationQueues', level=3, num='5.34.18'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.SyncReplica', level=3, num='5.34.19'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.RestartReplica', level=3, num='5.34.20'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Flush', level=3, num='5.34.21'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Flush.Distributed', level=3, num='5.34.22'), - Heading(name='RQ.SRS-006.RBAC.Privileges.System.Flush.Logs', level=3, num='5.34.23'), - Heading(name='Sources', level=2, num='5.35'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources', level=3, num='5.35.1'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.File', level=3, num='5.35.2'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.URL', level=3, num='5.35.3'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.Remote', level=3, num='5.35.4'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.MySQL', level=3, num='5.35.5'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.ODBC', level=3, num='5.35.6'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.JDBC', level=3, num='5.35.7'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.HDFS', level=3, num='5.35.8'), - Heading(name='RQ.SRS-006.RBAC.Privileges.Sources.S3', level=3, num='5.35.9'), - Heading(name='RQ.SRS-006.RBAC.Privileges.GrantOption', level=2, num='5.36'), - Heading(name='RQ.SRS-006.RBAC.Privileges.All', level=2, num='5.37'), - Heading(name='RQ.SRS-006.RBAC.Privileges.RoleAll', level=2, num='5.38'), - Heading(name='RQ.SRS-006.RBAC.Privileges.None', level=2, num='5.39'), - Heading(name='RQ.SRS-006.RBAC.Privileges.AdminOption', level=2, num='5.40'), - Heading(name='References', level=1, num='6'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Terminology", level=1, num="3"), + Heading(name="Privilege Definitions", level=1, num="4"), + Heading(name="Requirements", level=1, num="5"), + Heading(name="Generic", level=2, num="5.1"), + Heading(name="RQ.SRS-006.RBAC", level=3, num="5.1.1"), + Heading(name="Login", level=2, num="5.2"), + Heading(name="RQ.SRS-006.RBAC.Login", level=3, num="5.2.1"), + Heading(name="RQ.SRS-006.RBAC.Login.DefaultUser", level=3, num="5.2.2"), + Heading(name="User", level=2, num="5.3"), + Heading(name="RQ.SRS-006.RBAC.User", level=3, num="5.3.1"), + Heading(name="RQ.SRS-006.RBAC.User.Roles", level=3, num="5.3.2"), + Heading(name="RQ.SRS-006.RBAC.User.Privileges", level=3, num="5.3.3"), + Heading(name="RQ.SRS-006.RBAC.User.Variables", level=3, num="5.3.4"), + Heading( + name="RQ.SRS-006.RBAC.User.Variables.Constraints", level=3, num="5.3.5" ), + Heading(name="RQ.SRS-006.RBAC.User.SettingsProfile", level=3, num="5.3.6"), + Heading(name="RQ.SRS-006.RBAC.User.Quotas", level=3, num="5.3.7"), + Heading(name="RQ.SRS-006.RBAC.User.RowPolicies", level=3, num="5.3.8"), + Heading(name="RQ.SRS-006.RBAC.User.DefaultRole", level=3, num="5.3.9"), + Heading(name="RQ.SRS-006.RBAC.User.RoleSelection", level=3, num="5.3.10"), + Heading(name="RQ.SRS-006.RBAC.User.ShowCreate", level=3, num="5.3.11"), + Heading(name="RQ.SRS-006.RBAC.User.ShowPrivileges", level=3, num="5.3.12"), + Heading(name="RQ.SRS-006.RBAC.User.Use.DefaultRole", level=3, num="5.3.13"), + Heading( + name="RQ.SRS-006.RBAC.User.Use.AllRolesWhenNoDefaultRole", + level=3, + num="5.3.14", + ), + Heading(name="Create User", level=3, num="5.3.15"), + Heading(name="RQ.SRS-006.RBAC.User.Create", level=4, num="5.3.15.1"), + Heading( + name="RQ.SRS-006.RBAC.User.Create.IfNotExists", level=4, num="5.3.15.2" + ), + Heading(name="RQ.SRS-006.RBAC.User.Create.Replace", level=4, num="5.3.15.3"), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.NoPassword", + level=4, + num="5.3.15.4", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.NoPassword.Login", + level=4, + num="5.3.15.5", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.PlainText", + level=4, + num="5.3.15.6", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.PlainText.Login", + level=4, + num="5.3.15.7", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Password", + level=4, + num="5.3.15.8", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Password.Login", + level=4, + num="5.3.15.9", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash", + level=4, + num="5.3.15.10", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.Sha256Hash.Login", + level=4, + num="5.3.15.11", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password", + level=4, + num="5.3.15.12", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Password.Login", + level=4, + num="5.3.15.13", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash", + level=4, + num="5.3.15.14", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Password.DoubleSha1Hash.Login", + level=4, + num="5.3.15.15", + ), + Heading(name="RQ.SRS-006.RBAC.User.Create.Host.Name", level=4, num="5.3.15.16"), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Host.Regexp", level=4, num="5.3.15.17" + ), + Heading(name="RQ.SRS-006.RBAC.User.Create.Host.IP", level=4, num="5.3.15.18"), + Heading(name="RQ.SRS-006.RBAC.User.Create.Host.Any", level=4, num="5.3.15.19"), + Heading(name="RQ.SRS-006.RBAC.User.Create.Host.None", level=4, num="5.3.15.20"), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Host.Local", level=4, num="5.3.15.21" + ), + Heading(name="RQ.SRS-006.RBAC.User.Create.Host.Like", level=4, num="5.3.15.22"), + Heading( + name="RQ.SRS-006.RBAC.User.Create.Host.Default", level=4, num="5.3.15.23" + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.DefaultRole", level=4, num="5.3.15.24" + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.DefaultRole.None", + level=4, + num="5.3.15.25", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Create.DefaultRole.All", level=4, num="5.3.15.26" + ), + Heading(name="RQ.SRS-006.RBAC.User.Create.Settings", level=4, num="5.3.15.27"), + Heading(name="RQ.SRS-006.RBAC.User.Create.OnCluster", level=4, num="5.3.15.28"), + Heading(name="RQ.SRS-006.RBAC.User.Create.Syntax", level=4, num="5.3.15.29"), + Heading(name="Alter User", level=3, num="5.3.16"), + Heading(name="RQ.SRS-006.RBAC.User.Alter", level=4, num="5.3.16.1"), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.OrderOfEvaluation", level=4, num="5.3.16.2" + ), + Heading(name="RQ.SRS-006.RBAC.User.Alter.IfExists", level=4, num="5.3.16.3"), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Cluster", level=4, num="5.3.16.4"), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Rename", level=4, num="5.3.16.5"), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Password.PlainText", + level=4, + num="5.3.16.6", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Password.Sha256Password", + level=4, + num="5.3.16.7", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Password.DoubleSha1Password", + level=4, + num="5.3.16.8", + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Host.AddDrop", level=4, num="5.3.16.9" + ), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Host.Local", level=4, num="5.3.16.10"), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Host.Name", level=4, num="5.3.16.11"), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Host.Regexp", level=4, num="5.3.16.12" + ), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Host.IP", level=4, num="5.3.16.13"), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Host.Like", level=4, num="5.3.16.14"), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Host.Any", level=4, num="5.3.16.15"), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Host.None", level=4, num="5.3.16.16"), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.DefaultRole", level=4, num="5.3.16.17" + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.DefaultRole.All", level=4, num="5.3.16.18" + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.DefaultRole.AllExcept", + level=4, + num="5.3.16.19", + ), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Settings", level=4, num="5.3.16.20"), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Settings.Min", level=4, num="5.3.16.21" + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Settings.Max", level=4, num="5.3.16.22" + ), + Heading( + name="RQ.SRS-006.RBAC.User.Alter.Settings.Profile", level=4, num="5.3.16.23" + ), + Heading(name="RQ.SRS-006.RBAC.User.Alter.Syntax", level=4, num="5.3.16.24"), + Heading(name="Show Create User", level=3, num="5.3.17"), + Heading(name="RQ.SRS-006.RBAC.User.ShowCreateUser", level=4, num="5.3.17.1"), + Heading( + name="RQ.SRS-006.RBAC.User.ShowCreateUser.For", level=4, num="5.3.17.2" + ), + Heading( + name="RQ.SRS-006.RBAC.User.ShowCreateUser.Syntax", level=4, num="5.3.17.3" + ), + Heading(name="Drop User", level=3, num="5.3.18"), + Heading(name="RQ.SRS-006.RBAC.User.Drop", level=4, num="5.3.18.1"), + Heading(name="RQ.SRS-006.RBAC.User.Drop.IfExists", level=4, num="5.3.18.2"), + Heading(name="RQ.SRS-006.RBAC.User.Drop.OnCluster", level=4, num="5.3.18.3"), + Heading(name="RQ.SRS-006.RBAC.User.Drop.Syntax", level=4, num="5.3.18.4"), + Heading(name="Role", level=2, num="5.4"), + Heading(name="RQ.SRS-006.RBAC.Role", level=3, num="5.4.1"), + Heading(name="RQ.SRS-006.RBAC.Role.Privileges", level=3, num="5.4.2"), + Heading(name="RQ.SRS-006.RBAC.Role.Variables", level=3, num="5.4.3"), + Heading(name="RQ.SRS-006.RBAC.Role.SettingsProfile", level=3, num="5.4.4"), + Heading(name="RQ.SRS-006.RBAC.Role.Quotas", level=3, num="5.4.5"), + Heading(name="RQ.SRS-006.RBAC.Role.RowPolicies", level=3, num="5.4.6"), + Heading(name="Create Role", level=3, num="5.4.7"), + Heading(name="RQ.SRS-006.RBAC.Role.Create", level=4, num="5.4.7.1"), + Heading(name="RQ.SRS-006.RBAC.Role.Create.IfNotExists", level=4, num="5.4.7.2"), + Heading(name="RQ.SRS-006.RBAC.Role.Create.Replace", level=4, num="5.4.7.3"), + Heading(name="RQ.SRS-006.RBAC.Role.Create.Settings", level=4, num="5.4.7.4"), + Heading(name="RQ.SRS-006.RBAC.Role.Create.Syntax", level=4, num="5.4.7.5"), + Heading(name="Alter Role", level=3, num="5.4.8"), + Heading(name="RQ.SRS-006.RBAC.Role.Alter", level=4, num="5.4.8.1"), + Heading(name="RQ.SRS-006.RBAC.Role.Alter.IfExists", level=4, num="5.4.8.2"), + Heading(name="RQ.SRS-006.RBAC.Role.Alter.Cluster", level=4, num="5.4.8.3"), + Heading(name="RQ.SRS-006.RBAC.Role.Alter.Rename", level=4, num="5.4.8.4"), + Heading(name="RQ.SRS-006.RBAC.Role.Alter.Settings", level=4, num="5.4.8.5"), + Heading(name="RQ.SRS-006.RBAC.Role.Alter.Syntax", level=4, num="5.4.8.6"), + Heading(name="Drop Role", level=3, num="5.4.9"), + Heading(name="RQ.SRS-006.RBAC.Role.Drop", level=4, num="5.4.9.1"), + Heading(name="RQ.SRS-006.RBAC.Role.Drop.IfExists", level=4, num="5.4.9.2"), + Heading(name="RQ.SRS-006.RBAC.Role.Drop.Cluster", level=4, num="5.4.9.3"), + Heading(name="RQ.SRS-006.RBAC.Role.Drop.Syntax", level=4, num="5.4.9.4"), + Heading(name="Show Create Role", level=3, num="5.4.10"), + Heading(name="RQ.SRS-006.RBAC.Role.ShowCreate", level=4, num="5.4.10.1"), + Heading(name="RQ.SRS-006.RBAC.Role.ShowCreate.Syntax", level=4, num="5.4.10.2"), + Heading(name="Partial Revokes", level=2, num="5.5"), + Heading(name="RQ.SRS-006.RBAC.PartialRevokes", level=3, num="5.5.1"), + Heading(name="RQ.SRS-006.RBAC.PartialRevoke.Syntax", level=3, num="5.5.2"), + Heading(name="Settings Profile", level=2, num="5.6"), + Heading(name="RQ.SRS-006.RBAC.SettingsProfile", level=3, num="5.6.1"), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Constraints", level=3, num="5.6.2" + ), + Heading(name="Create Settings Profile", level=3, num="5.6.3"), + Heading(name="RQ.SRS-006.RBAC.SettingsProfile.Create", level=4, num="5.6.3.1"), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.IfNotExists", + level=4, + num="5.6.3.2", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Replace", + level=4, + num="5.6.3.3", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Variables", + level=4, + num="5.6.3.4", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Value", + level=4, + num="5.6.3.5", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Variables.Constraints", + level=4, + num="5.6.3.6", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment", + level=4, + num="5.6.3.7", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.None", + level=4, + num="5.6.3.8", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.All", + level=4, + num="5.6.3.9", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Assignment.AllExcept", + level=4, + num="5.6.3.10", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Inherit", + level=4, + num="5.6.3.11", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.OnCluster", + level=4, + num="5.6.3.12", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Create.Syntax", + level=4, + num="5.6.3.13", + ), + Heading(name="Alter Settings Profile", level=3, num="5.6.4"), + Heading(name="RQ.SRS-006.RBAC.SettingsProfile.Alter", level=4, num="5.6.4.1"), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.IfExists", + level=4, + num="5.6.4.2", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Rename", level=4, num="5.6.4.3" + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables", + level=4, + num="5.6.4.4", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Value", + level=4, + num="5.6.4.5", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Variables.Constraints", + level=4, + num="5.6.4.6", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment", + level=4, + num="5.6.4.7", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.None", + level=4, + num="5.6.4.8", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.All", + level=4, + num="5.6.4.9", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.AllExcept", + level=4, + num="5.6.4.10", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.Inherit", + level=4, + num="5.6.4.11", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Assignment.OnCluster", + level=4, + num="5.6.4.12", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Alter.Syntax", level=4, num="5.6.4.13" + ), + Heading(name="Drop Settings Profile", level=3, num="5.6.5"), + Heading(name="RQ.SRS-006.RBAC.SettingsProfile.Drop", level=4, num="5.6.5.1"), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Drop.IfExists", level=4, num="5.6.5.2" + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Drop.OnCluster", + level=4, + num="5.6.5.3", + ), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.Drop.Syntax", level=4, num="5.6.5.4" + ), + Heading(name="Show Create Settings Profile", level=3, num="5.6.6"), + Heading( + name="RQ.SRS-006.RBAC.SettingsProfile.ShowCreateSettingsProfile", + level=4, + num="5.6.6.1", + ), + Heading(name="Quotas", level=2, num="5.7"), + Heading(name="RQ.SRS-006.RBAC.Quotas", level=3, num="5.7.1"), + Heading(name="RQ.SRS-006.RBAC.Quotas.Keyed", level=3, num="5.7.2"), + Heading(name="RQ.SRS-006.RBAC.Quotas.Queries", level=3, num="5.7.3"), + Heading(name="RQ.SRS-006.RBAC.Quotas.Errors", level=3, num="5.7.4"), + Heading(name="RQ.SRS-006.RBAC.Quotas.ResultRows", level=3, num="5.7.5"), + Heading(name="RQ.SRS-006.RBAC.Quotas.ReadRows", level=3, num="5.7.6"), + Heading(name="RQ.SRS-006.RBAC.Quotas.ResultBytes", level=3, num="5.7.7"), + Heading(name="RQ.SRS-006.RBAC.Quotas.ReadBytes", level=3, num="5.7.8"), + Heading(name="RQ.SRS-006.RBAC.Quotas.ExecutionTime", level=3, num="5.7.9"), + Heading(name="Create Quotas", level=3, num="5.7.10"), + Heading(name="RQ.SRS-006.RBAC.Quota.Create", level=4, num="5.7.10.1"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.IfNotExists", level=4, num="5.7.10.2" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.Replace", level=4, num="5.7.10.3"), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.Cluster", level=4, num="5.7.10.4"), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.Interval", level=4, num="5.7.10.5"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.Interval.Randomized", + level=4, + num="5.7.10.6", + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.Queries", level=4, num="5.7.10.7"), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.Errors", level=4, num="5.7.10.8"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.ResultRows", level=4, num="5.7.10.9" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.ReadRows", level=4, num="5.7.10.10"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.ResultBytes", level=4, num="5.7.10.11" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.ReadBytes", level=4, num="5.7.10.12" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.ExecutionTime", level=4, num="5.7.10.13" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.NoLimits", level=4, num="5.7.10.14"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.TrackingOnly", level=4, num="5.7.10.15" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.KeyedBy", level=4, num="5.7.10.16"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.KeyedByOptions", level=4, num="5.7.10.17" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.Assignment", level=4, num="5.7.10.18" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.Assignment.None", + level=4, + num="5.7.10.19", + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.Assignment.All", level=4, num="5.7.10.20" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Create.Assignment.Except", + level=4, + num="5.7.10.21", + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Create.Syntax", level=4, num="5.7.10.22"), + Heading(name="Alter Quota", level=3, num="5.7.11"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter", level=4, num="5.7.11.1"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.IfExists", level=4, num="5.7.11.2"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.Rename", level=4, num="5.7.11.3"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.Cluster", level=4, num="5.7.11.4"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.Interval", level=4, num="5.7.11.5"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.Interval.Randomized", + level=4, + num="5.7.11.6", + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.Queries", level=4, num="5.7.11.7"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.Errors", level=4, num="5.7.11.8"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.ResultRows", level=4, num="5.7.11.9"), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.ReadRows", level=4, num="5.7.11.10"), + Heading( + name="RQ.SRS-006.RBAC.Quota.ALter.ResultBytes", level=4, num="5.7.11.11" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.ReadBytes", level=4, num="5.7.11.12"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.ExecutionTime", level=4, num="5.7.11.13" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.NoLimits", level=4, num="5.7.11.14"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.TrackingOnly", level=4, num="5.7.11.15" + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.KeyedBy", level=4, num="5.7.11.16"), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.KeyedByOptions", level=4, num="5.7.11.17" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment", level=4, num="5.7.11.18" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment.None", level=4, num="5.7.11.19" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment.All", level=4, num="5.7.11.20" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.Alter.Assignment.Except", + level=4, + num="5.7.11.21", + ), + Heading(name="RQ.SRS-006.RBAC.Quota.Alter.Syntax", level=4, num="5.7.11.22"), + Heading(name="Drop Quota", level=3, num="5.7.12"), + Heading(name="RQ.SRS-006.RBAC.Quota.Drop", level=4, num="5.7.12.1"), + Heading(name="RQ.SRS-006.RBAC.Quota.Drop.IfExists", level=4, num="5.7.12.2"), + Heading(name="RQ.SRS-006.RBAC.Quota.Drop.Cluster", level=4, num="5.7.12.3"), + Heading(name="RQ.SRS-006.RBAC.Quota.Drop.Syntax", level=4, num="5.7.12.4"), + Heading(name="Show Quotas", level=3, num="5.7.13"), + Heading(name="RQ.SRS-006.RBAC.Quota.ShowQuotas", level=4, num="5.7.13.1"), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.IntoOutfile", level=4, num="5.7.13.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.Format", level=4, num="5.7.13.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.Settings", level=4, num="5.7.13.4" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowQuotas.Syntax", level=4, num="5.7.13.5" + ), + Heading(name="Show Create Quota", level=3, num="5.7.14"), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Name", level=4, num="5.7.14.1" + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Current", + level=4, + num="5.7.14.2", + ), + Heading( + name="RQ.SRS-006.RBAC.Quota.ShowCreateQuota.Syntax", level=4, num="5.7.14.3" + ), + Heading(name="Row Policy", level=2, num="5.8"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy", level=3, num="5.8.1"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Condition", level=3, num="5.8.2"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Restriction", level=3, num="5.8.3"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Nesting", level=3, num="5.8.4"), + Heading(name="Create Row Policy", level=3, num="5.8.5"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Create", level=4, num="5.8.5.1"), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.IfNotExists", level=4, num="5.8.5.2" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Replace", level=4, num="5.8.5.3" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.OnCluster", level=4, num="5.8.5.4" + ), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Create.On", level=4, num="5.8.5.5"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Create.Access", level=4, num="5.8.5.6"), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Access.Permissive", + level=4, + num="5.8.5.7", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Access.Restrictive", + level=4, + num="5.8.5.8", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.ForSelect", level=4, num="5.8.5.9" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Condition", level=4, num="5.8.5.10" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment", level=4, num="5.8.5.11" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.None", + level=4, + num="5.8.5.12", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.All", + level=4, + num="5.8.5.13", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Assignment.AllExcept", + level=4, + num="5.8.5.14", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Create.Syntax", level=4, num="5.8.5.15" + ), + Heading(name="Alter Row Policy", level=3, num="5.8.6"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Alter", level=4, num="5.8.6.1"), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.IfExists", level=4, num="5.8.6.2" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.ForSelect", level=4, num="5.8.6.3" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.OnCluster", level=4, num="5.8.6.4" + ), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Alter.On", level=4, num="5.8.6.5"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Alter.Rename", level=4, num="5.8.6.6"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Alter.Access", level=4, num="5.8.6.7"), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Permissive", + level=4, + num="5.8.6.8", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Access.Restrictive", + level=4, + num="5.8.6.9", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Condition", level=4, num="5.8.6.10" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Condition.None", + level=4, + num="5.8.6.11", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment", level=4, num="5.8.6.12" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.None", + level=4, + num="5.8.6.13", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.All", + level=4, + num="5.8.6.14", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Alter.Assignment.AllExcept", + level=4, + num="5.8.6.15", + ), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Alter.Syntax", level=4, num="5.8.6.16"), + Heading(name="Drop Row Policy", level=3, num="5.8.7"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Drop", level=4, num="5.8.7.1"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Drop.IfExists", level=4, num="5.8.7.2"), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Drop.On", level=4, num="5.8.7.3"), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.Drop.OnCluster", level=4, num="5.8.7.4" + ), + Heading(name="RQ.SRS-006.RBAC.RowPolicy.Drop.Syntax", level=4, num="5.8.7.5"), + Heading(name="Show Create Row Policy", level=3, num="5.8.8"), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy", level=4, num="5.8.8.1" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.On", + level=4, + num="5.8.8.2", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.ShowCreateRowPolicy.Syntax", + level=4, + num="5.8.8.3", + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies", level=4, num="5.8.8.4" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.On", level=4, num="5.8.8.5" + ), + Heading( + name="RQ.SRS-006.RBAC.RowPolicy.ShowRowPolicies.Syntax", + level=4, + num="5.8.8.6", + ), + Heading(name="Set Default Role", level=2, num="5.9"), + Heading(name="RQ.SRS-006.RBAC.SetDefaultRole", level=3, num="5.9.1"), + Heading( + name="RQ.SRS-006.RBAC.SetDefaultRole.CurrentUser", level=3, num="5.9.2" + ), + Heading(name="RQ.SRS-006.RBAC.SetDefaultRole.All", level=3, num="5.9.3"), + Heading(name="RQ.SRS-006.RBAC.SetDefaultRole.AllExcept", level=3, num="5.9.4"), + Heading(name="RQ.SRS-006.RBAC.SetDefaultRole.None", level=3, num="5.9.5"), + Heading(name="RQ.SRS-006.RBAC.SetDefaultRole.Syntax", level=3, num="5.9.6"), + Heading(name="Set Role", level=2, num="5.10"), + Heading(name="RQ.SRS-006.RBAC.SetRole", level=3, num="5.10.1"), + Heading(name="RQ.SRS-006.RBAC.SetRole.Default", level=3, num="5.10.2"), + Heading(name="RQ.SRS-006.RBAC.SetRole.None", level=3, num="5.10.3"), + Heading(name="RQ.SRS-006.RBAC.SetRole.All", level=3, num="5.10.4"), + Heading(name="RQ.SRS-006.RBAC.SetRole.AllExcept", level=3, num="5.10.5"), + Heading(name="RQ.SRS-006.RBAC.SetRole.Syntax", level=3, num="5.10.6"), + Heading(name="Grant", level=2, num="5.11"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.To", level=3, num="5.11.1"), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.ToCurrentUser", level=3, num="5.11.2" + ), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Select", level=3, num="5.11.3"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Insert", level=3, num="5.11.4"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Alter", level=3, num="5.11.5"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Create", level=3, num="5.11.6"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Drop", level=3, num="5.11.7"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Truncate", level=3, num="5.11.8"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Optimize", level=3, num="5.11.9"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Show", level=3, num="5.11.10"), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.KillQuery", level=3, num="5.11.11" + ), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.AccessManagement", + level=3, + num="5.11.12", + ), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.System", level=3, num="5.11.13"), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.Introspection", level=3, num="5.11.14" + ), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Sources", level=3, num="5.11.15"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.DictGet", level=3, num="5.11.16"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.None", level=3, num="5.11.17"), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.All", level=3, num="5.11.18"), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.GrantOption", level=3, num="5.11.19" + ), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.On", level=3, num="5.11.20"), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.PrivilegeColumns", + level=3, + num="5.11.21", + ), + Heading( + name="RQ.SRS-006.RBAC.Grant.Privilege.OnCluster", level=3, num="5.11.22" + ), + Heading(name="RQ.SRS-006.RBAC.Grant.Privilege.Syntax", level=3, num="5.11.23"), + Heading(name="Revoke", level=2, num="5.12"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Cluster", level=3, num="5.12.1"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Select", level=3, num="5.12.2"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Insert", level=3, num="5.12.3"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Alter", level=3, num="5.12.4"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Create", level=3, num="5.12.5"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Drop", level=3, num="5.12.6"), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.Truncate", level=3, num="5.12.7" + ), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.Optimize", level=3, num="5.12.8" + ), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Show", level=3, num="5.12.9"), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.KillQuery", level=3, num="5.12.10" + ), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.AccessManagement", + level=3, + num="5.12.11", + ), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.System", level=3, num="5.12.12"), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.Introspection", + level=3, + num="5.12.13", + ), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.Sources", level=3, num="5.12.14" + ), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.DictGet", level=3, num="5.12.15" + ), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.PrivilegeColumns", + level=3, + num="5.12.16", + ), + Heading( + name="RQ.SRS-006.RBAC.Revoke.Privilege.Multiple", level=3, num="5.12.17" + ), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.All", level=3, num="5.12.18"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.None", level=3, num="5.12.19"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.On", level=3, num="5.12.20"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.From", level=3, num="5.12.21"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Privilege.Syntax", level=3, num="5.12.22"), + Heading(name="Grant Role", level=2, num="5.13"), + Heading(name="RQ.SRS-006.RBAC.Grant.Role", level=3, num="5.13.1"), + Heading(name="RQ.SRS-006.RBAC.Grant.Role.CurrentUser", level=3, num="5.13.2"), + Heading(name="RQ.SRS-006.RBAC.Grant.Role.AdminOption", level=3, num="5.13.3"), + Heading(name="RQ.SRS-006.RBAC.Grant.Role.OnCluster", level=3, num="5.13.4"), + Heading(name="RQ.SRS-006.RBAC.Grant.Role.Syntax", level=3, num="5.13.5"), + Heading(name="Revoke Role", level=2, num="5.14"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Role", level=3, num="5.14.1"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Role.Keywords", level=3, num="5.14.2"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Role.Cluster", level=3, num="5.14.3"), + Heading(name="RQ.SRS-006.RBAC.Revoke.AdminOption", level=3, num="5.14.4"), + Heading(name="RQ.SRS-006.RBAC.Revoke.Role.Syntax", level=3, num="5.14.5"), + Heading(name="Show Grants", level=2, num="5.15"), + Heading(name="RQ.SRS-006.RBAC.Show.Grants", level=3, num="5.15.1"), + Heading(name="RQ.SRS-006.RBAC.Show.Grants.For", level=3, num="5.15.2"), + Heading(name="RQ.SRS-006.RBAC.Show.Grants.Syntax", level=3, num="5.15.3"), + Heading(name="Table Privileges", level=2, num="5.16"), + Heading(name="RQ.SRS-006.RBAC.Table.PublicTables", level=3, num="5.16.1"), + Heading(name="RQ.SRS-006.RBAC.Table.SensitiveTables", level=3, num="5.16.2"), + Heading(name="Distributed Tables", level=2, num="5.17"), + Heading(name="RQ.SRS-006.RBAC.DistributedTable.Create", level=3, num="5.17.1"), + Heading(name="RQ.SRS-006.RBAC.DistributedTable.Select", level=3, num="5.17.2"), + Heading(name="RQ.SRS-006.RBAC.DistributedTable.Insert", level=3, num="5.17.3"), + Heading( + name="RQ.SRS-006.RBAC.DistributedTable.SpecialTables", level=3, num="5.17.4" + ), + Heading( + name="RQ.SRS-006.RBAC.DistributedTable.LocalUser", level=3, num="5.17.5" + ), + Heading( + name="RQ.SRS-006.RBAC.DistributedTable.SameUserDifferentNodesDifferentPrivileges", + level=3, + num="5.17.6", + ), + Heading(name="Views", level=2, num="5.18"), + Heading(name="View", level=3, num="5.18.1"), + Heading(name="RQ.SRS-006.RBAC.View", level=4, num="5.18.1.1"), + Heading(name="RQ.SRS-006.RBAC.View.Create", level=4, num="5.18.1.2"), + Heading(name="RQ.SRS-006.RBAC.View.Select", level=4, num="5.18.1.3"), + Heading(name="RQ.SRS-006.RBAC.View.Drop", level=4, num="5.18.1.4"), + Heading(name="Materialized View", level=3, num="5.18.2"), + Heading(name="RQ.SRS-006.RBAC.MaterializedView", level=4, num="5.18.2.1"), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Create", level=4, num="5.18.2.2" + ), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Select", level=4, num="5.18.2.3" + ), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Select.TargetTable", + level=4, + num="5.18.2.4", + ), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Select.SourceTable", + level=4, + num="5.18.2.5", + ), + Heading(name="RQ.SRS-006.RBAC.MaterializedView.Drop", level=4, num="5.18.2.6"), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.ModifyQuery", level=4, num="5.18.2.7" + ), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Insert", level=4, num="5.18.2.8" + ), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Insert.SourceTable", + level=4, + num="5.18.2.9", + ), + Heading( + name="RQ.SRS-006.RBAC.MaterializedView.Insert.TargetTable", + level=4, + num="5.18.2.10", + ), + Heading(name="Live View", level=3, num="5.18.3"), + Heading(name="RQ.SRS-006.RBAC.LiveView", level=4, num="5.18.3.1"), + Heading(name="RQ.SRS-006.RBAC.LiveView.Create", level=4, num="5.18.3.2"), + Heading(name="RQ.SRS-006.RBAC.LiveView.Select", level=4, num="5.18.3.3"), + Heading(name="RQ.SRS-006.RBAC.LiveView.Drop", level=4, num="5.18.3.4"), + Heading(name="RQ.SRS-006.RBAC.LiveView.Refresh", level=4, num="5.18.3.5"), + Heading(name="Select", level=2, num="5.19"), + Heading(name="RQ.SRS-006.RBAC.Select", level=3, num="5.19.1"), + Heading(name="RQ.SRS-006.RBAC.Select.Column", level=3, num="5.19.2"), + Heading(name="RQ.SRS-006.RBAC.Select.Cluster", level=3, num="5.19.3"), + Heading(name="RQ.SRS-006.RBAC.Select.TableEngines", level=3, num="5.19.4"), + Heading(name="Insert", level=2, num="5.20"), + Heading(name="RQ.SRS-006.RBAC.Insert", level=3, num="5.20.1"), + Heading(name="RQ.SRS-006.RBAC.Insert.Column", level=3, num="5.20.2"), + Heading(name="RQ.SRS-006.RBAC.Insert.Cluster", level=3, num="5.20.3"), + Heading(name="RQ.SRS-006.RBAC.Insert.TableEngines", level=3, num="5.20.4"), + Heading(name="Alter", level=2, num="5.21"), + Heading(name="Alter Column", level=3, num="5.21.1"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterColumn", level=4, num="5.21.1.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Grant", level=4, num="5.21.1.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Revoke", + level=4, + num="5.21.1.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Column", + level=4, + num="5.21.1.4", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.Cluster", + level=4, + num="5.21.1.5", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterColumn.TableEngines", + level=4, + num="5.21.1.6", + ), + Heading(name="Alter Index", level=3, num="5.21.2"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterIndex", level=4, num="5.21.2.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.Grant", level=4, num="5.21.2.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.Revoke", level=4, num="5.21.2.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.Cluster", + level=4, + num="5.21.2.4", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterIndex.TableEngines", + level=4, + num="5.21.2.5", + ), + Heading(name="Alter Constraint", level=3, num="5.21.3"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint", level=4, num="5.21.3.1" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.Grant", + level=4, + num="5.21.3.2", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.Revoke", + level=4, + num="5.21.3.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.Cluster", + level=4, + num="5.21.3.4", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterConstraint.TableEngines", + level=4, + num="5.21.3.5", + ), + Heading(name="Alter TTL", level=3, num="5.21.4"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterTTL", level=4, num="5.21.4.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.Grant", level=4, num="5.21.4.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.Revoke", level=4, num="5.21.4.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.Cluster", level=4, num="5.21.4.4" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterTTL.TableEngines", + level=4, + num="5.21.4.5", + ), + Heading(name="Alter Settings", level=3, num="5.21.5"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterSettings", level=4, num="5.21.5.1" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.Grant", + level=4, + num="5.21.5.2", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.Revoke", + level=4, + num="5.21.5.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.Cluster", + level=4, + num="5.21.5.4", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterSettings.TableEngines", + level=4, + num="5.21.5.5", + ), + Heading(name="Alter Update", level=3, num="5.21.6"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterUpdate", level=4, num="5.21.6.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate.Grant", level=4, num="5.21.6.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate.Revoke", + level=4, + num="5.21.6.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterUpdate.TableEngines", + level=4, + num="5.21.6.4", + ), + Heading(name="Alter Delete", level=3, num="5.21.7"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterDelete", level=4, num="5.21.7.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterDelete.Grant", level=4, num="5.21.7.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterDelete.Revoke", + level=4, + num="5.21.7.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterDelete.TableEngines", + level=4, + num="5.21.7.4", + ), + Heading(name="Alter Freeze Partition", level=3, num="5.21.8"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterFreeze", level=4, num="5.21.8.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze.Grant", level=4, num="5.21.8.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze.Revoke", + level=4, + num="5.21.8.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterFreeze.TableEngines", + level=4, + num="5.21.8.4", + ), + Heading(name="Alter Fetch Partition", level=3, num="5.21.9"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterFetch", level=4, num="5.21.9.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterFetch.Grant", level=4, num="5.21.9.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterFetch.Revoke", level=4, num="5.21.9.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterFetch.TableEngines", + level=4, + num="5.21.9.4", + ), + Heading(name="Alter Move Partition", level=3, num="5.21.10"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterMove", level=4, num="5.21.10.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterMove.Grant", level=4, num="5.21.10.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterMove.Revoke", level=4, num="5.21.10.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterMove.TableEngines", + level=4, + num="5.21.10.4", + ), + Heading(name="Create", level=2, num="5.22"), + Heading(name="RQ.SRS-006.RBAC.Privileges.CreateTable", level=3, num="5.22.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.CreateDatabase", level=3, num="5.22.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.CreateDictionary", level=3, num="5.22.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.CreateTemporaryTable", + level=3, + num="5.22.4", + ), + Heading(name="Attach", level=2, num="5.23"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AttachDatabase", level=3, num="5.23.1" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AttachDictionary", level=3, num="5.23.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AttachTemporaryTable", + level=3, + num="5.23.3", + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.AttachTable", level=3, num="5.23.4"), + Heading(name="Drop", level=2, num="5.24"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DropTable", level=3, num="5.24.1"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DropDatabase", level=3, num="5.24.2"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.DropDictionary", level=3, num="5.24.3" + ), + Heading(name="Detach", level=2, num="5.25"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DetachTable", level=3, num="5.25.1"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DetachView", level=3, num="5.25.2"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.DetachDatabase", level=3, num="5.25.3" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.DetachDictionary", level=3, num="5.25.4" + ), + Heading(name="Truncate", level=2, num="5.26"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Truncate", level=3, num="5.26.1"), + Heading(name="Optimize", level=2, num="5.27"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Optimize", level=3, num="5.27.1"), + Heading(name="Kill Query", level=2, num="5.28"), + Heading(name="RQ.SRS-006.RBAC.Privileges.KillQuery", level=3, num="5.28.1"), + Heading(name="Kill Mutation", level=2, num="5.29"), + Heading(name="RQ.SRS-006.RBAC.Privileges.KillMutation", level=3, num="5.29.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.KillMutation.AlterUpdate", + level=3, + num="5.29.2", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDelete", + level=3, + num="5.29.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.KillMutation.AlterDropColumn", + level=3, + num="5.29.4", + ), + Heading(name="Show", level=2, num="5.30"), + Heading(name="RQ.SRS-006.RBAC.ShowTables.Privilege", level=3, num="5.30.1"), + Heading( + name="RQ.SRS-006.RBAC.ShowTables.RequiredPrivilege", level=3, num="5.30.2" + ), + Heading( + name="RQ.SRS-006.RBAC.ExistsTable.RequiredPrivilege", level=3, num="5.30.3" + ), + Heading( + name="RQ.SRS-006.RBAC.CheckTable.RequiredPrivilege", level=3, num="5.30.4" + ), + Heading(name="RQ.SRS-006.RBAC.ShowDatabases.Privilege", level=3, num="5.30.5"), + Heading( + name="RQ.SRS-006.RBAC.ShowDatabases.RequiredPrivilege", + level=3, + num="5.30.6", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateDatabase.RequiredPrivilege", + level=3, + num="5.30.7", + ), + Heading( + name="RQ.SRS-006.RBAC.UseDatabase.RequiredPrivilege", level=3, num="5.30.8" + ), + Heading(name="RQ.SRS-006.RBAC.ShowColumns.Privilege", level=3, num="5.30.9"), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateTable.RequiredPrivilege", + level=3, + num="5.30.10", + ), + Heading( + name="RQ.SRS-006.RBAC.DescribeTable.RequiredPrivilege", + level=3, + num="5.30.11", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowDictionaries.Privilege", level=3, num="5.30.12" + ), + Heading( + name="RQ.SRS-006.RBAC.ShowDictionaries.RequiredPrivilege", + level=3, + num="5.30.13", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateDictionary.RequiredPrivilege", + level=3, + num="5.30.14", + ), + Heading( + name="RQ.SRS-006.RBAC.ExistsDictionary.RequiredPrivilege", + level=3, + num="5.30.15", + ), + Heading(name="Access Management", level=2, num="5.31"), + Heading(name="RQ.SRS-006.RBAC.Privileges.CreateUser", level=3, num="5.31.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.CreateUser.DefaultRole", + level=3, + num="5.31.2", + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterUser", level=3, num="5.31.3"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DropUser", level=3, num="5.31.4"), + Heading(name="RQ.SRS-006.RBAC.Privileges.CreateRole", level=3, num="5.31.5"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterRole", level=3, num="5.31.6"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DropRole", level=3, num="5.31.7"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.CreateRowPolicy", level=3, num="5.31.8" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterRowPolicy", level=3, num="5.31.9" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.DropRowPolicy", level=3, num="5.31.10" + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.CreateQuota", level=3, num="5.31.11"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AlterQuota", level=3, num="5.31.12"), + Heading(name="RQ.SRS-006.RBAC.Privileges.DropQuota", level=3, num="5.31.13"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.CreateSettingsProfile", + level=3, + num="5.31.14", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.AlterSettingsProfile", + level=3, + num="5.31.15", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.DropSettingsProfile", + level=3, + num="5.31.16", + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.RoleAdmin", level=3, num="5.31.17"), + Heading(name="Show Access", level=3, num="5.31.18"), + Heading(name="RQ.SRS-006.RBAC.ShowUsers.Privilege", level=4, num="5.31.18.1"), + Heading( + name="RQ.SRS-006.RBAC.ShowUsers.RequiredPrivilege", level=4, num="5.31.18.2" + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateUser.RequiredPrivilege", + level=4, + num="5.31.18.3", + ), + Heading(name="RQ.SRS-006.RBAC.ShowRoles.Privilege", level=4, num="5.31.18.4"), + Heading( + name="RQ.SRS-006.RBAC.ShowRoles.RequiredPrivilege", level=4, num="5.31.18.5" + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateRole.RequiredPrivilege", + level=4, + num="5.31.18.6", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowRowPolicies.Privilege", level=4, num="5.31.18.7" + ), + Heading( + name="RQ.SRS-006.RBAC.ShowRowPolicies.RequiredPrivilege", + level=4, + num="5.31.18.8", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateRowPolicy.RequiredPrivilege", + level=4, + num="5.31.18.9", + ), + Heading(name="RQ.SRS-006.RBAC.ShowQuotas.Privilege", level=4, num="5.31.18.10"), + Heading( + name="RQ.SRS-006.RBAC.ShowQuotas.RequiredPrivilege", + level=4, + num="5.31.18.11", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateQuota.RequiredPrivilege", + level=4, + num="5.31.18.12", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowSettingsProfiles.Privilege", + level=4, + num="5.31.18.13", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowSettingsProfiles.RequiredPrivilege", + level=4, + num="5.31.18.14", + ), + Heading( + name="RQ.SRS-006.RBAC.ShowCreateSettingsProfile.RequiredPrivilege", + level=4, + num="5.31.18.15", + ), + Heading(name="dictGet", level=2, num="5.32"), + Heading(name="RQ.SRS-006.RBAC.dictGet.Privilege", level=3, num="5.32.1"), + Heading( + name="RQ.SRS-006.RBAC.dictGet.RequiredPrivilege", level=3, num="5.32.2" + ), + Heading( + name="RQ.SRS-006.RBAC.dictGet.Type.RequiredPrivilege", level=3, num="5.32.3" + ), + Heading( + name="RQ.SRS-006.RBAC.dictGet.OrDefault.RequiredPrivilege", + level=3, + num="5.32.4", + ), + Heading( + name="RQ.SRS-006.RBAC.dictHas.RequiredPrivilege", level=3, num="5.32.5" + ), + Heading( + name="RQ.SRS-006.RBAC.dictGetHierarchy.RequiredPrivilege", + level=3, + num="5.32.6", + ), + Heading( + name="RQ.SRS-006.RBAC.dictIsIn.RequiredPrivilege", level=3, num="5.32.7" + ), + Heading(name="Introspection", level=2, num="5.33"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Introspection", level=3, num="5.33.1"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.Introspection.addressToLine", + level=3, + num="5.33.2", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.Introspection.addressToSymbol", + level=3, + num="5.33.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.Introspection.demangle", + level=3, + num="5.33.4", + ), + Heading(name="System", level=2, num="5.34"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Shutdown", level=3, num="5.34.1" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.DropCache", level=3, num="5.34.2" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.DropCache.DNS", + level=3, + num="5.34.3", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.DropCache.Mark", + level=3, + num="5.34.4", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.DropCache.Uncompressed", + level=3, + num="5.34.5", + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.System.Reload", level=3, num="5.34.6"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Reload.Config", + level=3, + num="5.34.7", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionary", + level=3, + num="5.34.8", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Reload.Dictionaries", + level=3, + num="5.34.9", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Reload.EmbeddedDictionaries", + level=3, + num="5.34.10", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Merges", level=3, num="5.34.11" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.TTLMerges", level=3, num="5.34.12" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Fetches", level=3, num="5.34.13" + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.System.Moves", level=3, num="5.34.14"), + Heading(name="RQ.SRS-006.RBAC.Privileges.System.Sends", level=3, num="5.34.15"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Sends.Distributed", + level=3, + num="5.34.16", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Sends.Replicated", + level=3, + num="5.34.17", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.ReplicationQueues", + level=3, + num="5.34.18", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.SyncReplica", level=3, num="5.34.19" + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.RestartReplica", + level=3, + num="5.34.20", + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.System.Flush", level=3, num="5.34.21"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Flush.Distributed", + level=3, + num="5.34.22", + ), + Heading( + name="RQ.SRS-006.RBAC.Privileges.System.Flush.Logs", level=3, num="5.34.23" + ), + Heading(name="Sources", level=2, num="5.35"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources", level=3, num="5.35.1"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.File", level=3, num="5.35.2"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.URL", level=3, num="5.35.3"), + Heading( + name="RQ.SRS-006.RBAC.Privileges.Sources.Remote", level=3, num="5.35.4" + ), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.MySQL", level=3, num="5.35.5"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.ODBC", level=3, num="5.35.6"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.JDBC", level=3, num="5.35.7"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.HDFS", level=3, num="5.35.8"), + Heading(name="RQ.SRS-006.RBAC.Privileges.Sources.S3", level=3, num="5.35.9"), + Heading(name="RQ.SRS-006.RBAC.Privileges.GrantOption", level=2, num="5.36"), + Heading(name="RQ.SRS-006.RBAC.Privileges.All", level=2, num="5.37"), + Heading(name="RQ.SRS-006.RBAC.Privileges.RoleAll", level=2, num="5.38"), + Heading(name="RQ.SRS-006.RBAC.Privileges.None", level=2, num="5.39"), + Heading(name="RQ.SRS-006.RBAC.Privileges.AdminOption", level=2, num="5.40"), + Heading(name="References", level=1, num="6"), + ), requirements=( RQ_SRS_006_RBAC, RQ_SRS_006_RBAC_Login, @@ -9977,8 +11280,8 @@ SRS_006_ClickHouse_Role_Based_Access_Control = Specification( RQ_SRS_006_RBAC_Privileges_RoleAll, RQ_SRS_006_RBAC_Privileges_None, RQ_SRS_006_RBAC_Privileges_AdminOption, - ), - content=''' + ), + content=""" # SRS-006 ClickHouse Role Based Access Control # Software Requirements Specification @@ -14498,4 +15801,5 @@ the user has that role with `ADMIN OPTION` privilege. [Git]: https://git-scm.com/ [MySQL]: https://dev.mysql.com/doc/refman/8.0/en/account-management-statements.html [PostgreSQL]: https://www.postgresql.org/docs/12/user-manag.html -''') +""", +) diff --git a/tests/testflows/rbac/tests/privileges/admin_option.py b/tests/testflows/rbac/tests/privileges/admin_option.py index f6115839bf5..467eab0ef4d 100644 --- a/tests/testflows/rbac/tests/privileges/admin_option.py +++ b/tests/testflows/rbac/tests/privileges/admin_option.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to grant role with `ADMIN OPTION` privilege granted directly. - """ + """Check that a user is able to grant role with `ADMIN OPTION` privilege granted directly.""" user_name = f"user_{getuid()}" @@ -19,10 +19,10 @@ def privileges_granted_directly(self, node=None): Suite(test=grant_role)(grant_target_name=user_name, user_name=user_name) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to grant role with `ADMIN OPTION` privilege granted through a role. - """ + """Check that a user is able to grant role with `ADMIN OPTION` privilege granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -37,10 +37,10 @@ def privileges_granted_via_role(self, node=None): Suite(test=grant_role)(grant_target_name=role_name, user_name=user_name) + @TestSuite def grant_role(self, grant_target_name, user_name, node=None): - """Check that user is able to execute to grant roles if and only if they have role with `ADMIN OPTION`. - """ + """Check that user is able to execute to grant roles if and only if they have role with `ADMIN OPTION`.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -59,8 +59,12 @@ def grant_role(self, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't grant a role"): - node.query(f"GRANT {grant_role_name} TO {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {grant_role_name} TO {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("Grant role with privilege"): grant_role_name = f"grant_role_{getuid()}" @@ -69,10 +73,15 @@ def grant_role(self, grant_target_name, user_name, node=None): with user(node, target_user_name), role(node, grant_role_name): with When(f"I grant ADMIN OPTION"): - node.query(f"GRANT {grant_role_name} TO {grant_target_name} WITH ADMIN OPTION") + node.query( + f"GRANT {grant_role_name} TO {grant_target_name} WITH ADMIN OPTION" + ) with Then("I check the user can grant a role"): - node.query(f"GRANT {grant_role_name} TO {target_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"GRANT {grant_role_name} TO {target_user_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("Grant role on cluster"): grant_role_name = f"grant_role_{getuid()}" @@ -86,14 +95,21 @@ def grant_role(self, grant_target_name, user_name, node=None): node.query(f"CREATE USER {target_user_name} ON CLUSTER sharded_cluster") with When("I grant ADMIN OPTION privilege"): - node.query(f"GRANT {grant_role_name} TO {grant_target_name} WITH ADMIN OPTION") + node.query( + f"GRANT {grant_role_name} TO {grant_target_name} WITH ADMIN OPTION" + ) with Then("I check the user can grant a role"): - node.query(f"GRANT {grant_role_name} TO {target_user_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"GRANT {grant_role_name} TO {target_user_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP ROLE IF EXISTS {grant_role_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROLE IF EXISTS {grant_role_name} ON CLUSTER sharded_cluster" + ) with Scenario("Grant role with revoked privilege"): grant_role_name = f"grant_role_{getuid()}" @@ -102,24 +118,30 @@ def grant_role(self, grant_target_name, user_name, node=None): with user(node, target_user_name), role(node, grant_role_name): with When(f"I grant ADMIN OPTION"): - node.query(f"GRANT {grant_role_name} TO {grant_target_name} WITH ADMIN OPTION") + node.query( + f"GRANT {grant_role_name} TO {grant_target_name} WITH ADMIN OPTION" + ) with And(f"I revoke ADMIN OPTION"): node.query(f"REVOKE {grant_role_name} FROM {grant_target_name}") with Then("I check the user cannot grant a role"): - node.query(f"GRANT {grant_role_name} TO {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {grant_role_name} TO {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("admin option") @Requirements( RQ_SRS_006_RBAC_Privileges_AdminOption("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ADMIN OPTION. - """ + """Check the RBAC functionality of ADMIN OPTION.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/all_role.py b/tests/testflows/rbac/tests/privileges/all_role.py index 629848a2746..a246237cb3e 100644 --- a/tests/testflows/rbac/tests/privileges/all_role.py +++ b/tests/testflows/rbac/tests/privileges/all_role.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestScenario def privilege_check(self, node=None): - '''Check that a role named ALL only grants privileges that it already has. - ''' + """Check that a role named ALL only grants privileges that it already has.""" user_name = f"user_{getuid()}" @@ -21,8 +21,9 @@ def privilege_check(self, node=None): node.query(f"GRANT ALL TO {user_name}") with Then("I check the user doesn't have any privileges"): - output = node.query("SHOW TABLES", settings=[("user",user_name)]).output - assert output == '', error() + output = node.query("SHOW TABLES", settings=[("user", user_name)]).output + assert output == "", error() + @TestFeature @Name("all role") @@ -30,8 +31,7 @@ def privilege_check(self, node=None): RQ_SRS_006_RBAC_Privileges_RoleAll("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of the role 'ALL'. - """ + """Check the RBAC functionality of the role 'ALL'.""" self.context.node = self.context.cluster.node(node) Scenario(run=privilege_check, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_column.py b/tests/testflows/rbac/tests/privileges/alter/alter_column.py index 2be20d4e667..05ce47c8852 100755 --- a/tests/testflows/rbac/tests/privileges/alter/alter_column.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_column.py @@ -9,26 +9,27 @@ import rbac.helper.errors as errors from rbac.helper.tables import table_types subprivileges = { - "ADD COLUMN" : 1 << 0, - "CLEAR COLUMN" : 1 << 1, - "MODIFY COLUMN" : 1 << 2, + "ADD COLUMN": 1 << 0, + "CLEAR COLUMN": 1 << 1, + "MODIFY COLUMN": 1 << 2, "RENAME COLUMN": 1 << 3, "COMMENT COLUMN": 1 << 4, "DROP COLUMN": 1 << 5, } aliases = { - "ADD COLUMN" : ["ALTER ADD COLUMN", "ADD COLUMN"], + "ADD COLUMN": ["ALTER ADD COLUMN", "ADD COLUMN"], "CLEAR COLUMN": ["ALTER CLEAR COLUMN", "CLEAR COLUMN"], - "MODIFY COLUMN" : ["ALTER MODIFY COLUMN", "MODIFY COLUMN"], - "RENAME COLUMN" : ["ALTER RENAME COLUMN", "RENAME COLUMN"], + "MODIFY COLUMN": ["ALTER MODIFY COLUMN", "MODIFY COLUMN"], + "RENAME COLUMN": ["ALTER RENAME COLUMN", "RENAME COLUMN"], "COMMENT COLUMN": ["ALTER COMMENT COLUMN", "COMMENT COLUMN"], "DROP COLUMN": ["ALTER DROP COLUMN", "DROP COLUMN"], - "ALTER COLUMN" : ["ALTER COLUMN", "ALL"], #super-privilege + "ALTER COLUMN": ["ALTER COLUMN", "ALL"], # super-privilege } # extra permutation is for 'ALTER COLUMN' super-privilege -permutation_count = (1 << len(subprivileges)) +permutation_count = 1 << len(subprivileges) + def permutations(table_type): """Uses stress flag and table type, returns list of all permutations to run @@ -45,8 +46,13 @@ def permutations(table_type): # "COMMENT COLUMN", "DROP COLUMN", "NONE", "DROP, RENAME, CLEAR", all, and # "ALTER COLUMN" # ] - return [1 << index for index in range(len(subprivileges))] + \ - [0, int('101010', 2), permutation_count-1, permutation_count] + return [1 << index for index in range(len(subprivileges))] + [ + 0, + int("101010", 2), + permutation_count - 1, + permutation_count, + ] + def alter_column_privileges(grants: int): """Takes in an integer, and returns the corresponding set of tests to grant and @@ -59,33 +65,47 @@ def alter_column_privileges(grants: int): # extra iteration for ALTER COLUMN if grants >= permutation_count: - privileges.append(aliases["ALTER COLUMN"][grants-permutation_count]) - elif grants==0: # No privileges + privileges.append(aliases["ALTER COLUMN"][grants - permutation_count]) + elif grants == 0: # No privileges privileges.append("NONE") else: - if (grants & subprivileges["ADD COLUMN"]): - privileges.append(aliases["ADD COLUMN"][grants % len(aliases["ADD COLUMN"])]) - if (grants & subprivileges["CLEAR COLUMN"]): - privileges.append(aliases["CLEAR COLUMN"][grants % len(aliases["CLEAR COLUMN"])]) - if (grants & subprivileges["MODIFY COLUMN"]): - privileges.append(aliases["MODIFY COLUMN"][grants % len(aliases["MODIFY COLUMN"])]) - if (grants & subprivileges["RENAME COLUMN"]): - privileges.append(aliases["RENAME COLUMN"][grants % len(aliases["RENAME COLUMN"])]) - if (grants & subprivileges["COMMENT COLUMN"]): - privileges.append(aliases["COMMENT COLUMN"][grants % len(aliases["COMMENT COLUMN"])]) - if (grants & subprivileges["DROP COLUMN"]): - privileges.append(aliases["DROP COLUMN"][grants % len(aliases["DROP COLUMN"])]) + if grants & subprivileges["ADD COLUMN"]: + privileges.append( + aliases["ADD COLUMN"][grants % len(aliases["ADD COLUMN"])] + ) + if grants & subprivileges["CLEAR COLUMN"]: + privileges.append( + aliases["CLEAR COLUMN"][grants % len(aliases["CLEAR COLUMN"])] + ) + if grants & subprivileges["MODIFY COLUMN"]: + privileges.append( + aliases["MODIFY COLUMN"][grants % len(aliases["MODIFY COLUMN"])] + ) + if grants & subprivileges["RENAME COLUMN"]: + privileges.append( + aliases["RENAME COLUMN"][grants % len(aliases["RENAME COLUMN"])] + ) + if grants & subprivileges["COMMENT COLUMN"]: + privileges.append( + aliases["COMMENT COLUMN"][grants % len(aliases["COMMENT COLUMN"])] + ) + if grants & subprivileges["DROP COLUMN"]: + privileges.append( + aliases["DROP COLUMN"][grants % len(aliases["DROP COLUMN"])] + ) note(f"Testing privileges: {privileges}") - return ', '.join(privileges) + return ", ".join(privileges) + def on_columns(privileges, columns): """For column-based tests. Takes in string output of alter_column_privileges() and adds columns for those privileges. """ - privileges = privileges.split(',') + privileges = privileges.split(",") privileges = [privilege + f"({columns})" for privilege in privileges] - return ', '.join(privileges) + return ", ".join(privileges) + def alter_column_privilege_handler(grants, table, user, node, columns=None): """For all 6 subprivileges, if the privilege is granted: run test to ensure correct behavior, @@ -97,69 +117,87 @@ def alter_column_privilege_handler(grants, table, user, node, columns=None): note(f"GRANTS: {grants}") # testing ALTER COLUMN is the same as testing all subprivileges - if grants > permutation_count-1: - grants = permutation_count-1 + if grants > permutation_count - 1: + grants = permutation_count - 1 # if 'columns' is not passed then one iteration with column = None columns = columns.split(",") if columns != None else [None] for column in columns: # will always run 6 tests per column depending on granted privileges - if (grants & subprivileges["ADD COLUMN"]): + if grants & subprivileges["ADD COLUMN"]: with When("I check add column when privilege is granted"): check_add_column_when_privilege_is_granted(table, user, node, column) else: with When("I check add column when privilege is not granted"): - check_add_column_when_privilege_is_not_granted(table, user, node, column) - if (grants & subprivileges["CLEAR COLUMN"]): + check_add_column_when_privilege_is_not_granted( + table, user, node, column + ) + if grants & subprivileges["CLEAR COLUMN"]: with When("I check clear column when privilege is granted"): check_clear_column_when_privilege_is_granted(table, user, node, column) else: with When("I check clear column when privilege is not granted"): - check_clear_column_when_privilege_is_not_granted(table, user, node, column) - if (grants & subprivileges["MODIFY COLUMN"]): + check_clear_column_when_privilege_is_not_granted( + table, user, node, column + ) + if grants & subprivileges["MODIFY COLUMN"]: with When("I check modify column when privilege is granted"): check_modify_column_when_privilege_is_granted(table, user, node, column) else: with When("I check modify column when privilege is not granted"): - check_modify_column_when_privilege_is_not_granted(table, user, node, column) - if (grants & subprivileges["RENAME COLUMN"]): + check_modify_column_when_privilege_is_not_granted( + table, user, node, column + ) + if grants & subprivileges["RENAME COLUMN"]: with When("I check rename column when privilege is granted"): check_rename_column_when_privilege_is_granted(table, user, node, column) else: with When("I check rename column when privilege is not granted"): - check_rename_column_when_privilege_is_not_granted(table, user, node, column) - if (grants & subprivileges["COMMENT COLUMN"]): + check_rename_column_when_privilege_is_not_granted( + table, user, node, column + ) + if grants & subprivileges["COMMENT COLUMN"]: with When("I check comment column when privilege is granted"): - check_comment_column_when_privilege_is_granted(table, user, node, column) + check_comment_column_when_privilege_is_granted( + table, user, node, column + ) else: with When("I check comment column when privilege is not granted"): - check_comment_column_when_privilege_is_not_granted(table, user, node, column) - if (grants & subprivileges["DROP COLUMN"]): + check_comment_column_when_privilege_is_not_granted( + table, user, node, column + ) + if grants & subprivileges["DROP COLUMN"]: with When("I check drop column when privilege is granted"): check_drop_column_when_privilege_is_granted(table, user, node, column) else: with When("I check drop column when privilege is not granted"): - check_drop_column_when_privilege_is_not_granted(table, user, node, column) + check_drop_column_when_privilege_is_not_granted( + table, user, node, column + ) + def check_add_column_when_privilege_is_granted(table, user, node, column=None): """Ensures ADD COLUMN runs as expected when the privilege is granted to the specified user. """ if column is None: - column = 'add' + column = "add" with Given(f"I add column '{column}'"): - node.query(f"ALTER TABLE {table} ADD COLUMN {column} String", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} ADD COLUMN {column} String", settings=[("user", user)] + ) with Then("I insert data to tree"): - node.query(f"INSERT INTO {table} ({column}) VALUES ('3.4')") #String + node.query(f"INSERT INTO {table} ({column}) VALUES ('3.4')") # String with Then("I verify that the column was successfully added"): - column_data = node.query(f"SELECT {column} FROM {table} FORMAT JSONEachRow").output - column_data_list = column_data.split('\n') - output_rows = [{f"{column}":"3.4"}, {f"{column}":""}] + column_data = node.query( + f"SELECT {column} FROM {table} FORMAT JSONEachRow" + ).output + column_data_list = column_data.split("\n") + output_rows = [{f"{column}": "3.4"}, {f"{column}": ""}] for row in column_data_list: assert json.loads(row) in output_rows, error() @@ -167,12 +205,13 @@ def check_add_column_when_privilege_is_granted(table, user, node, column=None): with Finally(f"I drop column '{column}'"): node.query(f"ALTER TABLE {table} DROP COLUMN {column}") + def check_clear_column_when_privilege_is_granted(table, user, node, column=None): """Ensures CLEAR COLUMN runs as expected when the privilege is granted to the specified user. """ if column is None: - column = 'clear' + column = "clear" with Given(f"I add the column {column}"): node.query(f"ALTER TABLE {table} ADD COLUMN {column} String") @@ -181,25 +220,29 @@ def check_clear_column_when_privilege_is_granted(table, user, node, column=None) node.query(f"INSERT INTO {table} ({column}) VALUES ('ready to be cleared')") with When(f"I clear column '{column}'"): - node.query(f"ALTER TABLE {table} CLEAR COLUMN {column}", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} CLEAR COLUMN {column}", settings=[("user", user)] + ) with Then("I verify that the column was successfully cleared"): - column_data = node.query(f"SELECT {column} FROM {table} FORMAT JSONEachRow").output - column_data_list = column_data.split('\n') + column_data = node.query( + f"SELECT {column} FROM {table} FORMAT JSONEachRow" + ).output + column_data_list = column_data.split("\n") for row in column_data_list: - assert json.loads(row) == {f"{column}":""}, error() + assert json.loads(row) == {f"{column}": ""}, error() with Finally(f"I drop column '{column}'"): node.query(f"ALTER TABLE {table} DROP COLUMN {column}") + def check_modify_column_when_privilege_is_granted(table, user, node, column=None): """Ensures MODIFY COLUMN runs as expected when the privilege is granted to the specified user. """ if column is None: - column = 'modify' + column = "modify" with Given(f"I add the column {column}"): node.query(f"ALTER TABLE {table} ADD COLUMN {column} String DEFAULT '0'") @@ -208,25 +251,36 @@ def check_modify_column_when_privilege_is_granted(table, user, node, column=None node.query(f"INSERT INTO {table} ({column}) VALUES ('3.4')") with When(f"I modify column '{column}' to type Float"): - node.query(f"ALTER TABLE {table} MODIFY COLUMN {column} Float64", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} MODIFY COLUMN {column} Float64", + settings=[("user", user)], + ) with And("I run optimize table to ensure above UPDATE command is done"): node.query(f"OPTIMIZE TABLE {table} FINAL", timeout=900) with Then("I verify that the column type was modified"): - with When(f"I try to insert a String (old type) to column {column}, throws exception"): - exitcode, message = errors.cannot_parse_string_as_float('hello') - node.query(f"INSERT INTO {table} ({column}) VALUES ('hello')", - exitcode=exitcode, message=message) + with When( + f"I try to insert a String (old type) to column {column}, throws exception" + ): + exitcode, message = errors.cannot_parse_string_as_float("hello") + node.query( + f"INSERT INTO {table} ({column}) VALUES ('hello')", + exitcode=exitcode, + message=message, + ) - with And(f"I try to insert float data (correct type) to column {column}, will accept"): + with And( + f"I try to insert float data (correct type) to column {column}, will accept" + ): node.query(f"INSERT INTO {table} ({column}) VALUES (30.01)") with And("I verify that the date was inserted correctly"): - column_data = node.query(f"SELECT {column} FROM {table} FORMAT JSONEachRow").output - column_data_list = column_data.split('\n') - output_rows = [{f"{column}":30.01}, {f"{column}":3.4}, {f"{column}":0}] + column_data = node.query( + f"SELECT {column} FROM {table} FORMAT JSONEachRow" + ).output + column_data_list = column_data.split("\n") + output_rows = [{f"{column}": 30.01}, {f"{column}": 3.4}, {f"{column}": 0}] for row in column_data_list: assert json.loads(row) in output_rows, error() @@ -234,12 +288,13 @@ def check_modify_column_when_privilege_is_granted(table, user, node, column=None with Finally(f"I drop column '{column}'"): node.query(f"ALTER TABLE {table} DROP COLUMN {column}") + def check_rename_column_when_privilege_is_granted(table, user, node, column=None): """Ensures RENAME COLUMN runs as expected when the privilege is granted to the specified user. """ if column is None: - column = 'rename' + column = "rename" new_column = f"{column}_new" @@ -248,31 +303,43 @@ def check_rename_column_when_privilege_is_granted(table, user, node, column=None with And("I get the initial contents of the column"): # could be either str or float depending on MODIFY COLUMN - initial_column_data = node.query(f"SELECT {column} FROM {table} ORDER BY {column}" - " FORMAT JSONEachRow").output + initial_column_data = node.query( + f"SELECT {column} FROM {table} ORDER BY {column}" " FORMAT JSONEachRow" + ).output with When(f"I rename column '{column}' to '{new_column}'"): - node.query(f"ALTER TABLE {table} RENAME COLUMN {column} TO {new_column}", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} RENAME COLUMN {column} TO {new_column}", + settings=[("user", user)], + ) with Then("I verify that the column was successfully renamed"): with When("I verify that the original column does not exist"): exitcode, message = errors.missing_columns(column) - node.query(f"SELECT {column} FROM {table} FORMAT JSONEachRow", - exitcode=exitcode, message=message) + node.query( + f"SELECT {column} FROM {table} FORMAT JSONEachRow", + exitcode=exitcode, + message=message, + ) - with And("I verify that the new column does exist as expected, with same values"): - new_column_data = node.query(f"SELECT {new_column} FROM {table} ORDER BY" - f" {new_column} FORMAT JSONEachRow").output + with And( + "I verify that the new column does exist as expected, with same values" + ): + new_column_data = node.query( + f"SELECT {new_column} FROM {table} ORDER BY" + f" {new_column} FORMAT JSONEachRow" + ).output - if initial_column_data == '': + if initial_column_data == "": assert initial_column_data == new_column_data, error() else: - new_column_data_list = new_column_data.split('\n') - initial_column_data_list = initial_column_data.split('\n') + new_column_data_list = new_column_data.split("\n") + initial_column_data_list = initial_column_data.split("\n") for new, initial in zip(new_column_data_list, initial_column_data_list): - assert json.loads(new)[new_column] == json.loads(initial)[column], error() + assert ( + json.loads(new)[new_column] == json.loads(initial)[column] + ), error() with Finally(f"I use default user to undo rename"): node.query(f"ALTER TABLE {table} RENAME COLUMN {new_column} TO {column}") @@ -280,28 +347,31 @@ def check_rename_column_when_privilege_is_granted(table, user, node, column=None with Finally(f"I drop column '{column}'"): node.query(f"ALTER TABLE {table} DROP COLUMN {column}") -def check_comment_column_when_privilege_is_granted(table, user, node, column='x'): + +def check_comment_column_when_privilege_is_granted(table, user, node, column="x"): """Ensures COMMENT COLUMN runs as expected when the privilege is granted to the specified user. """ if column is None: - column = 'comment' + column = "comment" with Given(f"I add the column {column}"): node.query(f"ALTER TABLE {table} ADD COLUMN {column} String") with And(f"I alter {column} with comment"): - node.query(f"ALTER TABLE {table} COMMENT COLUMN {column} 'This is a comment.'", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} COMMENT COLUMN {column} 'This is a comment.'", + settings=[("user", user)], + ) with Then(f"I verify that the specified comment is present for {column}"): table_data = node.query(f"DESCRIBE TABLE {table} FORMAT JSONEachRow").output - table_data_list = table_data.split('\n') + table_data_list = table_data.split("\n") for row in table_data_list: row = json.loads(row) - if row['name'] == column: - assert row['comment'] == "This is a comment.", error() + if row["name"] == column: + assert row["comment"] == "This is a comment.", error() with Finally(f"I drop column '{column}'"): node.query(f"ALTER TABLE {table} DROP COLUMN {column}") @@ -313,6 +383,7 @@ def check_comment_column_when_privilege_is_granted(table, user, node, column='x' error() + def check_drop_column_when_privilege_is_granted(table, user, node, column=None): """Ensures DROP COLUMN runs as expected when the privilege is granted to the specified user. @@ -324,136 +395,200 @@ def check_drop_column_when_privilege_is_granted(table, user, node, column=None): else: exitcode, message = errors.wrong_column_name("fake_column") - node.query(f"ALTER TABLE {table} DROP COLUMN fake_column", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP COLUMN fake_column", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) if column is None: - column = 'drop' + column = "drop" with Given(f"I add the column {column}"): node.query(f"ALTER TABLE {table} ADD COLUMN {column} String") with Then(f"I drop column {column} which exists"): - node.query(f"ALTER TABLE {table} DROP COLUMN {column}", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} DROP COLUMN {column}", settings=[("user", user)] + ) with And(f"I verify that {column} has been dropped"): exitcode, message = errors.wrong_column_name(column) - node.query(f"ALTER TABLE {table} DROP COLUMN {column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP COLUMN {column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_add_column_when_privilege_is_not_granted(table, user, node, column=None): """Ensures ADD COLUMN errors as expected without the required privilege for the specified user. """ if column is None: - column = 'add' + column = "add" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} ADD COLUMN {column} String", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} ADD COLUMN {column} String", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) with Then("I try to ADD COLUMN"): - node.query(f"ALTER TABLE {table} ADD COLUMN {column} String", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} ADD COLUMN {column} String", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_clear_column_when_privilege_is_not_granted(table, user, node, column=None): """Ensures CLEAR COLUMN errors as expected without the required privilege for the specified user. """ if column is None: - column = 'clear' + column = "clear" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} CLEAR COLUMN {column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} CLEAR COLUMN {column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) with And(f"I grant NONE to the user"): node.query(f"GRANT NONE TO {user}") with Then("I try to CLEAR COLUMN"): - node.query(f"ALTER TABLE {table} CLEAR COLUMN {column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} CLEAR COLUMN {column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_modify_column_when_privilege_is_not_granted(table, user, node, column=None): """Ensures MODIFY COLUMN errors as expected without the required privilege for the specified user. """ if column is None: - column = 'modify' + column = "modify" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY COLUMN {column} String", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MODIFY COLUMN {column} String", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) with And(f"I grant NONE to the user"): node.query(f"GRANT NONE TO {user}") with Then("I try to MODIFY COLUMN"): - node.query(f"ALTER TABLE {table} MODIFY COLUMN {column} String", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MODIFY COLUMN {column} String", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_rename_column_when_privilege_is_not_granted(table, user, node, column=None): """Ensures RENAME COLUMN errors as expected without the required privilege for the specified user. """ if column is None: - column = 'rename' + column = "rename" new_column = f"{column}_new" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} RENAME COLUMN {column} TO {new_column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} RENAME COLUMN {column} TO {new_column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) with And(f"I grant NONE to the user"): node.query(f"GRANT NONE TO {user}") with Then("I try to RENAME COLUMN"): - node.query(f"ALTER TABLE {table} RENAME COLUMN {column} TO {new_column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} RENAME COLUMN {column} TO {new_column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_comment_column_when_privilege_is_not_granted(table, user, node, column=None): """Ensures COMMENT COLUMN errors as expected without the required privilege for the specified user. """ if column is None: - column = 'comment' + column = "comment" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} COMMENT COLUMN {column} 'This is a comment.'", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} COMMENT COLUMN {column} 'This is a comment.'", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) with And(f"I grant NONE to the user"): node.query(f"GRANT NONE TO {user}") with When("I try to COMMENT COLUMN"): - node.query(f"ALTER TABLE {table} COMMENT COLUMN {column} 'This is a comment.'", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} COMMENT COLUMN {column} 'This is a comment.'", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_drop_column_when_privilege_is_not_granted(table, user, node, column=None): """Ensures DROP COLUMN errors as expected without the required privilege for the specified user. """ if column is None: - column = 'drop' + column = "drop" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} DROP COLUMN {column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP COLUMN {column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) with And(f"I grant NONE to the user"): node.query(f"GRANT NONE TO {user}") with Then("I try to DROP COLUMN"): - node.query(f"ALTER TABLE {table} DROP COLUMN {column}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP COLUMN {column}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_some_privileges(self, permutation, table_type, node=None): @@ -475,6 +610,7 @@ def user_with_some_privileges(self, permutation, table_type, node=None): with Then(f"I try to ALTER COLUMN"): alter_column_privilege_handler(permutation, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterColumn_Revoke("1.0"), @@ -502,27 +638,41 @@ def user_with_revoked_privileges(self, permutation, table_type, node=None): # No privileges granted alter_column_privilege_handler(0, table_name, user_name, node) + @TestScenario -@Examples("grant_columns revoke_columns alter_columns_fail", [ - ("t1", "t1", "t2"), - ("t1,t3", "t1", "t2"), - ("t1,t3,t4", "t1,t3,t4", "t2"), -]) +@Examples( + "grant_columns revoke_columns alter_columns_fail", + [ + ("t1", "t1", "t2"), + ("t1,t3", "t1", "t2"), + ("t1,t3,t4", "t1,t3,t4", "t2"), + ], +) def user_with_privileges_on_columns(self, table_type, permutation, node=None): """Passes in examples to user_column_privileges() below to test granting of sub-privileges on columns """ - examples=Examples("grant_columns revoke_columns alter_columns_fail table_type permutation", - [tuple(list(row)+[table_type, permutation]) for row in self.examples]) + examples = Examples( + "grant_columns revoke_columns alter_columns_fail table_type permutation", + [tuple(list(row) + [table_type, permutation]) for row in self.examples], + ) Scenario(test=user_column_privileges, examples=examples)() + @TestOutline @Requirements( RQ_SRS_006_RBAC_Privileges_AlterColumn_Column("1.0"), ) -def user_column_privileges(self, grant_columns, revoke_columns, alter_columns_fail, table_type, - permutation, node=None): +def user_column_privileges( + self, + grant_columns, + revoke_columns, + alter_columns_fail, + table_type, + permutation, + node=None, +): """Check that user is able to alter on granted columns and unable to alter on not granted or revoked columns. """ @@ -538,22 +688,33 @@ def user_column_privileges(self, grant_columns, revoke_columns, alter_columns_fa with When(f"granted={privileges_on_columns}"): with table(node, table_name, table_type), user(node, user_name): with When(f"I grant subprivileges"): - node.query(f"GRANT {privileges_on_columns} ON {table_name} TO {user_name}") + node.query( + f"GRANT {privileges_on_columns} ON {table_name} TO {user_name}" + ) if alter_columns_fail is not None: with When(f"I try to alter on not granted columns, fails"): # Permutation 0: no privileges for any permutation on these columns - alter_column_privilege_handler(0, table_name, user_name, node, columns=alter_columns_fail) + alter_column_privilege_handler( + 0, table_name, user_name, node, columns=alter_columns_fail + ) with Then(f"I try to ALTER COLUMN"): - alter_column_privilege_handler(permutation, table_name, user_name, node, columns=grant_columns) + alter_column_privilege_handler( + permutation, table_name, user_name, node, columns=grant_columns + ) if revoke_columns is not None: with When(f"I revoke alter column privilege for columns"): - node.query(f"REVOKE {privileges_on_columns} ON {table_name} FROM {user_name}") + node.query( + f"REVOKE {privileges_on_columns} ON {table_name} FROM {user_name}" + ) with And("I try to alter revoked columns"): - alter_column_privilege_handler(0, table_name, user_name, node, columns=alter_columns_fail) + alter_column_privilege_handler( + 0, table_name, user_name, node, columns=alter_columns_fail + ) + @TestScenario @Requirements( @@ -572,7 +733,9 @@ def role_with_some_privileges(self, permutation, table_type, node=None): role_name = f"role_{getuid()}" with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with Given("I grant the alter column privilege to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -582,6 +745,7 @@ def role_with_some_privileges(self, permutation, table_type, node=None): with Then(f"I try to ALTER COLUMN"): alter_column_privilege_handler(permutation, table_name, user_name, node) + @TestScenario def user_with_revoked_role(self, permutation, table_type, node=None): """Check that user with a role that has alter column privilege on a table is unable to @@ -596,7 +760,9 @@ def user_with_revoked_role(self, permutation, table_type, node=None): role_name = f"role_{getuid()}" with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with When("I grant privileges to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -610,27 +776,41 @@ def user_with_revoked_role(self, permutation, table_type, node=None): # Permutation 0: no privileges for any permutation on these columns alter_column_privilege_handler(0, table_name, user_name, node) + @TestScenario -@Examples("grant_columns revoke_columns alter_columns_fail", [ - ("t1", "t1", "t2"), - ("t1,t3", "t1", "t2"), - ("t1,t3,t4", "t1,t3,t4", "t2"), -]) +@Examples( + "grant_columns revoke_columns alter_columns_fail", + [ + ("t1", "t1", "t2"), + ("t1,t3", "t1", "t2"), + ("t1,t3,t4", "t1,t3,t4", "t2"), + ], +) def role_with_privileges_on_columns(self, table_type, permutation, node=None): """Passes in examples to role_column_privileges() below to test granting of subprivileges on columns """ - examples=Examples("grant_columns revoke_columns alter_columns_fail table_type permutation", - [tuple(list(row)+[table_type, permutation]) for row in self.examples]) + examples = Examples( + "grant_columns revoke_columns alter_columns_fail table_type permutation", + [tuple(list(row) + [table_type, permutation]) for row in self.examples], + ) Scenario(test=user_column_privileges, examples=examples)() + @TestOutline @Requirements( RQ_SRS_006_RBAC_Privileges_AlterColumn_Column("1.0"), ) -def role_column_privileges(self, grant_columns, revoke_columns, alter_columns_fail, table_type, - permutation, node=None): +def role_column_privileges( + self, + grant_columns, + revoke_columns, + alter_columns_fail, + table_type, + permutation, + node=None, +): """Check that user is able to alter column from granted columns and unable to alter column from not granted or revoked columns. """ @@ -645,9 +825,13 @@ def role_column_privileges(self, grant_columns, revoke_columns, alter_columns_fa privileges_on_columns = on_columns(privileges, grant_columns) with When(f"granted={privileges_on_columns}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with When(f"I grant subprivileges"): - node.query(f"GRANT {privileges_on_columns} ON {table_name} TO {role_name}") + node.query( + f"GRANT {privileges_on_columns} ON {table_name} TO {role_name}" + ) with And("I grant the role to a user"): node.query(f"GRANT {role_name} TO {user_name}") @@ -655,17 +839,26 @@ def role_column_privileges(self, grant_columns, revoke_columns, alter_columns_fa if alter_columns_fail is not None: with When(f"I try to alter on not granted columns, fails"): # Permutation 0: no privileges for any permutation on these columns - alter_column_privilege_handler(0, table_name, user_name, node, columns=alter_columns_fail) + alter_column_privilege_handler( + 0, table_name, user_name, node, columns=alter_columns_fail + ) with Then(f"I try to ALTER COLUMN"): - alter_column_privilege_handler(permutation, table_name, user_name, node, columns=grant_columns) + alter_column_privilege_handler( + permutation, table_name, user_name, node, columns=grant_columns + ) if revoke_columns is not None: with When(f"I revoke alter column privilege for columns"): - node.query(f"REVOKE {privileges_on_columns} ON {table_name} FROM {role_name}") + node.query( + f"REVOKE {privileges_on_columns} ON {table_name} FROM {role_name}" + ) with And("I try to alter failed columns"): - alter_column_privilege_handler(0, table_name, user_name, node, columns=revoke_columns) + alter_column_privilege_handler( + 0, table_name, user_name, node, columns=revoke_columns + ) + @TestScenario @Requirements( @@ -686,48 +879,58 @@ def user_with_privileges_on_cluster(self, permutation, table_type, node=None): with table(node, table_name, table_type): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) with When("I grant alter column privileges on a cluster"): - node.query(f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}" + ) with Then(f"I try to ALTER COLUMN"): - alter_column_privilege_handler(permutation, table_name, user_name, node) + alter_column_privilege_handler( + permutation, table_name, user_name, node + ) finally: with Finally("I drop the user on a cluster"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestSuite def scenario_parallelization(self, table_type, permutation): args = {"table_type": table_type, "permutation": permutation} with Pool(7) as pool: try: for scenario in loads(current_module(), Scenario): - Scenario(test=scenario, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Scenario( + test=scenario, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterColumn("1.0"), RQ_SRS_006_RBAC_Privileges_AlterColumn_TableEngines("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter column") def feature(self, stress=None, node="clickhouse1"): - """Runs test suites above which check correctness over scenarios and permutations. - """ + """Runs test suites above which check correctness over scenarios and permutations.""" self.context.node = self.context.cluster.node(node) if stress is not None: self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue @@ -738,6 +941,11 @@ def feature(self, stress=None, node="clickhouse1"): for permutation in permutations(table_type): privileges = alter_column_privileges(permutation) args = {"table_type": table_type, "permutation": permutation} - Suite(test=scenario_parallelization, name=privileges, parallel=True, executor=pool)(**args) + Suite( + test=scenario_parallelization, + name=privileges, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_constraint.py b/tests/testflows/rbac/tests/privileges/alter/alter_constraint.py index c24109c8052..d1156aadc9f 100755 --- a/tests/testflows/rbac/tests/privileges/alter/alter_constraint.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_constraint.py @@ -9,18 +9,19 @@ import rbac.helper.errors as errors from rbac.helper.tables import table_types subprivileges = { - "ADD CONSTRAINT" : 1 << 0, - "DROP CONSTRAINT" : 1 << 1, + "ADD CONSTRAINT": 1 << 0, + "DROP CONSTRAINT": 1 << 1, } aliases = { - "ADD CONSTRAINT" : ["ALTER ADD CONSTRAINT", "ADD CONSTRAINT"], + "ADD CONSTRAINT": ["ALTER ADD CONSTRAINT", "ADD CONSTRAINT"], "DROP CONSTRAINT": ["ALTER DROP CONSTRAINT", "DROP CONSTRAINT"], - "ALTER CONSTRAINT": ["ALTER CONSTRAINT", "CONSTRAINT", "ALL"] # super-privilege + "ALTER CONSTRAINT": ["ALTER CONSTRAINT", "CONSTRAINT", "ALL"], # super-privilege } # Extra permutation is for 'ALTER CONSTRAINT' super-privilege -permutation_count = (1 << len(subprivileges)) +permutation_count = 1 << len(subprivileges) + def permutations(): """Returns list of all permutations to run. @@ -28,6 +29,7 @@ def permutations(): """ return [*range(permutation_count + len(aliases["ALTER CONSTRAINT"]))] + def alter_constraint_privileges(grants: int): """Takes in an integer, and returns the corresponding set of tests to grant and not grant using the binary string. Each integer corresponds to a unique permutation @@ -38,62 +40,75 @@ def alter_constraint_privileges(grants: int): # Extra iteration for ALTER CONSTRAINT if grants >= permutation_count: - privileges.append(aliases["ALTER CONSTRAINT"][grants-permutation_count]) - elif grants==0: # No privileges + privileges.append(aliases["ALTER CONSTRAINT"][grants - permutation_count]) + elif grants == 0: # No privileges privileges.append("NONE") else: - if (grants & subprivileges["ADD CONSTRAINT"]): - privileges.append(aliases["ADD CONSTRAINT"][grants % len(aliases["ADD CONSTRAINT"])]) - if (grants & subprivileges["DROP CONSTRAINT"]): - privileges.append(aliases["DROP CONSTRAINT"][grants % len(aliases["DROP CONSTRAINT"])]) + if grants & subprivileges["ADD CONSTRAINT"]: + privileges.append( + aliases["ADD CONSTRAINT"][grants % len(aliases["ADD CONSTRAINT"])] + ) + if grants & subprivileges["DROP CONSTRAINT"]: + privileges.append( + aliases["DROP CONSTRAINT"][grants % len(aliases["DROP CONSTRAINT"])] + ) note(f"Testing these privileges: {privileges}") - return ', '.join(privileges) + return ", ".join(privileges) + def alter_constraint_privilege_handler(grants, table, user, node): """For all 2 subprivileges, if the privilege is granted: run test to ensure correct behavior, and if the privilege is not granted, run test to ensure correct behavior there as well """ # Testing ALTER CONSTRAINT and CONSTRAINT is the same as testing all subprivileges - if grants > permutation_count-1: - grants = permutation_count-1 + if grants > permutation_count - 1: + grants = permutation_count - 1 - if (grants & subprivileges["ADD CONSTRAINT"]): + if grants & subprivileges["ADD CONSTRAINT"]: with When("I check add constraint when privilege is granted"): check_add_constraint_when_privilege_is_granted(table, user, node) else: with When("I check add constraint when privilege is not granted"): check_add_constraint_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["DROP CONSTRAINT"]): + if grants & subprivileges["DROP CONSTRAINT"]: with When("I check drop constraint when privilege is granted"): check_drop_constraint_when_privilege_is_granted(table, user, node) else: with When("I check drop constraint when privilege is not granted"): check_drop_constraint_when_privilege_is_not_granted(table, user, node) + def check_add_constraint_when_privilege_is_granted(table, user, node): - """Ensures ADD CONSTRAINT runs as expected when the privilege is granted to the specified user - """ + """Ensures ADD CONSTRAINT runs as expected when the privilege is granted to the specified user""" constraint = "add" with Given(f"I add constraint '{constraint}'"): - node.query(f"ALTER TABLE {table} ADD CONSTRAINT {constraint} CHECK x>5", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} ADD CONSTRAINT {constraint} CHECK x>5", + settings=[("user", user)], + ) with Then("I verify that the constraint is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"CONSTRAINT {constraint} CHECK x > 5" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert f"CONSTRAINT {constraint} CHECK x > 5" in output["statement"], error() with Finally(f"I drop constraint {constraint}"): node.query(f"ALTER TABLE {table} DROP constraint {constraint}") + def check_drop_constraint_when_privilege_is_granted(table, user, node): - """Ensures DROP CONSTRAINT runs as expected when the privilege is granted to the specified user - """ + """Ensures DROP CONSTRAINT runs as expected when the privilege is granted to the specified user""" with But("I try to drop nonexistent constraint, throws exception"): exitcode, message = errors.wrong_constraint_name("fake_constraint") - node.query(f"ALTER TABLE {table} DROP CONSTRAINT fake_constraint", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP CONSTRAINT fake_constraint", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) constraint = "drop" @@ -101,32 +116,47 @@ def check_drop_constraint_when_privilege_is_granted(table, user, node): node.query(f"ALTER TABLE {table} ADD CONSTRAINT {constraint} CHECK x>5") with Then(f"I drop constraint {constraint} which exists"): - node.query(f"ALTER TABLE {table} DROP CONSTRAINT {constraint}", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} DROP CONSTRAINT {constraint}", + settings=[("user", user)], + ) with Then("I verify that the constraint is not in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"CONSTRAINT {constraint} CHECK x > 5" not in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert ( + f"CONSTRAINT {constraint} CHECK x > 5" not in output["statement"] + ), error() + def check_add_constraint_when_privilege_is_not_granted(table, user, node): - """Ensures ADD CONSTRAINT errors as expected without the required privilege for the specified user - """ + """Ensures ADD CONSTRAINT errors as expected without the required privilege for the specified user""" constraint = "add" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} ADD CONSTRAINT {constraint} CHECK x>5", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} ADD CONSTRAINT {constraint} CHECK x>5", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_drop_constraint_when_privilege_is_not_granted(table, user, node): - """Ensures DROP CONSTRAINT errors as expected without the required privilege for the specified user - """ + """Ensures DROP CONSTRAINT errors as expected without the required privilege for the specified user""" constraint = "drop" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} DROP CONSTRAINT {constraint}", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP CONSTRAINT {constraint}", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_some_privileges(self, table_type, node=None): @@ -148,7 +178,10 @@ def user_with_some_privileges(self, table_type, node=None): node.query(f"GRANT {privileges} ON {table_name} TO {user_name}") with Then(f"I try to ALTER CONSTRAINT"): - alter_constraint_privilege_handler(permutation, table_name, user_name, node) + alter_constraint_privilege_handler( + permutation, table_name, user_name, node + ) + @TestScenario @Requirements( @@ -179,6 +212,7 @@ def user_with_revoked_privileges(self, table_type, node=None): # Permutation 0: no privileges alter_constraint_privilege_handler(0, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterConstraint_Grant("1.0"), @@ -198,7 +232,9 @@ def role_with_some_privileges(self, table_type, node=None): privileges = alter_constraint_privileges(permutation) with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with Given("I grant the ALTER CONSTRAINT privilege to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -206,7 +242,10 @@ def role_with_some_privileges(self, table_type, node=None): node.query(f"GRANT {role_name} TO {user_name}") with Then(f"I try to ALTER CONSTRAINT"): - alter_constraint_privilege_handler(permutation, table_name, user_name, node) + alter_constraint_privilege_handler( + permutation, table_name, user_name, node + ) + @TestScenario def user_with_revoked_role(self, table_type, node=None): @@ -224,7 +263,9 @@ def user_with_revoked_role(self, table_type, node=None): privileges = alter_constraint_privileges(permutation) with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with When("I grant privileges to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -238,6 +279,7 @@ def user_with_revoked_role(self, table_type, node=None): # Permutation 0: no privileges for any permutation alter_constraint_privilege_handler(0, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterConstraint_Cluster("1.0"), @@ -259,27 +301,32 @@ def user_with_privileges_on_cluster(self, table_type, node=None): with table(node, table_name, table_type): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) with When("I grant ALTER CONSTRAINT privileges on a cluster"): - node.query(f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}" + ) with Then(f"I try to ALTER CONSTRAINT"): - alter_constraint_privilege_handler(permutation, table_name, user_name, node) + alter_constraint_privilege_handler( + permutation, table_name, user_name, node + ) finally: with Finally("I drop the user on a cluster"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterConstraint("1.0"), RQ_SRS_006_RBAC_Privileges_AlterConstraint_TableEngines("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter constraint") def feature(self, stress=None, node="clickhouse1"): self.context.node = self.context.cluster.node(node) @@ -288,17 +335,22 @@ def feature(self, stress=None, node="clickhouse1"): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue - args = {"table_type" : table_type} + args = {"table_type": table_type} with Example(str(example)): with Pool(5) as pool: try: for scenario in loads(current_module(), Scenario): - Scenario(test=scenario, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Scenario( + test=scenario, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_delete.py b/tests/testflows/rbac/tests/privileges/alter/alter_delete.py index 93d520f91bd..89e55e57ad0 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_delete.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_delete.py @@ -7,6 +7,7 @@ import rbac.helper.errors as errors aliases = {"ALTER DELETE", "DELETE", "ALL"} + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, privilege, node=None): """Check that user is only able to execute ALTER DELETE when they have required privilege, @@ -21,8 +22,16 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute ALTER DELETE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that {user_name} is only able to execute ALTER DELETE with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): with user(node, user_name), role(node, role_name): @@ -30,12 +39,20 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER DELETE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute ALTER DELETE with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, privilege, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege", setup=instrument_clickhouse_server_log): @@ -50,8 +67,12 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to delete columns without privilege"): - node.query(f"ALTER TABLE {table_name} DELETE WHERE 1", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} DELETE WHERE 1", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" @@ -62,9 +83,14 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with Then("I attempt to delete columns"): - node.query(f"ALTER TABLE {table_name} DELETE WHERE 1", settings = [("user", user_name)]) + node.query( + f"ALTER TABLE {table_name} DELETE WHERE 1", + settings=[("user", user_name)], + ) - with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user with revoked privilege", setup=instrument_clickhouse_server_log + ): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): @@ -73,25 +99,29 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with And("I revoke the delete privilege"): - node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}" + ) with Then("I attempt to delete columns"): - node.query(f"ALTER TABLE {table_name} DELETE WHERE 1", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} DELETE WHERE 1", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterDelete("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter delete") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ALTER DELETE. - """ + """Check the RBAC functionality of ALTER DELETE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -100,7 +130,7 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue @@ -108,4 +138,6 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): with Example(str(example)): for alias in aliases: with Suite(alias, test=privilege_granted_directly_or_via_role): - privilege_granted_directly_or_via_role(table_type=table_type, privilege=alias) + privilege_granted_directly_or_via_role( + table_type=table_type, privilege=alias + ) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py b/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py index b4ff0b65fd4..5a30231b9d5 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_fetch.py @@ -7,10 +7,10 @@ import rbac.helper.errors as errors aliases = {"ALTER FETCH PARTITION", "FETCH PARTITION", "ALL"} + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, privilege, node=None): - """Check that user is only able to execute ALTER FETCH PARTITION when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ALTER FETCH PARTITION when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -20,8 +20,16 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute ALTER FETCH PARTITION with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that {user_name} is only able to execute ALTER FETCH PARTITION with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): with user(node, user_name), role(node, role_name): @@ -29,12 +37,20 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER FETCH PARTITION with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute ALTER FETCH PARTITION with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, privilege, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege", setup=instrument_clickhouse_server_log): @@ -49,8 +65,12 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to fetch a partition without privilege"): - node.query(f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/'", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/'", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" @@ -60,10 +80,16 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with Then("I attempt to fetch a partition"): - node.query(f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/'", settings = [("user", user_name)], - exitcode=231, message="DB::Exception: No node") + node.query( + f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/'", + settings=[("user", user_name)], + exitcode=231, + message="DB::Exception: No node", + ) - with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user with revoked privilege", setup=instrument_clickhouse_server_log + ): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): @@ -71,38 +97,47 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with And("I revoke the fetch privilege"): - node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}" + ) with Then("I attempt to fetch a partition"): - node.query(f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/'", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} FETCH PARTITION 1 FROM '/clickhouse/'", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterFetch("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), +) +@Examples( + "table_type", + [ + ("ReplicatedMergeTree-sharded_cluster",), + ("ReplicatedMergeTree-one_shard_cluster",), + ("ReplicatedReplacingMergeTree-sharded_cluster",), + ("ReplicatedReplacingMergeTree-one_shard_cluster",), + ("ReplicatedSummingMergeTree-sharded_cluster",), + ("ReplicatedSummingMergeTree-one_shard_cluster",), + ("ReplicatedAggregatingMergeTree-sharded_cluster",), + ("ReplicatedAggregatingMergeTree-one_shard_cluster",), + ("ReplicatedCollapsingMergeTree-sharded_cluster",), + ("ReplicatedCollapsingMergeTree-one_shard_cluster",), + ("ReplicatedVersionedCollapsingMergeTree-sharded_cluster",), + ("ReplicatedVersionedCollapsingMergeTree-one_shard_cluster",), + ("ReplicatedGraphiteMergeTree-sharded_cluster",), + ("ReplicatedGraphiteMergeTree-one_shard_cluster",), + ], ) -@Examples("table_type",[ - ("ReplicatedMergeTree-sharded_cluster",), - ("ReplicatedMergeTree-one_shard_cluster",), - ("ReplicatedReplacingMergeTree-sharded_cluster",), - ("ReplicatedReplacingMergeTree-one_shard_cluster",), - ("ReplicatedSummingMergeTree-sharded_cluster",), - ("ReplicatedSummingMergeTree-one_shard_cluster",), - ("ReplicatedAggregatingMergeTree-sharded_cluster",), - ("ReplicatedAggregatingMergeTree-one_shard_cluster",), - ("ReplicatedCollapsingMergeTree-sharded_cluster",), - ("ReplicatedCollapsingMergeTree-one_shard_cluster",), - ("ReplicatedVersionedCollapsingMergeTree-sharded_cluster",), - ("ReplicatedVersionedCollapsingMergeTree-one_shard_cluster",), - ("ReplicatedGraphiteMergeTree-sharded_cluster",), - ("ReplicatedGraphiteMergeTree-one_shard_cluster",) -]) @Name("alter fetch") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ALTER FETCH. - """ + """Check the RBAC functionality of ALTER FETCH.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -111,12 +146,17 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example - if table_type != "ReplicatedMergeTree-sharded_cluster" and not self.context.stress: + if ( + table_type != "ReplicatedMergeTree-sharded_cluster" + and not self.context.stress + ): continue with Example(str(example)): for alias in aliases: with Suite(alias, test=privilege_granted_directly_or_via_role): - privilege_granted_directly_or_via_role(table_type=table_type, privilege=alias) + privilege_granted_directly_or_via_role( + table_type=table_type, privilege=alias + ) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_freeze.py b/tests/testflows/rbac/tests/privileges/alter/alter_freeze.py index 775e2be270d..0f0e8ee6ee5 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_freeze.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_freeze.py @@ -7,10 +7,10 @@ import rbac.helper.errors as errors aliases = {"ALTER FREEZE PARTITION", "FREEZE PARTITION", "ALL"} + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, privilege, node=None): - """Check that user is only able to execute ALTER FREEZE PARTITION when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ALTER FREEZE PARTITION when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -19,19 +19,37 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Scenario("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute ALTER FREEZE PARTITION with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that {user_name} is only able to execute ALTER FREEZE PARTITION with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) - with Scenario("user with privilege via role", setup=instrument_clickhouse_server_log): + with Scenario( + "user with privilege via role", setup=instrument_clickhouse_server_log + ): with user(node, user_name), role(node, role_name): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER FREEZE PARTITION with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute ALTER FREEZE PARTITION with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, privilege, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege", setup=instrument_clickhouse_server_log): @@ -46,8 +64,12 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to freeze partitions without privilege"): - node.query(f"ALTER TABLE {table_name} FREEZE", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} FREEZE", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" @@ -58,9 +80,13 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with Then("I attempt to freeze partitions"): - node.query(f"ALTER TABLE {table_name} FREEZE", settings = [("user", user_name)]) + node.query( + f"ALTER TABLE {table_name} FREEZE", settings=[("user", user_name)] + ) - with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user with revoked privilege", setup=instrument_clickhouse_server_log + ): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): @@ -68,25 +94,29 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No with When("I grant the freeze privilege"): node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with And("I revoke the freeze privilege"): - node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}" + ) with Then("I attempt to freeze partitions"): - node.query(f"ALTER TABLE {table_name} FREEZE", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} FREEZE", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterFreeze("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter freeze") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ALTER FREEZE. - """ + """Check the RBAC functionality of ALTER FREEZE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -95,7 +125,7 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue @@ -103,4 +133,6 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): with Example(str(example)): for alias in aliases: with Suite(alias, test=privilege_granted_directly_or_via_role): - privilege_granted_directly_or_via_role(table_type=table_type, privilege=alias) + privilege_granted_directly_or_via_role( + table_type=table_type, privilege=alias + ) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_index.py b/tests/testflows/rbac/tests/privileges/alter/alter_index.py index 9bb1d72a004..d3190948eb7 100755 --- a/tests/testflows/rbac/tests/privileges/alter/alter_index.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_index.py @@ -11,26 +11,27 @@ import rbac.helper.errors as errors from rbac.helper.tables import table_types subprivileges = { - "ORDER BY" : 1 << 0, + "ORDER BY": 1 << 0, "SAMPLE BY": 1 << 1, - "ADD INDEX" : 1 << 2, - "MATERIALIZE INDEX" : 1 << 3, + "ADD INDEX": 1 << 2, + "MATERIALIZE INDEX": 1 << 3, "CLEAR INDEX": 1 << 4, "DROP INDEX": 1 << 5, } aliases = { - "ORDER BY" : ["ALTER ORDER BY", "ALTER MODIFY ORDER BY", "MODIFY ORDER BY"], + "ORDER BY": ["ALTER ORDER BY", "ALTER MODIFY ORDER BY", "MODIFY ORDER BY"], "SAMPLE BY": ["ALTER SAMPLE BY", "ALTER MODIFY SAMPLE BY", "MODIFY SAMPLE BY"], - "ADD INDEX" : ["ALTER ADD INDEX", "ADD INDEX"], - "MATERIALIZE INDEX" : ["ALTER MATERIALIZE INDEX", "MATERIALIZE INDEX"], + "ADD INDEX": ["ALTER ADD INDEX", "ADD INDEX"], + "MATERIALIZE INDEX": ["ALTER MATERIALIZE INDEX", "MATERIALIZE INDEX"], "CLEAR INDEX": ["ALTER CLEAR INDEX", "CLEAR INDEX"], "DROP INDEX": ["ALTER DROP INDEX", "DROP INDEX"], - "ALTER INDEX": ["ALTER INDEX", "INDEX", "ALL"] # super-privilege + "ALTER INDEX": ["ALTER INDEX", "INDEX", "ALL"], # super-privilege } # Extra permutation is for 'ALTER INDEX' super-privilege -permutation_count = (1 << len(subprivileges)) +permutation_count = 1 << len(subprivileges) + def permutations(table_type): """Uses stress flag and table type, returns list of all permutations to run @@ -44,8 +45,14 @@ def permutations(table_type): # *Selected permutations currently stand as [1,2,4,8,16,32,0,42,63,64,65]. # Testing ["ORDER BY", "SAMPLE BY", "ADD INDEX", "MATERIALIZE INDEX", "CLEAR INDEX", # "DROP INDEX", "NONE", {"DROP, MATERIALIZE, SAMPLE BY"}, all, "ALTER INDEX", and "INDEX"] - return [1 << index for index in range(len(subprivileges))] + \ - [0, int('101010', 2), permutation_count-1, permutation_count, permutation_count+1] + return [1 << index for index in range(len(subprivileges))] + [ + 0, + int("101010", 2), + permutation_count - 1, + permutation_count, + permutation_count + 1, + ] + def alter_index_privileges(grants: int): """Takes in an integer, and returns the corresponding set of tests to grant and @@ -57,85 +64,94 @@ def alter_index_privileges(grants: int): # Extra iteration for ALTER INDEX if grants >= permutation_count: - privileges.append(aliases["ALTER INDEX"][grants-permutation_count]) - elif grants==0: # No privileges + privileges.append(aliases["ALTER INDEX"][grants - permutation_count]) + elif grants == 0: # No privileges privileges.append("NONE") else: - if (grants & subprivileges["ORDER BY"]): + if grants & subprivileges["ORDER BY"]: privileges.append(aliases["ORDER BY"][grants % len(aliases["ORDER BY"])]) - if (grants & subprivileges["SAMPLE BY"]): + if grants & subprivileges["SAMPLE BY"]: privileges.append(aliases["SAMPLE BY"][grants % len(aliases["SAMPLE BY"])]) - if (grants & subprivileges["ADD INDEX"]): + if grants & subprivileges["ADD INDEX"]: privileges.append(aliases["ADD INDEX"][grants % len(aliases["ADD INDEX"])]) - if (grants & subprivileges["MATERIALIZE INDEX"]): - privileges.append(aliases["MATERIALIZE INDEX"][grants % len(aliases["MATERIALIZE INDEX"])]) - if (grants & subprivileges["CLEAR INDEX"]): - privileges.append(aliases["CLEAR INDEX"][grants % len(aliases["CLEAR INDEX"])]) - if (grants & subprivileges["DROP INDEX"]): - privileges.append(aliases["DROP INDEX"][grants % len(aliases["DROP INDEX"])]) + if grants & subprivileges["MATERIALIZE INDEX"]: + privileges.append( + aliases["MATERIALIZE INDEX"][grants % len(aliases["MATERIALIZE INDEX"])] + ) + if grants & subprivileges["CLEAR INDEX"]: + privileges.append( + aliases["CLEAR INDEX"][grants % len(aliases["CLEAR INDEX"])] + ) + if grants & subprivileges["DROP INDEX"]: + privileges.append( + aliases["DROP INDEX"][grants % len(aliases["DROP INDEX"])] + ) note(f"Testing these privileges: {privileges}") - return ', '.join(privileges) + return ", ".join(privileges) + def alter_index_privilege_handler(grants, table, user, node): """For all 5 subprivileges, if the privilege is granted: run test to ensure correct behavior, and if the privilege is not granted, run test to ensure correct behavior there as well. """ # Testing ALTER INDEX and INDEX is the same as testing all subprivileges - if grants > permutation_count-1: - grants = permutation_count-1 + if grants > permutation_count - 1: + grants = permutation_count - 1 - if (grants & subprivileges["ORDER BY"]): + if grants & subprivileges["ORDER BY"]: with When("I check order by when privilege is granted"): check_order_by_when_privilege_is_granted(table, user, node) else: with When("I check order by when privilege is not granted"): check_order_by_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["SAMPLE BY"]): + if grants & subprivileges["SAMPLE BY"]: with When("I check sample by when privilege is granted"): check_sample_by_when_privilege_is_granted(table, user, node) else: with When("I check sample by when privilege is not granted"): check_sample_by_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["ADD INDEX"]): + if grants & subprivileges["ADD INDEX"]: with When("I check add index when privilege is granted"): check_add_index_when_privilege_is_granted(table, user, node) else: with When("I check add index when privilege is not granted"): check_add_index_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["MATERIALIZE INDEX"]): + if grants & subprivileges["MATERIALIZE INDEX"]: with When("I check materialize index when privilege is granted"): check_materialize_index_when_privilege_is_granted(table, user, node) else: with When("I check materialize index when privilege is not granted"): check_materialize_index_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["CLEAR INDEX"]): + if grants & subprivileges["CLEAR INDEX"]: with When("I check clear index when privilege is granted"): check_clear_index_when_privilege_is_granted(table, user, node) else: with When("I check clear index when privilege is not granted"): check_clear_index_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["DROP INDEX"]): + if grants & subprivileges["DROP INDEX"]: with When("I check drop index when privilege is granted"): check_drop_index_when_privilege_is_granted(table, user, node) else: with When("I check drop index when privilege is not granted"): check_drop_index_when_privilege_is_not_granted(table, user, node) + def check_order_by_when_privilege_is_granted(table, user, node): - """Ensures ORDER BY runs as expected when the privilege is granted to the specified user - """ + """Ensures ORDER BY runs as expected when the privilege is granted to the specified user""" column = "order" with Given("I run sanity check"): - node.query(f"ALTER TABLE {table} MODIFY ORDER BY b", settings = [("user", user)]) + node.query(f"ALTER TABLE {table} MODIFY ORDER BY b", settings=[("user", user)]) with And("I add new column and modify order using that column"): - node.query(f"ALTER TABLE {table} ADD COLUMN {column} UInt32, MODIFY ORDER BY (b, {column})") + node.query( + f"ALTER TABLE {table} ADD COLUMN {column} UInt32, MODIFY ORDER BY (b, {column})" + ) with When(f"I insert random data into the ordered-by column {column}"): - data = random.sample(range(1,1000),100) - values = ', '.join(f'({datum})' for datum in data) + data = random.sample(range(1, 1000), 100) + values = ", ".join(f"({datum})" for datum in data) node.query(f"INSERT INTO {table}({column}) VALUES {values}") with Then("I synchronize with optimize table"): @@ -144,154 +160,216 @@ def check_order_by_when_privilege_is_granted(table, user, node): with And("I verify that the added data is ordered in the table"): data.sort() note(data) - column_data = node.query(f"SELECT {column} FROM {table} FORMAT JSONEachRow").output - column_data = column_data.split('\n') + column_data = node.query( + f"SELECT {column} FROM {table} FORMAT JSONEachRow" + ).output + column_data = column_data.split("\n") for row, datum in zip(column_data[:10], data[:10]): - assert json.loads(row) == {column:datum}, error() + assert json.loads(row) == {column: datum}, error() with And("I verify that the sorting key is present in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"ORDER BY (b, {column})" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert f"ORDER BY (b, {column})" in output["statement"], error() with But(f"I cannot drop the required column {column}"): exitcode, message = errors.missing_columns(column) - node.query(f"ALTER TABLE {table} DROP COLUMN {column}", - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP COLUMN {column}", + exitcode=exitcode, + message=message, + ) + def check_sample_by_when_privilege_is_granted(table, user, node): - """Ensures SAMPLE BY runs as expected when the privilege is granted to the specified user - """ - column = 'sample' + """Ensures SAMPLE BY runs as expected when the privilege is granted to the specified user""" + column = "sample" with When(f"I add sample by clause"): - node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY b", - settings = [("user", user)]) + node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY b", settings=[("user", user)]) with Then("I verify that the sample is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"SAMPLE BY b" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert f"SAMPLE BY b" in output["statement"], error() + def check_add_index_when_privilege_is_granted(table, user, node): - """Ensures ADD INDEX runs as expected when the privilege is granted to the specified user - """ + """Ensures ADD INDEX runs as expected when the privilege is granted to the specified user""" index = "add" - with Given(f"I add index '{index}'"): # Column x: String - node.query(f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1", - settings = [("user", user)]) + with Given(f"I add index '{index}'"): # Column x: String + node.query( + f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1", + settings=[("user", user)], + ) with Then("I verify that the index is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"INDEX {index} x TYPE set(0) GRANULARITY 1" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert ( + f"INDEX {index} x TYPE set(0) GRANULARITY 1" in output["statement"] + ), error() with Finally(f"I drop index {index}"): node.query(f"ALTER TABLE {table} DROP INDEX {index}") + def check_materialize_index_when_privilege_is_granted(table, user, node): - """Ensures MATERIALIZE INDEX runs as expected when the privilege is granted to the specified user - """ + """Ensures MATERIALIZE INDEX runs as expected when the privilege is granted to the specified user""" index = "materialize" with Given(f"I add index '{index}'"): - node.query(f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1") + node.query( + f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1" + ) with When(f"I materialize index '{index}'"): - node.query(f"ALTER TABLE {table} MATERIALIZE INDEX {index} IN PARTITION 1 SETTINGS mutations_sync = 2", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} MATERIALIZE INDEX {index} IN PARTITION 1 SETTINGS mutations_sync = 2", + settings=[("user", user)], + ) with Then("I verify that the index is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"INDEX {index} x TYPE set(0) GRANULARITY 1" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert ( + f"INDEX {index} x TYPE set(0) GRANULARITY 1" in output["statement"] + ), error() with Finally(f"I drop index {index}"): node.query(f"ALTER TABLE {table} DROP INDEX {index}") + def check_clear_index_when_privilege_is_granted(table, user, node): - """Ensures CLEAR INDEX runs as expected when the privilege is granted to the specified user - """ + """Ensures CLEAR INDEX runs as expected when the privilege is granted to the specified user""" index = "clear" - with Given(f"I add index '{index}'"): # Column x: String - node.query(f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1") + with Given(f"I add index '{index}'"): # Column x: String + node.query( + f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1" + ) with When(f"I clear index {index}"): node.query(f"ALTER TABLE {table} CLEAR INDEX {index} IN PARTITION 1") with Then("I verify that the index is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"INDEX {index} x TYPE set(0) GRANULARITY 1" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert ( + f"INDEX {index} x TYPE set(0) GRANULARITY 1" in output["statement"] + ), error() with Finally(f"I drop index {index}"): node.query(f"ALTER TABLE {table} DROP INDEX {index}") + def check_drop_index_when_privilege_is_granted(table, user, node): - """Ensures DROP INDEX runs as expected when the privilege is granted to the specified user - """ + """Ensures DROP INDEX runs as expected when the privilege is granted to the specified user""" with When("I try to drop nonexistent index, throws exception"): exitcode, message = errors.wrong_index_name("fake_index") - node.query(f"ALTER TABLE {table} DROP INDEX fake_index", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP INDEX fake_index", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) index = "drop" with Given(f"I add the index"): - node.query(f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1") + node.query( + f"ALTER TABLE {table} ADD INDEX {index}(x) TYPE set(0) GRANULARITY 1" + ) with Then(f"I drop index {index} which exists"): - node.query(f"ALTER TABLE {table} DROP INDEX {index}", - settings = [("user", user)]) + node.query(f"ALTER TABLE {table} DROP INDEX {index}", settings=[("user", user)]) with And("I verify that the index is not in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"INDEX {index} x TYPE set(0) GRANULARITY 1" not in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert ( + f"INDEX {index} x TYPE set(0) GRANULARITY 1" not in output["statement"] + ), error() + def check_order_by_when_privilege_is_not_granted(table, user, node): - """Ensures ORDER BY errors as expected without the required privilege for the specified user - """ + """Ensures ORDER BY errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY ORDER BY b", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MODIFY ORDER BY b", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_sample_by_when_privilege_is_not_granted(table, user, node): - """Ensures SAMPLE BY errors as expected without the required privilege for the specified user - """ + """Ensures SAMPLE BY errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY b", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MODIFY SAMPLE BY b", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_add_index_when_privilege_is_not_granted(table, user, node): - """Ensures ADD INDEX errors as expected without the required privilege for the specified user - """ + """Ensures ADD INDEX errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} ADD INDEX index1 b * length(x) TYPE set(1000) GRANULARITY 4", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} ADD INDEX index1 b * length(x) TYPE set(1000) GRANULARITY 4", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_materialize_index_when_privilege_is_not_granted(table, user, node): - """Ensures MATERIALIZE INDEX errors as expected without the required privilege for the specified user - """ + """Ensures MATERIALIZE INDEX errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MATERIALIZE INDEX index1", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MATERIALIZE INDEX index1", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_clear_index_when_privilege_is_not_granted(table, user, node): - """Ensures CLEAR INDEX errors as expected without the required privilege for the specified user - """ + """Ensures CLEAR INDEX errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} CLEAR INDEX index1 IN PARTITION 1", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} CLEAR INDEX index1 IN PARTITION 1", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_drop_index_when_privilege_is_not_granted(table, user, node): - """Ensures DROP INDEX errors as expected without the required privilege for the specified user - """ + """Ensures DROP INDEX errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} DROP INDEX index1", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} DROP INDEX index1", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_some_privileges(self, table_type, node=None): @@ -313,7 +391,10 @@ def user_with_some_privileges(self, table_type, node=None): node.query(f"GRANT {privileges} ON {table_name} TO {user_name}") with Then(f"I try to ALTER INDEX with given privileges"): - alter_index_privilege_handler(permutation, table_name, user_name, node) + alter_index_privilege_handler( + permutation, table_name, user_name, node + ) + @TestScenario @Requirements( @@ -344,6 +425,7 @@ def user_with_revoked_privileges(self, table_type, node=None): # Permutation 0: no privileges alter_index_privilege_handler(0, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterIndex_Grant("1.0"), @@ -363,7 +445,9 @@ def role_with_some_privileges(self, table_type, node=None): privileges = alter_index_privileges(permutation) with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with Given("I grant the ALTER INDEX privilege to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -371,7 +455,10 @@ def role_with_some_privileges(self, table_type, node=None): node.query(f"GRANT {role_name} TO {user_name}") with Then(f"I try to ALTER INDEX with given privileges"): - alter_index_privilege_handler(permutation, table_name, user_name, node) + alter_index_privilege_handler( + permutation, table_name, user_name, node + ) + @TestScenario def user_with_revoked_role(self, table_type, node=None): @@ -389,7 +476,9 @@ def user_with_revoked_role(self, table_type, node=None): privileges = alter_index_privileges(permutation) with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with When("I grant privileges to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -403,6 +492,7 @@ def user_with_revoked_role(self, table_type, node=None): # Permutation 0: no privileges for any permutation on these columns alter_index_privilege_handler(0, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterIndex_Cluster("1.0"), @@ -424,27 +514,34 @@ def user_with_privileges_on_cluster(self, table_type, node=None): with table(node, table_name, table_type): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) - with When("I grant ALTER INDEX privileges needed for iteration on a cluster"): - node.query(f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}") + with When( + "I grant ALTER INDEX privileges needed for iteration on a cluster" + ): + node.query( + f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}" + ) with Then(f"I try to ALTER INDEX with given privileges"): - alter_index_privilege_handler(permutation, table_name, user_name, node) + alter_index_privilege_handler( + permutation, table_name, user_name, node + ) finally: with Finally("I drop the user on cluster"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterIndex("1.0"), RQ_SRS_006_RBAC_Privileges_AlterIndex_TableEngines("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter index") def feature(self, stress=None, parallel=None, node="clickhouse1"): self.context.node = self.context.cluster.node(node) @@ -455,17 +552,22 @@ def feature(self, stress=None, parallel=None, node="clickhouse1"): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue - args = {"table_type" : table_type} + args = {"table_type": table_type} with Example(str(example)): with Pool(5) as pool: try: for scenario in loads(current_module(), Scenario): - Scenario(test=scenario, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Scenario( + test=scenario, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_move.py b/tests/testflows/rbac/tests/privileges/alter/alter_move.py index a8094716fe4..8d2fc79c0d4 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_move.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_move.py @@ -5,12 +5,18 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors -aliases = {"ALTER MOVE PARTITION", "ALTER MOVE PART", "MOVE PARTITION", "MOVE PART", "ALL"} +aliases = { + "ALTER MOVE PARTITION", + "ALTER MOVE PART", + "MOVE PARTITION", + "MOVE PART", + "ALL", +} + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, privilege, node=None): - """Check that user is only able to execute ALTER MOVE PARTITION when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ALTER MOVE PARTITION when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -20,8 +26,16 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute ALTER MOVE PARTITION with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that {user_name} is only able to execute ALTER MOVE PARTITION with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): with user(node, user_name), role(node, role_name): @@ -29,12 +43,20 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER MOVE PARTITION with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute ALTER MOVE PARTITION with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, privilege, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege", setup=instrument_clickhouse_server_log): @@ -50,121 +72,199 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to move partition without privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) - with Scenario("user without ALTER MOVE PARTITION privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user without ALTER MOVE PARTITION privilege", + setup=instrument_clickhouse_server_log, + ): source_table_name = f"source_merge_tree_{getuid()}" target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): - with When(f"I grant SELECT and ALTER DELETE privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE ON {source_table_name} TO {grant_target_name}") + with When( + f"I grant SELECT and ALTER DELETE privileges on {source_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT SELECT, ALTER DELETE ON {source_table_name} TO {grant_target_name}" + ) with And(f"I grant INSERT on {target_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {target_table_name} TO {grant_target_name}") + node.query( + f"GRANT INSERT ON {target_table_name} TO {grant_target_name}" + ) with Then("I attempt to move partitions without ALTER MOVE privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) - with Scenario("user with ALTER MOVE PARTITION privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user with ALTER MOVE PARTITION privilege", + setup=instrument_clickhouse_server_log, + ): source_table_name = f"source_merge_tree_{getuid()}" target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): - with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + with When( + f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}" + ) with And(f"I grant INSERT on {target_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {target_table_name} TO {grant_target_name}") + node.query( + f"GRANT INSERT ON {target_table_name} TO {grant_target_name}" + ) with Then("I attempt to move partitions with ALTER MOVE privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", settings = [("user", user_name)]) + node.query( + f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", + settings=[("user", user_name)], + ) - with Scenario("user with revoked ALTER MOVE PARTITION privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user with revoked ALTER MOVE PARTITION privilege", + setup=instrument_clickhouse_server_log, + ): source_table_name = f"source_merge_tree_{getuid()}" target_table_name = f"target_merge_tree_{getuid()}" with table(node, f"{source_table_name},{target_table_name}", table_type): - with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + with When( + f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}" + ) with And(f"I grant INSERT on {target_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {target_table_name} TO {grant_target_name}") + node.query( + f"GRANT INSERT ON {target_table_name} TO {grant_target_name}" + ) with And("I revoke ALTER MOVE PARTITION privilege"): - node.query(f"REVOKE {privilege} ON {source_table_name} FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {source_table_name} FROM {grant_target_name}" + ) with Then("I attempt to move partition"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {target_table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) - with Scenario("move partition to source table of a materialized view", setup=instrument_clickhouse_server_log): + with Scenario( + "move partition to source table of a materialized view", + setup=instrument_clickhouse_server_log, + ): source_table_name = f"source_merge_tree_{getuid()}" mat_view_name = f"mat_view_{getuid()}" mat_view_source_table_name = f"mat_view_source_merge_tree_{getuid()}" - with table(node, f"{source_table_name},{mat_view_source_table_name}", table_type): + with table( + node, f"{source_table_name},{mat_view_source_table_name}", table_type + ): try: with Given("I have a materialized view"): - node.query(f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}" + ) - with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + with When( + f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}" + ) - with And(f"I grant INSERT on {mat_view_source_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {mat_view_source_table_name} TO {grant_target_name}") + with And( + f"I grant INSERT on {mat_view_source_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT INSERT ON {mat_view_source_table_name} TO {grant_target_name}" + ) with Then("I attempt to move partitions with ALTER MOVE privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {mat_view_source_table_name}", settings = [("user", user_name)]) + node.query( + f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {mat_view_source_table_name}", + settings=[("user", user_name)], + ) finally: with Finally("I drop the materialized view"): node.query(f"DROP VIEW IF EXISTS {mat_view_name}") - with Scenario("move partition to implicit target table of a materialized view", setup=instrument_clickhouse_server_log): + with Scenario( + "move partition to implicit target table of a materialized view", + setup=instrument_clickhouse_server_log, + ): source_table_name = f"source_merge_tree_{getuid()}" mat_view_name = f"mat_view_{getuid()}" mat_view_source_table_name = f"mat_view_source_merge_tree_{getuid()}" - implicit_table_name = f"\\\".inner.{mat_view_name}\\\"" + implicit_table_name = f'\\".inner.{mat_view_name}\\"' - with table(node, f"{source_table_name},{mat_view_source_table_name}", table_type): + with table( + node, f"{source_table_name},{mat_view_source_table_name}", table_type + ): try: with Given("I have a materialized view"): - node.query(f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {mat_view_name} ENGINE = {table_type} PARTITION BY y ORDER BY d AS SELECT * FROM {mat_view_source_table_name}" + ) - with When(f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}"): - node.query(f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}") + with When( + f"I grant SELECT, ALTER DELETE, and ALTER MOVE PARTITION privileges on {source_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT SELECT, ALTER DELETE, {privilege} ON {source_table_name} TO {grant_target_name}" + ) - with And(f"I grant INSERT on {implicit_table_name} to {grant_target_name}"): - node.query(f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}") + with And( + f"I grant INSERT on {implicit_table_name} to {grant_target_name}" + ): + node.query( + f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}" + ) with Then("I attempt to move partitions with ALTER MOVE privilege"): - node.query(f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {implicit_table_name}", settings = [("user", user_name)]) + node.query( + f"ALTER TABLE {source_table_name} MOVE PARTITION 1 TO TABLE {implicit_table_name}", + settings=[("user", user_name)], + ) finally: with Finally("I drop the materialized view"): node.query(f"DROP VIEW IF EXISTS {mat_view_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterMove("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter move") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ALTER MOVE. - """ + """Check the RBAC functionality of ALTER MOVE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -173,7 +273,7 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue @@ -181,4 +281,6 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): with Example(str(example)): for alias in aliases: with Suite(alias, test=privilege_granted_directly_or_via_role): - privilege_granted_directly_or_via_role(table_type=table_type, privilege=alias) + privilege_granted_directly_or_via_role( + table_type=table_type, privilege=alias + ) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_quota.py b/tests/testflows/rbac/tests/privileges/alter/alter_quota.py index faad7c001f4..ae8326a0eab 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_quota.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_quota.py @@ -7,6 +7,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def quota(node, name): try: @@ -19,10 +20,10 @@ def quota(node, name): with Finally("I drop the quota"): node.query(f"DROP QUOTA IF EXISTS {name}") + @TestSuite def alter_quota_granted_directly(self, node=None): - """Check that a user is able to execute `ALTER QUOTA` with privileges are granted directly. - """ + """Check that a user is able to execute `ALTER QUOTA` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -31,15 +32,22 @@ def alter_quota_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=alter_quota, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in alter_quota.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=alter_quota, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in alter_quota.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def alter_quota_granted_via_role(self, node=None): - """Check that a user is able to execute `ALTER QUOTA` with privileges are granted through a role. - """ + """Check that a user is able to execute `ALTER QUOTA` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -52,20 +60,30 @@ def alter_quota_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=alter_quota, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in alter_quota.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=alter_quota, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in alter_quota.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("ALTER QUOTA",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("ALTER QUOTA",), + ], +) def alter_quota(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `ALTER QUOTA` when they have the necessary privilege. - """ + """Check that user is only able to execute `ALTER QUOTA` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -83,8 +101,12 @@ def alter_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't alter a quota"): - node.query(f"ALTER QUOTA {alter_quota_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER QUOTA {alter_quota_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("ALTER QUOTA with privilege"): alter_quota_name = f"alter_quota_{getuid()}" @@ -95,25 +117,34 @@ def alter_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a user"): - node.query(f"ALTER QUOTA {alter_quota_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER QUOTA {alter_quota_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("ALTER QUOTA on cluster"): alter_quota_name = f"alter_quota_{getuid()}" try: with Given("I have a quota on a cluster"): - node.query(f"CREATE QUOTA {alter_quota_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE QUOTA {alter_quota_name} ON CLUSTER sharded_cluster" + ) with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a quota"): - node.query(f"ALTER QUOTA {alter_quota_name} ON CLUSTER sharded_cluster", - settings = [("user", f"{user_name}")]) + node.query( + f"ALTER QUOTA {alter_quota_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the quota"): - node.query(f"DROP QUOTA IF EXISTS {alter_quota_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP QUOTA IF EXISTS {alter_quota_name} ON CLUSTER sharded_cluster" + ) with Scenario("ALTER QUOTA with revoked privilege"): alter_quota_name = f"alter_quota_{getuid()}" @@ -127,19 +158,23 @@ def alter_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user can't alter a quota"): - node.query(f"ALTER QUOTA {alter_quota_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER QUOTA {alter_quota_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("alter quota") @Requirements( RQ_SRS_006_RBAC_Privileges_AlterQuota("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ALTER QUOTA. - """ + """Check the RBAC functionality of ALTER QUOTA.""" self.context.node = self.context.cluster.node(node) Suite(run=alter_quota_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_role.py b/tests/testflows/rbac/tests/privileges/alter/alter_role.py index 49e8baa191b..4be7123a969 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_role.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_role.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `ALTER ROLE` with privileges are granted directly. - """ + """Check that a user is able to execute `ALTER ROLE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=alter_role, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in alter_role.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=alter_role, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in alter_role.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `ALTER ROLE` with privileges are granted through a role. - """ + """Check that a user is able to execute `ALTER ROLE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=alter_role, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in alter_role.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=alter_role, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in alter_role.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("ALTER ROLE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("ALTER ROLE",), + ], +) def alter_role(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `ALTER ROLE` when they have the necessary privilege. - """ + """Check that user is only able to execute `ALTER ROLE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -69,8 +86,12 @@ def alter_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't alter a role"): - node.query(f"ALTER ROLE {alter_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER ROLE {alter_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("ALTER ROLE with privilege"): alter_role_name = f"alter_role_{getuid()}" @@ -81,7 +102,9 @@ def alter_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a role"): - node.query(f"ALTER ROLE {alter_role_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER ROLE {alter_role_name}", settings=[("user", f"{user_name}")] + ) with Scenario("ALTER ROLE on cluster"): alter_role_name = f"alter_role_{getuid()}" @@ -94,11 +117,16 @@ def alter_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a role"): - node.query(f"ALTER ROLE {alter_role_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER ROLE {alter_role_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP ROLE IF EXISTS {alter_role_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROLE IF EXISTS {alter_role_name} ON CLUSTER sharded_cluster" + ) with Scenario("ALTER ROLE with revoked privilege"): alter_role_name = f"alter_role_{getuid()}" @@ -111,19 +139,23 @@ def alter_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot alter a role"): - node.query(f"ALTER ROLE {alter_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER ROLE {alter_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("alter role") @Requirements( RQ_SRS_006_RBAC_Privileges_AlterRole("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ALTER ROLE. - """ + """Check the RBAC functionality of ALTER ROLE.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_row_policy.py b/tests/testflows/rbac/tests/privileges/alter/alter_row_policy.py index a0d1e4271bc..36a83051d5a 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_row_policy.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_row_policy.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `ALTER ROW POLICY` with privileges are granted directly. - """ + """Check that a user is able to execute `ALTER ROW POLICY` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=alter_row_policy, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in alter_row_policy.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=alter_row_policy, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in alter_row_policy.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `ALTER ROW POLICY` with privileges are granted through a role. - """ + """Check that a user is able to execute `ALTER ROW POLICY` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,21 +45,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=alter_row_policy, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in alter_row_policy.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=alter_row_policy, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in alter_row_policy.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("ALTER ROW POLICY",), - ("ALTER POLICY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("ALTER ROW POLICY",), + ("ALTER POLICY",), + ], +) def alter_row_policy(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `ALTER ROW POLICY` when they have the necessary privilege. - """ + """Check that user is only able to execute `ALTER ROW POLICY` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -73,12 +90,18 @@ def alter_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't alter a row policy"): - node.query(f"ALTER ROW POLICY {alter_row_policy_name} ON {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER ROW POLICY {alter_row_policy_name} ON {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON {table_name}" + ) with Scenario("ALTER ROW POLICY with privilege"): alter_row_policy_name = f"alter_row_policy_{getuid()}" @@ -92,11 +115,16 @@ def alter_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a row policy"): - node.query(f"ALTER ROW POLICY {alter_row_policy_name} ON {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER ROW POLICY {alter_row_policy_name} ON {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON {table_name}" + ) with Scenario("ALTER ROW POLICY on cluster"): alter_row_policy_name = f"alter_row_policy_{getuid()}" @@ -104,17 +132,24 @@ def alter_row_policy(self, privilege, grant_target_name, user_name, node=None): try: with Given("I have a row policy on a cluster"): - node.query(f"CREATE ROW POLICY {alter_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {alter_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a row policy"): - node.query(f"ALTER ROW POLICY {alter_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER ROW POLICY {alter_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with Scenario("ALTER ROW POLICY with revoked privilege"): alter_row_policy_name = f"alter_row_policy_{getuid()}" @@ -131,16 +166,21 @@ def alter_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot alter row policy"): - node.query(f"ALTER ROW POLICY {alter_row_policy_name} ON {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER ROW POLICY {alter_row_policy_name} ON {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: - with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON {table_name}") + with Finally("I drop the row policy"): + node.query( + f"DROP ROW POLICY IF EXISTS {alter_row_policy_name} ON {table_name}" + ) + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Restriction("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Restriction("1.0")) def no_grants(self, node=None): """Check that user is unable to select from a table without a row policy after a row policy has been altered to have a condition. @@ -162,22 +202,22 @@ def no_grants(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() with When("I alter the row policy to have a condition"): node.query(f"ALTER POLICY {pol_name} ON {table_name} FOR SELECT USING 1") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Permissive("1.0"), ) def permissive(self, node=None): - """Check that user is able to see from a table when they have a PERMISSIVE policy. - """ + """Check that user is able to see from a table when they have a PERMISSIVE policy.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -194,19 +234,19 @@ def permissive(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1), (2)") with When("I alter a row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Restrictive("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Restrictive("1.0")) def restrictive(self, node=None): - """Check that user is able to see values they have a RESTRICTIVE policy for. - """ + """Check that user is able to see values they have a RESTRICTIVE policy for.""" table_name = f"table_{getuid()}" perm_pol_name = f"perm_pol_{getuid()}" @@ -224,25 +264,29 @@ def restrictive(self, node=None): row_policy(name=perm_pol_name, table=table_name) with And("I alter a row policy to be permissive"): - node.query(f"ALTER ROW POLICY {perm_pol_name} ON {table_name} FOR SELECT USING y=1 OR y=2 TO default") + node.query( + f"ALTER ROW POLICY {perm_pol_name} ON {table_name} FOR SELECT USING y=1 OR y=2 TO default" + ) with And("I alter a row policy to be restrictive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} AS RESTRICTIVE FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} AS RESTRICTIVE FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1), (2)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Alter_ForSelect("1.0"), ) def for_select(self, node=None): - """Check that user is able to see values allowed by the row policy condition in the FOR SELECT clause. - """ + """Check that user is able to see values allowed by the row policy condition in the FOR SELECT clause.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -259,19 +303,19 @@ def for_select(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Given("I alter therow policy to use FOR SELECT"): - node.query(f"Alter ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default") + node.query( + f"Alter ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default" + ) with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Alter_Condition("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Alter_Condition("1.0")) def condition(self, node=None): - """Check that user is able to see values allowed by the row policy condition. - """ + """Check that user is able to see values allowed by the row policy condition.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -288,19 +332,19 @@ def condition(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with When("I alter a row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Alter_Condition_None("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Alter_Condition_None("1.0")) def remove_condition(self, node=None): - """Check that user is able to see the table after row policy condition has been removed. - """ + """Check that user is able to see the table after row policy condition has been removed.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -314,25 +358,27 @@ def remove_condition(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy has a condition"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with When("I alter a row policy to not have a condition"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING NONE") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING NONE" + ) with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Alter_IfExists("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Alter_IfExists("1.0")) def if_exists(self, node=None): - """Check that a row policy altered using IF EXISTS restricts rows as expected. - """ + """Check that a row policy altered using IF EXISTS restricts rows as expected.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -349,19 +395,19 @@ def if_exists(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with When("I have alter a row policy to be permissive using IF EXISTS clause"): - node.query(f"ALTER ROW POLICY IF EXISTS {pol_name} ON {table_name} FOR SELECT USING 1 TO default") + node.query( + f"ALTER ROW POLICY IF EXISTS {pol_name} ON {table_name} FOR SELECT USING 1 TO default" + ) with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Alter_Rename("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Alter_Rename("1.0")) def rename(self, node=None): - """Check that a row policy altered using RENAME restricts rows as expected. - """ + """Check that a row policy altered using RENAME restricts rows as expected.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -379,26 +425,28 @@ def rename(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with And("The row policy is permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with When("I have alter a row policy by renaming it"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} RENAME TO {pol_new_name}") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} RENAME TO {pol_new_name}" + ) with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_new_name} ON {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Alter_OnCluster("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Alter_OnCluster("1.0")) def on_cluster(self, node=None): - """Check that a row policy altered using ON CLUSTER applies to the nodes of the cluster correctly. - """ + """Check that a row policy altered using ON CLUSTER applies to the nodes of the cluster correctly.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -409,10 +457,14 @@ def on_cluster(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy on a cluster on that table"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("The table has some values on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -421,27 +473,31 @@ def on_cluster(self, node=None): node2.query(f"INSERT INTO {table_name} (x) VALUES (1)") with When("I alter the row policy to have a condition"): - node.query(f"ALTER ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1" + ) with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() with And("I select from another node on the cluster"): output = node2.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE {table_name} ON CLUSTER sharded_cluster") + @TestScenario def diff_policies_on_diff_nodes(self, node=None): - """Check that a row policy altered on a node, does not effect row policy on a different node. - """ + """Check that a row policy altered on a node, does not effect row policy on a different node.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -452,10 +508,14 @@ def diff_policies_on_diff_nodes(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy on the cluster"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("The table has some values on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -464,30 +524,34 @@ def diff_policies_on_diff_nodes(self, node=None): node2.query(f"INSERT INTO {table_name} (x) VALUES (1)") with When("I alter the row policy on the first node"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() with And("I select from another node on the cluster"): output = node2.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE {table_name} ON CLUSTER sharded_cluster") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0"), ) def assignment(self, node=None): - """Check that user is able to see rows from a table when they have PERMISSIVE policy assigned to them. - """ + """Check that user is able to see rows from a table when they have PERMISSIVE policy assigned to them.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -501,7 +565,9 @@ def assignment(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy is permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") @@ -511,15 +577,15 @@ def assignment(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_None("1.0"), ) def assignment_none(self, node=None): - """Check that no one is affected when a row policy is altered to be assigned to NONE. - """ + """Check that no one is affected when a row policy is altered to be assigned to NONE.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -533,7 +599,9 @@ def assignment_none(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy is permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") @@ -543,15 +611,15 @@ def assignment_none(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_All("1.0"), ) def assignment_all(self, node=None): - """Check that everyone is effected with a row policy is altered to be assigned to ALL. - """ + """Check that everyone is effected with a row policy is altered to be assigned to ALL.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -565,7 +633,9 @@ def assignment_all(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy is permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") @@ -575,15 +645,15 @@ def assignment_all(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_AllExcept("1.0"), ) def assignment_all_except(self, node=None): - """Check that everyone is except the specified user is effect by a row policy is altered to be assigned to ALL EXCEPT. - """ + """Check that everyone is except the specified user is effect by a row policy is altered to be assigned to ALL EXCEPT.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -597,22 +667,25 @@ def assignment_all_except(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy is permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with When("I alter a row policy to be assigned to ALL EXCEPT default"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} TO ALL EXCEPT default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} TO ALL EXCEPT default" + ) with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def nested_view(self, node=None): """Check that if a user has a row policy on a table and a view is altered to use a condition on that table, the user is only able to access the rows specified by the assigned policies. @@ -638,20 +711,21 @@ def nested_view(self, node=None): node.query(f"CREATE VIEW {view_name} AS SELECT * FROM {table_name}") with When("I alter the row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def nested_live_view_before_policy(self, node=None): """Check that if a live view exists on a table and then a row policy is created, the user is only able to select rows specified by the assigned policies from the view. @@ -666,8 +740,12 @@ def nested_live_view_before_policy(self, node=None): with table(node, table_name): try: - with Given("I add allow_experimental_live_view to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + with Given( + "I add allow_experimental_live_view to the default query settings" + ): + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(("allow_experimental_live_view", 1)) with And("I have a row policy"): @@ -677,30 +755,40 @@ def nested_live_view_before_policy(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with And("There exists a live view on the table"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}" + ) with When("I alter the row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the live view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") - with And("I remove allow_experimental_live_view from the default query settings", flags=TE): + with And( + "I remove allow_experimental_live_view from the default query settings", + flags=TE, + ): if default_query_settings: try: - default_query_settings.pop(default_query_settings.index(("allow_experimental_live_view", 1))) + default_query_settings.pop( + default_query_settings.index( + ("allow_experimental_live_view", 1) + ) + ) except ValueError: pass + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def nested_live_view_after_policy(self, node=None): """Check that if a user has a row policy on a table and a materialized view is created on that table, the user is only able to select rows specified by the assigned policies from the view. @@ -715,8 +803,12 @@ def nested_live_view_after_policy(self, node=None): with table(node, table_name): try: - with Given("I add allow_experimental_live_view to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + with Given( + "I add allow_experimental_live_view to the default query settings" + ): + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(("allow_experimental_live_view", 1)) with And("I have a row policy"): @@ -726,30 +818,40 @@ def nested_live_view_after_policy(self, node=None): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with When("I alter the row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with And("I create a live view on the table"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the live view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") - with And("I remove allow_experimental_live_view from the default query settings", flags=TE): + with And( + "I remove allow_experimental_live_view from the default query settings", + flags=TE, + ): if default_query_settings: try: - default_query_settings.pop(default_query_settings.index(("allow_experimental_live_view", 1))) + default_query_settings.pop( + default_query_settings.index( + ("allow_experimental_live_view", 1) + ) + ) except ValueError: pass + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def nested_mat_view_before_policy(self, node=None): """Check that if a materialized view exists on a table and then a row policy is created, the user is only able to select rows specified by the assigned policies from the view. @@ -768,26 +870,29 @@ def nested_mat_view_before_policy(self, node=None): row_policy(name=pol_name, table=table_name) with And("There exists a mat view on the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with When("I alter the row policy"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the materialized view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def nested_mat_view_after_policy(self, node=None): """Check that if a user has a row policy on a table and a materialized view is created on that table, the user is only able to select rows specified by the assigned policies from the view. @@ -806,26 +911,29 @@ def nested_mat_view_after_policy(self, node=None): row_policy(name=pol_name, table=table_name) with And("I alter the row policy"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with When("I create a mat view on the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the materialized view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def populate_mat_view(self, node=None): """Check that if a user has a row policy on a table and a materialized view is created using POPULATE from that table, the user can only select the rows from the materialized view specified in the row policy. @@ -844,26 +952,29 @@ def populate_mat_view(self, node=None): row_policy(name=pol_name, table=table_name) with And("I alter a row policy on the table"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with When("I create a mat view populated by the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the materialized view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def dist_table(self, node=None): """Check that if a user has a row policy on a table and a distributed table is created on that table, the user is only able to access the rows specified by the assigned policies. @@ -879,27 +990,37 @@ def dist_table(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I have a distributed table"): - node.query(f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())" + ) with And("The table has some values on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") with When("I alter the row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} ON CLUSTER sharded_cluster FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} ON CLUSTER sharded_cluster FOR SELECT USING 1" + ) with Then("I select from the distributed table"): output = node.query(f"SELECT * FROM {dist_table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") @@ -907,13 +1028,11 @@ def dist_table(self, node=None): with And("I drop the distributed table", flags=TE): node.query(f"DROP TABLE IF EXISTS {dist_table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def dist_table_diff_policies_on_diff_nodes(self, node=None): - """Check that user is only able to select from the distributed table what is allowed by the row policies on each node. - """ + """Check that user is only able to select from the distributed table what is allowed by the row policies on each node.""" table_name = f"table_{getuid()}" dist_table_name = f"dist_table_{getuid()}" @@ -925,13 +1044,19 @@ def dist_table_diff_policies_on_diff_nodes(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I have a distributed table"): - node.query(f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())" + ) with And("The table has some values on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -940,15 +1065,19 @@ def dist_table_diff_policies_on_diff_nodes(self, node=None): node2.query(f"INSERT INTO {table_name} (x) VALUES (2)") with When("I alter the row policy to be permissive on the first node"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with Then("I select from the distributed table"): output = node.query(f"SELECT * FROM {dist_table_name}").output - assert '1' not in output and '2' in output, error() + assert "1" not in output and "2" in output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name} ON CLUSTER sharded_cluster" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") @@ -956,10 +1085,9 @@ def dist_table_diff_policies_on_diff_nodes(self, node=None): with And("I drop the distributed table", flags=TE): node.query(f"DROP TABLE IF EXISTS {dist_table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def dist_table_on_dist_table(self, node=None): """Check that if a user has a row policy on a table and a distributed table is created on that table, and another distributed table is created on top of that, @@ -976,40 +1104,55 @@ def dist_table_on_dist_table(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I have a distributed table on a cluster"): - node.query(f"CREATE TABLE {dist_table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())" + ) with And("I have a distributed table on the other distributed table"): - node.query(f"CREATE TABLE {dist_table_2_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {dist_table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_2_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {dist_table_name}, rand())" + ) with And("The table has some values on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") with When("I alter the row policy to be permissive on the first node"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with Then("I select from the second distributed table"): output = node.query(f"SELECT * FROM {dist_table_2_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") with And("I drop the distributed table", flags=TE): - node.query(f"DROP TABLE IF EXISTS {dist_table_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP TABLE IF EXISTS {dist_table_name} ON CLUSTER sharded_cluster" + ) with And("I drop the outer distributed table", flags=TE): node.query(f"DROP TABLE IF EXISTS {dist_table_2_name}") + @TestScenario def policy_before_table(self, node=None): """Check that if the policy is created and altered before the table, @@ -1025,7 +1168,9 @@ def policy_before_table(self, node=None): row_policy(name=pol_name, table=table_name) with And("I alter the row policy"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with table(node, table_name): with When("The table has some values"): @@ -1033,7 +1178,8 @@ def policy_before_table(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() + @TestScenario @Requirements( @@ -1056,20 +1202,26 @@ def dict(self, node=None): row_policy(name=pol_name, table=table_name) with And("I have a table"): - node.query(f"CREATE TABLE {table_name} (key UInt64, val UInt64 DEFAULT 5) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} (key UInt64, val UInt64 DEFAULT 5) ENGINE = Memory" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (key) VALUES (1),(2)") with And("I create a dict on the table"): - node.query(f"CREATE DICTIONARY {dict_name} (key UInt64 DEFAULT 0, val UInt64 DEFAULT 5) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE {table_name} PASSWORD '' DB 'default')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())") + node.query( + f"CREATE DICTIONARY {dict_name} (key UInt64 DEFAULT 0, val UInt64 DEFAULT 5) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE {table_name} PASSWORD '' DB 'default')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())" + ) with When("I alter the row policy to be permissive"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING key=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING key=1 TO default" + ) with Then("I try to select from the dict"): output = node.query(f"SELECT * FROM {dict_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the materialized view", flags=TE): @@ -1084,11 +1236,10 @@ def dict(self, node=None): @Requirements( RQ_SRS_006_RBAC_Privileges_AlterRowPolicy("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ALTER ROW POLICY. - """ + """Check the RBAC functionality of ALTER ROW POLICY.""" self.context.node = self.context.cluster.node(node) self.context.node2 = self.context.cluster.node("clickhouse2") @@ -1116,7 +1267,10 @@ def feature(self, node="clickhouse1"): Scenario(run=populate_mat_view, setup=instrument_clickhouse_server_log) Scenario(run=dist_table, setup=instrument_clickhouse_server_log) Scenario(run=dist_table_on_dist_table, setup=instrument_clickhouse_server_log) - Scenario(run=dist_table_diff_policies_on_diff_nodes, setup=instrument_clickhouse_server_log) + Scenario( + run=dist_table_diff_policies_on_diff_nodes, + setup=instrument_clickhouse_server_log, + ) Scenario(run=diff_policies_on_diff_nodes, setup=instrument_clickhouse_server_log) Scenario(run=policy_before_table, setup=instrument_clickhouse_server_log) Scenario(run=dict, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_settings.py b/tests/testflows/rbac/tests/privileges/alter/alter_settings.py index a1a2b824a11..f9d81342b84 100755 --- a/tests/testflows/rbac/tests/privileges/alter/alter_settings.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_settings.py @@ -8,26 +8,41 @@ from rbac.helper.common import * import rbac.helper.errors as errors from rbac.helper.tables import table_types -aliases = {"ALTER SETTINGS", "ALTER SETTING", "ALTER MODIFY SETTING", "MODIFY SETTING", "ALL"} +aliases = { + "ALTER SETTINGS", + "ALTER SETTING", + "ALTER MODIFY SETTING", + "MODIFY SETTING", + "ALL", +} + def check_alter_settings_when_privilege_is_granted(table, user, node): - """Ensures ADD SETTINGS runs as expected when the privilege is granted to the specified user - """ + """Ensures ADD SETTINGS runs as expected when the privilege is granted to the specified user""" with Given("I check that the modified setting is not already in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert "merge_with_ttl_timeout = 5" not in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert "merge_with_ttl_timeout = 5" not in output["statement"], error() with And(f"I modify settings"): - node.query(f"ALTER TABLE {table} MODIFY SETTING merge_with_ttl_timeout=5", - settings=[("user", user)]) + node.query( + f"ALTER TABLE {table} MODIFY SETTING merge_with_ttl_timeout=5", + settings=[("user", user)], + ) with Then("I verify that the setting is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert "SETTINGS index_granularity = 8192, merge_with_ttl_timeout = 5" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert ( + "SETTINGS index_granularity = 8192, merge_with_ttl_timeout = 5" + in output["statement"] + ), error() + def check_alter_settings_when_privilege_is_not_granted(table, user, node): - """Ensures CLEAR SETTINGS runs as expected when the privilege is granted to the specified user - """ + """Ensures CLEAR SETTINGS runs as expected when the privilege is granted to the specified user""" with When("I grant the user NONE privilege"): node.query(f"GRANT NONE TO {user}") @@ -36,8 +51,13 @@ def check_alter_settings_when_privilege_is_not_granted(table, user, node): with Then("I try to use ALTER SETTING, has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY SETTING merge_with_ttl_timeout=5", - settings=[("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MODIFY SETTING merge_with_ttl_timeout=5", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_privileges(self, privilege, table_type, node=None): @@ -80,7 +100,10 @@ def user_with_revoked_privileges(self, privilege, table_type, node=None): node.query(f"REVOKE {privilege} ON {table_name} FROM {user_name}") with When(f"I try to ALTER SETTINGS"): - check_alter_settings_when_privilege_is_not_granted(table_name, user_name, node) + check_alter_settings_when_privilege_is_not_granted( + table_name, user_name, node + ) + @TestScenario @Requirements( @@ -97,7 +120,9 @@ def role_with_some_privileges(self, privilege, table_type, node=None): user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with Given("I grant the alter settings privilege to a role"): node.query(f"GRANT {privilege} ON {table_name} TO {role_name}") @@ -107,6 +132,7 @@ def role_with_some_privileges(self, privilege, table_type, node=None): with Then(f"I try to ALTER SETTINGS"): check_alter_settings_when_privilege_is_granted(table_name, user_name, node) + @TestScenario def user_with_revoked_role(self, privilege, table_type, node=None): """Check that user with a role that has alter settings privilege on a table is unable to @@ -119,7 +145,9 @@ def user_with_revoked_role(self, privilege, table_type, node=None): user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with When("I grant privileges to a role"): node.query(f"GRANT {privilege} ON {table_name} TO {role_name}") @@ -130,7 +158,10 @@ def user_with_revoked_role(self, privilege, table_type, node=None): node.query(f"REVOKE {role_name} FROM {user_name}") with And("I alter settings on the table"): - check_alter_settings_when_privilege_is_not_granted(table_name, user_name, node) + check_alter_settings_when_privilege_is_not_granted( + table_name, user_name, node + ) + @TestScenario @Requirements( @@ -150,27 +181,37 @@ def user_with_privileges_on_cluster(self, privilege, table_type, node=None): with table(node, table_name, table_type): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) with When("I grant alter settings privileges on a cluster"): - node.query(f"GRANT ON CLUSTER sharded_cluster ALTER SETTINGS ON {table_name} TO {user_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster ALTER SETTINGS ON {table_name} TO {user_name}" + ) with Then(f"I try to ALTER SETTINGS"): - check_alter_settings_when_privilege_is_granted(table_name, user_name, node) + check_alter_settings_when_privilege_is_granted( + table_name, user_name, node + ) with When("I revoke alter settings privileges on a cluster"): - node.query(f"REVOKE ON CLUSTER sharded_cluster ALTER SETTINGS ON {table_name} FROM {user_name}") + node.query( + f"REVOKE ON CLUSTER sharded_cluster ALTER SETTINGS ON {table_name} FROM {user_name}" + ) with Then(f"I try to ALTER SETTINGS"): - check_alter_settings_when_privilege_is_not_granted(table_name, user_name, node) + check_alter_settings_when_privilege_is_not_granted( + table_name, user_name, node + ) finally: with Finally("I drop the user on a cluster"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestSuite def scenario_parallelization(self, table_type, privilege): - """Runs all scenarios in parallel for a given privilege. - """ + """Runs all scenarios in parallel for a given privilege.""" args = {"table_type": table_type, "privilege": privilege} with Pool(4) as pool: @@ -180,27 +221,25 @@ def scenario_parallelization(self, table_type, privilege): finally: join() + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterSettings("1.0"), RQ_SRS_006_RBAC_Privileges_AlterSettings_TableEngines("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter settings") def feature(self, stress=None, node="clickhouse1"): - """Runs test suites above which check correctness over scenarios and permutations - """ + """Runs test suites above which check correctness over scenarios and permutations""" self.context.node = self.context.cluster.node(node) if stress is not None: self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue @@ -210,6 +249,12 @@ def feature(self, stress=None, node="clickhouse1"): try: for alias in aliases: args = {"table_type": table_type, "privilege": alias} - Suite(test=scenario_parallelization, name=alias, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Suite( + test=scenario_parallelization, + name=alias, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_settings_profile.py b/tests/testflows/rbac/tests/privileges/alter/alter_settings_profile.py index cd4648305f7..406ce704cff 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_settings_profile.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_settings_profile.py @@ -7,6 +7,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def settings_profile(node, name): try: @@ -19,10 +20,10 @@ def settings_profile(node, name): with Finally("I drop the settings_profile"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {name}") + @TestSuite def alter_settings_profile_granted_directly(self, node=None): - """Check that a user is able to execute `ALTER SETTINGS PROFILE` with privileges are granted directly. - """ + """Check that a user is able to execute `ALTER SETTINGS PROFILE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -31,15 +32,22 @@ def alter_settings_profile_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=alter_settings_profile, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in alter_settings_profile.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=alter_settings_profile, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in alter_settings_profile.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def alter_settings_profile_granted_via_role(self, node=None): - """Check that a user is able to execute `ALTER SETTINGS PROFILE` with privileges are granted through a role. - """ + """Check that a user is able to execute `ALTER SETTINGS PROFILE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -52,21 +60,31 @@ def alter_settings_profile_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=alter_settings_profile, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in alter_settings_profile.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=alter_settings_profile, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in alter_settings_profile.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("ALTER SETTINGS PROFILE",), - ("ALTER PROFILE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("ALTER SETTINGS PROFILE",), + ("ALTER PROFILE",), + ], +) def alter_settings_profile(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `ALTER SETTINGS PROFILE` when they have the necessary privilege. - """ + """Check that user is only able to execute `ALTER SETTINGS PROFILE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -84,8 +102,12 @@ def alter_settings_profile(self, privilege, grant_target_name, user_name, node=N node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't alter a settings_profile"): - node.query(f"ALTER SETTINGS PROFILE {alter_settings_profile_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER SETTINGS PROFILE {alter_settings_profile_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("ALTER SETTINGS PROFILE with privilege"): alter_settings_profile_name = f"alter_settings_profile_{getuid()}" @@ -96,25 +118,34 @@ def alter_settings_profile(self, privilege, grant_target_name, user_name, node=N node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a user"): - node.query(f"ALTER SETTINGS PROFILE {alter_settings_profile_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER SETTINGS PROFILE {alter_settings_profile_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("ALTER SETTINGS PROFILE on cluster"): alter_settings_profile_name = f"alter_settings_profile_{getuid()}" try: with Given("I have a settings_profile on a cluster"): - node.query(f"CREATE SETTINGS PROFILE {alter_settings_profile_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE SETTINGS PROFILE {alter_settings_profile_name} ON CLUSTER sharded_cluster" + ) with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a settings_profile"): - node.query(f"ALTER SETTINGS PROFILE {alter_settings_profile_name} ON CLUSTER sharded_cluster", - settings = [("user", f"{user_name}")]) + node.query( + f"ALTER SETTINGS PROFILE {alter_settings_profile_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the settings_profile"): - node.query(f"DROP SETTINGS PROFILE IF EXISTS {alter_settings_profile_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP SETTINGS PROFILE IF EXISTS {alter_settings_profile_name} ON CLUSTER sharded_cluster" + ) with Scenario("ALTER SETTINGS PROFILE with revoked privilege"): alter_settings_profile_name = f"alter_settings_profile_{getuid()}" @@ -128,20 +159,30 @@ def alter_settings_profile(self, privilege, grant_target_name, user_name, node=N node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user can't alter a settings_profile"): - node.query(f"ALTER SETTINGS PROFILE {alter_settings_profile_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER SETTINGS PROFILE {alter_settings_profile_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("alter settings profile") @Requirements( RQ_SRS_006_RBAC_Privileges_AlterSettingsProfile("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ALTER SETTINGS PROFILE. - """ + """Check the RBAC functionality of ALTER SETTINGS PROFILE.""" self.context.node = self.context.cluster.node(node) - Suite(run=alter_settings_profile_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=alter_settings_profile_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=alter_settings_profile_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=alter_settings_profile_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_ttl.py b/tests/testflows/rbac/tests/privileges/alter/alter_ttl.py index 419cf880f30..50742c26eeb 100755 --- a/tests/testflows/rbac/tests/privileges/alter/alter_ttl.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_ttl.py @@ -9,16 +9,17 @@ import rbac.helper.errors as errors from rbac.helper.tables import table_types subprivileges = { - "TTL" : 1 << 0, - "MATERIALIZE TTL" : 1 << 1, + "TTL": 1 << 0, + "MATERIALIZE TTL": 1 << 1, } aliases = { - "TTL" : ["ALTER TTL", "ALTER MODIFY TTL", "MODIFY TTL"], + "TTL": ["ALTER TTL", "ALTER MODIFY TTL", "MODIFY TTL"], "MATERIALIZE TTL": ["ALTER MATERIALIZE TTL", "MATERIALIZE TTL", "ALL"], } -permutation_count = (1 << len(subprivileges)) +permutation_count = 1 << len(subprivileges) + def permutations(): """Returns list of all permutations to run. @@ -26,6 +27,7 @@ def permutations(): """ return [*range(permutation_count)] + def alter_ttl_privileges(grants: int): """Takes in an integer, and returns the corresponding set of tests to grant and not grant using the binary string. Each integer corresponds to a unique permutation @@ -34,75 +36,93 @@ def alter_ttl_privileges(grants: int): note(grants) privileges = [] - if grants==0: # No privileges + if grants == 0: # No privileges privileges.append("NONE") else: - if (grants & subprivileges["TTL"]): + if grants & subprivileges["TTL"]: privileges.append(f"ALTER TTL") - if (grants & subprivileges["MATERIALIZE TTL"]): + if grants & subprivileges["MATERIALIZE TTL"]: privileges.append(f"ALTER MATERIALIZE TTL") note(f"Testing these privileges: {privileges}") - return ', '.join(privileges) + return ", ".join(privileges) + def alter_ttl_privilege_handler(grants, table, user, node): """For all 2 subprivileges, if the privilege is granted: run test to ensure correct behavior, and if the privilege is not granted, run test to ensure correct behavior there as well """ - if (grants & subprivileges["TTL"]): + if grants & subprivileges["TTL"]: with When("I check ttl when privilege is granted"): check_ttl_when_privilege_is_granted(table, user, node) else: with When("I check ttl when privilege is not granted"): check_ttl_when_privilege_is_not_granted(table, user, node) - if (grants & subprivileges["MATERIALIZE TTL"]): + if grants & subprivileges["MATERIALIZE TTL"]: with When("I check materialize ttl when privilege is granted"): check_materialize_ttl_when_privilege_is_granted(table, user, node) else: with When("I check materialize ttl when privilege is not granted"): check_materialize_ttl_when_privilege_is_not_granted(table, user, node) + def check_ttl_when_privilege_is_granted(table, user, node): - """Ensures ALTER TTL runs as expected when the privilege is granted to the specified user - """ + """Ensures ALTER TTL runs as expected when the privilege is granted to the specified user""" with Given(f"I modify TTL"): - node.query(f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 1 DAY;", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 1 DAY;", + settings=[("user", user)], + ) with Then("I verify that the TTL clause is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert "TTL d + toIntervalDay(1)" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert "TTL d + toIntervalDay(1)" in output["statement"], error() + def check_materialize_ttl_when_privilege_is_granted(table, user, node): - """Ensures MATERIALIZE TTL runs as expected when the privilege is granted to the specified user - """ + """Ensures MATERIALIZE TTL runs as expected when the privilege is granted to the specified user""" with Given("I modify TTL so it exists"): node.query(f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 1 MONTH;") with Then("I materialize the TTL"): - node.query(f"ALTER TABLE {table} MATERIALIZE TTL IN PARTITION 2", - settings = [("user", user)]) + node.query( + f"ALTER TABLE {table} MATERIALIZE TTL IN PARTITION 2", + settings=[("user", user)], + ) with Then("I verify that the TTL clause is in the table"): - output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert "TTL d + toIntervalMonth(1)" in output['statement'], error() + output = json.loads( + node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output + ) + assert "TTL d + toIntervalMonth(1)" in output["statement"], error() + def check_ttl_when_privilege_is_not_granted(table, user, node): - """Ensures ALTER TTL errors as expected without the required privilege for the specified user - """ + """Ensures ALTER TTL errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 1 DAY;", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 1 DAY;", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + def check_materialize_ttl_when_privilege_is_not_granted(table, user, node): - """Ensures MATERIALIZE TTL errors as expected without the required privilege for the specified user - """ + """Ensures MATERIALIZE TTL errors as expected without the required privilege for the specified user""" with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MATERIALIZE TTL IN PARTITION 4", - settings = [("user", user)], exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table} MATERIALIZE TTL IN PARTITION 4", + settings=[("user", user)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_some_privileges(self, table_type, node=None): @@ -124,7 +144,10 @@ def user_with_some_privileges(self, table_type, node=None): node.query(f"GRANT {privileges} ON {table_name} TO {user_name}") with Then(f"I try to ALTER TTL"): - alter_ttl_privilege_handler(permutation, table_name, user_name, node) + alter_ttl_privilege_handler( + permutation, table_name, user_name, node + ) + @TestScenario @Requirements( @@ -155,6 +178,7 @@ def user_with_revoked_privileges(self, table_type, node=None): # Permutation 0: no privileges alter_ttl_privilege_handler(0, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterTTL_Grant("1.0"), @@ -174,7 +198,9 @@ def role_with_some_privileges(self, table_type, node=None): privileges = alter_ttl_privileges(permutation) with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with Given("I grant the ALTER TTL privilege to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -182,7 +208,10 @@ def role_with_some_privileges(self, table_type, node=None): node.query(f"GRANT {role_name} TO {user_name}") with Then(f"I try to ALTER TTL"): - alter_ttl_privilege_handler(permutation, table_name, user_name, node) + alter_ttl_privilege_handler( + permutation, table_name, user_name, node + ) + @TestScenario def user_with_revoked_role(self, table_type, node=None): @@ -200,7 +229,9 @@ def user_with_revoked_role(self, table_type, node=None): privileges = alter_ttl_privileges(permutation) with When(f"granted={privileges}"): - with table(node, table_name, table_type), user(node, user_name), role(node, role_name): + with table(node, table_name, table_type), user(node, user_name), role( + node, role_name + ): with When("I grant privileges to a role"): node.query(f"GRANT {privileges} ON {table_name} TO {role_name}") @@ -214,6 +245,7 @@ def user_with_revoked_role(self, table_type, node=None): # Permutation 0: no privileges for any permutation alter_ttl_privilege_handler(0, table_name, user_name, node) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_AlterTTL_Cluster("1.0"), @@ -235,27 +267,32 @@ def user_with_privileges_on_cluster(self, table_type, node=None): with table(node, table_name, table_type): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) with When("I grant ALTER TTL privileges on a cluster"): - node.query(f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster {privileges} ON {table_name} TO {user_name}" + ) with Then(f"I try to ALTER TTL"): - alter_ttl_privilege_handler(permutation, table_name, user_name, node) + alter_ttl_privilege_handler( + permutation, table_name, user_name, node + ) finally: with Finally("I drop the user on a cluster"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterTTL("1.0"), RQ_SRS_006_RBAC_Privileges_AlterTTL_TableEngines("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter ttl") def feature(self, stress=None, node="clickhouse1"): self.context.node = self.context.cluster.node(node) @@ -264,17 +301,22 @@ def feature(self, stress=None, node="clickhouse1"): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue - args = {"table_type" : table_type} + args = {"table_type": table_type} with Example(str(example)): with Pool(5) as pool: try: for scenario in loads(current_module(), Scenario): - Scenario(test=scenario, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Scenario( + test=scenario, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_update.py b/tests/testflows/rbac/tests/privileges/alter/alter_update.py index 9f3f4e19041..1e0850a59bb 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_update.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_update.py @@ -7,10 +7,10 @@ import rbac.helper.errors as errors aliases = {"ALTER UPDATE", "UPDATE", "ALL"} + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, privilege, node=None): - """Check that user is only able to execute ALTER UPDATE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ALTER UPDATE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -19,19 +19,35 @@ def privilege_granted_directly_or_via_role(self, table_type, privilege, node=Non with Suite("user with direct privilege", setup=instrument_clickhouse_server_log): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute ALTER UPDATE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that {user_name} is only able to execute ALTER UPDATE with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with privilege via role", setup=instrument_clickhouse_server_log): with user(node, user_name), role(node, role_name): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute ALTER UPDATE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, privilege=privilege, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute ALTER UPDATE with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + privilege=privilege, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, privilege, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege", setup=instrument_clickhouse_server_log): @@ -46,8 +62,12 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to update a column without privilege"): - node.query(f"ALTER TABLE {table_name} UPDATE a = x WHERE 1", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} UPDATE a = x WHERE 1", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege", setup=instrument_clickhouse_server_log): table_name = f"merge_tree_{getuid()}" @@ -58,9 +78,14 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with Then("I attempt to update a column"): - node.query(f"ALTER TABLE {table_name} UPDATE a = x WHERE 1", settings = [("user", user_name)]) + node.query( + f"ALTER TABLE {table_name} UPDATE a = x WHERE 1", + settings=[("user", user_name)], + ) - with Scenario("user with revoked privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "user with revoked privilege", setup=instrument_clickhouse_server_log + ): table_name = f"merge_tree_{getuid()}" with table(node, table_name, table_type): @@ -69,26 +94,30 @@ def privilege_check(grant_target_name, user_name, table_type, privilege, node=No node.query(f"GRANT {privilege} ON {table_name} TO {grant_target_name}") with And("I revoke the update privilege"): - node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {table_name} FROM {grant_target_name}" + ) with Then("I attempt to update a column"): - node.query(f"ALTER TABLE {table_name} UPDATE a = x WHERE 1", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {table_name} UPDATE a = x WHERE 1", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AlterUpdate("1.0"), RQ_SRS_006_RBAC_Privileges_AlterUpdate_TableEngines("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("alter update") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ALTER UPDATE. - """ + """Check the RBAC functionality of ALTER UPDATE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -97,7 +126,7 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue @@ -105,4 +134,6 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): with Example(str(example)): for alias in aliases: with Suite(alias, test=privilege_granted_directly_or_via_role): - privilege_granted_directly_or_via_role(table_type=table_type, privilege=alias) + privilege_granted_directly_or_via_role( + table_type=table_type, privilege=alias + ) diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_user.py b/tests/testflows/rbac/tests/privileges/alter/alter_user.py index bcf3014c9be..d4522ee29e8 100644 --- a/tests/testflows/rbac/tests/privileges/alter/alter_user.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_user.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def alter_user_granted_directly(self, node=None): - """Check that a user is able to execute `ALTER USER` with privileges are granted directly. - """ + """Check that a user is able to execute `ALTER USER` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def alter_user_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=alter_user, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in alter_user.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=alter_user, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in alter_user.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def alter_user_granted_via_role(self, node=None): - """Check that a user is able to execute `ALTER USER` with privileges are granted through a role. - """ + """Check that a user is able to execute `ALTER USER` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def alter_user_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=alter_user, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in alter_user.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=alter_user, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in alter_user.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("ALTER USER",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("ALTER USER",), + ], +) def alter_user(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `ALTER USER` when they have the necessary privilege. - """ + """Check that user is only able to execute `ALTER USER` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -69,8 +86,12 @@ def alter_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't alter a user"): - node.query(f"ALTER USER {alter_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER USER {alter_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("ALTER USER with privilege"): alter_user_name = f"alter_user_{getuid()}" @@ -79,7 +100,9 @@ def alter_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a user"): - node.query(f"ALTER USER {alter_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER USER {alter_user_name}", settings=[("user", f"{user_name}")] + ) with Scenario("ALTER USER on cluster"): alter_user_name = f"alter_user_{getuid()}" @@ -91,12 +114,16 @@ def alter_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can alter a user"): - node.query(f"ALTER USER {alter_user_name} ON CLUSTER sharded_cluster", - settings = [("user", f"{user_name}")]) + node.query( + f"ALTER USER {alter_user_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP USER IF EXISTS {alter_user_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP USER IF EXISTS {alter_user_name} ON CLUSTER sharded_cluster" + ) with Scenario("ALTER USER with revoked privilege"): alter_user_name = f"alter_user_{getuid()}" @@ -108,19 +135,23 @@ def alter_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user can't alter a user"): - node.query(f"ALTER USER {alter_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"ALTER USER {alter_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("alter user") @Requirements( RQ_SRS_006_RBAC_Privileges_AlterUser("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ALTER USER. - """ + """Check the RBAC functionality of ALTER USER.""" self.context.node = self.context.cluster.node(node) Suite(run=alter_user_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_database.py b/tests/testflows/rbac/tests/privileges/attach/attach_database.py index 3fecbe2571f..5e5009d1c2f 100644 --- a/tests/testflows/rbac/tests/privileges/attach/attach_database.py +++ b/tests/testflows/rbac/tests/privileges/attach/attach_database.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute ATTACH DATABASE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ATTACH DATABASE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE DATABASE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE DATABASE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DATABASE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE DATABASE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to attach a database without privilege"): - node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -55,11 +67,17 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with Then("I attempt to attach a database"): - node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=80, message="DB::Exception: Received from localhost:9000. DB::Exception: Database engine must be specified for ATTACH DATABASE query") + node.query( + f"ATTACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=80, + message="DB::Exception: Received from localhost:9000. DB::Exception: Database engine must be specified for ATTACH DATABASE query", + ) finally: with Finally("I drop the database"): @@ -70,14 +88,22 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with And("I revoke the create database privilege"): - node.query(f"REVOKE CREATE DATABASE ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE CREATE DATABASE ON {db_name}.* FROM {grant_target_name}" + ) with Then("I attempt to attach a database"): - node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -88,14 +114,20 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to attach a database"): - node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -109,23 +141,27 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to attach a database"): - node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=80, message="DB::Exception: Received from localhost:9000. DB::Exception: Database engine must be specified for ATTACH DATABASE query") + node.query( + f"ATTACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=80, + message="DB::Exception: Received from localhost:9000. DB::Exception: Database engine must be specified for ATTACH DATABASE query", + ) finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AttachDatabase("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("attach database") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ATTACH DATABASE. - """ + """Check the RBAC functionality of ATTACH DATABASE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -133,5 +169,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py b/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py index fbebdc0003d..678863aee2a 100644 --- a/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py +++ b/tests/testflows/rbac/tests/privileges/attach/attach_dictionary.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute ATTACH DICTIONARY when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ATTACH DICTIONARY when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE DICTIONARY with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DICTIONARY with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to attach a dictionary without privilege"): - node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): @@ -55,11 +67,17 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant create dictionary privilege"): - node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with Then("I attempt to attach a dictionary"): - node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], - exitcode=134, message=f"DB::Exception: Table `{dict_name}` doesn't exist.") + node.query( + f"ATTACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=134, + message=f"DB::Exception: Table `{dict_name}` doesn't exist.", + ) finally: with Finally("I drop the dictionary"): @@ -70,14 +88,22 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create dictionary privilege"): - node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with And("I revoke the create dictionary privilege"): - node.query(f"REVOKE CREATE DICTIONARY ON {dict_name} FROM {grant_target_name}") + node.query( + f"REVOKE CREATE DICTIONARY ON {dict_name} FROM {grant_target_name}" + ) with Then("I attempt to attach a dictionary"): - node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): @@ -88,14 +114,20 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to attach a database"): - node.query(f"ATTACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -109,23 +141,27 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to attach a dictionary"): - node.query(f"ATTACH DICTIONARY {dict_name}", settings = [("user", user_name)], - exitcode=134, message=f"DB::Exception: Table `{dict_name}` doesn't exist.") + node.query( + f"ATTACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=134, + message=f"DB::Exception: Table `{dict_name}` doesn't exist.", + ) finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AttachDictionary("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("attach dictionary") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ATTACH DICTIONARY. - """ + """Check the RBAC functionality of ATTACH DICTIONARY.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -133,5 +169,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_table.py b/tests/testflows/rbac/tests/privileges/attach/attach_table.py index 411140506ea..1bbb51c75e1 100644 --- a/tests/testflows/rbac/tests/privileges/attach/attach_table.py +++ b/tests/testflows/rbac/tests/privileges/attach/attach_table.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute ATTACH TABLE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ATTACH TABLE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE TABLE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE TABLE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE TABLE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE TABLE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to attach a table without privilege"): - node.query(f"ATTACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): @@ -58,8 +70,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT CREATE TABLE ON *.* TO {grant_target_name}") with Then("I attempt to attach a table"): - node.query(f"ATTACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=134, message=f"DB::Exception: Table `{table_name}` doesn't exist.") + node.query( + f"ATTACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=134, + message=f"DB::Exception: Table `{table_name}` doesn't exist.", + ) finally: with Finally("I drop the table"): @@ -76,8 +92,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE CREATE TABLE ON *.* FROM {grant_target_name}") with Then("I attempt to attach a table"): - node.query(f"ATTACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): @@ -94,8 +114,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to attach a table"): - node.query(f"ATTACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): @@ -109,23 +133,27 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to attach a table"): - node.query(f"ATTACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=134, message=f"DB::Exception: Table `{table_name}` doesn't exist.") + node.query( + f"ATTACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=134, + message=f"DB::Exception: Table `{table_name}` doesn't exist.", + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AttachTable("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("attach table") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ATTACH TABLE. - """ + """Check the RBAC functionality of ATTACH TABLE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -133,5 +161,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py b/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py index 2662a24d5a2..0a2cb6087a3 100644 --- a/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py +++ b/tests/testflows/rbac/tests/privileges/attach/attach_temp_table.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute ATTACH TEMPORARY TABLE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute ATTACH TEMPORARY TABLE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE TEMPORARY TABLE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE TEMPORARY TABLE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with When("I attempt to attach a temporary table without privilege"): - node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the temporary table"): @@ -55,10 +67,15 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant create temporary table privilege"): - node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + node.query( + f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}" + ) with Then("I attempt to attach a temporary table"): - node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)]) + node.query( + f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + ) finally: with Finally("I drop the temporary table"): @@ -69,14 +86,22 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create temporary table privilege"): - node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + node.query( + f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}" + ) with And("I revoke the create temporary table privilege"): - node.query(f"REVOKE CREATE TEMPORARY TABLE ON *.* FROM {grant_target_name}") + node.query( + f"REVOKE CREATE TEMPORARY TABLE ON *.* FROM {grant_target_name}" + ) with Then("I attempt to attach a temporary table"): - node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the temporary table"): @@ -87,14 +112,20 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create temporary table privilege"): - node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + node.query( + f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to attach a temporary table"): - node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the temporary table"): @@ -108,22 +139,25 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to attach a temporary table"): - node.query(f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)]) + node.query( + f"ATTACH TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + ) finally: with Finally("I drop the temporary table"): node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_AttachTemporaryTable("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("attach temporary table") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of ATTACH TEMPORARY TABLE. - """ + """Check the RBAC functionality of ATTACH TEMPORARY TABLE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -131,5 +165,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/create_database.py b/tests/testflows/rbac/tests/privileges/create/create_database.py index 8367d49e050..8a8b71b19a3 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_database.py +++ b/tests/testflows/rbac/tests/privileges/create/create_database.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute CREATE DATABASE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute CREATE DATABASE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE DATABASE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE DATABASE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DATABASE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE DATABASE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to create a database without privilege"): - node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -55,10 +67,12 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with Then("I attempt to create a database"): - node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)]) + node.query(f"CREATE DATABASE {db_name}", settings=[("user", user_name)]) finally: with Finally("I drop the database"): @@ -69,14 +83,22 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with And("I revoke the create database privilege"): - node.query(f"REVOKE CREATE DATABASE ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE CREATE DATABASE ON {db_name}.* FROM {grant_target_name}" + ) with Then("I attempt to create a database"): - node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -87,14 +109,20 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create database privilege"): - node.query(f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}") + node.query( + f"GRANT CREATE DATABASE ON {db_name}.* TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to create a database"): - node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -108,22 +136,22 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to create a database"): - node.query(f"CREATE DATABASE {db_name}", settings = [("user", user_name)]) + node.query(f"CREATE DATABASE {db_name}", settings=[("user", user_name)]) finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_CreateDatabase("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("create database") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of CREATE DATABASE. - """ + """Check the RBAC functionality of CREATE DATABASE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -131,5 +159,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/create_dictionary.py b/tests/testflows/rbac/tests/privileges/create/create_dictionary.py index 73734f5d556..76125aa0239 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_dictionary.py +++ b/tests/testflows/rbac/tests/privileges/create/create_dictionary.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute CREATE DICTIONARY when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute CREATE DICTIONARY when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE DICTIONARY with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE DICTIONARY with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to create a dictionary without privilege"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): @@ -55,10 +67,15 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant create dictionary privilege"): - node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with Then("I attempt to create a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)]) + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", + settings=[("user", user_name)], + ) finally: with Finally("I drop the dictionary"): @@ -69,33 +86,46 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create dictionary privilege"): - node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with And("I revoke the create dictionary privilege"): - node.query(f"REVOKE CREATE DICTIONARY ON {dict_name} FROM {grant_target_name}") + node.query( + f"REVOKE CREATE DICTIONARY ON {dict_name} FROM {grant_target_name}" + ) with Then("I attempt to create a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") - with Scenario("user with revoked ALL privilege"): dict_name = f"dict_{getuid()}" try: with When("I grant the create dictionary privilege"): - node.query(f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT CREATE DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to create a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): @@ -109,22 +139,25 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to create a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", settings = [("user", user_name)]) + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)", + settings=[("user", user_name)], + ) finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_CreateDictionary("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("create dictionary") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of CREATE DICTIONARY. - """ + """Check the RBAC functionality of CREATE DICTIONARY.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -132,5 +165,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/create_quota.py b/tests/testflows/rbac/tests/privileges/create/create_quota.py index d6e50ea904e..207bb786c89 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_quota.py +++ b/tests/testflows/rbac/tests/privileges/create/create_quota.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `CREATE QUOTA` with privileges are granted directly. - """ + """Check that a user is able to execute `CREATE QUOTA` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=create_quota, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in create_quota.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_quota, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in create_quota.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `CREATE QUOTA` with privileges are granted through a role. - """ + """Check that a user is able to execute `CREATE QUOTA` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=create_quota, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in create_quota.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_quota, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in create_quota.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("CREATE QUOTA",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("CREATE QUOTA",), + ], +) def create_quota(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `CREATE QUOTA` when they have the necessary privilege. - """ + """Check that user is only able to execute `CREATE QUOTA` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -68,8 +85,12 @@ def create_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't create a quota"): - node.query(f"CREATE QUOTA {create_quota_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE QUOTA {create_quota_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the quota"): @@ -83,7 +104,10 @@ def create_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a quota"): - node.query(f"CREATE QUOTA {create_quota_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE QUOTA {create_quota_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the quota"): @@ -97,11 +121,16 @@ def create_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a quota"): - node.query(f"CREATE QUOTA {create_quota_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE QUOTA {create_quota_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the quota"): - node.query(f"DROP QUOTA IF EXISTS {create_quota_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP QUOTA IF EXISTS {create_quota_name} ON CLUSTER sharded_cluster" + ) with Scenario("CREATE QUOTA with revoked privilege"): create_quota_name = f"create_quota_{getuid()}" @@ -114,23 +143,27 @@ def create_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot create a quota"): - node.query(f"CREATE QUOTA {create_quota_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE QUOTA {create_quota_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the quota"): node.query(f"DROP QUOTA IF EXISTS {create_quota_name}") + @TestFeature @Name("create quota") @Requirements( RQ_SRS_006_RBAC_Privileges_CreateQuota("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of CREATE QUOTA. - """ + """Check the RBAC functionality of CREATE QUOTA.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/create/create_role.py b/tests/testflows/rbac/tests/privileges/create/create_role.py index c442036b625..403b02c8c63 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_role.py +++ b/tests/testflows/rbac/tests/privileges/create/create_role.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `CREATE ROLE` with privileges are granted directly. - """ + """Check that a user is able to execute `CREATE ROLE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=create_role, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in create_role.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_role, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in create_role.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `CREATE ROLE` with privileges are granted through a role. - """ + """Check that a user is able to execute `CREATE ROLE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=create_role, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in create_role.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_role, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in create_role.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("CREATE ROLE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("CREATE ROLE",), + ], +) def create_role(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `CREATE ROLE` when they have the necessary privilege. - """ + """Check that user is only able to execute `CREATE ROLE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -68,8 +85,12 @@ def create_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't create a role"): - node.query(f"CREATE ROLE {create_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE ROLE {create_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the role"): @@ -83,7 +104,10 @@ def create_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a role"): - node.query(f"CREATE ROLE {create_role_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE ROLE {create_role_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the role"): @@ -97,11 +121,16 @@ def create_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a role"): - node.query(f"CREATE ROLE {create_role_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE ROLE {create_role_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the role"): - node.query(f"DROP ROLE IF EXISTS {create_role_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROLE IF EXISTS {create_role_name} ON CLUSTER sharded_cluster" + ) with Scenario("CREATE ROLE with revoked privilege"): create_role_name = f"create_role_{getuid()}" @@ -114,23 +143,27 @@ def create_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot create a role"): - node.query(f"CREATE ROLE {create_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE ROLE {create_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the role"): node.query(f"DROP ROLE IF EXISTS {create_role_name}") + @TestFeature @Name("create role") @Requirements( RQ_SRS_006_RBAC_Privileges_CreateRole("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of CREATE ROLE. - """ + """Check the RBAC functionality of CREATE ROLE.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/create/create_row_policy.py b/tests/testflows/rbac/tests/privileges/create/create_row_policy.py index 8e670333492..81b9d093e7e 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_row_policy.py +++ b/tests/testflows/rbac/tests/privileges/create/create_row_policy.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `CREATE ROW POLICY` with privileges are granted directly. - """ + """Check that a user is able to execute `CREATE ROW POLICY` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=create_row_policy, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in create_row_policy.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_row_policy, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in create_row_policy.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `CREATE ROW POLICY` with privileges are granted through a role. - """ + """Check that a user is able to execute `CREATE ROW POLICY` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,21 +45,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=create_row_policy, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in create_row_policy.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_row_policy, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in create_row_policy.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("CREATE ROW POLICY",), - ("CREATE POLICY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("CREATE ROW POLICY",), + ("CREATE POLICY",), + ], +) def create_row_policy(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `CREATE ROW POLICY` when they have the necessary privilege. - """ + """Check that user is only able to execute `CREATE ROW POLICY` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -70,12 +87,18 @@ def create_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't create a row policy"): - node.query(f"CREATE ROW POLICY {create_row_policy_name} ON {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE ROW POLICY {create_row_policy_name} ON {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON {table_name}" + ) with Scenario("CREATE ROW POLICY with privilege"): create_row_policy_name = f"create_row_policy_{getuid()}" @@ -86,11 +109,16 @@ def create_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a row policy"): - node.query(f"CREATE ROW POLICY {create_row_policy_name} ON {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE ROW POLICY {create_row_policy_name} ON {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON {table_name}" + ) with Scenario("CREATE ROW POLICY on cluster"): create_row_policy_name = f"create_row_policy_{getuid()}" @@ -101,11 +129,16 @@ def create_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a row policy"): - node.query(f"CREATE ROW POLICY {create_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE ROW POLICY {create_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with Scenario("CREATE ROW POLICY with revoked privilege"): create_row_policy_name = f"create_row_policy_{getuid()}" @@ -119,17 +152,22 @@ def create_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot create a row policy"): - node.query(f"CREATE ROW POLICY {create_row_policy_name} ON {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE ROW POLICY {create_row_policy_name} ON {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {create_row_policy_name} ON {table_name}" + ) + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Restriction("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Restriction("1.0")) def no_grants(self, node=None): """Check that user is unable to select from a table without a row policy after a row policy with a condition has been created on that table. @@ -151,26 +189,28 @@ def no_grants(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() with When("I create a row policy with a condition"): - node.query(f"CREATE ROW POLICY OR REPLACE {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY OR REPLACE {pol_name} ON {table_name} FOR SELECT USING 1" + ) with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Create_Access_Permissive("1.0"), ) def permissive(self, node=None): - """Check that user is able to see from a table when they have a PERMISSIVE policy. - """ + """Check that user is able to see from a table when they have a PERMISSIVE policy.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -181,26 +221,26 @@ def permissive(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1), (2)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Create_Access_Restrictive("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Create_Access_Restrictive("1.0")) def restrictive(self, node=None): - """Check that user is able to see values they have a RESTRICTIVE policy for. - """ + """Check that user is able to see values they have a RESTRICTIVE policy for.""" table_name = f"table_{getuid()}" perm_pol_name = f"perm_pol_{getuid()}" @@ -212,17 +252,21 @@ def restrictive(self, node=None): with table(node, table_name): try: with Given("I have a permissive row policy"): - node.query(f"CREATE ROW POLICY {perm_pol_name} ON {table_name} FOR SELECT USING y=1 OR y=2 TO default") + node.query( + f"CREATE ROW POLICY {perm_pol_name} ON {table_name} FOR SELECT USING y=1 OR y=2 TO default" + ) with And("I have a restrictive row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS RESTRICTIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS RESTRICTIVE FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1), (2)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the restrictive row policy", flags=TE): @@ -231,13 +275,13 @@ def restrictive(self, node=None): with And("I drop the permissive row policy", flags=TE): node.query(f"DROP ROW POLICY IF EXISTS {perm_pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Create_ForSelect("1.0"), ) def for_select(self, node=None): - """Check that user is able to see values allowed by the row policy condition in the FOR SELECT clause. - """ + """Check that user is able to see values allowed by the row policy condition in the FOR SELECT clause.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -248,26 +292,26 @@ def for_select(self, node=None): with table(node, table_name): try: with Given("I have a restrictive row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Create_Condition("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Create_Condition("1.0")) def condition(self, node=None): - """Check that user is able to see values allowed by the row policy condition. - """ + """Check that user is able to see values allowed by the row policy condition.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -278,26 +322,26 @@ def condition(self, node=None): with table(node, table_name): try: with Given("I have a restrictive row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Create_IfNotExists("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Create_IfNotExists("1.0")) def if_not_exists(self, node=None): - """Check that a row policy created using IF NOT EXISTS does not replace a row policy with the same name. - """ + """Check that a row policy created using IF NOT EXISTS does not replace a row policy with the same name.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -308,33 +352,37 @@ def if_not_exists(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() - with When("I create another row policy with the same name using IF NOT EXISTS"): - node.query(f"CREATE ROW POLICY IF NOT EXISTS {pol_name} ON {table_name}") + with When( + "I create another row policy with the same name using IF NOT EXISTS" + ): + node.query( + f"CREATE ROW POLICY IF NOT EXISTS {pol_name} ON {table_name}" + ) with Then("I select from the table again"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Create_Replace("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Create_Replace("1.0")) def or_replace(self, node=None): - """Check that a row policy created using OR REPLACE does replace the row policy with the same name. - """ + """Check that a row policy created using OR REPLACE does replace the row policy with the same name.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -345,33 +393,37 @@ def or_replace(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() - with When("I create another row policy with the same name using OR REPLACE"): - node.query(f"CREATE ROW POLICY OR REPLACE {pol_name} ON {table_name} AS RESTRICTIVE FOR SELECT USING 1 TO default") + with When( + "I create another row policy with the same name using OR REPLACE" + ): + node.query( + f"CREATE ROW POLICY OR REPLACE {pol_name} ON {table_name} AS RESTRICTIVE FOR SELECT USING 1 TO default" + ) with Then("I can no longer select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert output == '', error() + assert output == "", error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Create_OnCluster("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Create_OnCluster("1.0")) def on_cluster(self, node=None): - """Check that a row policy created using ON CLUSTER applies to the nodes of the cluster correctly. - """ + """Check that a row policy created using ON CLUSTER applies to the nodes of the cluster correctly.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -382,10 +434,14 @@ def on_cluster(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1" + ) with When("I insert some values into the table on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -395,23 +451,25 @@ def on_cluster(self, node=None): with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() with And("I select from another node on the cluster"): output = node2.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE {table_name} ON CLUSTER sharded_cluster") + @TestScenario def diff_policies_on_diff_nodes(self, node=None): - """Check that a row policy created on a node, does not effect a different node. - """ + """Check that a row policy created on a node, does not effect a different node.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -422,10 +480,14 @@ def diff_policies_on_diff_nodes(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy on one node"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with When("I insert some values into the table on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -435,11 +497,11 @@ def diff_policies_on_diff_nodes(self, node=None): with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() with And("I select from another node on the cluster"): output = node2.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -448,13 +510,13 @@ def diff_policies_on_diff_nodes(self, node=None): with And("I drop the table", flags=TE): node.query(f"DROP TABLE {table_name} ON CLUSTER sharded_cluster") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0"), ) def assignment(self, node=None): - """Check that user is able to see rows from a table when they have PERMISSIVE policy assigned to them. - """ + """Check that user is able to see rows from a table when they have PERMISSIVE policy assigned to them.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -465,26 +527,28 @@ def assignment(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_None("1.0"), ) def assignment_none(self, node=None): - """Check that no one is affected when a row policy is assigned to NONE. - """ + """Check that no one is affected when a row policy is assigned to NONE.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -495,26 +559,28 @@ def assignment_none(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO NONE") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO NONE" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_All("1.0"), ) def assignment_all(self, node=None): - """Check that everyone is effected with a row policy is assigned to ALL. - """ + """Check that everyone is effected with a row policy is assigned to ALL.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -525,26 +591,28 @@ def assignment_all(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO ALL") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO ALL" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_AllExcept("1.0"), ) def assignment_all_except(self, node=None): - """Check that everyone is except the specified user is effect by a row policy assigned to ALL EXCEPT. - """ + """Check that everyone is except the specified user is effect by a row policy assigned to ALL EXCEPT.""" table_name = f"table_{getuid()}" pol_name = f"pol_{getuid()}" @@ -555,19 +623,22 @@ def assignment_all_except(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO ALL EXCEPT default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO ALL EXCEPT default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1)") with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0"), @@ -587,7 +658,9 @@ def nested_view(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") @@ -597,7 +670,7 @@ def nested_view(self, node=None): with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -606,6 +679,7 @@ def nested_view(self, node=None): with And("I drop the view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0"), @@ -624,22 +698,30 @@ def nested_live_view_after_policy(self, node=None): with table(node, table_name): try: - with Given("I add allow_experimental_live_view to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + with Given( + "I add allow_experimental_live_view to the default query settings" + ): + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(("allow_experimental_live_view", 1)) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with And("I create a live view on the table"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -648,13 +730,21 @@ def nested_live_view_after_policy(self, node=None): with And("I drop the live view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") - with And("I remove allow_experimental_live_view from the default query settings", flags=TE): + with And( + "I remove allow_experimental_live_view from the default query settings", + flags=TE, + ): if default_query_settings: try: - default_query_settings.pop(default_query_settings.index(("allow_experimental_live_view", 1))) + default_query_settings.pop( + default_query_settings.index( + ("allow_experimental_live_view", 1) + ) + ) except ValueError: pass + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0"), @@ -673,22 +763,30 @@ def nested_live_view_before_policy(self, node=None): with table(node, table_name): try: - with Given("I add allow_experimental_live_view to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + with Given( + "I add allow_experimental_live_view to the default query settings" + ): + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(("allow_experimental_live_view", 1)) with And("There is a live view on the table"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}" + ) with And("There is a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with When("I insert values into the table"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -697,13 +795,21 @@ def nested_live_view_before_policy(self, node=None): with And("I drop the live view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") - with And("I remove allow_experimental_live_view from the default query settings", flags=TE): + with And( + "I remove allow_experimental_live_view from the default query settings", + flags=TE, + ): if default_query_settings: try: - default_query_settings.pop(default_query_settings.index(("allow_experimental_live_view", 1))) + default_query_settings.pop( + default_query_settings.index( + ("allow_experimental_live_view", 1) + ) + ) except ValueError: pass + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0"), @@ -723,17 +829,21 @@ def nested_mat_view_after_policy(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with When("I create a view on the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with And("I insert some values on the table"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -742,6 +852,7 @@ def nested_mat_view_after_policy(self, node=None): with And("I drop the materialized view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0"), @@ -761,17 +872,21 @@ def nested_mat_view_before_policy(self, node=None): with table(node, table_name): try: with Given("I have a view on the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with And("I have some values on the table"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with When("I create a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -780,6 +895,7 @@ def nested_mat_view_before_policy(self, node=None): with And("I drop the materialized view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def populate_mat_view(self, node=None): """Check that if a user has a row policy on a table and a materialized view is created using POPULATE from that table, @@ -796,17 +912,21 @@ def populate_mat_view(self, node=None): with table(node, table_name): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with When("I create a mat view with POPULATE from the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}" + ) with Then("I try to select from the view"): output = node.query(f"SELECT * FROM {view_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -815,10 +935,9 @@ def populate_mat_view(self, node=None): with And("I drop the materialized view", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def dist_table(self, node=None): """Check that if a user has a row policy on a table and a distributed table is created on that table, the user is only able to select rows specified by the assigned policies from the distributed table. @@ -834,24 +953,32 @@ def dist_table(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1" + ) with And("I have a distributed table"): - node.query(f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())" + ) with When("I insert some values into the table on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") with Then("I select from the table"): output = node.query(f"SELECT * FROM {dist_table_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") @@ -859,10 +986,9 @@ def dist_table(self, node=None): with And("I drop the distributed table", flags=TE): node.query(f"DROP TABLE IF EXISTS {dist_table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def dist_table_diff_policies_on_diff_nodes(self, node=None): """Check that the user can only access the rows of the distributed table that are allowed by row policies on the the source tables. The row policies are different on different nodes. @@ -878,13 +1004,19 @@ def dist_table_diff_policies_on_diff_nodes(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("I have a distributed table"): - node.query(f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())" + ) with When("I insert some values into the table on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -894,11 +1026,13 @@ def dist_table_diff_policies_on_diff_nodes(self, node=None): with Then("I select from the table"): output = node.query(f"SELECT * FROM {dist_table_name}").output - assert '2' in output and '1' not in output, error() + assert "2" in output and "1" not in output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") @@ -906,10 +1040,9 @@ def dist_table_diff_policies_on_diff_nodes(self, node=None): with And("I drop the distributed table", flags=TE): node.query(f"DROP TABLE IF EXISTS {dist_table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0")) def dist_table_on_dist_table(self, node=None): """Check that if a user has a row policy on a table and a distributed table is created on that table, and another distributed table is created on top of that, @@ -926,53 +1059,72 @@ def dist_table_on_dist_table(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1" + ) with And("I have a distributed table on a cluster"): - node.query(f"CREATE TABLE {dist_table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Distributed(sharded_cluster, default, {table_name}, rand())" + ) with And("I have a distributed table on the other distributed table"): - node.query(f"CREATE TABLE {dist_table_2_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {dist_table_name}, rand())") + node.query( + f"CREATE TABLE {dist_table_2_name} (x UInt64) ENGINE = Distributed(sharded_cluster, default, {dist_table_name}, rand())" + ) with When("I insert some values into the table on the first node"): node.query(f"INSERT INTO {dist_table_2_name} (x) VALUES (1)") with Then("I select from the table"): output = node.query(f"SELECT * FROM {dist_table_2_name}").output - assert '' == output, error() + assert "" == output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") with And("I drop the distributed table", flags=TE): - node.query(f"DROP TABLE IF EXISTS {dist_table_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP TABLE IF EXISTS {dist_table_name} ON CLUSTER sharded_cluster" + ) with And("I drop the outer distributed table", flags=TE): node.query(f"DROP TABLE IF EXISTS {dist_table_2_name}") + @TestScenario def no_table(self, node=None): - """Check that row policy is not created when the table is not specified. - """ + """Check that row policy is not created when the table is not specified.""" pol_name = f"pol_{getuid()}" if node is None: node = self.context.node with When("I try to create a row policy without a table"): - node.query(f"CREATE ROW POLICY {pol_name}", - exitcode=62, message='Exception: Syntax error') + node.query( + f"CREATE ROW POLICY {pol_name}", + exitcode=62, + message="Exception: Syntax error", + ) with And("I try to create a row policy on a database"): - node.query(f"CREATE ROW POLICY {pol_name} ON default.*", - exitcode=62, message='Exception: Syntax error') + node.query( + f"CREATE ROW POLICY {pol_name} ON default.*", + exitcode=62, + message="Exception: Syntax error", + ) + @TestScenario def policy_before_table(self, node=None): @@ -987,7 +1139,9 @@ def policy_before_table(self, node=None): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING y=1 TO default" + ) with table(node, table_name): with When("The table has some values"): @@ -995,12 +1149,13 @@ def policy_before_table(self, node=None): with Then("I try to select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON {table_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Nesting("1.0"), @@ -1019,20 +1174,26 @@ def dict(self, node=None): try: with Given("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING key=1 TO default") + node.query( + f"CREATE ROW POLICY {pol_name} ON {table_name} AS PERMISSIVE FOR SELECT USING key=1 TO default" + ) with And("I have a table"): - node.query(f"CREATE TABLE {table_name} (key UInt64, val UInt64 DEFAULT 5) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} (key UInt64, val UInt64 DEFAULT 5) ENGINE = Memory" + ) with When("The table has some values"): node.query(f"INSERT INTO {table_name} (key) VALUES (1),(2)") with And("I create a dict on the table"): - node.query(f"CREATE DICTIONARY {dict_name} (key UInt64 DEFAULT 0, val UInt64 DEFAULT 5) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE {table_name} PASSWORD '' DB 'default')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())") + node.query( + f"CREATE DICTIONARY {dict_name} (key UInt64 DEFAULT 0, val UInt64 DEFAULT 5) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE {table_name} PASSWORD '' DB 'default')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())" + ) with Then("I try to select from the dict"): output = node.query(f"SELECT * FROM {dict_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() finally: with Finally("I drop the row policy", flags=TE): @@ -1044,16 +1205,16 @@ def dict(self, node=None): with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestFeature @Name("create row policy") @Requirements( RQ_SRS_006_RBAC_Privileges_CreateRowPolicy("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of CREATE ROW POLICY. - """ + """Check the RBAC functionality of CREATE ROW POLICY.""" self.context.node = self.context.cluster.node(node) self.context.node2 = self.context.cluster.node("clickhouse2") @@ -1080,7 +1241,10 @@ def feature(self, node="clickhouse1"): Scenario(run=populate_mat_view, setup=instrument_clickhouse_server_log) Scenario(run=dist_table, setup=instrument_clickhouse_server_log) Scenario(run=dist_table_on_dist_table, setup=instrument_clickhouse_server_log) - Scenario(run=dist_table_diff_policies_on_diff_nodes, setup=instrument_clickhouse_server_log) + Scenario( + run=dist_table_diff_policies_on_diff_nodes, + setup=instrument_clickhouse_server_log, + ) Scenario(run=diff_policies_on_diff_nodes, setup=instrument_clickhouse_server_log) Scenario(run=no_table, setup=instrument_clickhouse_server_log) Scenario(run=policy_before_table, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/create/create_settings_profile.py b/tests/testflows/rbac/tests/privileges/create/create_settings_profile.py index 938de560391..95511afa260 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_settings_profile.py +++ b/tests/testflows/rbac/tests/privileges/create/create_settings_profile.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `CREATE SETTINGS PROFILE` with privileges are granted directly. - """ + """Check that a user is able to execute `CREATE SETTINGS PROFILE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=create_settings_profile, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in create_settings_profile.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_settings_profile, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in create_settings_profile.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `CREATE SETTINGS PROFILE` with privileges are granted through a role. - """ + """Check that a user is able to execute `CREATE SETTINGS PROFILE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,21 +45,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=create_settings_profile, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in create_settings_profile.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=create_settings_profile, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in create_settings_profile.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("CREATE SETTINGS PROFILE",), - ("CREATE PROFILE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("CREATE SETTINGS PROFILE",), + ("CREATE PROFILE",), + ], +) def create_settings_profile(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `CREATE SETTINGS PROFILE` when they have the necessary privilege. - """ + """Check that user is only able to execute `CREATE SETTINGS PROFILE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -69,12 +86,18 @@ def create_settings_profile(self, privilege, grant_target_name, user_name, node= node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't create a settings_profile"): - node.query(f"CREATE SETTINGS PROFILE {create_settings_profile_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE SETTINGS PROFILE {create_settings_profile_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the settings_profile"): - node.query(f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name}") + node.query( + f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name}" + ) with Scenario("CREATE SETTINGS PROFILE with privilege"): create_settings_profile_name = f"create_settings_profile_{getuid()}" @@ -84,11 +107,16 @@ def create_settings_profile(self, privilege, grant_target_name, user_name, node= node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a settings_profile"): - node.query(f"CREATE SETTINGS PROFILE {create_settings_profile_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE SETTINGS PROFILE {create_settings_profile_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the settings_profile"): - node.query(f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name}") + node.query( + f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name}" + ) with Scenario("CREATE SETTINGS PROFILE on cluster"): create_settings_profile_name = f"create_settings_profile_{getuid()}" @@ -98,11 +126,16 @@ def create_settings_profile(self, privilege, grant_target_name, user_name, node= node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a settings_profile"): - node.query(f"CREATE SETTINGS PROFILE {create_settings_profile_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE SETTINGS PROFILE {create_settings_profile_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the settings_profile"): - node.query(f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name} ON CLUSTER sharded_cluster" + ) with Scenario("CREATE SETTINGS PROFILE with revoked privilege"): create_settings_profile_name = f"create_settings_profile_{getuid()}" @@ -115,23 +148,29 @@ def create_settings_profile(self, privilege, grant_target_name, user_name, node= node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot create a settings_profile"): - node.query(f"CREATE SETTINGS PROFILE {create_settings_profile_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE SETTINGS PROFILE {create_settings_profile_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the settings_profile"): - node.query(f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name}") + node.query( + f"DROP SETTINGS PROFILE IF EXISTS {create_settings_profile_name}" + ) + @TestFeature @Name("create settings profile") @Requirements( RQ_SRS_006_RBAC_Privileges_CreateSettingsProfile("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of CREATE SETTINGS PROFILE. - """ + """Check the RBAC functionality of CREATE SETTINGS PROFILE.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/create/create_table.py b/tests/testflows/rbac/tests/privileges/create/create_table.py index 8f0a9f43771..9d10d4fc9f0 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_table.py +++ b/tests/testflows/rbac/tests/privileges/create/create_table.py @@ -5,13 +5,11 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_Privileges_None("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_None("1.0")) def create_without_create_table_privilege(self, node=None): - """Check that user is unable to create a table without CREATE TABLE privilege. - """ + """Check that user is unable to create a table without CREATE TABLE privilege.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -30,18 +28,24 @@ def create_without_create_table_privilege(self, node=None): with And("I grant the user USAGE privilege"): node.query(f"GRANT USAGE ON *.* TO {user_name}") - with Then("I try to create a table without CREATE TABLE privilege as the user"): - node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I try to create a table without CREATE TABLE privilege as the user" + ): + node.query( + f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_create_table_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to create a table with CREATE TABLE privilege, either granted directly or through a role. - """ + """Check that user is able to create a table with CREATE TABLE privilege, either granted directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -50,21 +54,25 @@ def create_with_create_table_privilege_granted_directly_or_via_role(self, node=N with user(node, f"{user_name}"): - Scenario(test=create_with_create_table_privilege, - name="create with create table privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_create_table_privilege, + name="create with create table privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_create_table_privilege, - name="create with create table privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_create_table_privilege, + name="create with create table privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_create_table_privilege(self, grant_target_name, user_name, node=None): - """Check that user is able to create a table with the granted privileges. - """ + """Check that user is able to create a table with the granted privileges.""" table_name = f"table_{getuid()}" if node is None: @@ -77,19 +85,20 @@ def create_with_create_table_privilege(self, grant_target_name, user_name, node= node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") with Then("I try to create a table without privilege as the user"): - node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_Privileges_All("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_All("1.0")) def create_with_all_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to create a table with ALL privilege, either granted directly or through a role. - """ + """Check that user is able to create a table with ALL privilege, either granted directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -98,21 +107,25 @@ def create_with_all_privilege_granted_directly_or_via_role(self, node=None): with user(node, f"{user_name}"): - Scenario(test=create_with_all_privilege, - name="create with ALL privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_all_privilege, + name="create with ALL privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_all_privilege, - name="create with ALL privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_all_privilege, + name="create with ALL privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_all_privilege(self, grant_target_name, user_name, node=None): - """Check that user is able to create a table with the granted privileges. - """ + """Check that user is able to create a table with the granted privileges.""" table_name = f"table_{getuid()}" if node is None: @@ -125,16 +138,21 @@ def create_with_all_privilege(self, grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I try to create a table without privilege as the user"): - node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario -def create_with_revoked_create_table_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to create table after the CREATE TABLE privilege is revoked, either directly or from a role. - """ +def create_with_revoked_create_table_privilege_revoked_directly_or_from_role( + self, node=None +): + """Check that user is unable to create table after the CREATE TABLE privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -143,21 +161,27 @@ def create_with_revoked_create_table_privilege_revoked_directly_or_from_role(sel with user(node, f"{user_name}"): - Scenario(test=create_with_revoked_create_table_privilege, - name="create with create table privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_table_privilege, + name="create with create table privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_revoked_create_table_privilege, - name="create with create table privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_table_privilege, + name="create with create table privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def create_with_revoked_create_table_privilege(self, grant_target_name, user_name, node=None): - """Revoke CREATE TABLE privilege and check the user is unable to create a table. - """ +def create_with_revoked_create_table_privilege( + self, grant_target_name, user_name, node=None +): + """Revoke CREATE TABLE privilege and check the user is unable to create a table.""" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -175,17 +199,21 @@ def create_with_revoked_create_table_privilege(self, grant_target_name, user_nam node.query(f"REVOKE CREATE TABLE ON {table_name} FROM {grant_target_name}") with Then("I try to create a table on the table as the user"): - node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_all_privileges_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to create table after ALL privileges are revoked, either directly or from a role. - """ + """Check that user is unable to create table after ALL privileges are revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -194,21 +222,25 @@ def create_with_all_privileges_revoked_directly_or_from_role(self, node=None): with user(node, f"{user_name}"): - Scenario(test=create_with_revoked_all_privilege, - name="create with all privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_revoked_all_privilege, + name="create with all privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_revoked_all_privilege, - name="create with all privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_revoked_all_privilege, + name="create with all privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_revoked_all_privilege(self, grant_target_name, user_name, node=None): - """Revoke ALL privilege and check the user is unable to create a table. - """ + """Revoke ALL privilege and check the user is unable to create a table.""" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -226,13 +258,18 @@ def create_with_revoked_all_privilege(self, grant_target_name, user_name, node=N node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I try to create a table on the table as the user"): - node.query(f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x Int8) ENGINE = Memory", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_without_source_table_privilege(self, node=None): """Check that user is unable to create a table without select @@ -256,14 +293,21 @@ def create_without_source_table_privilege(self, node=None): with And("I grant INSERT privilege"): node.query(f"GRANT INSERT ON {table_name} TO {user_name}") - with Then("I try to create a table without select privilege on the table"): - node.query(f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I try to create a table without select privilege on the table" + ): + node.query( + f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_without_insert_privilege(self, node=None): """Check that user is unable to create a table without insert @@ -287,13 +331,20 @@ def create_without_insert_privilege(self, node=None): with And("I grant SELECT privilege"): node.query(f"GRANT SELECT ON {source_table_name} TO {user_name}") - with Then("I try to create a table without select privilege on the table"): - node.query(f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I try to create a table without select privilege on the table" + ): + node.query( + f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_source_table_privilege_granted_directly_or_via_role(self, node=None): """Check that a user is able to create a table if and only if the user has create table privilege and @@ -307,21 +358,25 @@ def create_with_source_table_privilege_granted_directly_or_via_role(self, node=N with user(node, f"{user_name}"): - Scenario(test=create_with_source_table_privilege, - name="create with create table and select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create table and select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_source_table_privilege, - name="create with create table and select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create table and select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Check that user is unable to create a table without SELECT privilege on the source table. - """ + """Check that user is unable to create a table without SELECT privilege on the source table.""" table_name = f"table_{getuid()}" source_table_name = f"source_table_{getuid()}" @@ -338,16 +393,22 @@ def create_with_source_table_privilege(self, user_name, grant_target_name, node= node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") with And("I grant SELECT privilege"): - node.query(f"GRANT SELECT ON {source_table_name} TO {grant_target_name}") + node.query( + f"GRANT SELECT ON {source_table_name} TO {grant_target_name}" + ) with And("I try to create a table on the table as the user"): node.query(f"DROP TABLE IF EXISTS {table_name}") - node.query(f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table_name} ENGINE = Memory AS SELECT * FROM {source_table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a table where the stored query has two subqueries @@ -362,16 +423,21 @@ def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None) with user(node, f"{user_name}"): - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_subquery(self, user_name, grant_target_name, node=None): @@ -399,28 +465,71 @@ def create_with_subquery(self, user_name, grant_target_name, node=None): node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") with Then("I attempt to CREATE TABLE as the user with create privilege"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with When("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with When("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")]) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a table where the stored query includes a `JOIN` statement @@ -435,16 +544,21 @@ def create_with_join_query_privilege_granted_directly_or_via_role(self, node=Non with user(node, f"{user_name}"): - Scenario(test=create_with_join_query, - name="create with join query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_query, - name="create with join query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_query(self, grant_target_name, user_name, node=None): @@ -470,28 +584,62 @@ def create_with_join_query(self, grant_target_name, user_name, node=None): node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") with Then("I attempt to create table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with When("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with When("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_union_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a table where the stored query includes a `UNION ALL` statement @@ -506,16 +654,21 @@ def create_with_union_query_privilege_granted_directly_or_via_role(self, node=No with user(node, f"{user_name}"): - Scenario(test=create_with_union_query, - name="create with union query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_union_query, + name="create with union query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_union_query, - name="create with union query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_union_query, + name="create with union query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_union_query(self, grant_target_name, user_name, node=None): @@ -541,30 +694,66 @@ def create_with_union_query(self, grant_target_name, user_name, node=None): node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") with Then("I attempt to create table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with When("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with When("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario -def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): +def create_with_join_union_subquery_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to create a table with a stored query that includes `UNION ALL`, `JOIN` and two subqueries if and only if the user has SELECT privilege on all of the tables, either granted directly or through a role. """ @@ -576,16 +765,21 @@ def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, with user(node, f"{user_name}"): - Scenario(test=create_with_join_union_subquery, - name="create with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_union_subquery, - name="create with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_union_subquery(self, grant_target_name, user_name, node=None): @@ -604,46 +798,103 @@ def create_with_join_union_subquery(self, grant_target_name, user_name, node=Non if node is None: node = self.context.node - with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with table( + node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}" + ): with user(node, f"{user_name}"): try: with When("I grant CREATE TABLE privilege"): - node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") + node.query( + f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}" + ) with And("I grant INSERT privilege"): node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") - with Then("I attempt to create table as the user with CREATE TABLE privilege"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + with Then( + "I attempt to create table as the user with CREATE TABLE privilege" + ): + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=5): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table3_name, table4_name) as tables_granted: + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table3_name, + table4_name, + ) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a table"): node.query(f"DROP TABLE IF EXISTS {table_name}") with Then("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ): with Given("I don't have a table"): node.query(f"DROP TABLE IF EXISTS {table_name}") with Then("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")]) + node.query( + create_table_query.format( + table_name=table_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_with_nested_tables_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a table with a stored query that includes other tables if and only if @@ -657,16 +908,21 @@ def create_with_nested_tables_privilege_granted_directly_or_via_role(self, node= with user(node, f"{user_name}"): - Scenario(test=create_with_nested_tables, - name="create with nested tables, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_nested_tables, + name="create with nested tables, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_nested_tables, - name="create with nested tables, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_nested_tables, + name="create with nested tables, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_nested_tables(self, grant_target_name, user_name, node=None): @@ -692,9 +948,15 @@ def create_with_nested_tables(self, grant_target_name, user_name, node=None): try: with Given("I have some tables"): - node.query(f"CREATE TABLE {table1_name} ENGINE = Memory AS SELECT y FROM {table0_name}") - node.query(f"CREATE TABLE {table3_name} ENGINE = Memory AS SELECT y FROM {table2_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)") - node.query(f"CREATE TABLE {table5_name} ENGINE = Memory AS SELECT y FROM {table4_name} JOIN {table3_name} USING y") + node.query( + f"CREATE TABLE {table1_name} ENGINE = Memory AS SELECT y FROM {table0_name}" + ) + node.query( + f"CREATE TABLE {table3_name} ENGINE = Memory AS SELECT y FROM {table2_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)" + ) + node.query( + f"CREATE TABLE {table5_name} ENGINE = Memory AS SELECT y FROM {table4_name} JOIN {table3_name} USING y" + ) with When("I grant CREATE TABLE privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -702,32 +964,82 @@ def create_with_nested_tables(self, grant_target_name, user_name, node=None): with And("I grant INSERT privilege"): node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") - with Then("I attempt to create table as the user with CREATE TABLE privilege"): - node.query(create_table_query.format(table_name=table_name, table5_name=table5_name, table6_name=table6_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + with Then( + "I attempt to create table as the user with CREATE TABLE privilege" + ): + node.query( + create_table_query.format( + table_name=table_name, + table5_name=table5_name, + table6_name=table6_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,2,3,7,11,15,31,39,79,95],permutations(table_count=7))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, table5_name, table6_name, table3_name, table4_name, table1_name, table2_name, table0_name) as tables_granted: + for permutation in ( + [0, 1, 2, 3, 7, 11, 15, 31, 39, 79, 95], + permutations(table_count=7), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + table5_name, + table6_name, + table3_name, + table4_name, + table1_name, + table2_name, + table0_name, + ) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a table"): node.query(f"DROP TABLE IF EXISTS {table3_name}") with Then("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table5_name=table5_name, table6_name=table6_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_table_query.format( + table_name=table_name, + table5_name=table5_name, + table6_name=table6_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=7))+1, - grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name, table5_name, table6_name): + with grant_select_on_table( + node, + max(permutations(table_count=7)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + table5_name, + table6_name, + ): with Given("I don't have a table"): node.query(f"DROP TABLE IF EXISTS {table_name}") with Then("I attempt to create a table as the user"): - node.query(create_table_query.format(table_name=table_name, table5_name=table5_name, table6_name=table6_name), - settings = [("user", f"{user_name}")]) + node.query( + create_table_query.format( + table_name=table_name, + table5_name=table5_name, + table6_name=table6_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally(f"I drop {table_name}"): @@ -742,10 +1054,10 @@ def create_with_nested_tables(self, grant_target_name, user_name, node=None): with And(f"I drop {table5_name}"): node.query(f"DROP TABLE IF EXISTS {table5_name}") + @TestScenario def create_as_another_table(self, node=None): - """Check that user is able to create a table as another table with only CREATE TABLE privilege. - """ + """Check that user is able to create a table as another table with only CREATE TABLE privilege.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" source_table_name = f"source_table_{getuid()}" @@ -762,16 +1074,19 @@ def create_as_another_table(self, node=None): node.query(f"GRANT CREATE TABLE ON {table_name} TO {user_name}") with Then("I try to create a table as another table"): - node.query(f"CREATE TABLE {table_name} AS {source_table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table_name} AS {source_table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the tables"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_as_numbers(self, node=None): - """Check that user is able to create a table as numbers table function. - """ + """Check that user is able to create a table as numbers table function.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -789,16 +1104,19 @@ def create_as_numbers(self, node=None): node.query(f"GRANT INSERT ON {table_name} TO {user_name}") with Then("I try to create a table without select privilege on the table"): - node.query(f"CREATE TABLE {table_name} AS numbers(5)", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table_name} AS numbers(5)", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the tables"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestScenario def create_as_merge(self, node=None): - """Check that user is able to create a table as merge table function. - """ + """Check that user is able to create a table as merge table function.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" source_table_name = f"source_table_{getuid()}" @@ -818,20 +1136,23 @@ def create_as_merge(self, node=None): node.query(f"GRANT SELECT ON {source_table_name} TO {user_name}") with Then("I try to create a table as another table"): - node.query(f"CREATE TABLE {table_name} AS merge(default,'{source_table_name}')", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table_name} AS merge(default,'{source_table_name}')", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the tables"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_CreateTable("1.0"), ) @Name("create table") def feature(self, stress=None, node="clickhouse1"): - """Check the RBAC functionality of CREATE TABLE. - """ + """Check the RBAC functionality of CREATE TABLE.""" self.context.node = self.context.cluster.node(node) if stress is not None: diff --git a/tests/testflows/rbac/tests/privileges/create/create_temp_table.py b/tests/testflows/rbac/tests/privileges/create/create_temp_table.py index ac38e0269cf..0cc3211bddf 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_temp_table.py +++ b/tests/testflows/rbac/tests/privileges/create/create_temp_table.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute CREATE TEMPORARY TABLE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute CREATE TEMPORARY TABLE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute CREATE TEMPORARY TABLE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute CREATE TEMPORARY TABLE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute CREATE TEMPORARY TABLE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -43,8 +51,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to create a temporary table without privilege"): - node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the temporary table"): @@ -55,10 +67,15 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant create temporary table privilege"): - node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + node.query( + f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}" + ) with Then("I attempt to create aa temporary table"): - node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)]) + node.query( + f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + ) finally: with Finally("I drop the temporary table"): @@ -69,14 +86,22 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create temporary table privilege"): - node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + node.query( + f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}" + ) with And("I revoke the create temporary table privilege"): - node.query(f"REVOKE CREATE TEMPORARY TABLE ON *.* FROM {grant_target_name}") + node.query( + f"REVOKE CREATE TEMPORARY TABLE ON *.* FROM {grant_target_name}" + ) with Then("I attempt to create a temporary table"): - node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the temporary table"): @@ -87,14 +112,20 @@ def privilege_check(grant_target_name, user_name, node=None): try: with When("I grant the create temporary table privilege"): - node.query(f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}") + node.query( + f"GRANT CREATE TEMPORARY TABLE ON *.* TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to create a temporary table"): - node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the temporary table"): @@ -108,22 +139,25 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to create aa temporary table"): - node.query(f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", settings = [("user", user_name)]) + node.query( + f"CREATE TEMPORARY TABLE {temp_table_name} (x Int8)", + settings=[("user", user_name)], + ) finally: with Finally("I drop the temporary table"): node.query(f"DROP TEMPORARY TABLE IF EXISTS {temp_table_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_CreateTemporaryTable("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("create temporary table") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of CREATE TEMPORARY TABLE. - """ + """Check the RBAC functionality of CREATE TEMPORARY TABLE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -131,5 +165,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/create/create_user.py b/tests/testflows/rbac/tests/privileges/create/create_user.py index b055deecea2..c2722b5b9b0 100644 --- a/tests/testflows/rbac/tests/privileges/create/create_user.py +++ b/tests/testflows/rbac/tests/privileges/create/create_user.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def create_user_granted_directly(self, node=None): - """Check that a user is able to execute `CREATE USER` with privileges are granted directly. - """ + """Check that a user is able to execute `CREATE USER` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def create_user_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=create_user, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in create_user.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=create_user, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in create_user.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def create_user_granted_via_role(self, node=None): - """Check that a user is able to execute `CREATE USER` with privileges are granted through a role. - """ + """Check that a user is able to execute `CREATE USER` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def create_user_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=create_user, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in create_user.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=create_user, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in create_user.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("CREATE USER",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("CREATE USER",), + ], +) def create_user(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `CREATE USER` when they have the necessary privilege. - """ + """Check that user is only able to execute `CREATE USER` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -68,8 +85,12 @@ def create_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't create a user"): - node.query(f"CREATE USER {create_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE USER {create_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): @@ -83,7 +104,10 @@ def create_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a user"): - node.query(f"CREATE USER {create_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE USER {create_user_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): @@ -97,12 +121,16 @@ def create_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can create a user"): - node.query(f"CREATE USER {create_user_name} ON CLUSTER sharded_cluster", - settings = [("user", f"{user_name}")]) + node.query( + f"CREATE USER {create_user_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP USER IF EXISTS {create_user_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP USER IF EXISTS {create_user_name} ON CLUSTER sharded_cluster" + ) with Scenario("CREATE USER with revoked privilege"): create_user_name = f"create_user_{getuid()}" @@ -115,17 +143,21 @@ def create_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user can't create a user"): - node.query(f"CREATE USER {create_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE USER {create_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): node.query(f"DROP USER IF EXISTS {create_user_name}") + @TestSuite def default_role_granted_directly(self, node=None): - """Check that a user is able to execute `CREATE USER` with `DEFAULT ROLE` with privileges are granted directly. - """ + """Check that a user is able to execute `CREATE USER` with `DEFAULT ROLE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -136,10 +168,10 @@ def default_role_granted_directly(self, node=None): Suite(test=default_role)(grant_target_name=user_name, user_name=user_name) + @TestSuite def default_role_granted_via_role(self, node=None): - """Check that a user is able to execute `CREATE USER` with `DEFAULT ROLE` with privileges are granted through a role. - """ + """Check that a user is able to execute `CREATE USER` with `DEFAULT ROLE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -154,6 +186,7 @@ def default_role_granted_via_role(self, node=None): Suite(test=default_role)(grant_target_name=role_name, user_name=user_name) + @TestSuite @Requirements( RQ_SRS_006_RBAC_Privileges_CreateUser_DefaultRole("1.0"), @@ -177,8 +210,12 @@ def default_role(self, grant_target_name, user_name, node=None): node.query(f"GRANT CREATE USER ON *.* TO {grant_target_name}") with Then("I check the user can't create a user"): - node.query(f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): @@ -194,10 +231,15 @@ def default_role(self, grant_target_name, user_name, node=None): node.query(f"GRANT CREATE USER ON *.* TO {grant_target_name}") with And(f"I grant the role with ADMIN OPTION"): - node.query(f"GRANT {default_role_name} TO {grant_target_name} WITH ADMIN OPTION") + node.query( + f"GRANT {default_role_name} TO {grant_target_name} WITH ADMIN OPTION" + ) with Then("I check the user can create a user"): - node.query(f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", settings=[("user",user_name)]) + node.query( + f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", + settings=[("user", user_name)], + ) finally: with Finally("I drop the user"): @@ -209,21 +251,29 @@ def default_role(self, grant_target_name, user_name, node=None): try: with Given("I have role on a cluster"): - node.query(f"CREATE ROLE {default_role_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE ROLE {default_role_name} ON CLUSTER sharded_cluster" + ) with When(f"I grant CREATE USER"): node.query(f"GRANT CREATE USER ON *.* TO {grant_target_name}") with And(f"I grant the role with ADMIN OPTION"): - node.query(f"GRANT {default_role_name} TO {grant_target_name} WITH ADMIN OPTION") + node.query( + f"GRANT {default_role_name} TO {grant_target_name} WITH ADMIN OPTION" + ) with Then("I check the user can create a user"): - node.query(f"CREATE USER {create_user_name} ON CLUSTER sharded_cluster DEFAULT ROLE {default_role_name}", - settings = [("user", f"{user_name}")]) + node.query( + f"CREATE USER {create_user_name} ON CLUSTER sharded_cluster DEFAULT ROLE {default_role_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP USER IF EXISTS {create_user_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP USER IF EXISTS {create_user_name} ON CLUSTER sharded_cluster" + ) with And("I drop the role from the cluster"): node.query(f"DROP ROLE {default_role_name} ON CLUSTER sharded_cluster") @@ -238,14 +288,20 @@ def default_role(self, grant_target_name, user_name, node=None): node.query(f"GRANT CREATE USER ON *.* TO {grant_target_name}") with And(f"I grant the role with ADMIN OPTION"): - node.query(f"GRANT {default_role_name} TO {grant_target_name} WITH ADMIN OPTION") + node.query( + f"GRANT {default_role_name} TO {grant_target_name} WITH ADMIN OPTION" + ) with And(f"I revoke the role"): node.query(f"REVOKE {default_role_name} FROM {grant_target_name}") with Then("I check the user can't create a user"): - node.query(f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): @@ -258,25 +314,30 @@ def default_role(self, grant_target_name, user_name, node=None): with role(node, default_role_name): try: with When(f"I grant ACCESS MANAGEMENT "): - node.query(f"GRANT ACCESS MANAGEMENT ON *.* TO {grant_target_name}") + node.query( + f"GRANT ACCESS MANAGEMENT ON *.* TO {grant_target_name}" + ) with Then("I check the user can create a user"): - node.query(f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", settings=[("user",user_name)]) + node.query( + f"CREATE USER {create_user_name} DEFAULT ROLE {default_role_name}", + settings=[("user", user_name)], + ) finally: with Finally("I drop the user"): node.query(f"DROP USER IF EXISTS {create_user_name}") + @TestFeature @Name("create user") @Requirements( RQ_SRS_006_RBAC_Privileges_CreateUser("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of CREATE USER. - """ + """Check the RBAC functionality of CREATE USER.""" self.context.node = self.context.cluster.node(node) Suite(run=create_user_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_database.py b/tests/testflows/rbac/tests/privileges/detach/detach_database.py index 12eeb39aa1b..848de0bb682 100644 --- a/tests/testflows/rbac/tests/privileges/detach/detach_database.py +++ b/tests/testflows/rbac/tests/privileges/detach/detach_database.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DETACH DATABASE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DETACH DATABASE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DETACH DATABASE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DETACH DATABASE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH DATABASE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DETACH DATABASE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege", setup=instrument_clickhouse_server_log): @@ -46,8 +54,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to detach the database"): - node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the database", flags=TE): node.query(f"ATTACH DATABASE IF NOT EXISTS {db_name}") @@ -65,7 +77,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") with Then("I attempt to detach a database"): - node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)]) + node.query(f"DETACH DATABASE {db_name}", settings=[("user", user_name)]) finally: with Finally("I reattach the database", flags=TE): @@ -84,11 +96,17 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") with And("I revoke the drop database privilege"): - node.query(f"REVOKE DROP DATABASE ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE DROP DATABASE ON {db_name}.* FROM {grant_target_name}" + ) with Then("I attempt to detach a database"): - node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the database", flags=TE): @@ -110,8 +128,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to detach a database"): - node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the database", flags=TE): @@ -130,7 +152,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to detach a database"): - node.query(f"DETACH DATABASE {db_name}", settings = [("user", user_name)]) + node.query(f"DETACH DATABASE {db_name}", settings=[("user", user_name)]) finally: with Finally("I reattach the database", flags=TE): @@ -138,16 +160,16 @@ def privilege_check(grant_target_name, user_name, node=None): with And("I drop the database", flags=TE): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DetachDatabase("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("detach database") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DETACH DATABASE. - """ + """Check the RBAC functionality of DETACH DATABASE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -155,5 +177,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py b/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py index 17b37ce6dc0..bd24d0a400a 100644 --- a/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py +++ b/tests/testflows/rbac/tests/privileges/detach/detach_dictionary.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DETACH DICTIONARY when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DETACH DICTIONARY when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DETACH DICTIONARY with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DETACH DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH DICTIONARY with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DETACH DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -37,7 +45,9 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant the user NONE privilege"): node.query(f"GRANT NONE TO {grant_target_name}") @@ -46,7 +56,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to detach a dictionary without privilege"): - node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"DETACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the dictionary", flags=TE): @@ -59,13 +74,19 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant drop dictionary privilege"): - node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with Then("I attempt to detach a dictionary"): - node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)]) + node.query( + f"DETACH DICTIONARY {dict_name}", settings=[("user", user_name)] + ) finally: with Finally("I reattach the dictionary", flags=TE): @@ -78,16 +99,27 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant the drop dictionary privilege"): - node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with And("I revoke the drop dictionary privilege"): - node.query(f"REVOKE DROP DICTIONARY ON {dict_name} FROM {grant_target_name}") + node.query( + f"REVOKE DROP DICTIONARY ON {dict_name} FROM {grant_target_name}" + ) with Then("I attempt to detach a dictionary"): - node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"DETACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the dictionary", flags=TE): @@ -100,16 +132,25 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant the drop dictionary privilege"): - node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to detach a dictionary"): - node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"DETACH DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the dictionary", flags=TE): @@ -122,13 +163,17 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to detach a dictionary"): - node.query(f"DETACH DICTIONARY {dict_name}", settings = [("user", user_name)]) + node.query( + f"DETACH DICTIONARY {dict_name}", settings=[("user", user_name)] + ) finally: with Finally("I reattach the dictionary", flags=TE): @@ -136,16 +181,16 @@ def privilege_check(grant_target_name, user_name, node=None): with And("I drop the dictionary", flags=TE): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DetachDictionary("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("detach dictionary") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DETACH DICTIONARY. - """ + """Check the RBAC functionality of DETACH DICTIONARY.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -153,5 +198,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_table.py b/tests/testflows/rbac/tests/privileges/detach/detach_table.py index b5a01b361fc..421464ac501 100644 --- a/tests/testflows/rbac/tests/privileges/detach/detach_table.py +++ b/tests/testflows/rbac/tests/privileges/detach/detach_table.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DETACH TABLE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DETACH TABLE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DETACH TABLE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DETACH TABLE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH TABLE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DETACH TABLE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -46,8 +54,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with When("I attempt to detach a table without privilege"): - node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the table", flags=TE): @@ -66,7 +78,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP TABLE ON *.* TO {grant_target_name}") with Then("I attempt to detach a table"): - node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)]) + node.query(f"DETACH TABLE {table_name}", settings=[("user", user_name)]) finally: with Finally("I reattach the table", flags=TE): @@ -88,8 +100,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE DROP TABLE ON *.* FROM {grant_target_name}") with Then("I attempt to detach a table"): - node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the table", flags=TE): @@ -111,8 +127,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to detach a table"): - node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the table", flags=TE): @@ -131,7 +151,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to detach a table"): - node.query(f"DETACH TABLE {table_name}", settings = [("user", user_name)]) + node.query(f"DETACH TABLE {table_name}", settings=[("user", user_name)]) finally: with Finally("I reattach the table", flags=TE): @@ -139,16 +159,16 @@ def privilege_check(grant_target_name, user_name, node=None): with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DetachTable("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("detach table") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DETACH TABLE. - """ + """Check the RBAC functionality of DETACH TABLE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -156,5 +176,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/detach/detach_view.py b/tests/testflows/rbac/tests/privileges/detach/detach_view.py index c3c9f70a35a..03dd6247ccb 100644 --- a/tests/testflows/rbac/tests/privileges/detach/detach_view.py +++ b/tests/testflows/rbac/tests/privileges/detach/detach_view.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DETACH VIEW when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DETACH VIEW when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DETACH VIEW with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DETACH VIEW with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DETACH VIEW with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DETACH VIEW with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -46,8 +54,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with When("I attempt to drop a view without privilege"): - node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH VIEW {view_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the view as a table", flags=TE): @@ -66,7 +78,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to drop a view"): - node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)]) + node.query(f"DETACH VIEW {view_name}", settings=[("user", user_name)]) finally: with Finally("I reattach the view as a table", flags=TE): @@ -88,8 +100,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE DROP VIEW ON {view_name} FROM {grant_target_name}") with Then("I attempt to drop a view"): - node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH VIEW {view_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the view as a table", flags=TE): @@ -111,8 +127,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to drop a view"): - node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DETACH VIEW {view_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I reattach the view as a table", flags=TE): @@ -131,7 +151,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to drop a view"): - node.query(f"DETACH VIEW {view_name}", settings = [("user", user_name)]) + node.query(f"DETACH VIEW {view_name}", settings=[("user", user_name)]) finally: with Finally("I reattach the view as a table", flags=TE): @@ -139,16 +159,16 @@ def privilege_check(grant_target_name, user_name, node=None): with And("I drop the table", flags=TE): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DetachView("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("detach view") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DETACH VIEW. - """ + """Check the RBAC functionality of DETACH VIEW.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -156,5 +176,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/dictGet.py b/tests/testflows/rbac/tests/privileges/dictGet.py index 4bee598bb9b..14a38f78f8d 100644 --- a/tests/testflows/rbac/tests/privileges/dictGet.py +++ b/tests/testflows/rbac/tests/privileges/dictGet.py @@ -6,17 +6,21 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def dict_setup(node, table_name, dict_name, type="UInt64"): - """Setup and teardown of table and dictionary needed for the tests. - """ + """Setup and teardown of table and dictionary needed for the tests.""" try: with Given("I have a table"): - node.query(f"CREATE TABLE {table_name} (x UInt64, y UInt64, z {type}) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} (x UInt64, y UInt64, z {type}) ENGINE = Memory" + ) with And("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name} (x UInt64 HIERARCHICAL IS_OBJECT_ID, y UInt64 HIERARCHICAL, z {type}) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' password '' db 'default' table '{table_name}')) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name} (x UInt64 HIERARCHICAL IS_OBJECT_ID, y UInt64 HIERARCHICAL, z {type}) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' password '' db 'default' table '{table_name}')) LIFETIME(0)" + ) yield @@ -27,10 +31,10 @@ def dict_setup(node, table_name, dict_name, type="UInt64"): with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestSuite def dictGet_granted_directly(self, node=None): - """Run dictGet checks with privileges granted directly. - """ + """Run dictGet checks with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -39,15 +43,22 @@ def dictGet_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=dictGet_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictGet_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGet_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictGet_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictGet_granted_via_role(self, node=None): - """Run dictGet checks with privileges granted through a role. - """ + """Run dictGet checks with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -60,25 +71,33 @@ def dictGet_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictGet_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictGet_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGet_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictGet_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("dictGet", "dict"), - ("dictHas", "dict"), - ("dictGetHierarchy", "dict"), - ("dictIsIn", "dict"), -]) -@Requirements( - RQ_SRS_006_RBAC_dictGet_RequiredPrivilege("1.0") +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("dictGet", "dict"), + ("dictHas", "dict"), + ("dictGetHierarchy", "dict"), + ("dictIsIn", "dict"), + ], ) +@Requirements(RQ_SRS_006_RBAC_dictGet_RequiredPrivilege("1.0")) def dictGet_check(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is able to execute `dictGet` if and only if they have the necessary privileges. - """ + """Check that user is able to execute `dictGet` if and only if they have the necessary privileges.""" if node is None: node = self.context.node @@ -100,7 +119,12 @@ def dictGet_check(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to dictGet without privilege"): - node.query(f"SELECT dictGet ({dict_name},'y',toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGet ({dict_name},'y',toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -110,7 +134,10 @@ def dictGet_check(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I attempt to dictGet with privilege"): - node.query(f"SELECT dictGet ({dict_name},'y',toUInt64(1))", settings = [("user", user_name)]) + node.query( + f"SELECT dictGet ({dict_name},'y',toUInt64(1))", + settings=[("user", user_name)], + ) with Scenario("user with revoked privilege"): @@ -123,12 +150,17 @@ def dictGet_check(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with When("I attempt to dictGet without privilege"): - node.query(f"SELECT dictGet ({dict_name},'y',toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGet ({dict_name},'y',toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def dictGetOrDefault_granted_directly(self, node=None): - """Run dictGetOrDefault checks with privileges granted directly. - """ + """Run dictGetOrDefault checks with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -137,15 +169,22 @@ def dictGetOrDefault_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=dictGetOrDefault_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictGetOrDefault_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGetOrDefault_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictGetOrDefault_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictGetOrDefault_granted_via_role(self, node=None): - """Run dictGetOrDefault checks with privileges granted through a role. - """ + """Run dictGetOrDefault checks with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -158,25 +197,35 @@ def dictGetOrDefault_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictGetOrDefault_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictGetOrDefault_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGetOrDefault_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictGetOrDefault_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("dictGet", "dict"), - ("dictHas", "dict"), - ("dictGetHierarchy", "dict"), - ("dictIsIn", "dict"), -]) -@Requirements( - RQ_SRS_006_RBAC_dictGet_OrDefault_RequiredPrivilege("1.0") +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("dictGet", "dict"), + ("dictHas", "dict"), + ("dictGetHierarchy", "dict"), + ("dictIsIn", "dict"), + ], ) -def dictGetOrDefault_check(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is able to execute `dictGetOrDefault` if and only if they have the necessary privileges. - """ +@Requirements(RQ_SRS_006_RBAC_dictGet_OrDefault_RequiredPrivilege("1.0")) +def dictGetOrDefault_check( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is able to execute `dictGetOrDefault` if and only if they have the necessary privileges.""" if node is None: node = self.context.node @@ -198,7 +247,12 @@ def dictGetOrDefault_check(self, privilege, on, grant_target_name, user_name, no node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to dictGetOrDefault without privilege"): - node.query(f"SELECT dictGetOrDefault ({dict_name},'y',toUInt64(1),toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGetOrDefault ({dict_name},'y',toUInt64(1),toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -208,7 +262,10 @@ def dictGetOrDefault_check(self, privilege, on, grant_target_name, user_name, no node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I attempt to dictGetOrDefault with privilege"): - node.query(f"SELECT dictGetOrDefault ({dict_name},'y',toUInt64(1),toUInt64(1))", settings = [("user", user_name)]) + node.query( + f"SELECT dictGetOrDefault ({dict_name},'y',toUInt64(1),toUInt64(1))", + settings=[("user", user_name)], + ) with Scenario("user with revoked privilege"): @@ -221,12 +278,17 @@ def dictGetOrDefault_check(self, privilege, on, grant_target_name, user_name, no node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with When("I attempt to dictGetOrDefault without privilege"): - node.query(f"SELECT dictGetOrDefault ({dict_name},'y',toUInt64(1),toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGetOrDefault ({dict_name},'y',toUInt64(1),toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def dictHas_granted_directly(self, node=None): - """Run dictHas checks with privileges granted directly. - """ + """Run dictHas checks with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -235,15 +297,22 @@ def dictHas_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=dictHas_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictHas_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictHas_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictHas_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictHas_granted_via_role(self, node=None): - """Run checks with privileges granted through a role. - """ + """Run checks with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -256,25 +325,33 @@ def dictHas_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictHas_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictHas_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictHas_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictHas_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("dictGet", "dict"), - ("dictHas", "dict"), - ("dictGetHierarchy", "dict"), - ("dictIsIn", "dict"), -]) -@Requirements( - RQ_SRS_006_RBAC_dictHas_RequiredPrivilege("1.0") +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("dictGet", "dict"), + ("dictHas", "dict"), + ("dictGetHierarchy", "dict"), + ("dictIsIn", "dict"), + ], ) +@Requirements(RQ_SRS_006_RBAC_dictHas_RequiredPrivilege("1.0")) def dictHas_check(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is able to execute `dictHas` if and only if they have the necessary privileges. - """ + """Check that user is able to execute `dictHas` if and only if they have the necessary privileges.""" if node is None: node = self.context.node @@ -296,7 +373,12 @@ def dictHas_check(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to dictHas without privilege"): - node.query(f"SELECT dictHas({dict_name},toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictHas({dict_name},toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -306,7 +388,10 @@ def dictHas_check(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I attempt to dictHas with privilege"): - node.query(f"SELECT dictHas({dict_name},toUInt64(1))", settings = [("user", user_name)]) + node.query( + f"SELECT dictHas({dict_name},toUInt64(1))", + settings=[("user", user_name)], + ) with Scenario("user with revoked privilege"): @@ -319,12 +404,17 @@ def dictHas_check(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with When("I attempt to dictHas without privilege"): - node.query(f"SELECT dictHas({dict_name},toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictHas({dict_name},toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def dictGetHierarchy_granted_directly(self, node=None): - """Run dictGetHierarchy checks with privileges granted directly. - """ + """Run dictGetHierarchy checks with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -332,15 +422,22 @@ def dictGetHierarchy_granted_directly(self, node=None): node = self.context.node with user(node, f"{user_name}"): - Suite(run=dictGetHierarchy_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictGetHierarchy_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGetHierarchy_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictGetHierarchy_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictGetHierarchy_granted_via_role(self, node=None): - """Run checks with privileges granted through a role. - """ + """Run checks with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -353,25 +450,35 @@ def dictGetHierarchy_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictGetHierarchy_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictGetHierarchy_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGetHierarchy_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictGetHierarchy_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("dictGet", "dict"), - ("dictHas", "dict"), - ("dictGetHierarchy", "dict"), - ("dictIsIn", "dict"), -]) -@Requirements( - RQ_SRS_006_RBAC_dictGetHierarchy_RequiredPrivilege("1.0") +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("dictGet", "dict"), + ("dictHas", "dict"), + ("dictGetHierarchy", "dict"), + ("dictIsIn", "dict"), + ], ) -def dictGetHierarchy_check(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is able to execute `dictGetHierarchy` if and only if they have the necessary privileges. - """ +@Requirements(RQ_SRS_006_RBAC_dictGetHierarchy_RequiredPrivilege("1.0")) +def dictGetHierarchy_check( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is able to execute `dictGetHierarchy` if and only if they have the necessary privileges.""" if node is None: node = self.context.node @@ -393,7 +500,12 @@ def dictGetHierarchy_check(self, privilege, on, grant_target_name, user_name, no node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to dictGetHierarchy without privilege"): - node.query(f"SELECT dictGetHierarchy({dict_name},toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGetHierarchy({dict_name},toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -403,7 +515,10 @@ def dictGetHierarchy_check(self, privilege, on, grant_target_name, user_name, no node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I attempt to dictGetHierarchy with privilege"): - node.query(f"SELECT dictGetHierarchy({dict_name},toUInt64(1))", settings = [("user", user_name)]) + node.query( + f"SELECT dictGetHierarchy({dict_name},toUInt64(1))", + settings=[("user", user_name)], + ) with Scenario("user with revoked privilege"): @@ -416,12 +531,17 @@ def dictGetHierarchy_check(self, privilege, on, grant_target_name, user_name, no node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with When("I attempt to dictGetHierarchy without privilege"): - node.query(f"SELECT dictGetHierarchy({dict_name},toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGetHierarchy({dict_name},toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def dictIsIn_granted_directly(self, node=None): - """Run dictIsIn checks with privileges granted directly. - """ + """Run dictIsIn checks with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -429,15 +549,22 @@ def dictIsIn_granted_directly(self, node=None): node = self.context.node with user(node, f"{user_name}"): - Suite(run=dictIsIn_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictIsIn_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictIsIn_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictIsIn_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictIsIn_granted_via_role(self, node=None): - """Run checks with privileges granted through a role. - """ + """Run checks with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -450,25 +577,33 @@ def dictIsIn_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictIsIn_check, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictIsIn_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictIsIn_check, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictIsIn_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("dictGet", "dict"), - ("dictHas", "dict"), - ("dictGetHierarchy", "dict"), - ("dictIsIn", "dict"), -]) -@Requirements( - RQ_SRS_006_RBAC_dictIsIn_RequiredPrivilege("1.0") +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("dictGet", "dict"), + ("dictHas", "dict"), + ("dictGetHierarchy", "dict"), + ("dictIsIn", "dict"), + ], ) +@Requirements(RQ_SRS_006_RBAC_dictIsIn_RequiredPrivilege("1.0")) def dictIsIn_check(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is able to execute `dictIsIn` if and only if they have the necessary privileges. - """ + """Check that user is able to execute `dictIsIn` if and only if they have the necessary privileges.""" if node is None: node = self.context.node @@ -490,7 +625,12 @@ def dictIsIn_check(self, privilege, on, grant_target_name, user_name, node=None) node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to dictIsIn without privilege"): - node.query(f"SELECT dictIsIn({dict_name},toUInt64(1),toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictIsIn({dict_name},toUInt64(1),toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -500,7 +640,10 @@ def dictIsIn_check(self, privilege, on, grant_target_name, user_name, node=None) node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I attempt to dictIsIn with privilege"): - node.query(f"SELECT dictIsIn({dict_name},toUInt64(1),toUInt64(1))", settings = [("user", user_name)]) + node.query( + f"SELECT dictIsIn({dict_name},toUInt64(1),toUInt64(1))", + settings=[("user", user_name)], + ) with Scenario("user with revoked privilege"): @@ -513,28 +656,36 @@ def dictIsIn_check(self, privilege, on, grant_target_name, user_name, node=None) node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with When("I attempt to dictIsIn without privilege"): - node.query(f"SELECT dictIsIn({dict_name},toUInt64(1),toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictIsIn({dict_name},toUInt64(1),toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite -@Examples("type",[ - ("Int8",), - ("Int16",), - ("Int32",), - ("Int64",), - ("UInt8",), - ("UInt16",), - ("UInt32",), - ("UInt64",), - ("Float32",), - ("Float64",), - ("Date",), - ("DateTime",), - ("UUID",), - ("String",), -]) +@Examples( + "type", + [ + ("Int8",), + ("Int16",), + ("Int32",), + ("Int64",), + ("UInt8",), + ("UInt16",), + ("UInt32",), + ("UInt64",), + ("Float32",), + ("Float64",), + ("Date",), + ("DateTime",), + ("UUID",), + ("String",), + ], +) def dictGetType_granted_directly(self, type, node=None): - """Run checks on dictGet with a type specified with privileges granted directly. - """ + """Run checks on dictGet with a type specified with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -542,31 +693,41 @@ def dictGetType_granted_directly(self, type, node=None): node = self.context.node with user(node, f"{user_name}"): - Suite(run=dictGetType_check, - examples=Examples("privilege on grant_target_name user_name type", [ - tuple(list(row)+[user_name,user_name,type]) for row in dictGetType_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGetType_check, + examples=Examples( + "privilege on grant_target_name user_name type", + [ + tuple(list(row) + [user_name, user_name, type]) + for row in dictGetType_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite -@Examples("type",[ - ("Int8",), - ("Int16",), - ("Int32",), - ("Int64",), - ("UInt8",), - ("UInt16",), - ("UInt32",), - ("UInt64",), - ("Float32",), - ("Float64",), - ("Date",), - ("DateTime",), - ("UUID",), - ("String",), -]) +@Examples( + "type", + [ + ("Int8",), + ("Int16",), + ("Int32",), + ("Int64",), + ("UInt8",), + ("UInt16",), + ("UInt32",), + ("UInt64",), + ("Float32",), + ("Float64",), + ("Date",), + ("DateTime",), + ("UUID",), + ("String",), + ], +) def dictGetType_granted_via_role(self, type, node=None): - """Run checks on dictGet with a type specified with privileges granted through a role. - """ + """Run checks on dictGet with a type specified with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -579,25 +740,35 @@ def dictGetType_granted_via_role(self, type, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictGetType_check, - examples=Examples("privilege on grant_target_name user_name type", [ - tuple(list(row)+[role_name,user_name,type]) for row in dictGetType_check.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictGetType_check, + examples=Examples( + "privilege on grant_target_name user_name type", + [ + tuple(list(row) + [role_name, user_name, type]) + for row in dictGetType_check.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("dictGet", "dict"), - ("dictHas", "dict"), - ("dictGetHierarchy", "dict"), - ("dictIsIn", "dict"), -]) -@Requirements( - RQ_SRS_006_RBAC_dictGet_Type_RequiredPrivilege("1.0") +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("dictGet", "dict"), + ("dictHas", "dict"), + ("dictGetHierarchy", "dict"), + ("dictIsIn", "dict"), + ], ) -def dictGetType_check(self, privilege, on, grant_target_name, user_name, type, node=None): - """Check that user is able to execute `dictGet` if and only if they have the necessary privileges. - """ +@Requirements(RQ_SRS_006_RBAC_dictGet_Type_RequiredPrivilege("1.0")) +def dictGetType_check( + self, privilege, on, grant_target_name, user_name, type, node=None +): + """Check that user is able to execute `dictGet` if and only if they have the necessary privileges.""" if node is None: node = self.context.node @@ -619,7 +790,12 @@ def dictGetType_check(self, privilege, on, grant_target_name, user_name, type, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to dictGet without privilege"): - node.query(f"SELECT dictGet{type}({dict_name},'z',toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGet{type}({dict_name},'z',toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -629,7 +805,10 @@ def dictGetType_check(self, privilege, on, grant_target_name, user_name, type, n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I attempt to dictGet with privilege"): - node.query(f"SELECT dictGet{type}({dict_name},'z',toUInt64(1))", settings = [("user", user_name)]) + node.query( + f"SELECT dictGet{type}({dict_name},'z',toUInt64(1))", + settings=[("user", user_name)], + ) with Scenario("user with revoked privilege"): @@ -642,18 +821,23 @@ def dictGetType_check(self, privilege, on, grant_target_name, user_name, type, n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with When("I attempt to dictGet without privilege"): - node.query(f"SELECT dictGet{type}({dict_name},'z',toUInt64(1))", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT dictGet{type}({dict_name},'z',toUInt64(1))", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_dictGet_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("dictGet") def feature(self, stress=None, node="clickhouse1"): - """Check the RBAC functionality of dictGet. - """ + """Check the RBAC functionality of dictGet.""" self.context.node = self.context.cluster.node(node) if stress is not None: @@ -661,24 +845,84 @@ def feature(self, stress=None, node="clickhouse1"): with Pool(20) as pool: try: - Suite(run=dictGet_granted_directly, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictGet_granted_via_role, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictGetOrDefault_granted_directly, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictGetOrDefault_granted_via_role, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictHas_granted_directly, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictHas_granted_via_role, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictGetHierarchy_granted_directly, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictGetHierarchy_granted_via_role, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictIsIn_granted_directly, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) - Suite(run=dictIsIn_granted_via_role, setup=instrument_clickhouse_server_log, parallel=True, executor=pool) + Suite( + run=dictGet_granted_directly, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictGet_granted_via_role, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictGetOrDefault_granted_directly, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictGetOrDefault_granted_via_role, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictHas_granted_directly, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictHas_granted_via_role, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictGetHierarchy_granted_directly, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictGetHierarchy_granted_via_role, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictIsIn_granted_directly, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) + Suite( + run=dictIsIn_granted_via_role, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + ) for example in dictGetType_granted_directly.examples: - type, = example - args = {"type" : type} + (type,) = example + args = {"type": type} with Example(example): - Suite(test=dictGetType_granted_directly, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) - Suite(test=dictGetType_granted_via_role, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Suite( + test=dictGetType_granted_directly, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) + Suite( + test=dictGetType_granted_via_role, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/distributed_table.py b/tests/testflows/rbac/tests/privileges/distributed_table.py index c99e6363b4d..5291a5609db 100755 --- a/tests/testflows/rbac/tests/privileges/distributed_table.py +++ b/tests/testflows/rbac/tests/privileges/distributed_table.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestStep(Given) def user(self, name, node=None): - """Create a user with a given name. - """ + """Create a user with a given name.""" if node is None: node = self.context.node @@ -20,10 +20,10 @@ def user(self, name, node=None): with Finally(f"I delete user {name}"): node.query(f"DROP USER IF EXISTS {name} ON CLUSTER one_shard_cluster") + @TestStep(Given) def role(self, name, node=None): - """Create a role with a given name. - """ + """Create a role with a given name.""" if node is None: node = self.context.node @@ -35,17 +35,19 @@ def role(self, name, node=None): with Finally(f"I delete role {name}"): node.query(f"DROP ROLE IF EXISTS {name} ON CLUSTER one_shard_cluster") + @TestStep(Given) def table(self, name, cluster=None, node=None): - """Create a table with given name and on specified cluster, if specified. - """ + """Create a table with given name and on specified cluster, if specified.""" if node is None: node = self.context.node try: if cluster: with Given(f"I create table {name}"): node.query(f"DROP TABLE IF EXISTS {name}") - node.query(f"CREATE TABLE {name} ON CLUSTER {cluster} (a UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {name} ON CLUSTER {cluster} (a UInt64) ENGINE = Memory" + ) else: with Given(f"I create table {name}"): node.query(f"DROP TABLE IF EXISTS {name}") @@ -59,26 +61,26 @@ def table(self, name, cluster=None, node=None): with Finally(f"I delete role {name}"): node.query(f"DROP ROLE IF EXISTS {name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_DistributedTable_Create("1.0"), ) def create(self): - """Check the RBAC functionality of distributed table with CREATE. - """ - create_scenarios=[ - create_without_privilege, - create_with_privilege_granted_directly_or_via_role, - create_with_all_privilege_granted_directly_or_via_role, + """Check the RBAC functionality of distributed table with CREATE.""" + create_scenarios = [ + create_without_privilege, + create_with_privilege_granted_directly_or_via_role, + create_with_all_privilege_granted_directly_or_via_role, ] for scenario in create_scenarios: Scenario(run=scenario, setup=instrument_clickhouse_server_log) + @TestScenario def create_without_privilege(self, node=None): - """Check that user is unable to create a distributed table without privileges. - """ + """Check that user is unable to create a distributed table without privileges.""" user_name = f"user_{getuid()}" table0_name = f"table0_{getuid()}" @@ -104,8 +106,13 @@ def create_without_privilege(self, node=None): node.query(f"GRANT USAGE ON *.* TO {user_name}") with Then("I attempt to create the distributed table without privilege"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_privilege_granted_directly_or_via_role(self, node=None): @@ -121,8 +128,9 @@ def create_with_privilege_granted_directly_or_via_role(self, node=None): with Given("I have a user"): user(name=user_name) - Scenario(test=create_with_privilege, - name="create with privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=create_with_privilege, name="create with privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with Given("I have a user"): user(name=user_name) @@ -133,8 +141,10 @@ def create_with_privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=create_with_privilege, - name="create with privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_privilege, name="create with privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_privilege(self, user_name, grant_target_name, node=None): @@ -158,8 +168,12 @@ def create_with_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT CREATE ON {table1_name} TO {grant_target_name}") with Then("I attempt to create the distributed table as the user"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke the create table privilege"): node.query(f"REVOKE CREATE TABLE ON {table1_name} FROM {grant_target_name}") @@ -168,19 +182,27 @@ def create_with_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT REMOTE ON *.* to {grant_target_name}") with Then("I attempt to create the distributed table as the user"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant create table privilege"): node.query(f"GRANT CREATE ON {table1_name} TO {grant_target_name}") with Then("I attempt to create the distributed table as the user"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestScenario def create_with_all_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a distributed table if and only if @@ -195,8 +217,9 @@ def create_with_all_privilege_granted_directly_or_via_role(self, node=None): with Given("I have a user"): user(name=user_name) - Scenario(test=create_with_privilege, - name="create with privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=create_with_privilege, name="create with privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with Given("I have a user"): user(name=user_name) @@ -207,13 +230,14 @@ def create_with_all_privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=create_with_privilege, - name="create with privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_privilege, name="create with privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_privilege(self, user_name, grant_target_name, node=None): - """Grant ALL privilege and check the user is able is create the table. - """ + """Grant ALL privilege and check the user is able is create the table.""" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -231,32 +255,35 @@ def create_with_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I create the distributed table as the user"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_DistributedTable_Select("1.0"), ) def select(self): - """Check the RBAC functionality of distributed table with SELECT. - """ + """Check the RBAC functionality of distributed table with SELECT.""" select_scenarios = [ select_without_privilege, select_with_privilege_granted_directly_or_via_role, - select_with_all_privilege_granted_directly_or_via_role + select_with_all_privilege_granted_directly_or_via_role, ] for scenario in select_scenarios: Scenario(run=scenario, setup=instrument_clickhouse_server_log) + @TestScenario def select_without_privilege(self, node=None): - """Check that user is unable to select from a distributed table without privileges. - """ + """Check that user is unable to select from a distributed table without privileges.""" user_name = f"user_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -275,7 +302,9 @@ def select_without_privilege(self, node=None): table(name=table0_name, cluster=cluster) with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant the user NONE privilege"): node.query(f"GRANT NONE TO {user_name}") @@ -284,12 +313,17 @@ def select_without_privilege(self, node=None): node.query(f"GRANT USAGE ON *.* TO {user_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestScenario def select_with_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a distributed table if and only if @@ -304,8 +338,9 @@ def select_with_privilege_granted_directly_or_via_role(self, node=None): with Given("I have a user"): user(name=user_name) - Scenario(test=select_with_privilege, - name="select with privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=select_with_privilege, name="select with privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with Given("I have a user"): user(name=user_name) @@ -316,8 +351,10 @@ def select_with_privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=select_with_privilege, - name="select with privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_privilege, name="select with privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_privilege(self, user_name, grant_target_name, node=None): @@ -338,14 +375,20 @@ def select_with_privilege(self, user_name, grant_target_name, node=None): table(name=table0_name, cluster=cluster) with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke select privilege on the distributed table"): node.query(f"REVOKE SELECT ON {table1_name} FROM {grant_target_name}") @@ -354,19 +397,26 @@ def select_with_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT SELECT ON {table0_name} to {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant the user select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestScenario def select_with_all_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a distributed table if and only if @@ -381,8 +431,9 @@ def select_with_all_privilege_granted_directly_or_via_role(self, node=None): with Given("I have a user"): user(name=user_name) - Scenario(test=select_with_privilege, - name="select with privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=select_with_privilege, name="select with privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with Given("I have a user"): user(name=user_name) @@ -393,13 +444,14 @@ def select_with_all_privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=select_with_privilege, - name="select with privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_privilege, name="select with privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_privilege(self, user_name, grant_target_name, node=None): - """Grant ALL and check the user is able to select from the distributed table. - """ + """Grant ALL and check the user is able to select from the distributed table.""" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -414,25 +466,29 @@ def select_with_privilege(self, user_name, grant_target_name, node=None): table(name=table0_name, cluster=cluster) with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_DistributedTable_Insert("1.0"), ) def insert(self): - """Check the RBAC functionality of distributed table with INSERT. - """ + """Check the RBAC functionality of distributed table with INSERT.""" insert_scenarios = [ insert_without_privilege, insert_with_privilege_granted_directly_or_via_role, @@ -441,10 +497,10 @@ def insert(self): for scenario in insert_scenarios: Scenario(run=scenario, setup=instrument_clickhouse_server_log) + @TestScenario def insert_without_privilege(self, node=None): - """Check that user is unable to insert into a distributed table without privileges. - """ + """Check that user is unable to insert into a distributed table without privileges.""" user_name = f"user_{getuid()}" table0_name = f"table0_{getuid()}" @@ -465,7 +521,9 @@ def insert_without_privilege(self, node=None): table(name=table0_name, cluster=cluster) with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant the user NONE privilege"): node.query(f"GRANT NONE TO {user_name}") @@ -474,12 +532,17 @@ def insert_without_privilege(self, node=None): node.query(f"GRANT USAGE ON *.* TO {user_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestScenario def insert_with_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a distributed table if and only if @@ -494,8 +557,9 @@ def insert_with_privilege_granted_directly_or_via_role(self, node=None): with Given("I have a user"): user(name=user_name) - Scenario(test=insert_with_privilege, - name="insert with privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=insert_with_privilege, name="insert with privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with Given("I have a user"): user(name=user_name) @@ -506,8 +570,10 @@ def insert_with_privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=insert_with_privilege, - name="insert with privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_with_privilege, name="insert with privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def insert_with_privilege(self, user_name, grant_target_name, node=None): @@ -528,14 +594,20 @@ def insert_with_privilege(self, user_name, grant_target_name, node=None): table(name=table0_name, cluster=cluster) with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant insert privilege on the distributed table"): node.query(f"GRANT INSERT ON {table1_name} TO {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke the insert privilege on the distributed table"): node.query(f"REVOKE INSERT ON {table1_name} FROM {grant_target_name}") @@ -544,32 +616,47 @@ def insert_with_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT INSERT ON {table0_name} to {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant insert privilege on the distributed table"): node.query(f"GRANT INSERT ON {table1_name} TO {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_DistributedTable_SpecialTables("1.0"), @@ -590,8 +677,11 @@ def special_cases(self): for scenario in special_case_scenarios: Scenario(run=scenario, setup=instrument_clickhouse_server_log) + @TestScenario -def select_with_table_on_materialized_view_privilege_granted_directly_or_via_role(self, node=None): +def select_with_table_on_materialized_view_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to SELECT from a distributed table that uses a materialized view if and only if they have SELECT privilege on the distributed table and the materialized view it is built on. """ @@ -604,8 +694,10 @@ def select_with_table_on_materialized_view_privilege_granted_directly_or_via_rol with Given("I have a user"): user(name=user_name) - Scenario(test=select_with_table_on_source_table_of_materialized_view, - name="select with table on source table of materialized view, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_table_on_source_table_of_materialized_view, + name="select with table on source table of materialized view, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with Given("I have a user"): user(name=user_name) @@ -616,11 +708,16 @@ def select_with_table_on_materialized_view_privilege_granted_directly_or_via_rol with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=select_with_table_on_source_table_of_materialized_view, - name="select with table on source table of materialized view, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_table_on_source_table_of_materialized_view, + name="select with table on source table of materialized view, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def select_with_table_on_materialized_view(self, user_name, grant_target_name, node=None): +def select_with_table_on_materialized_view( + self, user_name, grant_target_name, node=None +): """Grant SELECT on the distributed table and the materialized view seperately, check that the user is unable to select from the distributed table, grant privilege on both and check the user is able to select. """ @@ -639,10 +736,14 @@ def select_with_table_on_materialized_view(self, user_name, grant_target_name, n table(name=table0_name, cluster=cluster) with And("I have a materialized view on a cluster"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} ENGINE = Memory() AS SELECT * FROM {table0_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} ENGINE = Memory() AS SELECT * FROM {table0_name}" + ) with And("I have a distributed table on the materialized view"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {view_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {view_name}, rand())" + ) with When("I grant the user NONE privilege"): node.query(f"GRANT NONE TO {grant_target_name}") @@ -651,15 +752,23 @@ def select_with_table_on_materialized_view(self, user_name, grant_target_name, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke the select privilege on the distributed table"): node.query(f"REVOKE SELECT ON {table1_name} FROM {grant_target_name}") @@ -668,27 +777,39 @@ def select_with_table_on_materialized_view(self, user_name, grant_target_name, n node.query(f"GRANT SELECT ON {view_name} to {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the distributed table"): @@ -697,8 +818,11 @@ def select_with_table_on_materialized_view(self, user_name, grant_target_name, n with And("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def select_with_table_on_source_table_of_materialized_view_privilege_granted_directly_or_via_role(self, node=None): +def select_with_table_on_source_table_of_materialized_view_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to SELECT from a distributed table that uses the source table of a materialized view if and only if they have SELECT privilege on the distributed table and the table it is using. """ @@ -711,8 +835,10 @@ def select_with_table_on_source_table_of_materialized_view_privilege_granted_dir with Given("I have a user"): user(name=user_name) - Scenario(test=select_with_table_on_source_table_of_materialized_view, - name="select with table on source table of materialized view, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_table_on_source_table_of_materialized_view, + name="select with table on source table of materialized view, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with Given("I have a user"): user(name=user_name) @@ -723,11 +849,16 @@ def select_with_table_on_source_table_of_materialized_view_privilege_granted_dir with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=select_with_table_on_source_table_of_materialized_view, - name="select with table on source table of materialized view, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_table_on_source_table_of_materialized_view, + name="select with table on source table of materialized view, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def select_with_table_on_source_table_of_materialized_view(self, user_name, grant_target_name, node=None): +def select_with_table_on_source_table_of_materialized_view( + self, user_name, grant_target_name, node=None +): """Grant SELECT on the distributed table and the source table seperately, check that the user is unable to select from the distributed table, grant privilege on both and check the user is able to select. """ @@ -746,17 +877,27 @@ def select_with_table_on_source_table_of_materialized_view(self, user_name, gran table(name=table0_name, cluster=cluster) with And("I have a materialized view on a cluster"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} ENGINE = Memory() AS SELECT * FROM {table0_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} ENGINE = Memory() AS SELECT * FROM {table0_name}" + ) - with And("I have a distributed table using the source table of the materialized view"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + with And( + "I have a distributed table using the source table of the materialized view" + ): + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke select privilege on the distributed table"): node.query(f"REVOKE SELECT ON {table1_name} FROM {grant_target_name}") @@ -765,27 +906,39 @@ def select_with_table_on_source_table_of_materialized_view(self, user_name, gran node.query(f"GRANT SELECT ON {table0_name} to {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the distributed table"): @@ -794,8 +947,11 @@ def select_with_table_on_source_table_of_materialized_view(self, user_name, gran with And("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def select_with_table_on_distributed_table_privilege_granted_directly_or_via_role(self, node=None): +def select_with_table_on_distributed_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to SELECT from a distributed table that uses another distributed table if and only if they have SELECT privilege on the distributed table, the distributed table it is using and the table that the second distributed table is using. """ @@ -808,8 +964,10 @@ def select_with_table_on_distributed_table_privilege_granted_directly_or_via_rol with Given("I have a user"): user(name=user_name) - Scenario(test=select_with_table_on_distributed_table, - name="select with table on distributed table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_table_on_distributed_table, + name="select with table on distributed table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with Given("I have a user"): user(name=user_name) @@ -820,11 +978,16 @@ def select_with_table_on_distributed_table_privilege_granted_directly_or_via_rol with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=select_with_table_on_distributed_table, - name="select with table on distributed table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_table_on_distributed_table, + name="select with table on distributed table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestScenario -def select_with_table_on_distributed_table(self, user_name, grant_target_name, node=None): +def select_with_table_on_distributed_table( + self, user_name, grant_target_name, node=None +): """Grant SELECT privilege seperately on the distributed table, the distributed table it is using and the table that the second distributed table is using, check that user is unable to select from the distributed table, grant privilege on all three and check the user is able to select. """ @@ -843,40 +1006,75 @@ def select_with_table_on_distributed_table(self, user_name, grant_target_name, n table(name=table0_name, cluster=cluster) with And("I have a distributed table on a cluster"): - node.query(f"CREATE TABLE {table1_name} ON CLUSTER {cluster} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} ON CLUSTER {cluster} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with And("I have a distributed table on that distributed table"): - node.query(f"CREATE TABLE {table2_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table1_name}, rand())") + node.query( + f"CREATE TABLE {table2_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table1_name}, rand())" + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): - with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table2_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I attempt to select from the distributed table as the user" + ): + node.query( + f"SELECT * FROM {table2_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table2_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table2_name}", + settings=[("user", f"{user_name}")], + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table2_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table2_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt to select from the distributed table as the user"): - node.query(f"SELECT * FROM {table2_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table2_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the first distributed table"): @@ -885,8 +1083,11 @@ def select_with_table_on_distributed_table(self, user_name, grant_target_name, n with And("I drop the other distributed table"): node.query(f"DROP TABLE IF EXISTS {table2_name}") + @TestScenario -def insert_with_table_on_materialized_view_privilege_granted_directly_or_via_role(self, node=None): +def insert_with_table_on_materialized_view_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to INSERT into a distributed table that uses a materialized view if and only if they have INSERT privilege on the distributed table and the materialized view it is built on. """ @@ -899,8 +1100,10 @@ def insert_with_table_on_materialized_view_privilege_granted_directly_or_via_rol with Given("I have a user"): user(name=user_name) - Scenario(test=insert_with_table_on_materialized_view, - name="insert with table on materialized view, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_with_table_on_materialized_view, + name="insert with table on materialized view, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with Given("I have a user"): user(name=user_name) @@ -911,11 +1114,16 @@ def insert_with_table_on_materialized_view_privilege_granted_directly_or_via_rol with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=insert_with_table_on_materialized_view, - name="insert with table on materialized view, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_with_table_on_materialized_view, + name="insert with table on materialized view, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def insert_with_table_on_materialized_view(self, user_name, grant_target_name, node=None): +def insert_with_table_on_materialized_view( + self, user_name, grant_target_name, node=None +): """Grant INSERT on the distributed table and the materialized view seperately, check that the user is unable to insert into the distributed table, grant privilege on both and check the user is able to insert. """ @@ -938,17 +1146,25 @@ def insert_with_table_on_materialized_view(self, user_name, grant_target_name, n table(name=table1_name, cluster=cluster) with And("I have a materialized view on a cluster"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} TO {table0_name} AS SELECT * FROM {table1_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} TO {table0_name} AS SELECT * FROM {table1_name}" + ) with And("I have a distributed table on the materialized view"): - node.query(f"CREATE TABLE {table2_name} (a UInt64) ENGINE = Distributed({cluster}, default, {view_name}, rand())") + node.query( + f"CREATE TABLE {table2_name} (a UInt64) ENGINE = Distributed({cluster}, default, {view_name}, rand())" + ) with When("I grant insert privilege on the distributed table"): node.query(f"GRANT INSERT ON {table2_name} TO {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke the insert privilege on the distributed table"): node.query(f"REVOKE INSERT ON {table2_name} FROM {grant_target_name}") @@ -957,27 +1173,41 @@ def insert_with_table_on_materialized_view(self, user_name, grant_target_name, n node.query(f"GRANT INSERT ON {view_name} to {grant_target_name}") with Then("I attempt insert into the distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant insert privilege on the distributed table"): node.query(f"GRANT INSERT ON {table2_name} TO {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt insert into the distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the distributed table"): @@ -986,8 +1216,11 @@ def insert_with_table_on_materialized_view(self, user_name, grant_target_name, n with And("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def insert_with_table_on_source_table_of_materialized_view_privilege_granted_directly_or_via_role(self, node=None): +def insert_with_table_on_source_table_of_materialized_view_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to INSERT into a distributed table that uses the source table of a materialized view if and only if they have INSERT privilege on the distributed table and the table it is using. """ @@ -1000,8 +1233,10 @@ def insert_with_table_on_source_table_of_materialized_view_privilege_granted_dir with Given("I have a user"): user(name=user_name) - Scenario(test=insert_with_table_on_source_table_of_materialized_view, - name="insert with table on source table of materialized view, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_with_table_on_source_table_of_materialized_view, + name="insert with table on source table of materialized view, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with Given("I have a user"): user(name=user_name) @@ -1012,11 +1247,16 @@ def insert_with_table_on_source_table_of_materialized_view_privilege_granted_dir with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=insert_with_table_on_source_table_of_materialized_view, - name="insert with table on source table of materialized view, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_with_table_on_source_table_of_materialized_view, + name="insert with table on source table of materialized view, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def insert_with_table_on_source_table_of_materialized_view(self, user_name, grant_target_name, node=None): +def insert_with_table_on_source_table_of_materialized_view( + self, user_name, grant_target_name, node=None +): """Grant INSERT on the distributed table and the source table seperately, check that the user is unable to insert into the distributed table, grant privilege on both and check the user is able to insert. """ @@ -1035,17 +1275,25 @@ def insert_with_table_on_source_table_of_materialized_view(self, user_name, gran table(name=table0_name, cluster=cluster) with And("I have a materialized view on a cluster"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} ENGINE = Memory() AS SELECT * FROM {table0_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ON CLUSTER {cluster} ENGINE = Memory() AS SELECT * FROM {table0_name}" + ) with And("I have a distributed table on the materialized view"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant insert privilege on the distributed table"): node.query(f"GRANT INSERT ON {table1_name} TO {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke insert privilege on the distributed table"): node.query(f"REVOKE INSERT ON {table1_name} FROM {grant_target_name}") @@ -1054,27 +1302,41 @@ def insert_with_table_on_source_table_of_materialized_view(self, user_name, gran node.query(f"GRANT INSERT ON {table0_name} to {grant_target_name}") with Then("I attempt insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant insert privilege on the distributed table"): node.query(f"GRANT INSERT ON {table1_name} TO {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt to insert into the distributed table as the user"): - node.query(f"INSERT INTO {table1_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table1_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the distributed table"): @@ -1083,8 +1345,11 @@ def insert_with_table_on_source_table_of_materialized_view(self, user_name, gran with And("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def insert_with_table_on_distributed_table_privilege_granted_directly_or_via_role(self, node=None): +def insert_with_table_on_distributed_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to INSERT into a distributed table that uses another distributed table if and only if they have INSERT privilege on the distributed table, the distributed table it is using and the table that the second distributed table is using. """ @@ -1097,8 +1362,10 @@ def insert_with_table_on_distributed_table_privilege_granted_directly_or_via_rol with Given("I have a user"): user(name=user_name) - Scenario(test=insert_with_table_on_distributed_table, - name="insert with table on distributed table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_with_table_on_distributed_table, + name="insert with table on distributed table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with Given("I have a user"): user(name=user_name) @@ -1109,11 +1376,16 @@ def insert_with_table_on_distributed_table_privilege_granted_directly_or_via_rol with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name} ON CLUSTER one_shard_cluster") - Scenario(test=insert_with_table_on_distributed_table, - name="insert with table on distributed table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_with_table_on_distributed_table, + name="insert with table on distributed table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def insert_with_table_on_distributed_table(self, user_name, grant_target_name, node=None): +def insert_with_table_on_distributed_table( + self, user_name, grant_target_name, node=None +): """Grant INSERT privilege seperately on the distributed table, the distributed table it is using and the table that the second distributed table is using, check that user is unable to insert into the distributed table, grant privilege on all three and check the user is able to insert. """ @@ -1132,17 +1404,25 @@ def insert_with_table_on_distributed_table(self, user_name, grant_target_name, n table(name=table0_name, cluster=cluster) with And("I have a distributed table on a cluster"): - node.query(f"CREATE TABLE {table1_name} ON CLUSTER {cluster} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} ON CLUSTER {cluster} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with And("I have a distributed table on that distributed table"): - node.query(f"CREATE TABLE {table2_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table1_name}, rand())") + node.query( + f"CREATE TABLE {table2_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table1_name}, rand())" + ) with When("I grant insert privilege on the outer distributed table"): node.query(f"GRANT INSERT ON {table2_name} TO {grant_target_name}") with Then("I attempt to insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke the insert privilege on the outer distributed table"): node.query(f"REVOKE INSERT ON {table2_name} FROM {grant_target_name}") @@ -1151,8 +1431,12 @@ def insert_with_table_on_distributed_table(self, user_name, grant_target_name, n node.query(f"GRANT INSERT ON {table1_name} to {grant_target_name}") with Then("I attempt insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke the insert privilege on the inner distributed table"): node.query(f"REVOKE INSERT ON {table1_name} FROM {grant_target_name}") @@ -1161,34 +1445,52 @@ def insert_with_table_on_distributed_table(self, user_name, grant_target_name, n node.query(f"GRANT INSERT ON {table0_name} to {grant_target_name}") with Then("I attempt insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant insert privilege on the inner distributed table"): node.query(f"GRANT INSERT ON {table1_name} to {grant_target_name}") with Then("I attempt insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant insert privilege on the outer distributed table"): node.query(f"GRANT INSERT ON {table2_name} to {grant_target_name}") with Then("I attempt insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* To {grant_target_name}") with Then("I attempt insert into the outer distributed table as the user"): - node.query(f"INSERT INTO {table2_name} VALUES (8888)", settings = [("user", f"{user_name}")]) + node.query( + f"INSERT INTO {table2_name} VALUES (8888)", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the outer distributed table"): @@ -1197,16 +1499,28 @@ def insert_with_table_on_distributed_table(self, user_name, grant_target_name, n with And("I drop the inner distributed table"): node.query(f"DROP TABLE IF EXISTS {table2_name}") + @TestOutline(Scenario) -@Examples("cluster", [ - ("sharded_cluster12", Description("two node cluster with two shards where one shard is" - " on clickhouse1 and another on clickhouse2 accessed from clickhouse1")), - ("one_shard_cluster12", Description("two node cluster with only one shard and two replicas" - " where one replica is on clickhouse1 and another on clickhouse2 accessed from clickhouse1")), -]) -@Requirements( - RQ_SRS_006_RBAC_DistributedTable_LocalUser("1.0") +@Examples( + "cluster", + [ + ( + "sharded_cluster12", + Description( + "two node cluster with two shards where one shard is" + " on clickhouse1 and another on clickhouse2 accessed from clickhouse1" + ), + ), + ( + "one_shard_cluster12", + Description( + "two node cluster with only one shard and two replicas" + " where one replica is on clickhouse1 and another on clickhouse2 accessed from clickhouse1" + ), + ), + ], ) +@Requirements(RQ_SRS_006_RBAC_DistributedTable_LocalUser("1.0")) def local_user(self, cluster, node=None): """Check that a user that exists locally and not present on the remote nodes is able to execute queries they have privileges to. @@ -1227,7 +1541,9 @@ def local_user(self, cluster, node=None): table(name=table0_name, cluster=cluster) with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} (a UInt64) ENGINE = Distributed({cluster}, default, {table0_name}, rand())" + ) with When("I grant select privilege on the distributed table"): node.query(f"GRANT SELECT ON {table1_name} TO {user_name}") @@ -1236,7 +1552,9 @@ def local_user(self, cluster, node=None): node.query(f"GRANT SELECT ON {table0_name} TO {user_name}") with Then("I select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) with When("I revoke ALL privileges"): node.query(f"REVOKE ALL ON *.* FROM {user_name}") @@ -1245,7 +1563,9 @@ def local_user(self, cluster, node=None): node.query(f"GRANT ALL ON *.* To {user_name}") with Then("I select from the distributed table as the user"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the user"): @@ -1254,6 +1574,7 @@ def local_user(self, cluster, node=None): with And("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_DistributedTable_SameUserDifferentNodesDifferentPrivileges("1.0") @@ -1282,7 +1603,9 @@ def multiple_node_user(self, node=None): table(name=table0_name, cluster="sharded_cluster12") with And("I have a distributed table"): - node.query(f"CREATE TABLE {table1_name} ON CLUSTER sharded_cluster12 (a UInt64) ENGINE = Distributed(sharded_cluster12, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table1_name} ON CLUSTER sharded_cluster12 (a UInt64) ENGINE = Distributed(sharded_cluster12, default, {table0_name}, rand())" + ) with When("I grant select privilege on the distributed table on one node"): node.query(f"GRANT SELECT ON {table1_name} TO {user_name}") @@ -1290,12 +1613,22 @@ def multiple_node_user(self, node=None): with And("I grant select privilege on the other table on one node"): node.query(f"GRANT SELECT ON {table0_name} TO {user_name}") - with Then("I select from the distributed table on the node where the user has privileges"): - node.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")]) + with Then( + "I select from the distributed table on the node where the user has privileges" + ): + node.query( + f"SELECT * FROM {table1_name}", settings=[("user", f"{user_name}")] + ) - with And("I select from the distributed table on the node the user doesn't have privileges"): - node2.query(f"SELECT * FROM {table1_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with And( + "I select from the distributed table on the node the user doesn't have privileges" + ): + node2.query( + f"SELECT * FROM {table1_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): @@ -1304,19 +1637,40 @@ def multiple_node_user(self, node=None): with And("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table1_name}") + @TestOutline(Feature) -@Examples("cluster", [ - ("cluster1", Description("one node cluster with clickhouse1 accessed from clickhouse1")), - ("sharded_cluster23", Description("two node cluster with two shards where one shard is" - " on clickhouse2 and another on clickhouse3 accessed from clickhouse1")), - ("sharded_cluster12", Description("two node cluster with two shards where one shard is" - " on clickhouse1 and another on clickhouse2 accessed from clickhouse1")), - ("one_shard_cluster12", Description("two node cluster with only one shard and two replicas" - " where one replica is on clickhouse1 and another on clickhouse2 accessed from clickhouse1")), -]) +@Examples( + "cluster", + [ + ( + "cluster1", + Description("one node cluster with clickhouse1 accessed from clickhouse1"), + ), + ( + "sharded_cluster23", + Description( + "two node cluster with two shards where one shard is" + " on clickhouse2 and another on clickhouse3 accessed from clickhouse1" + ), + ), + ( + "sharded_cluster12", + Description( + "two node cluster with two shards where one shard is" + " on clickhouse1 and another on clickhouse2 accessed from clickhouse1" + ), + ), + ( + "one_shard_cluster12", + Description( + "two node cluster with only one shard and two replicas" + " where one replica is on clickhouse1 and another on clickhouse2 accessed from clickhouse1" + ), + ), + ], +) def cluster_tests(self, cluster, node=None): - """Scenarios to be run on different cluster configurations. - """ + """Scenarios to be run on different cluster configurations.""" self.context.cluster_name = cluster with Pool(3) as pool: @@ -1326,15 +1680,14 @@ def cluster_tests(self, cluster, node=None): finally: join() + @TestFeature @Requirements( - RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_All("1.0"), RQ_SRS_006_RBAC_Privileges_None("1.0") ) @Name("distributed table") def feature(self, node="clickhouse1"): - """Check the RBAC functionality of queries executed using distributed tables. - """ + """Check the RBAC functionality of queries executed using distributed tables.""" self.context.node = self.context.cluster.node(node) self.context.node2 = self.context.cluster.node("clickhouse2") self.context.node3 = self.context.cluster.node("clickhouse3") diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_database.py b/tests/testflows/rbac/tests/privileges/drop/drop_database.py index 274003e763f..3001285ef37 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_database.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_database.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DROP DATABASE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DROP DATABASE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DROP DATABASE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DROP DATABASE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DROP DATABASE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DROP DATABASE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -46,8 +54,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to drop the database"): - node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") @@ -63,7 +75,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") with Then("I attempt to drop a database"): - node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)]) + node.query(f"DROP DATABASE {db_name}", settings=[("user", user_name)]) finally: with Finally("I drop the database"): @@ -80,11 +92,17 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP DATABASE ON {db_name}.* TO {grant_target_name}") with And("I revoke the drop database privilege"): - node.query(f"REVOKE DROP DATABASE ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE DROP DATABASE ON {db_name}.* FROM {grant_target_name}" + ) with Then("I attempt to drop a database"): - node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -104,8 +122,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to drop a database"): - node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): @@ -122,22 +144,22 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to drop a database"): - node.query(f"DROP DATABASE {db_name}", settings = [("user", user_name)]) + node.query(f"DROP DATABASE {db_name}", settings=[("user", user_name)]) finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DropDatabase("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("drop database") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DROP DATABASE. - """ + """Check the RBAC functionality of DROP DATABASE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -145,5 +167,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py b/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py index c3f07885bd5..7d5958945b1 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_dictionary.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DROP DICTIONARY when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DROP DICTIONARY when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -14,19 +14,27 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DROP DICTIONARY with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DROP DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DROP DICTIONARY with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DROP DICTIONARY with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -34,7 +42,9 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant the user NONE privilege"): node.query(f"GRANT NONE TO {grant_target_name}") @@ -43,7 +53,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to drop a dictionary without privilege"): - node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"DROP DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): @@ -54,13 +69,19 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant drop dictionary privilege"): - node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with Then("I attempt to drop aa dictionary"): - node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)]) + node.query( + f"DROP DICTIONARY {dict_name}", settings=[("user", user_name)] + ) finally: with Finally("I drop the dictionary"): @@ -71,16 +92,27 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant the drop dictionary privilege"): - node.query(f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}") + node.query( + f"GRANT DROP DICTIONARY ON {dict_name} TO {grant_target_name}" + ) with And("I revoke the drop dictionary privilege"): - node.query(f"REVOKE DROP DICTIONARY ON {dict_name} FROM {grant_target_name}") + node.query( + f"REVOKE DROP DICTIONARY ON {dict_name} FROM {grant_target_name}" + ) with Then("I attempt to drop a dictionary"): - node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)], exitcode=exitcode, message=message) + node.query( + f"DROP DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): @@ -91,27 +123,32 @@ def privilege_check(grant_target_name, user_name, node=None): try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with When("I grant ALL privilege"): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I drop the dictionary"): - node.query(f"DROP DICTIONARY {dict_name}", settings = [("user", user_name)]) + node.query( + f"DROP DICTIONARY {dict_name}", settings=[("user", user_name)] + ) finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DropDictionary("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("drop dictionary") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DROP DICTIONARY. - """ + """Check the RBAC functionality of DROP DICTIONARY.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -119,5 +156,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_quota.py b/tests/testflows/rbac/tests/privileges/drop/drop_quota.py index b8727556a26..f2202d3fb67 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_quota.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_quota.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `DROP QUOTA` with privileges are granted directly. - """ + """Check that a user is able to execute `DROP QUOTA` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=drop_quota, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in drop_quota.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_quota, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in drop_quota.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `DROP QUOTA` with privileges are granted through a role. - """ + """Check that a user is able to execute `DROP QUOTA` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=drop_quota, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in drop_quota.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_quota, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in drop_quota.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("DROP QUOTA",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("DROP QUOTA",), + ], +) def drop_quota(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `DROP QUOTA` when they have the necessary privilege. - """ + """Check that user is only able to execute `DROP QUOTA` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -71,8 +88,12 @@ def drop_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't drop a quota"): - node.query(f"DROP QUOTA {drop_row_policy_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP QUOTA {drop_row_policy_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the quota"): @@ -89,7 +110,10 @@ def drop_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a quota"): - node.query(f"DROP QUOTA {drop_row_policy_name}", settings = [("user", f"{user_name}")]) + node.query( + f"DROP QUOTA {drop_row_policy_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the quota"): @@ -100,17 +124,24 @@ def drop_quota(self, privilege, grant_target_name, user_name, node=None): try: with Given("I have a quota on a cluster"): - node.query(f"CREATE QUOTA {drop_row_policy_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE QUOTA {drop_row_policy_name} ON CLUSTER sharded_cluster" + ) with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a quota"): - node.query(f"DROP QUOTA {drop_row_policy_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"DROP QUOTA {drop_row_policy_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP QUOTA IF EXISTS {drop_row_policy_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP QUOTA IF EXISTS {drop_row_policy_name} ON CLUSTER sharded_cluster" + ) with Scenario("DROP QUOTA with revoked privilege"): drop_row_policy_name = f"drop_row_policy_{getuid()}" @@ -126,22 +157,26 @@ def drop_quota(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot drop quota"): - node.query(f"DROP QUOTA {drop_row_policy_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP QUOTA {drop_row_policy_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: - with Finally("I drop the quota"): + with Finally("I drop the quota"): node.query(f"DROP QUOTA IF EXISTS {drop_row_policy_name}") + @TestFeature @Name("drop quota") @Requirements( RQ_SRS_006_RBAC_Privileges_DropQuota("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of DROP QUOTA. - """ + """Check the RBAC functionality of DROP QUOTA.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_role.py b/tests/testflows/rbac/tests/privileges/drop/drop_role.py index ca9eb1b0947..df63911620d 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_role.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_role.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `DROP ROLE` with privileges are granted directly. - """ + """Check that a user is able to execute `DROP ROLE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=drop_role, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in drop_role.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_role, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in drop_role.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `DROP ROLE` with privileges are granted through a role. - """ + """Check that a user is able to execute `DROP ROLE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=drop_role, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in drop_role.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_role, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in drop_role.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("DROP ROLE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("DROP ROLE",), + ], +) def drop_role(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `DROP ROLE` when they have the necessary privilege. - """ + """Check that user is only able to execute `DROP ROLE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -69,8 +86,12 @@ def drop_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't drop a role"): - node.query(f"DROP ROLE {drop_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP ROLE {drop_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("DROP ROLE with privilege"): drop_role_name = f"drop_role_{getuid()}" @@ -81,7 +102,9 @@ def drop_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a role"): - node.query(f"DROP ROLE {drop_role_name}", settings = [("user", f"{user_name}")]) + node.query( + f"DROP ROLE {drop_role_name}", settings=[("user", f"{user_name}")] + ) with Scenario("DROP ROLE on cluster"): drop_role_name = f"drop_role_{getuid()}" @@ -94,11 +117,16 @@ def drop_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a role"): - node.query(f"DROP ROLE {drop_role_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"DROP ROLE {drop_role_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP ROLE IF EXISTS {drop_role_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROLE IF EXISTS {drop_role_name} ON CLUSTER sharded_cluster" + ) with Scenario("DROP ROLE with revoked privilege"): drop_role_name = f"drop_role_{getuid()}" @@ -111,19 +139,23 @@ def drop_role(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user can't drop a role"): - node.query(f"DROP ROLE {drop_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP ROLE {drop_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("drop role") @Requirements( RQ_SRS_006_RBAC_Privileges_DropRole("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of DROP ROLE. - """ + """Check the RBAC functionality of DROP ROLE.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_row_policy.py b/tests/testflows/rbac/tests/privileges/drop/drop_row_policy.py index ad7fed94df0..59c65990aea 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_row_policy.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_row_policy.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `DROP ROW POLICY` with privileges are granted directly. - """ + """Check that a user is able to execute `DROP ROW POLICY` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=drop_row_policy, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in drop_row_policy.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_row_policy, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in drop_row_policy.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `DROP ROW POLICY` with privileges are granted through a role. - """ + """Check that a user is able to execute `DROP ROW POLICY` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,21 +45,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=drop_row_policy, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in drop_row_policy.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_row_policy, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in drop_row_policy.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("DROP ROW POLICY",), - ("DROP POLICY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("DROP ROW POLICY",), + ("DROP POLICY",), + ], +) def drop_row_policy(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `DROP ROW POLICY` when they have the necessary privilege. - """ + """Check that user is only able to execute `DROP ROW POLICY` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -73,12 +90,18 @@ def drop_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't drop a row policy"): - node.query(f"DROP ROW POLICY {drop_row_policy_name} ON {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP ROW POLICY {drop_row_policy_name} ON {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON {table_name}" + ) with Scenario("DROP ROW POLICY with privilege"): drop_row_policy_name = f"drop_row_policy_{getuid()}" @@ -92,11 +115,16 @@ def drop_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a row policy"): - node.query(f"DROP ROW POLICY {drop_row_policy_name} ON {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"DROP ROW POLICY {drop_row_policy_name} ON {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON {table_name}" + ) with Scenario("DROP ROW POLICY on cluster"): drop_row_policy_name = f"drop_row_policy_{getuid()}" @@ -104,17 +132,24 @@ def drop_row_policy(self, privilege, grant_target_name, user_name, node=None): try: with Given("I have a row policy on a cluster"): - node.query(f"CREATE ROW POLICY {drop_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"CREATE ROW POLICY {drop_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a row policy"): - node.query(f"DROP ROW POLICY {drop_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"DROP ROW POLICY {drop_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with Scenario("DROP ROW POLICY with revoked privilege"): drop_row_policy_name = f"drop_row_policy_{getuid()}" @@ -131,19 +166,23 @@ def drop_row_policy(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot drop row policy"): - node.query(f"DROP ROW POLICY {drop_row_policy_name} ON {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP ROW POLICY {drop_row_policy_name} ON {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: - with Finally("I drop the row policy"): - node.query(f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON {table_name}") + with Finally("I drop the row policy"): + node.query( + f"DROP ROW POLICY IF EXISTS {drop_row_policy_name} ON {table_name}" + ) + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_RowPolicy_Restriction("1.0") -) +@Requirements(RQ_SRS_006_RBAC_RowPolicy_Restriction("1.0")) def drop_all_pol_with_conditions(self, node=None): - """Check that when all policies with conditions are dropped, the table becomes unrestricted. - """ + """Check that when all policies with conditions are dropped, the table becomes unrestricted.""" if node is None: node = self.context.node @@ -157,29 +196,31 @@ def drop_all_pol_with_conditions(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy has a condition"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING 1" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with And("I can't see any of the rows on the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '' == output, error() + assert "" == output, error() with When("I drop the row policy"): node.query(f"DROP ROW POLICY {pol_name} ON {table_name}") with Then("I select all the rows from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' in output, error() + assert "1" in output and "2" in output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), ) def drop_on(self, node=None): - """Check that when a row policy is dropped, users are able to access rows restricted by that policy. - """ + """Check that when a row policy is dropped, users are able to access rows restricted by that policy.""" if node is None: node = self.context.node @@ -193,29 +234,31 @@ def drop_on(self, node=None): row_policy(name=pol_name, table=table_name) with And("The row policy has a condition"): - node.query(f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default") + node.query( + f"ALTER ROW POLICY {pol_name} ON {table_name} FOR SELECT USING y=1 TO default" + ) with And("The table has some values"): node.query(f"INSERT INTO {table_name} (y) VALUES (1),(2)") with And("I can't see one of the rows on the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' not in output, error() + assert "1" in output and "2" not in output, error() with When("I drop the row policy"): node.query(f"DROP ROW POLICY {pol_name} ON {table_name}") with Then("I select all the rows from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output and '2' in output, error() + assert "1" in output and "2" in output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_RowPolicy_Drop_OnCluster("1.0"), ) def drop_on_cluster(self, node=None): - """Check that when a row policy is dropped on a cluster, it works on all nodes. - """ + """Check that when a row policy is dropped on a cluster, it works on all nodes.""" if node is None: node = self.context.node @@ -226,10 +269,14 @@ def drop_on_cluster(self, node=None): try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (x UInt64) ENGINE = Memory" + ) with And("I have a row policy"): - node.query(f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1") + node.query( + f"CREATE ROW POLICY {pol_name} ON CLUSTER sharded_cluster ON {table_name} FOR SELECT USING 1" + ) with And("There are some values on the table on the first node"): node.query(f"INSERT INTO {table_name} (x) VALUES (1)") @@ -238,33 +285,37 @@ def drop_on_cluster(self, node=None): node2.query(f"INSERT INTO {table_name} (x) VALUES (1)") with When("I drop the row policy on cluster"): - node.query(f"DROP ROW POLICY {pol_name} ON {table_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROW POLICY {pol_name} ON {table_name} ON CLUSTER sharded_cluster" + ) with Then("I select from the table"): output = node.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() with And("I select from another node on the cluster"): output = node2.query(f"SELECT * FROM {table_name}").output - assert '1' in output, error() + assert "1" in output, error() finally: with Finally("I drop the row policy", flags=TE): - node.query(f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}") + node.query( + f"DROP ROW POLICY IF EXISTS {pol_name} ON CLUSTER sharded_cluster ON {table_name}" + ) with And("I drop the table", flags=TE): node.query(f"DROP TABLE {table_name} ON CLUSTER sharded_cluster") + @TestFeature @Name("drop row policy") @Requirements( RQ_SRS_006_RBAC_Privileges_DropRowPolicy("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of DROP ROW POLICY. - """ + """Check the RBAC functionality of DROP ROW POLICY.""" self.context.node = self.context.cluster.node(node) self.context.node2 = self.context.cluster.node("clickhouse2") diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_settings_profile.py b/tests/testflows/rbac/tests/privileges/drop/drop_settings_profile.py index 3aa9ef2c369..e3876984801 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_settings_profile.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_settings_profile.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `DROP SETTINGS PROFILE` with privileges are granted directly. - """ + """Check that a user is able to execute `DROP SETTINGS PROFILE` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=drop_settings_profile, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in drop_settings_profile.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_settings_profile, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in drop_settings_profile.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `DROP SETTINGS PROFILE` with privileges are granted through a role. - """ + """Check that a user is able to execute `DROP SETTINGS PROFILE` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,21 +45,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=drop_settings_profile, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in drop_settings_profile.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=drop_settings_profile, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in drop_settings_profile.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("DROP SETTINGS PROFILE",), - ("DROP PROFILE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("DROP SETTINGS PROFILE",), + ("DROP PROFILE",), + ], +) def drop_settings_profile(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `DROP SETTINGS PROFILE` when they have the necessary privilege. - """ + """Check that user is only able to execute `DROP SETTINGS PROFILE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -72,8 +89,12 @@ def drop_settings_profile(self, privilege, grant_target_name, user_name, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't drop a settings_profile"): - node.query(f"DROP SETTINGS PROFILE {drop_row_policy_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP SETTINGS PROFILE {drop_row_policy_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the settings_profile"): @@ -90,7 +111,10 @@ def drop_settings_profile(self, privilege, grant_target_name, user_name, node=No node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a settings_profile"): - node.query(f"DROP SETTINGS PROFILE {drop_row_policy_name}", settings = [("user", f"{user_name}")]) + node.query( + f"DROP SETTINGS PROFILE {drop_row_policy_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the settings_profile"): @@ -101,17 +125,24 @@ def drop_settings_profile(self, privilege, grant_target_name, user_name, node=No try: with Given("I have a settings_profile on a cluster"): - node.query(f"CREATE SETTINGS PROFILE {drop_row_policy_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE SETTINGS PROFILE {drop_row_policy_name} ON CLUSTER sharded_cluster" + ) with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a settings_profile"): - node.query(f"DROP SETTINGS PROFILE {drop_row_policy_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"DROP SETTINGS PROFILE {drop_row_policy_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP SETTINGS PROFILE IF EXISTS {drop_row_policy_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP SETTINGS PROFILE IF EXISTS {drop_row_policy_name} ON CLUSTER sharded_cluster" + ) with Scenario("DROP SETTINGS PROFILE with revoked privilege"): drop_row_policy_name = f"drop_row_policy_{getuid()}" @@ -127,22 +158,26 @@ def drop_settings_profile(self, privilege, grant_target_name, user_name, node=No node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot drop settings_profile"): - node.query(f"DROP SETTINGS PROFILE {drop_row_policy_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP SETTINGS PROFILE {drop_row_policy_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: - with Finally("I drop the settings_profile"): + with Finally("I drop the settings_profile"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {drop_row_policy_name}") + @TestFeature @Name("drop settings profile") @Requirements( RQ_SRS_006_RBAC_Privileges_DropSettingsProfile("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of DROP SETTINGS PROFILE. - """ + """Check the RBAC functionality of DROP SETTINGS PROFILE.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_table.py b/tests/testflows/rbac/tests/privileges/drop/drop_table.py index 1fd394daf96..daafa250462 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_table.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_table.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute DROP TABLE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute DROP TABLE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute DROP TABLE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute DROP TABLE with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute DROP TABLE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute DROP TABLE with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -46,8 +54,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to drop a table without privilege"): - node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): @@ -64,7 +76,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT DROP TABLE ON *.* TO {grant_target_name}") with Then("I attempt to drop a table"): - node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)]) + node.query(f"DROP TABLE {table_name}", settings=[("user", user_name)]) finally: with Finally("I drop the table"): @@ -83,8 +95,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE DROP TABLE ON *.* FROM {grant_target_name}") with Then("I attempt to drop a table"): - node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): @@ -103,8 +119,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to drop a table"): - node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the table"): @@ -121,22 +141,22 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I drop the table"): - node.query(f"DROP TABLE {table_name}", settings = [("user", user_name)]) + node.query(f"DROP TABLE {table_name}", settings=[("user", user_name)]) finally: with Finally("I drop the table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_DropTable("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("drop table") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of DROP TABLE. - """ + """Check the RBAC functionality of DROP TABLE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -144,5 +164,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/drop/drop_user.py b/tests/testflows/rbac/tests/privileges/drop/drop_user.py index c3f1df8ae15..8c8d77d46f0 100644 --- a/tests/testflows/rbac/tests/privileges/drop/drop_user.py +++ b/tests/testflows/rbac/tests/privileges/drop/drop_user.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def drop_user_granted_directly(self, node=None): - """Check that a user is able to execute `DROP USER` with privileges are granted directly. - """ + """Check that a user is able to execute `DROP USER` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def drop_user_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=drop_user, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in drop_user.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=drop_user, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in drop_user.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def drop_user_granted_via_role(self, node=None): - """Check that a user is able to execute `DROP USER` with privileges are granted through a role. - """ + """Check that a user is able to execute `DROP USER` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,20 +45,30 @@ def drop_user_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=drop_user, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in drop_user.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=drop_user, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in drop_user.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("DROP USER",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("DROP USER",), + ], +) def drop_user(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `DROP USER` when they have the necessary privilege. - """ + """Check that user is only able to execute `DROP USER` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -70,8 +87,12 @@ def drop_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with When("I check the user can't drop a user"): - node.query(f"DROP USER {drop_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP USER {drop_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("DROP USER with privilege"): drop_user_name = f"drop_user_{getuid()}" @@ -81,7 +102,9 @@ def drop_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a user"): - node.query(f"DROP USER {drop_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"DROP USER {drop_user_name}", settings=[("user", f"{user_name}")] + ) with Scenario("DROP USER on cluster"): drop_user_name = f"drop_user_{getuid()}" @@ -94,12 +117,16 @@ def drop_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can drop a user"): - node.query(f"DROP USER {drop_user_name} ON CLUSTER sharded_cluster", - settings = [("user", f"{user_name}")]) + node.query( + f"DROP USER {drop_user_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP USER IF EXISTS {drop_user_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP USER IF EXISTS {drop_user_name} ON CLUSTER sharded_cluster" + ) with Scenario("DROP USER with revoked privilege"): drop_user_name = f"drop_user_{getuid()}" @@ -112,19 +139,23 @@ def drop_user(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user can't drop a user"): - node.query(f"DROP USER {drop_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DROP USER {drop_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("drop user") @Requirements( RQ_SRS_006_RBAC_Privileges_DropUser("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of DROP USER. - """ + """Check the RBAC functionality of DROP USER.""" self.context.node = self.context.cluster.node(node) Suite(run=drop_user_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/feature.py b/tests/testflows/rbac/tests/privileges/feature.py index e68d71675ab..58d24d1f1f7 100755 --- a/tests/testflows/rbac/tests/privileges/feature.py +++ b/tests/testflows/rbac/tests/privileges/feature.py @@ -2,95 +2,401 @@ from testflows.core import * from rbac.helper.common import * + @TestFeature @Name("privileges") def feature(self): - """Check RBAC privileges. - """ + """Check RBAC privileges.""" with Pool(10) as pool: try: - Feature(run=load("rbac.tests.privileges.insert", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.select", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.public_tables", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.distributed_table", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.grant_option", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.truncate", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.optimize", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.kill_query", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.kill_mutation", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.role_admin", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.dictGet", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.introspection", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.sources", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.admin_option", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.all_role", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.insert", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.select", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.public_tables", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.distributed_table", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.grant_option", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.truncate", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.optimize", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.kill_query", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.kill_mutation", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.role_admin", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.dictGet", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.introspection", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.sources", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.admin_option", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.all_role", "feature"), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.show.show_tables", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_dictionaries", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_databases", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_columns", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_users", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_roles", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_quotas", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_settings_profiles", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.show.show_row_policies", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.show.show_tables", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_dictionaries", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_databases", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_columns", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_users", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_roles", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_quotas", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load( + "rbac.tests.privileges.show.show_settings_profiles", "feature" + ), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.show.show_row_policies", "feature"), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.alter.alter_column", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_index", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_constraint", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_ttl", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_settings", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_update", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_delete", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_freeze", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_fetch", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_move", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_user", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_role", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_row_policy", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_quota", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.alter.alter_settings_profile", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.alter.alter_column", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_index", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_constraint", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_ttl", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_settings", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_update", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_delete", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_freeze", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_fetch", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_move", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_user", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_role", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_row_policy", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.alter.alter_quota", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load( + "rbac.tests.privileges.alter.alter_settings_profile", "feature" + ), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.create.create_database", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_dictionary", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_temp_table", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_table", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_user", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_role", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_row_policy", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_quota", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.create.create_settings_profile", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.create.create_database", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_dictionary", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_temp_table", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_table", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_user", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_role", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_row_policy", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.create.create_quota", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load( + "rbac.tests.privileges.create.create_settings_profile", "feature" + ), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.attach.attach_database", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.attach.attach_dictionary", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.attach.attach_temp_table", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.attach.attach_table", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.attach.attach_database", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.attach.attach_dictionary", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.attach.attach_temp_table", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.attach.attach_table", "feature"), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.drop.drop_database", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_dictionary", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_table", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_user", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_role", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_row_policy", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_quota", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.drop.drop_settings_profile", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.drop.drop_database", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_dictionary", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_table", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_user", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_role", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_row_policy", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_quota", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.drop.drop_settings_profile", "feature"), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.detach.detach_database", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.detach.detach_dictionary", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.detach.detach_table", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.detach.detach_view", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.detach.detach_database", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.detach.detach_dictionary", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.detach.detach_table", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.detach.detach_view", "feature"), + parallel=True, + executor=pool, + ) - Feature(run=load("rbac.tests.privileges.system.drop_cache", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.reload", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.flush", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.merges", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.moves", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.replication_queues", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.ttl_merges", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.restart_replica", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.sends", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.sync_replica", "feature"), parallel=True, executor=pool) - Feature(run=load("rbac.tests.privileges.system.fetches", "feature"), parallel=True, executor=pool) + Feature( + run=load("rbac.tests.privileges.system.drop_cache", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.reload", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.flush", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.merges", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.moves", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.replication_queues", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.ttl_merges", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.restart_replica", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.sends", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.sync_replica", "feature"), + parallel=True, + executor=pool, + ) + Feature( + run=load("rbac.tests.privileges.system.fetches", "feature"), + parallel=True, + executor=pool, + ) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/grant_option.py b/tests/testflows/rbac/tests/privileges/grant_option.py index ea5ff0ba66a..795b336969a 100644 --- a/tests/testflows/rbac/tests/privileges/grant_option.py +++ b/tests/testflows/rbac/tests/privileges/grant_option.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def grant_option(self, table_type, privilege, node=None): """Check that user is able to execute GRANT and REVOKE privilege statements if and only if they have the privilege WITH GRANT OPTION, @@ -19,104 +20,213 @@ def grant_option(self, table_type, privilege, node=None): with Suite("user with direct privilege granting to user"): with user(node, f"{user0_name},{user1_name}"): - with When(f"I run checks that grant and revoke privilege from {user0_name} to {user1_name}"): - grant_option_check(grant_option_target=user0_name, grant_target=user1_name, user_name=user0_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that grant and revoke privilege from {user0_name} to {user1_name}" + ): + grant_option_check( + grant_option_target=user0_name, + grant_target=user1_name, + user_name=user0_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with direct privilege granting to role"): with user(node, user0_name), role(node, role1_name): - with When(f"I run checks that grant and revoke privilege from {user0_name} to {role1_name}"): - grant_option_check(grant_option_target=user0_name, grant_target=role1_name, user_name=user0_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that grant and revoke privilege from {user0_name} to {role1_name}" + ): + grant_option_check( + grant_option_target=user0_name, + grant_target=role1_name, + user_name=user0_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with privilege via role granting to user"): with user(node, f"{user0_name},{user1_name}"), role(node, role0_name): with When("I grant the role to the user"): node.query(f"GRANT {role0_name} TO {user0_name}") - with When(f"I run checks that grant and revoke privilege from {user0_name} with {role0_name} to {user1_name}"): - grant_option_check(grant_option_target=role0_name, grant_target=user1_name, user_name=user0_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that grant and revoke privilege from {user0_name} with {role0_name} to {user1_name}" + ): + grant_option_check( + grant_option_target=role0_name, + grant_target=user1_name, + user_name=user0_name, + table_type=table_type, + privilege=privilege, + node=node, + ) with Suite("user with privilege via role granting to role"): with user(node, user0_name), role(node, f"{role0_name},{role1_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role0_name} TO {user0_name}") - with When(f"I run checks that grant and revoke privilege from {user0_name} with {role0_name} to {role1_name}"): - grant_option_check(grant_option_target=role0_name, grant_target=role1_name, user_name=user0_name, table_type=table_type, privilege=privilege, node=node) + with When( + f"I run checks that grant and revoke privilege from {user0_name} with {role0_name} to {role1_name}" + ): + grant_option_check( + grant_option_target=role0_name, + grant_target=role1_name, + user_name=user0_name, + table_type=table_type, + privilege=privilege, + node=node, + ) -def grant_option_check(grant_option_target, grant_target, user_name, table_type, privilege, node=None): - """Run different scenarios to check the user's access with different privileges. - """ + +def grant_option_check( + grant_option_target, grant_target, user_name, table_type, privilege, node=None +): + """Run different scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") - with Scenario("grant by user without privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "grant by user without privilege", setup=instrument_clickhouse_server_log + ): table_name = f"merge_tree_{getuid()}" with table(node, name=table_name, table_type_name=table_type): with Then("I attempt to grant delete privilege without privilege"): - node.query(f"GRANT {privilege} ON {table_name} TO {grant_target}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {privilege} ON {table_name} TO {grant_target}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) - with Scenario("grant by user with grant option privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "grant by user with grant option privilege", + setup=instrument_clickhouse_server_log, + ): table_name = f"merge_tree_{getuid()}" with table(node, name=table_name, table_type_name=table_type): with When("I grant delete privilege"): - node.query(f"GRANT {privilege} ON {table_name} TO {grant_option_target} WITH GRANT OPTION") + node.query( + f"GRANT {privilege} ON {table_name} TO {grant_option_target} WITH GRANT OPTION" + ) with Then("I attempt to grant delete privilege"): - node.query(f"GRANT {privilege} ON {table_name} TO {grant_target}", settings = [("user", user_name)]) + node.query( + f"GRANT {privilege} ON {table_name} TO {grant_target}", + settings=[("user", user_name)], + ) - with Scenario("revoke by user with grant option privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "revoke by user with grant option privilege", + setup=instrument_clickhouse_server_log, + ): table_name = f"merge_tree_{getuid()}" with table(node, name=table_name, table_type_name=table_type): with When("I grant delete privilege"): - node.query(f"GRANT {privilege} ON {table_name} TO {grant_option_target} WITH GRANT OPTION") + node.query( + f"GRANT {privilege} ON {table_name} TO {grant_option_target} WITH GRANT OPTION" + ) with Then("I attempt to revoke delete privilege"): - node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_target}", settings = [("user", user_name)]) + node.query( + f"REVOKE {privilege} ON {table_name} FROM {grant_target}", + settings=[("user", user_name)], + ) - with Scenario("grant by user with revoked grant option privilege", setup=instrument_clickhouse_server_log): + with Scenario( + "grant by user with revoked grant option privilege", + setup=instrument_clickhouse_server_log, + ): table_name = f"merge_tree_{getuid()}" with table(node, name=table_name, table_type_name=table_type): - with When(f"I grant delete privilege with grant option to {grant_option_target}"): - node.query(f"GRANT {privilege} ON {table_name} TO {grant_option_target} WITH GRANT OPTION") - with And(f"I revoke delete privilege with grant option from {grant_option_target}"): - node.query(f"REVOKE {privilege} ON {table_name} FROM {grant_option_target}") + with When( + f"I grant delete privilege with grant option to {grant_option_target}" + ): + node.query( + f"GRANT {privilege} ON {table_name} TO {grant_option_target} WITH GRANT OPTION" + ) + with And( + f"I revoke delete privilege with grant option from {grant_option_target}" + ): + node.query( + f"REVOKE {privilege} ON {table_name} FROM {grant_option_target}" + ) with Then("I attempt to grant delete privilege"): - node.query(f"GRANT {privilege} ON {table_name} TO {grant_target}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {privilege} ON {table_name} TO {grant_target}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_GrantOption("1.0"), ) -@Examples("privilege", [ - ("ALTER MOVE PARTITION",), ("ALTER MOVE PART",), ("MOVE PARTITION",), ("MOVE PART",), - ("ALTER DELETE",), ("DELETE",), - ("ALTER FETCH PARTITION",), ("FETCH PARTITION",), - ("ALTER FREEZE PARTITION",), ("FREEZE PARTITION",), - ("ALTER UPDATE",), ("UPDATE",), - ("ALTER ADD COLUMN",), ("ADD COLUMN",), - ("ALTER CLEAR COLUMN",), ("CLEAR COLUMN",), - ("ALTER MODIFY COLUMN",), ("MODIFY COLUMN",), - ("ALTER RENAME COLUMN",), ("RENAME COLUMN",), - ("ALTER COMMENT COLUMN",), ("COMMENT COLUMN",), - ("ALTER DROP COLUMN",), ("DROP COLUMN",), - ("ALTER COLUMN",), - ("ALTER SETTINGS",), ("ALTER SETTING",), ("ALTER MODIFY SETTING",), ("MODIFY SETTING",), - ("ALTER ORDER BY",), ("ALTER MODIFY ORDER BY",), ("MODIFY ORDER BY",), - ("ALTER SAMPLE BY",), ("ALTER MODIFY SAMPLE BY",), ("MODIFY SAMPLE BY",), - ("ALTER ADD INDEX",), ("ADD INDEX",), - ("ALTER MATERIALIZE INDEX",), ("MATERIALIZE INDEX",), - ("ALTER CLEAR INDEX",), ("CLEAR INDEX",), - ("ALTER DROP INDEX",), ("DROP INDEX",), - ("ALTER INDEX",), ("INDEX",), - ("ALTER TTL",), ("ALTER MODIFY TTL",), ("MODIFY TTL",), - ("ALTER MATERIALIZE TTL",), ("MATERIALIZE TTL",), - ("ALTER ADD CONSTRAINT",), ("ADD CONSTRAINT",), - ("ALTER DROP CONSTRAINT",), ("DROP CONSTRAINT",), - ("ALTER CONSTRAINT",), ("CONSTRAINT",), - ("INSERT",), - ("SELECT",), -]) +@Examples( + "privilege", + [ + ("ALTER MOVE PARTITION",), + ("ALTER MOVE PART",), + ("MOVE PARTITION",), + ("MOVE PART",), + ("ALTER DELETE",), + ("DELETE",), + ("ALTER FETCH PARTITION",), + ("FETCH PARTITION",), + ("ALTER FREEZE PARTITION",), + ("FREEZE PARTITION",), + ("ALTER UPDATE",), + ("UPDATE",), + ("ALTER ADD COLUMN",), + ("ADD COLUMN",), + ("ALTER CLEAR COLUMN",), + ("CLEAR COLUMN",), + ("ALTER MODIFY COLUMN",), + ("MODIFY COLUMN",), + ("ALTER RENAME COLUMN",), + ("RENAME COLUMN",), + ("ALTER COMMENT COLUMN",), + ("COMMENT COLUMN",), + ("ALTER DROP COLUMN",), + ("DROP COLUMN",), + ("ALTER COLUMN",), + ("ALTER SETTINGS",), + ("ALTER SETTING",), + ("ALTER MODIFY SETTING",), + ("MODIFY SETTING",), + ("ALTER ORDER BY",), + ("ALTER MODIFY ORDER BY",), + ("MODIFY ORDER BY",), + ("ALTER SAMPLE BY",), + ("ALTER MODIFY SAMPLE BY",), + ("MODIFY SAMPLE BY",), + ("ALTER ADD INDEX",), + ("ADD INDEX",), + ("ALTER MATERIALIZE INDEX",), + ("MATERIALIZE INDEX",), + ("ALTER CLEAR INDEX",), + ("CLEAR INDEX",), + ("ALTER DROP INDEX",), + ("DROP INDEX",), + ("ALTER INDEX",), + ("INDEX",), + ("ALTER TTL",), + ("ALTER MODIFY TTL",), + ("MODIFY TTL",), + ("ALTER MATERIALIZE TTL",), + ("MATERIALIZE TTL",), + ("ALTER ADD CONSTRAINT",), + ("ADD CONSTRAINT",), + ("ALTER DROP CONSTRAINT",), + ("DROP CONSTRAINT",), + ("ALTER CONSTRAINT",), + ("CONSTRAINT",), + ("INSERT",), + ("SELECT",), + ], +) @Name("grant option") def feature(self, stress=None, node="clickhouse1"): - """Check the RBAC functionality of privileges with GRANT OPTION. - """ + """Check the RBAC functionality of privileges with GRANT OPTION.""" self.context.node = self.context.cluster.node(node) if stress is not None: @@ -125,8 +235,14 @@ def feature(self, stress=None, node="clickhouse1"): with Pool(12) as pool: try: for example in self.examples: - privilege, = example + (privilege,) = example args = {"table_type": "MergeTree", "privilege": privilege} - Suite(test=grant_option, name=privilege, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Suite( + test=grant_option, + name=privilege, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/insert.py b/tests/testflows/rbac/tests/privileges/insert.py index 650e65b2fb0..4130249c742 100755 --- a/tests/testflows/rbac/tests/privileges/insert.py +++ b/tests/testflows/rbac/tests/privileges/insert.py @@ -8,20 +8,23 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + def input_output_equality_check(node, input_columns, input_data, table_name): data_list = [x.strip("'") for x in input_data.split(",")] input_dict = dict(zip(input_columns.split(","), data_list)) - output_dict = json.loads(node.query(f"select {input_columns} from {table_name} format JSONEachRow").output) - output_dict = {k:str(v) for (k,v) in output_dict.items()} + output_dict = json.loads( + node.query( + f"select {input_columns} from {table_name} format JSONEachRow" + ).output + ) + output_dict = {k: str(v) for (k, v) in output_dict.items()} return input_dict == output_dict + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_Privileges_None("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_None("1.0")) def without_privilege(self, table_type, node=None): - """Check that user without insert privilege on a table is not able to insert on that table. - """ + """Check that user without insert privilege on a table is not able to insert on that table.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -40,16 +43,20 @@ def without_privilege(self, table_type, node=None): with Then("I run INSERT without privilege"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Grant_Privilege_Insert("1.0"), ) def user_with_privilege(self, table_type, node=None): - """Check that user can insert into a table on which they have insert privilege. - """ + """Check that user can insert into a table on which they have insert privilege.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -63,20 +70,25 @@ def user_with_privilege(self, table_type, node=None): node.query(f"GRANT INSERT ON {table_name} TO {user_name}") with And("I use INSERT"): - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + ) with Then("I check the insert functioned"): - output = node.query(f"SELECT d FROM {table_name} FORMAT JSONEachRow").output + output = node.query( + f"SELECT d FROM {table_name} FORMAT JSONEachRow" + ).output assert output == '{"d":"2020-01-01"}', error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_Privileges_All("1.0"), RQ_SRS_006_RBAC_Grant_Privilege_Insert("1.0"), ) def all_privilege(self, table_type, node=None): - """Check that user can insert into a table on which they have insert privilege. - """ + """Check that user can insert into a table on which they have insert privilege.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -91,19 +103,24 @@ def all_privilege(self, table_type, node=None): node.query(f"GRANT ALL ON *.* TO {user_name}") with And("I use INSERT"): - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + ) with Then("I check the insert functioned"): - output = node.query(f"SELECT d FROM {table_name} FORMAT JSONEachRow").output + output = node.query( + f"SELECT d FROM {table_name} FORMAT JSONEachRow" + ).output assert output == '{"d":"2020-01-01"}', error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_Revoke_Privilege_Insert("1.0"), ) def user_with_revoked_privilege(self, table_type, node=None): - """Check that user is unable to insert into a table after insert privilege on that table has been revoked from user. - """ + """Check that user is unable to insert into a table after insert privilege on that table has been revoked from user.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -121,13 +138,17 @@ def user_with_revoked_privilege(self, table_type, node=None): with Then("I use INSERT"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_all_revoked_privilege(self, table_type, node=None): - """Check that user is unable to insert into a table after ALL privilege has been revoked from user. - """ + """Check that user is unable to insert into a table after ALL privilege has been revoked from user.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -145,28 +166,52 @@ def user_with_all_revoked_privilege(self, table_type, node=None): with Then("I use INSERT"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_privilege_on_columns(self, table_type): - Scenario(run=user_column_privileges, - examples=Examples("grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass table_type", - [tuple(list(row)+[table_type]) for row in user_column_privileges.examples])) + Scenario( + run=user_column_privileges, + examples=Examples( + "grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass table_type", + [ + tuple(list(row) + [table_type]) + for row in user_column_privileges.examples + ], + ), + ) @TestOutline @Requirements( RQ_SRS_006_RBAC_Insert_Column("1.0"), ) -@Examples("grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass", [ - ("d", "d", "x", "d", '\'woo\'', '\'2020-01-01\''), - ("d,a", "d", "x", "d", '\'woo\'', '\'2020-01-01\''), - ("d,a,b", "d,a,b", "x", "d,b", '\'woo\'', '\'2020-01-01\',9'), - ("d,a,b", "b", "y", "d,a,b", '9', '\'2020-01-01\',\'woo\',9') -]) -def user_column_privileges(self, grant_columns, insert_columns_pass, data_fail, data_pass, table_type, - revoke_columns=None, insert_columns_fail=None, node=None): +@Examples( + "grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass", + [ + ("d", "d", "x", "d", "'woo'", "'2020-01-01'"), + ("d,a", "d", "x", "d", "'woo'", "'2020-01-01'"), + ("d,a,b", "d,a,b", "x", "d,b", "'woo'", "'2020-01-01',9"), + ("d,a,b", "b", "y", "d,a,b", "9", "'2020-01-01','woo',9"), + ], +) +def user_column_privileges( + self, + grant_columns, + insert_columns_pass, + data_fail, + data_pass, + table_type, + revoke_columns=None, + insert_columns_fail=None, + node=None, +): """Check that user is able to insert on columns where insert privilege is granted and unable to insert on columns where insert privilege has not been granted or has been revoked. """ @@ -181,31 +226,48 @@ def user_column_privileges(self, grant_columns, insert_columns_pass, data_fail, with user(node, user_name): with When("I grant insert privilege"): - node.query(f"GRANT INSERT({grant_columns}) ON {table_name} TO {user_name}") + node.query( + f"GRANT INSERT({grant_columns}) ON {table_name} TO {user_name}" + ) if insert_columns_fail is not None: with And("I insert into a column without insert privilege"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} ({insert_columns_fail}) VALUES ({data_fail})", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} ({insert_columns_fail}) VALUES ({data_fail})", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with And("I insert into granted column"): - node.query(f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", - settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", + settings=[("user", user_name)], + ) with Then("I check the insert functioned"): - input_equals_output = input_output_equality_check(node, insert_columns_pass, data_pass, table_name) + input_equals_output = input_output_equality_check( + node, insert_columns_pass, data_pass, table_name + ) assert input_equals_output, error() if revoke_columns is not None: with When("I revoke insert privilege from columns"): - node.query(f"REVOKE INSERT({revoke_columns}) ON {table_name} FROM {user_name}") + node.query( + f"REVOKE INSERT({revoke_columns}) ON {table_name} FROM {user_name}" + ) with And("I insert into revoked columns"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( @@ -233,12 +295,18 @@ def role_with_privilege(self, table_type, node=None): node.query(f"GRANT {role_name} TO {user_name}") with And("I insert into the table"): - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + ) with Then("I check the data matches the input"): - output = node.query(f"SELECT d FROM {table_name} FORMAT JSONEachRow").output + output = node.query( + f"SELECT d FROM {table_name} FORMAT JSONEachRow" + ).output assert output == '{"d":"2020-01-01"}', error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_Revoke_Privilege_Insert("1.0"), @@ -270,8 +338,13 @@ def role_with_revoked_privilege(self, table_type, node=None): with And("I insert into the table"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_revoked_role(self, table_type, node=None): @@ -300,27 +373,52 @@ def user_with_revoked_role(self, table_type, node=None): with And("I insert into the table"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def role_with_privilege_on_columns(self, table_type): - Scenario(run=role_column_privileges, - examples=Examples("grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass table_type", - [tuple(list(row)+[table_type]) for row in role_column_privileges.examples])) + Scenario( + run=role_column_privileges, + examples=Examples( + "grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass table_type", + [ + tuple(list(row) + [table_type]) + for row in role_column_privileges.examples + ], + ), + ) + @TestOutline @Requirements( RQ_SRS_006_RBAC_Insert_Column("1.0"), ) -@Examples("grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass", [ - ("d", "d", "x", "d", '\'woo\'', '\'2020-01-01\''), - ("d,a", "d", "x", "d", '\'woo\'', '\'2020-01-01\''), - ("d,a,b", "d,a,b", "x", "d,b", '\'woo\'', '\'2020-01-01\',9'), - ("d,a,b", "b", "y", "d,a,b", '9', '\'2020-01-01\',\'woo\',9') -]) -def role_column_privileges(self, grant_columns, insert_columns_pass, data_fail, data_pass, - table_type, revoke_columns=None, insert_columns_fail=None, node=None): +@Examples( + "grant_columns revoke_columns insert_columns_fail insert_columns_pass data_fail data_pass", + [ + ("d", "d", "x", "d", "'woo'", "'2020-01-01'"), + ("d,a", "d", "x", "d", "'woo'", "'2020-01-01'"), + ("d,a,b", "d,a,b", "x", "d,b", "'woo'", "'2020-01-01',9"), + ("d,a,b", "b", "y", "d,a,b", "9", "'2020-01-01','woo',9"), + ], +) +def role_column_privileges( + self, + grant_columns, + insert_columns_pass, + data_fail, + data_pass, + table_type, + revoke_columns=None, + insert_columns_fail=None, + node=None, +): """Check that user with a role is able to insert on columns where insert privilege is granted to the role and unable to insert on columns where insert privilege has not been granted or has been revoked from the role. """ @@ -335,7 +433,9 @@ def role_column_privileges(self, grant_columns, insert_columns_pass, data_fail, with user(node, user_name), role(node, role_name): with When("I grant insert privilege"): - node.query(f"GRANT INSERT({grant_columns}) ON {table_name} TO {role_name}") + node.query( + f"GRANT INSERT({grant_columns}) ON {table_name} TO {role_name}" + ) with And("I grant the role to a user"): node.query(f"GRANT {role_name} TO {user_name}") @@ -344,26 +444,41 @@ def role_column_privileges(self, grant_columns, insert_columns_pass, data_fail, with And("I insert into columns without insert privilege"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} ({insert_columns_fail}) VALUES ({data_fail})", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} ({insert_columns_fail}) VALUES ({data_fail})", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with And("I insert into granted column"): - node.query(f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", - settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", + settings=[("user", user_name)], + ) with Then("I check the insert functioned"): - input_equals_output = input_output_equality_check(node, insert_columns_pass, data_pass, table_name) + input_equals_output = input_output_equality_check( + node, insert_columns_pass, data_pass, table_name + ) assert input_equals_output, error() if revoke_columns is not None: with When("I revoke insert privilege from columns"): - node.query(f"REVOKE INSERT({revoke_columns}) ON {table_name} FROM {role_name}") + node.query( + f"REVOKE INSERT({revoke_columns}) ON {table_name} FROM {role_name}" + ) with And("I insert into revoked columns"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", - settings=[("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} ({insert_columns_pass}) VALUES ({data_pass})", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( @@ -383,40 +498,68 @@ def user_with_privilege_on_cluster(self, table_type, node=None): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) - with When("I grant insert privilege on a cluster without the node with the table"): - node.query(f"GRANT ON CLUSTER sharded_cluster23 INSERT ON {table_name} TO {user_name}") + with When( + "I grant insert privilege on a cluster without the node with the table" + ): + node.query( + f"GRANT ON CLUSTER sharded_cluster23 INSERT ON {table_name} TO {user_name}" + ) with And("I insert into the table expecting a fail"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with And("I grant insert privilege on cluster including all nodes"): - node.query(f"GRANT ON CLUSTER sharded_cluster INSERT ON {table_name} TO {user_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster INSERT ON {table_name} TO {user_name}" + ) - with And("I revoke insert privilege on cluster without the node with the table"): - node.query(f"REVOKE ON CLUSTER sharded_cluster23 INSERT ON {table_name} FROM {user_name}") + with And( + "I revoke insert privilege on cluster without the node with the table" + ): + node.query( + f"REVOKE ON CLUSTER sharded_cluster23 INSERT ON {table_name} FROM {user_name}" + ) with And("I insert into the table"): - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + ) with And("I check that I can read inserted data"): - output = node.query(f"SELECT d FROM {table_name} FORMAT JSONEachRow").output + output = node.query( + f"SELECT d FROM {table_name} FORMAT JSONEachRow" + ).output assert output == '{"d":"2020-01-01"}', error() with And("I revoke insert privilege on cluster with all nodes"): - node.query(f"REVOKE ON CLUSTER sharded_cluster INSERT ON {table_name} FROM {user_name}") + node.query( + f"REVOKE ON CLUSTER sharded_cluster INSERT ON {table_name} FROM {user_name}" + ) with Then("I insert into table expecting fail"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestScenario @Requirements( RQ_SRS_006_RBAC_Insert_Cluster("1.0"), @@ -436,59 +579,81 @@ def role_with_privilege_on_cluster(self, table_type, node=None): try: with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) with And("I have a role on a cluster"): - node.query(f"CREATE ROLE OR REPLACE {role_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE ROLE OR REPLACE {role_name} ON CLUSTER sharded_cluster" + ) with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And("I grant insert privilege on a cluster without the node with the table"): - node.query(f"GRANT ON CLUSTER sharded_cluster23 INSERT ON {table_name} TO {role_name}") + with And( + "I grant insert privilege on a cluster without the node with the table" + ): + node.query( + f"GRANT ON CLUSTER sharded_cluster23 INSERT ON {table_name} TO {role_name}" + ) with And("I insert into the table expecting a fail"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with And("I grant insert privilege on cluster including all nodes"): - node.query(f"GRANT ON CLUSTER sharded_cluster INSERT ON {table_name} TO {role_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster INSERT ON {table_name} TO {role_name}" + ) with And("I revoke insert privilege on cluster without the table node"): - node.query(f"REVOKE ON CLUSTER sharded_cluster23 INSERT ON {table_name} FROM {role_name}") + node.query( + f"REVOKE ON CLUSTER sharded_cluster23 INSERT ON {table_name} FROM {role_name}" + ) with And("I insert into the table"): - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)]) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + ) with And("I check that I can read inserted data"): - output = node.query(f"SELECT d FROM {table_name} FORMAT JSONEachRow").output + output = node.query( + f"SELECT d FROM {table_name} FORMAT JSONEachRow" + ).output assert output == '{"d":"2020-01-01"}', error() with And("I revoke insert privilege on cluster with all nodes"): - node.query(f"REVOKE ON CLUSTER sharded_cluster INSERT ON {table_name} FROM {role_name}") + node.query( + f"REVOKE ON CLUSTER sharded_cluster INSERT ON {table_name} FROM {role_name}" + ) with Then("I insert into table expecting fail"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the user"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestOutline(Feature) -@Requirements( - RQ_SRS_006_RBAC_Insert("1.0"), - RQ_SRS_006_RBAC_Insert_TableEngines("1.0") -) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Requirements(RQ_SRS_006_RBAC_Insert("1.0"), RQ_SRS_006_RBAC_Insert_TableEngines("1.0")) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("insert") def feature(self, table_type, stress=None, node="clickhouse1"): - """Check the RBAC functionality of INSERT. - """ - args = {"table_type" : table_type} + """Check the RBAC functionality of INSERT.""" + args = {"table_type": table_type} self.context.node = self.context.cluster.node(node) @@ -502,6 +667,11 @@ def feature(self, table_type, stress=None, node="clickhouse1"): with Pool(10) as pool: try: for scenario in loads(current_module(), Scenario): - Scenario(test=scenario, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Scenario( + test=scenario, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/introspection.py b/tests/testflows/rbac/tests/privileges/introspection.py index a8d62cf8618..b36085ced96 100644 --- a/tests/testflows/rbac/tests/privileges/introspection.py +++ b/tests/testflows/rbac/tests/privileges/introspection.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def allow_introspection_functions(node): setting = ("allow_introspection_functions", 1) @@ -12,21 +13,25 @@ def allow_introspection_functions(node): try: with Given("I add allow_introspection_functions to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(setting) yield finally: - with Finally("I remove allow_introspection_functions from the default query settings"): + with Finally( + "I remove allow_introspection_functions from the default query settings" + ): if default_query_settings: try: default_query_settings.pop(default_query_settings.index(setting)) except ValueError: pass + @TestSuite def addressToLine_privileges_granted_directly(self, node=None): - """Check that a user is able to execute `addressToLine` with privileges are granted directly. - """ + """Check that a user is able to execute `addressToLine` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -35,15 +40,22 @@ def addressToLine_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=addressToLine, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in addressToLine.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=addressToLine, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in addressToLine.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def addressToLine_privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `addressToLine` with privileges are granted through a role. - """ + """Check that a user is able to execute `addressToLine` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -56,24 +68,34 @@ def addressToLine_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=addressToLine, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in addressToLine.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=addressToLine, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in addressToLine.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("INTROSPECTION",), - ("INTROSPECTION FUNCTIONS",), - ("addressToLine",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("INTROSPECTION",), + ("INTROSPECTION FUNCTIONS",), + ("addressToLine",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Introspection_addressToLine("1.0"), ) def addressToLine(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `addressToLine` when they have the necessary privilege. - """ + """Check that user is only able to execute `addressToLine` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -88,8 +110,12 @@ def addressToLine(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use addressToLine"): - node.query(f"WITH addressToLine(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"WITH addressToLine(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("addressToLine with privilege"): @@ -97,7 +123,10 @@ def addressToLine(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use addressToLine"): - node.query(f"WITH addressToLine(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", settings = [("user", f"{user_name}")]) + node.query( + f"WITH addressToLine(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", f"{user_name}")], + ) with Scenario("addressToLine with revoked privilege"): @@ -108,13 +137,17 @@ def addressToLine(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use addressToLine"): - node.query(f"WITH addressToLine(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"WITH addressToLine(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def addressToSymbol_privileges_granted_directly(self, node=None): - """Check that a user is able to execute `addressToSymbol` with privileges are granted directly. - """ + """Check that a user is able to execute `addressToSymbol` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -123,15 +156,22 @@ def addressToSymbol_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=addressToSymbol, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in addressToSymbol.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=addressToSymbol, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in addressToSymbol.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def addressToSymbol_privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `addressToSymbol` with privileges are granted through a role. - """ + """Check that a user is able to execute `addressToSymbol` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -144,24 +184,34 @@ def addressToSymbol_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=addressToSymbol, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in addressToSymbol.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=addressToSymbol, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in addressToSymbol.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("INTROSPECTION",), - ("INTROSPECTION FUNCTIONS",), - ("addressToSymbol",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("INTROSPECTION",), + ("INTROSPECTION FUNCTIONS",), + ("addressToSymbol",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Introspection_addressToSymbol("1.0"), ) def addressToSymbol(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `addressToSymbol` when they have the necessary privilege. - """ + """Check that user is only able to execute `addressToSymbol` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -176,8 +226,12 @@ def addressToSymbol(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use addressToSymbol"): - node.query(f"WITH addressToSymbol(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"WITH addressToSymbol(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("addressToSymbol with privilege"): @@ -185,7 +239,10 @@ def addressToSymbol(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use addressToSymbol"): - node.query(f"WITH addressToSymbol(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", settings = [("user", f"{user_name}")]) + node.query( + f"WITH addressToSymbol(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", f"{user_name}")], + ) with Scenario("addressToSymbol with revoked privilege"): @@ -196,13 +253,17 @@ def addressToSymbol(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use addressToSymbol"): - node.query(f"WITH addressToSymbol(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"WITH addressToSymbol(toUInt64(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def demangle_privileges_granted_directly(self, node=None): - """Check that a user is able to execute `demangle` with privileges are granted directly. - """ + """Check that a user is able to execute `demangle` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -211,15 +272,22 @@ def demangle_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=demangle, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in demangle.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=demangle, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in demangle.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def demangle_privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `demangle` with privileges are granted through a role. - """ + """Check that a user is able to execute `demangle` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -232,24 +300,34 @@ def demangle_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=demangle, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in demangle.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=demangle, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in demangle.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("INTROSPECTION",), - ("INTROSPECTION FUNCTIONS",), - ("demangle",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("INTROSPECTION",), + ("INTROSPECTION FUNCTIONS",), + ("demangle",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Introspection_demangle("1.0"), ) def demangle(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `demangle` when they have the necessary privilege. - """ + """Check that user is only able to execute `demangle` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -264,8 +342,12 @@ def demangle(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use demangle"): - node.query(f"WITH demangle(toString(dummy)) AS addr SELECT 1 WHERE addr = ''", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"WITH demangle(toString(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("demangle with privilege"): @@ -273,7 +355,10 @@ def demangle(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use demangle"): - node.query(f"WITH demangle(toString(dummy)) AS addr SELECT 1 WHERE addr = ''", settings = [("user", f"{user_name}")]) + node.query( + f"WITH demangle(toString(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", f"{user_name}")], + ) with Scenario("demangle with revoked privilege"): @@ -284,25 +369,47 @@ def demangle(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use demangle"): - node.query(f"WITH demangle(toString(dummy)) AS addr SELECT 1 WHERE addr = ''", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"WITH demangle(toString(dummy)) AS addr SELECT 1 WHERE addr = ''", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("introspection") @Requirements( RQ_SRS_006_RBAC_Privileges_Introspection("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of INTROSPECTION. - """ + """Check the RBAC functionality of INTROSPECTION.""" self.context.node = self.context.cluster.node(node) with allow_introspection_functions(self.context.node): - Suite(run=addressToLine_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=addressToLine_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=addressToSymbol_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=addressToSymbol_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=demangle_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=demangle_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=addressToLine_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=addressToLine_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=addressToSymbol_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=addressToSymbol_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=demangle_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=demangle_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/kill_mutation.py b/tests/testflows/rbac/tests/privileges/kill_mutation.py index 9a27836cad4..ce810ff8625 100644 --- a/tests/testflows/rbac/tests/privileges/kill_mutation.py +++ b/tests/testflows/rbac/tests/privileges/kill_mutation.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def no_privilege(self, node=None): - """Check that user doesn't need privileges to execute `KILL MUTATION` with no mutations. - """ + """Check that user doesn't need privileges to execute `KILL MUTATION` with no mutations.""" if node is None: node = self.context.node @@ -24,7 +24,10 @@ def no_privilege(self, node=None): node.query(f"GRANT USAGE ON *.* TO {user_name}") with Then("I attempt to kill mutation on table"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) with Scenario("kill mutation on cluster"): user_name = f"user_{getuid()}" @@ -41,7 +44,11 @@ def no_privilege(self, node=None): node.query(f"GRANT USAGE ON *.* TO {user_name}") with Then("I attempt to kill mutation on cluster"): - node.query(f"KILL MUTATION ON CLUSTER sharded_cluster WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION ON CLUSTER sharded_cluster WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) + @TestSuite def privileges_granted_directly(self, node=None): @@ -60,6 +67,7 @@ def privileges_granted_directly(self, node=None): Suite(test=delete)(user_name=user_name, grant_target_name=user_name) Suite(test=drop_column)(user_name=user_name, grant_target_name=user_name) + @TestSuite def privileges_granted_via_role(self, node=None): """Check that a user is able to execute `KILL MUTATION` on a table with a mutation @@ -81,10 +89,9 @@ def privileges_granted_via_role(self, node=None): Suite(test=delete)(user_name=user_name, grant_target_name=role_name) Suite(test=drop_column)(user_name=user_name, grant_target_name=role_name) + @TestSuite -@Requirements( - RQ_SRS_006_RBAC_Privileges_KillMutation_AlterUpdate("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_KillMutation_AlterUpdate("1.0")) def update(self, user_name, grant_target_name, node=None): """Check that the user is able to execute `KILL MUTATION` after `ALTER UPDATE` if and only if the user has `ALTER UPDATE` privilege. @@ -112,8 +119,12 @@ def update(self, user_name, grant_target_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER UPDATE with privilege"): table_name = f"merge_tree_{getuid()}" @@ -127,7 +138,10 @@ def update(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALTER UPDATE ON {table_name} TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) with Scenario("KILL ALTER UPDATE with revoked privilege"): table_name = f"merge_tree_{getuid()}" @@ -141,11 +155,17 @@ def update(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALTER UPDATE ON {table_name} TO {grant_target_name}") with And("I revoke the ALTER UPDATE privilege"): - node.query(f"REVOKE ALTER UPDATE ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE ALTER UPDATE ON {table_name} FROM {grant_target_name}" + ) with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER UPDATE with revoked ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -162,8 +182,12 @@ def update(self, user_name, grant_target_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER UPDATE with ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -177,12 +201,14 @@ def update(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) + @TestSuite -@Requirements( - RQ_SRS_006_RBAC_Privileges_KillMutation_AlterDelete("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_KillMutation_AlterDelete("1.0")) def delete(self, user_name, grant_target_name, node=None): """Check that the user is able to execute `KILL MUTATION` after `ALTER DELETE` if and only if the user has `ALTER DELETE` privilege. @@ -210,8 +236,12 @@ def delete(self, user_name, grant_target_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER DELETE with privilege"): table_name = f"merge_tree_{getuid()}" @@ -225,7 +255,10 @@ def delete(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALTER DELETE ON {table_name} TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) with Scenario("KILL ALTER DELETE with revoked privilege"): table_name = f"merge_tree_{getuid()}" @@ -239,11 +272,17 @@ def delete(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALTER DELETE ON {table_name} TO {grant_target_name}") with And("I revoke the ALTER DELETE privilege"): - node.query(f"REVOKE ALTER DELETE ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE ALTER DELETE ON {table_name} FROM {grant_target_name}" + ) with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER DELETE with revoked ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -260,8 +299,12 @@ def delete(self, user_name, grant_target_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER DELETE with ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -275,12 +318,14 @@ def delete(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) + @TestSuite -@Requirements( - RQ_SRS_006_RBAC_Privileges_KillMutation_AlterDropColumn("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_KillMutation_AlterDropColumn("1.0")) def drop_column(self, user_name, grant_target_name, node=None): """Check that the user is able to execute `KILL MUTATION` after `ALTER DROP COLUMN` if and only if the user has `ALTER DROP COLUMN` privilege. @@ -308,8 +353,12 @@ def drop_column(self, user_name, grant_target_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER DROP COLUMN with privilege"): table_name = f"merge_tree_{getuid()}" @@ -320,10 +369,15 @@ def drop_column(self, user_name, grant_target_name, node=None): node.query(f"ALTER TABLE {table_name} DROP COLUMN x") with When("I grant the ALTER DROP COLUMN privilege"): - node.query(f"GRANT ALTER DROP COLUMN ON {table_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER DROP COLUMN ON {table_name} TO {grant_target_name}" + ) with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) with Scenario("KILL ALTER DROP COLUMN with revoked privilege"): table_name = f"merge_tree_{getuid()}" @@ -334,14 +388,22 @@ def drop_column(self, user_name, grant_target_name, node=None): node.query(f"ALTER TABLE {table_name} DROP COLUMN x") with When("I grant the ALTER DROP COLUMN privilege"): - node.query(f"GRANT ALTER DROP COLUMN ON {table_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER DROP COLUMN ON {table_name} TO {grant_target_name}" + ) with And("I revoke the ALTER DROP COLUMN privilege"): - node.query(f"REVOKE ALTER DROP COLUMN ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE ALTER DROP COLUMN ON {table_name} FROM {grant_target_name}" + ) with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER DROP COLUMN with revoked privilege"): table_name = f"merge_tree_{getuid()}" @@ -352,14 +414,20 @@ def drop_column(self, user_name, grant_target_name, node=None): node.query(f"ALTER TABLE {table_name} DROP COLUMN x") with When("I grant the ALTER DROP COLUMN privilege"): - node.query(f"GRANT ALTER DROP COLUMN ON {table_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER DROP COLUMN ON {table_name} TO {grant_target_name}" + ) with And("I revoke ALL privilege"): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)], - exitcode=exitcode, message="Exception: Not allowed to kill mutation.") + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + exitcode=exitcode, + message="Exception: Not allowed to kill mutation.", + ) with Scenario("KILL ALTER DROP COLUMN with ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -373,18 +441,21 @@ def drop_column(self, user_name, grant_target_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I try to KILL MUTATION"): - node.query(f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", settings = [("user", user_name)]) + node.query( + f"KILL MUTATION WHERE database = 'default' AND table = '{table_name}'", + settings=[("user", user_name)], + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_KillMutation("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("kill mutation") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of KILL MUTATION. - """ + """Check the RBAC functionality of KILL MUTATION.""" self.context.node = self.context.cluster.node(node) if parallel is not None: diff --git a/tests/testflows/rbac/tests/privileges/kill_query.py b/tests/testflows/rbac/tests/privileges/kill_query.py index d1f96e23fd8..397d783a097 100644 --- a/tests/testflows/rbac/tests/privileges/kill_query.py +++ b/tests/testflows/rbac/tests/privileges/kill_query.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, node=None): - """Check that user is only able to execute KILL QUERY when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute KILL QUERY when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,12 @@ def privilege_granted_directly_or_via_role(self, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute KILL QUERY with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, node=node) + with When( + f"I run checks that {user_name} is only able to execute KILL QUERY with required privileges" + ): + privilege_check( + grant_target_name=user_name, user_name=user_name, node=node + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +28,16 @@ def privilege_granted_directly_or_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute KILL QUERY with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute KILL QUERY with required privileges" + ): + privilege_check( + grant_target_name=role_name, user_name=user_name, node=node + ) + def privilege_check(grant_target_name, user_name, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -41,8 +49,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to kill a query without privilege"): - node.query(f"KILL QUERY WHERE user ='default'", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"KILL QUERY WHERE user ='default'", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): @@ -50,7 +62,7 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT KILL QUERY TO {grant_target_name}") with Then("I attempt to kill a query"): - node.query(f"KILL QUERY WHERE 1", settings = [("user", user_name)]) + node.query(f"KILL QUERY WHERE 1", settings=[("user", user_name)]) with Scenario("user with revoked privilege"): @@ -61,8 +73,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE KILL QUERY TO {grant_target_name}") with Then("I attempt to kill a query"): - node.query(f"KILL QUERY WHERE 1", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"KILL QUERY WHERE 1", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with revoked ALL privilege"): @@ -73,8 +89,12 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to kill a query"): - node.query(f"KILL QUERY WHERE 1", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"KILL QUERY WHERE 1", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("execute on cluster"): @@ -82,7 +102,9 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT KILL QUERY TO {grant_target_name}") with Then("I attempt to kill a query"): - node.query(f"KILL QUERY ON CLUSTER WHERE 1'", settings = [("user", user_name)]) + node.query( + f"KILL QUERY ON CLUSTER WHERE 1'", settings=[("user", user_name)] + ) with Scenario("user with ALL privilege"): @@ -93,18 +115,18 @@ def privilege_check(grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* ON {grant_target_name}") with Then("I attempt to kill a query"): - node.query(f"KILL QUERY WHERE 1", settings = [("user", user_name)]) + node.query(f"KILL QUERY WHERE 1", settings=[("user", user_name)]) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_KillQuery("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) @Name("kill query") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of KILL QUERY. - """ + """Check the RBAC functionality of KILL QUERY.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -112,5 +134,8 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): if stress is not None: self.context.stress = stress - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role() diff --git a/tests/testflows/rbac/tests/privileges/optimize.py b/tests/testflows/rbac/tests/privileges/optimize.py index 7d3f41a43b4..0e984a17a9b 100644 --- a/tests/testflows/rbac/tests/privileges/optimize.py +++ b/tests/testflows/rbac/tests/privileges/optimize.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, node=None): - """Check that user is only able to execute OPTIMIZE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute OPTIMIZE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,15 @@ def privilege_granted_directly_or_via_role(self, table_type, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute OPTIMIZE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, node=node) + with When( + f"I run checks that {user_name} is only able to execute OPTIMIZE with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + node=node, + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +31,19 @@ def privilege_granted_directly_or_via_role(self, table_type, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute OPTIMIZE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute OPTIMIZE with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -44,8 +58,12 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to optimize a table without privilege"): - node.query(f"OPTIMIZE TABLE {table_name} FINAL", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"OPTIMIZE TABLE {table_name} FINAL", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): table_name = f"merge_tree_{getuid()}" @@ -56,7 +74,9 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT OPTIMIZE ON {table_name} TO {grant_target_name}") with Then("I attempt to optimize a table"): - node.query(f"OPTIMIZE TABLE {table_name}", settings = [("user", user_name)]) + node.query( + f"OPTIMIZE TABLE {table_name}", settings=[("user", user_name)] + ) with Scenario("user with revoked privilege"): table_name = f"merge_tree_{getuid()}" @@ -70,8 +90,12 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"REVOKE OPTIMIZE ON {table_name} FROM {grant_target_name}") with Then("I attempt to optimize a table"): - node.query(f"OPTIMIZE TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"OPTIMIZE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with revoked ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -85,25 +109,36 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to optimize a table"): - node.query(f"OPTIMIZE TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"OPTIMIZE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("execute on cluster"): table_name = f"merge_tree_{getuid()}" try: with Given("I have a table on a cluster"): - node.query(f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) ENGINE = MergeTree() PARTITION BY y ORDER BY d") + node.query( + f"CREATE TABLE {table_name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) ENGINE = MergeTree() PARTITION BY y ORDER BY d" + ) with When("I grant the optimize privilege"): node.query(f"GRANT OPTIMIZE ON {table_name} TO {grant_target_name}") with Then("I attempt to optimize a table"): - node.query(f"OPTIMIZE TABLE {table_name} ON CLUSTER sharded_cluster", settings = [("user", user_name)]) + node.query( + f"OPTIMIZE TABLE {table_name} ON CLUSTER sharded_cluster", + settings=[("user", user_name)], + ) finally: with Finally("I drop the table from the cluster"): - node.query(f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster" + ) with Scenario("user with ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -117,20 +152,21 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to optimize a table"): - node.query(f"OPTIMIZE TABLE {table_name}", settings = [("user", user_name)]) + node.query( + f"OPTIMIZE TABLE {table_name}", settings=[("user", user_name)] + ) + + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_Optimize("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("optimize") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of OPTIMIZE. - """ + """Check the RBAC functionality of OPTIMIZE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -139,11 +175,14 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue with Example(str(example)): - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role(table_type=table_type) diff --git a/tests/testflows/rbac/tests/privileges/public_tables.py b/tests/testflows/rbac/tests/privileges/public_tables.py index 52232022ed6..7ce67d7f0b0 100755 --- a/tests/testflows/rbac/tests/privileges/public_tables.py +++ b/tests/testflows/rbac/tests/privileges/public_tables.py @@ -6,13 +6,13 @@ from testflows.asserts import error from rbac.requirements import * from rbac.helper.common import * + @TestScenario @Requirements( RQ_SRS_006_RBAC_Table_PublicTables("1.0"), ) def public_tables(self, node=None): - """Check that a user with no privilege is able to select from public tables. - """ + """Check that a user with no privilege is able to select from public tables.""" user_name = f"user_{getuid()}" if node is None: node = self.context.node @@ -20,24 +20,33 @@ def public_tables(self, node=None): with user(node, f"{user_name}"): with When("I check the user is able to select on system.one"): - node.query("SELECT count(*) FROM system.one", settings = [("user",user_name)]) + node.query( + "SELECT count(*) FROM system.one", settings=[("user", user_name)] + ) with And("I check the user is able to select on system.numbers"): - node.query("SELECT * FROM system.numbers LIMIT 1", settings = [("user",user_name)]) + node.query( + "SELECT * FROM system.numbers LIMIT 1", settings=[("user", user_name)] + ) with And("I check the user is able to select on system.contributors"): - node.query("SELECT count(*) FROM system.contributors", settings = [("user",user_name)]) + node.query( + "SELECT count(*) FROM system.contributors", + settings=[("user", user_name)], + ) with And("I check the user is able to select on system.functions"): - node.query("SELECT count(*) FROM system.functions", settings = [("user",user_name)]) + node.query( + "SELECT count(*) FROM system.functions", settings=[("user", user_name)] + ) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Table_SensitiveTables("1.0"), ) def sensitive_tables(self, node=None): - """Check that a user with no privilege is not able to see from these tables. - """ + """Check that a user with no privilege is not able to see from these tables.""" user_name = f"user_{getuid()}" if node is None: node = self.context.node @@ -47,53 +56,84 @@ def sensitive_tables(self, node=None): node.query("SELECT 1") with When("I select from processes"): - output = node.query("SELECT count(*) FROM system.processes", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.processes", settings=[("user", user_name)] + ).output assert output == 0, error() with And("I select from query_log"): - output = node.query("SELECT count(*) FROM system.query_log", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.query_log", settings=[("user", user_name)] + ).output assert output == 0, error() with And("I select from query_thread_log"): - output = node.query("SELECT count(*) FROM system.query_thread_log", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.query_thread_log", + settings=[("user", user_name)], + ).output assert output == 0, error() with And("I select from query_views_log"): - output = node.query("SELECT count(*) FROM system.query_views_log", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.query_views_log", + settings=[("user", user_name)], + ).output assert output == 0, error() with And("I select from clusters"): - output = node.query("SELECT count(*) FROM system.clusters", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.clusters", settings=[("user", user_name)] + ).output assert output == 0, error() with And("I select from events"): - output = node.query("SELECT count(*) FROM system.events", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.events", settings=[("user", user_name)] + ).output assert output == 0, error() with And("I select from graphite_retentions"): - output = node.query("SELECT count(*) FROM system.graphite_retentions", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.graphite_retentions", + settings=[("user", user_name)], + ).output assert output == 0, error() with And("I select from stack_trace"): - output = node.query("SELECT count(*) FROM system.stack_trace", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.stack_trace", + settings=[("user", user_name)], + ).output assert output == 0, error() with And("I select from trace_log"): - output = node.query("SELECT count(*) FROM system.trace_log", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.trace_log", settings=[("user", user_name)] + ).output assert output == 0, error() with And("I select from user_directories"): - output = node.query("SELECT count(*) FROM system.user_directories", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.user_directories", + settings=[("user", user_name)], + ).output assert output == 0, error() with And("I select from zookeeper"): - output = node.query("SELECT count(*) FROM system.zookeeper WHERE path = '/clickhouse' ", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.zookeeper WHERE path = '/clickhouse' ", + settings=[("user", user_name)], + ).output assert output == 0, error() with And("I select from macros"): - output = node.query("SELECT count(*) FROM system.macros", settings = [("user",user_name)]).output + output = node.query( + "SELECT count(*) FROM system.macros", settings=[("user", user_name)] + ).output assert output == 0, error() + @TestFeature @Name("public tables") def feature(self, node="clickhouse1"): diff --git a/tests/testflows/rbac/tests/privileges/role_admin.py b/tests/testflows/rbac/tests/privileges/role_admin.py index 8deea7874cd..191d4cb13c5 100644 --- a/tests/testflows/rbac/tests/privileges/role_admin.py +++ b/tests/testflows/rbac/tests/privileges/role_admin.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to grant role with `ROLE ADMIN` privilege granted directly. - """ + """Check that a user is able to grant role with `ROLE ADMIN` privilege granted directly.""" user_name = f"user_{getuid()}" @@ -19,10 +19,10 @@ def privileges_granted_directly(self, node=None): Suite(test=role_admin)(grant_target_name=user_name, user_name=user_name) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to grant role with `ROLE ADMIN` privilege granted through a role. - """ + """Check that a user is able to grant role with `ROLE ADMIN` privilege granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -37,10 +37,10 @@ def privileges_granted_via_role(self, node=None): Suite(test=role_admin)(grant_target_name=role_name, user_name=user_name) + @TestSuite def role_admin(self, grant_target_name, user_name, node=None): - """Check that user is able to execute to grant roles if and only if they have `ROLE ADMIN`. - """ + """Check that user is able to execute to grant roles if and only if they have `ROLE ADMIN`.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -59,8 +59,12 @@ def role_admin(self, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't grant a role"): - node.query(f"GRANT {role_admin_name} TO {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {role_admin_name} TO {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("Grant role with privilege"): role_admin_name = f"role_admin_{getuid()}" @@ -72,7 +76,10 @@ def role_admin(self, grant_target_name, user_name, node=None): node.query(f"GRANT ROLE ADMIN ON *.* TO {grant_target_name}") with Then("I check the user can grant a role"): - node.query(f"GRANT {role_admin_name} TO {target_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"GRANT {role_admin_name} TO {target_user_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("Grant role on cluster"): role_admin_name = f"role_admin_{getuid()}" @@ -89,11 +96,16 @@ def role_admin(self, grant_target_name, user_name, node=None): node.query(f"GRANT ROLE ADMIN ON *.* TO {grant_target_name}") with Then("I check the user can grant a role"): - node.query(f"GRANT {role_admin_name} TO {target_user_name} ON CLUSTER sharded_cluster", settings = [("user", f"{user_name}")]) + node.query( + f"GRANT {role_admin_name} TO {target_user_name} ON CLUSTER sharded_cluster", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the user"): - node.query(f"DROP ROLE IF EXISTS {role_admin_name} ON CLUSTER sharded_cluster") + node.query( + f"DROP ROLE IF EXISTS {role_admin_name} ON CLUSTER sharded_cluster" + ) with Scenario("Grant role with revoked privilege"): role_admin_name = f"role_admin_{getuid()}" @@ -108,8 +120,12 @@ def role_admin(self, grant_target_name, user_name, node=None): node.query(f"REVOKE ROLE ADMIN ON *.* FROM {grant_target_name}") with Then("I check the user cannot grant a role"): - node.query(f"GRANT {role_admin_name} TO {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {role_admin_name} TO {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("Grant role with revoked ALL privilege"): role_admin_name = f"role_admin_{getuid()}" @@ -124,8 +140,12 @@ def role_admin(self, grant_target_name, user_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I check the user cannot grant a role"): - node.query(f"GRANT {role_admin_name} TO {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"GRANT {role_admin_name} TO {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("Grant role with ALL privilege"): role_admin_name = f"role_admin_{getuid()}" @@ -137,18 +157,21 @@ def role_admin(self, grant_target_name, user_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I check the user can grant a role"): - node.query(f"GRANT {role_admin_name} TO {target_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"GRANT {role_admin_name} TO {target_user_name}", + settings=[("user", f"{user_name}")], + ) + @TestFeature @Name("role admin") @Requirements( RQ_SRS_006_RBAC_Privileges_RoleAdmin("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of ROLE ADMIN. - """ + """Check the RBAC functionality of ROLE ADMIN.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/select.py b/tests/testflows/rbac/tests/privileges/select.py index b1a95b4be0b..add90c7789d 100755 --- a/tests/testflows/rbac/tests/privileges/select.py +++ b/tests/testflows/rbac/tests/privileges/select.py @@ -8,13 +8,11 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_Privileges_None("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_None("1.0")) def without_privilege(self, table_type, node=None): - """Check that user without select privilege on a table is not able to select on that table. - """ + """Check that user without select privilege on a table is not able to select on that table.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -34,16 +32,20 @@ def without_privilege(self, table_type, node=None): with Then("I run SELECT without privilege"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT * FROM {table_name}", settings = [("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_006_RBAC_Grant_Privilege_Select("1.0"), ) def user_with_privilege(self, table_type, node=None): - """Check that user can select from a table on which they have select privilege. - """ + """Check that user can select from a table on which they have select privilege.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -61,18 +63,18 @@ def user_with_privilege(self, table_type, node=None): node.query(f"GRANT SELECT ON {table_name} TO {user_name}") with Then("I verify SELECT command"): - user_select = node.query(f"SELECT d FROM {table_name}", settings = [("user",user_name)]) + user_select = node.query( + f"SELECT d FROM {table_name}", settings=[("user", user_name)] + ) default = node.query(f"SELECT d FROM {table_name}") assert user_select.output == default.output, error() + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_Privileges_All("1.0") -) +@Requirements(RQ_SRS_006_RBAC_Privileges_All("1.0")) def user_with_all_privilege(self, table_type, node=None): - """Check that user can select from a table if have ALL privilege. - """ + """Check that user can select from a table if have ALL privilege.""" user_name = f"user_{getuid()}" table_name = f"table_{getuid()}" @@ -90,11 +92,14 @@ def user_with_all_privilege(self, table_type, node=None): node.query(f"GRANT ALL ON *.* TO {user_name}") with Then("I verify SELECT command"): - user_select = node.query(f"SELECT d FROM {table_name}", settings = [("user",user_name)]) + user_select = node.query( + f"SELECT d FROM {table_name}", settings=[("user", user_name)] + ) default = node.query(f"SELECT d FROM {table_name}") assert user_select.output == default.output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_Revoke_Privilege_Select("1.0"), @@ -121,8 +126,13 @@ def user_with_revoked_privilege(self, table_type, node=None): with Then("I use SELECT, throws exception"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT * FROM {table_name}", settings = [("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_revoked_all_privilege(self, table_type, node=None): @@ -147,26 +157,51 @@ def user_with_revoked_all_privilege(self, table_type, node=None): with Then("I use SELECT, throws exception"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT * FROM {table_name}", settings = [("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_privilege_on_columns(self, table_type): - Scenario(run=user_column_privileges, - examples=Examples("grant_columns revoke_columns select_columns_fail select_columns_pass data_pass table_type", - [tuple(list(row)+[table_type]) for row in user_column_privileges.examples])) + Scenario( + run=user_column_privileges, + examples=Examples( + "grant_columns revoke_columns select_columns_fail select_columns_pass data_pass table_type", + [ + tuple(list(row) + [table_type]) + for row in user_column_privileges.examples + ], + ), + ) + @TestOutline @Requirements( RQ_SRS_006_RBAC_Select_Column("1.0"), ) -@Examples("grant_columns revoke_columns select_columns_fail select_columns_pass data_pass", [ - ("d", "d", "x", "d", '\'2020-01-01\''), - ("d,a", "d", "x", "d", '\'2020-01-01\''), - ("d,a,b", "d,a,b", "x", "d,b", '\'2020-01-01\',9'), - ("d,a,b", "b", "y", "d,a,b", '\'2020-01-01\',\'woo\',9') -]) -def user_column_privileges(self, grant_columns, select_columns_pass, data_pass, table_type, revoke_columns=None, select_columns_fail=None, node=None): +@Examples( + "grant_columns revoke_columns select_columns_fail select_columns_pass data_pass", + [ + ("d", "d", "x", "d", "'2020-01-01'"), + ("d,a", "d", "x", "d", "'2020-01-01'"), + ("d,a,b", "d,a,b", "x", "d,b", "'2020-01-01',9"), + ("d,a,b", "b", "y", "d,a,b", "'2020-01-01','woo',9"), + ], +) +def user_column_privileges( + self, + grant_columns, + select_columns_pass, + data_pass, + table_type, + revoke_columns=None, + select_columns_fail=None, + node=None, +): """Check that user is able to select on granted columns and unable to select on not granted or revoked columns. """ @@ -179,7 +214,9 @@ def user_column_privileges(self, grant_columns, select_columns_pass, data_pass, with table(node, table_name, table_type), user(node, user_name): with Given("The table has some data on some columns"): - node.query(f"INSERT INTO {table_name} ({select_columns_pass}) VALUES ({data_pass})") + node.query( + f"INSERT INTO {table_name} ({select_columns_pass}) VALUES ({data_pass})" + ) with When("I grant select privilege"): node.query(f"GRANT SELECT({grant_columns}) ON {table_name} TO {user_name}") @@ -188,22 +225,37 @@ def user_column_privileges(self, grant_columns, select_columns_pass, data_pass, with And("I select from not granted column"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT ({select_columns_fail}) FROM {table_name}", - settings = [("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT ({select_columns_fail}) FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Then("I select from granted column, verify correct result"): - user_select = node.query(f"SELECT ({select_columns_pass}) FROM {table_name}", settings = [("user",user_name)]) + user_select = node.query( + f"SELECT ({select_columns_pass}) FROM {table_name}", + settings=[("user", user_name)], + ) default = node.query(f"SELECT ({select_columns_pass}) FROM {table_name}") assert user_select.output == default.output if revoke_columns is not None: with When("I revoke select privilege for columns from user"): - node.query(f"REVOKE SELECT({revoke_columns}) ON {table_name} FROM {user_name}") + node.query( + f"REVOKE SELECT({revoke_columns}) ON {table_name} FROM {user_name}" + ) with And("I select from revoked columns"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT ({select_columns_pass}) FROM {table_name}", settings = [("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT ({select_columns_pass}) FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( @@ -236,10 +288,13 @@ def role_with_privilege(self, table_type, node=None): node.query(f"GRANT {role_name} TO {user_name}") with Then("I verify SELECT command"): - user_select = node.query(f"SELECT d FROM {table_name}", settings = [("user",user_name)]) + user_select = node.query( + f"SELECT d FROM {table_name}", settings=[("user", user_name)] + ) default = node.query(f"SELECT d FROM {table_name}") assert user_select.output == default.output, error() + @TestScenario @Requirements( RQ_SRS_006_RBAC_Revoke_Privilege_Select("1.0"), @@ -270,8 +325,13 @@ def role_with_revoked_privilege(self, table_type, node=None): with And("I select from the table"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT * FROM {table_name}", settings = [("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def user_with_revoked_role(self, table_type, node=None): @@ -300,26 +360,51 @@ def user_with_revoked_role(self, table_type, node=None): with And("I select from the table"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT * FROM {table_name}", settings = [("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario def role_with_privilege_on_columns(self, table_type): - Scenario(run=role_column_privileges, - examples=Examples("grant_columns revoke_columns select_columns_fail select_columns_pass data_pass table_type", - [tuple(list(row)+[table_type]) for row in role_column_privileges.examples])) + Scenario( + run=role_column_privileges, + examples=Examples( + "grant_columns revoke_columns select_columns_fail select_columns_pass data_pass table_type", + [ + tuple(list(row) + [table_type]) + for row in role_column_privileges.examples + ], + ), + ) + @TestOutline @Requirements( RQ_SRS_006_RBAC_Select_Column("1.0"), ) -@Examples("grant_columns revoke_columns select_columns_fail select_columns_pass data_pass", [ - ("d", "d", "x", "d", '\'2020-01-01\''), - ("d,a", "d", "x", "d", '\'2020-01-01\''), - ("d,a,b", "d,a,b", "x", "d,b", '\'2020-01-01\',9'), - ("d,a,b", "b", "y", "d,a,b", '\'2020-01-01\',\'woo\',9') -]) -def role_column_privileges(self, grant_columns, select_columns_pass, data_pass, table_type, revoke_columns=None, select_columns_fail=None, node=None): +@Examples( + "grant_columns revoke_columns select_columns_fail select_columns_pass data_pass", + [ + ("d", "d", "x", "d", "'2020-01-01'"), + ("d,a", "d", "x", "d", "'2020-01-01'"), + ("d,a,b", "d,a,b", "x", "d,b", "'2020-01-01',9"), + ("d,a,b", "b", "y", "d,a,b", "'2020-01-01','woo',9"), + ], +) +def role_column_privileges( + self, + grant_columns, + select_columns_pass, + data_pass, + table_type, + revoke_columns=None, + select_columns_fail=None, + node=None, +): """Check that user is able to select from granted columns and unable to select from not granted or revoked columns. """ @@ -333,12 +418,16 @@ def role_column_privileges(self, grant_columns, select_columns_pass, data_pass, with table(node, table_name, table_type): with Given("The table has some data on some columns"): - node.query(f"INSERT INTO {table_name} ({select_columns_pass}) VALUES ({data_pass})") + node.query( + f"INSERT INTO {table_name} ({select_columns_pass}) VALUES ({data_pass})" + ) with user(node, user_name), role(node, role_name): with When("I grant select privilege"): - node.query(f"GRANT SELECT({grant_columns}) ON {table_name} TO {role_name}") + node.query( + f"GRANT SELECT({grant_columns}) ON {table_name} TO {role_name}" + ) with And("I grant the role to a user"): node.query(f"GRANT {role_name} TO {user_name}") @@ -346,23 +435,36 @@ def role_column_privileges(self, grant_columns, select_columns_pass, data_pass, if select_columns_fail is not None: with And("I select from not granted column"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT ({select_columns_fail}) FROM {table_name}", - settings = [("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT ({select_columns_fail}) FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Then("I verify SELECT command"): - user_select = node.query(f"SELECT d FROM {table_name}", settings = [("user",user_name)]) + user_select = node.query( + f"SELECT d FROM {table_name}", settings=[("user", user_name)] + ) default = node.query(f"SELECT d FROM {table_name}") assert user_select.output == default.output, error() if revoke_columns is not None: with When("I revoke select privilege for columns from role"): - node.query(f"REVOKE SELECT({revoke_columns}) ON {table_name} FROM {role_name}") + node.query( + f"REVOKE SELECT({revoke_columns}) ON {table_name} FROM {role_name}" + ) with And("I select from revoked columns"): exitcode, message = errors.not_enough_privileges(name=user_name) - node.query(f"SELECT ({select_columns_pass}) FROM {table_name}", - settings = [("user",user_name)], exitcode=exitcode, message=message) + node.query( + f"SELECT ({select_columns_pass}) FROM {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( @@ -385,13 +487,19 @@ def user_with_privilege_on_cluster(self, table_type, node=None): node.query(f"INSERT INTO {table_name} (d) VALUES ('2020-01-01')") with Given("I have a user on a cluster"): - node.query(f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster") + node.query( + f"CREATE USER OR REPLACE {user_name} ON CLUSTER sharded_cluster" + ) with When("I grant select privilege on a cluster"): - node.query(f"GRANT ON CLUSTER sharded_cluster SELECT ON {table_name} TO {user_name}") + node.query( + f"GRANT ON CLUSTER sharded_cluster SELECT ON {table_name} TO {user_name}" + ) with Then("I verify SELECT command"): - user_select = node.query(f"SELECT d FROM {table_name}", settings = [("user",user_name)]) + user_select = node.query( + f"SELECT d FROM {table_name}", settings=[("user", user_name)] + ) default = node.query(f"SELECT d FROM {table_name}") assert user_select.output == default.output, error() @@ -399,28 +507,28 @@ def user_with_privilege_on_cluster(self, table_type, node=None): with Finally("I drop the user"): node.query(f"DROP USER {user_name} ON CLUSTER sharded_cluster") + @TestOutline(Feature) -@Requirements( - RQ_SRS_006_RBAC_Select("1.0"), - RQ_SRS_006_RBAC_Select_TableEngines("1.0") -) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Requirements(RQ_SRS_006_RBAC_Select("1.0"), RQ_SRS_006_RBAC_Select_TableEngines("1.0")) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("select") def feature(self, table_type, stress=None, node="clickhouse1"): - """Check the RBAC functionality of SELECT. - """ + """Check the RBAC functionality of SELECT.""" self.context.node = self.context.cluster.node(node) if stress is not None: self.context.stress = stress - args = {"table_type" : table_type} + args = {"table_type": table_type} with Pool(10) as pool: try: for scenario in loads(current_module(), Scenario): - Scenario(test=scenario, setup=instrument_clickhouse_server_log, parallel=True, executor=pool)(**args) + Scenario( + test=scenario, + setup=instrument_clickhouse_server_log, + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/tests/testflows/rbac/tests/privileges/show/show_columns.py b/tests/testflows/rbac/tests/privileges/show/show_columns.py index 108200e7a57..25bafe46a4e 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_columns.py +++ b/tests/testflows/rbac/tests/privileges/show/show_columns.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def describe_with_privilege_granted_directly(self, node=None): """Check that user is able to execute DESCRIBE on a table if and only if @@ -18,7 +19,10 @@ def describe_with_privilege_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(test=describe)(grant_target_name=user_name, user_name=user_name, table_name=table_name) + Suite(test=describe)( + grant_target_name=user_name, user_name=user_name, table_name=table_name + ) + @TestSuite def describe_with_privilege_granted_via_role(self, node=None): @@ -37,15 +41,17 @@ def describe_with_privilege_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(test=describe)(grant_target_name=role_name, user_name=user_name, table_name=table_name) + Suite(test=describe)( + grant_target_name=role_name, user_name=user_name, table_name=table_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_DescribeTable_RequiredPrivilege("1.0"), ) def describe(self, grant_target_name, user_name, table_name, node=None): - """Check that user is able to execute DESCRIBE only when they have SHOW COLUMNS privilege. - """ + """Check that user is able to execute DESCRIBE only when they have SHOW COLUMNS privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -62,8 +68,12 @@ def describe(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I attempt to DESCRIBE {table_name}"): - node.query(f"DESCRIBE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DESCRIBE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("DESCRIBE with privilege"): @@ -71,7 +81,9 @@ def describe(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT SHOW COLUMNS ON {table_name} TO {grant_target_name}") with Then(f"I attempt to DESCRIBE {table_name}"): - node.query(f"DESCRIBE TABLE {table_name}", settings=[("user",user_name)]) + node.query( + f"DESCRIBE TABLE {table_name}", settings=[("user", user_name)] + ) with Scenario("DESCRIBE with revoked privilege"): @@ -79,11 +91,17 @@ def describe(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT SHOW COLUMNS ON {table_name} TO {grant_target_name}") with And(f"I revoke SHOW COLUMNS on the table"): - node.query(f"REVOKE SHOW COLUMNS ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE SHOW COLUMNS ON {table_name} FROM {grant_target_name}" + ) with Then(f"I attempt to DESCRIBE {table_name}"): - node.query(f"DESCRIBE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DESCRIBE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("DESCRIBE with revoked ALL privilege"): @@ -94,8 +112,12 @@ def describe(self, grant_target_name, user_name, table_name, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then(f"I attempt to DESCRIBE {table_name}"): - node.query(f"DESCRIBE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"DESCRIBE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("DESCRIBE with ALL privilege"): @@ -103,7 +125,10 @@ def describe(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then(f"I attempt to DESCRIBE {table_name}"): - node.query(f"DESCRIBE TABLE {table_name}", settings=[("user",user_name)]) + node.query( + f"DESCRIBE TABLE {table_name}", settings=[("user", user_name)] + ) + @TestSuite def show_create_with_privilege_granted_directly(self, node=None): @@ -118,7 +143,10 @@ def show_create_with_privilege_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(test=show_create)(grant_target_name=user_name, user_name=user_name, table_name=table_name) + Suite(test=show_create)( + grant_target_name=user_name, user_name=user_name, table_name=table_name + ) + @TestSuite def show_create_with_privilege_granted_via_role(self, node=None): @@ -137,15 +165,17 @@ def show_create_with_privilege_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(test=show_create)(grant_target_name=role_name, user_name=user_name, table_name=table_name) + Suite(test=show_create)( + grant_target_name=role_name, user_name=user_name, table_name=table_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateTable_RequiredPrivilege("1.0"), ) def show_create(self, grant_target_name, user_name, table_name, node=None): - """Check that user is able to execute SHOW CREATE on a table only when they have SHOW COLUMNS privilege. - """ + """Check that user is able to execute SHOW CREATE on a table only when they have SHOW COLUMNS privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -162,8 +192,12 @@ def show_create(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {table_name}"): - node.query(f"SHOW CREATE TABLE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE with privilege"): @@ -171,7 +205,9 @@ def show_create(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT SHOW COLUMNS ON {table_name} TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {table_name}"): - node.query(f"SHOW CREATE TABLE {table_name}", settings=[("user",user_name)]) + node.query( + f"SHOW CREATE TABLE {table_name}", settings=[("user", user_name)] + ) with Scenario("SHOW CREATE with revoked privilege"): @@ -179,11 +215,17 @@ def show_create(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT SHOW COLUMNS ON {table_name} TO {grant_target_name}") with And(f"I revoke SHOW COLUMNS on the table"): - node.query(f"REVOKE SHOW COLUMNS ON {table_name} FROM {grant_target_name}") + node.query( + f"REVOKE SHOW COLUMNS ON {table_name} FROM {grant_target_name}" + ) with Then(f"I attempt to SHOW CREATE {table_name}"): - node.query(f"SHOW CREATE TABLE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE with ALL privilege"): @@ -191,21 +233,35 @@ def show_create(self, grant_target_name, user_name, table_name, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {table_name}"): - node.query(f"SHOW CREATE TABLE {table_name}", settings=[("user",user_name)]) + node.query( + f"SHOW CREATE TABLE {table_name}", settings=[("user", user_name)] + ) + @TestFeature @Name("show columns") @Requirements( RQ_SRS_006_RBAC_ShowColumns_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW COLUMNS. - """ + """Check the RBAC functionality of SHOW COLUMNS.""" self.context.node = self.context.cluster.node(node) - Suite(run=describe_with_privilege_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=describe_with_privilege_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=show_create_with_privilege_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=show_create_with_privilege_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=describe_with_privilege_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=describe_with_privilege_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=show_create_with_privilege_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=show_create_with_privilege_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/show/show_databases.py b/tests/testflows/rbac/tests/privileges/show/show_databases.py index 39a46947afe..b9e0dfc75d7 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_databases.py +++ b/tests/testflows/rbac/tests/privileges/show/show_databases.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def dict_privileges_granted_directly(self, node=None): """Check that a user is able to execute `USE` and `SHOW CREATE` @@ -20,10 +21,18 @@ def dict_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): db_name = f"db_name_{getuid()}" - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name db_name", [ - tuple(list(row)+[user_name,user_name,db_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name db_name", + [ + tuple(list(row) + [user_name, user_name, db_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dict_privileges_granted_via_role(self, node=None): @@ -44,39 +53,69 @@ def dict_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name db_name", [ - tuple(list(row)+[role_name,user_name,db_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name db_name", + [ + tuple(list(row) + [role_name, user_name, db_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SHOW","*.*"), - ("SHOW DATABASES","db"), - ("CREATE DATABASE","db"), - ("DROP DATABASE","db"), -]) -def check_privilege(self, privilege, on, grant_target_name, user_name, db_name, node=None): - """Run checks for commands that require SHOW DATABASE privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SHOW", "*.*"), + ("SHOW DATABASES", "db"), + ("CREATE DATABASE", "db"), + ("DROP DATABASE", "db"), + ], +) +def check_privilege( + self, privilege, on, grant_target_name, user_name, db_name, node=None +): + """Run checks for commands that require SHOW DATABASE privilege.""" if node is None: node = self.context.node on = on.replace("db", f"{db_name}") - Suite(test=show_db)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, db_name=db_name) - Suite(test=use)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, db_name=db_name) - Suite(test=show_create)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, db_name=db_name) + Suite(test=show_db)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + db_name=db_name, + ) + Suite(test=use)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + db_name=db_name, + ) + Suite(test=show_create)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + db_name=db_name, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowDatabases_RequiredPrivilege("1.0"), ) def show_db(self, privilege, on, grant_target_name, user_name, db_name, node=None): - """Check that user is only able to see a database in SHOW DATABASES when they have a privilege on that database. - """ + """Check that user is only able to see a database in SHOW DATABASES when they have a privilege on that database.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -96,8 +135,10 @@ def show_db(self, privilege, on, grant_target_name, user_name, db_name, node=Non node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user doesn't see the database"): - output = node.query("SHOW DATABASES", settings = [("user", f"{user_name}")]).output - assert output == '', error() + output = node.query( + "SHOW DATABASES", settings=[("user", f"{user_name}")] + ).output + assert output == "", error() with Scenario("SHOW DATABASES with privilege"): @@ -105,7 +146,11 @@ def show_db(self, privilege, on, grant_target_name, user_name, db_name, node=Non node.query(f"GRANT {privilege} ON {db_name}.* TO {grant_target_name}") with Then("I check the user does see a database"): - output = node.query("SHOW DATABASES", settings = [("user", f"{user_name}")], message = f'{db_name}') + output = node.query( + "SHOW DATABASES", + settings=[("user", f"{user_name}")], + message=f"{db_name}", + ) with Scenario("SHOW DATABASES with revoked privilege"): @@ -113,16 +158,21 @@ def show_db(self, privilege, on, grant_target_name, user_name, db_name, node=Non node.query(f"GRANT {privilege} ON {db_name}.* TO {grant_target_name}") with And(f"I revoke {privilege} on the database"): - node.query(f"REVOKE {privilege} ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {db_name}.* FROM {grant_target_name}" + ) with Then("I check the user does not see a database"): - output = node.query("SHOW DATABASES", settings = [("user", f"{user_name}")]).output - assert output == f'', error() + output = node.query( + "SHOW DATABASES", settings=[("user", f"{user_name}")] + ).output + assert output == f"", error() finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_UseDatabase_RequiredPrivilege("1.0"), @@ -149,8 +199,12 @@ def use(self, privilege, on, grant_target_name, user_name, db_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I attempt to USE {db_name}"): - node.query(f"USE {db_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"USE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("USE with privilege"): @@ -158,7 +212,7 @@ def use(self, privilege, on, grant_target_name, user_name, db_name, node=None): node.query(f"GRANT {privilege} ON {db_name}.* TO {grant_target_name}") with Then(f"I attempt to USE {db_name}"): - node.query(f"USE {db_name}", settings=[("user",user_name)]) + node.query(f"USE {db_name}", settings=[("user", user_name)]) with Scenario("USE with revoked privilege"): @@ -166,16 +220,23 @@ def use(self, privilege, on, grant_target_name, user_name, db_name, node=None): node.query(f"GRANT {privilege} ON {db_name}.* TO {grant_target_name}") with And(f"I revoke {privilege} on the database"): - node.query(f"REVOKE {privilege} ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {db_name}.* FROM {grant_target_name}" + ) with Then(f"I attempt to USE {db_name}"): - node.query(f"USE {db_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"USE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateDatabase_RequiredPrivilege("1.0"), @@ -202,8 +263,12 @@ def show_create(self, privilege, on, grant_target_name, user_name, db_name, node node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {db_name}"): - node.query(f"SHOW CREATE DATABASE {db_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE with privilege"): @@ -211,7 +276,9 @@ def show_create(self, privilege, on, grant_target_name, user_name, db_name, node node.query(f"GRANT {privilege} ON {db_name}.* TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {db_name}"): - node.query(f"SHOW CREATE DATABASE {db_name}", settings=[("user",user_name)]) + node.query( + f"SHOW CREATE DATABASE {db_name}", settings=[("user", user_name)] + ) with Scenario("SHOW CREATE with revoked privilege"): @@ -219,26 +286,32 @@ def show_create(self, privilege, on, grant_target_name, user_name, db_name, node node.query(f"GRANT {privilege} ON {db_name}.* TO {grant_target_name}") with And(f"I revoke {privilege} on the database"): - node.query(f"REVOKE {privilege} ON {db_name}.* FROM {grant_target_name}") + node.query( + f"REVOKE {privilege} ON {db_name}.* FROM {grant_target_name}" + ) with Then(f"I attempt to SHOW CREATE {db_name}"): - node.query(f"SHOW CREATE DATABASE {db_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE DATABASE {db_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the database"): node.query(f"DROP DATABASE IF EXISTS {db_name}") + @TestFeature @Name("show databases") @Requirements( RQ_SRS_006_RBAC_ShowDatabases_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW DATABASES. - """ + """Check the RBAC functionality of SHOW DATABASES.""" self.context.node = self.context.cluster.node(node) Suite(run=dict_privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_dictionaries.py b/tests/testflows/rbac/tests/privileges/show/show_dictionaries.py index 5b717b5f47c..9c571b8836e 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_dictionaries.py +++ b/tests/testflows/rbac/tests/privileges/show/show_dictionaries.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def dict_privileges_granted_directly(self, node=None): """Check that a user is able to execute `SHOW CREATE` and `EXISTS` @@ -20,10 +21,18 @@ def dict_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): dict_name = f"dict_name_{getuid()}" - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name dict_name", [ - tuple(list(row)+[user_name,user_name,dict_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name dict_name", + [ + tuple(list(row) + [user_name, user_name, dict_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dict_privileges_granted_via_role(self, node=None): @@ -44,31 +53,62 @@ def dict_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name dict_name", [ - tuple(list(row)+[role_name,user_name,dict_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name dict_name", + [ + tuple(list(row) + [role_name, user_name, dict_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SHOW","*.*"), - ("SHOW DICTIONARIES","dict"), - ("CREATE DICTIONARY","dict"), - ("DROP DICTIONARY","dict"), -]) -def check_privilege(self, privilege, on, grant_target_name, user_name, dict_name, node=None): - """Run checks for commands that require SHOW DICTIONARY privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SHOW", "*.*"), + ("SHOW DICTIONARIES", "dict"), + ("CREATE DICTIONARY", "dict"), + ("DROP DICTIONARY", "dict"), + ], +) +def check_privilege( + self, privilege, on, grant_target_name, user_name, dict_name, node=None +): + """Run checks for commands that require SHOW DICTIONARY privilege.""" if node is None: node = self.context.node on = on.replace("dict", f"{dict_name}") - Suite(test=show_dict)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, dict_name=dict_name) - Suite(test=exists)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, dict_name=dict_name) - Suite(test=show_create)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, dict_name=dict_name) + Suite(test=show_dict)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + dict_name=dict_name, + ) + Suite(test=exists)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + dict_name=dict_name, + ) + Suite(test=show_create)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + dict_name=dict_name, + ) + @TestSuite @Requirements( @@ -85,7 +125,9 @@ def show_dict(self, privilege, on, grant_target_name, user_name, dict_name, node try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with Scenario("SHOW DICTIONARIES without privilege"): @@ -96,15 +138,21 @@ def show_dict(self, privilege, on, grant_target_name, user_name, dict_name, node node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user doesn't see the dictionary"): - output = node.query("SHOW DICTIONARIES", settings = [("user", f"{user_name}")]).output - assert output == '', error() + output = node.query( + "SHOW DICTIONARIES", settings=[("user", f"{user_name}")] + ).output + assert output == "", error() with Scenario("SHOW DICTIONARIES with privilege"): with When(f"I grant {privilege} on the dictionary"): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user does see a dictionary"): - node.query("SHOW DICTIONARIES", settings = [("user", f"{user_name}")], message=f"{dict_name}") + node.query( + "SHOW DICTIONARIES", + settings=[("user", f"{user_name}")], + message=f"{dict_name}", + ) with Scenario("SHOW DICTIONARIES with revoked privilege"): with When(f"I grant {privilege} on the dictionary"): @@ -114,13 +162,16 @@ def show_dict(self, privilege, on, grant_target_name, user_name, dict_name, node node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user does not see a dictionary"): - output = node.query("SHOW DICTIONARIES", settings = [("user", f"{user_name}")]).output - assert output == f'', error() + output = node.query( + "SHOW DICTIONARIES", settings=[("user", f"{user_name}")] + ).output + assert output == f"", error() finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_ExistsDictionary_RequiredPrivilege("1.0"), @@ -136,7 +187,9 @@ def exists(self, privilege, on, grant_target_name, user_name, dict_name, node=No try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with Scenario("EXISTS without privilege"): @@ -147,15 +200,19 @@ def exists(self, privilege, on, grant_target_name, user_name, dict_name, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I check if {dict_name} EXISTS"): - node.query(f"EXISTS {dict_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"EXISTS {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("EXISTS with privilege"): with When(f"I grant {privilege} on the dictionary"): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then(f"I check if {dict_name} EXISTS"): - node.query(f"EXISTS {dict_name}", settings=[("user",user_name)]) + node.query(f"EXISTS {dict_name}", settings=[("user", user_name)]) with Scenario("EXISTS with revoked privilege"): with When(f"I grant {privilege} on the dictionary"): @@ -165,18 +222,25 @@ def exists(self, privilege, on, grant_target_name, user_name, dict_name, node=No node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then(f"I check if {dict_name} EXISTS"): - node.query(f"EXISTS {dict_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"EXISTS {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateDictionary_RequiredPrivilege("1.0"), ) -def show_create(self, privilege, on, grant_target_name, user_name, dict_name, node=None): +def show_create( + self, privilege, on, grant_target_name, user_name, dict_name, node=None +): """Check that user is able to execute SHOW CREATE on a dictionary if and only if the user has SHOW DICTIONARY privilege on that dictionary. """ @@ -187,7 +251,9 @@ def show_create(self, privilege, on, grant_target_name, user_name, dict_name, no try: with Given("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)") + node.query( + f"CREATE DICTIONARY {dict_name}(x Int32, y Int32) PRIMARY KEY x LAYOUT(FLAT()) SOURCE(CLICKHOUSE()) LIFETIME(0)" + ) with Scenario("SHOW CREATE without privilege"): @@ -198,15 +264,22 @@ def show_create(self, privilege, on, grant_target_name, user_name, dict_name, no node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {dict_name}"): - node.query(f"SHOW CREATE DICTIONARY {dict_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE with privilege"): with When(f"I grant {privilege} on the dictionary"): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then(f"I attempt to SHOW CREATE {dict_name}"): - node.query(f"SHOW CREATE DICTIONARY {dict_name}", settings=[("user",user_name)]) + node.query( + f"SHOW CREATE DICTIONARY {dict_name}", + settings=[("user", user_name)], + ) with Scenario("SHOW CREATE with revoked privilege"): with When(f"I grant {privilege} on the dictionary"): @@ -216,23 +289,27 @@ def show_create(self, privilege, on, grant_target_name, user_name, dict_name, no node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then(f"I attempt to SHOW CREATE {dict_name}"): - node.query(f"SHOW CREATE DICTIONARY {dict_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE DICTIONARY {dict_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the dictionary"): node.query(f"DROP DICTIONARY IF EXISTS {dict_name}") + @TestFeature @Name("show dictionaries") @Requirements( RQ_SRS_006_RBAC_ShowDictionaries_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW DICTIONARIES. - """ + """Check the RBAC functionality of SHOW DICTIONARIES.""" self.context.node = self.context.cluster.node(node) Suite(run=dict_privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_quotas.py b/tests/testflows/rbac/tests/privileges/show/show_quotas.py index 20476ae759b..d7556db6d07 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_quotas.py +++ b/tests/testflows/rbac/tests/privileges/show/show_quotas.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def quota(node, name): try: @@ -17,10 +18,10 @@ def quota(node, name): with Finally("I drop the quota"): node.query(f"DROP QUOTA IF EXISTS {name}") + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `SHOW QUOTAS` with privileges are granted directly. - """ + """Check that a user is able to execute `SHOW QUOTAS` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -29,15 +30,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `SHOW QUOTAS` with privileges are granted through a role. - """ + """Check that a user is able to execute `SHOW QUOTAS` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -50,36 +58,50 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("SHOW ACCESS",), - ("SHOW QUOTAS",), - ("SHOW CREATE QUOTA",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("SHOW ACCESS",), + ("SHOW QUOTAS",), + ("SHOW CREATE QUOTA",), + ], +) def check_privilege(self, privilege, grant_target_name, user_name, node=None): - """Run checks for commands that require SHOW QUOTAS privilege. - """ + """Run checks for commands that require SHOW QUOTAS privilege.""" if node is None: node = self.context.node - Suite(test=show_quotas)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=show_create)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=show_quotas)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + Suite(test=show_create)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowQuotas_RequiredPrivilege("1.0"), ) def show_quotas(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW QUOTAS` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW QUOTAS` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -94,8 +116,12 @@ def show_quotas(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW QUOTAS"): - node.query(f"SHOW QUOTAS", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW QUOTAS", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW QUOTAS with privilege"): @@ -103,7 +129,7 @@ def show_quotas(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW QUOTAS"): - node.query(f"SHOW QUOTAS", settings = [("user", f"{user_name}")]) + node.query(f"SHOW QUOTAS", settings=[("user", f"{user_name}")]) with Scenario("SHOW QUOTAS with revoked privilege"): @@ -114,16 +140,20 @@ def show_quotas(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW QUOTAS"): - node.query(f"SHOW QUOTAS", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW QUOTAS", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateQuota_RequiredPrivilege("1.0"), ) def show_create(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW CREATE QUOTA` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW CREATE QUOTA` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -141,8 +171,12 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW CREATE QUOTA"): - node.query(f"SHOW CREATE QUOTA {target_quota_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE QUOTA {target_quota_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE QUOTA with privilege"): target_quota_name = f"target_quota_{getuid()}" @@ -153,7 +187,10 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW CREATE QUOTA"): - node.query(f"SHOW CREATE QUOTA {target_quota_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SHOW CREATE QUOTA {target_quota_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SHOW CREATE QUOTA with revoked privilege"): target_quota_name = f"target_quota_{getuid()}" @@ -167,19 +204,23 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW CREATE QUOTA"): - node.query(f"SHOW CREATE QUOTA {target_quota_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE QUOTA {target_quota_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("show quotas") @Requirements( RQ_SRS_006_RBAC_ShowQuotas_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW QUOTAS. - """ + """Check the RBAC functionality of SHOW QUOTAS.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_roles.py b/tests/testflows/rbac/tests/privileges/show/show_roles.py index 14d038102dd..6fe3906eeb6 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_roles.py +++ b/tests/testflows/rbac/tests/privileges/show/show_roles.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `SHOW ROLES` with privileges are granted directly. - """ + """Check that a user is able to execute `SHOW ROLES` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `SHOW ROLES` with privileges are granted through a role. - """ + """Check that a user is able to execute `SHOW ROLES` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,36 +45,50 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("SHOW ACCESS",), - ("SHOW ROLES",), - ("SHOW CREATE ROLE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("SHOW ACCESS",), + ("SHOW ROLES",), + ("SHOW CREATE ROLE",), + ], +) def check_privilege(self, privilege, grant_target_name, user_name, node=None): - """Run checks for commands that require SHOW ROLES privilege. - """ + """Run checks for commands that require SHOW ROLES privilege.""" if node is None: node = self.context.node - Suite(test=show_roles)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=show_create)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=show_roles)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + Suite(test=show_create)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowRoles_RequiredPrivilege("1.0"), ) def show_roles(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW ROLES` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW ROLES` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -82,8 +103,12 @@ def show_roles(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW ROLES"): - node.query(f"SHOW ROLES", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW ROLES", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW ROLES with privilege"): @@ -91,7 +116,7 @@ def show_roles(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW ROLES"): - node.query(f"SHOW ROLES", settings = [("user", f"{user_name}")]) + node.query(f"SHOW ROLES", settings=[("user", f"{user_name}")]) with Scenario("SHOW ROLES with revoked privilege"): @@ -102,16 +127,20 @@ def show_roles(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW ROLES"): - node.query(f"SHOW ROLES", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW ROLES", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateRole_RequiredPrivilege("1.0"), ) def show_create(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW CREATE ROLE` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW CREATE ROLE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -129,8 +158,12 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW CREATE ROLE"): - node.query(f"SHOW CREATE ROLE {target_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE ROLE {target_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE ROLE with privilege"): target_role_name = f"target_role_{getuid()}" @@ -141,7 +174,10 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW CREATE ROLE"): - node.query(f"SHOW CREATE ROLE {target_role_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SHOW CREATE ROLE {target_role_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SHOW CREATE ROLE with revoked privilege"): target_role_name = f"target_role_{getuid()}" @@ -155,19 +191,23 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW CREATE ROLE"): - node.query(f"SHOW CREATE ROLE {target_role_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE ROLE {target_role_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("show roles") @Requirements( RQ_SRS_006_RBAC_ShowRoles_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW ROLES. - """ + """Check the RBAC functionality of SHOW ROLES.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_row_policies.py b/tests/testflows/rbac/tests/privileges/show/show_row_policies.py index 789c4c95223..5591c5bd7a1 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_row_policies.py +++ b/tests/testflows/rbac/tests/privileges/show/show_row_policies.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def row_policy(node, name, table): try: @@ -17,10 +18,10 @@ def row_policy(node, name, table): with Finally("I drop the row policy"): node.query(f"DROP ROW POLICY IF EXISTS {name} ON {table}") + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `SHOW ROW POLICIES` with privileges are granted directly. - """ + """Check that a user is able to execute `SHOW ROW POLICIES` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -29,15 +30,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `SHOW ROW POLICIES` with privileges are granted through a role. - """ + """Check that a user is able to execute `SHOW ROW POLICIES` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -50,38 +58,52 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("SHOW ACCESS",), - ("SHOW ROW POLICIES",), - ("SHOW POLICIES",), - ("SHOW CREATE ROW POLICY",), - ("SHOW CREATE POLICY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("SHOW ACCESS",), + ("SHOW ROW POLICIES",), + ("SHOW POLICIES",), + ("SHOW CREATE ROW POLICY",), + ("SHOW CREATE POLICY",), + ], +) def check_privilege(self, privilege, grant_target_name, user_name, node=None): - """Run checks for commands that require SHOW ROW POLICIES privilege. - """ + """Run checks for commands that require SHOW ROW POLICIES privilege.""" if node is None: node = self.context.node - Suite(test=show_row_policies)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=show_create)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=show_row_policies)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + Suite(test=show_create)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowRowPolicies_RequiredPrivilege("1.0"), ) def show_row_policies(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW ROW POLICIES` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW ROW POLICIES` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -96,8 +118,12 @@ def show_row_policies(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW ROW POLICIES"): - node.query(f"SHOW ROW POLICIES", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW ROW POLICIES", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW ROW POLICIES with privilege"): @@ -105,7 +131,7 @@ def show_row_policies(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW ROW POLICIES"): - node.query(f"SHOW ROW POLICIES", settings = [("user", f"{user_name}")]) + node.query(f"SHOW ROW POLICIES", settings=[("user", f"{user_name}")]) with Scenario("SHOW ROW POLICIES with revoked privilege"): @@ -116,16 +142,20 @@ def show_row_policies(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW ROW POLICIES"): - node.query(f"SHOW ROW POLICIES", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW ROW POLICIES", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateRowPolicy_RequiredPrivilege("1.0"), ) def show_create(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW CREATE ROW POLICY` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW CREATE ROW POLICY` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -144,8 +174,12 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW CREATE ROW POLICY"): - node.query(f"SHOW CREATE ROW POLICY {target_row_policy_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE ROW POLICY {target_row_policy_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE ROW POLICY with privilege"): target_row_policy_name = f"target_row_policy_{getuid()}" @@ -157,7 +191,10 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW CREATE ROW POLICY"): - node.query(f"SHOW CREATE ROW POLICY {target_row_policy_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SHOW CREATE ROW POLICY {target_row_policy_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SHOW CREATE ROW POLICY with revoked privilege"): target_row_policy_name = f"target_row_policy_{getuid()}" @@ -172,19 +209,23 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW CREATE ROW POLICY"): - node.query(f"SHOW CREATE ROW POLICY {target_row_policy_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE ROW POLICY {target_row_policy_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("show row policies") @Requirements( RQ_SRS_006_RBAC_ShowRowPolicies_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW ROW POLICYS. - """ + """Check the RBAC functionality of SHOW ROW POLICYS.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_settings_profiles.py b/tests/testflows/rbac/tests/privileges/show/show_settings_profiles.py index 18ca0ee7f6e..1342b420afe 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_settings_profiles.py +++ b/tests/testflows/rbac/tests/privileges/show/show_settings_profiles.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def settings_profile(node, name): try: @@ -17,10 +18,10 @@ def settings_profile(node, name): with Finally("I drop the settings_profile"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {name}") + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `SHOW SETTINGS PROFILES` with privileges are granted directly. - """ + """Check that a user is able to execute `SHOW SETTINGS PROFILES` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -29,15 +30,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `SHOW SETTINGS PROFILES` with privileges are granted through a role. - """ + """Check that a user is able to execute `SHOW SETTINGS PROFILES` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -50,38 +58,52 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("SHOW ACCESS",), - ("SHOW SETTINGS PROFILES",), - ("SHOW PROFILES",), - ("SHOW CREATE SETTINGS PROFILE",), - ("SHOW CREATE PROFILE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("SHOW ACCESS",), + ("SHOW SETTINGS PROFILES",), + ("SHOW PROFILES",), + ("SHOW CREATE SETTINGS PROFILE",), + ("SHOW CREATE PROFILE",), + ], +) def check_privilege(self, privilege, grant_target_name, user_name, node=None): - """Run checks for commands that require SHOW SETTINGS PROFILES privilege. - """ + """Run checks for commands that require SHOW SETTINGS PROFILES privilege.""" if node is None: node = self.context.node - Suite(test=show_settings_profiles)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=show_create)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=show_settings_profiles)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + Suite(test=show_create)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowSettingsProfiles_RequiredPrivilege("1.0"), ) def show_settings_profiles(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW SETTINGS PROFILES` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW SETTINGS PROFILES` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -96,8 +118,12 @@ def show_settings_profiles(self, privilege, grant_target_name, user_name, node=N node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW SETTINGS PROFILES"): - node.query(f"SHOW SETTINGS PROFILES", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW SETTINGS PROFILES", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW SETTINGS PROFILES with privilege"): @@ -105,7 +131,7 @@ def show_settings_profiles(self, privilege, grant_target_name, user_name, node=N node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW SETTINGS PROFILES"): - node.query(f"SHOW SETTINGS PROFILES", settings = [("user", f"{user_name}")]) + node.query(f"SHOW SETTINGS PROFILES", settings=[("user", f"{user_name}")]) with Scenario("SHOW SETTINGS PROFILES with revoked privilege"): @@ -116,16 +142,20 @@ def show_settings_profiles(self, privilege, grant_target_name, user_name, node=N node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW SETTINGS PROFILES"): - node.query(f"SHOW SETTINGS PROFILES", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW SETTINGS PROFILES", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateSettingsProfile_RequiredPrivilege("1.0"), ) def show_create(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW CREATE SETTINGS PROFILE` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW CREATE SETTINGS PROFILE` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -143,8 +173,12 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW CREATE SETTINGS PROFILE"): - node.query(f"SHOW CREATE SETTINGS PROFILE {target_settings_profile_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE SETTINGS PROFILE {target_settings_profile_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE SETTINGS PROFILE with privilege"): target_settings_profile_name = f"target_settings_profile_{getuid()}" @@ -155,7 +189,10 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW CREATE SETTINGS PROFILE"): - node.query(f"SHOW CREATE SETTINGS PROFILE {target_settings_profile_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SHOW CREATE SETTINGS PROFILE {target_settings_profile_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SHOW CREATE SETTINGS PROFILE with revoked privilege"): target_settings_profile_name = f"target_settings_profile_{getuid()}" @@ -169,19 +206,23 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW CREATE SETTINGS PROFILE"): - node.query(f"SHOW CREATE SETTINGS PROFILE {target_settings_profile_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE SETTINGS PROFILE {target_settings_profile_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("show settings profiles") @Requirements( RQ_SRS_006_RBAC_ShowSettingsProfiles_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW SETTINGS PROFILES. - """ + """Check the RBAC functionality of SHOW SETTINGS PROFILES.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_tables.py b/tests/testflows/rbac/tests/privileges/show/show_tables.py index d445550c032..f6eacef4164 100755 --- a/tests/testflows/rbac/tests/privileges/show/show_tables.py +++ b/tests/testflows/rbac/tests/privileges/show/show_tables.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def table_privileges_granted_directly(self, node=None): """Check that a user is able to execute `CHECK` and `EXISTS` @@ -20,10 +21,18 @@ def table_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[user_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [user_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def table_privileges_granted_via_role(self, node=None): @@ -44,41 +53,73 @@ def table_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[role_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [role_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SHOW", "*.*"), - ("SHOW TABLES", "table"), - ("SELECT", "table"), - ("INSERT", "table"), - ("ALTER", "table"), - ("SELECT(a)", "table"), - ("INSERT(a)", "table"), - ("ALTER(a)", "table"), -]) -def check_privilege(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Run checks for commands that require SHOW TABLE privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SHOW", "*.*"), + ("SHOW TABLES", "table"), + ("SELECT", "table"), + ("INSERT", "table"), + ("ALTER", "table"), + ("SELECT(a)", "table"), + ("INSERT(a)", "table"), + ("ALTER(a)", "table"), + ], +) +def check_privilege( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Run checks for commands that require SHOW TABLE privilege.""" if node is None: node = self.context.node - Suite(test=show_tables)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) - Suite(test=exists)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) - Suite(test=check)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) + Suite(test=show_tables)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + Suite(test=exists)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + Suite(test=check)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowTables_RequiredPrivilege("1.0"), ) -def show_tables(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to see a table in SHOW TABLES when they have a privilege on that table. - """ +def show_tables( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to see a table in SHOW TABLES when they have a privilege on that table.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -97,8 +138,10 @@ def show_tables(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user doesn't see the table"): - output = node.query("SHOW TABLES", settings = [("user", f"{user_name}")]).output - assert output == '', error() + output = node.query( + "SHOW TABLES", settings=[("user", f"{user_name}")] + ).output + assert output == "", error() with Scenario("SHOW TABLES with privilege"): @@ -106,7 +149,11 @@ def show_tables(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user does see a table"): - node.query("SHOW TABLES", settings = [("user", f"{user_name}")], message=f"{table_name}") + node.query( + "SHOW TABLES", + settings=[("user", f"{user_name}")], + message=f"{table_name}", + ) with Scenario("SHOW TABLES with revoked privilege"): @@ -117,8 +164,11 @@ def show_tables(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user does not see a table"): - output = node.query("SHOW TABLES", settings = [("user", f"{user_name}")]).output - assert output == '', error() + output = node.query( + "SHOW TABLES", settings=[("user", f"{user_name}")] + ).output + assert output == "", error() + @TestSuite @Requirements( @@ -147,8 +197,12 @@ def exists(self, privilege, on, grant_target_name, user_name, table_name, node=N node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I check if {table_name} EXISTS"): - node.query(f"EXISTS {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"EXISTS {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("EXISTS with privilege"): @@ -156,7 +210,7 @@ def exists(self, privilege, on, grant_target_name, user_name, table_name, node=N node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then(f"I check if {table_name} EXISTS"): - node.query(f"EXISTS {table_name}", settings=[("user",user_name)]) + node.query(f"EXISTS {table_name}", settings=[("user", user_name)]) with Scenario("EXISTS with revoked privilege"): @@ -167,8 +221,13 @@ def exists(self, privilege, on, grant_target_name, user_name, table_name, node=N node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then(f"I check if {table_name} EXISTS"): - node.query(f"EXISTS {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"EXISTS {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite @Requirements( @@ -197,8 +256,12 @@ def check(self, privilege, on, grant_target_name, user_name, table_name, node=No node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then(f"I CHECK {table_name}"): - node.query(f"CHECK TABLE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CHECK TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("CHECK with privilege"): @@ -206,7 +269,7 @@ def check(self, privilege, on, grant_target_name, user_name, table_name, node=No node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then(f"I CHECK {table_name}"): - node.query(f"CHECK TABLE {table_name}", settings=[("user",user_name)]) + node.query(f"CHECK TABLE {table_name}", settings=[("user", user_name)]) with Scenario("CHECK with revoked privilege"): @@ -217,19 +280,23 @@ def check(self, privilege, on, grant_target_name, user_name, table_name, node=No node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then(f"I CHECK {table_name}"): - node.query(f"CHECK TABLE {table_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CHECK TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("show tables") @Requirements( RQ_SRS_006_RBAC_ShowTables_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW TABLES. - """ + """Check the RBAC functionality of SHOW TABLES.""" self.context.node = self.context.cluster.node(node) Suite(run=table_privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/show/show_users.py b/tests/testflows/rbac/tests/privileges/show/show_users.py index aa5c97297b5..f3406c4134b 100644 --- a/tests/testflows/rbac/tests/privileges/show/show_users.py +++ b/tests/testflows/rbac/tests/privileges/show/show_users.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Check that a user is able to execute `SHOW USERS` with privileges are granted directly. - """ + """Check that a user is able to execute `SHOW USERS` with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Check that a user is able to execute `SHOW USERS` with privileges are granted through a role. - """ + """Check that a user is able to execute `SHOW USERS` with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,36 +45,50 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("ACCESS MANAGEMENT",), - ("SHOW ACCESS",), - ("SHOW USERS",), - ("SHOW CREATE USER",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("ACCESS MANAGEMENT",), + ("SHOW ACCESS",), + ("SHOW USERS",), + ("SHOW CREATE USER",), + ], +) def check_privilege(self, privilege, grant_target_name, user_name, node=None): - """Run checks for commands that require SHOW USERS privilege. - """ + """Run checks for commands that require SHOW USERS privilege.""" if node is None: node = self.context.node - Suite(test=show_users)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=show_create)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=show_users)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + Suite(test=show_create)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowUsers_RequiredPrivilege("1.0"), ) def show_users(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW USERS` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW USERS` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -82,8 +103,12 @@ def show_users(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW USERS"): - node.query(f"SHOW USERS", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW USERS", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW USERS with privilege"): @@ -91,7 +116,7 @@ def show_users(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW USERS"): - node.query(f"SHOW USERS", settings = [("user", f"{user_name}")]) + node.query(f"SHOW USERS", settings=[("user", f"{user_name}")]) with Scenario("SHOW USERS with revoked privilege"): @@ -102,16 +127,20 @@ def show_users(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW USERS"): - node.query(f"SHOW USERS", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW USERS", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite @Requirements( RQ_SRS_006_RBAC_ShowCreateUser_RequiredPrivilege("1.0"), ) def show_create(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SHOW CREATE USER` when they have the necessary privilege. - """ + """Check that user is only able to execute `SHOW CREATE USER` when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -129,8 +158,12 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SHOW CREATE USER"): - node.query(f"SHOW CREATE USER {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE USER {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SHOW CREATE USER with privilege"): target_user_name = f"target_user_{getuid()}" @@ -141,7 +174,10 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SHOW CREATE USER"): - node.query(f"SHOW CREATE USER {target_user_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SHOW CREATE USER {target_user_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SHOW CREATE USER with revoked privilege"): target_user_name = f"target_user_{getuid()}" @@ -155,19 +191,23 @@ def show_create(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SHOW CREATE USER"): - node.query(f"SHOW CREATE USER {target_user_name}", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SHOW CREATE USER {target_user_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("show users") @Requirements( RQ_SRS_006_RBAC_ShowUsers_Privilege("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SHOW USERS. - """ + """Check the RBAC functionality of SHOW USERS.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/sources.py b/tests/testflows/rbac/tests/privileges/sources.py index e473c623955..96e02198845 100644 --- a/tests/testflows/rbac/tests/privileges/sources.py +++ b/tests/testflows/rbac/tests/privileges/sources.py @@ -5,10 +5,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def file_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `File` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `File` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -17,15 +17,19 @@ def file_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=file, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in file.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=file, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in file.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def file_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `File` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `File` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -38,30 +42,37 @@ def file_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=file, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in file.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=file, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in file.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("FILE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("FILE",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_File("1.0"), ) def file(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `File` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `File` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node with Scenario("File source without privilege"): - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -73,8 +84,12 @@ def file(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the File source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=File('')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=File('')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("File source with privilege"): @@ -82,8 +97,12 @@ def file(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the File source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=File('')", settings = [("user", f"{user_name}")], - exitcode=0, message=None) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=File('')", + settings=[("user", f"{user_name}")], + exitcode=0, + message=None, + ) with Scenario("File source with revoked privilege"): @@ -94,13 +113,17 @@ def file(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the File source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=File('')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=File('')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def url_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `URL` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `URL` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -109,15 +132,19 @@ def url_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=url, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in url.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=url, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in url.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def url_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `URL` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `URL` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -130,30 +157,37 @@ def url_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=url, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in url.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=url, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in url.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("URL",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("URL",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_URL("1.0"), ) def url(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `URL` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `URL` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node - table_name = f'table_{getuid()}' - + table_name = f"table_{getuid()}" + with Scenario("URL source without privilege"): with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -165,16 +199,22 @@ def url(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the URL source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=URL('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=URL('127.0.0.1', 'TSV')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("URL source with privilege"): with When(f"I grant {privilege}"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the URL source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=URL('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=URL('127.0.0.1', 'TSV')", + settings=[("user", f"{user_name}")], + ) with Scenario("URL source with revoked privilege"): with When(f"I grant {privilege}"): @@ -184,13 +224,17 @@ def url(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the URL source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=URL('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=URL('127.0.0.1', 'TSV')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def remote_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a Remote source with privileges are granted directly. - """ + """Check that a user is able to create a table from a Remote source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -199,15 +243,19 @@ def remote_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=remote, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in remote.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=remote, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in remote.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def remote_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a Remote source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a Remote source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -220,30 +268,37 @@ def remote_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=remote, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in remote.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=remote, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in remote.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("REMOTE",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("REMOTE",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_Remote("1.0"), ) def remote(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a remote source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a remote source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node with Scenario("Remote source without privilege"): - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -255,8 +310,12 @@ def remote(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the Remote source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE = Distributed('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE = Distributed('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("Remote source with privilege"): @@ -264,8 +323,12 @@ def remote(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the Remote source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE = Distributed('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE = Distributed('127.0.0.1')", + settings=[("user", f"{user_name}")], + exitcode=42, + message="Exception: Storage", + ) with Scenario("Remote source with revoked privilege"): @@ -276,13 +339,17 @@ def remote(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the Remote source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE = Distributed('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE = Distributed('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def MySQL_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `MySQL` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `MySQL` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -291,15 +358,19 @@ def MySQL_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=MySQL, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in MySQL.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=MySQL, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in MySQL.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def MySQL_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `MySQL` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `MySQL` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -312,30 +383,37 @@ def MySQL_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=MySQL, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in MySQL.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=MySQL, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in MySQL.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("MYSQL",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("MYSQL",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_MySQL("1.0"), ) def MySQL(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `MySQL` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `MySQL` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node with Scenario("MySQL source without privilege"): - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -347,8 +425,12 @@ def MySQL(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the MySQL source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=MySQL('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=MySQL('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("MySQL source with privilege"): @@ -356,8 +438,12 @@ def MySQL(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the MySQL source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=MySQL('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=MySQL('127.0.0.1')", + settings=[("user", f"{user_name}")], + exitcode=42, + message="Exception: Storage", + ) with Scenario("MySQL source with revoked privilege"): @@ -368,13 +454,17 @@ def MySQL(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the MySQL source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=MySQL('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=MySQL('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def ODBC_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `ODBC` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `ODBC` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -383,15 +473,19 @@ def ODBC_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=ODBC, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in ODBC.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=ODBC, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in ODBC.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def ODBC_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `ODBC` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `ODBC` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -404,30 +498,37 @@ def ODBC_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=ODBC, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in ODBC.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=ODBC, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in ODBC.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("ODBC",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("ODBC",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_ODBC("1.0"), ) def ODBC(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `ODBC` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `ODBC` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node with Scenario("ODBC source without privilege"): - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -439,8 +540,12 @@ def ODBC(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the ODBC source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=ODBC('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=ODBC('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("ODBC source with privilege"): @@ -448,8 +553,12 @@ def ODBC(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the ODBC source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=ODBC('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=ODBC('127.0.0.1')", + settings=[("user", f"{user_name}")], + exitcode=42, + message="Exception: Storage", + ) with Scenario("ODBC source with revoked privilege"): @@ -460,13 +569,17 @@ def ODBC(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the ODBC source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=ODBC('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=ODBC('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def JDBC_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `JDBC` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `JDBC` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -475,15 +588,19 @@ def JDBC_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=JDBC, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in JDBC.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=JDBC, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in JDBC.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def JDBC_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `JDBC` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `JDBC` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -496,30 +613,37 @@ def JDBC_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=JDBC, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in JDBC.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=JDBC, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in JDBC.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("JDBC",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("JDBC",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_JDBC("1.0"), ) def JDBC(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `JDBC` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `JDBC` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node with Scenario("JDBC source without privilege"): - table_name = f'table_{getuid()}' + table_name = f"table_{getuid()}" with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -531,8 +655,12 @@ def JDBC(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the JDBC source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=JDBC('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=JDBC('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("JDBC source with privilege"): @@ -540,8 +668,12 @@ def JDBC(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the JDBC source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=JDBC('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=JDBC('127.0.0.1')", + settings=[("user", f"{user_name}")], + exitcode=42, + message="Exception: Storage", + ) with Scenario("JDBC source with revoked privilege"): @@ -552,13 +684,17 @@ def JDBC(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the JDBC source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=JDBC('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=JDBC('127.0.0.1')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def HDFS_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `HDFS` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `HDFS` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -567,15 +703,19 @@ def HDFS_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=HDFS, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in HDFS.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=HDFS, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in HDFS.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def HDFS_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `HDFS` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `HDFS` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -588,30 +728,38 @@ def HDFS_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=HDFS, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in HDFS.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=HDFS, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in HDFS.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("HDFS",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("HDFS",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_HDFS("1.0"), ) def HDFS(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `HDFS` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `HDFS` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node + table_name = f"table_{getuid()}" + with Scenario("HDFS source without privilege"): - table_name = f'table_{getuid()}' with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -623,8 +771,12 @@ def HDFS(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the HDFS source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=HDFS('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=HDFS('hdfs://127.0.0.1:8020/path', 'TSV')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("HDFS source with privilege"): @@ -632,8 +784,10 @@ def HDFS(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the HDFS source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=HDFS('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=HDFS('hdfs://127.0.0.1:8020/path', 'TSV')", + settings=[("user", f"{user_name}")], + ) with Scenario("HDFS source with revoked privilege"): @@ -644,13 +798,17 @@ def HDFS(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the HDFS source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=HDFS('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=HDFS('hdfs://127.0.0.1:8020/path', 'TSV')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def S3_privileges_granted_directly(self, node=None): - """Check that a user is able to create a table from a `S3` source with privileges are granted directly. - """ + """Check that a user is able to create a table from a `S3` source with privileges are granted directly.""" user_name = f"user_{getuid()}" @@ -659,15 +817,19 @@ def S3_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=S3, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in S3.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=S3, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in S3.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def S3_privileges_granted_via_role(self, node=None): - """Check that a user is able to create a table from a `S3` source with privileges are granted through a role. - """ + """Check that a user is able to create a table from a `S3` source with privileges are granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -680,30 +842,38 @@ def S3_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=S3, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in S3.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=S3, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in S3.examples], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SOURCES",), - ("S3",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SOURCES",), + ("S3",), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_Sources_S3("1.0"), ) def S3(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to to create a table from a `S3` source when they have the necessary privilege. - """ + """Check that user is only able to to create a table from a `S3` source when they have the necessary privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: node = self.context.node + table_name = f"table_{getuid()}" + with Scenario("S3 source without privilege"): - table_name = f'table_{getuid()}' with Given("The user has table privilege"): node.query(f"GRANT CREATE TABLE ON {table_name} TO {grant_target_name}") @@ -715,8 +885,12 @@ def S3(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use the S3 source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=S3('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=S3('https://my.amazonaws.com/mybucket/mydata', 'TSV')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("S3 source with privilege"): @@ -724,8 +898,10 @@ def S3(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use the S3 source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=S3('127.0.0.1')", settings = [("user", f"{user_name}")], - exitcode=42, message='Exception: Storage') + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=S3('https://my.amazonaws.com/mybucket/mydata', 'TSV')", + settings=[("user", f"{user_name}")], + ) with Scenario("S3 source with revoked privilege"): @@ -736,27 +912,35 @@ def S3(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use the S3 source"): - node.query(f"CREATE TABLE {table_name} (x String) ENGINE=S3('127.0.0.1')", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"CREATE TABLE {table_name} (x String) ENGINE=S3('https://my.amazonaws.com/mybucket/mydata', 'TSV')", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("sources") @Requirements( RQ_SRS_006_RBAC_Privileges_Sources("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SOURCES. - """ + """Check the RBAC functionality of SOURCES.""" self.context.node = self.context.cluster.node(node) Suite(run=file_privileges_granted_directly, setup=instrument_clickhouse_server_log) Suite(run=file_privileges_granted_via_role, setup=instrument_clickhouse_server_log) Suite(run=url_privileges_granted_directly, setup=instrument_clickhouse_server_log) Suite(run=url_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=remote_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=remote_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=remote_privileges_granted_directly, setup=instrument_clickhouse_server_log + ) + Suite( + run=remote_privileges_granted_via_role, setup=instrument_clickhouse_server_log + ) Suite(run=MySQL_privileges_granted_directly, setup=instrument_clickhouse_server_log) Suite(run=MySQL_privileges_granted_via_role, setup=instrument_clickhouse_server_log) Suite(run=ODBC_privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/system/drop_cache.py b/tests/testflows/rbac/tests/privileges/system/drop_cache.py index 8f1a6caeaac..cda6838b974 100644 --- a/tests/testflows/rbac/tests/privileges/system/drop_cache.py +++ b/tests/testflows/rbac/tests/privileges/system/drop_cache.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def dns_cache_privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM DROP DNS CACHE` if and only if @@ -17,10 +18,18 @@ def dns_cache_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=dns_cache, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dns_cache.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dns_cache, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dns_cache.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dns_cache_privileges_granted_via_role(self, node=None): @@ -38,28 +47,38 @@ def dns_cache_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dns_cache, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dns_cache.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dns_cache, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dns_cache.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_DropCache_DNS("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM DROP CACHE",), - ("SYSTEM DROP DNS CACHE",), - ("DROP CACHE",), - ("DROP DNS CACHE",), - ("SYSTEM DROP DNS",), - ("DROP DNS",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM DROP CACHE",), + ("SYSTEM DROP DNS CACHE",), + ("DROP CACHE",), + ("DROP DNS CACHE",), + ("SYSTEM DROP DNS",), + ("DROP DNS",), + ], +) def dns_cache(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM DROP DNS CACHE` privilege. - """ + """Run checks for `SYSTEM DROP DNS CACHE` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -74,8 +93,12 @@ def dns_cache(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user is unable to execute SYSTEM DROP DNS CACHE"): - node.query("SYSTEM DROP DNS CACHE", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM DROP DNS CACHE", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM DROP DNS CACHE with privilege"): @@ -83,7 +106,7 @@ def dns_cache(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user is bale to execute SYSTEM DROP DNS CACHE"): - node.query("SYSTEM DROP DNS CACHE", settings = [("user", f"{user_name}")]) + node.query("SYSTEM DROP DNS CACHE", settings=[("user", f"{user_name}")]) with Scenario("SYSTEM DROP DNS CACHE with revoked privilege"): @@ -94,8 +117,13 @@ def dns_cache(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user is unable to execute SYSTEM DROP DNS CACHE"): - node.query("SYSTEM DROP DNS CACHE", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM DROP DNS CACHE", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def mark_cache_privileges_granted_directly(self, node=None): @@ -109,10 +137,18 @@ def mark_cache_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=mark_cache, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in mark_cache.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=mark_cache, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in mark_cache.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def mark_cache_privileges_granted_via_role(self, node=None): @@ -130,28 +166,38 @@ def mark_cache_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=mark_cache, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in mark_cache.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=mark_cache, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in mark_cache.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_DropCache_Mark("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM DROP CACHE",), - ("SYSTEM DROP MARK CACHE",), - ("DROP CACHE",), - ("DROP MARK CACHE",), - ("SYSTEM DROP MARK",), - ("DROP MARKS",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM DROP CACHE",), + ("SYSTEM DROP MARK CACHE",), + ("DROP CACHE",), + ("DROP MARK CACHE",), + ("SYSTEM DROP MARK",), + ("DROP MARKS",), + ], +) def mark_cache(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM DROP MARK CACHE` privilege. - """ + """Run checks for `SYSTEM DROP MARK CACHE` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -166,8 +212,12 @@ def mark_cache(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user is unable to execute SYSTEM DROP MARK CACHE"): - node.query("SYSTEM DROP MARK CACHE", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM DROP MARK CACHE", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM DROP MARK CACHE with privilege"): @@ -175,7 +225,7 @@ def mark_cache(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user is bale to execute SYSTEM DROP MARK CACHE"): - node.query("SYSTEM DROP MARK CACHE", settings = [("user", f"{user_name}")]) + node.query("SYSTEM DROP MARK CACHE", settings=[("user", f"{user_name}")]) with Scenario("SYSTEM DROP MARK CACHE with revoked privilege"): @@ -186,8 +236,13 @@ def mark_cache(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user is unable to execute SYSTEM DROP MARK CACHE"): - node.query("SYSTEM DROP MARK CACHE", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM DROP MARK CACHE", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def uncompressed_cache_privileges_granted_directly(self, node=None): @@ -201,10 +256,18 @@ def uncompressed_cache_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=uncompressed_cache, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in uncompressed_cache.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=uncompressed_cache, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in uncompressed_cache.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def uncompressed_cache_privileges_granted_via_role(self, node=None): @@ -222,28 +285,38 @@ def uncompressed_cache_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=uncompressed_cache, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in uncompressed_cache.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=uncompressed_cache, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in uncompressed_cache.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_DropCache_Uncompressed("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM DROP CACHE",), - ("SYSTEM DROP UNCOMPRESSED CACHE",), - ("DROP CACHE",), - ("DROP UNCOMPRESSED CACHE",), - ("SYSTEM DROP UNCOMPRESSED",), - ("DROP UNCOMPRESSED",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM DROP CACHE",), + ("SYSTEM DROP UNCOMPRESSED CACHE",), + ("DROP CACHE",), + ("DROP UNCOMPRESSED CACHE",), + ("SYSTEM DROP UNCOMPRESSED",), + ("DROP UNCOMPRESSED",), + ], +) def uncompressed_cache(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM DROP UNCOMPRESSED CACHE` privilege. - """ + """Run checks for `SYSTEM DROP UNCOMPRESSED CACHE` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -257,9 +330,15 @@ def uncompressed_cache(self, privilege, grant_target_name, user_name, node=None) with And("I grant the user USAGE privilege"): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") - with Then("I check the user is unable to execute SYSTEM DROP UNCOMPRESSED CACHE"): - node.query("SYSTEM DROP UNCOMPRESSED CACHE", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I check the user is unable to execute SYSTEM DROP UNCOMPRESSED CACHE" + ): + node.query( + "SYSTEM DROP UNCOMPRESSED CACHE", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM DROP UNCOMPRESSED CACHE with privilege"): @@ -267,7 +346,9 @@ def uncompressed_cache(self, privilege, grant_target_name, user_name, node=None) node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user is bale to execute SYSTEM DROP UNCOMPRESSED CACHE"): - node.query("SYSTEM DROP UNCOMPRESSED CACHE", settings = [("user", f"{user_name}")]) + node.query( + "SYSTEM DROP UNCOMPRESSED CACHE", settings=[("user", f"{user_name}")] + ) with Scenario("SYSTEM DROP UNCOMPRESSED CACHE with revoked privilege"): @@ -277,25 +358,49 @@ def uncompressed_cache(self, privilege, grant_target_name, user_name, node=None) with And(f"I revoke {privilege} on the table"): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") - with Then("I check the user is unable to execute SYSTEM DROP UNCOMPRESSED CACHE"): - node.query("SYSTEM DROP UNCOMPRESSED CACHE", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I check the user is unable to execute SYSTEM DROP UNCOMPRESSED CACHE" + ): + node.query( + "SYSTEM DROP UNCOMPRESSED CACHE", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system drop cache") @Requirements( RQ_SRS_006_RBAC_Privileges_System_DropCache("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM DROP CACHE. - """ + """Check the RBAC functionality of SYSTEM DROP CACHE.""" self.context.node = self.context.cluster.node(node) - Suite(run=dns_cache_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=dns_cache_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=mark_cache_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=mark_cache_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=uncompressed_cache_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=uncompressed_cache_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=dns_cache_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=dns_cache_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=mark_cache_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=mark_cache_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=uncompressed_cache_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=uncompressed_cache_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/system/fetches.py b/tests/testflows/rbac/tests/privileges/system/fetches.py index 3aba1b71566..28c0be6c8b5 100644 --- a/tests/testflows/rbac/tests/privileges/system/fetches.py +++ b/tests/testflows/rbac/tests/privileges/system/fetches.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def replicated_privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM FETCHES` commands if and only if @@ -17,10 +18,18 @@ def replicated_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_replicated_privilege, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_replicated_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_replicated_privilege, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_replicated_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def replicated_privileges_granted_via_role(self, node=None): @@ -38,35 +47,59 @@ def replicated_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_replicated_privilege, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_replicated_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_replicated_privilege, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_replicated_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM FETCHES", "table"), - ("SYSTEM STOP FETCHES", "table"), - ("SYSTEM START FETCHES", "table"), - ("START FETCHES", "table"), - ("STOP FETCHES", "table"), -]) -def check_replicated_privilege(self, privilege, on, grant_target_name, user_name, node=None): - """Run checks for commands that require SYSTEM FETCHES privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM FETCHES", "table"), + ("SYSTEM STOP FETCHES", "table"), + ("SYSTEM START FETCHES", "table"), + ("START FETCHES", "table"), + ("STOP FETCHES", "table"), + ], +) +def check_replicated_privilege( + self, privilege, on, grant_target_name, user_name, node=None +): + """Run checks for commands that require SYSTEM FETCHES privilege.""" if node is None: node = self.context.node - Suite(test=start_replication_queues)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=stop_replication_queues)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=start_replication_queues)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + ) + Suite(test=stop_replication_queues)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + ) + @TestSuite -def start_replication_queues(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM START FETCHES` when they have privilege. - """ +def start_replication_queues( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is only able to execute `SYSTEM START FETCHES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -86,8 +119,12 @@ def start_replication_queues(self, privilege, on, grant_target_name, user_name, node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start fetches"): - node.query(f"SYSTEM START FETCHES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START FETCHES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START FETCHES with privilege"): @@ -95,7 +132,10 @@ def start_replication_queues(self, privilege, on, grant_target_name, user_name, node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start fetches"): - node.query(f"SYSTEM START FETCHES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START FETCHES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START FETCHES with revoked privilege"): @@ -106,13 +146,19 @@ def start_replication_queues(self, privilege, on, grant_target_name, user_name, node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start fetches"): - node.query(f"SYSTEM START FETCHES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START FETCHES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite -def stop_replication_queues(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM STOP FETCHES` when they have privilege. - """ +def stop_replication_queues( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is only able to execute `SYSTEM STOP FETCHES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -132,8 +178,12 @@ def stop_replication_queues(self, privilege, on, grant_target_name, user_name, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop fetches"): - node.query(f"SYSTEM STOP FETCHES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP FETCHES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP FETCHES with privilege"): @@ -141,7 +191,10 @@ def stop_replication_queues(self, privilege, on, grant_target_name, user_name, n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start fetches"): - node.query(f"SYSTEM STOP FETCHES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP FETCHES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP FETCHES with revoked privilege"): @@ -152,20 +205,30 @@ def stop_replication_queues(self, privilege, on, grant_target_name, user_name, n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start fetches"): - node.query(f"SYSTEM STOP FETCHES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP FETCHES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system fetches") @Requirements( RQ_SRS_006_RBAC_Privileges_System_Fetches("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM FETCHES. - """ + """Check the RBAC functionality of SYSTEM FETCHES.""" self.context.node = self.context.cluster.node(node) - Suite(run=replicated_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=replicated_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=replicated_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=replicated_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/system/flush.py b/tests/testflows/rbac/tests/privileges/system/flush.py index 8c540fa1286..f225639ee46 100644 --- a/tests/testflows/rbac/tests/privileges/system/flush.py +++ b/tests/testflows/rbac/tests/privileges/system/flush.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM FLUSH LOGS` commands if and only if @@ -17,10 +18,18 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=flush_logs, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in flush_logs.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=flush_logs, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in flush_logs.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): @@ -38,25 +47,35 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=flush_logs, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in flush_logs.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=flush_logs, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in flush_logs.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM FLUSH", "*.*"), - ("SYSTEM FLUSH LOGS", "*.*"), - ("FLUSH LOGS", "*.*"), -]) +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM FLUSH", "*.*"), + ("SYSTEM FLUSH LOGS", "*.*"), + ("FLUSH LOGS", "*.*"), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Flush_Logs("1.0"), ) def flush_logs(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM START REPLICATED FLUSH` when they have privilege. - """ + """Check that user is only able to execute `SYSTEM START REPLICATED FLUSH` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -71,8 +90,12 @@ def flush_logs(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't flush logs"): - node.query(f"SYSTEM FLUSH LOGS", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM FLUSH LOGS", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM FLUSH LOGS with privilege"): @@ -80,7 +103,7 @@ def flush_logs(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can flush logs"): - node.query(f"SYSTEM FLUSH LOGS", settings = [("user", f"{user_name}")]) + node.query(f"SYSTEM FLUSH LOGS", settings=[("user", f"{user_name}")]) with Scenario("SYSTEM FLUSH LOGS with revoked privilege"): @@ -91,8 +114,13 @@ def flush_logs(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't flush logs"): - node.query(f"SYSTEM FLUSH LOGS", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM FLUSH LOGS", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def distributed_privileges_granted_directly(self, node=None): @@ -107,10 +135,18 @@ def distributed_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(run=flush_distributed, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[user_name,user_name,table_name]) for row in flush_distributed.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=flush_distributed, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [user_name, user_name, table_name]) + for row in flush_distributed.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def distributed_privileges_granted_via_role(self, node=None): @@ -129,25 +165,37 @@ def distributed_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=flush_distributed, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[role_name,user_name,table_name]) for row in flush_distributed.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=flush_distributed, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [role_name, user_name, table_name]) + for row in flush_distributed.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM FLUSH", "*.*"), - ("SYSTEM FLUSH DISTRIBUTED", "table"), - ("FLUSH DISTRIBUTED", "table"), -]) +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM FLUSH", "*.*"), + ("SYSTEM FLUSH DISTRIBUTED", "table"), + ("FLUSH DISTRIBUTED", "table"), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Flush_Distributed("1.0"), ) -def flush_distributed(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM FLUSH DISTRIBUTED` when they have privilege. - """ +def flush_distributed( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM FLUSH DISTRIBUTED` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table0_name = f"table0_{getuid()}" @@ -159,7 +207,9 @@ def flush_distributed(self, privilege, on, grant_target_name, user_name, table_n with table(node, table0_name): try: with Given("I have a distributed table"): - node.query(f"CREATE TABLE {table_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())" + ) with Scenario("SYSTEM FLUSH DISTRIBUTED without privilege"): @@ -170,8 +220,12 @@ def flush_distributed(self, privilege, on, grant_target_name, user_name, table_n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't flush distributed"): - node.query(f"SYSTEM FLUSH DISTRIBUTED {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM FLUSH DISTRIBUTED {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM FLUSH DISTRIBUTED with privilege"): @@ -179,7 +233,10 @@ def flush_distributed(self, privilege, on, grant_target_name, user_name, table_n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can flush distributed"): - node.query(f"SYSTEM FLUSH DISTRIBUTED {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM FLUSH DISTRIBUTED {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM FLUSH DISTRIBUTED with revoked privilege"): @@ -190,8 +247,12 @@ def flush_distributed(self, privilege, on, grant_target_name, user_name, table_n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't flush distributed"): - node.query(f"SYSTEM FLUSH DISTRIBUTED {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM FLUSH DISTRIBUTED {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the distributed table"): @@ -203,14 +264,19 @@ def flush_distributed(self, privilege, on, grant_target_name, user_name, table_n @Requirements( RQ_SRS_006_RBAC_Privileges_System_Flush("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM FLUSH. - """ + """Check the RBAC functionality of SYSTEM FLUSH.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) Suite(run=privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=distributed_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=distributed_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=distributed_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=distributed_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/system/merges.py b/tests/testflows/rbac/tests/privileges/system/merges.py index 324b9c0b4ec..35d32220b4d 100644 --- a/tests/testflows/rbac/tests/privileges/system/merges.py +++ b/tests/testflows/rbac/tests/privileges/system/merges.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM MERGES` commands if and only if @@ -18,10 +19,18 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[user_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [user_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): @@ -40,35 +49,61 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[role_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [role_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM MERGES", "table"), - ("SYSTEM STOP MERGES", "table"), - ("SYSTEM START MERGES", "table"), - ("START MERGES", "table"), - ("STOP MERGES", "table"), -]) -def check_privilege(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Run checks for commands that require SYSTEM MERGES privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM MERGES", "table"), + ("SYSTEM STOP MERGES", "table"), + ("SYSTEM START MERGES", "table"), + ("START MERGES", "table"), + ("STOP MERGES", "table"), + ], +) +def check_privilege( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Run checks for commands that require SYSTEM MERGES privilege.""" if node is None: node = self.context.node - Suite(test=start_merges)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) - Suite(test=stop_merges)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) + Suite(test=start_merges)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + Suite(test=stop_merges)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + @TestSuite -def start_merges(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM START MERGES` when they have privilege. - """ +def start_merges( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM START MERGES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -87,8 +122,12 @@ def start_merges(self, privilege, on, grant_target_name, user_name, table_name, node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start merges"): - node.query(f"SYSTEM START MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START MERGES with privilege"): @@ -96,7 +135,10 @@ def start_merges(self, privilege, on, grant_target_name, user_name, table_name, node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start merges"): - node.query(f"SYSTEM START MERGES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START MERGES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START MERGES with revoked privilege"): @@ -107,13 +149,19 @@ def start_merges(self, privilege, on, grant_target_name, user_name, table_name, node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start merges"): - node.query(f"SYSTEM START MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite -def stop_merges(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM STOP MERGES` when they have privilege. - """ +def stop_merges( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM STOP MERGES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -132,8 +180,12 @@ def stop_merges(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop merges"): - node.query(f"SYSTEM STOP MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP MERGES with privilege"): @@ -141,7 +193,10 @@ def stop_merges(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can stop merges"): - node.query(f"SYSTEM STOP MERGES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP MERGES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP MERGES with revoked privilege"): @@ -152,19 +207,23 @@ def stop_merges(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't stop merges"): - node.query(f"SYSTEM STOP MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system merges") @Requirements( RQ_SRS_006_RBAC_Privileges_System_Merges("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM MERGES. - """ + """Check the RBAC functionality of SYSTEM MERGES.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/system/moves.py b/tests/testflows/rbac/tests/privileges/system/moves.py index 2a75ff39aaf..17ce6d931b3 100644 --- a/tests/testflows/rbac/tests/privileges/system/moves.py +++ b/tests/testflows/rbac/tests/privileges/system/moves.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM MOVES` commands if and only if @@ -18,10 +19,18 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[user_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [user_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): @@ -40,35 +49,61 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[role_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [role_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM MOVES", "table"), - ("SYSTEM STOP MOVES", "table"), - ("SYSTEM START MOVES", "table"), - ("START MOVES", "table"), - ("STOP MOVES", "table"), -]) -def check_privilege(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Run checks for commands that require SYSTEM MOVES privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM MOVES", "table"), + ("SYSTEM STOP MOVES", "table"), + ("SYSTEM START MOVES", "table"), + ("START MOVES", "table"), + ("STOP MOVES", "table"), + ], +) +def check_privilege( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Run checks for commands that require SYSTEM MOVES privilege.""" if node is None: node = self.context.node - Suite(test=start_moves)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) - Suite(test=stop_moves)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) + Suite(test=start_moves)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + Suite(test=stop_moves)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + @TestSuite -def start_moves(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM START MOVES` when they have privilege. - """ +def start_moves( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM START MOVES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -87,8 +122,12 @@ def start_moves(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start moves"): - node.query(f"SYSTEM START MOVES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START MOVES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START MOVES with privilege"): @@ -96,7 +135,10 @@ def start_moves(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start moves"): - node.query(f"SYSTEM START MOVES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START MOVES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START MOVES with revoked privilege"): @@ -107,13 +149,19 @@ def start_moves(self, privilege, on, grant_target_name, user_name, table_name, n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start moves"): - node.query(f"SYSTEM START MOVES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START MOVES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite -def stop_moves(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM STOP MOVES` when they have privilege. - """ +def stop_moves( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM STOP MOVES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -132,8 +180,12 @@ def stop_moves(self, privilege, on, grant_target_name, user_name, table_name, no node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop moves"): - node.query(f"SYSTEM STOP MOVES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP MOVES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP MOVES with privilege"): @@ -141,7 +193,10 @@ def stop_moves(self, privilege, on, grant_target_name, user_name, table_name, no node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can stop moves"): - node.query(f"SYSTEM STOP MOVES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP MOVES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP MOVES with revoked privilege"): @@ -152,19 +207,23 @@ def stop_moves(self, privilege, on, grant_target_name, user_name, table_name, no node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't stop moves"): - node.query(f"SYSTEM STOP MOVES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP MOVES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system moves") @Requirements( RQ_SRS_006_RBAC_Privileges_System_Moves("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM MOVES. - """ + """Check the RBAC functionality of SYSTEM MOVES.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/system/reload.py b/tests/testflows/rbac/tests/privileges/system/reload.py index 08df5803287..d0c7a2caea8 100644 --- a/tests/testflows/rbac/tests/privileges/system/reload.py +++ b/tests/testflows/rbac/tests/privileges/system/reload.py @@ -5,17 +5,21 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def dict_setup(node, table_name, dict_name): - """Setup and teardown of table and dictionary needed for the tests. - """ + """Setup and teardown of table and dictionary needed for the tests.""" try: with Given("I have a table"): - node.query(f"CREATE TABLE {table_name} (key UInt64, val UInt64) Engine=Memory()") + node.query( + f"CREATE TABLE {table_name} (key UInt64, val UInt64) Engine=Memory()" + ) with And("I have a dictionary"): - node.query(f"CREATE DICTIONARY {dict_name} (key UInt64 DEFAULT 0, val UInt64 DEFAULT 10) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE '{table_name}' PASSWORD '' DB 'default')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())") + node.query( + f"CREATE DICTIONARY {dict_name} (key UInt64 DEFAULT 0, val UInt64 DEFAULT 10) PRIMARY KEY key SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE '{table_name}' PASSWORD '' DB 'default')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT())" + ) yield @@ -26,6 +30,7 @@ def dict_setup(node, table_name, dict_name): with And("I drop the table", flags=TE): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestSuite def config_privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM RELOAD CONFIG` if and only if @@ -38,10 +43,15 @@ def config_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=config, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in config.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=config, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [user_name, user_name]) for row in config.examples], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def config_privileges_granted_via_role(self, node=None): @@ -59,25 +69,32 @@ def config_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=config, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in config.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=config, + examples=Examples( + "privilege grant_target_name user_name", + [tuple(list(row) + [role_name, user_name]) for row in config.examples], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Reload_Config("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM RELOAD",), - ("SYSTEM RELOAD CONFIG",), - ("RELOAD CONFIG",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM RELOAD",), + ("SYSTEM RELOAD CONFIG",), + ("RELOAD CONFIG",), + ], +) def config(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM RELOAD CONFIG` privilege. - """ + """Run checks for `SYSTEM RELOAD CONFIG` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -92,8 +109,12 @@ def config(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user is unable to execute SYSTEM RELOAD CONFIG"): - node.query("SYSTEM RELOAD CONFIG", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM RELOAD CONFIG", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM RELOAD CONFIG with privilege"): @@ -101,7 +122,7 @@ def config(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user is bale to execute SYSTEM RELOAD CONFIG"): - node.query("SYSTEM RELOAD CONFIG", settings = [("user", f"{user_name}")]) + node.query("SYSTEM RELOAD CONFIG", settings=[("user", f"{user_name}")]) with Scenario("SYSTEM RELOAD CONFIG with revoked privilege"): @@ -112,8 +133,13 @@ def config(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user is unable to execute SYSTEM RELOAD CONFIG"): - node.query("SYSTEM RELOAD CONFIG", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM RELOAD CONFIG", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def dictionary_privileges_granted_directly(self, node=None): @@ -127,10 +153,18 @@ def dictionary_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=dictionary, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictionary.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictionary, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictionary.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictionary_privileges_granted_via_role(self, node=None): @@ -148,26 +182,36 @@ def dictionary_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictionary, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictionary.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictionary, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictionary.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Reload_Dictionary("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM RELOAD",), - ("SYSTEM RELOAD DICTIONARIES",), - ("RELOAD DICTIONARIES",), - ("RELOAD DICTIONARY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM RELOAD",), + ("SYSTEM RELOAD DICTIONARIES",), + ("RELOAD DICTIONARIES",), + ("RELOAD DICTIONARY",), + ], +) def dictionary(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM RELOAD DICTIONARY` privilege. - """ + """Run checks for `SYSTEM RELOAD DICTIONARY` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -187,8 +231,12 @@ def dictionary(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user is unable to execute SYSTEM RELOAD DICTIONARY"): - node.query(f"SYSTEM RELOAD DICTIONARY default.{dict_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM RELOAD DICTIONARY default.{dict_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM RELOAD DICTIONARY with privilege"): @@ -201,7 +249,10 @@ def dictionary(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user is bale to execute SYSTEM RELOAD DICTIONARY"): - node.query(f"SYSTEM RELOAD DICTIONARY default.{dict_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM RELOAD DICTIONARY default.{dict_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM RELOAD DICTIONARY with revoked privilege"): @@ -217,8 +268,13 @@ def dictionary(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user is unable to execute SYSTEM RELOAD DICTIONARY"): - node.query(f"SYSTEM RELOAD DICTIONARY default.{dict_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM RELOAD DICTIONARY default.{dict_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def dictionaries_privileges_granted_directly(self, node=None): @@ -232,10 +288,18 @@ def dictionaries_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=dictionaries, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in dictionaries.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictionaries, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in dictionaries.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def dictionaries_privileges_granted_via_role(self, node=None): @@ -253,26 +317,36 @@ def dictionaries_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=dictionaries, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in dictionaries.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=dictionaries, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in dictionaries.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Reload_Dictionaries("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM RELOAD",), - ("SYSTEM RELOAD DICTIONARIES",), - ("RELOAD DICTIONARIES",), - ("RELOAD DICTIONARY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM RELOAD",), + ("SYSTEM RELOAD DICTIONARIES",), + ("RELOAD DICTIONARIES",), + ("RELOAD DICTIONARY",), + ], +) def dictionaries(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM RELOAD DICTIONARIES` privilege. - """ + """Run checks for `SYSTEM RELOAD DICTIONARIES` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -287,8 +361,12 @@ def dictionaries(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user is unable to execute SYSTEM RELOAD DICTIONARIES"): - node.query("SYSTEM RELOAD DICTIONARIES", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM RELOAD DICTIONARIES", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM RELOAD DICTIONARIES with privilege"): @@ -296,7 +374,9 @@ def dictionaries(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user is bale to execute SYSTEM RELOAD DICTIONARIES"): - node.query("SYSTEM RELOAD DICTIONARIES", settings = [("user", f"{user_name}")]) + node.query( + "SYSTEM RELOAD DICTIONARIES", settings=[("user", f"{user_name}")] + ) with Scenario("SYSTEM RELOAD DICTIONARIES with revoked privilege"): @@ -307,8 +387,13 @@ def dictionaries(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user is unable to execute SYSTEM RELOAD DICTIONARIES"): - node.query("SYSTEM RELOAD DICTIONARIES", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + "SYSTEM RELOAD DICTIONARIES", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def embedded_dictionaries_privileges_granted_directly(self, node=None): @@ -322,10 +407,18 @@ def embedded_dictionaries_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=embedded_dictionaries, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in embedded_dictionaries.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=embedded_dictionaries, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in embedded_dictionaries.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def embedded_dictionaries_privileges_granted_via_role(self, node=None): @@ -343,25 +436,35 @@ def embedded_dictionaries_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=embedded_dictionaries, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in embedded_dictionaries.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=embedded_dictionaries, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in embedded_dictionaries.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Reload_EmbeddedDictionaries("1.0"), ) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM RELOAD",), - ("SYSTEM RELOAD EMBEDDED DICTIONARIES",), - ("SYSTEM RELOAD DICTIONARY",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM RELOAD",), + ("SYSTEM RELOAD EMBEDDED DICTIONARIES",), + ("SYSTEM RELOAD DICTIONARY",), + ], +) def embedded_dictionaries(self, privilege, grant_target_name, user_name, node=None): - """Run checks for `SYSTEM RELOAD EMBEDDED DICTIONARIES` privilege. - """ + """Run checks for `SYSTEM RELOAD EMBEDDED DICTIONARIES` privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -375,17 +478,28 @@ def embedded_dictionaries(self, privilege, grant_target_name, user_name, node=No with And("I grant the user USAGE privilege"): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") - with Then("I check the user is unable to execute SYSTEM RELOAD EMBEDDED DICTIONARIES"): - node.query("SYSTEM RELOAD EMBEDDED DICTIONARIES", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I check the user is unable to execute SYSTEM RELOAD EMBEDDED DICTIONARIES" + ): + node.query( + "SYSTEM RELOAD EMBEDDED DICTIONARIES", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM RELOAD EMBEDDED DICTIONARIES with privilege"): with When(f"I grant {privilege} on the table"): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") - with Then("I check the user is bale to execute SYSTEM RELOAD EMBEDDED DICTIONARIES"): - node.query("SYSTEM RELOAD EMBEDDED DICTIONARIES", settings = [("user", f"{user_name}")]) + with Then( + "I check the user is bale to execute SYSTEM RELOAD EMBEDDED DICTIONARIES" + ): + node.query( + "SYSTEM RELOAD EMBEDDED DICTIONARIES", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM RELOAD EMBEDDED DICTIONARIES with revoked privilege"): @@ -395,27 +509,55 @@ def embedded_dictionaries(self, privilege, grant_target_name, user_name, node=No with And(f"I revoke {privilege} on the table"): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") - with Then("I check the user is unable to execute SYSTEM RELOAD EMBEDDED DICTIONARIES"): - node.query("SYSTEM RELOAD EMBEDDED DICTIONARIES", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I check the user is unable to execute SYSTEM RELOAD EMBEDDED DICTIONARIES" + ): + node.query( + "SYSTEM RELOAD EMBEDDED DICTIONARIES", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system reload") @Requirements( RQ_SRS_006_RBAC_Privileges_System_Reload("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM RELOAD. - """ + """Check the RBAC functionality of SYSTEM RELOAD.""" self.context.node = self.context.cluster.node(node) - Suite(run=config_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=config_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=dictionary_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=dictionary_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=dictionaries_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=dictionaries_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=embedded_dictionaries_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=embedded_dictionaries_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=config_privileges_granted_directly, setup=instrument_clickhouse_server_log + ) + Suite( + run=config_privileges_granted_via_role, setup=instrument_clickhouse_server_log + ) + Suite( + run=dictionary_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=dictionary_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=dictionaries_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=dictionaries_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=embedded_dictionaries_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=embedded_dictionaries_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/system/replication_queues.py b/tests/testflows/rbac/tests/privileges/system/replication_queues.py index 47f12b7c866..7bf5f0d8ad5 100644 --- a/tests/testflows/rbac/tests/privileges/system/replication_queues.py +++ b/tests/testflows/rbac/tests/privileges/system/replication_queues.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def replicated_privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM REPLICATION QUEUES` commands if and only if @@ -17,10 +18,18 @@ def replicated_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_replicated_privilege, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_replicated_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_replicated_privilege, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_replicated_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def replicated_privileges_granted_via_role(self, node=None): @@ -38,35 +47,59 @@ def replicated_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_replicated_privilege, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_replicated_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_replicated_privilege, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_replicated_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM REPLICATION QUEUES", "table"), - ("SYSTEM STOP REPLICATION QUEUES", "table"), - ("SYSTEM START REPLICATION QUEUES", "table"), - ("START REPLICATION QUEUES", "table"), - ("STOP REPLICATION QUEUES", "table"), -]) -def check_replicated_privilege(self, privilege, on, grant_target_name, user_name, node=None): - """Run checks for commands that require SYSTEM REPLICATION QUEUES privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM REPLICATION QUEUES", "table"), + ("SYSTEM STOP REPLICATION QUEUES", "table"), + ("SYSTEM START REPLICATION QUEUES", "table"), + ("START REPLICATION QUEUES", "table"), + ("STOP REPLICATION QUEUES", "table"), + ], +) +def check_replicated_privilege( + self, privilege, on, grant_target_name, user_name, node=None +): + """Run checks for commands that require SYSTEM REPLICATION QUEUES privilege.""" if node is None: node = self.context.node - Suite(test=start_replication_queues)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=stop_replication_queues)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=start_replication_queues)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + ) + Suite(test=stop_replication_queues)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + ) + @TestSuite -def start_replication_queues(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM START REPLICATION QUEUES` when they have privilege. - """ +def start_replication_queues( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is only able to execute `SYSTEM START REPLICATION QUEUES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -86,8 +119,12 @@ def start_replication_queues(self, privilege, on, grant_target_name, user_name, node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start sends"): - node.query(f"SYSTEM START REPLICATION QUEUES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START REPLICATION QUEUES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START REPLICATION QUEUES with privilege"): @@ -95,7 +132,10 @@ def start_replication_queues(self, privilege, on, grant_target_name, user_name, node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start sends"): - node.query(f"SYSTEM START REPLICATION QUEUES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START REPLICATION QUEUES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START REPLICATION QUEUES with revoked privilege"): @@ -106,13 +146,19 @@ def start_replication_queues(self, privilege, on, grant_target_name, user_name, node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start sends"): - node.query(f"SYSTEM START REPLICATION QUEUES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START REPLICATION QUEUES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite -def stop_replication_queues(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM STOP REPLICATION QUEUES` when they have privilege. - """ +def stop_replication_queues( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is only able to execute `SYSTEM STOP REPLICATION QUEUES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -132,8 +178,12 @@ def stop_replication_queues(self, privilege, on, grant_target_name, user_name, n node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop sends"): - node.query(f"SYSTEM STOP REPLICATION QUEUES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP REPLICATION QUEUES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP REPLICATION QUEUES with privilege"): @@ -141,7 +191,10 @@ def stop_replication_queues(self, privilege, on, grant_target_name, user_name, n node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start sends"): - node.query(f"SYSTEM STOP REPLICATION QUEUES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP REPLICATION QUEUES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP REPLICATION QUEUES with revoked privilege"): @@ -152,20 +205,30 @@ def stop_replication_queues(self, privilege, on, grant_target_name, user_name, n node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start sends"): - node.query(f"SYSTEM STOP REPLICATION QUEUES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP REPLICATION QUEUES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system replication queues") @Requirements( RQ_SRS_006_RBAC_Privileges_System_ReplicationQueues("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM REPLICATION QUEUES. - """ + """Check the RBAC functionality of SYSTEM REPLICATION QUEUES.""" self.context.node = self.context.cluster.node(node) - Suite(run=replicated_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=replicated_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=replicated_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=replicated_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/system/restart_replica.py b/tests/testflows/rbac/tests/privileges/system/restart_replica.py index 4e3d5f7b060..cf45a784b03 100644 --- a/tests/testflows/rbac/tests/privileges/system/restart_replica.py +++ b/tests/testflows/rbac/tests/privileges/system/restart_replica.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM RESTART REPLICA` commands if and only if @@ -17,10 +18,18 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=restart_replica, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in restart_replica.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=restart_replica, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in restart_replica.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): @@ -38,21 +47,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=restart_replica, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in restart_replica.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=restart_replica, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in restart_replica.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM RESTART REPLICA", "table"), - ("RESTART REPLICA", "table"), -]) +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM RESTART REPLICA", "table"), + ("RESTART REPLICA", "table"), + ], +) def restart_replica(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM RESTARTE REPLICA` when they have privilege. - """ + """Check that user is only able to execute `SYSTEM RESTARTE REPLICA` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -72,8 +91,12 @@ def restart_replica(self, privilege, on, grant_target_name, user_name, node=None node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't restart replica"): - node.query(f"SYSTEM RESTART REPLICA {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM RESTART REPLICA {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM RESTART REPLICA with privilege"): @@ -81,7 +104,10 @@ def restart_replica(self, privilege, on, grant_target_name, user_name, node=None node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can restart replica"): - node.query(f"SYSTEM RESTART REPLICA {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM RESTART REPLICA {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM RESTART REPLICA with revoked privilege"): @@ -92,19 +118,23 @@ def restart_replica(self, privilege, on, grant_target_name, user_name, node=None node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't restart replica"): - node.query(f"SYSTEM RESTART REPLICA {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM RESTART REPLICA {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system restart replica") @Requirements( RQ_SRS_006_RBAC_Privileges_System_RestartReplica("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM RESTART REPLICA. - """ + """Check the RBAC functionality of SYSTEM RESTART REPLICA.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/system/sends.py b/tests/testflows/rbac/tests/privileges/system/sends.py index 4acd173d922..ee298c50cb1 100644 --- a/tests/testflows/rbac/tests/privileges/system/sends.py +++ b/tests/testflows/rbac/tests/privileges/system/sends.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def replicated_privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM REPLICATED SENDS` commands if and only if @@ -17,10 +18,18 @@ def replicated_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_replicated_privilege, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_replicated_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_replicated_privilege, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_replicated_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def replicated_privileges_granted_via_role(self, node=None): @@ -38,43 +47,67 @@ def replicated_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_replicated_privilege, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_replicated_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_replicated_privilege, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_replicated_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM SENDS", "*.*"), - ("SYSTEM START SENDS", "*.*"), - ("SYSTEM STOP SENDS", "*.*"), - ("START SENDS", "*.*"), - ("STOP SENDS", "*.*"), - ("SYSTEM REPLICATED SENDS", "table"), - ("SYSTEM STOP REPLICATED SENDS", "table"), - ("SYSTEM START REPLICATED SENDS", "table"), - ("START REPLICATED SENDS", "table"), - ("STOP REPLICATED SENDS", "table"), -]) +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM SENDS", "*.*"), + ("SYSTEM START SENDS", "*.*"), + ("SYSTEM STOP SENDS", "*.*"), + ("START SENDS", "*.*"), + ("STOP SENDS", "*.*"), + ("SYSTEM REPLICATED SENDS", "table"), + ("SYSTEM STOP REPLICATED SENDS", "table"), + ("SYSTEM START REPLICATED SENDS", "table"), + ("START REPLICATED SENDS", "table"), + ("STOP REPLICATED SENDS", "table"), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Sends_Replicated("1.0"), ) -def check_replicated_privilege(self, privilege, on, grant_target_name, user_name, node=None): - """Run checks for commands that require SYSTEM REPLICATED SENDS privilege. - """ +def check_replicated_privilege( + self, privilege, on, grant_target_name, user_name, node=None +): + """Run checks for commands that require SYSTEM REPLICATED SENDS privilege.""" if node is None: node = self.context.node - Suite(test=start_replicated_sends)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=stop_replicated_sends)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=start_replicated_sends)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + ) + Suite(test=stop_replicated_sends)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + ) + @TestSuite -def start_replicated_sends(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM START REPLICATED SENDS` when they have privilege. - """ +def start_replicated_sends( + self, privilege, on, grant_target_name, user_name, node=None +): + """Check that user is only able to execute `SYSTEM START REPLICATED SENDS` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -94,8 +127,12 @@ def start_replicated_sends(self, privilege, on, grant_target_name, user_name, no node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start sends"): - node.query(f"SYSTEM START REPLICATED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START REPLICATED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START REPLICATED SENDS with privilege"): @@ -103,7 +140,10 @@ def start_replicated_sends(self, privilege, on, grant_target_name, user_name, no node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start sends"): - node.query(f"SYSTEM START REPLICATED SENDS {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START REPLICATED SENDS {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START REPLICATED SENDS with revoked privilege"): @@ -114,13 +154,17 @@ def start_replicated_sends(self, privilege, on, grant_target_name, user_name, no node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start sends"): - node.query(f"SYSTEM START REPLICATED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START REPLICATED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def stop_replicated_sends(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM STOP REPLICATED SENDS` when they have privilege. - """ + """Check that user is only able to execute `SYSTEM STOP REPLICATED SENDS` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -140,8 +184,12 @@ def stop_replicated_sends(self, privilege, on, grant_target_name, user_name, nod node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop sends"): - node.query(f"SYSTEM STOP REPLICATED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP REPLICATED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP REPLICATED SENDS with privilege"): @@ -149,7 +197,10 @@ def stop_replicated_sends(self, privilege, on, grant_target_name, user_name, nod node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can stop sends"): - node.query(f"SYSTEM STOP REPLICATED SENDS {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP REPLICATED SENDS {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP REPLICATED SENDS with revoked privilege"): @@ -160,8 +211,13 @@ def stop_replicated_sends(self, privilege, on, grant_target_name, user_name, nod node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't stop sends"): - node.query(f"SYSTEM STOP REPLICATED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP REPLICATED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite def distributed_privileges_granted_directly(self, node=None): @@ -176,10 +232,18 @@ def distributed_privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(run=check_distributed_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[user_name,user_name,table_name]) for row in check_distributed_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_distributed_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [user_name, user_name, table_name]) + for row in check_distributed_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def distributed_privileges_granted_via_role(self, node=None): @@ -198,43 +262,69 @@ def distributed_privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_distributed_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[role_name,user_name,table_name]) for row in check_distributed_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_distributed_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [role_name, user_name, table_name]) + for row in check_distributed_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM SENDS", "*.*"), - ("SYSTEM START SENDS", "*.*"), - ("SYSTEM STOP SENDS", "*.*"), - ("START SENDS", "*.*"), - ("STOP SENDS", "*.*"), - ("SYSTEM DISTRIBUTED SENDS", "table"), - ("SYSTEM STOP DISTRIBUTED SENDS", "table"), - ("SYSTEM START DISTRIBUTED SENDS", "table"), - ("START DISTRIBUTED SENDS", "table"), - ("STOP DISTRIBUTED SENDS", "table"), -]) +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM SENDS", "*.*"), + ("SYSTEM START SENDS", "*.*"), + ("SYSTEM STOP SENDS", "*.*"), + ("START SENDS", "*.*"), + ("STOP SENDS", "*.*"), + ("SYSTEM DISTRIBUTED SENDS", "table"), + ("SYSTEM STOP DISTRIBUTED SENDS", "table"), + ("SYSTEM START DISTRIBUTED SENDS", "table"), + ("START DISTRIBUTED SENDS", "table"), + ("STOP DISTRIBUTED SENDS", "table"), + ], +) @Requirements( RQ_SRS_006_RBAC_Privileges_System_Sends_Distributed("1.0"), ) -def check_distributed_privilege(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Run checks for commands that require SYSTEM DISTRIBUTED SENDS privilege. - """ +def check_distributed_privilege( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Run checks for commands that require SYSTEM DISTRIBUTED SENDS privilege.""" if node is None: node = self.context.node - Suite(test=start_distributed_moves)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) - Suite(test=stop_distributed_moves)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) + Suite(test=start_distributed_moves)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + Suite(test=stop_distributed_moves)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + @TestSuite -def start_distributed_moves(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM START DISTRIBUTED SENDS` when they have privilege. - """ +def start_distributed_moves( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM START DISTRIBUTED SENDS` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table0_name = f"table0_{getuid()}" @@ -246,7 +336,9 @@ def start_distributed_moves(self, privilege, on, grant_target_name, user_name, t with table(node, table0_name): try: with Given("I have a distributed table"): - node.query(f"CREATE TABLE {table_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())" + ) with Scenario("SYSTEM START DISTRIBUTED SENDS without privilege"): @@ -257,8 +349,12 @@ def start_distributed_moves(self, privilege, on, grant_target_name, user_name, t node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start merges"): - node.query(f"SYSTEM START DISTRIBUTED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START DISTRIBUTED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START DISTRIBUTED SENDS with privilege"): @@ -266,7 +362,10 @@ def start_distributed_moves(self, privilege, on, grant_target_name, user_name, t node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start merges"): - node.query(f"SYSTEM START DISTRIBUTED SENDS {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START DISTRIBUTED SENDS {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START DISTRIBUTED SENDS with revoked privilege"): @@ -277,17 +376,23 @@ def start_distributed_moves(self, privilege, on, grant_target_name, user_name, t node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start merges"): - node.query(f"SYSTEM START DISTRIBUTED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START DISTRIBUTED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestSuite -def stop_distributed_moves(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM STOP DISTRIBUTED SENDS` when they have privilege. - """ +def stop_distributed_moves( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM STOP DISTRIBUTED SENDS` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table0_name = f"table0_{getuid()}" @@ -299,7 +404,9 @@ def stop_distributed_moves(self, privilege, on, grant_target_name, user_name, ta with table(node, table0_name): try: with Given("I have a distributed table"): - node.query(f"CREATE TABLE {table_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())") + node.query( + f"CREATE TABLE {table_name} (a UInt64) ENGINE = Distributed(sharded_cluster, default, {table0_name}, rand())" + ) with Scenario("SYSTEM STOP DISTRIBUTED SENDS without privilege"): @@ -310,8 +417,12 @@ def stop_distributed_moves(self, privilege, on, grant_target_name, user_name, ta node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop merges"): - node.query(f"SYSTEM STOP DISTRIBUTED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP DISTRIBUTED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP DISTRIBUTED SENDS with privilege"): @@ -319,7 +430,10 @@ def stop_distributed_moves(self, privilege, on, grant_target_name, user_name, ta node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can stop merges"): - node.query(f"SYSTEM STOP DISTRIBUTED SENDS {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP DISTRIBUTED SENDS {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP DISTRIBUTED SENDS with revoked privilege"): @@ -330,25 +444,41 @@ def stop_distributed_moves(self, privilege, on, grant_target_name, user_name, ta node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't stop merges"): - node.query(f"SYSTEM STOP DISTRIBUTED SENDS {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP DISTRIBUTED SENDS {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the distributed table"): node.query(f"DROP TABLE IF EXISTS {table_name}") + @TestFeature @Name("system sends") @Requirements( RQ_SRS_006_RBAC_Privileges_System_Sends("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM SENDS. - """ + """Check the RBAC functionality of SYSTEM SENDS.""" self.context.node = self.context.cluster.node(node) - Suite(run=replicated_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=replicated_privileges_granted_via_role, setup=instrument_clickhouse_server_log) - Suite(run=distributed_privileges_granted_directly, setup=instrument_clickhouse_server_log) - Suite(run=distributed_privileges_granted_via_role, setup=instrument_clickhouse_server_log) + Suite( + run=replicated_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=replicated_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=distributed_privileges_granted_directly, + setup=instrument_clickhouse_server_log, + ) + Suite( + run=distributed_privileges_granted_via_role, + setup=instrument_clickhouse_server_log, + ) diff --git a/tests/testflows/rbac/tests/privileges/system/shutdown.py b/tests/testflows/rbac/tests/privileges/system/shutdown.py index 26752ef4d01..2b09b7d8585 100644 --- a/tests/testflows/rbac/tests/privileges/system/shutdown.py +++ b/tests/testflows/rbac/tests/privileges/system/shutdown.py @@ -7,10 +7,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): - """Run checks with privileges granted directly. - """ + """Run checks with privileges granted directly.""" user_name = f"user_{getuid()}" @@ -19,15 +19,22 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): - """Run checks with privileges granted through a role. - """ + """Run checks with privileges granted through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" @@ -40,33 +47,47 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in check_privilege.examples - ], args=Args(name="privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in check_privilege.examples + ], + args=Args(name="privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege",[ - ("ALL",), - ("SYSTEM",), - ("SYSTEM SHUTDOWN",), - ("SHUTDOWN",), - ("SYSTEM KILL",), -]) +@Examples( + "privilege", + [ + ("ALL",), + ("SYSTEM",), + ("SYSTEM SHUTDOWN",), + ("SHUTDOWN",), + ("SYSTEM KILL",), + ], +) def check_privilege(self, privilege, grant_target_name, user_name, node=None): - """Run checks for commands that require SYSTEM SHUTDOWN privilege. - """ + """Run checks for commands that require SYSTEM SHUTDOWN privilege.""" if node is None: node = self.context.node - Suite(test=shutdown)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) - Suite(test=kill)(privilege=privilege, grant_target_name=grant_target_name, user_name=user_name) + Suite(test=shutdown)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + Suite(test=kill)( + privilege=privilege, grant_target_name=grant_target_name, user_name=user_name + ) + @TestSuite def shutdown(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM SHUTDOWN` when they have the necessary privilege. - """ + """Check that user is only able to execute `SYSTEM SHUTDOWN` when they have the necessary privilege.""" cluster = self.context.cluster exitcode, message = errors.not_enough_privileges(name=user_name) @@ -83,8 +104,12 @@ def shutdown(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SYSTEM SHUTDOWN"): - node.query(f"SYSTEM SHUTDOWN", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM SHUTDOWN", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM SHUTDOWN with privilege"): timeout = 60 @@ -94,13 +119,13 @@ def shutdown(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SYSTEM SHUTDOWN"): - node.query(f"SYSTEM SHUTDOWN", settings = [("user", f"{user_name}")]) + node.query(f"SYSTEM SHUTDOWN", settings=[("user", f"{user_name}")]) with And("I close all connections to the node"): node.close_bashes() with And("I check that system is down"): - command = f"echo -e \"SELECT 1\" | {cluster.docker_compose} exec -T {node.name} clickhouse client -n" + command = f'echo -e "SELECT 1" | {cluster.docker_compose} exec -T {node.name} clickhouse client -n' start_time = time.time() @@ -127,13 +152,17 @@ def shutdown(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SYSTEM SHUTDOWN"): - node.query(f"SYSTEM SHUTDOWN", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM SHUTDOWN", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestSuite def kill(self, privilege, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM KILL` when they have the necessary privilege. - """ + """Check that user is only able to execute `SYSTEM KILL` when they have the necessary privilege.""" cluster = self.context.cluster exitcode, message = errors.not_enough_privileges(name=user_name) @@ -150,8 +179,12 @@ def kill(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't use SYSTEM KILL"): - node.query(f"SYSTEM KILL", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM KILL", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM KILL with privilege"): timeout = 60 @@ -161,7 +194,7 @@ def kill(self, privilege, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON *.* TO {grant_target_name}") with Then("I check the user can use SYSTEM KILL"): - command = f"echo -e \"SYSTEM KILL\" | clickhouse client -n" + command = f'echo -e "SYSTEM KILL" | clickhouse client -n' with By("executing command", description=command): self.context.cluster.bash(node.name).send(command) @@ -169,7 +202,7 @@ def kill(self, privilege, grant_target_name, user_name, node=None): node.close_bashes() with And("I check that system is down"): - command = f"echo -e \"SELECT 1\" | {cluster.docker_compose} exec -T {node.name} clickhouse client -n" + command = f'echo -e "SELECT 1" | {cluster.docker_compose} exec -T {node.name} clickhouse client -n' start_time = time.time() @@ -196,19 +229,23 @@ def kill(self, privilege, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON *.* FROM {grant_target_name}") with Then("I check the user cannot use SYSTEM KILL"): - node.query(f"SYSTEM KILL", settings=[("user",user_name)], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM KILL", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system shutdown") @Requirements( RQ_SRS_006_RBAC_Privileges_System_Shutdown("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM SHUTDOWN. - """ + """Check the RBAC functionality of SYSTEM SHUTDOWN.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/system/sync_replica.py b/tests/testflows/rbac/tests/privileges/system/sync_replica.py index 14681ad31ae..6bb7f9820a9 100644 --- a/tests/testflows/rbac/tests/privileges/system/sync_replica.py +++ b/tests/testflows/rbac/tests/privileges/system/sync_replica.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM SYNC REPLICA` commands if and only if @@ -17,10 +18,18 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): - Suite(run=sync_replica, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[user_name,user_name]) for row in sync_replica.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=sync_replica, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [user_name, user_name]) + for row in sync_replica.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): @@ -38,21 +47,31 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=sync_replica, - examples=Examples("privilege on grant_target_name user_name", [ - tuple(list(row)+[role_name,user_name]) for row in sync_replica.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=sync_replica, + examples=Examples( + "privilege on grant_target_name user_name", + [ + tuple(list(row) + [role_name, user_name]) + for row in sync_replica.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM SYNC REPLICA", "table"), - ("SYNC REPLICA", "table"), -]) +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM SYNC REPLICA", "table"), + ("SYNC REPLICA", "table"), + ], +) def sync_replica(self, privilege, on, grant_target_name, user_name, node=None): - """Check that user is only able to execute `SYSTEM SYNCE REPLICA` when they have privilege. - """ + """Check that user is only able to execute `SYSTEM SYNCE REPLICA` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) table_name = f"table_name_{getuid()}" @@ -72,8 +91,12 @@ def sync_replica(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't sync replica"): - node.query(f"SYSTEM SYNC REPLICA {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM SYNC REPLICA {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM SYNC REPLICA with privilege"): @@ -81,7 +104,10 @@ def sync_replica(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can sync replica"): - node.query(f"SYSTEM SYNC REPLICA {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM SYNC REPLICA {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM SYNC REPLICA with revoked privilege"): @@ -92,19 +118,23 @@ def sync_replica(self, privilege, on, grant_target_name, user_name, node=None): node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't sync replica"): - node.query(f"SYSTEM SYNC REPLICA {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM SYNC REPLICA {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system sync replica") @Requirements( RQ_SRS_006_RBAC_Privileges_System_SyncReplica("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM SYNC REPLICA. - """ + """Check the RBAC functionality of SYSTEM SYNC REPLICA.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/system/ttl_merges.py b/tests/testflows/rbac/tests/privileges/system/ttl_merges.py index a59cc530a6d..5f6d1c9475f 100644 --- a/tests/testflows/rbac/tests/privileges/system/ttl_merges.py +++ b/tests/testflows/rbac/tests/privileges/system/ttl_merges.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privileges_granted_directly(self, node=None): """Check that a user is able to execute `SYSTEM TTL MERGES` commands if and only if @@ -18,10 +19,18 @@ def privileges_granted_directly(self, node=None): with user(node, f"{user_name}"): table_name = f"table_name_{getuid()}" - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[user_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [user_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestSuite def privileges_granted_via_role(self, node=None): @@ -40,35 +49,61 @@ def privileges_granted_via_role(self, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Suite(run=check_privilege, - examples=Examples("privilege on grant_target_name user_name table_name", [ - tuple(list(row)+[role_name,user_name,table_name]) for row in check_privilege.examples - ], args=Args(name="check privilege={privilege}", format_name=True))) + Suite( + run=check_privilege, + examples=Examples( + "privilege on grant_target_name user_name table_name", + [ + tuple(list(row) + [role_name, user_name, table_name]) + for row in check_privilege.examples + ], + args=Args(name="check privilege={privilege}", format_name=True), + ), + ) + @TestOutline(Suite) -@Examples("privilege on",[ - ("ALL", "*.*"), - ("SYSTEM", "*.*"), - ("SYSTEM TTL MERGES", "table"), - ("SYSTEM STOP TTL MERGES", "table"), - ("SYSTEM START TTL MERGES", "table"), - ("START TTL MERGES", "table"), - ("STOP TTL MERGES", "table"), -]) -def check_privilege(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Run checks for commands that require SYSTEM TTL MERGES privilege. - """ +@Examples( + "privilege on", + [ + ("ALL", "*.*"), + ("SYSTEM", "*.*"), + ("SYSTEM TTL MERGES", "table"), + ("SYSTEM STOP TTL MERGES", "table"), + ("SYSTEM START TTL MERGES", "table"), + ("START TTL MERGES", "table"), + ("STOP TTL MERGES", "table"), + ], +) +def check_privilege( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Run checks for commands that require SYSTEM TTL MERGES privilege.""" if node is None: node = self.context.node - Suite(test=start_ttl_merges)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) - Suite(test=stop_ttl_merges)(privilege=privilege, on=on, grant_target_name=grant_target_name, user_name=user_name, table_name=table_name) + Suite(test=start_ttl_merges)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + Suite(test=stop_ttl_merges)( + privilege=privilege, + on=on, + grant_target_name=grant_target_name, + user_name=user_name, + table_name=table_name, + ) + @TestSuite -def start_ttl_merges(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM START TTL MERGES` when they have privilege. - """ +def start_ttl_merges( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM START TTL MERGES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -87,8 +122,12 @@ def start_ttl_merges(self, privilege, on, grant_target_name, user_name, table_na node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't start merges"): - node.query(f"SYSTEM START TTL MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START TTL MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM START TTL MERGES with privilege"): @@ -96,7 +135,10 @@ def start_ttl_merges(self, privilege, on, grant_target_name, user_name, table_na node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can start merges"): - node.query(f"SYSTEM START TTL MERGES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM START TTL MERGES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM START TTL MERGES with revoked privilege"): @@ -107,13 +149,19 @@ def start_ttl_merges(self, privilege, on, grant_target_name, user_name, table_na node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't start merges"): - node.query(f"SYSTEM START TTL MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM START TTL MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestSuite -def stop_ttl_merges(self, privilege, on, grant_target_name, user_name, table_name, node=None): - """Check that user is only able to execute `SYSTEM STOP TTL MERGES` when they have privilege. - """ +def stop_ttl_merges( + self, privilege, on, grant_target_name, user_name, table_name, node=None +): + """Check that user is only able to execute `SYSTEM STOP TTL MERGES` when they have privilege.""" exitcode, message = errors.not_enough_privileges(name=user_name) if node is None: @@ -132,8 +180,12 @@ def stop_ttl_merges(self, privilege, on, grant_target_name, user_name, table_nam node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I check the user can't stop merges"): - node.query(f"SYSTEM STOP TTL MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP TTL MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with Scenario("SYSTEM STOP TTL MERGES with privilege"): @@ -141,7 +193,10 @@ def stop_ttl_merges(self, privilege, on, grant_target_name, user_name, table_nam node.query(f"GRANT {privilege} ON {on} TO {grant_target_name}") with Then("I check the user can stop merges"): - node.query(f"SYSTEM STOP TTL MERGES {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SYSTEM STOP TTL MERGES {table_name}", + settings=[("user", f"{user_name}")], + ) with Scenario("SYSTEM STOP TTL MERGES with revoked privilege"): @@ -152,19 +207,23 @@ def stop_ttl_merges(self, privilege, on, grant_target_name, user_name, table_nam node.query(f"REVOKE {privilege} ON {on} FROM {grant_target_name}") with Then("I check the user can't stop merges"): - node.query(f"SYSTEM STOP TTL MERGES {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SYSTEM STOP TTL MERGES {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("system ttl merges") @Requirements( RQ_SRS_006_RBAC_Privileges_System_TTLMerges("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) def feature(self, node="clickhouse1"): - """Check the RBAC functionality of SYSTEM TTL MERGES. - """ + """Check the RBAC functionality of SYSTEM TTL MERGES.""" self.context.node = self.context.cluster.node(node) Suite(run=privileges_granted_directly, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/tests/privileges/truncate.py b/tests/testflows/rbac/tests/privileges/truncate.py index df81913f0a8..8e107da3c0f 100644 --- a/tests/testflows/rbac/tests/privileges/truncate.py +++ b/tests/testflows/rbac/tests/privileges/truncate.py @@ -2,10 +2,10 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite def privilege_granted_directly_or_via_role(self, table_type, node=None): - """Check that user is only able to execute TRUNCATE when they have required privilege, either directly or via role. - """ + """Check that user is only able to execute TRUNCATE when they have required privilege, either directly or via role.""" role_name = f"role_{getuid()}" user_name = f"user_{getuid()}" @@ -15,8 +15,15 @@ def privilege_granted_directly_or_via_role(self, table_type, node=None): with Suite("user with direct privilege"): with user(node, user_name): - with When(f"I run checks that {user_name} is only able to execute TRUNCATE with required privileges"): - privilege_check(grant_target_name=user_name, user_name=user_name, table_type=table_type, node=node) + with When( + f"I run checks that {user_name} is only able to execute TRUNCATE with required privileges" + ): + privilege_check( + grant_target_name=user_name, + user_name=user_name, + table_type=table_type, + node=node, + ) with Suite("user with privilege via role"): with user(node, user_name), role(node, role_name): @@ -24,12 +31,19 @@ def privilege_granted_directly_or_via_role(self, table_type, node=None): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - with And(f"I run checks that {user_name} with {role_name} is only able to execute TRUNCATE with required privileges"): - privilege_check(grant_target_name=role_name, user_name=user_name, table_type=table_type, node=node) + with And( + f"I run checks that {user_name} with {role_name} is only able to execute TRUNCATE with required privileges" + ): + privilege_check( + grant_target_name=role_name, + user_name=user_name, + table_type=table_type, + node=node, + ) + def privilege_check(grant_target_name, user_name, table_type, node=None): - """Run scenarios to check the user's access with different privileges. - """ + """Run scenarios to check the user's access with different privileges.""" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") with Scenario("user without privilege"): @@ -44,8 +58,12 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT USAGE ON *.* TO {grant_target_name}") with Then("I attempt to truncate a table without privilege"): - node.query(f"TRUNCATE TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"TRUNCATE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with privilege"): table_name = f"merge_tree_{getuid()}" @@ -56,7 +74,9 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT TRUNCATE ON {table_name} TO {grant_target_name}") with Then("I attempt to truncate a table"): - node.query(f"TRUNCATE TABLE {table_name}", settings = [("user", user_name)]) + node.query( + f"TRUNCATE TABLE {table_name}", settings=[("user", user_name)] + ) with Scenario("user with revoked privilege"): table_name = f"merge_tree_{getuid()}" @@ -70,8 +90,12 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"REVOKE TRUNCATE ON {table_name} FROM {grant_target_name}") with Then("I attempt to truncate a table"): - node.query(f"TRUNCATE TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"TRUNCATE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("user with revoked ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -85,8 +109,12 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"REVOKE ALL ON *.* FROM {grant_target_name}") with Then("I attempt to truncate a table"): - node.query(f"TRUNCATE TABLE {table_name}", settings = [("user", user_name)], - exitcode=exitcode, message=message) + node.query( + f"TRUNCATE TABLE {table_name}", + settings=[("user", user_name)], + exitcode=exitcode, + message=message, + ) with Scenario("execute on cluster"): table_name = f"merge_tree_{getuid()}" @@ -97,7 +125,10 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT TRUNCATE ON {table_name} TO {grant_target_name}") with Then("I attempt to truncate a table"): - node.query(f"TRUNCATE TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster", settings = [("user", user_name)]) + node.query( + f"TRUNCATE TABLE IF EXISTS {table_name} ON CLUSTER sharded_cluster", + settings=[("user", user_name)], + ) with Scenario("user with ALL privilege"): table_name = f"merge_tree_{getuid()}" @@ -111,21 +142,21 @@ def privilege_check(grant_target_name, user_name, table_type, node=None): node.query(f"GRANT ALL ON *.* TO {grant_target_name}") with Then("I attempt to truncate a table"): - node.query(f"TRUNCATE TABLE {table_name}", settings = [("user", user_name)]) + node.query( + f"TRUNCATE TABLE {table_name}", settings=[("user", user_name)] + ) + @TestFeature @Requirements( RQ_SRS_006_RBAC_Privileges_Truncate("1.0"), RQ_SRS_006_RBAC_Privileges_All("1.0"), - RQ_SRS_006_RBAC_Privileges_None("1.0") + RQ_SRS_006_RBAC_Privileges_None("1.0"), ) -@Examples("table_type", [ - (key,) for key in table_types.keys() -]) +@Examples("table_type", [(key,) for key in table_types.keys()]) @Name("truncate") def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check the RBAC functionality of TRUNCATE. - """ + """Check the RBAC functionality of TRUNCATE.""" self.context.node = self.context.cluster.node(node) if parallel is not None: @@ -134,11 +165,14 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): self.context.stress = stress for example in self.examples: - table_type, = example + (table_type,) = example if table_type != "MergeTree" and not self.context.stress: continue with Example(str(example)): - with Suite(test=privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log): + with Suite( + test=privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ): privilege_granted_directly_or_via_role(table_type=table_type) diff --git a/tests/testflows/rbac/tests/syntax/alter_quota.py b/tests/testflows/rbac/tests/syntax/alter_quota.py index 6ccafc4dbcd..34ed1b00f8d 100755 --- a/tests/testflows/rbac/tests/syntax/alter_quota.py +++ b/tests/testflows/rbac/tests/syntax/alter_quota.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("alter quota") @Args(format_description=False) @@ -33,13 +34,17 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE USER user0") node.query(f"CREATE ROLE role0") - with Scenario("I alter quota with no options", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter("1.0")]): + with Scenario( + "I alter quota with no options", + requirements=[RQ_SRS_006_RBAC_Quota_Alter("1.0")], + ): with When("I alter quota"): node.query("ALTER QUOTA quota0") - with Scenario("I alter quota that does not exist, throws an exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter("1.0")]): + with Scenario( + "I alter quota that does not exist, throws an exception", + requirements=[RQ_SRS_006_RBAC_Quota_Alter("1.0")], + ): quota = "quota1" cleanup_quota(quota) with When(f"I alter quota {quota}, which does not exist"): @@ -47,24 +52,32 @@ def feature(self, node="clickhouse1"): node.query(f"ALTER QUOTA {quota}", exitcode=exitcode, message=message) del quota - with Scenario("I alter quota with if exists, quota does exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_IfExists("1.0")]): + with Scenario( + "I alter quota with if exists, quota does exist", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_IfExists("1.0")], + ): node.query("ALTER QUOTA IF EXISTS quota0") - with Scenario("I alter quota with if exists, quota does not exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_IfExists("1.0")]): + with Scenario( + "I alter quota with if exists, quota does not exist", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_IfExists("1.0")], + ): quota = "quota1" cleanup_quota(quota) with When(f"I alter quota {quota}, which does not exist, with IF EXISTS"): node.query(f"ALTER QUOTA IF EXISTS {quota}") del quota - with Scenario("I alter quota using rename, target available", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Rename("1.0")]): + with Scenario( + "I alter quota using rename, target available", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Rename("1.0")], + ): node.query("ALTER QUOTA quota0 RENAME TO quota0") - with Scenario("I alter quota using rename, target unavailable", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Rename("1.0")]): + with Scenario( + "I alter quota using rename, target unavailable", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Rename("1.0")], + ): new_quota = "quota1" try: @@ -72,39 +85,72 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE QUOTA IF NOT EXISTS {new_quota}") with When(f"I try to rename to {new_quota}"): - exitcode, message = errors.cannot_rename_quota(name="quota0", name_new=new_quota) - node.query(f"ALTER QUOTA quota0 RENAME TO {new_quota}", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_rename_quota( + name="quota0", name_new=new_quota + ) + node.query( + f"ALTER QUOTA quota0 RENAME TO {new_quota}", + exitcode=exitcode, + message=message, + ) finally: with Finally(f"I cleanup target name {new_quota}"): node.query(f"DROP QUOTA IF EXISTS {new_quota}") del new_quota - keys = ['none', 'user name', 'ip address', 'client key', 'client key or user name', 'client key or ip address'] + keys = [ + "none", + "user name", + "ip address", + "client key", + "client key or user name", + "client key or ip address", + ] for key in keys: - with Scenario(f"I alter quota keyed by {key}", requirements=[ + with Scenario( + f"I alter quota keyed by {key}", + requirements=[ RQ_SRS_006_RBAC_Quota_Alter_KeyedBy("1.0"), - RQ_SRS_006_RBAC_Quota_Alter_KeyedByOptions("1.0")]): + RQ_SRS_006_RBAC_Quota_Alter_KeyedByOptions("1.0"), + ], + ): with When("I alter quota with a key"): node.query(f"ALTER QUOTA quota0 KEYED BY '{key}'") - with Scenario("I alter quota for randomized interval", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Interval_Randomized("1.0")]): + with Scenario( + "I alter quota for randomized interval", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Interval_Randomized("1.0")], + ): with When("I alter quota on a randomized interval"): node.query("ALTER QUOTA quota0 FOR RANDOMIZED INTERVAL 1 DAY NO LIMITS") - intervals = ['SECOND', 'MINUTE', 'HOUR', 'DAY', 'MONTH'] + intervals = ["SECOND", "MINUTE", "HOUR", "DAY", "MONTH"] for i, interval in enumerate(intervals): - with Scenario(f"I alter quota for interval {interval}", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Interval("1.0")]): + with Scenario( + f"I alter quota for interval {interval}", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Interval("1.0")], + ): with When(f"I alter quota for {interval}"): - node.query(f"ALTER QUOTA quota0 FOR INTERVAL 1 {interval} NO LIMITS") + node.query( + f"ALTER QUOTA quota0 FOR INTERVAL 1 {interval} NO LIMITS" + ) - constraints = ['MAX QUERIES', 'MAX ERRORS', 'MAX RESULT ROWS', - 'MAX RESULT BYTES', 'MAX READ ROWS', 'MAX READ BYTES', 'MAX EXECUTION TIME', - 'NO LIMITS', 'TRACKING ONLY'] + constraints = [ + "MAX QUERIES", + "MAX ERRORS", + "MAX RESULT ROWS", + "MAX RESULT BYTES", + "MAX READ ROWS", + "MAX READ BYTES", + "MAX EXECUTION TIME", + "NO LIMITS", + "TRACKING ONLY", + ] for i, constraint in enumerate(constraints): - with Scenario(f"I alter quota for {constraint.lower()}", requirements=[ + with Scenario( + f"I alter quota for {constraint.lower()}", + requirements=[ RQ_SRS_006_RBAC_Quota_Alter_Queries("1.0"), RQ_SRS_006_RBAC_Quota_Alter_Errors("1.0"), RQ_SRS_006_RBAC_Quota_Alter_ResultRows("1.0"), @@ -113,70 +159,106 @@ def feature(self, node="clickhouse1"): RQ_SRS_006_RBAC_Quota_Alter_ReadBytes("1.0"), RQ_SRS_006_RBAC_Quota_Alter_ExecutionTime("1.0"), RQ_SRS_006_RBAC_Quota_Alter_NoLimits("1.0"), - RQ_SRS_006_RBAC_Quota_Alter_TrackingOnly("1.0")]): + RQ_SRS_006_RBAC_Quota_Alter_TrackingOnly("1.0"), + ], + ): with When("I alter quota for a constraint"): - node.query(f"ALTER QUOTA quota0 FOR INTERVAL 1 DAY {constraint}{' 1024' if constraint.startswith('MAX') else ''}") + node.query( + f"ALTER QUOTA quota0 FOR INTERVAL 1 DAY {constraint}{' 1024' if constraint.startswith('MAX') else ''}" + ) - with Scenario("I create quota for multiple constraints", requirements=[ + with Scenario( + "I create quota for multiple constraints", + requirements=[ RQ_SRS_006_RBAC_Quota_Alter_Interval("1.0"), - RQ_SRS_006_RBAC_Quota_Alter_Queries("1.0")]): - node.query("ALTER QUOTA quota0 \ + RQ_SRS_006_RBAC_Quota_Alter_Queries("1.0"), + ], + ): + node.query( + "ALTER QUOTA quota0 \ FOR INTERVAL 1 DAY NO LIMITS, \ FOR INTERVAL 2 DAY MAX QUERIES 124, \ - FOR INTERVAL 1 MONTH TRACKING ONLY") + FOR INTERVAL 1 MONTH TRACKING ONLY" + ) - with Scenario("I alter quota to assign to one role", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")]): + with Scenario( + "I alter quota to assign to one role", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")], + ): with When("I alter quota to a role"): node.query("ALTER QUOTA quota0 TO role0") - with Scenario("I alter quota to assign to role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")]): + with Scenario( + "I alter quota to assign to role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") with Then(f"I alter a quota, assign to role {role}, which does not exist"): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"ALTER QUOTA quota0 TO {role}", exitcode=exitcode, message=message) + node.query( + f"ALTER QUOTA quota0 TO {role}", exitcode=exitcode, message=message + ) del role - with Scenario("I alter quota to assign to all except role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")]): + with Scenario( + "I alter quota to assign to all except role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I alter a quota, assign to all except role {role}, which does not exist"): + with Then( + f"I alter a quota, assign to all except role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"ALTER QUOTA quota0 TO ALL EXCEPT {role}", exitcode=exitcode, message=message) + node.query( + f"ALTER QUOTA quota0 TO ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter quota to assign to one role and one user", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")]): + with Scenario( + "I alter quota to assign to one role and one user", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment("1.0")], + ): with When("I alter quota to a role and a user"): node.query("ALTER QUOTA quota0 TO role0, user0") - with Scenario("I alter quota assigned to none", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment_None("1.0")]): + with Scenario( + "I alter quota assigned to none", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment_None("1.0")], + ): with When("I alter quota to none"): node.query("ALTER QUOTA quota0 TO NONE") - with Scenario("I alter quota to assign to all", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment_All("1.0")]): + with Scenario( + "I alter quota to assign to all", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment_All("1.0")], + ): with When("I alter quota to all"): node.query("ALTER QUOTA quota0 TO ALL") - with Scenario("I alter quota to assign to all except one role", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment_Except("1.0")]): + with Scenario( + "I alter quota to assign to all except one role", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment_Except("1.0")], + ): with When("I alter quota to all except one role"): node.query("ALTER QUOTA quota0 TO ALL EXCEPT role0") - with Scenario("I alter quota to assign to all except multiple roles", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Assignment_Except("1.0")]): + with Scenario( + "I alter quota to assign to all except multiple roles", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Assignment_Except("1.0")], + ): with When("I alter quota to all except one multiple roles"): node.query("ALTER QUOTA quota0 TO ALL EXCEPT role0, user0") - with Scenario("I alter quota on cluster", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Cluster("1.0")]): + with Scenario( + "I alter quota on cluster", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Cluster("1.0")], + ): try: with Given("I have a quota on a cluster"): node.query("CREATE QUOTA quota1 ON CLUSTER sharded_cluster") @@ -184,23 +266,33 @@ def feature(self, node="clickhouse1"): with When("I run alter quota command on a cluster"): node.query("ALTER QUOTA quota1 ON CLUSTER sharded_cluster") with And("I run alter quota command on a cluster with a key"): - node.query("ALTER QUOTA quota1 ON CLUSTER sharded_cluster KEYED BY 'none'") + node.query( + "ALTER QUOTA quota1 ON CLUSTER sharded_cluster KEYED BY 'none'" + ) with And("I run alter quota command on a cluster with an interval"): - node.query("ALTER QUOTA quota1 ON CLUSTER sharded_cluster FOR INTERVAL 1 DAY TRACKING ONLY") + node.query( + "ALTER QUOTA quota1 ON CLUSTER sharded_cluster FOR INTERVAL 1 DAY TRACKING ONLY" + ) with And("I run alter quota command on a cluster for all"): node.query("ALTER QUOTA quota1 ON CLUSTER sharded_cluster TO ALL") finally: with Finally("I drop the quota"): node.query("DROP QUOTA IF EXISTS quota1 ON CLUSTER sharded_cluster") - with Scenario("I alter quota on nonexistent cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Alter_Cluster("1.0")]): + with Scenario( + "I alter quota on nonexistent cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Alter_Cluster("1.0")], + ): with When("I run alter quota on a cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("ALTER QUOTA quota0 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "ALTER QUOTA quota0 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the quota and all the users and roles"): node.query(f"DROP QUOTA IF EXISTS quota0") node.query(f"DROP USER IF EXISTS user0") - node.query(f"DROP ROLE IF EXISTS role0") \ No newline at end of file + node.query(f"DROP ROLE IF EXISTS role0") diff --git a/tests/testflows/rbac/tests/syntax/alter_role.py b/tests/testflows/rbac/tests/syntax/alter_role.py index 5068302fc84..eb826e32a77 100755 --- a/tests/testflows/rbac/tests/syntax/alter_role.py +++ b/tests/testflows/rbac/tests/syntax/alter_role.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("alter role") def feature(self, node="clickhouse1"): @@ -23,7 +24,7 @@ def feature(self, node="clickhouse1"): try: with Given("I have a role"): node.query(f"CREATE ROLE OR REPLACE {role}") - if profile != None: #create profile when name is given + if profile != None: # create profile when name is given with Given("And I have a profile"): node.query(f"CREATE SETTINGS PROFILE OR REPLACE {profile}") yield @@ -38,14 +39,17 @@ def feature(self, node="clickhouse1"): with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") - with Scenario("I alter role with no options", requirements=[ - RQ_SRS_006_RBAC_Role_Alter("1.0")]): + with Scenario( + "I alter role with no options", requirements=[RQ_SRS_006_RBAC_Role_Alter("1.0")] + ): with setup("role0"): with When("I alter role"): node.query("ALTER ROLE role0") - with Scenario("I alter role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Alter("1.0")]): + with Scenario( + "I alter role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Alter("1.0")], + ): role = "role0" cleanup_role(role) with When(f"I alter role {role} that does not exist"): @@ -53,43 +57,61 @@ def feature(self, node="clickhouse1"): node.query(f"ALTER ROLE {role}", exitcode=exitcode, message=message) del role - with Scenario("I alter role if exists, role does exist", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_IfExists("1.0")]): + with Scenario( + "I alter role if exists, role does exist", + requirements=[RQ_SRS_006_RBAC_Role_Alter_IfExists("1.0")], + ): with setup("role1"): with When("I alter role with if exists"): node.query("ALTER ROLE IF EXISTS role1") - with Scenario("I alter role if exists, role does not exist", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_IfExists("1.0")]): + with Scenario( + "I alter role if exists, role does not exist", + requirements=[RQ_SRS_006_RBAC_Role_Alter_IfExists("1.0")], + ): role = "role0" cleanup_role(role) with When(f"I alter role {role} that does not exist"): node.query(f"ALTER ROLE IF EXISTS {role}") del role - with Scenario("I alter role on cluster", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Cluster("1.0")]): + with Scenario( + "I alter role on cluster", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Cluster("1.0")], + ): try: with Given("I have a role on a cluster"): node.query("CREATE ROLE role1 ON CLUSTER sharded_cluster") with When("I run alter role on a cluster"): node.query("ALTER ROLE role1 ON CLUSTER sharded_cluster") with And("I rename role on a cluster"): - node.query("ALTER ROLE role1 ON CLUSTER sharded_cluster RENAME TO role2") + node.query( + "ALTER ROLE role1 ON CLUSTER sharded_cluster RENAME TO role2" + ) with And("I alter role with settings on a cluster"): - node.query("ALTER ROLE role2 ON CLUSTER sharded_cluster SETTINGS max_memory_usage=10000000 READONLY") + node.query( + "ALTER ROLE role2 ON CLUSTER sharded_cluster SETTINGS max_memory_usage=10000000 READONLY" + ) finally: with Finally("I drop the role"): node.query("DROP ROLE IF EXISTS role1,role2 ON CLUSTER sharded_cluster") - with Scenario("I alter role on nonexistent cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Cluster("1.0")]): + with Scenario( + "I alter role on nonexistent cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Cluster("1.0")], + ): with When("I run alter role on a cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("ALTER ROLE role1 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "ALTER ROLE role1 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) - with Scenario("I alter role to rename, new name is available", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Rename("1.0")]): + with Scenario( + "I alter role to rename, new name is available", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Rename("1.0")], + ): with setup("role2"): new_role = "role3" try: @@ -102,95 +124,144 @@ def feature(self, node="clickhouse1"): node.query(f"DROP ROLE IF EXISTS {new_role}") del new_role - with Scenario("I alter role to rename, new name is not available, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Rename("1.0")]): + with Scenario( + "I alter role to rename, new name is not available, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Rename("1.0")], + ): with setup("role2a"): new_role = "role3a" try: with Given(f"Ensure target name {new_role} is NOT available"): node.query(f"CREATE ROLE IF NOT EXISTS {new_role}") with When(f"I try to rename to {new_role}"): - exitcode, message = errors.cannot_rename_role(name="role2a", name_new=new_role) - node.query(f"ALTER ROLE role2a RENAME TO {new_role}", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_rename_role( + name="role2a", name_new=new_role + ) + node.query( + f"ALTER ROLE role2a RENAME TO {new_role}", + exitcode=exitcode, + message=message, + ) finally: with Finally(f"I cleanup target name {new_role}"): node.query(f"DROP ROLE IF EXISTS {new_role}") del new_role - with Scenario("I alter role settings profile", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings profile", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role4"): with When("I alter role with settings profile"): - node.query("ALTER ROLE role4 SETTINGS PROFILE default, max_memory_usage=10000000 READONLY") + node.query( + "ALTER ROLE role4 SETTINGS PROFILE default, max_memory_usage=10000000 READONLY" + ) - with Scenario("I alter role settings profile, profile does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings profile, profile does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role4a"): with Given("I ensure profile profile0 does not exist"): node.query("DROP SETTINGS PROFILE IF EXISTS profile0") with When("I alter role with settings profile that does not exist"): - exitcode, message = errors.settings_profile_not_found_in_disk("profile0") - node.query("ALTER ROLE role4a SETTINGS PROFILE profile0", exitcode=exitcode, message=message) + exitcode, message = errors.settings_profile_not_found_in_disk( + "profile0" + ) + node.query( + "ALTER ROLE role4a SETTINGS PROFILE profile0", + exitcode=exitcode, + message=message, + ) - with Scenario("I alter role settings profile multiple", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings profile multiple", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role4b", profile="profile0"): with When("I alter role with multiple profiles"): - node.query("ALTER ROLE role4b SETTINGS PROFILE default, PROFILE profile0, \ - max_memory_usage=10000000 READONLY") + node.query( + "ALTER ROLE role4b SETTINGS PROFILE default, PROFILE profile0, \ + max_memory_usage=10000000 READONLY" + ) - with Scenario("I alter role settings without profile", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings without profile", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role5"): with When("I alter role with settings and no profile"): - node.query("ALTER ROLE role5 SETTINGS max_memory_usage=10000000 READONLY") + node.query( + "ALTER ROLE role5 SETTINGS max_memory_usage=10000000 READONLY" + ) - with Scenario("I alter role settings, variable does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings, variable does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role5a"): with When("I alter role using settings and nonexistent value"): exitcode, message = errors.unknown_setting("fake_setting") - node.query("ALTER ROLE role5a SETTINGS fake_setting = 100000001", exitcode=exitcode, message=message) + node.query( + "ALTER ROLE role5a SETTINGS fake_setting = 100000001", + exitcode=exitcode, + message=message, + ) - - with Scenario("I alter role settings without profile multiple", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings without profile multiple", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role6"): with When("I alter role with multiple settings and no profile"): - node.query("ALTER ROLE role6 SETTINGS max_memory_usage=10000000 READONLY, \ - max_rows_to_read MIN 20 MAX 25") + node.query( + "ALTER ROLE role6 SETTINGS max_memory_usage=10000000 READONLY, \ + max_rows_to_read MIN 20 MAX 25" + ) - with Scenario("I alter role settings with multiple profiles multiple variables", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings with multiple profiles multiple variables", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role7", profile="profile1"): with When("I alter role with multiple settings and profiles"): - node.query("ALTER ROLE role7 SETTINGS PROFILE default, PROFILE profile1, \ - max_memory_usage=10000000 READONLY, max_rows_to_read MIN 20 MAX 25") + node.query( + "ALTER ROLE role7 SETTINGS PROFILE default, PROFILE profile1, \ + max_memory_usage=10000000 READONLY, max_rows_to_read MIN 20 MAX 25" + ) - with Scenario("I alter role settings readonly", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings readonly", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role8"): with When("I alter role with readonly"): node.query("ALTER ROLE role8 SETTINGS max_memory_usage READONLY") - with Scenario("I alter role settings writable", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings writable", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role9"): with When("I alter role with writable"): node.query("ALTER ROLE role9 SETTINGS max_memory_usage WRITABLE") - with Scenario("I alter role settings min, with and without = sign", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings min, with and without = sign", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role10"): with When("I set min, no equals"): node.query("ALTER ROLE role10 SETTINGS max_memory_usage MIN 200") with When("I set min, yes equals"): node.query("ALTER ROLE role10 SETTINGS max_memory_usage MIN = 200") - with Scenario("I alter role settings max, with and without = sign", requirements=[ - RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")]): + with Scenario( + "I alter role settings max, with and without = sign", + requirements=[RQ_SRS_006_RBAC_Role_Alter_Settings("1.0")], + ): with setup("role11"): with When("I set max, no equals"): node.query("ALTER ROLE role11 SETTINGS max_memory_usage MAX 2000") with When("I set max, yes equals"): - node.query("ALTER ROLE role11 SETTINGS max_memory_usage MAX = 200") \ No newline at end of file + node.query("ALTER ROLE role11 SETTINGS max_memory_usage MAX = 200") diff --git a/tests/testflows/rbac/tests/syntax/alter_row_policy.py b/tests/testflows/rbac/tests/syntax/alter_row_policy.py index 6422a81fec2..55eb3060699 100755 --- a/tests/testflows/rbac/tests/syntax/alter_row_policy.py +++ b/tests/testflows/rbac/tests/syntax/alter_row_policy.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("alter row policy") @Args(format_description=False) @@ -42,165 +43,279 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE ROLE role0") node.query(f"CREATE ROLE role1") - with Scenario("I alter row policy with no options", requirements=[ + with Scenario( + "I alter row policy with no options", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy0"): with When("I alter row policy"): node.query("ALTER ROW POLICY policy0 ON default.foo") - with Scenario("I alter row policy using short syntax with no options", requirements=[ + with Scenario( + "I alter row policy using short syntax with no options", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy1"): with When("I alter row policy short form"): node.query("ALTER POLICY policy1 ON default.foo") - with Scenario("I alter row policy, does not exist, throws exception", requirements=[ + with Scenario( + "I alter row policy, does not exist, throws exception", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): policy = "policy2" cleanup_policy(policy) with When(f"I alter row policy {policy} that doesn't exist"): - exitcode, message = errors.row_policy_not_found_in_disk(name=f"{policy} ON default.foo") - node.query(f"ALTER ROW POLICY {policy} ON default.foo", exitcode=exitcode, message=message) + exitcode, message = errors.row_policy_not_found_in_disk( + name=f"{policy} ON default.foo" + ) + node.query( + f"ALTER ROW POLICY {policy} ON default.foo", + exitcode=exitcode, + message=message, + ) del policy - with Scenario("I alter row policy if exists", requirements=[ + with Scenario( + "I alter row policy if exists", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_IfExists("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy2"): with When("I alter row policy using if exists"): node.query("ALTER ROW POLICY IF EXISTS policy2 ON default.foo") - with Scenario("I alter row policy if exists, policy does not exist", requirements=[ + with Scenario( + "I alter row policy if exists, policy does not exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_IfExists("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): policy = "policy2" cleanup_policy(policy) with When(f"I alter row policy {policy} that doesn't exist"): node.query(f"ALTER ROW POLICY IF EXISTS {policy} ON default.foo") del policy - with Scenario("I alter row policy to rename, target available", requirements=[ + with Scenario( + "I alter row policy to rename, target available", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Rename("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy3"): with When("I alter row policy with rename"): - node.query("ALTER ROW POLICY policy3 ON default.foo RENAME TO policy3") + node.query( + "ALTER ROW POLICY policy3 ON default.foo RENAME TO policy3" + ) - with Scenario("I alter row policy to rename, target unavailable", requirements=[ + with Scenario( + "I alter row policy to rename, target unavailable", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Rename("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy3"): new_policy = "policy4" try: with Given(f"Ensure target name {new_policy} is NOT available"): - node.query(f"CREATE ROW POLICY IF NOT EXISTS {new_policy} ON default.foo") + node.query( + f"CREATE ROW POLICY IF NOT EXISTS {new_policy} ON default.foo" + ) with When(f"I try to rename to {new_policy}"): - exitcode, message = errors.cannot_rename_row_policy(name="policy3 ON default.foo", - name_new=f"{new_policy} ON default.foo") - node.query(f"ALTER ROW POLICY policy3 ON default.foo RENAME TO {new_policy}", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_rename_row_policy( + name="policy3 ON default.foo", + name_new=f"{new_policy} ON default.foo", + ) + node.query( + f"ALTER ROW POLICY policy3 ON default.foo RENAME TO {new_policy}", + exitcode=exitcode, + message=message, + ) finally: with Finally(f"I cleanup target name {new_policy}"): - node.query(f"DROP ROW POLICY IF EXISTS {new_policy} ON default.foo") + node.query( + f"DROP ROW POLICY IF EXISTS {new_policy} ON default.foo" + ) del new_policy - with Scenario("I alter row policy to permissive", requirements=[ + with Scenario( + "I alter row policy to permissive", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Permissive("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy4"): with When("I alter row policy as permissive"): node.query("ALTER ROW POLICY policy4 ON default.foo AS PERMISSIVE") - with Scenario("I alter row policy to restrictive", requirements=[ + with Scenario( + "I alter row policy to restrictive", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Access_Restrictive("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy5"): with When("I alter row policy as restrictive"): node.query("ALTER ROW POLICY policy5 ON default.foo AS RESTRICTIVE") - with Scenario("I alter row policy for select", requirements=[ + with Scenario( + "I alter row policy for select", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_ForSelect("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy6"): with When("I alter row policy using for select"): - node.query("ALTER ROW POLICY policy6 ON default.foo FOR SELECT USING x > 10") + node.query( + "ALTER ROW POLICY policy6 ON default.foo FOR SELECT USING x > 10" + ) - with Scenario("I alter row policy using condition", requirements=[ + with Scenario( + "I alter row policy using condition", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Condition("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy6"): with When("I alter row policy wtih condition"): node.query("ALTER ROW POLICY policy6 ON default.foo USING x > 10") - with Scenario("I alter row policy using condition none", requirements=[ + with Scenario( + "I alter row policy using condition none", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Condition_None("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy7"): with When("I alter row policy using no condition"): node.query("ALTER ROW POLICY policy7 ON default.foo USING NONE") - with Scenario("I alter row policy to one role", requirements=[ + with Scenario( + "I alter row policy to one role", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy8"): with When("I alter row policy to a role"): node.query("ALTER ROW POLICY policy8 ON default.foo TO role0") - with Scenario("I alter row policy to assign to role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0")]): + with Scenario( + "I alter row policy to assign to role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0")], + ): role = "role2" with cleanup("policy8a"): with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I alter a row policy, assign to role {role}, which does not exist"): + with Then( + f"I alter a row policy, assign to role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"ALTER ROW POLICY policy8a ON default.foo TO {role}", exitcode=exitcode, message=message) + node.query( + f"ALTER ROW POLICY policy8a ON default.foo TO {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter row policy to assign to all excpet role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0")]): + with Scenario( + "I alter row policy to assign to all excpet role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0")], + ): role = "role2" with cleanup("policy8a"): with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I alter a row policy, assign to all except role {role}, which does not exist"): + with Then( + f"I alter a row policy, assign to all except role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"ALTER ROW POLICY policy8a ON default.foo TO ALL EXCEPT {role}", exitcode=exitcode, message=message) + node.query( + f"ALTER ROW POLICY policy8a ON default.foo TO ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter row policy assigned to multiple roles", requirements=[ + with Scenario( + "I alter row policy assigned to multiple roles", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy9"): with When("I alter row policy to multiple roles"): - node.query("ALTER ROW POLICY policy9 ON default.foo TO role0, role1") + node.query( + "ALTER ROW POLICY policy9 ON default.foo TO role0, role1" + ) - with Scenario("I alter row policy assigned to all", requirements=[ + with Scenario( + "I alter row policy assigned to all", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_All("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy10"): with When("I alter row policy to all"): node.query("ALTER ROW POLICY policy10 ON default.foo TO ALL") - with Scenario("I alter row policy assigned to all except one role", requirements=[ + with Scenario( + "I alter row policy assigned to all except one role", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_AllExcept("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy11"): with When("I alter row policy to all except"): - node.query("ALTER ROW POLICY policy11 ON default.foo TO ALL EXCEPT role0") + node.query( + "ALTER ROW POLICY policy11 ON default.foo TO ALL EXCEPT role0" + ) - with Scenario("I alter row policy assigned to all except multiple roles", requirements=[ + with Scenario( + "I alter row policy assigned to all except multiple roles", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_AllExcept("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy12"): with When("I alter row policy to all except multiple roles"): - node.query("ALTER ROW POLICY policy12 ON default.foo TO ALL EXCEPT role0, role1") + node.query( + "ALTER ROW POLICY policy12 ON default.foo TO ALL EXCEPT role0, role1" + ) - with Scenario("I alter row policy assigned to none", requirements=[ + with Scenario( + "I alter row policy assigned to none", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_Assignment_None("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with cleanup("policy12"): with When("I alter row policy to no assignment"): node.query("ALTER ROW POLICY policy12 ON default.foo TO NONE") @@ -208,37 +323,65 @@ def feature(self, node="clickhouse1"): # Official syntax: ON CLUSTER cluster_name ON database.table # Working syntax: both orderings of ON CLUSTER and TABLE clauses work - with Scenario("I alter row policy on cluster", requirements=[ + with Scenario( + "I alter row policy on cluster", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): try: with Given("I have a row policy"): - node.query("CREATE ROW POLICY policy13 ON CLUSTER sharded_cluster ON default.foo") + node.query( + "CREATE ROW POLICY policy13 ON CLUSTER sharded_cluster ON default.foo" + ) with When("I run alter row policy command"): - node.query("ALTER ROW POLICY policy13 ON CLUSTER sharded_cluster ON default.foo") + node.query( + "ALTER ROW POLICY policy13 ON CLUSTER sharded_cluster ON default.foo" + ) finally: with Finally("I drop the row policy"): - node.query("DROP ROW POLICY IF EXISTS policy13 ON CLUSTER sharded_cluster ON default.foo") + node.query( + "DROP ROW POLICY IF EXISTS policy13 ON CLUSTER sharded_cluster ON default.foo" + ) - with Scenario("I alter row policy on fake cluster, throws exception", requirements=[ + with Scenario( + "I alter row policy on fake cluster, throws exception", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): with When("I run alter row policy command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("ALTER ROW POLICY policy13 ON CLUSTER fake_cluster ON default.foo", exitcode=exitcode, message=message) + node.query( + "ALTER ROW POLICY policy13 ON CLUSTER fake_cluster ON default.foo", + exitcode=exitcode, + message=message, + ) - with Scenario("I alter row policy on cluster after table", requirements=[ + with Scenario( + "I alter row policy on cluster after table", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Alter_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Alter_On("1.0"), + ], + ): try: with Given("I have a row policy"): - node.query("CREATE ROW POLICY policy14 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "CREATE ROW POLICY policy14 ON default.foo ON CLUSTER sharded_cluster" + ) with When("I run create row policy command"): - node.query("ALTER ROW POLICY policy14 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "ALTER ROW POLICY policy14 ON default.foo ON CLUSTER sharded_cluster" + ) finally: with Finally("I drop the row policy"): - node.query("DROP ROW POLICY IF EXISTS policy14 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "DROP ROW POLICY IF EXISTS policy14 ON default.foo ON CLUSTER sharded_cluster" + ) finally: with Finally("I drop the table and the roles"): node.query(f"DROP TABLE IF EXISTS default.foo") - node.query(f"DROP ROLE IF EXISTS role0, role1") \ No newline at end of file + node.query(f"DROP ROLE IF EXISTS role0, role1") diff --git a/tests/testflows/rbac/tests/syntax/alter_settings_profile.py b/tests/testflows/rbac/tests/syntax/alter_settings_profile.py index 4533f6aea65..bd78a76bd51 100755 --- a/tests/testflows/rbac/tests/syntax/alter_settings_profile.py +++ b/tests/testflows/rbac/tests/syntax/alter_settings_profile.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("alter settings profile") @Args(format_description=False) @@ -31,28 +32,49 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE USER user0") node.query(f"CREATE ROLE role0") - with Scenario("I alter settings profile with no options", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter("1.0")]): + with Scenario( + "I alter settings profile with no options", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter("1.0")], + ): with When("I alter settings profile"): node.query("ALTER SETTINGS PROFILE profile0") - with Scenario("I alter settings profile short form", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter("1.0")]): + with Scenario( + "I alter settings profile short form", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter("1.0")], + ): with When("I short form alter settings profile"): node.query("ALTER PROFILE profile0") - with Scenario("I alter settings profile that does not exist, throws exception", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter("1.0")]): + with Scenario( + "I alter settings profile that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter("1.0")], + ): profile = "profile1" cleanup_profile(profile) with When(f"I alter settings profile {profile} that doesn't exist"): - exitcode, message = errors.settings_profile_not_found_in_disk(name=profile) - node.query(f"ALTER SETTINGS PROFILE {profile}", exitcode=exitcode, message=message) + exitcode, message = errors.settings_profile_not_found_in_disk( + name=profile + ) + node.query( + f"ALTER SETTINGS PROFILE {profile}", + exitcode=exitcode, + message=message, + ) del profile - with Scenario("I alter settings profile if exists", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_IfExists("1.0")]): + with Scenario( + "I alter settings profile if exists", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_IfExists("1.0")], + ): with When("I alter settings profile using if exists"): node.query("ALTER SETTINGS PROFILE IF EXISTS profile0") - with Scenario("I alter settings profile if exists, profile does not exist", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_IfExists("1.0")]): + with Scenario( + "I alter settings profile if exists, profile does not exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_IfExists("1.0")], + ): profile = "profile1" cleanup_profile(profile) @@ -61,11 +83,17 @@ def feature(self, node="clickhouse1"): del profile - with Scenario("I alter settings profile to rename, target available", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Rename("1.0")]): + with Scenario( + "I alter settings profile to rename, target available", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Rename("1.0")], + ): with When("I alter settings profile by renaming it"): node.query("ALTER SETTINGS PROFILE profile0 RENAME TO profile0") - with Scenario("I alter settings profile to rename, target unavailable", requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Rename("1.0")]): + with Scenario( + "I alter settings profile to rename, target unavailable", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Rename("1.0")], + ): new_profile = "profile1" try: @@ -73,157 +101,293 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE SETTINGS PROFILE IF NOT EXISTS {new_profile}") with When(f"I try to rename to {new_profile}"): - exitcode, message = errors.cannot_rename_settings_profile(name="profile0", name_new=new_profile) - node.query(f"ALTER SETTINGS PROFILE profile0 RENAME TO {new_profile}", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_rename_settings_profile( + name="profile0", name_new=new_profile + ) + node.query( + f"ALTER SETTINGS PROFILE profile0 RENAME TO {new_profile}", + exitcode=exitcode, + message=message, + ) finally: with Finally(f"I cleanup target name {new_profile}"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {new_profile}") del new_profile - with Scenario("I alter settings profile with a setting value", requirements=[ + with Scenario( + "I alter settings profile with a setting value", + requirements=[ RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables("1.0"), - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0")]): + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0"), + ], + ): with When("I alter settings profile using settings"): - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage = 100000001") + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage = 100000001" + ) - with Scenario("I alter settings profile with a setting value, does not exist, throws exception", requirements=[ + with Scenario( + "I alter settings profile with a setting value, does not exist, throws exception", + requirements=[ RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables("1.0"), - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0")]): + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0"), + ], + ): with When("I alter settings profile using settings and nonexistent value"): exitcode, message = errors.unknown_setting("fake_setting") - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS fake_setting = 100000001", exitcode=exitcode, message=message) + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS fake_setting = 100000001", + exitcode=exitcode, + message=message, + ) - with Scenario("I alter settings profile with a min setting value", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0")]): + with Scenario( + "I alter settings profile with a min setting value", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0") + ], + ): with When("I alter settings profile using 2 minimum formats"): - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MIN 100000001") - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MIN = 100000001") + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MIN 100000001" + ) + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MIN = 100000001" + ) - with Scenario("I alter settings profile with a max setting value", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0")]): + with Scenario( + "I alter settings profile with a max setting value", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0") + ], + ): with When("I alter settings profile using 2 maximum formats"): - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MAX 100000001") - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MAX = 100000001") + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MAX 100000001" + ) + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MAX = 100000001" + ) - with Scenario("I alter settings profile with min and max setting values", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0")]): + with Scenario( + "I alter settings profile with min and max setting values", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0") + ], + ): with When("I alter settings profile with both min and max"): - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MIN 100000001 MAX 200000001") + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage MIN 100000001 MAX 200000001" + ) - with Scenario("I alter settings profile with a readonly setting", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0")]): + with Scenario( + "I alter settings profile with a readonly setting", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0") + ], + ): with When("I alter settings profile with with readonly"): - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage READONLY") + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage READONLY" + ) - with Scenario("I alter settings profile with a writable setting", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0")]): + with Scenario( + "I alter settings profile with a writable setting", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Constraints("1.0") + ], + ): with When("I alter settings profile with writable"): - node.query("ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage WRITABLE") + node.query( + "ALTER SETTINGS PROFILE profile0 SETTINGS max_memory_usage WRITABLE" + ) - with Scenario("I alter settings profile with inherited settings", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_Inherit("1.0")]): + with Scenario( + "I alter settings profile with inherited settings", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_Inherit("1.0") + ], + ): with When("I alter settings profile with inherit"): node.query("ALTER SETTINGS PROFILE profile0 SETTINGS INHERIT 'default'") - with Scenario("I alter settings profile with inherit, parent profile does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_Inherit("1.0")]): + with Scenario( + "I alter settings profile with inherit, parent profile does not exist, throws exception", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_Inherit("1.0") + ], + ): profile = "profile3" with Given(f"I ensure that profile {profile} does not exist"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") with When("I alter settings profile inherit from nonexistant parent"): exitcode, message = errors.settings_profile_not_found_in_disk(profile) - node.query(f"ALTER PROFILE profile0 SETTINGS INHERIT {profile}", exitcode=exitcode, message=message) + node.query( + f"ALTER PROFILE profile0 SETTINGS INHERIT {profile}", + exitcode=exitcode, + message=message, + ) del profile - with Scenario("I alter settings profile with multiple settings", requirements=[ + with Scenario( + "I alter settings profile with multiple settings", + requirements=[ RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables("1.0"), - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0")]): + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0"), + ], + ): with When("I alter settings profile with multiple settings"): - node.query("ALTER SETTINGS PROFILE profile0" - " SETTINGS max_memory_usage = 100000001" - " SETTINGS max_memory_usage_for_user = 100000001") + node.query( + "ALTER SETTINGS PROFILE profile0" + " SETTINGS max_memory_usage = 100000001" + " SETTINGS max_memory_usage_for_user = 100000001" + ) - with Scenario("I alter settings profile with multiple settings short form", requirements=[ + with Scenario( + "I alter settings profile with multiple settings short form", + requirements=[ RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables("1.0"), - RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0")]): + RQ_SRS_006_RBAC_SettingsProfile_Alter_Variables_Value("1.0"), + ], + ): with When("I alter settings profile with short form multiple settings"): - node.query("ALTER SETTINGS PROFILE profile0" - " SETTINGS max_memory_usage = 100000001," - " max_memory_usage_for_user = 100000001") + node.query( + "ALTER SETTINGS PROFILE profile0" + " SETTINGS max_memory_usage = 100000001," + " max_memory_usage_for_user = 100000001" + ) - with Scenario("I alter settings profile assigned to one role", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")]): + with Scenario( + "I alter settings profile assigned to one role", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")], + ): with When("I alter settings profile with assignment to role"): node.query("ALTER SETTINGS PROFILE profile0 TO role0") - with Scenario("I alter settings profile to assign to role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")]): + with Scenario( + "I alter settings profile to assign to role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I alter a settings profile, assign to role {role}, which does not exist"): + with Then( + f"I alter a settings profile, assign to role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"ALTER SETTINGS PROFILE profile0 TO {role}", exitcode=exitcode, message=message) + node.query( + f"ALTER SETTINGS PROFILE profile0 TO {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter settings profile to assign to all except role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")]): + with Scenario( + "I alter settings profile to assign to all except role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I alter a settings profile, assign to all except role {role}, which does not exist"): + with Then( + f"I alter a settings profile, assign to all except role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"ALTER SETTINGS PROFILE profile0 TO ALL EXCEPT {role}", exitcode=exitcode, message=message) + node.query( + f"ALTER SETTINGS PROFILE profile0 TO ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter settings profile assigned to multiple roles", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")]): + with Scenario( + "I alter settings profile assigned to multiple roles", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment("1.0")], + ): with When("I alter settings profile with assignment to multiple roles"): node.query("ALTER SETTINGS PROFILE profile0 TO role0, user0") - with Scenario("I alter settings profile assigned to all", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_All("1.0")]): + with Scenario( + "I alter settings profile assigned to all", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_All("1.0")], + ): with When("I alter settings profile with assignment to all"): node.query("ALTER SETTINGS PROFILE profile0 TO ALL") - with Scenario("I alter settings profile assigned to all except one role", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_AllExcept("1.0")]): + with Scenario( + "I alter settings profile assigned to all except one role", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_AllExcept("1.0") + ], + ): with When("I alter settings profile with assignment to all except a role"): node.query("ALTER SETTINGS PROFILE profile0 TO ALL EXCEPT role0") - with Scenario("I alter settings profile assigned to all except multiple roles", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_AllExcept("1.0")]): - with When("I alter settings profile with assignmentto all except multiple roles"): + with Scenario( + "I alter settings profile assigned to all except multiple roles", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_AllExcept("1.0") + ], + ): + with When( + "I alter settings profile with assignmentto all except multiple roles" + ): node.query("ALTER SETTINGS PROFILE profile0 TO ALL EXCEPT role0, user0") - with Scenario("I alter settings profile assigned to none", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_None("1.0")]): + with Scenario( + "I alter settings profile assigned to none", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_None("1.0")], + ): with When("I alter settings profile with assignment to none"): node.query("ALTER SETTINGS PROFILE profile0 TO NONE") - with Scenario("I alter settings profile on cluster", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_OnCluster("1.0")]): + with Scenario( + "I alter settings profile on cluster", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_OnCluster("1.0") + ], + ): try: with Given("I have a settings profile on cluster"): - node.query("CREATE SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster") + node.query( + "CREATE SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster" + ) with When("I run alter settings profile command"): - node.query("ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster") + node.query( + "ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster" + ) with And("I alter settings profile with settings"): - node.query("ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster SETTINGS max_memory_usage = 100000001") + node.query( + "ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster SETTINGS max_memory_usage = 100000001" + ) with And("I alter settings profile with inherit"): - node.query("ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster SETTINGS INHERIT 'default'") + node.query( + "ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster SETTINGS INHERIT 'default'" + ) with And("I alter settings profile to all"): - node.query("ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster TO ALL") + node.query( + "ALTER SETTINGS PROFILE profile1 ON CLUSTER sharded_cluster TO ALL" + ) finally: with Finally("I drop the settings profile"): - node.query("DROP SETTINGS PROFILE IF EXISTS profile1 ON CLUSTER sharded_cluster") + node.query( + "DROP SETTINGS PROFILE IF EXISTS profile1 ON CLUSTER sharded_cluster" + ) - with Scenario("I alter settings profile on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_OnCluster("1.0")]): + with Scenario( + "I alter settings profile on fake cluster, throws exception", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Alter_Assignment_OnCluster("1.0") + ], + ): with When("I run alter settings profile command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("ALTER SETTINGS PROFILE profile1 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "ALTER SETTINGS PROFILE profile1 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the profile and all the users and roles"): diff --git a/tests/testflows/rbac/tests/syntax/alter_user.py b/tests/testflows/rbac/tests/syntax/alter_user.py index cf8a13008c9..d022176a598 100755 --- a/tests/testflows/rbac/tests/syntax/alter_user.py +++ b/tests/testflows/rbac/tests/syntax/alter_user.py @@ -6,6 +6,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("alter user") @Args(format_description=False) @@ -33,26 +34,33 @@ def feature(self, node="clickhouse1"): with Finally("I drop the user"): node.query(f"DROP USER IF EXISTS {user}") - with Scenario("I alter user, base command", requirements=[ - RQ_SRS_006_RBAC_User_Alter("1.0")]): + with Scenario( + "I alter user, base command", requirements=[RQ_SRS_006_RBAC_User_Alter("1.0")] + ): with setup("user0"): with When("I alter user"): node.query("ALTER USER user0") - with Scenario("I alter user that does not exist without if exists, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Alter("1.0")]): + with Scenario( + "I alter user that does not exist without if exists, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Alter("1.0")], + ): with When("I run alter user command, expecting error 192"): exitcode, message = errors.user_not_found_in_disk(name="user0") - node.query(f"ALTER USER user0",exitcode=exitcode, message=message) + node.query(f"ALTER USER user0", exitcode=exitcode, message=message) - with Scenario("I alter user with if exists", requirements=[ - RQ_SRS_006_RBAC_User_Alter_IfExists("1.0")]): + with Scenario( + "I alter user with if exists", + requirements=[RQ_SRS_006_RBAC_User_Alter_IfExists("1.0")], + ): with setup("user0"): with When(f"I alter user with if exists"): node.query(f"ALTER USER IF EXISTS user0") - with Scenario("I alter user that does not exist with if exists", requirements=[ - RQ_SRS_006_RBAC_User_Alter_IfExists("1.0")]): + with Scenario( + "I alter user that does not exist with if exists", + requirements=[RQ_SRS_006_RBAC_User_Alter_IfExists("1.0")], + ): user = "user0" with Given("I don't have a user"): node.query(f"DROP USER IF EXISTS {user}") @@ -60,8 +68,10 @@ def feature(self, node="clickhouse1"): node.query(f"ALTER USER IF EXISTS {user}") del user - with Scenario("I alter user on a cluster", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Cluster("1.0")]): + with Scenario( + "I alter user on a cluster", + requirements=[RQ_SRS_006_RBAC_User_Alter_Cluster("1.0")], + ): with Given("I have a user on a cluster"): node.query("CREATE USER OR REPLACE user0 ON CLUSTER sharded_cluster") with When("I alter user on a cluster"): @@ -69,106 +79,160 @@ def feature(self, node="clickhouse1"): with Finally("I drop user from cluster"): node.query("DROP USER IF EXISTS user0 ON CLUSTER sharded_cluster") - with Scenario("I alter user on a fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Cluster("1.0")]): + with Scenario( + "I alter user on a fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Alter_Cluster("1.0")], + ): with When("I alter user on a fake cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("ALTER USER user0 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "ALTER USER user0 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) - with Scenario("I alter user to rename, target available", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Rename("1.0")]): + with Scenario( + "I alter user to rename, target available", + requirements=[RQ_SRS_006_RBAC_User_Alter_Rename("1.0")], + ): with setup("user15"): with When("I alter user name"): node.query("ALTER USER user15 RENAME TO user15") - with Scenario("I alter user to rename, target unavailable", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Rename("1.0")]): + with Scenario( + "I alter user to rename, target unavailable", + requirements=[RQ_SRS_006_RBAC_User_Alter_Rename("1.0")], + ): with setup("user15"): new_user = "user16" try: with Given(f"Ensure target name {new_user} is NOT available"): node.query(f"CREATE USER IF NOT EXISTS {new_user}") with When(f"I try to rename to {new_user}"): - exitcode, message = errors.cannot_rename_user(name="user15", name_new=new_user) - node.query(f"ALTER USER user15 RENAME TO {new_user}", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_rename_user( + name="user15", name_new=new_user + ) + node.query( + f"ALTER USER user15 RENAME TO {new_user}", + exitcode=exitcode, + message=message, + ) finally: with Finally(f"I cleanup target name {new_user}"): node.query(f"DROP USER IF EXISTS {new_user}") del new_user - with Scenario("I alter user password plaintext password", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Password_PlainText("1.0")]): + with Scenario( + "I alter user password plaintext password", + requirements=[RQ_SRS_006_RBAC_User_Alter_Password_PlainText("1.0")], + ): with setup("user1"): with When("I alter user with plaintext password"): - node.query("ALTER USER user1 IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'mypassword'", step=When) + node.query( + "ALTER USER user1 IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'mypassword'", + step=When, + ) - with Scenario("I alter user password to sha256", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Password_Sha256Password("1.0")]): + with Scenario( + "I alter user password to sha256", + requirements=[RQ_SRS_006_RBAC_User_Alter_Password_Sha256Password("1.0")], + ): with setup("user2"): with When("I alter user with sha256_password"): password = hashlib.sha256("mypassword".encode("utf-8")).hexdigest() - node.query(f"ALTER USER user2 IDENTIFIED WITH SHA256_PASSWORD BY '{password}'",step=When) + node.query( + f"ALTER USER user2 IDENTIFIED WITH SHA256_PASSWORD BY '{password}'", + step=When, + ) - with Scenario("I alter user password to double_sha1_password", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Password_DoubleSha1Password("1.0")]): + with Scenario( + "I alter user password to double_sha1_password", + requirements=[RQ_SRS_006_RBAC_User_Alter_Password_DoubleSha1Password("1.0")], + ): with setup("user3"): with When("I alter user with double_sha1_password"): + def hash(password): return hashlib.sha1(password.encode("utf-8")).hexdigest() - password = hash(hash("mypassword")) - node.query(f"ALTER USER user3 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY '{password}'", step=When) - with Scenario("I alter user host local", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_Local("1.0")]): + password = hash(hash("mypassword")) + node.query( + f"ALTER USER user3 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY '{password}'", + step=When, + ) + + with Scenario( + "I alter user host local", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_Local("1.0")], + ): with setup("user4"): with When("I alter user with host local"): node.query("ALTER USER user4 HOST LOCAL") - with Scenario("I alter user host name", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_Name("1.0")]): + with Scenario( + "I alter user host name", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_Name("1.0")], + ): with setup("user5"): with When("I alter user with host name"): - node.query("ALTER USER user5 HOST NAME 'localhost', NAME 'clickhouse.com'") + node.query( + "ALTER USER user5 HOST NAME 'localhost', NAME 'clickhouse.com'" + ) - with Scenario("I alter user host regexp", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_Regexp("1.0")]): + with Scenario( + "I alter user host regexp", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_Regexp("1.0")], + ): with setup("user6"): with When("I alter user with host regexp"): node.query("ALTER USER user6 HOST REGEXP 'lo..*host', 'lo*host'") - with Scenario("I alter user host ip", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_IP("1.0")]): + with Scenario( + "I alter user host ip", requirements=[RQ_SRS_006_RBAC_User_Alter_Host_IP("1.0")] + ): with setup("user7"): with When("I alter user with host ip"): node.query("ALTER USER user7 HOST IP '127.0.0.1', IP '127.0.0.2'") - with Scenario("I alter user host like", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_Like("1.0")]): + with Scenario( + "I alter user host like", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_Like("1.0")], + ): with setup("user8"): with When("I alter user with host like"): node.query("ALTER USER user8 HOST LIKE '%.clickhouse.com'") - with Scenario("I alter user host any", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_Any("1.0")]): + with Scenario( + "I alter user host any", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_Any("1.0")], + ): with setup("user9"): with When("I alter user with host any"): node.query("ALTER USER user9 HOST ANY") - with Scenario("I alter user host many hosts", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_Like("1.0")]): + with Scenario( + "I alter user host many hosts", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_Like("1.0")], + ): with setup("user11"): with When("I alter user with multiple hosts"): - node.query("ALTER USER user11 HOST LIKE '%.clickhouse.com', \ - IP '127.0.0.2', NAME 'localhost', REGEXP 'lo*host'") + node.query( + "ALTER USER user11 HOST LIKE '%.clickhouse.com', \ + IP '127.0.0.2', NAME 'localhost', REGEXP 'lo*host'" + ) - with Scenario("I alter user default role set to none", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_None("1.0")]): + with Scenario( + "I alter user default role set to none", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_None("1.0")], + ): with setup("user12"): with When("I alter user with default role none"): node.query("ALTER USER user12 DEFAULT ROLE NONE") - with Scenario("I alter user default role set to all", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole_All("1.0")]): + with Scenario( + "I alter user default role set to all", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole_All("1.0")], + ): with setup("user13"): with When("I alter user with all roles set to default"): node.query("ALTER USER user13 DEFAULT ROLE ALL") @@ -183,120 +247,178 @@ def feature(self, node="clickhouse1"): with Finally(f"I drop the role {role}", flags=TE): node.query(f"DROP ROLE IF EXISTS {role}") - with Scenario("I alter user default role", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")]): + with Scenario( + "I alter user default role", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")], + ): with setup("user14"), setup_role("role2"): with Given("I have a user with a role"): node.query("GRANT role2 TO user14") with When("I alter user default role"): node.query("ALTER USER user14 DEFAULT ROLE role2") - with Scenario("I alter user default role, setting default role", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")]): + with Scenario( + "I alter user default role, setting default role", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")], + ): with setup("user14a"), setup_role("default"): with Given("I grant default role to the user"): node.query("GRANT default TO user14a") with When("I alter user default role"): node.query("ALTER USER user14a DEFAULT ROLE default") - with Scenario("I alter user default role, role doesn't exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")]): + with Scenario( + "I alter user default role, role doesn't exist, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")], + ): with setup("user12"): role = "role0" with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") with When(f"I alter user with default role {role}"): exitcode, message = errors.role_not_found_in_disk(role) - node.query(f"ALTER USER user12 DEFAULT ROLE {role}",exitcode=exitcode, message=message) + node.query( + f"ALTER USER user12 DEFAULT ROLE {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter user default role, all except role doesn't exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")]): + with Scenario( + "I alter user default role, all except role doesn't exist, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")], + ): with setup("user12"): role = "role0" with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") with When(f"I alter user with default role {role}"): exitcode, message = errors.role_not_found_in_disk(role) - node.query(f"ALTER USER user12 DEFAULT ROLE ALL EXCEPT {role}",exitcode=exitcode, message=message) + node.query( + f"ALTER USER user12 DEFAULT ROLE ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I alter user default role multiple", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")]): + with Scenario( + "I alter user default role multiple", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole("1.0")], + ): with setup("user15"), setup_role("second"), setup_role("third"): with Given("I have a user with multiple roles"): node.query("GRANT second,third TO user15") with When("I alter user default role to second, third"): node.query("ALTER USER user15 DEFAULT ROLE second, third") - with Scenario("I alter user default role set to all except", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole_AllExcept("1.0")]): + with Scenario( + "I alter user default role set to all except", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole_AllExcept("1.0")], + ): with setup("user16"), setup_role("second"): with Given("I have a user with a role"): node.query("GRANT second TO user16") with When("I alter user default role"): node.query("ALTER USER user16 DEFAULT ROLE ALL EXCEPT second") - with Scenario("I alter user default role multiple all except", requirements=[ - RQ_SRS_006_RBAC_User_Alter_DefaultRole_AllExcept("1.0")]): + with Scenario( + "I alter user default role multiple all except", + requirements=[RQ_SRS_006_RBAC_User_Alter_DefaultRole_AllExcept("1.0")], + ): with setup("user17"), setup_role("second"), setup_role("third"): with Given("I have a user with multiple roles"): node.query("GRANT second,third TO user17") with When("I alter user default role to all except second"): node.query("ALTER USER user17 DEFAULT ROLE ALL EXCEPT second") - with Scenario("I alter user settings profile", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Settings("1.0"), \ - RQ_SRS_006_RBAC_User_Alter_Settings_Profile("1.0")]): + with Scenario( + "I alter user settings profile", + requirements=[ + RQ_SRS_006_RBAC_User_Alter_Settings("1.0"), + RQ_SRS_006_RBAC_User_Alter_Settings_Profile("1.0"), + ], + ): with setup("user18"): try: with Given("I have a profile"): node.query(f"CREATE SETTINGS PROFILE profile10") with When("I alter user with settings and set profile to profile1"): - node.query("ALTER USER user18 SETTINGS PROFILE profile10, max_memory_usage = 100 MIN 0 MAX 1000 READONLY") + node.query( + "ALTER USER user18 SETTINGS PROFILE profile10, max_memory_usage = 100 MIN 0 MAX 1000 READONLY" + ) finally: with Finally("I drop the profile"): node.query(f"DROP SETTINGS PROFILE profile10") - with Scenario("I alter user settings profile, fake profile, throws exception", requirements=[ + with Scenario( + "I alter user settings profile, fake profile, throws exception", + requirements=[ RQ_SRS_006_RBAC_User_Alter_Settings("1.0"), - RQ_SRS_006_RBAC_User_Alter_Settings_Profile("1.0")]): + RQ_SRS_006_RBAC_User_Alter_Settings_Profile("1.0"), + ], + ): with setup("user18a"): profile = "profile0" with Given(f"I ensure that profile {profile} does not exist"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") - with When(f"I alter user with Settings and set profile to fake profile {profile}"): + with When( + f"I alter user with Settings and set profile to fake profile {profile}" + ): exitcode, message = errors.settings_profile_not_found_in_disk(profile) - node.query("ALTER USER user18a SETTINGS PROFILE profile0", exitcode=exitcode, message=message) + node.query( + "ALTER USER user18a SETTINGS PROFILE profile0", + exitcode=exitcode, + message=message, + ) del profile - with Scenario("I alter user settings with a fake setting, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Settings("1.0")]): - with setup("user18b"): - with When("I alter settings profile using settings and nonexistent value"): - exitcode, message = errors.unknown_setting("fake_setting") - node.query("ALTER USER user18b SETTINGS fake_setting = 100000001", exitcode=exitcode, message=message) + with Scenario( + "I alter user settings with a fake setting, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Alter_Settings("1.0")], + ): + with setup("user18b"): + with When("I alter settings profile using settings and nonexistent value"): + exitcode, message = errors.unknown_setting("fake_setting") + node.query( + "ALTER USER user18b SETTINGS fake_setting = 100000001", + exitcode=exitcode, + message=message, + ) - with Scenario("I alter user settings without profile (no equals)", requirements=[ + with Scenario( + "I alter user settings without profile (no equals)", + requirements=[ RQ_SRS_006_RBAC_User_Alter_Settings("1.0"), RQ_SRS_006_RBAC_User_Alter_Settings_Min("1.0"), - RQ_SRS_006_RBAC_User_Alter_Settings_Max("1.0")]): + RQ_SRS_006_RBAC_User_Alter_Settings_Max("1.0"), + ], + ): with setup("user19"): with When("I alter user with settings without profile using no equals"): - node.query("ALTER USER user19 SETTINGS max_memory_usage=10000000 MIN 100000 MAX 1000000000 READONLY") + node.query( + "ALTER USER user19 SETTINGS max_memory_usage=10000000 MIN 100000 MAX 1000000000 READONLY" + ) - #equals sign (=) syntax verify - with Scenario("I alter user settings without profile (yes equals)", requirements=[ + # equals sign (=) syntax verify + with Scenario( + "I alter user settings without profile (yes equals)", + requirements=[ RQ_SRS_006_RBAC_User_Alter_Settings("1.0"), RQ_SRS_006_RBAC_User_Alter_Settings_Min("1.0"), - RQ_SRS_006_RBAC_User_Alter_Settings_Max("1.0")]): + RQ_SRS_006_RBAC_User_Alter_Settings_Max("1.0"), + ], + ): with setup("user20"): with When("I alter user with settings without profile using equals"): - node.query("ALTER USER user20 SETTINGS max_memory_usage=10000000 MIN=100000 MAX=1000000000 READONLY") + node.query( + "ALTER USER user20 SETTINGS max_memory_usage=10000000 MIN=100000 MAX=1000000000 READONLY" + ) - #Add requirement to host: add/drop - with Scenario("I alter user to add host", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_AddDrop("1.0")]): + # Add requirement to host: add/drop + with Scenario( + "I alter user to add host", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_AddDrop("1.0")], + ): with setup("user21"): with When("I alter user by adding local host"): node.query("ALTER USER user21 ADD HOST LOCAL") @@ -309,8 +431,10 @@ def feature(self, node="clickhouse1"): with And("I alter user by adding host name"): node.query("ALTER USER user21 ADD HOST NAME 'localhost'") - with Scenario("I alter user to remove host", requirements=[ - RQ_SRS_006_RBAC_User_Alter_Host_AddDrop("1.0")]): + with Scenario( + "I alter user to remove host", + requirements=[RQ_SRS_006_RBAC_User_Alter_Host_AddDrop("1.0")], + ): with setup("user22"): with When("I alter user by removing local host"): node.query("ALTER USER user22 DROP HOST LOCAL") diff --git a/tests/testflows/rbac/tests/syntax/create_quota.py b/tests/testflows/rbac/tests/syntax/create_quota.py index 33dbbf9c153..8301d918c8c 100755 --- a/tests/testflows/rbac/tests/syntax/create_quota.py +++ b/tests/testflows/rbac/tests/syntax/create_quota.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("create quota") @Args(format_description=False) @@ -34,39 +35,51 @@ def feature(self, node="clickhouse1"): def create_quota(quota): with And(f"I ensure I do have quota {quota}"): - node.query(f"CREATE QUOTA OR REPLACE {quota}") + node.query(f"CREATE QUOTA OR REPLACE {quota}") try: with Given("I have a user and a role"): node.query(f"CREATE USER user0") node.query(f"CREATE ROLE role0") - with Scenario("I create quota with no options", requirements=[ - RQ_SRS_006_RBAC_Quota_Create("1.0")]): + with Scenario( + "I create quota with no options", + requirements=[RQ_SRS_006_RBAC_Quota_Create("1.0")], + ): with cleanup("quota0"): with When("I create a quota with no options"): node.query("CREATE QUOTA quota0") - with Scenario("I create quota that already exists, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Create("1.0")]): + with Scenario( + "I create quota that already exists, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Create("1.0")], + ): quota = "quota0" with cleanup(quota): create_quota(quota) - with When(f"I create a quota {quota} that already exists without IF EXISTS, throws exception"): + with When( + f"I create a quota {quota} that already exists without IF EXISTS, throws exception" + ): exitcode, message = errors.cannot_insert_quota(name=quota) - node.query(f"CREATE QUOTA {quota}", exitcode=exitcode, message=message) + node.query( + f"CREATE QUOTA {quota}", exitcode=exitcode, message=message + ) del quota - with Scenario("I create quota if not exists, quota does not exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_IfNotExists("1.0")]): + with Scenario( + "I create quota if not exists, quota does not exist", + requirements=[RQ_SRS_006_RBAC_Quota_Create_IfNotExists("1.0")], + ): quota = "quota1" with cleanup(quota): with When(f"I create a quota {quota} with if not exists"): node.query(f"CREATE QUOTA IF NOT EXISTS {quota}") del quota - with Scenario("I create quota if not exists, quota does exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_IfNotExists("1.0")]): + with Scenario( + "I create quota if not exists, quota does exist", + requirements=[RQ_SRS_006_RBAC_Quota_Create_IfNotExists("1.0")], + ): quota = "quota1" with cleanup(quota): create_quota(quota) @@ -74,16 +87,20 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE QUOTA IF NOT EXISTS {quota}") del quota - with Scenario("I create quota or replace, quota does not exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Replace("1.0")]): + with Scenario( + "I create quota or replace, quota does not exist", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Replace("1.0")], + ): quota = "quota2" with cleanup(quota): with When(f"I create a quota {quota} with or replace"): node.query(f"CREATE QUOTA OR REPLACE {quota}") del quota - with Scenario("I create quota or replace, quota does exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Replace("1.0")]): + with Scenario( + "I create quota or replace, quota does exist", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Replace("1.0")], + ): quota = "quota2" with cleanup(quota): create_quota(quota) @@ -91,36 +108,65 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE QUOTA OR REPLACE {quota}") del quota - keys = ['none', 'user name', 'ip address', 'client key', 'client key or user name', 'client key or ip address'] + keys = [ + "none", + "user name", + "ip address", + "client key", + "client key or user name", + "client key or ip address", + ] for i, key in enumerate(keys): - with Scenario(f"I create quota keyed by {key}", requirements=[ + with Scenario( + f"I create quota keyed by {key}", + requirements=[ RQ_SRS_006_RBAC_Quota_Create_KeyedBy("1.0"), - RQ_SRS_006_RBAC_Quota_Create_KeyedByOptions("1.0")]): - name = f'quota{3 + i}' + RQ_SRS_006_RBAC_Quota_Create_KeyedByOptions("1.0"), + ], + ): + name = f"quota{3 + i}" with cleanup(name): with When(f"I create a quota with {key}"): node.query(f"CREATE QUOTA {name} KEYED BY '{key}'") - with Scenario("I create quota for randomized interval", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Interval_Randomized("1.0")]): + with Scenario( + "I create quota for randomized interval", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Interval_Randomized("1.0")], + ): with cleanup("quota9"): with When("I create a quota for randomized interval"): - node.query("CREATE QUOTA quota9 FOR RANDOMIZED INTERVAL 1 DAY NO LIMITS") + node.query( + "CREATE QUOTA quota9 FOR RANDOMIZED INTERVAL 1 DAY NO LIMITS" + ) - intervals = ['SECOND', 'MINUTE', 'HOUR', 'DAY', 'MONTH'] + intervals = ["SECOND", "MINUTE", "HOUR", "DAY", "MONTH"] for i, interval in enumerate(intervals): - with Scenario(f"I create quota for interval {interval}", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Interval("1.0")]): - name = f'quota{10 + i}' + with Scenario( + f"I create quota for interval {interval}", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Interval("1.0")], + ): + name = f"quota{10 + i}" with cleanup(name): with When(f"I create a quota for {interval} interval"): - node.query(f"CREATE QUOTA {name} FOR INTERVAL 1 {interval} NO LIMITS") + node.query( + f"CREATE QUOTA {name} FOR INTERVAL 1 {interval} NO LIMITS" + ) - constraints = ['MAX QUERIES', 'MAX ERRORS', 'MAX RESULT ROWS', - 'MAX RESULT BYTES', 'MAX READ ROWS', 'MAX READ BYTES', 'MAX EXECUTION TIME', - 'NO LIMITS', 'TRACKING ONLY'] + constraints = [ + "MAX QUERIES", + "MAX ERRORS", + "MAX RESULT ROWS", + "MAX RESULT BYTES", + "MAX READ ROWS", + "MAX READ BYTES", + "MAX EXECUTION TIME", + "NO LIMITS", + "TRACKING ONLY", + ] for i, constraint in enumerate(constraints): - with Scenario(f"I create quota for {constraint.lower()}", requirements=[ + with Scenario( + f"I create quota for {constraint.lower()}", + requirements=[ RQ_SRS_006_RBAC_Quota_Create_Queries("1.0"), RQ_SRS_006_RBAC_Quota_Create_Errors("1.0"), RQ_SRS_006_RBAC_Quota_Create_ResultRows("1.0"), @@ -129,99 +175,149 @@ def feature(self, node="clickhouse1"): RQ_SRS_006_RBAC_Quota_Create_ReadBytes("1.0"), RQ_SRS_006_RBAC_Quota_Create_ExecutionTime("1.0"), RQ_SRS_006_RBAC_Quota_Create_NoLimits("1.0"), - RQ_SRS_006_RBAC_Quota_Create_TrackingOnly("1.0")]): - name = f'quota{15 + i}' + RQ_SRS_006_RBAC_Quota_Create_TrackingOnly("1.0"), + ], + ): + name = f"quota{15 + i}" with cleanup(name): with When(f"I create quota for {constraint.lower()}"): - node.query(f"CREATE QUOTA {name} FOR INTERVAL 1 DAY {constraint}{' 1024' if constraint.startswith('MAX') else ''}") + node.query( + f"CREATE QUOTA {name} FOR INTERVAL 1 DAY {constraint}{' 1024' if constraint.startswith('MAX') else ''}" + ) - with Scenario("I create quota for multiple constraints", requirements=[ + with Scenario( + "I create quota for multiple constraints", + requirements=[ RQ_SRS_006_RBAC_Quota_Create_Interval("1.0"), - RQ_SRS_006_RBAC_Quota_Create_Queries("1.0")]): + RQ_SRS_006_RBAC_Quota_Create_Queries("1.0"), + ], + ): with cleanup("quota23"): with When(f"I create quota for multiple constraints"): - node.query('CREATE QUOTA quota23 \ + node.query( + "CREATE QUOTA quota23 \ FOR INTERVAL 1 DAY NO LIMITS, \ FOR INTERVAL 2 DAY MAX QUERIES 124, \ - FOR INTERVAL 1 HOUR TRACKING ONLY') + FOR INTERVAL 1 HOUR TRACKING ONLY" + ) - with Scenario("I create quota assigned to one role", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")]): + with Scenario( + "I create quota assigned to one role", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")], + ): with cleanup("quota24"): with When("I create quota for role"): node.query("CREATE QUOTA quota24 TO role0") - with Scenario("I create quota to assign to role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")]): + with Scenario( + "I create quota to assign to role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") with Then(f"I create a quota, assign to role {role}, which does not exist"): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"CREATE QUOTA quota0 TO {role}", exitcode=exitcode, message=message) + node.query( + f"CREATE QUOTA quota0 TO {role}", exitcode=exitcode, message=message + ) del role - with Scenario("I create quota to assign to all except role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")]): + with Scenario( + "I create quota to assign to all except role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I create a quota, assign to all except role {role}, which does not exist"): + with Then( + f"I create a quota, assign to all except role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"CREATE QUOTA quota0 TO ALL EXCEPT {role}", exitcode=exitcode, message=message) + node.query( + f"CREATE QUOTA quota0 TO ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create quota assigned to no role", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment_None("1.0")]): + with Scenario( + "I create quota assigned to no role", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment_None("1.0")], + ): with When("I create quota for no role"): node.query("CREATE QUOTA quota24 TO NONE") - with Scenario("I create quota assigned to multiple roles", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")]): + with Scenario( + "I create quota assigned to multiple roles", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment("1.0")], + ): with cleanup("quota25"): with When("I create quota for multiple roles"): node.query("CREATE QUOTA quota25 TO role0, user0") - with Scenario("I create quota assigned to all", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment_All("1.0")]): + with Scenario( + "I create quota assigned to all", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment_All("1.0")], + ): with cleanup("quota26"): with When("I create quota for all"): node.query("CREATE QUOTA quota26 TO ALL") - with Scenario("I create quota assigned to all except one role", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment_Except("1.0")]): + with Scenario( + "I create quota assigned to all except one role", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment_Except("1.0")], + ): with cleanup("quota27"): with When("I create quota for all except one role"): node.query("CREATE QUOTA quota27 TO ALL EXCEPT role0") - with Scenario("I create quota assigned to all except multiple roles", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Assignment_Except("1.0")]): + with Scenario( + "I create quota assigned to all except multiple roles", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Assignment_Except("1.0")], + ): with cleanup("quota28"): with When("I create quota for all except multiple roles"): node.query("CREATE QUOTA quota28 TO ALL EXCEPT role0, user0") - with Scenario("I create quota on cluster", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Cluster("1.0")]): + with Scenario( + "I create quota on cluster", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Cluster("1.0")], + ): try: with When("I run create quota command on cluster"): node.query("CREATE QUOTA quota29 ON CLUSTER sharded_cluster") with When("I run create quota command on cluster, keyed"): - node.query("CREATE QUOTA OR REPLACE quota29 ON CLUSTER sharded_cluster KEYED BY 'none'") + node.query( + "CREATE QUOTA OR REPLACE quota29 ON CLUSTER sharded_cluster KEYED BY 'none'" + ) with When("I run create quota command on cluster, interval"): - node.query("CREATE QUOTA OR REPLACE quota29 ON CLUSTER sharded_cluster FOR INTERVAL 1 DAY TRACKING ONLY") + node.query( + "CREATE QUOTA OR REPLACE quota29 ON CLUSTER sharded_cluster FOR INTERVAL 1 DAY TRACKING ONLY" + ) with When("I run create quota command on cluster, assign"): - node.query("CREATE QUOTA OR REPLACE quota29 ON CLUSTER sharded_cluster TO ALL") + node.query( + "CREATE QUOTA OR REPLACE quota29 ON CLUSTER sharded_cluster TO ALL" + ) finally: with Finally("I drop the quota from cluster"): - node.query("DROP QUOTA IF EXISTS quota29 ON CLUSTER sharded_cluster") + node.query( + "DROP QUOTA IF EXISTS quota29 ON CLUSTER sharded_cluster" + ) - with Scenario("I create quota on nonexistent cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Create_Cluster("1.0")]): + with Scenario( + "I create quota on nonexistent cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Create_Cluster("1.0")], + ): with When("I run create quota on a cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("CREATE QUOTA quota0 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "CREATE QUOTA quota0 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop all the users and roles"): node.query(f"DROP USER IF EXISTS user0") - node.query(f"DROP ROLE IF EXISTS role0") \ No newline at end of file + node.query(f"DROP ROLE IF EXISTS role0") diff --git a/tests/testflows/rbac/tests/syntax/create_role.py b/tests/testflows/rbac/tests/syntax/create_role.py index 1cb10077570..993cdf822a5 100755 --- a/tests/testflows/rbac/tests/syntax/create_role.py +++ b/tests/testflows/rbac/tests/syntax/create_role.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("create role") @Args(format_description=False) @@ -30,16 +31,20 @@ def feature(self, node="clickhouse1"): def create_role(role): with Given(f"I ensure I do have role {role}"): - node.query(f"CREATE ROLE OR REPLACE {role}") + node.query(f"CREATE ROLE OR REPLACE {role}") - with Scenario("I create role with no options", requirements=[ - RQ_SRS_006_RBAC_Role_Create("1.0")]): + with Scenario( + "I create role with no options", + requirements=[RQ_SRS_006_RBAC_Role_Create("1.0")], + ): with cleanup("role0"): with When("I create role"): node.query("CREATE ROLE role0") - with Scenario("I create role that already exists, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Create("1.0")]): + with Scenario( + "I create role that already exists, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Create("1.0")], + ): role = "role0" with cleanup(role): with Given(f"I have role {role}"): @@ -49,16 +54,20 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE ROLE {role}", exitcode=exitcode, message=message) del role - with Scenario("I create role if not exists, role does not exist", requirements=[ - RQ_SRS_006_RBAC_Role_Create_IfNotExists("1.0")]): + with Scenario( + "I create role if not exists, role does not exist", + requirements=[RQ_SRS_006_RBAC_Role_Create_IfNotExists("1.0")], + ): role = "role1" with cleanup(role): with When(f"I create role {role} with if not exists"): node.query(f"CREATE ROLE IF NOT EXISTS {role}") del role - with Scenario("I create role if not exists, role does exist", requirements=[ - RQ_SRS_006_RBAC_Role_Create_IfNotExists("1.0")]): + with Scenario( + "I create role if not exists, role does exist", + requirements=[RQ_SRS_006_RBAC_Role_Create_IfNotExists("1.0")], + ): role = "role1" with cleanup(role): create_role(role) @@ -66,16 +75,20 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE ROLE IF NOT EXISTS {role}") del role - with Scenario("I create role or replace, role does not exist", requirements=[ - RQ_SRS_006_RBAC_Role_Create_Replace("1.0")]): + with Scenario( + "I create role or replace, role does not exist", + requirements=[RQ_SRS_006_RBAC_Role_Create_Replace("1.0")], + ): role = "role2" with cleanup(role): with When(f"I create role {role} with or replace"): node.query(f"CREATE ROLE OR REPLACE {role}") del role - with Scenario("I create role or replace, role does exist", requirements=[ - RQ_SRS_006_RBAC_Role_Create_Replace("1.0")]): + with Scenario( + "I create role or replace, role does exist", + requirements=[RQ_SRS_006_RBAC_Role_Create_Replace("1.0")], + ): role = "role2" with cleanup(role): create_role(role) @@ -83,42 +96,67 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE ROLE OR REPLACE {role}") del role - with Scenario("I create role on cluster", requirements=[ - RQ_SRS_006_RBAC_Role_Create("1.0")]): + with Scenario( + "I create role on cluster", requirements=[RQ_SRS_006_RBAC_Role_Create("1.0")] + ): try: with When("I have a role on a cluster"): node.query("CREATE ROLE role1 ON CLUSTER sharded_cluster") with And("I run create role or replace on a cluster"): node.query("CREATE ROLE OR REPLACE role1 ON CLUSTER sharded_cluster") with And("I create role with settings on a cluster"): - node.query("CREATE ROLE role2 ON CLUSTER sharded_cluster SETTINGS max_memory_usage=10000000 READONLY") + node.query( + "CREATE ROLE role2 ON CLUSTER sharded_cluster SETTINGS max_memory_usage=10000000 READONLY" + ) finally: with Finally("I drop the role"): node.query("DROP ROLE IF EXISTS role1,role2 ON CLUSTER sharded_cluster") - with Scenario("I create role on nonexistent cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Create("1.0")]): + with Scenario( + "I create role on nonexistent cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Create("1.0")], + ): with When("I run create role on a cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("CREATE ROLE role1 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "CREATE ROLE role1 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) - with Scenario("I create role with settings profile", requirements=[ - RQ_SRS_006_RBAC_Role_Create_Settings("1.0")]): + with Scenario( + "I create role with settings profile", + requirements=[RQ_SRS_006_RBAC_Role_Create_Settings("1.0")], + ): with cleanup("role3"): with When("I create role with settings profile"): - node.query("CREATE ROLE role3 SETTINGS PROFILE default, max_memory_usage=10000000 WRITABLE") + node.query( + "CREATE ROLE role3 SETTINGS PROFILE default, max_memory_usage=10000000 WRITABLE" + ) - with Scenario("I create role settings profile, fake profile, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Create_Settings("1.0")]): + with Scenario( + "I create role settings profile, fake profile, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Create_Settings("1.0")], + ): with cleanup("role4a"): with Given("I ensure profile profile0 does not exist"): node.query("DROP SETTINGS PROFILE IF EXISTS profile0") with When("I create role with settings profile that does not exist"): - exitcode, message = errors.settings_profile_not_found_in_disk("profile0") - node.query("CREATE ROLE role4a SETTINGS PROFILE profile0", exitcode=exitcode, message=message) + exitcode, message = errors.settings_profile_not_found_in_disk( + "profile0" + ) + node.query( + "CREATE ROLE role4a SETTINGS PROFILE profile0", + exitcode=exitcode, + message=message, + ) - with Scenario("I create role with settings without profile", requirements=[ - RQ_SRS_006_RBAC_Role_Create_Settings("1.0")]): + with Scenario( + "I create role with settings without profile", + requirements=[RQ_SRS_006_RBAC_Role_Create_Settings("1.0")], + ): with cleanup("role4"): with When("I create role with settings without profile"): - node.query("CREATE ROLE role4 SETTINGS max_memory_usage=10000000 READONLY") + node.query( + "CREATE ROLE role4 SETTINGS max_memory_usage=10000000 READONLY" + ) diff --git a/tests/testflows/rbac/tests/syntax/create_row_policy.py b/tests/testflows/rbac/tests/syntax/create_row_policy.py index 8bf83579dd5..cbc3b02a2e9 100755 --- a/tests/testflows/rbac/tests/syntax/create_row_policy.py +++ b/tests/testflows/rbac/tests/syntax/create_row_policy.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("create row policy") @Args(format_description=False) @@ -33,7 +34,7 @@ def feature(self, node="clickhouse1"): def create_policy(policy, on="default.foo"): with Given(f"I ensure I do have policy {policy} on {on}"): - node.query(f"CREATE ROW POLICY OR REPLACE {policy} ON {on}") + node.query(f"CREATE ROW POLICY OR REPLACE {policy} ON {on}") try: with Given("I have a table and some roles"): @@ -41,58 +42,94 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE ROLE role0") node.query(f"CREATE ROLE role1") - with Scenario("I create row policy with no options", requirements=[ + with Scenario( + "I create row policy with no options", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy0"): with When("I create row policy"): node.query("CREATE ROW POLICY policy0 ON default.foo") - with Scenario("I create row policy using short syntax with no options", requirements=[ + with Scenario( + "I create row policy using short syntax with no options", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy1"): with When("I create row policy short form"): node.query("CREATE POLICY policy1 ON default.foo") - with Scenario("I create row policy that already exists, throws exception", requirements=[ + with Scenario( + "I create row policy that already exists, throws exception", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): policy = "policy0" with cleanup(policy): create_policy(policy) with When(f"I create row policy {policy}"): - exitcode, message = errors.cannot_insert_row_policy(name=f"{policy} ON default.foo") - node.query(f"CREATE ROW POLICY {policy} ON default.foo", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_insert_row_policy( + name=f"{policy} ON default.foo" + ) + node.query( + f"CREATE ROW POLICY {policy} ON default.foo", + exitcode=exitcode, + message=message, + ) del policy - with Scenario("I create row policy if not exists, policy does not exist", requirements=[ + with Scenario( + "I create row policy if not exists, policy does not exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_IfNotExists("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy2"): with When("I create row policy with if not exists"): node.query("CREATE ROW POLICY IF NOT EXISTS policy2 ON default.foo") - with Scenario("I create row policy if not exists, policy does exist", requirements=[ + with Scenario( + "I create row policy if not exists, policy does exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_IfNotExists("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): policy = "policy2" with cleanup(policy): create_policy(policy) with When(f"I create row policy {policy} with if not exists"): - node.query(f"CREATE ROW POLICY IF NOT EXISTS {policy} ON default.foo") + node.query( + f"CREATE ROW POLICY IF NOT EXISTS {policy} ON default.foo" + ) del policy - with Scenario("I create row policy or replace, policy does not exist", requirements=[ + with Scenario( + "I create row policy or replace, policy does not exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Replace("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy3"): with When("I create row policy with or replace"): node.query("CREATE ROW POLICY OR REPLACE policy3 ON default.foo") - with Scenario("I create row policy or replace, policy does exist", requirements=[ + with Scenario( + "I create row policy or replace, policy does exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Replace("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): policy = "policy3" with cleanup(policy): create_policy(policy) @@ -100,126 +137,216 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE ROW POLICY OR REPLACE {policy} ON default.foo") del policy - with Scenario("I create row policy as permissive", requirements=[ + with Scenario( + "I create row policy as permissive", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Access_Permissive("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy4"): with When("I create row policy as permissive"): node.query("CREATE ROW POLICY policy4 ON default.foo AS PERMISSIVE") - with Scenario("I create row policy as restrictive", requirements=[ + with Scenario( + "I create row policy as restrictive", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Access_Restrictive("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy5"): with When("I create row policy as restrictive"): - node.query("CREATE ROW POLICY policy5 ON default.foo AS RESTRICTIVE") + node.query( + "CREATE ROW POLICY policy5 ON default.foo AS RESTRICTIVE" + ) - with Scenario("I create row policy for select", requirements=[ + with Scenario( + "I create row policy for select", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_ForSelect("1.0"), RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_Condition("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_Condition("1.0"), + ], + ): with cleanup("policy6"): with When("I create row policy with for select"): - node.query("CREATE ROW POLICY policy6 ON default.foo FOR SELECT USING x > 10") + node.query( + "CREATE ROW POLICY policy6 ON default.foo FOR SELECT USING x > 10" + ) - with Scenario("I create row policy using condition", requirements=[ + with Scenario( + "I create row policy using condition", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Condition("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy6"): with When("I create row policy with condition"): node.query("CREATE ROW POLICY policy6 ON default.foo USING x > 10") - with Scenario("I create row policy assigned to one role", requirements=[ + with Scenario( + "I create row policy assigned to one role", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy7"): with When("I create row policy for one role"): node.query("CREATE ROW POLICY policy7 ON default.foo TO role0") - with Scenario("I create row policy to assign to role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0")]): + with Scenario( + "I create row policy to assign to role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0")], + ): role = "role2" with cleanup("policy8a"): with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I create a row policy, assign to role {role}, which does not exist"): + with Then( + f"I create a row policy, assign to role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"CREATE ROW POLICY policy8a ON default.foo TO {role}", exitcode=exitcode, message=message) + node.query( + f"CREATE ROW POLICY policy8a ON default.foo TO {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create row policy to assign to all excpet role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0")]): + with Scenario( + "I create row policy to assign to all excpet role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0")], + ): role = "role2" with cleanup("policy8a"): with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I create a row policy, assign to all except role {role}, which does not exist"): + with Then( + f"I create a row policy, assign to all except role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"CREATE ROW POLICY policy8a ON default.foo TO ALL EXCEPT {role}", exitcode=exitcode, message=message) + node.query( + f"CREATE ROW POLICY policy8a ON default.foo TO ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create row policy assigned to multiple roles", requirements=[ + with Scenario( + "I create row policy assigned to multiple roles", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Assignment("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy8b"): with When("I create row policy for multiple roles"): - node.query("CREATE ROW POLICY policy8b ON default.foo TO role0, role1") + node.query( + "CREATE ROW POLICY policy8b ON default.foo TO role0, role1" + ) - with Scenario("I create row policy assigned to all", requirements=[ + with Scenario( + "I create row policy assigned to all", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_All("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy9"): with When("I create row policy for all"): node.query("CREATE ROW POLICY policy9 ON default.foo TO ALL") - with Scenario("I create row policy assigned to all except one role", requirements=[ + with Scenario( + "I create row policy assigned to all except one role", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_AllExcept("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy10"): with When("I create row policy for all except one"): - node.query("CREATE ROW POLICY policy10 ON default.foo TO ALL EXCEPT role0") + node.query( + "CREATE ROW POLICY policy10 ON default.foo TO ALL EXCEPT role0" + ) - with Scenario("I create row policy assigned to all except multiple roles", requirements=[ + with Scenario( + "I create row policy assigned to all except multiple roles", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_AllExcept("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy11"): with When("I create row policy for all except multiple roles"): - node.query("CREATE ROW POLICY policy11 ON default.foo TO ALL EXCEPT role0, role1") + node.query( + "CREATE ROW POLICY policy11 ON default.foo TO ALL EXCEPT role0, role1" + ) - with Scenario("I create row policy assigned to none", requirements=[ + with Scenario( + "I create row policy assigned to none", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_Assignment_None("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with cleanup("policy11"): with When("I create row policy for none"): node.query("CREATE ROW POLICY policy11 ON default.foo TO NONE") - with Scenario("I create row policy on cluster", requirements=[ + with Scenario( + "I create row policy on cluster", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): try: with When("I run create row policy command on cluster"): - node.query("CREATE ROW POLICY policy12 ON CLUSTER sharded_cluster ON default.foo") + node.query( + "CREATE ROW POLICY policy12 ON CLUSTER sharded_cluster ON default.foo" + ) finally: with Finally("I drop the row policy from cluster"): - node.query("DROP ROW POLICY IF EXISTS policy12 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "DROP ROW POLICY IF EXISTS policy12 ON default.foo ON CLUSTER sharded_cluster" + ) - with Scenario("I create row policy on fake cluster, throws exception", requirements=[ + with Scenario( + "I create row policy on fake cluster, throws exception", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): with When("I run create row policy command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("CREATE ROW POLICY policy13 ON CLUSTER fake_cluster ON default.foo", exitcode=exitcode, message=message) + node.query( + "CREATE ROW POLICY policy13 ON CLUSTER fake_cluster ON default.foo", + exitcode=exitcode, + message=message, + ) - with Scenario("I create row policy on cluster after table", requirements=[ + with Scenario( + "I create row policy on cluster after table", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Create_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Create_On("1.0"), + ], + ): try: with When("I run create row policy command on cluster"): - node.query("CREATE ROW POLICY policy12 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "CREATE ROW POLICY policy12 ON default.foo ON CLUSTER sharded_cluster" + ) finally: with Finally("I drop the row policy from cluster"): - node.query("DROP ROW POLICY IF EXISTS policy12 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "DROP ROW POLICY IF EXISTS policy12 ON default.foo ON CLUSTER sharded_cluster" + ) finally: with Finally("I drop the table and the roles"): node.query(f"DROP TABLE IF EXISTS default.foo") - node.query(f"DROP ROLE IF EXISTS role0, role1") \ No newline at end of file + node.query(f"DROP ROLE IF EXISTS role0, role1") diff --git a/tests/testflows/rbac/tests/syntax/create_settings_profile.py b/tests/testflows/rbac/tests/syntax/create_settings_profile.py index 8976ce6843a..dc04ea0eb4c 100755 --- a/tests/testflows/rbac/tests/syntax/create_settings_profile.py +++ b/tests/testflows/rbac/tests/syntax/create_settings_profile.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("create settings profile") @Args(format_description=False) @@ -32,37 +33,51 @@ def feature(self, node="clickhouse1"): def create_profile(profile): with Given(f"I ensure I do have profile {profile}"): - node.query(f"CREATE SETTINGS PROFILE OR REPLACE {profile}") + node.query(f"CREATE SETTINGS PROFILE OR REPLACE {profile}") try: with Given("I have a user and a role"): node.query(f"CREATE USER user0") node.query(f"CREATE ROLE role0") - with Scenario("I create settings profile with no options", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create("1.0")]): + with Scenario( + "I create settings profile with no options", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create("1.0")], + ): with cleanup("profile0"): with When("I create settings profile"): node.query("CREATE SETTINGS PROFILE profile0") - with Scenario("I create settings profile that already exists, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create("1.0")]): + with Scenario( + "I create settings profile that already exists, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create("1.0")], + ): profile = "profile0" with cleanup(profile): create_profile(profile) with When(f"I create settings profile {profile} that already exists"): - exitcode, message = errors.cannot_insert_settings_profile(name=profile) - node.query(f"CREATE SETTINGS PROFILE {profile}", exitcode=exitcode, message=message) + exitcode, message = errors.cannot_insert_settings_profile( + name=profile + ) + node.query( + f"CREATE SETTINGS PROFILE {profile}", + exitcode=exitcode, + message=message, + ) del profile - with Scenario("I create settings profile if not exists, profile does not exist", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_IfNotExists("1.0")]): + with Scenario( + "I create settings profile if not exists, profile does not exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_IfNotExists("1.0")], + ): with cleanup("profile1"): with When("I create settings profile with if not exists"): node.query("CREATE SETTINGS PROFILE IF NOT EXISTS profile1") - with Scenario("I create settings profile if not exists, profile does exist", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_IfNotExists("1.0")]): + with Scenario( + "I create settings profile if not exists, profile does exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_IfNotExists("1.0")], + ): profile = "profile1" with cleanup(profile): create_profile(profile) @@ -70,184 +85,326 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE SETTINGS PROFILE IF NOT EXISTS {profile}") del profile - with Scenario("I create settings profile or replace, profile does not exist", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Replace("1.0")]): + with Scenario( + "I create settings profile or replace, profile does not exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Replace("1.0")], + ): with cleanup("profile2"): with When("I create settings policy with or replace"): node.query("CREATE SETTINGS PROFILE OR REPLACE profile2") - with Scenario("I create settings profile or replace, profile does exist", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Replace("1.0")]): + with Scenario( + "I create settings profile or replace, profile does exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Replace("1.0")], + ): with cleanup("profile2"): create_profile("profile2") with When("I create settings policy with or replace"): node.query("CREATE SETTINGS PROFILE OR REPLACE profile2") - with Scenario("I create settings profile short form", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create("1.0")]): + with Scenario( + "I create settings profile short form", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create("1.0")], + ): with cleanup("profile3"): with When("I create settings profile short form"): node.query("CREATE PROFILE profile3") - with Scenario("I create settings profile with a setting value", requirements=[ + with Scenario( + "I create settings profile with a setting value", + requirements=[ RQ_SRS_006_RBAC_SettingsProfile_Create_Variables("1.0"), - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Value("1.0")]): + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Value("1.0"), + ], + ): with cleanup("profile4"): with When("I create settings profile with settings"): - node.query("CREATE SETTINGS PROFILE profile4 SETTINGS max_memory_usage = 100000001") + node.query( + "CREATE SETTINGS PROFILE profile4 SETTINGS max_memory_usage = 100000001" + ) - with Scenario("I create settings profile with a setting value, does not exist, throws exception", requirements=[ + with Scenario( + "I create settings profile with a setting value, does not exist, throws exception", + requirements=[ RQ_SRS_006_RBAC_SettingsProfile_Create_Variables("1.0"), - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Value("1.0")]): + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Value("1.0"), + ], + ): with When("I create settings profile using settings and nonexistent value"): exitcode, message = errors.unknown_setting("fake_setting") - node.query("CREATE SETTINGS PROFILE profile0 SETTINGS fake_setting = 100000001", exitcode=exitcode, message=message) + node.query( + "CREATE SETTINGS PROFILE profile0 SETTINGS fake_setting = 100000001", + exitcode=exitcode, + message=message, + ) - with Scenario("I create settings profile with a min setting value", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with a min setting value", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile5"), cleanup("profile6"): - with When("I create settings profile with min setting with and without equals"): - node.query("CREATE SETTINGS PROFILE profile5 SETTINGS max_memory_usage MIN 100000001") - node.query("CREATE SETTINGS PROFILE profile6 SETTINGS max_memory_usage MIN = 100000001") + with When( + "I create settings profile with min setting with and without equals" + ): + node.query( + "CREATE SETTINGS PROFILE profile5 SETTINGS max_memory_usage MIN 100000001" + ) + node.query( + "CREATE SETTINGS PROFILE profile6 SETTINGS max_memory_usage MIN = 100000001" + ) - with Scenario("I create settings profile with a max setting value", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with a max setting value", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile7"), cleanup("profile8"): - with When("I create settings profile with max setting with and without equals"): - node.query("CREATE SETTINGS PROFILE profile7 SETTINGS max_memory_usage MAX 100000001") - node.query("CREATE SETTINGS PROFILE profile8 SETTINGS max_memory_usage MAX = 100000001") + with When( + "I create settings profile with max setting with and without equals" + ): + node.query( + "CREATE SETTINGS PROFILE profile7 SETTINGS max_memory_usage MAX 100000001" + ) + node.query( + "CREATE SETTINGS PROFILE profile8 SETTINGS max_memory_usage MAX = 100000001" + ) - with Scenario("I create settings profile with min and max setting values", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with min and max setting values", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile9"): with When("I create settings profile with min and max setting"): - node.query("CREATE SETTINGS PROFILE profile9 SETTINGS max_memory_usage MIN 100000001 MAX 200000001") + node.query( + "CREATE SETTINGS PROFILE profile9 SETTINGS max_memory_usage MIN 100000001 MAX 200000001" + ) - with Scenario("I create settings profile with a readonly setting", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with a readonly setting", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile10"): with When("I create settings profile with readonly"): - node.query("CREATE SETTINGS PROFILE profile10 SETTINGS max_memory_usage READONLY") + node.query( + "CREATE SETTINGS PROFILE profile10 SETTINGS max_memory_usage READONLY" + ) - with Scenario("I create settings profile with a writable setting", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with a writable setting", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile21"): with When("I create settings profile with writable"): - node.query("CREATE SETTINGS PROFILE profile21 SETTINGS max_memory_usage WRITABLE") + node.query( + "CREATE SETTINGS PROFILE profile21 SETTINGS max_memory_usage WRITABLE" + ) - with Scenario("I create settings profile with inherited settings", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit("1.0")]): + with Scenario( + "I create settings profile with inherited settings", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit("1.0")], + ): with cleanup("profile11"): with When("I create settings profile with inherit"): - node.query("CREATE SETTINGS PROFILE profile11 SETTINGS INHERIT 'default'") + node.query( + "CREATE SETTINGS PROFILE profile11 SETTINGS INHERIT 'default'" + ) - with Scenario("I create settings profile with inherit/from profile, fake profile, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit("1.0")]): + with Scenario( + "I create settings profile with inherit/from profile, fake profile, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit("1.0")], + ): profile = "profile3" with Given(f"I ensure that profile {profile} does not exist"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") - sources = {"INHERIT","PROFILE"} + sources = {"INHERIT", "PROFILE"} for source in sources: - with When(f"I create settings profile {source} from nonexistant parent"): - exitcode, message = errors.settings_profile_not_found_in_disk(profile) - node.query(f"CREATE PROFILE profile0 SETTINGS {source} {profile}", exitcode=exitcode, message=message) + with When( + f"I create settings profile {source} from nonexistant parent" + ): + exitcode, message = errors.settings_profile_not_found_in_disk( + profile + ) + node.query( + f"CREATE PROFILE profile0 SETTINGS {source} {profile}", + exitcode=exitcode, + message=message, + ) del profile - with Scenario("I create settings profile with inherited settings other form", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit("1.0")]): + with Scenario( + "I create settings profile with inherited settings other form", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Inherit("1.0")], + ): with cleanup("profile12"): with When("I create settings profile with inherit short form"): node.query("CREATE PROFILE profile12 SETTINGS PROFILE 'default'") - with Scenario("I create settings profile with multiple settings", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with multiple settings", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile13"): with When("I create settings profile with multiple settings"): - node.query("CREATE SETTINGS PROFILE profile13" + node.query( + "CREATE SETTINGS PROFILE profile13" " SETTINGS max_memory_usage = 100000001" - " SETTINGS max_memory_usage_for_user = 100000001") + " SETTINGS max_memory_usage_for_user = 100000001" + ) - with Scenario("I create settings profile with multiple settings short form", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0")]): + with Scenario( + "I create settings profile with multiple settings short form", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Variables_Constraints("1.0") + ], + ): with cleanup("profile14"): - with When("I create settings profile with multiple settings short form"): - node.query("CREATE SETTINGS PROFILE profile14" + with When( + "I create settings profile with multiple settings short form" + ): + node.query( + "CREATE SETTINGS PROFILE profile14" " SETTINGS max_memory_usage = 100000001," - " max_memory_usage_for_user = 100000001") + " max_memory_usage_for_user = 100000001" + ) - with Scenario("I create settings profile assigned to one role", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")]): + with Scenario( + "I create settings profile assigned to one role", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")], + ): with cleanup("profile15"): with When("I create settings profile for a role"): node.query("CREATE SETTINGS PROFILE profile15 TO role0") - with Scenario("I create settings profile to assign to role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")]): + with Scenario( + "I create settings profile to assign to role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I create a settings profile, assign to role {role}, which does not exist"): + with Then( + f"I create a settings profile, assign to role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"CREATE SETTINGS PROFILE profile0 TO {role}", exitcode=exitcode, message=message) + node.query( + f"CREATE SETTINGS PROFILE profile0 TO {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create settings profile to assign to all except role that does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")]): + with Scenario( + "I create settings profile to assign to all except role that does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")], + ): role = "role1" with Given(f"I drop {role} if it exists"): node.query(f"DROP ROLE IF EXISTS {role}") - with Then(f"I create a settings profile, assign to all except role {role}, which does not exist"): + with Then( + f"I create a settings profile, assign to all except role {role}, which does not exist" + ): exitcode, message = errors.role_not_found_in_disk(name=role) - node.query(f"CREATE SETTINGS PROFILE profile0 TO ALL EXCEPT {role}", exitcode=exitcode, message=message) + node.query( + f"CREATE SETTINGS PROFILE profile0 TO ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create settings profile assigned to multiple roles", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")]): + with Scenario( + "I create settings profile assigned to multiple roles", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment("1.0")], + ): with cleanup("profile16"): with When("I create settings profile for multiple roles"): node.query("CREATE SETTINGS PROFILE profile16 TO role0, user0") - with Scenario("I create settings profile assigned to all", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_All("1.0")]): + with Scenario( + "I create settings profile assigned to all", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_All("1.0")], + ): with cleanup("profile17"): with When("I create settings profile for all"): node.query("CREATE SETTINGS PROFILE profile17 TO ALL") - with Scenario("I create settings profile assigned to all except one role",requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_AllExcept("1.0")]): + with Scenario( + "I create settings profile assigned to all except one role", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_AllExcept("1.0") + ], + ): with cleanup("profile18"): with When("I create settings profile for all except one role"): node.query("CREATE SETTINGS PROFILE profile18 TO ALL EXCEPT role0") - with Scenario("I create settings profile assigned to all except multiple roles", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_AllExcept("1.0")]): + with Scenario( + "I create settings profile assigned to all except multiple roles", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_AllExcept("1.0") + ], + ): with cleanup("profile19"): with When("I create settings profile for all except multiple roles"): - node.query("CREATE SETTINGS PROFILE profile19 TO ALL EXCEPT role0, user0") + node.query( + "CREATE SETTINGS PROFILE profile19 TO ALL EXCEPT role0, user0" + ) - with Scenario("I create settings profile assigned to none", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_None("1.0")]): + with Scenario( + "I create settings profile assigned to none", + requirements=[ + RQ_SRS_006_RBAC_SettingsProfile_Create_Assignment_None("1.0") + ], + ): with cleanup("profile22"): with When("I create settings profile for none"): node.query("CREATE SETTINGS PROFILE profile22 TO NONE") - with Scenario("I create settings profile on cluster", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_OnCluster("1.0")]): + with Scenario( + "I create settings profile on cluster", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_OnCluster("1.0")], + ): try: with When("I run create settings profile command"): - node.query("CREATE SETTINGS PROFILE profile20 ON CLUSTER sharded_cluster") - node.query("CREATE SETTINGS PROFILE OR REPLACE profile20 ON CLUSTER sharded_cluster SETTINGS max_memory_usage = 100000001") - node.query("CREATE SETTINGS PROFILE OR REPLACE profile20 ON CLUSTER sharded_cluster SETTINGS INHERIT 'default'") - node.query("CREATE SETTINGS PROFILE OR REPLACE profile20 ON CLUSTER sharded_cluster TO ALL") + node.query( + "CREATE SETTINGS PROFILE profile20 ON CLUSTER sharded_cluster" + ) + node.query( + "CREATE SETTINGS PROFILE OR REPLACE profile20 ON CLUSTER sharded_cluster SETTINGS max_memory_usage = 100000001" + ) + node.query( + "CREATE SETTINGS PROFILE OR REPLACE profile20 ON CLUSTER sharded_cluster SETTINGS INHERIT 'default'" + ) + node.query( + "CREATE SETTINGS PROFILE OR REPLACE profile20 ON CLUSTER sharded_cluster TO ALL" + ) finally: with Finally("I drop the settings profile"): - node.query("DROP SETTINGS PROFILE IF EXISTS profile20 ON CLUSTER sharded_cluster") + node.query( + "DROP SETTINGS PROFILE IF EXISTS profile20 ON CLUSTER sharded_cluster" + ) - with Scenario("I create settings profile on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Create_OnCluster("1.0")]): + with Scenario( + "I create settings profile on fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Create_OnCluster("1.0")], + ): with When("I run create settings profile command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("CREATE SETTINGS PROFILE profile1 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "CREATE SETTINGS PROFILE profile1 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop all the users and roles"): node.query(f"DROP USER IF EXISTS user0") diff --git a/tests/testflows/rbac/tests/syntax/create_user.py b/tests/testflows/rbac/tests/syntax/create_user.py index 326446e4620..20916e2a171 100755 --- a/tests/testflows/rbac/tests/syntax/create_user.py +++ b/tests/testflows/rbac/tests/syntax/create_user.py @@ -6,6 +6,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("create user") @Args(format_description=False) @@ -34,37 +35,51 @@ def feature(self, node="clickhouse1"): def create_user(user): with Given(f"I ensure I do have user {user}"): - node.query(f"CREATE USER OR REPLACE {user}") + node.query(f"CREATE USER OR REPLACE {user}") - with Scenario("I create user with no options", requirements=[ + with Scenario( + "I create user with no options", + requirements=[ RQ_SRS_006_RBAC_User_Create("1.0"), - RQ_SRS_006_RBAC_User_Create_Host_Default("1.0")]): + RQ_SRS_006_RBAC_User_Create_Host_Default("1.0"), + ], + ): with cleanup("user0"): with When("I create a user with no options"): node.query("CREATE USER user0") - with Scenario("I create user that already exists, throws exception", requirements=[ + with Scenario( + "I create user that already exists, throws exception", + requirements=[ RQ_SRS_006_RBAC_User_Create("1.0"), - RQ_SRS_006_RBAC_User_Create_Host_Default("1.0")]): + RQ_SRS_006_RBAC_User_Create_Host_Default("1.0"), + ], + ): user = "user0" with cleanup(user): create_user(user) - with When(f"I create a user {user} that already exists without IF EXISTS, throws exception"): + with When( + f"I create a user {user} that already exists without IF EXISTS, throws exception" + ): exitcode, message = errors.cannot_insert_user(name=user) node.query(f"CREATE USER {user}", exitcode=exitcode, message=message) del user - with Scenario("I create user with if not exists, user does not exist", requirements=[ - RQ_SRS_006_RBAC_User_Create_IfNotExists("1.0")]): + with Scenario( + "I create user with if not exists, user does not exist", + requirements=[RQ_SRS_006_RBAC_User_Create_IfNotExists("1.0")], + ): user = "user0" with cleanup(user): with When(f"I create a user {user} with if not exists"): node.query(f"CREATE USER IF NOT EXISTS {user}") del user - #Bug exists, mark as xfail - with Scenario("I create user with if not exists, user does exist", requirements=[ - RQ_SRS_006_RBAC_User_Create_IfNotExists("1.0")]): + # Bug exists, mark as xfail + with Scenario( + "I create user with if not exists, user does exist", + requirements=[RQ_SRS_006_RBAC_User_Create_IfNotExists("1.0")], + ): user = "user0" with cleanup(user): create_user(user) @@ -72,16 +87,20 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE USER IF NOT EXISTS {user}") del user - with Scenario("I create user or replace, user does not exist", requirements=[ - RQ_SRS_006_RBAC_User_Create_Replace("1.0")]): + with Scenario( + "I create user or replace, user does not exist", + requirements=[RQ_SRS_006_RBAC_User_Create_Replace("1.0")], + ): user = "user0" with cleanup(user): with When(f"I create a user {user} with or replace"): node.query(f"CREATE USER OR REPLACE {user}") del user - with Scenario("I create user or replace, user does exist", requirements=[ - RQ_SRS_006_RBAC_User_Create_Replace("1.0")]): + with Scenario( + "I create user or replace, user does exist", + requirements=[RQ_SRS_006_RBAC_User_Create_Replace("1.0")], + ): user = "user0" with cleanup(user): create_user(user) @@ -89,106 +108,156 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE USER OR REPLACE {user}") del user - with Scenario("I create user with no password", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_NoPassword("1.0")]): + with Scenario( + "I create user with no password", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_NoPassword("1.0")], + ): with cleanup("user1"): with When("I create a user with no password"): node.query("CREATE USER user1 IDENTIFIED WITH NO_PASSWORD") - with Scenario("I create user with plaintext password", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_PlainText("1.0")]): + with Scenario( + "I create user with plaintext password", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_PlainText("1.0")], + ): with cleanup("user1"): with When("I create a user with plaintext password"): - node.query("CREATE USER user1 IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'mypassword'") + node.query( + "CREATE USER user1 IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'mypassword'" + ) - with Scenario("I create user with sha256 password", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_Sha256Password("1.0")]): + with Scenario( + "I create user with sha256 password", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_Sha256Password("1.0")], + ): with cleanup("user2"): with When("I create a user with sha256 password"): password = hashlib.sha256("mypassword".encode("utf-8")).hexdigest() - node.query(f"CREATE USER user2 IDENTIFIED WITH SHA256_PASSWORD BY '{password}'") + node.query( + f"CREATE USER user2 IDENTIFIED WITH SHA256_PASSWORD BY '{password}'" + ) - with Scenario("I create user with sha256 password using IDENTIFIED BY", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_Sha256Password("1.0")]): + with Scenario( + "I create user with sha256 password using IDENTIFIED BY", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_Sha256Password("1.0")], + ): with cleanup("user2"): with When("I create a user with sha256 password using short form"): password = hashlib.sha256("mypassword".encode("utf-8")).hexdigest() node.query(f"CREATE USER user2 IDENTIFIED BY '{password}'") - with Scenario("I create user with sha256_hash password", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_Sha256Hash("1.0")]): + with Scenario( + "I create user with sha256_hash password", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_Sha256Hash("1.0")], + ): with cleanup("user3"): with When("I create a user with sha256_hash"): + def hash(password): return hashlib.sha256(password.encode("utf-8")).hexdigest() - password = hash(hash("mypassword")) - node.query(f"CREATE USER user3 IDENTIFIED WITH SHA256_HASH BY '{password}'") - with Scenario("I create user with double sha1 password", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Password("1.0")]): + password = hash(hash("mypassword")) + node.query( + f"CREATE USER user3 IDENTIFIED WITH SHA256_HASH BY '{password}'" + ) + + with Scenario( + "I create user with double sha1 password", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Password("1.0")], + ): with cleanup("user3"): with When("I create a user with double_sha1_password"): - node.query(f"CREATE USER user3 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY 'mypassword'") + node.query( + f"CREATE USER user3 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY 'mypassword'" + ) - with Scenario("I create user with double sha1 hash", requirements=[ - RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Hash("1.0")]): + with Scenario( + "I create user with double sha1 hash", + requirements=[RQ_SRS_006_RBAC_User_Create_Password_DoubleSha1Hash("1.0")], + ): with cleanup("user3"): with When("I create a user with double_sha1_hash"): + def hash(password): return hashlib.sha1(password.encode("utf-8")).hexdigest() - password = hash(hash("mypassword")) - node.query(f"CREATE USER user3 IDENTIFIED WITH DOUBLE_SHA1_HASH BY '{password}'") - with Scenario("I create user with host name", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_Name("1.0")]): + password = hash(hash("mypassword")) + node.query( + f"CREATE USER user3 IDENTIFIED WITH DOUBLE_SHA1_HASH BY '{password}'" + ) + + with Scenario( + "I create user with host name", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_Name("1.0")], + ): with cleanup("user4"): with When("I create a user with host name"): - node.query("CREATE USER user4 HOST NAME 'localhost', NAME 'clickhouse.com'") + node.query( + "CREATE USER user4 HOST NAME 'localhost', NAME 'clickhouse.com'" + ) - with Scenario("I create user with host regexp", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_Regexp("1.0")]): + with Scenario( + "I create user with host regexp", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_Regexp("1.0")], + ): with cleanup("user5"): with When("I create a user with host regexp"): - node.query("CREATE USER user5 HOST REGEXP 'lo.?*host', REGEXP 'lo*host'") + node.query( + "CREATE USER user5 HOST REGEXP 'lo.?*host', REGEXP 'lo*host'" + ) - with Scenario("I create user with host ip", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_IP("1.0")]): + with Scenario( + "I create user with host ip", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_IP("1.0")], + ): with cleanup("user6"): with When("I create a user with host ip"): node.query("CREATE USER user6 HOST IP '127.0.0.1', IP '127.0.0.2'") - with Scenario("I create user with host like", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_Like("1.0")]): + with Scenario( + "I create user with host like", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_Like("1.0")], + ): with cleanup("user7"): with When("I create a user with host like"): node.query("CREATE USER user7 HOST LIKE 'local%'") - with Scenario("I create user with host none", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_None("1.0")]): + with Scenario( + "I create user with host none", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_None("1.0")], + ): with cleanup("user7"): with When("I create a user with host none"): node.query("CREATE USER user7 HOST NONE") - with Scenario("I create user with host local", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_Local("1.0")]): + with Scenario( + "I create user with host local", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_Local("1.0")], + ): with cleanup("user7"): with When("I create a user with host local"): node.query("CREATE USER user7 HOST LOCAL") - with Scenario("I create user with host any", requirements=[ - RQ_SRS_006_RBAC_User_Create_Host_Any("1.0")]): + with Scenario( + "I create user with host any", + requirements=[RQ_SRS_006_RBAC_User_Create_Host_Any("1.0")], + ): with cleanup("user7"): with When("I create a user with host any"): node.query("CREATE USER user7 HOST ANY") - with Scenario("I create user with default role set to none", requirements=[ - RQ_SRS_006_RBAC_User_Create_DefaultRole_None("1.0")]): + with Scenario( + "I create user with default role set to none", + requirements=[RQ_SRS_006_RBAC_User_Create_DefaultRole_None("1.0")], + ): with cleanup("user8"): with When("I create a user with no default role"): node.query("CREATE USER user8 DEFAULT ROLE NONE") - with Scenario("I create user with default role", requirements=[ - RQ_SRS_006_RBAC_User_Create_DefaultRole("1.0")]): + with Scenario( + "I create user with default role", + requirements=[RQ_SRS_006_RBAC_User_Create_DefaultRole("1.0")], + ): with Given("I have a role"): node.query("CREATE ROLE default") with cleanup("user9"): @@ -197,66 +266,104 @@ def feature(self, node="clickhouse1"): with Finally("I drop the role"): node.query("DROP ROLE default") - with Scenario("I create user default role, role doesn't exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Create_DefaultRole("1.0")]): + with Scenario( + "I create user default role, role doesn't exist, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Create_DefaultRole("1.0")], + ): with cleanup("user12"): role = "role0" with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") with When(f"I create user with default role {role}"): exitcode, message = errors.role_not_found_in_disk(role) - node.query(f"CREATE USER user12 DEFAULT ROLE {role}",exitcode=exitcode, message=message) + node.query( + f"CREATE USER user12 DEFAULT ROLE {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create user default role, all except role doesn't exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Create_DefaultRole("1.0")]): + with Scenario( + "I create user default role, all except role doesn't exist, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Create_DefaultRole("1.0")], + ): with cleanup("user12"): role = "role0" with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") with When(f"I create user with default role {role}"): exitcode, message = errors.role_not_found_in_disk(role) - node.query(f"CREATE USER user12 DEFAULT ROLE ALL EXCEPT {role}",exitcode=exitcode, message=message) + node.query( + f"CREATE USER user12 DEFAULT ROLE ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) del role - with Scenario("I create user with all roles set to default", requirements=[ - RQ_SRS_006_RBAC_User_Create_DefaultRole_All("1.0")]): + with Scenario( + "I create user with all roles set to default", + requirements=[RQ_SRS_006_RBAC_User_Create_DefaultRole_All("1.0")], + ): with cleanup("user10"): with When("I create a user with all roles as default"): node.query("CREATE USER user10 DEFAULT ROLE ALL") - with Scenario("I create user with settings profile", requirements=[ - RQ_SRS_006_RBAC_User_Create_Settings("1.0")]): + with Scenario( + "I create user with settings profile", + requirements=[RQ_SRS_006_RBAC_User_Create_Settings("1.0")], + ): with cleanup("user11"): with When("I create a user with a settings profile"): - node.query("CREATE USER user11 SETTINGS PROFILE default, max_memory_usage=10000000 READONLY") + node.query( + "CREATE USER user11 SETTINGS PROFILE default, max_memory_usage=10000000 READONLY" + ) - with Scenario("I create user settings profile, fake profile, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Create_Settings("1.0")]): + with Scenario( + "I create user settings profile, fake profile, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Create_Settings("1.0")], + ): with cleanup("user18a"): profile = "profile0" with Given(f"I ensure that profile {profile} does not exist"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") - with When(f"I create user with Settings and set profile to fake profile {profile}"): + with When( + f"I create user with Settings and set profile to fake profile {profile}" + ): exitcode, message = errors.settings_profile_not_found_in_disk(profile) - node.query("CREATE USER user18a SETTINGS PROFILE profile0", exitcode=exitcode, message=message) + node.query( + "CREATE USER user18a SETTINGS PROFILE profile0", + exitcode=exitcode, + message=message, + ) del profile - with Scenario("I create user settings with a fake setting, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Create_Settings("1.0")]): + with Scenario( + "I create user settings with a fake setting, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Create_Settings("1.0")], + ): with cleanup("user18b"): with When("I create settings profile using settings and nonexistent value"): exitcode, message = errors.unknown_setting("fake_setting") - node.query("CREATE USER user18b SETTINGS fake_setting = 100000001", exitcode=exitcode, message=message) + node.query( + "CREATE USER user18b SETTINGS fake_setting = 100000001", + exitcode=exitcode, + message=message, + ) - with Scenario("I create user with settings without profile", requirements=[ - RQ_SRS_006_RBAC_User_Create_Settings("1.0")]): + with Scenario( + "I create user with settings without profile", + requirements=[RQ_SRS_006_RBAC_User_Create_Settings("1.0")], + ): with cleanup("user12"): with When("I create a user with settings and no profile"): - node.query("CREATE USER user12 SETTINGS max_memory_usage=10000000 READONLY") + node.query( + "CREATE USER user12 SETTINGS max_memory_usage=10000000 READONLY" + ) - with Scenario("I create user on cluster", requirements=[ - RQ_SRS_006_RBAC_User_Create_OnCluster("1.0")]): + with Scenario( + "I create user on cluster", + requirements=[RQ_SRS_006_RBAC_User_Create_OnCluster("1.0")], + ): try: with When("I create user on cluster"): node.query("CREATE USER user13 ON CLUSTER sharded_cluster") @@ -264,8 +371,14 @@ def feature(self, node="clickhouse1"): with Finally("I drop the user"): node.query("DROP USER user13 ON CLUSTER sharded_cluster") - with Scenario("I create user on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Create_OnCluster("1.0")]): - with When("I create user on fake cluster"): - exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("CREATE USER user14 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + with Scenario( + "I create user on fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Create_OnCluster("1.0")], + ): + with When("I create user on fake cluster"): + exitcode, message = errors.cluster_not_found("fake_cluster") + node.query( + "CREATE USER user14 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) diff --git a/tests/testflows/rbac/tests/syntax/drop_quota.py b/tests/testflows/rbac/tests/syntax/drop_quota.py index 879964e46fb..9692bdaddcb 100755 --- a/tests/testflows/rbac/tests/syntax/drop_quota.py +++ b/tests/testflows/rbac/tests/syntax/drop_quota.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("drop quota") def feature(self, node="clickhouse1"): @@ -30,14 +31,17 @@ def feature(self, node="clickhouse1"): with Given(f"I ensure that quota {quota} does not exist"): node.query(f"DROP QUOTA IF EXISTS {quota}") - with Scenario("I drop quota with no options", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop("1.0")]): + with Scenario( + "I drop quota with no options", requirements=[RQ_SRS_006_RBAC_Quota_Drop("1.0")] + ): with cleanup("quota0"): with When("I run drop quota command"): node.query("DROP QUOTA quota0") - with Scenario("I drop quota, does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop("1.0")]): + with Scenario( + "I drop quota, does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Quota_Drop("1.0")], + ): quota = "quota0" cleanup_quota(quota) with When("I run drop quota command, throws exception"): @@ -45,32 +49,41 @@ def feature(self, node="clickhouse1"): node.query(f"DROP QUOTA {quota}", exitcode=exitcode, message=message) del quota - with Scenario("I drop quota if exists, quota exists", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop_IfExists("1.0")]): + with Scenario( + "I drop quota if exists, quota exists", + requirements=[RQ_SRS_006_RBAC_Quota_Drop_IfExists("1.0")], + ): with cleanup("quota1"): with When("I run drop quota command"): node.query("DROP QUOTA IF EXISTS quota1") - with Scenario("I drop quota if exists, quota does not exist", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop_IfExists("1.0")]): + with Scenario( + "I drop quota if exists, quota does not exist", + requirements=[RQ_SRS_006_RBAC_Quota_Drop_IfExists("1.0")], + ): cleanup_quota("quota2") with When("I run drop quota command, quota does not exist"): node.query("DROP QUOTA IF EXISTS quota2") - with Scenario("I drop default quota, throws error", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop("1.0")]): + with Scenario( + "I drop default quota, throws error", + requirements=[RQ_SRS_006_RBAC_Quota_Drop("1.0")], + ): with When("I drop default quota"): exitcode, message = errors.cannot_remove_quota_default() node.query("DROP QUOTA default", exitcode=exitcode, message=message) - with Scenario("I drop multiple quotas", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop("1.0")]): + with Scenario( + "I drop multiple quotas", requirements=[RQ_SRS_006_RBAC_Quota_Drop("1.0")] + ): with cleanup("quota2"), cleanup("quota3"): with When("I run drop quota command"): node.query("DROP QUOTA quota2, quota3") - with Scenario("I drop quota on cluster", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop_Cluster("1.0")]): + with Scenario( + "I drop quota on cluster", + requirements=[RQ_SRS_006_RBAC_Quota_Drop_Cluster("1.0")], + ): try: with Given("I have a quota"): node.query("CREATE QUOTA quota4 ON CLUSTER sharded_cluster") @@ -80,8 +93,14 @@ def feature(self, node="clickhouse1"): with Finally("I drop the quota in case it still exists"): node.query("DROP QUOTA IF EXISTS quota4 ON CLUSTER sharded_cluster") - with Scenario("I drop quota on fake cluster", requirements=[ - RQ_SRS_006_RBAC_Quota_Drop_Cluster("1.0")]): + with Scenario( + "I drop quota on fake cluster", + requirements=[RQ_SRS_006_RBAC_Quota_Drop_Cluster("1.0")], + ): with When("I run drop quota command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("DROP QUOTA quota5 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "DROP QUOTA quota5 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) diff --git a/tests/testflows/rbac/tests/syntax/drop_role.py b/tests/testflows/rbac/tests/syntax/drop_role.py index 87810dc0184..7824b6509c6 100755 --- a/tests/testflows/rbac/tests/syntax/drop_role.py +++ b/tests/testflows/rbac/tests/syntax/drop_role.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("drop role") def feature(self, node="clickhouse1"): @@ -28,17 +29,19 @@ def feature(self, node="clickhouse1"): def cleanup_role(role): with Given(f"I ensure that role {role} does not exist"): - node.query(f"DROP ROLE IF EXISTS {role}") + node.query(f"DROP ROLE IF EXISTS {role}") - - with Scenario("I drop role with no options", requirements=[ - RQ_SRS_006_RBAC_Role_Drop("1.0")]): + with Scenario( + "I drop role with no options", requirements=[RQ_SRS_006_RBAC_Role_Drop("1.0")] + ): with setup("role0"): with When("I drop role"): node.query("DROP ROLE role0") - with Scenario("I drop role that doesn't exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_Role_Drop("1.0")]): + with Scenario( + "I drop role that doesn't exist, throws exception", + requirements=[RQ_SRS_006_RBAC_Role_Drop("1.0")], + ): role = "role0" cleanup_role(role) with When(f"I drop role {role}"): @@ -46,39 +49,54 @@ def feature(self, node="clickhouse1"): node.query(f"DROP ROLE {role}", exitcode=exitcode, message=message) del role - with Scenario("I drop multiple roles", requirements=[ - RQ_SRS_006_RBAC_Role_Drop("1.0")]): + with Scenario( + "I drop multiple roles", requirements=[RQ_SRS_006_RBAC_Role_Drop("1.0")] + ): with setup("role1"), setup("role2"): with When("I drop multiple roles"): node.query("DROP ROLE role1, role2") - with Scenario("I drop role that does not exist, using if exists", requirements=[ - RQ_SRS_006_RBAC_Role_Drop_IfExists("1.0")]): + with Scenario( + "I drop role that does not exist, using if exists", + requirements=[RQ_SRS_006_RBAC_Role_Drop_IfExists("1.0")], + ): with When("I drop role if exists"): node.query("DROP ROLE IF EXISTS role3") - with Scenario("I drop multiple roles where one does not exist", requirements=[ - RQ_SRS_006_RBAC_Role_Drop_IfExists("1.0")]): + with Scenario( + "I drop multiple roles where one does not exist", + requirements=[RQ_SRS_006_RBAC_Role_Drop_IfExists("1.0")], + ): with setup("role5"): with When("I drop multiple roles where one doesnt exist"): node.query("DROP ROLE IF EXISTS role3, role5") - with Scenario("I drop multiple roles where both do not exist", requirements=[ - RQ_SRS_006_RBAC_Role_Drop_IfExists("1.0")]): + with Scenario( + "I drop multiple roles where both do not exist", + requirements=[RQ_SRS_006_RBAC_Role_Drop_IfExists("1.0")], + ): with Given("I ensure role does not exist"): node.query("DROP ROLE IF EXISTS role6") with When("I drop the nonexistant roles"): node.query("DROP USER IF EXISTS role5, role6") - with Scenario("I drop role on cluster", requirements=[ - RQ_SRS_006_RBAC_Role_Drop_Cluster("1.0")]): + with Scenario( + "I drop role on cluster", + requirements=[RQ_SRS_006_RBAC_Role_Drop_Cluster("1.0")], + ): with Given("I have a role on cluster"): node.query("CREATE ROLE OR REPLACE role0 ON CLUSTER sharded_cluster") with When("I drop the role from the cluster"): node.query("DROP ROLE IF EXISTS role0 ON CLUSTER sharded_cluster") - with Scenario("I drop role on fake cluster", requirements=[ - RQ_SRS_006_RBAC_Role_Drop_Cluster("1.0")]): + with Scenario( + "I drop role on fake cluster", + requirements=[RQ_SRS_006_RBAC_Role_Drop_Cluster("1.0")], + ): with When("I run drop role command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("DROP ROLE role2 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "DROP ROLE role2 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) diff --git a/tests/testflows/rbac/tests/syntax/drop_row_policy.py b/tests/testflows/rbac/tests/syntax/drop_row_policy.py index 357f5084bb3..7efda97b721 100755 --- a/tests/testflows/rbac/tests/syntax/drop_row_policy.py +++ b/tests/testflows/rbac/tests/syntax/drop_row_policy.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("drop row policy") def feature(self, node="clickhouse1"): @@ -39,96 +40,163 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE TABLE default.foo (x UInt64, y String) Engine=Memory") node.query(f"CREATE TABLE default.foo2 (x UInt64, y String) Engine=Memory") - with Scenario("I drop row policy with no options", requirements=[ + with Scenario( + "I drop row policy with no options", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): with cleanup(["policy1"]): with When("I drop row policy"): node.query("DROP ROW POLICY policy1 ON default.foo") - with Scenario("I drop row policy using short syntax with no options", requirements=[ + with Scenario( + "I drop row policy using short syntax with no options", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): with cleanup(["policy2"]): with When("I drop row policy short form"): node.query("DROP POLICY policy2 ON default.foo") - with Scenario("I drop row policy, does not exist, throws exception", requirements=[ + with Scenario( + "I drop row policy, does not exist, throws exception", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): policy = "policy1" cleanup_policy(policy) with When("I drop row policy"): - exitcode, message = errors.row_policy_not_found_in_disk(name=f"{policy} ON default.foo") - node.query(f"DROP ROW POLICY {policy} ON default.foo", exitcode=exitcode, message=message) + exitcode, message = errors.row_policy_not_found_in_disk( + name=f"{policy} ON default.foo" + ) + node.query( + f"DROP ROW POLICY {policy} ON default.foo", + exitcode=exitcode, + message=message, + ) del policy - with Scenario("I drop row policy if exists, policy does exist", requirements=[ + with Scenario( + "I drop row policy if exists, policy does exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop_IfExists("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): with cleanup(["policy3"]): with When("I drop row policy if exists"): node.query("DROP ROW POLICY IF EXISTS policy3 ON default.foo") - with Scenario("I drop row policy if exists, policy doesn't exist", requirements=[ + with Scenario( + "I drop row policy if exists, policy doesn't exist", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop_IfExists("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): cleanup_policy("policy3") with When("I drop row policy if exists"): node.query("DROP ROW POLICY IF EXISTS policy3 ON default.foo") - with Scenario("I drop multiple row policies", requirements=[ + with Scenario( + "I drop multiple row policies", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): with cleanup(["policy3", "policy4"]): with When("I drop multiple row policies"): node.query("DROP ROW POLICY policy3, policy4 ON default.foo") - with Scenario("I drop row policy on multiple tables", requirements=[ + with Scenario( + "I drop row policy on multiple tables", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): - with cleanup(["policy3"], ["default.foo","default.foo2"]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): + with cleanup(["policy3"], ["default.foo", "default.foo2"]): with When("I drop row policy on multiple tables"): node.query("DROP ROW POLICY policy3 ON default.foo, default.foo2") - with Scenario("I drop multiple row policies on multiple tables", requirements=[ + with Scenario( + "I drop multiple row policies on multiple tables", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): - with cleanup(["policy3", "policy4"], ["default.foo","default.foo2"]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): + with cleanup(["policy3", "policy4"], ["default.foo", "default.foo2"]): with When("I drop the row policies from the tables"): - node.query("DROP ROW POLICY policy3 ON default.foo, policy4 ON default.foo2") + node.query( + "DROP ROW POLICY policy3 ON default.foo, policy4 ON default.foo2" + ) - with Scenario("I drop row policy on cluster", requirements=[ + with Scenario( + "I drop row policy on cluster", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): try: with Given("I have a row policy"): - node.query("CREATE ROW POLICY policy13 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "CREATE ROW POLICY policy13 ON default.foo ON CLUSTER sharded_cluster" + ) with When("I run drop row policy command"): - node.query("DROP ROW POLICY IF EXISTS policy13 ON CLUSTER sharded_cluster ON default.foo") + node.query( + "DROP ROW POLICY IF EXISTS policy13 ON CLUSTER sharded_cluster ON default.foo" + ) finally: with Finally("I drop the row policy in case it still exists"): - node.query("DROP ROW POLICY IF EXISTS policy13 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "DROP ROW POLICY IF EXISTS policy13 ON default.foo ON CLUSTER sharded_cluster" + ) - with Scenario("I drop row policy on cluster after table", requirements=[ + with Scenario( + "I drop row policy on cluster after table", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): try: with Given("I have a row policy"): - node.query("CREATE ROW POLICY policy12 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "CREATE ROW POLICY policy12 ON default.foo ON CLUSTER sharded_cluster" + ) with When("I run drop row policy command"): - node.query("DROP ROW POLICY IF EXISTS policy13 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "DROP ROW POLICY IF EXISTS policy13 ON default.foo ON CLUSTER sharded_cluster" + ) finally: with Finally("I drop the row policy in case it still exists"): - node.query("DROP ROW POLICY IF EXISTS policy12 ON default.foo ON CLUSTER sharded_cluster") + node.query( + "DROP ROW POLICY IF EXISTS policy12 ON default.foo ON CLUSTER sharded_cluster" + ) - with Scenario("I drop row policy on fake cluster throws exception", requirements=[ + with Scenario( + "I drop row policy on fake cluster throws exception", + requirements=[ RQ_SRS_006_RBAC_RowPolicy_Drop_OnCluster("1.0"), - RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0")]): + RQ_SRS_006_RBAC_RowPolicy_Drop_On("1.0"), + ], + ): with When("I run drop row policy command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("DROP ROW POLICY IF EXISTS policy14 ON default.foo ON CLUSTER fake_cluster", - exitcode=exitcode, message=message) + node.query( + "DROP ROW POLICY IF EXISTS policy14 ON default.foo ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the tables"): node.query(f"DROP TABLE IF EXISTS default.foo") diff --git a/tests/testflows/rbac/tests/syntax/drop_settings_profile.py b/tests/testflows/rbac/tests/syntax/drop_settings_profile.py index 514c3042679..de69bc7e0a7 100755 --- a/tests/testflows/rbac/tests/syntax/drop_settings_profile.py +++ b/tests/testflows/rbac/tests/syntax/drop_settings_profile.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("drop settings profile") def feature(self, node="clickhouse1"): @@ -30,64 +31,94 @@ def feature(self, node="clickhouse1"): with Given(f"I ensure that profile {profile} does not exist"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") - with Scenario("I drop settings profile with no options", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")]): + with Scenario( + "I drop settings profile with no options", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")], + ): with cleanup("profile0"): with When("I drop settings profile"): node.query("DROP SETTINGS PROFILE profile0") - with Scenario("I drop settings profile, does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")]): + with Scenario( + "I drop settings profile, does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")], + ): profile = "profile0" cleanup_profile(profile) with When("I drop settings profile"): exitcode, message = errors.settings_profile_not_found_in_disk(name=profile) - node.query("DROP SETTINGS PROFILE profile0", exitcode=exitcode, message=message) + node.query( + "DROP SETTINGS PROFILE profile0", exitcode=exitcode, message=message + ) del profile - with Scenario("I drop settings profile short form", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")]): + with Scenario( + "I drop settings profile short form", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")], + ): with cleanup("profile1"): with When("I drop settings profile short form"): node.query("DROP PROFILE profile1") - with Scenario("I drop settings profile if exists, profile does exist", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop_IfExists("1.0")]): + with Scenario( + "I drop settings profile if exists, profile does exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop_IfExists("1.0")], + ): with cleanup("profile2"): with When("I drop settings profile if exists"): node.query("DROP SETTINGS PROFILE IF EXISTS profile2") - with Scenario("I drop settings profile if exists, profile does not exist", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop_IfExists("1.0")]): + with Scenario( + "I drop settings profile if exists, profile does not exist", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop_IfExists("1.0")], + ): cleanup_profile("profile2") with When("I drop settings profile if exists"): node.query("DROP SETTINGS PROFILE IF EXISTS profile2") - with Scenario("I drop default settings profile, throws error", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")]): + with Scenario( + "I drop default settings profile, throws error", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")], + ): with When("I drop default profile"): exitcode, message = errors.cannot_remove_settings_profile_default() - node.query("DROP SETTINGS PROFILE default", exitcode=exitcode, message=message) + node.query( + "DROP SETTINGS PROFILE default", exitcode=exitcode, message=message + ) - with Scenario("I drop multiple settings profiles", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")]): + with Scenario( + "I drop multiple settings profiles", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop("1.0")], + ): with cleanup("profile3"), cleanup("profile4"): with When("I drop multiple settings profiles"): node.query("DROP SETTINGS PROFILE profile3, profile4") - with Scenario("I drop settings profile on cluster", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop_OnCluster("1.0")]): + with Scenario( + "I drop settings profile on cluster", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop_OnCluster("1.0")], + ): try: with Given("I have a settings profile"): - node.query("CREATE SETTINGS PROFILE profile5 ON CLUSTER sharded_cluster") + node.query( + "CREATE SETTINGS PROFILE profile5 ON CLUSTER sharded_cluster" + ) with When("I run drop settings profile command"): node.query("DROP SETTINGS PROFILE profile5 ON CLUSTER sharded_cluster") finally: with Finally("I drop the profile in case it still exists"): - node.query("DROP SETTINGS PROFILE IF EXISTS profile5 ON CLUSTER sharded_cluster") + node.query( + "DROP SETTINGS PROFILE IF EXISTS profile5 ON CLUSTER sharded_cluster" + ) - with Scenario("I drop settings profile on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_Drop_OnCluster("1.0")]): + with Scenario( + "I drop settings profile on fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_Drop_OnCluster("1.0")], + ): with When("I run drop settings profile command"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("DROP SETTINGS PROFILE profile6 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "DROP SETTINGS PROFILE profile6 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) diff --git a/tests/testflows/rbac/tests/syntax/drop_user.py b/tests/testflows/rbac/tests/syntax/drop_user.py index 9bd2433d487..287d61fdbe0 100755 --- a/tests/testflows/rbac/tests/syntax/drop_user.py +++ b/tests/testflows/rbac/tests/syntax/drop_user.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("drop user") def feature(self, node="clickhouse1"): @@ -30,69 +31,91 @@ def feature(self, node="clickhouse1"): with Given(f"I ensure that user {user} does not exist"): node.query(f"DROP USER IF EXISTS {user}") - with Scenario("I drop user with no options", requirements=[ - RQ_SRS_006_RBAC_User_Drop("1.0")]): + with Scenario( + "I drop user with no options", requirements=[RQ_SRS_006_RBAC_User_Drop("1.0")] + ): with setup("user0"): with When("I drop user"): node.query("DROP USER user0") - with Scenario("I drop user, does not exist, throws exception", requirements=[ - RQ_SRS_006_RBAC_User_Drop("1.0")]): - user = "user0" - cleanup_user(user) - with When(f"I drop user {user}"): - exitcode, message = errors.user_not_found_in_disk(name=user) - node.query(f"DROP USER {user}", exitcode=exitcode, message=message) - del user + with Scenario( + "I drop user, does not exist, throws exception", + requirements=[RQ_SRS_006_RBAC_User_Drop("1.0")], + ): + user = "user0" + cleanup_user(user) + with When(f"I drop user {user}"): + exitcode, message = errors.user_not_found_in_disk(name=user) + node.query(f"DROP USER {user}", exitcode=exitcode, message=message) + del user - with Scenario("I drop multiple users", requirements=[ - RQ_SRS_006_RBAC_User_Drop("1.0")]): + with Scenario( + "I drop multiple users", requirements=[RQ_SRS_006_RBAC_User_Drop("1.0")] + ): with setup("user1"), setup("user2"): with When("I drop multiple users"): node.query("DROP USER user1, user2") - with Scenario("I drop user if exists, user does exist", requirements=[ - RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")]): + with Scenario( + "I drop user if exists, user does exist", + requirements=[RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")], + ): with setup("user3"): with When("I drop user that exists"): node.query("DROP USER IF EXISTS user3") - with Scenario("I drop user if exists, user does not exist", requirements=[ - RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")]): + with Scenario( + "I drop user if exists, user does not exist", + requirements=[RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")], + ): cleanup_user("user3") with When("I drop nonexistant user"): node.query("DROP USER IF EXISTS user3") - with Scenario("I drop default user, throws error", requirements=[ - RQ_SRS_006_RBAC_User_Drop("1.0")]): + with Scenario( + "I drop default user, throws error", + requirements=[RQ_SRS_006_RBAC_User_Drop("1.0")], + ): with When("I drop user"): exitcode, message = errors.cannot_remove_user_default() node.query("DROP USER default", exitcode=exitcode, message=message) - with Scenario("I drop multiple users where one does not exist", requirements=[ - RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")]): + with Scenario( + "I drop multiple users where one does not exist", + requirements=[RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")], + ): with setup("user3"): with When("I drop multiple users where one does not exist"): node.query("DROP USER IF EXISTS user3, user4") - with Scenario("I drop multiple users where both do not exist", requirements=[ - RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")]): + with Scenario( + "I drop multiple users where both do not exist", + requirements=[RQ_SRS_006_RBAC_User_Drop_IfExists("1.0")], + ): with When("I drop the nonexistant users"): node.query("DROP USER IF EXISTS user5, user6") - with Scenario("I drop user from specific cluster", requirements=[ - RQ_SRS_006_RBAC_User_Drop_OnCluster("1.0")]): - try: + with Scenario( + "I drop user from specific cluster", + requirements=[RQ_SRS_006_RBAC_User_Drop_OnCluster("1.0")], + ): + try: with Given("I have a user on cluster"): node.query("CREATE USER user4 ON CLUSTER sharded_cluster") with When("I drop a user from the cluster"): node.query("DROP USER user4 ON CLUSTER sharded_cluster") - finally: - with Finally("I make sure the user is dropped"): - node.query("DROP USER IF EXISTS user4 ON CLUSTER sharded_cluster") + finally: + with Finally("I make sure the user is dropped"): + node.query("DROP USER IF EXISTS user4 ON CLUSTER sharded_cluster") - with Scenario("I drop user from fake cluster", requirements=[ - RQ_SRS_006_RBAC_User_Drop_OnCluster("1.0")]): + with Scenario( + "I drop user from fake cluster", + requirements=[RQ_SRS_006_RBAC_User_Drop_OnCluster("1.0")], + ): with When("I drop a user from the fake cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("DROP USER user5 ON CLUSTER fake_cluster", exitcode=exitcode, message=message) + node.query( + "DROP USER user5 ON CLUSTER fake_cluster", + exitcode=exitcode, + message=message, + ) diff --git a/tests/testflows/rbac/tests/syntax/feature.py b/tests/testflows/rbac/tests/syntax/feature.py index b7c23f8d7ee..0e8ea921d43 100755 --- a/tests/testflows/rbac/tests/syntax/feature.py +++ b/tests/testflows/rbac/tests/syntax/feature.py @@ -1,5 +1,6 @@ from testflows.core import * + @TestFeature @Name("syntax") def feature(self): @@ -12,10 +13,10 @@ def feature(self): Feature(run=load("rbac.tests.syntax.drop_role", "feature")) Feature(run=load("rbac.tests.syntax.show_create_role", "feature")) Feature(run=load("rbac.tests.syntax.grant_role", "feature")) - Feature(run=load("rbac.tests.syntax.grant_privilege","feature")) + Feature(run=load("rbac.tests.syntax.grant_privilege", "feature")) Feature(run=load("rbac.tests.syntax.show_grants", "feature")) Feature(run=load("rbac.tests.syntax.revoke_role", "feature")) - Feature(run=load("rbac.tests.syntax.revoke_privilege","feature")) + Feature(run=load("rbac.tests.syntax.revoke_privilege", "feature")) Feature(run=load("rbac.tests.syntax.create_row_policy", "feature")) Feature(run=load("rbac.tests.syntax.alter_row_policy", "feature")) Feature(run=load("rbac.tests.syntax.drop_row_policy", "feature")) @@ -31,4 +32,4 @@ def feature(self): Feature(run=load("rbac.tests.syntax.drop_settings_profile", "feature")) Feature(run=load("rbac.tests.syntax.show_create_settings_profile", "feature")) Feature(run=load("rbac.tests.syntax.set_default_role", "feature")) - Feature(run=load("rbac.tests.syntax.set_role","feature")) \ No newline at end of file + Feature(run=load("rbac.tests.syntax.set_role", "feature")) diff --git a/tests/testflows/rbac/tests/syntax/grant_privilege.py b/tests/testflows/rbac/tests/syntax/grant_privilege.py index 817a70498f4..ab422f38eec 100755 --- a/tests/testflows/rbac/tests/syntax/grant_privilege.py +++ b/tests/testflows/rbac/tests/syntax/grant_privilege.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @contextmanager def setup(node): try: @@ -19,31 +20,142 @@ def setup(node): node.query("DROP USER IF EXISTS user1") node.query("DROP ROLE IF EXISTS role1") + @TestOutline(Scenario) -@Examples("privilege on allow_column allow_introspection", [ - ("dictGet", ("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_DictGet("1.0"))), - ("INTROSPECTION", ("*.*",), False, True, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Introspection("1.0"))), - ("SELECT", ("db0.table0","db0.*","*.*","tb0","*"), True, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Select("1.0"))), - ("INSERT",("db0.table0","db0.*","*.*","tb0","*"), True, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Insert("1.0"))), - ("ALTER",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Alter("1.0"))), - ("CREATE",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Create("1.0"))), - ("DROP",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Drop("1.0"))), - ("TRUNCATE",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Truncate("1.0"))), - ("OPTIMIZE",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Optimize("1.0"))), - ("SHOW",("db0.table0","db0.*","*.*","tb0","*"), True, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Show("1.0"))), - ("KILL QUERY",("*.*",), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_KillQuery("1.0"))), - ("ACCESS MANAGEMENT",("*.*",), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_AccessManagement("1.0"))), - ("SYSTEM",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_System("1.0"))), - ("SOURCES",("*.*",), False, False, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Sources("1.0"))), - ("ALL",("*.*",), True, True, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_All("1.0"))), - ("ALL PRIVILEGES",("*.*",), True, True, Requirements(RQ_SRS_006_RBAC_Grant_Privilege_All("1.0"))), #alias for all - ],) -def grant_privileges(self, privilege, on, allow_column, allow_introspection, node="clickhouse1"): - grant_privilege(privilege=privilege, on=on, allow_column=allow_column, allow_introspection=allow_introspection, node=node) +@Examples( + "privilege on allow_column allow_introspection", + [ + ( + "dictGet", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_DictGet("1.0")), + ), + ( + "INTROSPECTION", + ("*.*",), + False, + True, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Introspection("1.0")), + ), + ( + "SELECT", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + True, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Select("1.0")), + ), + ( + "INSERT", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + True, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Insert("1.0")), + ), + ( + "ALTER", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Alter("1.0")), + ), + ( + "CREATE", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Create("1.0")), + ), + ( + "DROP", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Drop("1.0")), + ), + ( + "TRUNCATE", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Truncate("1.0")), + ), + ( + "OPTIMIZE", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Optimize("1.0")), + ), + ( + "SHOW", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + True, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Show("1.0")), + ), + ( + "KILL QUERY", + ("*.*",), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_KillQuery("1.0")), + ), + ( + "ACCESS MANAGEMENT", + ("*.*",), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_AccessManagement("1.0")), + ), + ( + "SYSTEM", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_System("1.0")), + ), + ( + "SOURCES", + ("*.*",), + False, + False, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_Sources("1.0")), + ), + ( + "ALL", + ("*.*",), + True, + True, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_All("1.0")), + ), + ( + "ALL PRIVILEGES", + ("*.*",), + True, + True, + Requirements(RQ_SRS_006_RBAC_Grant_Privilege_All("1.0")), + ), # alias for all + ], +) +def grant_privileges( + self, privilege, on, allow_column, allow_introspection, node="clickhouse1" +): + grant_privilege( + privilege=privilege, + on=on, + allow_column=allow_column, + allow_introspection=allow_introspection, + node=node, + ) + @TestOutline(Scenario) @Requirements(RQ_SRS_006_RBAC_Grant_Privilege_GrantOption("1.0")) -def grant_privilege(self, privilege, on, allow_column, allow_introspection, node="clickhouse1"): +def grant_privilege( + self, privilege, on, allow_column, allow_introspection, node="clickhouse1" +): node = self.context.cluster.node(node) for on_ in on: @@ -54,14 +166,22 @@ def grant_privilege(self, privilege, on, allow_column, allow_introspection, node settings.append(("allow_introspection_functions", 1)) node.query("SET allow_introspection_functions = 1") with When("I grant privilege without grant option"): - node.query(f"GRANT {privilege} ON {on_} TO user0", settings=settings) + node.query( + f"GRANT {privilege} ON {on_} TO user0", settings=settings + ) with When("I grant privilege with grant option"): - node.query(f"GRANT {privilege} ON {on_} TO user1 WITH GRANT OPTION", settings=settings) + node.query( + f"GRANT {privilege} ON {on_} TO user1 WITH GRANT OPTION", + settings=settings, + ) - if allow_column and ('*' not in on_): + if allow_column and ("*" not in on_): # Grant column specific for some column 'x' with When("I grant privilege with columns"): - node.query(f"GRANT {privilege}(x) ON {on_} TO user0", settings=settings) + node.query( + f"GRANT {privilege}(x) ON {on_} TO user0", settings=settings + ) + @TestFeature @Name("grant privilege") @@ -82,53 +202,83 @@ def feature(self, node="clickhouse1"): Scenario(run=grant_privileges) # with nonexistant object name, GRANT assumes type role - with Scenario("I grant privilege to role that does not exist", requirements=[ - RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")]): + with Scenario( + "I grant privilege to role that does not exist", + requirements=[RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")], + ): with Given("I ensure that role does not exist"): node.query("DROP ROLE IF EXISTS role0") with When("I grant privilege ON CLUSTER"): exitcode, message = errors.role_not_found_in_disk(name="role0") node.query("GRANT NONE TO role0", exitcode=exitcode, message=message) - with Scenario("I grant privilege ON CLUSTER", requirements=[ + with Scenario( + "I grant privilege ON CLUSTER", + requirements=[ RQ_SRS_006_RBAC_Grant_Privilege_OnCluster("1.0"), - RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Grant_Privilege_None("1.0"), + ], + ): with setup(node): with When("I grant privilege ON CLUSTER"): node.query("GRANT ON CLUSTER sharded_cluster NONE TO user0") - with Scenario("I grant privilege on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Grant_Privilege_OnCluster("1.0")]): + with Scenario( + "I grant privilege on fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Grant_Privilege_OnCluster("1.0")], + ): with setup(node): with When("I grant privilege ON CLUSTER"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("GRANT ON CLUSTER fake_cluster NONE TO user0", exitcode=exitcode, message=message) + node.query( + "GRANT ON CLUSTER fake_cluster NONE TO user0", + exitcode=exitcode, + message=message, + ) - with Scenario("I grant privilege to multiple users and roles", requirements=[ + with Scenario( + "I grant privilege to multiple users and roles", + requirements=[ RQ_SRS_006_RBAC_Grant_Privilege_To("1.0"), - RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Grant_Privilege_None("1.0"), + ], + ): with setup(node): with When("I grant privilege to several users"): node.query("GRANT NONE TO user0, user1, role1") - with Scenario("I grant privilege to current user", requirements=[ + with Scenario( + "I grant privilege to current user", + requirements=[ RQ_SRS_006_RBAC_Grant_Privilege_ToCurrentUser("1.0"), - RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Grant_Privilege_None("1.0"), + ], + ): with setup(node): with When("I grant privilege to current user"): - node.query("GRANT NONE TO CURRENT_USER", settings = [("user","user0")]) + node.query("GRANT NONE TO CURRENT_USER", settings=[("user", "user0")]) - with Scenario("I grant privilege NONE to default user, throws exception", requirements=[ + with Scenario( + "I grant privilege NONE to default user, throws exception", + requirements=[ RQ_SRS_006_RBAC_Grant_Privilege_ToCurrentUser("1.0"), - RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Grant_Privilege_None("1.0"), + ], + ): with setup(node): with When("I grant privilege to current user"): exitcode, message = errors.cannot_update_default() - node.query("GRANT NONE TO CURRENT_USER", exitcode=exitcode, message=message) + node.query( + "GRANT NONE TO CURRENT_USER", exitcode=exitcode, message=message + ) - with Scenario("I grant privilege with grant option", requirements=[ + with Scenario( + "I grant privilege with grant option", + requirements=[ RQ_SRS_006_RBAC_Grant_Privilege_GrantOption("1.0"), - RQ_SRS_006_RBAC_Grant_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Grant_Privilege_None("1.0"), + ], + ): with setup(node): with When("I grant privilege with grant option"): - node.query("GRANT NONE ON *.* TO user0 WITH GRANT OPTION") \ No newline at end of file + node.query("GRANT NONE ON *.* TO user0 WITH GRANT OPTION") diff --git a/tests/testflows/rbac/tests/syntax/grant_role.py b/tests/testflows/rbac/tests/syntax/grant_role.py index baede2445ee..321513cb8b7 100755 --- a/tests/testflows/rbac/tests/syntax/grant_role.py +++ b/tests/testflows/rbac/tests/syntax/grant_role.py @@ -4,6 +4,8 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * +from helpers.common import check_clickhouse_version + @TestFeature @Name("grant role") @@ -18,7 +20,7 @@ def feature(self, node="clickhouse1"): node = self.context.cluster.node(node) @contextmanager - def setup(users=0,roles=0): + def setup(users=0, roles=0): try: with Given("I have some users and roles"): for i in range(users): @@ -33,69 +35,94 @@ def feature(self, node="clickhouse1"): for j in range(roles): node.query(f"DROP ROLE IF EXISTS role{j}") - with Scenario("I grant a role to a user", requirements=[ - RQ_SRS_006_RBAC_Grant_Role("1.0")]): - with setup(1,1): + with Scenario( + "I grant a role to a user", requirements=[RQ_SRS_006_RBAC_Grant_Role("1.0")] + ): + with setup(1, 1): with When("I grant a role"): node.query("GRANT role0 TO user0") - with Scenario("I grant a nonexistent role to user", requirements=[ - RQ_SRS_006_RBAC_Grant_Role("1.0")]): - with setup(1,0): + with Scenario( + "I grant a nonexistent role to user", + requirements=[RQ_SRS_006_RBAC_Grant_Role("1.0")], + ): + with setup(1, 0): with When("I grant nonexistent role to a user"): exitcode, message = errors.role_not_found_in_disk(name="role0") node.query("GRANT role0 TO user0", exitcode=exitcode, message=message) # with nonexistent object name, GRANT assumes type role (treats user0 as role) - with Scenario("I grant a role to a nonexistent user", requirements=[ - RQ_SRS_006_RBAC_Grant_Role("1.0")]): - with setup(0,1): + with Scenario( + "I grant a role to a nonexistent user", + requirements=[RQ_SRS_006_RBAC_Grant_Role("1.0")], + ): + with setup(0, 1): with When("I grant role to a nonexistent user"): exitcode, message = errors.role_not_found_in_disk(name="user0") node.query("GRANT role0 TO user0", exitcode=exitcode, message=message) - with Scenario("I grant a nonexistent role to a nonexistent user", requirements=[ - RQ_SRS_006_RBAC_Grant_Role("1.0")]): - with setup(0,0): + with Scenario( + "I grant a nonexistent role to a nonexistent user", + requirements=[RQ_SRS_006_RBAC_Grant_Role("1.0")], + ): + with setup(0, 0): with When("I grant nonexistent role to a nonexistent user"): - exitcode, message = errors.role_not_found_in_disk(name="user0") + exitcode, message = ( + errors.role_not_found_in_disk(name="user0") + if check_clickhouse_version(">=21.09")(self) + else errors.role_not_found_in_disk(name="role0") + ) node.query("GRANT role0 TO user0", exitcode=exitcode, message=message) - with Scenario("I grant a role to multiple users", requirements=[ - RQ_SRS_006_RBAC_Grant_Role("1.0")]): - with setup(2,1): + with Scenario( + "I grant a role to multiple users", + requirements=[RQ_SRS_006_RBAC_Grant_Role("1.0")], + ): + with setup(2, 1): with When("I grant role to a multiple users"): node.query("GRANT role0 TO user0, user1") - with Scenario("I grant multiple roles to multiple users", requirements=[ - RQ_SRS_006_RBAC_Grant_Role("1.0")]): - with setup(2,2): + with Scenario( + "I grant multiple roles to multiple users", + requirements=[RQ_SRS_006_RBAC_Grant_Role("1.0")], + ): + with setup(2, 2): with When("I grant multiple roles to multiple users"): node.query("GRANT role0, role1 TO user0, user1") - with Scenario("I grant role to current user", requirements=[ - RQ_SRS_006_RBAC_Grant_Role_CurrentUser("1.0")]): - with setup(1,1): + with Scenario( + "I grant role to current user", + requirements=[RQ_SRS_006_RBAC_Grant_Role_CurrentUser("1.0")], + ): + with setup(1, 1): with Given("I have a user with access management privilege"): node.query("GRANT ACCESS MANAGEMENT ON *.* TO user0") with When("I grant role to current user"): - node.query("GRANT role0 TO CURRENT_USER", settings = [("user","user0")]) + node.query("GRANT role0 TO CURRENT_USER", settings=[("user", "user0")]) - with Scenario("I grant role to default user, throws exception", requirements=[ - RQ_SRS_006_RBAC_Grant_Role_CurrentUser("1.0")]): - with setup(1,1): + with Scenario( + "I grant role to default user, throws exception", + requirements=[RQ_SRS_006_RBAC_Grant_Role_CurrentUser("1.0")], + ): + with setup(1, 1): with When("I grant role to default user"): exitcode, message = errors.cannot_update_default() - node.query("GRANT role0 TO CURRENT_USER", exitcode=exitcode, message=message) + node.query( + "GRANT role0 TO CURRENT_USER", exitcode=exitcode, message=message + ) - with Scenario("I grant role to user with admin option", requirements=[ - RQ_SRS_006_RBAC_Grant_Role_AdminOption("1.0")]): - with setup(1,1): + with Scenario( + "I grant role to user with admin option", + requirements=[RQ_SRS_006_RBAC_Grant_Role_AdminOption("1.0")], + ): + with setup(1, 1): with When("I grant role to a user with admin option"): node.query("GRANT role0 TO user0 WITH ADMIN OPTION") - with Scenario("I grant role to user on cluster", requirements=[ - RQ_SRS_006_RBAC_Grant_Role_OnCluster("1.0")]): + with Scenario( + "I grant role to user on cluster", + requirements=[RQ_SRS_006_RBAC_Grant_Role_OnCluster("1.0")], + ): try: with Given("I have a user and a role on a cluster"): node.query("CREATE USER OR REPLACE user0 ON CLUSTER sharded_cluster") @@ -107,9 +134,15 @@ def feature(self, node="clickhouse1"): node.query("DROP USER IF EXISTS user0 ON CLUSTER sharded_cluster") node.query("DROP ROLE IF EXISTS role0 ON CLUSTER sharded_cluster") - with Scenario("I grant role to user on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Grant_Role_OnCluster("1.0")]): - with setup(1,1): + with Scenario( + "I grant role to user on fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Grant_Role_OnCluster("1.0")], + ): + with setup(1, 1): with When("I grant the role to the user"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("GRANT ON CLUSTER fake_cluster role0 TO user0", exitcode=exitcode, message=message) + node.query( + "GRANT ON CLUSTER fake_cluster role0 TO user0", + exitcode=exitcode, + message=message, + ) diff --git a/tests/testflows/rbac/tests/syntax/revoke_privilege.py b/tests/testflows/rbac/tests/syntax/revoke_privilege.py index 3e23f2ddfc9..023dd803018 100755 --- a/tests/testflows/rbac/tests/syntax/revoke_privilege.py +++ b/tests/testflows/rbac/tests/syntax/revoke_privilege.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @contextmanager def setup(node): try: @@ -21,32 +22,142 @@ def setup(node): @TestOutline(Scenario) -@Examples("privilege on allow_column allow_introspection", [ - ("dictGet", ("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_DictGet("1.0"))), - ("INTROSPECTION", ("*.*",), False, True, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Introspection("1.0"))), - ("SELECT", ("db0.table0","db0.*","*.*","tb0","*"), True, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Select("1.0"))), - ("INSERT",("db0.table0","db0.*","*.*","tb0","*"), True, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Insert("1.0"))), - ("ALTER",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Alter("1.0"))), - ("CREATE",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Create("1.0"))), - ("DROP",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Drop("1.0"))), - ("TRUNCATE",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Truncate("1.0"))), - ("OPTIMIZE",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Optimize("1.0"))), - ("SHOW",("db0.table0","db0.*","*.*","tb0","*"), True, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Show("1.0"))), - ("KILL QUERY",("*.*",), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_KillQuery("1.0"))), - ("ACCESS MANAGEMENT",("*.*",), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_AccessManagement("1.0"))), - ("SYSTEM",("db0.table0","db0.*","*.*","tb0","*"), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_System("1.0"))), - ("SOURCES",("*.*",), False, False, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Sources("1.0"))), - ("ALL",("*.*",), True, True, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_All("1.0"))), - ("ALL PRIVILEGES",("*.*",), True, True, Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_All("1.0"))), #alias for all - ],) -def revoke_privileges(self, privilege, on, allow_column, allow_introspection, node="clickhouse1"): - revoke_privilege(privilege=privilege, on=on, allow_column=allow_column, allow_introspection=allow_introspection, node=node) +@Examples( + "privilege on allow_column allow_introspection", + [ + ( + "dictGet", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_DictGet("1.0")), + ), + ( + "INTROSPECTION", + ("*.*",), + False, + True, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Introspection("1.0")), + ), + ( + "SELECT", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + True, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Select("1.0")), + ), + ( + "INSERT", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + True, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Insert("1.0")), + ), + ( + "ALTER", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Alter("1.0")), + ), + ( + "CREATE", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Create("1.0")), + ), + ( + "DROP", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Drop("1.0")), + ), + ( + "TRUNCATE", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Truncate("1.0")), + ), + ( + "OPTIMIZE", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Optimize("1.0")), + ), + ( + "SHOW", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + True, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Show("1.0")), + ), + ( + "KILL QUERY", + ("*.*",), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_KillQuery("1.0")), + ), + ( + "ACCESS MANAGEMENT", + ("*.*",), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_AccessManagement("1.0")), + ), + ( + "SYSTEM", + ("db0.table0", "db0.*", "*.*", "tb0", "*"), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_System("1.0")), + ), + ( + "SOURCES", + ("*.*",), + False, + False, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_Sources("1.0")), + ), + ( + "ALL", + ("*.*",), + True, + True, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_All("1.0")), + ), + ( + "ALL PRIVILEGES", + ("*.*",), + True, + True, + Requirements(RQ_SRS_006_RBAC_Revoke_Privilege_All("1.0")), + ), # alias for all + ], +) +def revoke_privileges( + self, privilege, on, allow_column, allow_introspection, node="clickhouse1" +): + revoke_privilege( + privilege=privilege, + on=on, + allow_column=allow_column, + allow_introspection=allow_introspection, + node=node, + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_006_RBAC_Revoke_Privilege_PrivilegeColumns("1.0"), ) -def revoke_privilege(self, privilege, on, allow_column, allow_introspection, node="clickhouse1"): +def revoke_privilege( + self, privilege, on, allow_column, allow_introspection, node="clickhouse1" +): node = self.context.cluster.node(node) for on_ in on: with When(f"I revoke {privilege} privilege from user on {on_}"): @@ -56,12 +167,18 @@ def revoke_privilege(self, privilege, on, allow_column, allow_introspection, nod settings.append(("allow_introspection_functions", 1)) node.query("SET allow_introspection_functions = 1") with When("I revoke privilege without columns"): - node.query(f"REVOKE {privilege} ON {on_} FROM user0", settings=settings) + node.query( + f"REVOKE {privilege} ON {on_} FROM user0", settings=settings + ) - if allow_column and ('*' not in on_): + if allow_column and ("*" not in on_): # Revoke column specific for some column 'x' with When("I revoke privilege with columns"): - node.query(f"REVOKE {privilege}(x) ON {on_} FROM user0", settings=settings) + node.query( + f"REVOKE {privilege}(x) ON {on_} FROM user0", + settings=settings, + ) + @TestFeature @Name("revoke privilege") @@ -80,83 +197,134 @@ def feature(self, node="clickhouse1"): Scenario(run=revoke_privileges) - with Scenario("I revoke privilege ON CLUSTER", requirements=[ + with Scenario( + "I revoke privilege ON CLUSTER", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_Cluster("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege ON CLUSTER"): node.query("REVOKE ON CLUSTER sharded_cluster NONE FROM user0") - with Scenario("I revoke privilege ON fake CLUSTER, throws exception", requirements=[ + with Scenario( + "I revoke privilege ON fake CLUSTER, throws exception", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_Cluster("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege ON CLUSTER"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("REVOKE ON CLUSTER fake_cluster NONE FROM user0", - exitcode=exitcode, message=message) + node.query( + "REVOKE ON CLUSTER fake_cluster NONE FROM user0", + exitcode=exitcode, + message=message, + ) - with Scenario("I revoke privilege from multiple users and roles", requirements=[ + with Scenario( + "I revoke privilege from multiple users and roles", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege from multiple users"): node.query("REVOKE NONE FROM user0, user1, role1") - with Scenario("I revoke privilege from current user", requirements=[ + with Scenario( + "I revoke privilege from current user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege from current user"): - node.query("REVOKE NONE FROM CURRENT_USER", settings = [("user","user0")]) + node.query( + "REVOKE NONE FROM CURRENT_USER", settings=[("user", "user0")] + ) - with Scenario("I revoke privilege from all users", requirements=[ + with Scenario( + "I revoke privilege from all users", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege from all users"): exitcode, message = errors.cannot_update_default() - node.query("REVOKE NONE FROM ALL", exitcode=exitcode,message=message) + node.query("REVOKE NONE FROM ALL", exitcode=exitcode, message=message) - with Scenario("I revoke privilege from default user", requirements=[ + with Scenario( + "I revoke privilege from default user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege from default user"): exitcode, message = errors.cannot_update_default() - node.query("REVOKE NONE FROM default", exitcode=exitcode,message=message) + node.query( + "REVOKE NONE FROM default", exitcode=exitcode, message=message + ) - #By default, ClickHouse treats unnamed object as role - with Scenario("I revoke privilege from nonexistent role, throws exception", requirements=[ + # By default, ClickHouse treats unnamed object as role + with Scenario( + "I revoke privilege from nonexistent role, throws exception", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): role = "role5" with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") with When(f"I revoke privilege from nonexistent role {role}"): exitcode, message = errors.role_not_found_in_disk(role) - node.query(f"REVOKE NONE FROM {role}", exitcode=exitcode,message=message) + node.query(f"REVOKE NONE FROM {role}", exitcode=exitcode, message=message) - with Scenario("I revoke privilege from ALL EXCEPT nonexistent role, throws exception", requirements=[ + with Scenario( + "I revoke privilege from ALL EXCEPT nonexistent role, throws exception", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): role = "role5" with Given(f"I ensure that role {role} does not exist"): node.query(f"DROP ROLE IF EXISTS {role}") with When(f"I revoke privilege from nonexistent role {role}"): exitcode, message = errors.role_not_found_in_disk(role) - node.query(f"REVOKE NONE FROM ALL EXCEPT {role}", exitcode=exitcode,message=message) + node.query( + f"REVOKE NONE FROM ALL EXCEPT {role}", + exitcode=exitcode, + message=message, + ) - with Scenario("I revoke privilege from all except some users and roles", requirements=[ + with Scenario( + "I revoke privilege from all except some users and roles", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege all except some users"): node.query("REVOKE NONE FROM ALL EXCEPT default, user0, role1") - with Scenario("I revoke privilege from all except current user", requirements=[ + with Scenario( + "I revoke privilege from all except current user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Privilege_From("1.0"), - RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0")]): + RQ_SRS_006_RBAC_Revoke_Privilege_None("1.0"), + ], + ): with setup(node): with When("I revoke privilege from all except current user"): - node.query("REVOKE NONE FROM ALL EXCEPT CURRENT_USER") \ No newline at end of file + node.query("REVOKE NONE FROM ALL EXCEPT CURRENT_USER") diff --git a/tests/testflows/rbac/tests/syntax/revoke_role.py b/tests/testflows/rbac/tests/syntax/revoke_role.py index 6fe72b14f7e..4e18ad13652 100755 --- a/tests/testflows/rbac/tests/syntax/revoke_role.py +++ b/tests/testflows/rbac/tests/syntax/revoke_role.py @@ -4,6 +4,8 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * +from helpers.common import check_clickhouse_version + @TestFeature @Name("revoke role") @@ -20,7 +22,7 @@ def feature(self, node="clickhouse1"): node = self.context.cluster.node(node) @contextmanager - def setup(users=2,roles=2): + def setup(users=2, roles=2): try: with Given("I have some users"): for i in range(users): @@ -37,124 +39,194 @@ def feature(self, node="clickhouse1"): for i in range(roles): node.query(f"DROP ROLE IF EXISTS role{i}") - with Scenario("I revoke a role from a user", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): + with Scenario( + "I revoke a role from a user", requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")] + ): with setup(): with When("I revoke a role"): node.query("REVOKE role0 FROM user0") - with Scenario("I revoke a nonexistent role from user", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): - with setup(1,0): + with Scenario( + "I revoke a nonexistent role from user", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): + with setup(1, 0): with When("I revoke nonexistent role from a user"): exitcode, message = errors.role_not_found_in_disk(name="role0") - node.query("REVOKE role0 FROM user0", exitcode=exitcode, message=message) + node.query( + "REVOKE role0 FROM user0", exitcode=exitcode, message=message + ) # with nonexistent object name, REVOKE assumes type role (treats user0 as role) - with Scenario("I revoke a role from a nonexistent user", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): - with setup(0,1): + with Scenario( + "I revoke a role from a nonexistent user", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): + with setup(0, 1): with When("I revoke role from a nonexistent user"): exitcode, message = errors.role_not_found_in_disk(name="user0") - node.query("REVOKE role0 FROM user0", exitcode=exitcode, message=message) + node.query( + "REVOKE role0 FROM user0", exitcode=exitcode, message=message + ) # with nonexistent object name, REVOKE assumes type role (treats user0 as role) - with Scenario("I revoke a role from ALL EXCEPT nonexistent user", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): - with setup(0,1): + with Scenario( + "I revoke a role from ALL EXCEPT nonexistent user", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): + with setup(0, 1): with When("I revoke role from a nonexistent user"): exitcode, message = errors.role_not_found_in_disk(name="user0") - node.query("REVOKE role0 FROM ALL EXCEPT user0", exitcode=exitcode, message=message) + node.query( + "REVOKE role0 FROM ALL EXCEPT user0", + exitcode=exitcode, + message=message, + ) - with Scenario("I revoke a nonexistent role from a nonexistent user", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): - with setup(0,0): + with Scenario( + "I revoke a nonexistent role from a nonexistent user", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): + with setup(0, 0): with When("I revoke nonexistent role from a nonexistent user"): - exitcode, message = errors.role_not_found_in_disk(name="user0") - node.query("REVOKE role0 FROM user0", exitcode=exitcode, message=message) + exitcode, message = ( + errors.role_not_found_in_disk(name="user0") + if check_clickhouse_version(">=21.09")(self) + else errors.role_not_found_in_disk(name="role0") + ) + node.query( + "REVOKE role0 FROM user0", exitcode=exitcode, message=message + ) - with Scenario("I revoke a role from multiple users", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): + with Scenario( + "I revoke a role from multiple users", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): with setup(): with When("I revoke a role from multiple users"): node.query("REVOKE role0 FROM user0, user1") - with Scenario("I revoke multiple roles from multiple users", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): + with Scenario( + "I revoke multiple roles from multiple users", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): with setup(): node.query("REVOKE role0, role1 FROM user0, user1") - #user is default, expect exception - with Scenario("I revoke a role from default user", requirements=[ + # user is default, expect exception + with Scenario( + "I revoke a role from default user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke a role from default user"): exitcode, message = errors.cannot_update_default() - node.query("REVOKE role0 FROM CURRENT_USER", exitcode=exitcode, message=message) + node.query( + "REVOKE role0 FROM CURRENT_USER", exitcode=exitcode, message=message + ) - #user is user0 - with Scenario("I revoke a role from current user", requirements=[ + # user is user0 + with Scenario( + "I revoke a role from current user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke a role from current user"): - node.query("REVOKE role0 FROM CURRENT_USER", settings = [("user","user0")]) + node.query( + "REVOKE role0 FROM CURRENT_USER", settings=[("user", "user0")] + ) - #user is default, expect exception - with Scenario("I revoke a role from all", requirements=[ + # user is default, expect exception + with Scenario( + "I revoke a role from all", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke a role from all"): exitcode, message = errors.cannot_update_default() node.query("REVOKE role0 FROM ALL", exitcode=exitcode, message=message) - #user is default, expect exception - with Scenario("I revoke multiple roles from all", requirements=[ + # user is default, expect exception + with Scenario( + "I revoke multiple roles from all", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke multiple roles from all"): exitcode, message = errors.cannot_update_default() - node.query("REVOKE role0, role1 FROM ALL", exitcode=exitcode, message=message) + node.query( + "REVOKE role0, role1 FROM ALL", exitcode=exitcode, message=message + ) - with Scenario("I revoke a role from all but current user", requirements=[ + with Scenario( + "I revoke a role from all but current user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke a role from all except current"): node.query("REVOKE role0 FROM ALL EXCEPT CURRENT_USER") - with Scenario("I revoke a role from all but default user", requirements=[ + with Scenario( + "I revoke a role from all but default user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke a role from all except default"): - node.query("REVOKE role0 FROM ALL EXCEPT default", - settings = [("user","user0")]) + node.query( + "REVOKE role0 FROM ALL EXCEPT default", settings=[("user", "user0")] + ) - with Scenario("I revoke multiple roles from all but default user", requirements=[ + with Scenario( + "I revoke multiple roles from all but default user", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Keywords("1.0"), + ], + ): with setup(): with When("I revoke multiple roles from all except default"): - node.query("REVOKE role0, role1 FROM ALL EXCEPT default", settings = [("user","user0")]) + node.query( + "REVOKE role0, role1 FROM ALL EXCEPT default", + settings=[("user", "user0")], + ) - with Scenario("I revoke a role from a role", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): + with Scenario( + "I revoke a role from a role", requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")] + ): with setup(): with When("I revoke a role from a role"): node.query("REVOKE role0 FROM role1") - with Scenario("I revoke a role from a role and a user", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role("1.0")]): + with Scenario( + "I revoke a role from a role and a user", + requirements=[RQ_SRS_006_RBAC_Revoke_Role("1.0")], + ): with setup(): with When("I revoke a role from multiple roles"): node.query("REVOKE role0 FROM role1, user0") - with Scenario("I revoke a role from a user on cluster", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0")]): + with Scenario( + "I revoke a role from a user on cluster", + requirements=[RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0")], + ): with Given("I have a role and a user on a cluster"): node.query("CREATE USER OR REPLACE user0 ON CLUSTER sharded_cluster") node.query("CREATE ROLE OR REPLACE role0 ON CLUSTER sharded_cluster") @@ -164,41 +236,59 @@ def feature(self, node="clickhouse1"): node.query("DROP USER IF EXISTS user0 ON CLUSTER sharded_cluster") node.query("DROP ROLE IF EXISTS role0 ON CLUSTER sharded_cluster") - with Scenario("I revoke a role on fake cluster, throws exception", requirements=[ - RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0")]): + with Scenario( + "I revoke a role on fake cluster, throws exception", + requirements=[RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0")], + ): with Given("I have a role and a user on a cluster"): node.query("CREATE USER OR REPLACE user0") node.query("CREATE ROLE OR REPLACE role0") with When("I revoke a role from user on a cluster"): exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("REVOKE ON CLUSTER fake_cluster role0 FROM user0", exitcode=exitcode, message=message) + node.query( + "REVOKE ON CLUSTER fake_cluster role0 FROM user0", + exitcode=exitcode, + message=message, + ) with Finally("I drop the user and role"): node.query("DROP USER IF EXISTS user0") node.query("DROP ROLE IF EXISTS role0") - with Scenario("I revoke multiple roles from multiple users on cluster", requirements=[ + with Scenario( + "I revoke multiple roles from multiple users on cluster", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0")]): + RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0"), + ], + ): with Given("I have multiple roles and multiple users on a cluster"): for i in range(2): node.query(f"CREATE USER OR REPLACE user{i} ON CLUSTER sharded_cluster") node.query(f"CREATE ROLE OR REPLACE role{i} ON CLUSTER sharded_cluster") with When("I revoke multiple roles from multiple users on cluster"): - node.query("REVOKE ON CLUSTER sharded_cluster role0, role1 FROM user0, user1") + node.query( + "REVOKE ON CLUSTER sharded_cluster role0, role1 FROM user0, user1" + ) with Finally("I drop the roles and users"): for i in range(2): node.query(f"DROP USER IF EXISTS user{i} ON CLUSTER sharded_cluster") node.query(f"DROP ROLE IF EXISTS role{i} ON CLUSTER sharded_cluster") - with Scenario("I revoke admin option for role from a user", requirements=[ - RQ_SRS_006_RBAC_Revoke_AdminOption("1.0")]): + with Scenario( + "I revoke admin option for role from a user", + requirements=[RQ_SRS_006_RBAC_Revoke_AdminOption("1.0")], + ): with setup(): with When("I revoke admin option for role from a user"): node.query("REVOKE ADMIN OPTION FOR role0 FROM user0") - with Scenario("I revoke admin option for multiple roles from multiple users", requirements=[ + with Scenario( + "I revoke admin option for multiple roles from multiple users", + requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), - RQ_SRS_006_RBAC_Revoke_AdminOption("1.0")]): + RQ_SRS_006_RBAC_Revoke_AdminOption("1.0"), + ], + ): with setup(): with When("I revoke admin option for multiple roles from multiple users"): node.query("REVOKE ADMIN OPTION FOR role0, role1 FROM user0, user1") diff --git a/tests/testflows/rbac/tests/syntax/set_default_role.py b/tests/testflows/rbac/tests/syntax/set_default_role.py index ed50810eba7..6fec27dea61 100755 --- a/tests/testflows/rbac/tests/syntax/set_default_role.py +++ b/tests/testflows/rbac/tests/syntax/set_default_role.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("set default role") @Args(format_description=False) @@ -18,7 +19,7 @@ def feature(self, node="clickhouse1"): node = self.context.cluster.node(node) @contextmanager - def setup(users=2,roles=2): + def setup(users=2, roles=2): try: with Given("I have some users"): for i in range(users): @@ -35,34 +36,58 @@ def feature(self, node="clickhouse1"): for i in range(roles): node.query(f"DROP ROLE IF EXISTS role{i}") - with Scenario("I set default a nonexistent role to user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): - with setup(1,0): + with Scenario( + "I set default a nonexistent role to user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): + with setup(1, 0): with When("I set default nonexistent role to a user"): exitcode, message = errors.role_not_found_in_disk(name="role0") - node.query("SET DEFAULT ROLE role0 TO user0", exitcode=exitcode, message=message) + node.query( + "SET DEFAULT ROLE role0 TO user0", + exitcode=exitcode, + message=message, + ) - with Scenario("I set default ALL EXCEPT a nonexistent role to user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): - with setup(1,0): + with Scenario( + "I set default ALL EXCEPT a nonexistent role to user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): + with setup(1, 0): with When("I set default nonexistent role to a user"): exitcode, message = errors.role_not_found_in_disk(name="role0") - node.query("SET DEFAULT ROLE ALL EXCEPT role0 TO user0", exitcode=exitcode, message=message) + node.query( + "SET DEFAULT ROLE ALL EXCEPT role0 TO user0", + exitcode=exitcode, + message=message, + ) - with Scenario("I set default a role to a nonexistent user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): - with setup(0,1): + with Scenario( + "I set default a role to a nonexistent user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): + with setup(0, 1): with When("I set default role to a nonexistent user"): exitcode, message = errors.user_not_found_in_disk(name="user0") - node.query("SET DEFAULT ROLE role0 TO user0", exitcode=exitcode, message=message) + node.query( + "SET DEFAULT ROLE role0 TO user0", + exitcode=exitcode, + message=message, + ) - #in SET DEFAULT ROLE, the nonexistent user is noticed first and becomes the thrown exception - with Scenario("I set default a nonexistent role to a nonexistent user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): - with setup(0,0): + # in SET DEFAULT ROLE, the nonexistent user is noticed first and becomes the thrown exception + with Scenario( + "I set default a nonexistent role to a nonexistent user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): + with setup(0, 0): with When("I set default nonexistent role to a nonexistent user"): exitcode, message = errors.user_not_found_in_disk(name="user0") - node.query("SET DEFAULT ROLE role0 TO user0", exitcode=exitcode, message=message) + node.query( + "SET DEFAULT ROLE role0 TO user0", + exitcode=exitcode, + message=message, + ) try: with Given("I have some roles and some users"): @@ -71,47 +96,70 @@ def feature(self, node="clickhouse1"): node.query(f"CREATE USER user{i}") node.query(f"GRANT role0, role1 TO user0, user1") - with Scenario("I set default role for a user to none", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole_None("1.0")]): + with Scenario( + "I set default role for a user to none", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole_None("1.0")], + ): with When("I set no roles default for user"): node.query("SET DEFAULT ROLE NONE TO user0") - with Scenario("I set one default role for a user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): + with Scenario( + "I set one default role for a user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): with When("I set a default role for user "): node.query("SET DEFAULT ROLE role0 TO user0") - with Scenario("I set one default role for user default, throws exception", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): + with Scenario( + "I set one default role for user default, throws exception", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): with When("I set a default role for default"): exitcode, message = errors.cannot_update_default() - node.query("SET DEFAULT ROLE role0 TO default", exitcode=exitcode, message=message) + node.query( + "SET DEFAULT ROLE role0 TO default", + exitcode=exitcode, + message=message, + ) - with Scenario("I set multiple default roles for a user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): + with Scenario( + "I set multiple default roles for a user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): with When("I set multiple default roles to user"): node.query("SET DEFAULT ROLE role0, role1 TO user0") - with Scenario("I set multiple default roles for multiple users", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole("1.0")]): + with Scenario( + "I set multiple default roles for multiple users", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole("1.0")], + ): with When("I set multiple default roles to multiple users"): node.query("SET DEFAULT ROLE role0, role1 TO user0, user1") - with Scenario("I set all roles as default for a user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole_All("1.0")]): + with Scenario( + "I set all roles as default for a user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole_All("1.0")], + ): with When("I set all roles default to user"): node.query("SET DEFAULT ROLE ALL TO user0") - with Scenario("I set all roles except one for a user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole_AllExcept("1.0")]): + with Scenario( + "I set all roles except one for a user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole_AllExcept("1.0")], + ): with When("I set all except one role default to user"): node.query("SET DEFAULT ROLE ALL EXCEPT role0 TO user0") - with Scenario("I set default role for current user", requirements=[ - RQ_SRS_006_RBAC_SetDefaultRole_CurrentUser("1.0")]): + with Scenario( + "I set default role for current user", + requirements=[RQ_SRS_006_RBAC_SetDefaultRole_CurrentUser("1.0")], + ): with When("I set default role to current user"): node.query("GRANT ACCESS MANAGEMENT ON *.* TO user0") - node.query("SET DEFAULT ROLE role0 TO CURRENT_USER", settings = [("user","user0")]) + node.query( + "SET DEFAULT ROLE role0 TO CURRENT_USER", + settings=[("user", "user0")], + ) finally: with Finally("I drop the roles and users"): diff --git a/tests/testflows/rbac/tests/syntax/set_role.py b/tests/testflows/rbac/tests/syntax/set_role.py index 3d3d4d00fac..bcf8db96ea7 100755 --- a/tests/testflows/rbac/tests/syntax/set_role.py +++ b/tests/testflows/rbac/tests/syntax/set_role.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("set role") @Args(format_description=False) @@ -29,63 +30,78 @@ def feature(self, node="clickhouse1"): for i in range(roles): node.query(f"DROP ROLE IF EXISTS role{i}") - with Scenario("I set default role for current user", requirements=[ - RQ_SRS_006_RBAC_SetRole_Default("1.0")]): + with Scenario( + "I set default role for current user", + requirements=[RQ_SRS_006_RBAC_SetRole_Default("1.0")], + ): with When("I set default role for current user"): node.query("SET ROLE DEFAULT") - with Scenario("I set no role for current user", requirements=[ - RQ_SRS_006_RBAC_SetRole_None("1.0")]): - with When("I set no role for current user"): - node.query("SET ROLE NONE") + with Scenario( + "I set no role for current user", + requirements=[RQ_SRS_006_RBAC_SetRole_None("1.0")], + ): + with When("I set no role for current user"): + node.query("SET ROLE NONE") - with Scenario("I set nonexistent role, throws exception", requirements=[ - RQ_SRS_006_RBAC_SetRole_None("1.0")]): - with Given("I ensure that role role5 does not exist"): - node.query("DROP ROLE IF EXISTS role5") - with When("I set nonexistent role for current user"): - exitcode, message = errors.role_not_found_in_disk("role5") - node.query("SET ROLE role5", exitcode=exitcode, message=message) + with Scenario( + "I set nonexistent role, throws exception", + requirements=[RQ_SRS_006_RBAC_SetRole_None("1.0")], + ): + with Given("I ensure that role role5 does not exist"): + node.query("DROP ROLE IF EXISTS role5") + with When("I set nonexistent role for current user"): + exitcode, message = errors.role_not_found_in_disk("role5") + node.query("SET ROLE role5", exitcode=exitcode, message=message) - with Scenario("I set nonexistent role, throws exception", requirements=[ - RQ_SRS_006_RBAC_SetRole_None("1.0")]): - with Given("I ensure that role role5 does not exist"): - node.query("DROP ROLE IF EXISTS role5") - with When("I set nonexistent role for current user"): - exitcode, message = errors.role_not_found_in_disk("role5") - node.query("SET ROLE ALL EXCEPT role5", exitcode=exitcode, message=message) + with Scenario( + "I set nonexistent role, throws exception", + requirements=[RQ_SRS_006_RBAC_SetRole_None("1.0")], + ): + with Given("I ensure that role role5 does not exist"): + node.query("DROP ROLE IF EXISTS role5") + with When("I set nonexistent role for current user"): + exitcode, message = errors.role_not_found_in_disk("role5") + node.query("SET ROLE ALL EXCEPT role5", exitcode=exitcode, message=message) - with Scenario("I set one role for current user", requirements=[ - RQ_SRS_006_RBAC_SetRole("1.0")]): + with Scenario( + "I set one role for current user", requirements=[RQ_SRS_006_RBAC_SetRole("1.0")] + ): with setup(1): with Given("I have a user"): node.query("CREATE USER OR REPLACE user0") with And("I grant user a role"): node.query("GRANT role0 TO user0") with When("I set role for the user"): - node.query("SET ROLE role0", settings = [("user","user0")]) + node.query("SET ROLE role0", settings=[("user", "user0")]) with Finally("I drop the user"): node.query("DROP USER user0") - with Scenario("I set multiple roles for current user", requirements=[ - RQ_SRS_006_RBAC_SetRole("1.0")]): + with Scenario( + "I set multiple roles for current user", + requirements=[RQ_SRS_006_RBAC_SetRole("1.0")], + ): with setup(2): with Given("I have a user"): node.query("CREATE USER OR REPLACE user0") with And("I grant user a role"): node.query("GRANT role0, role1 TO user0") with When("I set roles for the user"): - node.query("SET ROLE role0, role1", settings = [("user","user0")]) + node.query("SET ROLE role0, role1", settings=[("user", "user0")]) with Finally("I drop the user"): node.query("DROP USER user0") - with Scenario("I set all roles for current user", requirements=[ - RQ_SRS_006_RBAC_SetRole_All("1.0")]): + with Scenario( + "I set all roles for current user", + requirements=[RQ_SRS_006_RBAC_SetRole_All("1.0")], + ): with When("I set all roles for current user"): node.query("SET ROLE ALL") - with Scenario("I set all roles except one for current user", requirements=[ - RQ_SRS_006_RBAC_SetRole_AllExcept("1.0")]): + with Scenario( + "I set all roles except one for current user", + requirements=[RQ_SRS_006_RBAC_SetRole_AllExcept("1.0")], + ): with setup(1): with When("I run set role command"): - node.query("SET ROLE ALL EXCEPT role0") \ No newline at end of file + node.query("SET ROLE ALL EXCEPT role0") diff --git a/tests/testflows/rbac/tests/syntax/show_create_quota.py b/tests/testflows/rbac/tests/syntax/show_create_quota.py index f29b3f5bcc6..632e34f760c 100755 --- a/tests/testflows/rbac/tests/syntax/show_create_quota.py +++ b/tests/testflows/rbac/tests/syntax/show_create_quota.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show create quota") def feature(self, node="clickhouse1"): @@ -25,20 +26,26 @@ def feature(self, node="clickhouse1"): with Finally("I drop the quota"): node.query(f"DROP QUOTA IF EXISTS {quota}") - with Scenario("I show create quota", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Name("1.0")]): + with Scenario( + "I show create quota", + requirements=[RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Name("1.0")], + ): with cleanup("quota0"): with When("I run show create quota command"): node.query("SHOW CREATE QUOTA quota0") - with Scenario("I show create quota current", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Current("1.0")]): + with Scenario( + "I show create quota current", + requirements=[RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Current("1.0")], + ): with cleanup("quota1"): with When("I run show create quota command"): node.query("SHOW CREATE QUOTA CURRENT") - with Scenario("I show create quota current short form", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Current("1.0")]): + with Scenario( + "I show create quota current short form", + requirements=[RQ_SRS_006_RBAC_Quota_ShowCreateQuota_Current("1.0")], + ): with cleanup("quota2"): with When("I run show create quota command"): node.query("SHOW CREATE QUOTA") diff --git a/tests/testflows/rbac/tests/syntax/show_create_role.py b/tests/testflows/rbac/tests/syntax/show_create_role.py index 0b2adba96e2..f587285d85e 100755 --- a/tests/testflows/rbac/tests/syntax/show_create_role.py +++ b/tests/testflows/rbac/tests/syntax/show_create_role.py @@ -5,6 +5,7 @@ from testflows.core import * import rbac.helper.errors as errors from rbac.requirements import * + @TestFeature @Name("show create role") def feature(self, node="clickhouse1"): @@ -26,14 +27,17 @@ def feature(self, node="clickhouse1"): with Finally("I drop the role"): node.query(f"DROP ROLE IF EXISTS {role}") - with Scenario("I show create role", requirements=[ - RQ_SRS_006_RBAC_Role_ShowCreate("1.0")]): + with Scenario( + "I show create role", requirements=[RQ_SRS_006_RBAC_Role_ShowCreate("1.0")] + ): with setup("role0"): with When("I run show create role command"): node.query("SHOW CREATE ROLE role0") - with Scenario("I show create role, role doesn't exist, exception", requirements=[ - RQ_SRS_006_RBAC_Role_ShowCreate("1.0")]): + with Scenario( + "I show create role, role doesn't exist, exception", + requirements=[RQ_SRS_006_RBAC_Role_ShowCreate("1.0")], + ): with When("I run show create role to catch an exception"): exitcode, message = errors.role_not_found_in_disk(name="role0") - node.query("SHOW CREATE ROLE role0", exitcode=exitcode, message=message) \ No newline at end of file + node.query("SHOW CREATE ROLE role0", exitcode=exitcode, message=message) diff --git a/tests/testflows/rbac/tests/syntax/show_create_row_policy.py b/tests/testflows/rbac/tests/syntax/show_create_row_policy.py index cf43c0f2b41..70fd37ac58b 100755 --- a/tests/testflows/rbac/tests/syntax/show_create_row_policy.py +++ b/tests/testflows/rbac/tests/syntax/show_create_row_policy.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show create row policy") def feature(self, node="clickhouse1"): @@ -29,21 +30,27 @@ def feature(self, node="clickhouse1"): with Given("I have a table"): node.query(f"CREATE TABLE default.foo (x UInt64, y String) Engine=Memory") - with Scenario("I show create row policy", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy("1.0")]): + with Scenario( + "I show create row policy", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy("1.0")], + ): with cleanup("policy0"): with When("I run show create row policy command"): node.query("SHOW CREATE ROW POLICY policy0 ON default.foo") - with Scenario("I show create row policy on a table", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy_On("1.0")]): + with Scenario( + "I show create row policy on a table", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy_On("1.0")], + ): with cleanup("policy0"): with When("I run show create row policy command"): node.query("SHOW CREATE ROW POLICY policy0 ON default.foo") - with Scenario("I show create row policy using short syntax on a table", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy_On("1.0")]): - with cleanup("policy1",on="foo"): + with Scenario( + "I show create row policy using short syntax on a table", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowCreateRowPolicy_On("1.0")], + ): + with cleanup("policy1", on="foo"): with When("I run show create row policy command"): node.query("SHOW CREATE POLICY policy1 ON foo") finally: diff --git a/tests/testflows/rbac/tests/syntax/show_create_settings_profile.py b/tests/testflows/rbac/tests/syntax/show_create_settings_profile.py index 4af4e37951a..a4361b3531a 100755 --- a/tests/testflows/rbac/tests/syntax/show_create_settings_profile.py +++ b/tests/testflows/rbac/tests/syntax/show_create_settings_profile.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show create settings profile") def feature(self, node="clickhouse1"): @@ -25,14 +26,18 @@ def feature(self, node="clickhouse1"): with Finally("I drop the settings profile"): node.query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") - with Scenario("I show create settings profile", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_ShowCreateSettingsProfile("1.0")]): + with Scenario( + "I show create settings profile", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_ShowCreateSettingsProfile("1.0")], + ): with cleanup("profile0"): with When("I run show create settings profile command"): node.query("SHOW CREATE SETTINGS PROFILE profile0") - with Scenario("I show create settings profile short form", requirements=[ - RQ_SRS_006_RBAC_SettingsProfile_ShowCreateSettingsProfile("1.0")]): + with Scenario( + "I show create settings profile short form", + requirements=[RQ_SRS_006_RBAC_SettingsProfile_ShowCreateSettingsProfile("1.0")], + ): with cleanup("profile1"): with When("I run show create settings profile command"): node.query("SHOW CREATE PROFILE profile1") diff --git a/tests/testflows/rbac/tests/syntax/show_create_user.py b/tests/testflows/rbac/tests/syntax/show_create_user.py index 963e0d5d193..99da205b1a7 100755 --- a/tests/testflows/rbac/tests/syntax/show_create_user.py +++ b/tests/testflows/rbac/tests/syntax/show_create_user.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show create user") def feature(self, node="clickhouse1"): @@ -25,13 +26,17 @@ def feature(self, node="clickhouse1"): with Finally("I drop the user"): node.query(f"DROP USER IF EXISTS {user}") - with Scenario("I run show create on user with no options", requirements=[ - RQ_SRS_006_RBAC_User_ShowCreateUser_For("1.0")]): + with Scenario( + "I run show create on user with no options", + requirements=[RQ_SRS_006_RBAC_User_ShowCreateUser_For("1.0")], + ): with setup("user0"): with When("I run show create user command"): node.query("SHOW CREATE USER user0") - with Scenario("I run show create on current user", requirements=[ - RQ_SRS_006_RBAC_User_ShowCreateUser("1.0")]): + with Scenario( + "I run show create on current user", + requirements=[RQ_SRS_006_RBAC_User_ShowCreateUser("1.0")], + ): with When("I show create the current user"): - node.query("SHOW CREATE USER CURRENT_USER") \ No newline at end of file + node.query("SHOW CREATE USER CURRENT_USER") diff --git a/tests/testflows/rbac/tests/syntax/show_grants.py b/tests/testflows/rbac/tests/syntax/show_grants.py index 18165ba98a5..256f8aef527 100755 --- a/tests/testflows/rbac/tests/syntax/show_grants.py +++ b/tests/testflows/rbac/tests/syntax/show_grants.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show grants") def feature(self, node="clickhouse1"): @@ -25,13 +26,16 @@ def feature(self, node="clickhouse1"): with Finally("I drop the user"): node.query(f"DROP USER IF EXISTS {user}") - with Scenario("I show grants for user", requirements=[ - RQ_SRS_006_RBAC_Show_Grants_For("1.0")]): + with Scenario( + "I show grants for user", requirements=[RQ_SRS_006_RBAC_Show_Grants_For("1.0")] + ): with setup("user0"): with When("I run show grants command"): node.query("SHOW GRANTS FOR user0") - with Scenario("I show grants for current user", requirements=[ - RQ_SRS_006_RBAC_Show_Grants("1.0")]): + with Scenario( + "I show grants for current user", + requirements=[RQ_SRS_006_RBAC_Show_Grants("1.0")], + ): with When("I show grants"): - node.query("SHOW GRANTS") \ No newline at end of file + node.query("SHOW GRANTS") diff --git a/tests/testflows/rbac/tests/syntax/show_quotas.py b/tests/testflows/rbac/tests/syntax/show_quotas.py index 5fbae718a29..ec6b6edacec 100755 --- a/tests/testflows/rbac/tests/syntax/show_quotas.py +++ b/tests/testflows/rbac/tests/syntax/show_quotas.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show quotas") def feature(self, node="clickhouse1"): @@ -25,26 +26,33 @@ def feature(self, node="clickhouse1"): with Finally("I drop the quota"): node.query(f"DROP QUOTA IF EXISTS {quota}") - with Scenario("I show quotas", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowQuotas("1.0")]): + with Scenario( + "I show quotas", requirements=[RQ_SRS_006_RBAC_Quota_ShowQuotas("1.0")] + ): with cleanup("quota0"), cleanup("quota1"): with When("I run show quota command"): node.query("SHOW QUOTAS") - with Scenario("I show quotas into outfile", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowQuotas_IntoOutfile("1.0")]): + with Scenario( + "I show quotas into outfile", + requirements=[RQ_SRS_006_RBAC_Quota_ShowQuotas_IntoOutfile("1.0")], + ): with cleanup("quota0"), cleanup("quota1"): with When("I run show quota command"): node.query("SHOW QUOTAS INTO OUTFILE 'quotas.txt'") - with Scenario("I show quotas with format", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowQuotas_Format("1.0")]): + with Scenario( + "I show quotas with format", + requirements=[RQ_SRS_006_RBAC_Quota_ShowQuotas_Format("1.0")], + ): with cleanup("quota0"), cleanup("quota1"): with When("I run show quota command"): node.query("SHOW QUOTAS FORMAT TabSeparated") - with Scenario("I show quotas with settings", requirements=[ - RQ_SRS_006_RBAC_Quota_ShowQuotas("1.0")]): + with Scenario( + "I show quotas with settings", + requirements=[RQ_SRS_006_RBAC_Quota_ShowQuotas("1.0")], + ): with cleanup("quota0"), cleanup("quota1"): with When("I run show quota command"): node.query("SHOW QUOTAS SETTINGS max_memory_usage=5") diff --git a/tests/testflows/rbac/tests/syntax/show_row_policies.py b/tests/testflows/rbac/tests/syntax/show_row_policies.py index 0dc7f7f1d1a..81d59bd914b 100755 --- a/tests/testflows/rbac/tests/syntax/show_row_policies.py +++ b/tests/testflows/rbac/tests/syntax/show_row_policies.py @@ -4,6 +4,7 @@ from testflows.core import * from rbac.requirements import * + @TestFeature @Name("show row policies") def feature(self, node="clickhouse1"): @@ -29,26 +30,34 @@ def feature(self, node="clickhouse1"): with Given("I have a table"): node.query(f"CREATE TABLE default.foo (x UInt64, y String) Engine=Memory") - with Scenario("I show row policies", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies("1.0")]): + with Scenario( + "I show row policies", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies("1.0")], + ): with cleanup("policy0"): with When("I run drop row policy command"): node.query("SHOW ROW POLICIES") - with Scenario("I show row policies using short syntax", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies("1.0")]): + with Scenario( + "I show row policies using short syntax", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies("1.0")], + ): with cleanup("policy1"): with When("I run drop row policy command"): node.query("SHOW POLICIES") - with Scenario("I show row policies on a database table", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies_On("1.0")]): + with Scenario( + "I show row policies on a database table", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies_On("1.0")], + ): with cleanup("policy0"): with When("I run drop row policy command"): node.query("SHOW ROW POLICIES ON default.foo") - with Scenario("I show row policies on a table", requirements=[ - RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies_On("1.0")]): + with Scenario( + "I show row policies on a table", + requirements=[RQ_SRS_006_RBAC_RowPolicy_ShowRowPolicies_On("1.0")], + ): with cleanup("policy0"): with When("I run drop row policy command"): node.query("SHOW ROW POLICIES ON foo") diff --git a/tests/testflows/rbac/tests/views/feature.py b/tests/testflows/rbac/tests/views/feature.py index 67f0dadb862..4595339a4a6 100755 --- a/tests/testflows/rbac/tests/views/feature.py +++ b/tests/testflows/rbac/tests/views/feature.py @@ -2,14 +2,27 @@ from testflows.core import * from rbac.helper.common import * + @TestFeature @Name("views") def feature(self): with Pool(3) as pool: try: - Feature(test=load("rbac.tests.views.view", "feature"), parallel=True, executor=pool) - Feature(test=load("rbac.tests.views.live_view", "feature"), parallel=True, executor=pool) - Feature(test=load("rbac.tests.views.materialized_view", "feature"), parallel=True, executor=pool) + Feature( + test=load("rbac.tests.views.view", "feature"), + parallel=True, + executor=pool, + ) + Feature( + test=load("rbac.tests.views.live_view", "feature"), + parallel=True, + executor=pool, + ) + Feature( + test=load("rbac.tests.views.materialized_view", "feature"), + parallel=True, + executor=pool, + ) finally: join() diff --git a/tests/testflows/rbac/tests/views/live_view.py b/tests/testflows/rbac/tests/views/live_view.py index edda654d949..d4ee8f13c61 100755 --- a/tests/testflows/rbac/tests/views/live_view.py +++ b/tests/testflows/rbac/tests/views/live_view.py @@ -5,6 +5,7 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def allow_experimental_live_view(node): setting = ("allow_experimental_live_view", 1) @@ -12,38 +13,68 @@ def allow_experimental_live_view(node): try: with Given("I add allow_experimental_live_view to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(setting) yield finally: - with Finally("I remove allow_experimental_live_view from the default query settings"): + with Finally( + "I remove allow_experimental_live_view from the default query settings" + ): if default_query_settings: try: default_query_settings.pop(default_query_settings.index(setting)) except ValueError: pass + @TestSuite @Requirements( RQ_SRS_006_RBAC_LiveView_Create("1.0"), ) def create(self, node=None): - """Test the RBAC functionality of the `CREATE LIVE VIEW` command. - """ - Scenario(run=create_without_create_view_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_create_view_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_revoked_create_view_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_join_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `CREATE LIVE VIEW` command.""" + Scenario( + run=create_without_create_view_privilege, setup=instrument_clickhouse_server_log + ) + Scenario( + run=create_with_create_view_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_revoked_create_view_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_join_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def create_without_create_view_privilege(self, node=None): - """Check that user is unable to create a live view without CREATE VIEW privilege. - """ + """Check that user is unable to create a live view without CREATE VIEW privilege.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -51,34 +82,44 @@ def create_without_create_view_privilege(self, node=None): if node is None: node = self.context.node with user(node, f"{user_name}"): - with When("I try to create a live view without CREATE VIEW privilege as the user"): + with When( + "I try to create a live view without CREATE VIEW privilege as the user" + ): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT 1", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_create_view_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to create a live view with CREATE VIEW privilege, either granted directly or through a role. - """ + """Check that user is able to create a live view with CREATE VIEW privilege, either granted directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_create_view_privilege, - name="create with create view privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_create_view_privilege, + name="create with create view privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_create_view_privilege, - name="create with create view privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_create_view_privilege, + name="create with create view privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_create_view_privilege(self, grant_target_name, user_name, node=None): - """Check that user is able to create a live view with the granted privileges. - """ + """Check that user is able to create a live view with the granted privileges.""" view_name = f"view_{getuid()}" if node is None: @@ -89,35 +130,46 @@ def create_with_create_view_privilege(self, grant_target_name, user_name, node=N node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I try to create a live view without privilege as the user"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT 1", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def create_with_revoked_create_view_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to create live view after the CREATE VIEW privilege is revoked, either directly or from a role. - """ +def create_with_revoked_create_view_privilege_revoked_directly_or_from_role( + self, node=None +): + """Check that user is unable to create live view after the CREATE VIEW privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_revoked_create_view_privilege, - name="create with create view privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_view_privilege, + name="create with create view privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_revoked_create_view_privilege, - name="create with create view privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_view_privilege, + name="create with create view privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def create_with_revoked_create_view_privilege(self, grant_target_name, user_name, node=None): - """Revoke CREATE VIEW privilege and check the user is unable to create a live view. - """ +def create_with_revoked_create_view_privilege( + self, grant_target_name, user_name, node=None +): + """Revoke CREATE VIEW privilege and check the user is unable to create a live view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -130,8 +182,13 @@ def create_with_revoked_create_view_privilege(self, grant_target_name, user_name node.query(f"REVOKE CREATE VIEW ON {view_name} FROM {grant_target_name}") with Then("I try to create a live view on the table as the user"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT 1", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_without_source_table_privilege(self, node=None): @@ -150,9 +207,16 @@ def create_without_source_table_privilege(self, node=None): with When("I grant CREATE VIEW privilege to a user"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {user_name}") - with Then("I try to create a live view without select privilege on the table"): - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I try to create a live view without select privilege on the table" + ): + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_source_table_privilege_granted_directly_or_via_role(self, node=None): @@ -165,19 +229,23 @@ def create_with_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_source_table_privilege, - name="create with create view and select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create view and select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_source_table_privilege, - name="create with create view and select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create view and select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Check that user is unable to create a live view without SELECT privilege on the source table. - """ + """Check that user is unable to create a live view without SELECT privilege on the source table.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -193,16 +261,20 @@ def create_with_source_table_privilege(self, user_name, grant_target_name, node= with And("I try to create a live view on the table as the user"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + ) with Then("I check the view"): output = node.query(f"SELECT count(*) FROM {view_name}").output - assert output == '0', error() + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a live view where the stored query has two subqueries @@ -215,14 +287,19 @@ def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None) if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_subquery(self, user_name, grant_target_name, node=None): @@ -243,29 +320,72 @@ def create_with_subquery(self, user_name, grant_target_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to CREATE VIEW as the user with create privilege"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a live view where the stored query includes a `JOIN` statement @@ -278,14 +398,19 @@ def create_with_join_query_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_join_query, - name="create with join query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_query, - name="create with join query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_query(self, grant_target_name, user_name, node=None): @@ -305,29 +430,63 @@ def create_with_join_query(self, grant_target_name, user_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_join_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a live view with a stored query that includes `JOIN` and two subqueries @@ -339,14 +498,19 @@ def create_with_join_subquery_privilege_granted_directly_or_via_role(self, node= if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_join_subquery, - name="create with join subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_subquery, + name="create with join subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_subquery, - name="create with join subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_subquery, + name="create with join subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_subquery(self, grant_target_name, user_name, node=None): @@ -367,31 +531,79 @@ def create_with_join_subquery(self, grant_target_name, user_name, node=None): try: with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") - with Then("I attempt to create view as the user with CREATE VIEW privilege"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + with Then( + "I attempt to create view as the user with CREATE VIEW privilege" + ): + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=4): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table3_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table3_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=4))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=4)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a live view with a stored query that includes other views if and only if @@ -403,14 +615,19 @@ def create_with_nested_views_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_nested_views, - name="create with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_nested_views, + name="create with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_nested_views, - name="create with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_nested_views, + name="create with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_nested_views(self, grant_target_name, user_name, node=None): @@ -432,30 +649,79 @@ def create_with_nested_views(self, grant_target_name, user_name, node=None): try: with Given("I have some views"): node.query(f"CREATE VIEW {view0_name} AS SELECT y FROM {table0_name}") - node.query(f"CREATE VIEW {view1_name} AS SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {view0_name} WHERE y<2)") + node.query( + f"CREATE VIEW {view1_name} AS SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {view0_name} WHERE y<2)" + ) with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view2_name} TO {grant_target_name}") - with Then("I attempt to create view as the user with CREATE VIEW privilege"): - node.query(create_view_query.format(view2_name=view2_name, view1_name=view1_name, table2_name=table2_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + with Then( + "I attempt to create view as the user with CREATE VIEW privilege" + ): + node.query( + create_view_query.format( + view2_name=view2_name, + view1_name=view1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,2,3,10,14,15,26,27,30],permutations(table_count=5))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 2, 3, 10, 14, 15, 26, 27, 30], + permutations(table_count=5), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view2_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view2_name=view2_name, view1_name=view1_name, table2_name=table2_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view2_name=view2_name, + view1_name=view1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, view0_name, view1_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + view0_name, + view1_name, + table0_name, + table1_name, + table2_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view2_name}") with Then("I attempt to create a live view as the user"): - node.query(create_view_query.format(view2_name=view2_name, view1_name=view1_name, table2_name=table2_name), - settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view2_name=view2_name, + view1_name=view1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -466,27 +732,53 @@ def create_with_nested_views(self, grant_target_name, user_name, node=None): with And("I drop view2", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_LiveView_Select("1.0"), ) def select(self, node=None): - """Test the RBAC functionality of the `SELECT FROM live view` command. - """ - Scenario(run=select_without_select_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_select_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_select_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_join_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `SELECT FROM live view` command.""" + Scenario( + run=select_without_select_privilege, setup=instrument_clickhouse_server_log + ) + Scenario( + run=select_with_select_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_select_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_join_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def select_without_select_privilege(self, node=None): - """Check that user is unable to select on a view without view SELECT privilege. - """ + """Check that user is unable to select on a view without view SELECT privilege.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -500,36 +792,44 @@ def select_without_select_privilege(self, node=None): node.query(f"CREATE LIVE VIEW {view_name} AS SELECT 1") with Then("I try to select from view without privilege as the user"): - node.query(f"SELECT * FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_select_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to select from a view if and only if they have select privilege on that view, either directly or from a role. - """ + """Check that user is able to select from a view if and only if they have select privilege on that view, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_select_privilege, - name="select with select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_select_privilege, - name="select with select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_select_privilege(self, user_name, grant_target_name, node=None): - """Grant SELECT privilege on a view and check the user is able to SELECT from it. - """ + """Grant SELECT privilege on a view and check the user is able to SELECT from it.""" view_name = f"view_{getuid()}" if node is None: @@ -543,36 +843,42 @@ def select_with_select_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from view with privilege as the user"): - output = node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")]).output - assert output == '1', error() + output = node.query( + f"SELECT count(*) FROM {view_name}", settings=[("user", f"{user_name}")] + ).output + assert output == "1", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_select_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to select from a view if their SELECT privilege is revoked, either directly or from a role. - """ + """Check that user is unable to select from a view if their SELECT privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_revoked_select_privilege, - name="select with select privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_revoked_select_privilege, + name="select with select privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_revoked_select_privilege, - name="select with select privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_revoked_select_privilege, + name="select with select privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_revoked_select_privilege(self, user_name, grant_target_name, node=None): - """Grant and revoke SELECT privilege on a view and check the user is unable to SELECT from it. - """ + """Grant and revoke SELECT privilege on a view and check the user is unable to SELECT from it.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -589,17 +895,21 @@ def select_with_revoked_select_privilege(self, user_name, grant_target_name, nod node.query(f"REVOKE SELECT ON {view_name} FROM {grant_target_name}") with Then("I attempt to select from view with privilege as the user"): - node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_without_source_table_privilege(self, node=None): - """Check that user is unable to select from a view without SELECT privilege for the source table. - """ + """Check that user is unable to select from a view without SELECT privilege for the source table.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -608,7 +918,7 @@ def select_without_source_table_privilege(self, node=None): if node is None: node = self.context.node with table(node, f"{table_name}"): - with user(node, f"{user_name}"): + with user(node, f"{user_name}"): try: with When("I create a live view from the source table"): node.query(f"DROP VIEW IF EXISTS {view_name}") @@ -616,14 +926,21 @@ def select_without_source_table_privilege(self, node=None): with And("I grant view select privilege to the user"): node.query(f"GRANT SELECT ON {view_name} TO {user_name}") - with Then("I attempt to select from view without privilege on the source table"): - node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I attempt to select from view without privilege on the source table" + ): + node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_source_table_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view, with source table in the stored query, if and only if @@ -635,19 +952,23 @@ def select_with_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_source_table_privilege, - name="select with source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_source_table_privilege, + name="select with source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_source_table_privilege, - name="select with source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_source_table_privilege, + name="select with source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Grant SELECT privilege on view and the source table for that view and check the user is able to SELECT from the view. - """ + """Grant SELECT privilege on view and the source table for that view and check the user is able to SELECT from the view.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -657,20 +978,26 @@ def select_with_source_table_privilege(self, user_name, grant_target_name, node= try: with Given("I have a view with a source table"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table_name}" + ) with And("I grant select privileges"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I check the user is able to select from the view"): - output = node.query(f"SELECT count(*) FROM {view_name}", settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query has two subqueries if and only if @@ -682,19 +1009,23 @@ def select_with_subquery_privilege_granted_directly_or_via_role(self, node=None) if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_subquery, - name="select with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_subquery, + name="select with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_subquery, - name="select with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_subquery, + name="select with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_subquery(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -708,29 +1039,61 @@ def select_with_subquery(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a subquery"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Then("I attempt to select from a view as the user"): - output = node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query includes a `JOIN` statement if and only if @@ -742,19 +1105,23 @@ def select_with_join_query_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_join_query, - name="select with join, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_join_query, + name="select with join, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_join_query, - name="select with join, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_join_query, + name="select with join, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_join_query(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -767,28 +1134,54 @@ def select_with_join_query(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a JOIN statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table0_name} JOIN {table1_name} USING d") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT * FROM {table0_name} JOIN {table1_name} USING d" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_join_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view with a stored query that includes `JOIN` and two subqueries @@ -800,19 +1193,23 @@ def select_with_join_subquery_privilege_granted_directly_or_via_role(self, node= if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_join_subquery, - name="select with join subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_join_subquery, + name="select with join subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_join_subquery, - name="select with join subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_join_subquery, + name="select with join subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_join_subquery(self, grant_target_name, user_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -827,28 +1224,62 @@ def select_with_join_subquery(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE LIVE VIEW {view_name} AS SELECT y FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2)) JOIN {table3_name} USING y") + node.query( + f"CREATE LIVE VIEW {view_name} AS SELECT y FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2)) JOIN {table3_name} USING y" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=4): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name, table3_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=4))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=4)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view with a stored query that includes other views if and only if @@ -860,19 +1291,23 @@ def select_with_nested_views_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_nested_views, - name="select with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_nested_views, + name="select with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_nested_views, - name="select with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_nested_views, + name="select with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_nested_views(self, grant_target_name, user_name, node=None): - """Grant SELECT on views and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on views and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view0_name = f"view0_{getuid()}" view1_name = f"view1_{getuid()}" view2_name = f"view2_{getuid()}" @@ -887,25 +1322,67 @@ def select_with_nested_views(self, grant_target_name, user_name, node=None): with table(node, f"{table0_name},{table1_name},{table2_name}"): try: with Given("I have some views"): - node.query(f"CREATE LIVE VIEW {view0_name} AS SELECT y FROM {table0_name}") - node.query(f"CREATE LIVE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)") - node.query(f"CREATE LIVE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y") + node.query( + f"CREATE LIVE VIEW {view0_name} AS SELECT y FROM {table0_name}" + ) + node.query( + f"CREATE LIVE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)" + ) + node.query( + f"CREATE LIVE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y" + ) with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view2_name=view2_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view2_name=view2_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,3,5,7,21,29,31,53,55,61],permutations(table_count=6))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view2_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 3, 5, 7, 21, 29, 31, 53, 55, 61], + permutations(table_count=6), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view2_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view2_name=view2_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view2_name=view2_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=6))+1, grant_target_name, view0_name, view1_name, view2_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=6)) + 1, + grant_target_name, + view0_name, + view1_name, + view2_name, + table0_name, + table1_name, + table2_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view2_name=view2_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view2_name=view2_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -916,39 +1393,47 @@ def select_with_nested_views(self, grant_target_name, user_name, node=None): with And("I drop view2", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_LiveView_Drop("1.0"), ) def drop(self, node=None): - """Test the RBAC functionality of the `DROP VIEW` command. - """ - Scenario(run=drop_with_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=drop_with_revoked_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `DROP VIEW` command.""" + Scenario( + run=drop_with_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=drop_with_revoked_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def drop_with_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to drop view with DROP VIEW privilege if the user has privilege directly or through a role. - """ + """Check that user is able to drop view with DROP VIEW privilege if the user has privilege directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=drop_with_privilege, - name="drop privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=drop_with_privilege, name="drop privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=drop_with_privilege, - name="drop privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=drop_with_privilege, name="drop privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def drop_with_privilege(self, grant_target_name, user_name, node=None): - """Grant DROP VIEW privilege and check the user is able to successfully drop a view. - """ + """Grant DROP VIEW privilege and check the user is able to successfully drop a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.table_does_not_exist(name=f"default.{view_name}") @@ -963,7 +1448,7 @@ def drop_with_privilege(self, grant_target_name, user_name, node=None): node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") with And("I drop the view as the user"): - node.query(f"DROP VIEW {view_name}", settings = [("user",f"{user_name}")]) + node.query(f"DROP VIEW {view_name}", settings=[("user", f"{user_name}")]) with Then("I check the table does not exist"): node.query(f"SELECT * FROM {view_name}", exitcode=exitcode, message=message) @@ -972,29 +1457,31 @@ def drop_with_privilege(self, grant_target_name, user_name, node=None): with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def drop_with_revoked_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to drop view with DROP VIEW privilege revoked directly or from a role. - """ + """Check that user is unable to drop view with DROP VIEW privilege revoked directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=drop_with_revoked_privilege, - name="drop privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=drop_with_revoked_privilege, name="drop privilege revoked directly" + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=drop_with_revoked_privilege, - name="drop privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=drop_with_revoked_privilege, name="drop privilege revoked from a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): - """Revoke DROP VIEW privilege and check the user is unable to DROP a view. - """ + """Revoke DROP VIEW privilege and check the user is unable to DROP a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1011,22 +1498,33 @@ def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): node.query(f"REVOKE DROP VIEW ON {view_name} FROM {grant_target_name}") with Then("I drop the view as the user"): - node.query(f"DROP VIEW {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"DROP VIEW {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_LiveView_Refresh("1.0"), ) def refresh(self, node=None): - """Test the RBAC functionality of the `ALTER LIVE VIEW REFRESH` command. - """ - Scenario(run=refresh_with_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=refresh_with_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `ALTER LIVE VIEW REFRESH` command.""" + Scenario( + run=refresh_with_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=refresh_with_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def refresh_with_privilege_granted_directly_or_via_role(self, node=None): @@ -1039,19 +1537,21 @@ def refresh_with_privilege_granted_directly_or_via_role(self, node=None): if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=refresh_with_privilege, - name="refresh privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=refresh_with_privilege, name="refresh privilege granted directly" + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=refresh_with_privilege, - name="refresh privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=refresh_with_privilege, name="refresh privilege revoked from a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def refresh_with_privilege(self, grant_target_name, user_name, node=None): - """Grant REFRESH privilege and check that user is able to refresh a live view. - """ + """Grant REFRESH privilege and check that user is able to refresh a live view.""" view_name = f"view_{getuid()}" if node is None: @@ -1062,15 +1562,21 @@ def refresh_with_privilege(self, grant_target_name, user_name, node=None): node.query(f"CREATE LIVE VIEW {view_name} AS SELECT 1") with When("I grant REFRESH privilege"): - node.query(f"GRANT ALTER VIEW REFRESH ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW REFRESH ON {view_name} TO {grant_target_name}" + ) with Then("I attempt to refresh as the user"): - node.query(f"ALTER LIVE VIEW {view_name} REFRESH", settings = [("user",f"{user_name}")]) + node.query( + f"ALTER LIVE VIEW {view_name} REFRESH", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def refresh_with_privilege_revoked_directly_or_from_role(self, node=None): """Check that user is unable to refresh a live view with REFRESH privilege @@ -1082,19 +1588,23 @@ def refresh_with_privilege_revoked_directly_or_from_role(self, node=None): if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=refresh_with_revoked_privilege, - name="refresh privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=refresh_with_revoked_privilege, + name="refresh privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=refresh_with_revoked_privilege, - name="refresh privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=refresh_with_revoked_privilege, + name="refresh privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def refresh_with_revoked_privilege(self, grant_target_name, user_name, node=None): - """Revoke REFRESH privilege and check that user is unable to refresh a live view. - """ + """Revoke REFRESH privilege and check that user is unable to refresh a live view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1106,17 +1616,27 @@ def refresh_with_revoked_privilege(self, grant_target_name, user_name, node=None node.query(f"CREATE LIVE VIEW {view_name} AS SELECT 1") with When("I grant REFRESH privilege"): - node.query(f"GRANT ALTER VIEW REFRESH ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW REFRESH ON {view_name} TO {grant_target_name}" + ) with And("I revoke REFRESH privilege"): - node.query(f"REVOKE ALTER VIEW REFRESH ON {view_name} FROM {grant_target_name}") + node.query( + f"REVOKE ALTER VIEW REFRESH ON {view_name} FROM {grant_target_name}" + ) with Then("I attempt to refresh as the user"): - node.query(f"ALTER LIVE VIEW {view_name} REFRESH", settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"ALTER LIVE VIEW {view_name} REFRESH", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_LiveView("1.0"), diff --git a/tests/testflows/rbac/tests/views/materialized_view.py b/tests/testflows/rbac/tests/views/materialized_view.py index 0464332d327..6aedbd18c7a 100755 --- a/tests/testflows/rbac/tests/views/materialized_view.py +++ b/tests/testflows/rbac/tests/views/materialized_view.py @@ -5,49 +5,94 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @contextmanager def allow_experimental_alter_materialized_view_structure(node): setting = ("allow_experimental_alter_materialized_view_structure", 1) default_query_settings = None try: - with Given("I add allow_experimental_alter_materialized_view_structure to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) + with Given( + "I add allow_experimental_alter_materialized_view_structure to the default query settings" + ): + default_query_settings = getsattr( + current().context, "default_query_settings", [] + ) default_query_settings.append(setting) yield finally: - with Finally("I remove allow_experimental_alter_materialized_view_structure from the default query settings"): + with Finally( + "I remove allow_experimental_alter_materialized_view_structure from the default query settings" + ): if default_query_settings: try: default_query_settings.pop(default_query_settings.index(setting)) except ValueError: pass + @TestSuite @Requirements( RQ_SRS_006_RBAC_MaterializedView_Create("1.0"), ) def create(self, node=None): - """Test the RBAC functionality of the `CREATE MATERIALIZED VIEW` command. - """ - Scenario(run=create_without_create_view_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_create_view_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_revoked_create_view_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_union_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_join_union_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_target_table_privilege_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_populate_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_populate_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `CREATE MATERIALIZED VIEW` command.""" + Scenario( + run=create_without_create_view_privilege, setup=instrument_clickhouse_server_log + ) + Scenario( + run=create_with_create_view_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_revoked_create_view_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_union_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_join_union_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_target_table_privilege_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_populate_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_populate_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def create_without_create_view_privilege(self, node=None): - """Check that user is unable to create a view without CREATE VIEW privilege. - """ + """Check that user is unable to create a view without CREATE VIEW privilege.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -57,32 +102,40 @@ def create_without_create_view_privilege(self, node=None): with user(node, f"{user_name}"): with When("I try to create a view without CREATE VIEW privilege as the user"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_create_view_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to create a view with CREATE VIEW privilege, either granted directly or through a role. - """ + """Check that user is able to create a view with CREATE VIEW privilege, either granted directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_create_view_privilege, - name="create with create view privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_create_view_privilege, + name="create with create view privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_create_view_privilege, - name="create with create view privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_create_view_privilege, + name="create with create view privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_create_view_privilege(self, grant_target_name, user_name, node=None): - """Check that user is able to create a view with the granted privileges. - """ + """Check that user is able to create a view with the granted privileges.""" view_name = f"view_{getuid()}" if node is None: @@ -93,35 +146,46 @@ def create_with_create_view_privilege(self, grant_target_name, user_name, node=N node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I try to create a view without privilege as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1", + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def create_with_revoked_create_view_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to create view after the CREATE VIEW privilege is revoked, either directly or from a role. - """ +def create_with_revoked_create_view_privilege_revoked_directly_or_from_role( + self, node=None +): + """Check that user is unable to create view after the CREATE VIEW privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_revoked_create_view_privilege, - name="create with create view privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_view_privilege, + name="create with create view privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_revoked_create_view_privilege, - name="create with create view privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_view_privilege, + name="create with create view privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def create_with_revoked_create_view_privilege(self, grant_target_name, user_name, node=None): - """Revoke CREATE VIEW privilege and check the user is unable to create a view. - """ +def create_with_revoked_create_view_privilege( + self, grant_target_name, user_name, node=None +): + """Revoke CREATE VIEW privilege and check the user is unable to create a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -134,8 +198,13 @@ def create_with_revoked_create_view_privilege(self, grant_target_name, user_name node.query(f"REVOKE CREATE VIEW ON {view_name} FROM {grant_target_name}") with Then("I try to create a view on the table as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_without_source_table_privilege(self, node=None): @@ -155,8 +224,13 @@ def create_without_source_table_privilege(self, node=None): node.query(f"GRANT CREATE VIEW ON {view_name} TO {user_name}") with Then("I try to create a view without select privilege on the table"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_source_table_privilege_granted_directly_or_via_role(self, node=None): @@ -169,19 +243,23 @@ def create_with_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_source_table_privilege, - name="create with create view and select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create view and select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_source_table_privilege, - name="create with create view and select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create view and select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Check that user is unable to create a view without SELECT privilege on the source table. - """ + """Check that user is unable to create a view without SELECT privilege on the source table.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -197,16 +275,20 @@ def create_with_source_table_privilege(self, user_name, grant_target_name, node= with And("I try to create a view on the table as the user"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + ) with Then("I check the view"): output = node.query(f"SELECT count(*) FROM {view_name}").output - assert output == '0', error() + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view where the stored query has two subqueries @@ -219,14 +301,19 @@ def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None) if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_subquery(self, user_name, grant_target_name, node=None): @@ -247,29 +334,72 @@ def create_with_subquery(self, user_name, grant_target_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to CREATE VIEW as the user with create privilege"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view where the stored query includes a `JOIN` statement @@ -282,14 +412,19 @@ def create_with_join_query_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_join_query, - name="create with join query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_query, - name="create with join query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_query(self, grant_target_name, user_name, node=None): @@ -309,29 +444,63 @@ def create_with_join_query(self, grant_target_name, user_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_union_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view where the stored query includes a `UNION ALL` statement @@ -344,14 +513,19 @@ def create_with_union_query_privilege_granted_directly_or_via_role(self, node=No if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_union_query, - name="create with union query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_union_query, + name="create with union query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_union_query, - name="create with union query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_union_query, + name="create with union query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_union_query(self, grant_target_name, user_name, node=None): @@ -371,31 +545,67 @@ def create_with_union_query(self, grant_target_name, user_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): +def create_with_join_union_subquery_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to create a view with a stored query that includes `UNION ALL`, `JOIN` and two subqueries if and only if the user has SELECT privilege on all of the tables, either granted directly or through a role. """ @@ -405,14 +615,19 @@ def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_join_union_subquery, - name="create with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_union_subquery, - name="create with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_union_subquery(self, grant_target_name, user_name, node=None): @@ -430,38 +645,94 @@ def create_with_join_union_subquery(self, grant_target_name, user_name, node=Non if node is None: node = self.context.node - with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with table( + node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}" + ): with user(node, f"{user_name}"): try: with When("I grant CREATE VIEW privilege"): - node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") - with Then("I attempt to create view as the user with CREATE VIEW privilege"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}" + ) + with Then( + "I attempt to create view as the user with CREATE VIEW privilege" + ): + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=5): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table3_name, table4_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table3_name, + table4_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") -@TestScenario +@TestScenario def create_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view with a stored query that includes other views if and only if they have SELECT privilege on all the views and the source tables for those views. @@ -472,14 +743,19 @@ def create_with_nested_views_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_nested_views, - name="create with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_nested_views, + name="create with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_nested_views, - name="create with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_nested_views, + name="create with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_nested_views(self, grant_target_name, user_name, node=None): @@ -502,32 +778,89 @@ def create_with_nested_views(self, grant_target_name, user_name, node=None): with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name}"): try: with Given("I have some views"): - node.query(f"CREATE MATERIALIZED VIEW {view0_name} ENGINE = Memory AS SELECT y FROM {table0_name}") - node.query(f"CREATE MATERIALIZED VIEW {view1_name} ENGINE = Memory AS SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {view0_name} WHERE y<2)") - node.query(f"CREATE MATERIALIZED VIEW {view2_name} ENGINE = Memory AS SELECT y FROM {table2_name} JOIN {view1_name} USING y") + node.query( + f"CREATE MATERIALIZED VIEW {view0_name} ENGINE = Memory AS SELECT y FROM {table0_name}" + ) + node.query( + f"CREATE MATERIALIZED VIEW {view1_name} ENGINE = Memory AS SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {view0_name} WHERE y<2)" + ) + node.query( + f"CREATE MATERIALIZED VIEW {view2_name} ENGINE = Memory AS SELECT y FROM {table2_name} JOIN {view1_name} USING y" + ) with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view3_name} TO {grant_target_name}") - with Then("I attempt to create view as the user with CREATE VIEW privilege"): - node.query(create_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + with Then( + "I attempt to create view as the user with CREATE VIEW privilege" + ): + node.query( + create_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,2,3,7,11,15,31,39,79,95],permutations(table_count=7))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view2_name, table3_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 2, 3, 7, 11, 15, 31, 39, 79, 95], + permutations(table_count=7), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view2_name, + table3_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view3_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=7))+1, grant_target_name, view0_name, view1_name, view2_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=7)) + 1, + grant_target_name, + view0_name, + view1_name, + view2_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view3_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -540,6 +873,7 @@ def create_with_nested_views(self, grant_target_name, user_name, node=None): with And("I drop view3", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestScenario def create_with_target_table_privilege_directly_or_via_role(self, node=None): """Check that user is able to create a materialized view with a target table if and only if @@ -551,19 +885,23 @@ def create_with_target_table_privilege_directly_or_via_role(self, node=None): if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_target_table, - name="create with target table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_target_table, + name="create with target table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_target_table, - name="create with target table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_target_table, + name="create with target table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_target_table(self, grant_target_name, user_name, node=None): - """Check that user is unable to create a view without INSERT and SELECT privileges and is able to once both are granted. - """ + """Check that user is unable to create a view without INSERT and SELECT privileges and is able to once both are granted.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -575,33 +913,48 @@ def create_with_target_table(self, grant_target_name, user_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT on the target table"): node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I revoke SELECT on the target table"): node.query(f"REVOKE SELECT ON {table_name} FROM {grant_target_name}") with And("I grant INSERT privilege on the target table"): node.query(f"GRANT INSERT ON {table_name} TO {grant_target_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT on the target table"): node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I successfully create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", - settings = [("user", f"{user_name}")]) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_populate_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view with POPULATE specified if and only if @@ -615,21 +968,25 @@ def create_with_populate_privilege_granted_directly_or_via_role(self, node=None) with user(node, f"{user_name}"): - Scenario(test=create_with_populate, - name="create with populate privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_populate, + name="create with populate privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_populate, - name="create with populate privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_populate, + name="create with populate privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_populate(self, user_name, grant_target_name, node=None): - """Check that user is only able to create the view after INSERT privilege is granted. - """ + """Check that user is only able to create the view after INSERT privilege is granted.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -643,8 +1000,12 @@ def create_with_populate(self, user_name, grant_target_name, node=None): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT 1", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant INSERT privilege on the view"): node.query(f"GRANT INSERT ON {view_name} TO {grant_target_name}") @@ -653,15 +1014,20 @@ def create_with_populate(self, user_name, grant_target_name, node=None): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT 1", - settings = [("user", f"{user_name}")]) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT 1", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def create_with_populate_source_table_privilege_granted_directly_or_via_role(self, node=None): +def create_with_populate_source_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to create a view with POPULATE and a source table specified if and only if they have CREATE VIEW and INSERT privileges for the view, either directly or from a role. """ @@ -671,19 +1037,23 @@ def create_with_populate_source_table_privilege_granted_directly_or_via_role(sel if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_populate_source_table, - name="create with populate and source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_populate_source_table, + name="create with populate and source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_populate_source_table, - name="create with populate and source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_populate_source_table, + name="create with populate and source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_populate_source_table(self, user_name, grant_target_name, node=None): - """Check that user is only able to create the view after INSERT privilege is granted. - """ + """Check that user is only able to create the view after INSERT privilege is granted.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -695,50 +1065,92 @@ def create_with_populate_source_table(self, user_name, grant_target_name, node=N with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the source table"): node.query(f"GRANT SELECT ON {table_name} TO {user_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant INSERT privilege on the view"): node.query(f"GRANT INSERT ON {view_name} TO {grant_target_name}") with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")]) + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory POPULATE AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_MaterializedView_Select("1.0"), ) def select(self, node=None): - """Test the RBAC functionality of the `SELECT FROM materialized view` command - """ - Scenario(run=select_without_select_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_select_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_select_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_union_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_join_union_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_privilege_granted_directly_or_via_role_without_target_table_privilege, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `SELECT FROM materialized view` command""" + Scenario( + run=select_without_select_privilege, setup=instrument_clickhouse_server_log + ) + Scenario( + run=select_with_select_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_select_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_union_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_join_union_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_privilege_granted_directly_or_via_role_without_target_table_privilege, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def select_without_select_privilege(self, node=None): - """Check that user is unable to select on a view without view SELECT privilege. - """ + """Check that user is unable to select on a view without view SELECT privilege.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -749,39 +1161,49 @@ def select_without_select_privilege(self, node=None): try: with When("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with Then("I try to select from view without privilege as the user"): - node.query(f"SELECT * FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_select_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to select from a view if and only if they have select privilege on that view, either directly or from a role. - """ + """Check that user is able to select from a view if and only if they have select privilege on that view, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_select_privilege, - name="select with select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_select_privilege, - name="select with select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_select_privilege(self, user_name, grant_target_name, node=None): - """Grant SELECT privilege on a view and check the user is able to SELECT from it. - """ + """Grant SELECT privilege on a view and check the user is able to SELECT from it.""" view_name = f"view_{getuid()}" if node is None: @@ -789,42 +1211,50 @@ def select_with_select_privilege(self, user_name, grant_target_name, node=None): try: with When("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with And("I grant SELECT privilege for the view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from view with privilege as the user"): - output = node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")]).output - assert output == '1', error() + output = node.query( + f"SELECT count(*) FROM {view_name}", settings=[("user", f"{user_name}")] + ).output + assert output == "1", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_select_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to select from a view if their SELECT privilege is revoked, either directly or from a role. - """ + """Check that user is unable to select from a view if their SELECT privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_select_privilege, - name="select with select privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_select_privilege, - name="select with select privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_revoked_select_privilege(self, user_name, grant_target_name, node=None): - """Grant and revoke SELECT privilege on a view and check the user is unable to SELECT from it. - """ + """Grant and revoke SELECT privilege on a view and check the user is unable to SELECT from it.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -841,17 +1271,21 @@ def select_with_revoked_select_privilege(self, user_name, grant_target_name, nod node.query(f"REVOKE SELECT ON {view_name} FROM {grant_target_name}") with Then("I attempt to select from view with privilege as the user"): - node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_without_source_table_privilege(self, node=None): - """Check that user is unable to select from a view without SELECT privilege for the source table. - """ + """Check that user is unable to select from a view without SELECT privilege for the source table.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -860,22 +1294,31 @@ def select_without_source_table_privilege(self, node=None): if node is None: node = self.context.node with table(node, f"{table_name}"): - with user(node, f"{user_name}"): + with user(node, f"{user_name}"): try: with When("I create a view from the source table"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with And("I grant view select privilege to the user"): node.query(f"GRANT SELECT ON {view_name} TO {user_name}") - with Then("I attempt to select from view without privilege on the source table"): - node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I attempt to select from view without privilege on the source table" + ): + node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_source_table_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view, with source table in the stored query, if and only if @@ -887,19 +1330,23 @@ def select_with_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_source_table_privilege, - name="select with source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_source_table_privilege, + name="select with source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_source_table_privilege, - name="select with source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_source_table_privilege, + name="select with source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Grant SELECT privilege on view and the source table for that view and check the user is able to SELECT from the view. - """ + """Grant SELECT privilege on view and the source table for that view and check the user is able to SELECT from the view.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -909,20 +1356,26 @@ def select_with_source_table_privilege(self, user_name, grant_target_name, node= try: with Given("I have a view with a source table"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with And("I grant select privileges"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I check the user is able to select from the view"): - output = node.query(f"SELECT count(*) FROM {view_name}", settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query has two subqueries if and only if @@ -934,19 +1387,23 @@ def select_with_subquery_privilege_granted_directly_or_via_role(self, node=None) if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_subquery, - name="select with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_subquery, + name="select with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_subquery, - name="select with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_subquery, + name="select with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_subquery(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -960,29 +1417,61 @@ def select_with_subquery(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a subquery"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Then("I attempt to select from a view as the user"): - output = node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query includes a `JOIN` statement if and only if @@ -994,19 +1483,23 @@ def select_with_join_query_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_join_query, - name="select with join, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_join_query, + name="select with join, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_join_query, - name="select with join, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_join_query, + name="select with join, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_join_query(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -1019,28 +1512,54 @@ def select_with_join_query(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a JOIN statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table0_name} JOIN {table1_name} USING d") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table0_name} JOIN {table1_name} USING d" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_union_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query includes a `UNION ALL` statement if and only if @@ -1052,19 +1571,23 @@ def select_with_union_query_privilege_granted_directly_or_via_role(self, node=No if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_union_query, - name="select with union, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_union_query, + name="select with union, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_union_query, - name="select with union, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_union_query, + name="select with union, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_union_query(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -1077,30 +1600,58 @@ def select_with_union_query(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a UNION statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table0_name} UNION ALL SELECT * FROM {table1_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table0_name} UNION ALL SELECT * FROM {table1_name}" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def select_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): +def select_with_join_union_subquery_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to select from a view with a stored query that includes `UNION ALL`, `JOIN` and two subqueries if and only if the user has SELECT privilege on all the tables and the view, either directly or through a role. """ @@ -1110,19 +1661,23 @@ def select_with_join_union_subquery_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_join_union_subquery, - name="select with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_join_union_subquery, + name="select with join union subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_join_union_subquery, - name="select with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_join_union_subquery, + name="select with join union subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_join_union_subquery(self, grant_target_name, user_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -1134,32 +1689,70 @@ def select_with_join_union_subquery(self, grant_target_name, user_name, node=Non if node is None: node = self.context.node - with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with table( + node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}" + ): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT y FROM {table0_name} JOIN {table1_name} USING y UNION ALL SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table3_name} WHERE y IN (SELECT y FROM {table4_name} WHERE y<2))") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT y FROM {table0_name} JOIN {table1_name} USING y UNION ALL SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table3_name} WHERE y IN (SELECT y FROM {table4_name} WHERE y<2))" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=5): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view with a stored query that includes other views if and only if @@ -1171,19 +1764,23 @@ def select_with_nested_views_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_nested_views, - name="select with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_nested_views, + name="select with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_nested_views, - name="select with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_nested_views, + name="select with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_nested_views(self, grant_target_name, user_name, node=None): - """Grant SELECT on views and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on views and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view0_name = f"view0_{getuid()}" view1_name = f"view1_{getuid()}" view2_name = f"view2_{getuid()}" @@ -1201,25 +1798,71 @@ def select_with_nested_views(self, grant_target_name, user_name, node=None): try: with Given("I have some views"): node.query(f"CREATE VIEW {view0_name} AS SELECT y FROM {table0_name}") - node.query(f"CREATE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)") - node.query(f"CREATE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y") - node.query(f"CREATE VIEW {view3_name} AS SELECT y FROM {view2_name} UNION ALL SELECT y FROM {table3_name}") + node.query( + f"CREATE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)" + ) + node.query( + f"CREATE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y" + ) + node.query( + f"CREATE VIEW {view3_name} AS SELECT y FROM {view2_name} UNION ALL SELECT y FROM {table3_name}" + ) with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view3_name=view3_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view3_name=view3_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,3,5,7,13,15,23,31,45,63,95,127,173,237,247,253],permutations(table_count=8))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view3_name, table3_name, view2_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 3, 5, 7, 13, 15, 23, 31, 45, 63, 95, 127, 173, 237, 247, 253], + permutations(table_count=8), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view3_name, + table3_name, + view2_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view3_name=view3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view3_name=view3_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=8))+1, grant_target_name, view0_name, view1_name, view2_name, view3_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=8)) + 1, + grant_target_name, + view0_name, + view1_name, + view2_name, + view3_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view3_name=view3_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view3_name=view3_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -1232,40 +1875,52 @@ def select_with_nested_views(self, grant_target_name, user_name, node=None): with And("I drop view3", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestScenario -def select_with_privilege_granted_directly_or_via_role_without_target_table_privilege(self, node=None): - """Check that user is able to select from a materialized view without target table SELECT privilege. - """ +def select_with_privilege_granted_directly_or_via_role_without_target_table_privilege( + self, node=None +): + """Check that user is able to select from a materialized view without target table SELECT privilege.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_without_target_table_privilege, - name="select without target table privilege, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_without_target_table_privilege, + name="select without target table privilege, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_without_target_table_privilege, - name="select without target table privilege, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_without_target_table_privilege, + name="select without target table privilege, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def select_without_target_table_privilege(self, grant_target_name, user_name, node=None): - """GRANT the user SELECT privilege on the view and check the user is able to successfully SELECT from the view without target table privilege. - """ +def select_without_target_table_privilege( + self, grant_target_name, user_name, node=None +): + """GRANT the user SELECT privilege on the view and check the user is able to successfully SELECT from the view without target table privilege.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" if node is None: node = self.context.node try: with Given("I have a view"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1" + ) with When("I grant SELECT privilege on the view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from a view as the user"): - node.query(f"SELECT * FROM {view_name}", settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {view_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the view"): @@ -1274,17 +1929,26 @@ def select_without_target_table_privilege(self, grant_target_name, user_name, no @TestSuite def select_from_tables(self, node=None): - """Testing RBAC functionality of SELECT for tables related to materialized views - target tables, source tables. - """ - Scenario(run=select_from_implicit_target_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_from_explicit_target_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_from_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Testing RBAC functionality of SELECT for tables related to materialized views - target tables, source tables.""" + Scenario( + run=select_from_implicit_target_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_from_explicit_target_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_from_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_MaterializedView_Select_TargetTable("1.0") -) -def select_from_implicit_target_table_privilege_granted_directly_or_via_role(self, node=None): +@Requirements(RQ_SRS_006_RBAC_MaterializedView_Select_TargetTable("1.0")) +def select_from_implicit_target_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to SELECT from the implicit target table created from a materialized view if they have SELECT privilege on that table. """ @@ -1294,53 +1958,70 @@ def select_from_implicit_target_table_privilege_granted_directly_or_via_role(sel if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_from_implicit_target_table, - name="select from implicit target table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_from_implicit_target_table, + name="select from implicit target table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_from_implicit_target_table, - name="select from implicit target table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_from_implicit_target_table, + name="select from implicit target table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_from_implicit_target_table(self, grant_target_name, user_name, node=None): - """Grant SELECT on the implicit target table and check the user is able to SELECT only if they have SELECT privilege on the table. - """ + """Grant SELECT on the implicit target table and check the user is able to SELECT only if they have SELECT privilege on the table.""" view_name = f"view_{getuid()}" - implicit_table_name = f"\\\".inner.{view_name}\\\"" + implicit_table_name = f'\\".inner.{view_name}\\"' exitcode, message = errors.not_enough_privileges(name=f"{user_name}") if node is None: node = self.context.node try: with Given("I have a view"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with Then("I attempt to SELECT from the implicit target table as the user"): - node.query(f"SELECT * FROM {implicit_table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {implicit_table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to SELECT from the implicit target table as the user"): - node.query(f"SELECT * FROM {implicit_table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {implicit_table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the target table"): node.query(f"GRANT SELECT ON {implicit_table_name} TO {grant_target_name}") with Then("I attempt to SELECT from the implicit target table as the user"): - node.query(f"SELECT * FROM {implicit_table_name}", - settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {implicit_table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_MaterializedView_Select_TargetTable("1.0") -) -def select_from_explicit_target_table_privilege_granted_directly_or_via_role(self, node=None): +@Requirements(RQ_SRS_006_RBAC_MaterializedView_Select_TargetTable("1.0")) +def select_from_explicit_target_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to SELECT from the explicit target table created from a materialized view if they have SELECT privilege on that table. """ @@ -1350,19 +2031,23 @@ def select_from_explicit_target_table_privilege_granted_directly_or_via_role(sel if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_from_explicit_target_table, - name="select from explicit target table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_from_explicit_target_table, + name="select from explicit target table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_from_explicit_target_table, - name="select from explicit target table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_from_explicit_target_table, + name="select from explicit target table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_from_explicit_target_table(self, grant_target_name, user_name, node=None): - """Grant SELECT on the explicit target table and check the user is able to SELECT only if they have SELECT privilege on the table. - """ + """Grant SELECT on the explicit target table and check the user is able to SELECT only if they have SELECT privilege on the table.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1371,32 +2056,42 @@ def select_from_explicit_target_table(self, grant_target_name, user_name, node=N with table(node, f"{table_name}"): try: with Given("I have a view"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table_name} AS SELECT 1" + ) with Then("I attempt to SELECT from the explicit target table as the user"): - node.query(f"SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to SELECT from the explicit target table as the user"): - node.query(f"SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the target table"): node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I attempt to SELECT from the explicit target table as the user"): - node.query(f"SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -@Requirements( - RQ_SRS_006_RBAC_MaterializedView_Select_SourceTable("1.0") -) +@Requirements(RQ_SRS_006_RBAC_MaterializedView_Select_SourceTable("1.0")) def select_from_source_table_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to SELECT from the source table of a materialized view if they have SELECT privilege on that table. @@ -1407,19 +2102,23 @@ def select_from_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_from_source_table, - name="select from source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_from_source_table, + name="select from source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_from_source_table, - name="select from source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_from_source_table, + name="select from source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_from_source_table(self, grant_target_name, user_name, node=None): - """Grant SELECT on the source table and check the user is able to SELECT only if they have SELECT privilege on the table. - """ + """Grant SELECT on the source table and check the user is able to SELECT only if they have SELECT privilege on the table.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1428,61 +2127,80 @@ def select_from_source_table(self, grant_target_name, user_name, node=None): with table(node, f"{table_name}"): try: with Given("I have a view"): - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with Then("I attempt to SELECT from the source table as the user"): - node.query(f"SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to SELECT from the implicit target table as the user"): - node.query(f"SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant SELECT privilege on the target table"): node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I attempt to SELECT from the implicit target table as the user"): - node.query(f"SELECT * FROM {table_name}", - settings = [("user", f"{user_name}")]) + node.query( + f"SELECT * FROM {table_name}", settings=[("user", f"{user_name}")] + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_MaterializedView_Drop("1.0"), ) def drop(self, node=None): - """Test the RBAC functionality of the `DROP VIEW` command. - """ - Scenario(run=drop_with_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=drop_with_revoked_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `DROP VIEW` command.""" + Scenario( + run=drop_with_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=drop_with_revoked_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def drop_with_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to drop view with DROP VIEW privilege if the user has privilege directly or through a role. - """ + """Check that user is able to drop view with DROP VIEW privilege if the user has privilege directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=drop_with_privilege, - name="drop privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=drop_with_privilege, name="drop privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=drop_with_privilege, - name="drop privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=drop_with_privilege, name="drop privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def drop_with_privilege(self, grant_target_name, user_name, node=None): - """Grant DROP VIEW privilege and check the user is able to successfully drop a view. - """ + """Grant DROP VIEW privilege and check the user is able to successfully drop a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.table_does_not_exist(name=f"default.{view_name}") @@ -1491,13 +2209,15 @@ def drop_with_privilege(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant DROP VIEW privilege"): node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") with And("I drop the view as the user"): - node.query(f"DROP VIEW {view_name}", settings = [("user",f"{user_name}")]) + node.query(f"DROP VIEW {view_name}", settings=[("user", f"{user_name}")]) with Then("I check the table does not exist"): node.query(f"SELECT * FROM {view_name}", exitcode=exitcode, message=message) @@ -1506,29 +2226,31 @@ def drop_with_privilege(self, grant_target_name, user_name, node=None): with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def drop_with_revoked_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to drop view with DROP VIEW privilege revoked directly or from a role. - """ + """Check that user is unable to drop view with DROP VIEW privilege revoked directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=drop_with_revoked_privilege, - name="drop privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=drop_with_revoked_privilege, name="drop privilege revoked directly" + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=drop_with_revoked_privilege, - name="drop privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=drop_with_revoked_privilege, name="drop privilege revoked from a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): - """Revoke DROP VIEW privilege and check the user is unable to DROP a view. - """ + """Revoke DROP VIEW privilege and check the user is unable to DROP a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1537,7 +2259,9 @@ def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant DROP VIEW privilege"): node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") @@ -1546,57 +2270,92 @@ def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): node.query(f"REVOKE DROP VIEW ON {view_name} FROM {grant_target_name}") with Then("I drop the view as the user"): - node.query(f"DROP VIEW {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"DROP VIEW {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_MaterializedView_ModifyQuery("1.0"), ) def modify_query(self, node=None): - """Test the RBAC functionality of the `MODIFY QUERY` command. - """ + """Test the RBAC functionality of the `MODIFY QUERY` command.""" if node is None: node = self.context.node with allow_experimental_alter_materialized_view_structure(node): - Scenario(run=modify_query_with_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_union_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_join_union_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=modify_query_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + Scenario( + run=modify_query_with_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_union_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_join_union_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=modify_query_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def modify_query_with_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to modify view with MODIFY QUERY if the user has privilege directly or through a role. - """ + """Check that user is able to modify view with MODIFY QUERY if the user has privilege directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_privilege, - name="modify query privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_privilege, + name="modify query privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_privilege, - name="modify query privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_privilege, + name="modify query privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestScenario def modify_query_with_privilege(self, grant_target_name, user_name, node=None): - """Grant MODIFY QUERY and check that user is able to execute it. - """ + """Grant MODIFY QUERY and check that user is able to execute it.""" view_name = f"view_{getuid()}" if node is None: @@ -1604,41 +2363,52 @@ def modify_query_with_privilege(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant MODIFY QUERY privilege"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with Then("I modify the view query as the user"): - node.query(f"ALTER TABLE {view_name} MODIFY QUERY SELECT 2", settings = [("user",f"{user_name}")]) + node.query( + f"ALTER TABLE {view_name} MODIFY QUERY SELECT 2", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def modify_query_with_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to modify the view query with MODIFY QUERY if the privilege has been revoked, directly or from a role. - """ + """Check that user is unable to modify the view query with MODIFY QUERY if the privilege has been revoked, directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_revoked_privilege, - name="modify query privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_revoked_privilege, + name="modify query privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_revoked_privilege, - name="modify query privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_revoked_privilege, + name="modify query privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestScenario def modify_query_with_revoked_privilege(self, grant_target_name, user_name, node=None): - """Revoke MODIFY QUERY and check that user is unable to modify the view query. - """ + """Revoke MODIFY QUERY and check that user is unable to modify the view query.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1647,21 +2417,32 @@ def modify_query_with_revoked_privilege(self, grant_target_name, user_name, node try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant MODIFY QUERY privilege"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with And("I revoke MODIFY QUERY privilege"): - node.query(f"REVOKE ALTER VIEW MODIFY QUERY ON {view_name} FROM {grant_target_name}") + node.query( + f"REVOKE ALTER VIEW MODIFY QUERY ON {view_name} FROM {grant_target_name}" + ) with Then("I modify the view query as the user"): - node.query(f"ALTER TABLE {view_name} MODIFY QUERY SELECT 2", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"ALTER TABLE {view_name} MODIFY QUERY SELECT 2", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def modify_query_without_source_table_privilege(self, node=None): """Check that user is unable to modify the view query to have a source table in the stored query @@ -1675,24 +2456,37 @@ def modify_query_without_source_table_privilege(self, node=None): if node is None: node = self.context.node with table(node, f"{table_name}"): - with user(node, f"{user_name}"): + with user(node, f"{user_name}"): try: with When("I create a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with And("I grant view MODIFY QUERY privilege to the user"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {user_name}") - with Then("I attempt to use MODIFY QUERY on the view without privilege on the source table"): - node.query(f"ALTER TABLE {view_name} MODIFY QUERY SELECT * FROM {table_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {user_name}" + ) + with Then( + "I attempt to use MODIFY QUERY on the view without privilege on the source table" + ): + node.query( + f"ALTER TABLE {view_name} MODIFY QUERY SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def modify_query_with_source_table_privilege_granted_directly_or_via_role(self, node=None): +def modify_query_with_source_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to modify the view query to have a source table in the stored query, if and only if the user has SELECT privilege for the view and the source table, either directly or from a role. """ @@ -1702,19 +2496,25 @@ def modify_query_with_source_table_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_source_table_privilege, - name="modify query with source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_source_table_privilege, + name="modify query with source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_source_table_privilege, - name="modify query with source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_source_table_privilege, + name="modify query with source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def modify_query_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Grant MODIFY QUERY privilege on view and SELECT privilege on the new source table and check the user is able to modify the view query. - """ +def modify_query_with_source_table_privilege( + self, user_name, grant_target_name, node=None +): + """Grant MODIFY QUERY privilege on view and SELECT privilege on the new source table and check the user is able to modify the view query.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -1724,20 +2524,28 @@ def modify_query_with_source_table_privilege(self, user_name, grant_target_name, try: with Given("I have a view with a source table"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with And("I grant view MODIFY QUERY privilege"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with And("I grant table SELECT privilege"): node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I check the user is able to modify the view query"): - node.query(f"ALTER TABLE {view_name} MODIFY QUERY SELECT * FROM {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"ALTER TABLE {view_name} MODIFY QUERY SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def modify_query_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to modify the view query to use a query with two subqueries if and only if @@ -1749,14 +2557,19 @@ def modify_query_with_subquery_privilege_granted_directly_or_via_role(self, node if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_subquery, - name="modify query with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_subquery, + name="modify query with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_subquery, - name="modify query with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_subquery, + name="modify query with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def modify_query_with_subquery(self, user_name, grant_target_name, node=None): @@ -1776,31 +2589,82 @@ def modify_query_with_subquery(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a subquery"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant MODIFY QUERY privilege on view"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Then("I attempt to modify the view query as the user"): - output = node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def modify_query_with_join_query_privilege_granted_directly_or_via_role(self, node=None): +def modify_query_with_join_query_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to modify the view query to use a query that includes a `JOIN` statement if and only if the user has SELECT privilege on all the tables and MODIFY QUERY privilege on the view, either directly or through a role. """ @@ -1810,14 +2674,19 @@ def modify_query_with_join_query_privilege_granted_directly_or_via_role(self, no if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_join_query, - name="modify query with join, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_join_query, + name="modify query with join, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_join_query, - name="modify query with join, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_join_query, + name="modify query with join, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def modify_query_with_join_query(self, user_name, grant_target_name, node=None): @@ -1836,30 +2705,72 @@ def modify_query_with_join_query(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a JOIN statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant MODIFY QUERY privilege on view"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def modify_query_with_union_query_privilege_granted_directly_or_via_role(self, node=None): +def modify_query_with_union_query_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to modify the view query to include a `UNION ALL` statement if and only if the user has SELECT privilege on all the tables and MODIFY QUERY on the view, either directly or through a role. """ @@ -1869,19 +2780,23 @@ def modify_query_with_union_query_privilege_granted_directly_or_via_role(self, n if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_union_query, - name="modify query with union, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_union_query, + name="modify query with union, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_union_query, - name="modify query with union, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_union_query, + name="modify query with union, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def modify_query_with_union_query(self, user_name, grant_target_name, node=None): - """Grant MODIFY QUERY on the view and SELECT on the tables in the stored query and check the user is able modify the view query if and only if they have SELECT privilege on all of them. - """ + """Grant MODIFY QUERY on the view and SELECT on the tables in the stored query and check the user is able modify the view query if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -1894,31 +2809,72 @@ def modify_query_with_union_query(self, user_name, grant_target_name, node=None) try: with Given("I have a view with a UNION statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant MODIFY QUERY privilege on view"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def modify_query_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): +def modify_query_with_join_union_subquery_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to modify the view query to include `UNION ALL`, `JOIN` and two subqueries if and only if the user has SELECT privilege on all the tables and the view, either directly or through a role. """ @@ -1928,17 +2884,24 @@ def modify_query_with_join_union_subquery_privilege_granted_directly_or_via_role if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_join_union_subquery, - name="modify query with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_join_union_subquery, + name="modify query with join union subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_join_union_subquery, - name="modify query with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_join_union_subquery, + name="modify query with join union subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def modify_query_with_join_union_subquery(self, grant_target_name, user_name, node=None): +def modify_query_with_join_union_subquery( + self, grant_target_name, user_name, node=None +): """Grant MODIFY QUERY on the view and SELECT on tables in the modify query and check the user is able modify the view query if and only if they have SELECT privilege on all of them. """ @@ -1953,36 +2916,97 @@ def modify_query_with_join_union_subquery(self, grant_target_name, user_name, no if node is None: node = self.context.node - with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with table( + node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}" + ): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT 1" + ) with When("I grant MODIFY QUERY privilege on view"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view_name} TO {grant_target_name}" + ) with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=5): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), settings = [("user", f"{user_name}")]) + node.query( + modify_query_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def modify_query_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): +def modify_query_with_nested_views_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to modify the view query to include other views if and only if the user has MODIFY QUERY privilege on the view SELECT privilege on all of the views and the source tables for those views, either directly or through a role. """ @@ -1992,14 +3016,19 @@ def modify_query_with_nested_views_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=modify_query_with_nested_views, - name="modify query with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=modify_query_with_nested_views, + name="modify query with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=modify_query_with_nested_views, - name="modify query with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=modify_query_with_nested_views, + name="modify query with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def modify_query_with_nested_views(self, grant_target_name, user_name, node=None): @@ -2023,26 +3052,83 @@ def modify_query_with_nested_views(self, grant_target_name, user_name, node=None try: with Given("I have some views"): node.query(f"CREATE VIEW {view0_name} AS SELECT y FROM {table0_name}") - node.query(f"CREATE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)") - node.query(f"CREATE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y") + node.query( + f"CREATE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)" + ) + node.query( + f"CREATE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y" + ) node.query(f"CREATE VIEW {view3_name} AS SELECT 1") with When("I grant MODIFY QUERY privilege on view"): - node.query(f"GRANT ALTER VIEW MODIFY QUERY ON {view3_name} TO {grant_target_name}") + node.query( + f"GRANT ALTER VIEW MODIFY QUERY ON {view3_name} TO {grant_target_name}" + ) with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,2,3,7,11,15,31,39,79,95],permutations(table_count=7))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view2_name, table3_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 2, 3, 7, 11, 15, 31, 39, 79, 95], + permutations(table_count=7), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view2_name, + table3_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + modify_query_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=7))+1, grant_target_name, view0_name, view1_name, view2_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=7)) + 1, + grant_target_name, + view0_name, + view1_name, + view2_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Then("I attempt to modify the view query as the user"): - node.query(modify_query_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), settings = [("user", f"{user_name}")]) + node.query( + modify_query_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -2055,40 +3141,53 @@ def modify_query_with_nested_views(self, grant_target_name, user_name, node=None with And("I drop view3", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestSuite def insert(self, node=None): - """Check RBAC functionality of INSERT with materialized views. - """ - Scenario(run=insert_on_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=insert_with_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=insert_on_target_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Check RBAC functionality of INSERT with materialized views.""" + Scenario( + run=insert_on_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=insert_with_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=insert_on_target_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario @Requirements( RQ_SRS_006_RBAC_MaterializedView_Insert_SourceTable("1.0"), ) def insert_on_source_table_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to INSERT on the source table of the materialized view with only INSERT privilege on the source table. - """ + """Check that user is able to INSERT on the source table of the materialized view with only INSERT privilege on the source table.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=insert_on_source_table, - name="insert on source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_on_source_table, + name="insert on source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=insert_on_source_table, - name="insert on source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_on_source_table, + name="insert on source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def insert_on_source_table(self, grant_target_name, user_name, node=None): - """Grant SELECT on the source table to the user and check they are able to SELECT from it. - """ + """Grant SELECT on the source table to the user and check they are able to SELECT from it.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -2099,17 +3198,23 @@ def insert_on_source_table(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table0_name} AS SELECT * FROM {table1_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table0_name} AS SELECT * FROM {table1_name}" + ) with When("I grant INSERT on the source table"): node.query(f"GRANT INSERT ON {table1_name} TO {grant_target_name}") with Then("I attempt to insert into the source table"): - node.query(f"INSERT INTO {table1_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) + node.query( + f"INSERT INTO {table1_name}(d) VALUES ('2020-01-01')", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_MaterializedView_Insert("1.0"), @@ -2124,19 +3229,23 @@ def insert_with_privilege_granted_directly_or_via_role(self, node=None): if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=insert_with_insert_privilege, - name="insert on view, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_with_insert_privilege, + name="insert on view, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=insert_with_insert_privilege, - name="insert on view, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_with_insert_privilege, + name="insert on view, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def insert_with_insert_privilege(self, grant_target_name, user_name, node=None): - """Grant INSERT and check user is able to INSERT into the materialized view only if they have INSERT privilege for the view. - """ + """Grant INSERT and check user is able to INSERT into the materialized view only if they have INSERT privilege for the view.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -2148,18 +3257,23 @@ def insert_with_insert_privilege(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table0_name} AS SELECT * FROM {table1_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table0_name} AS SELECT * FROM {table1_name}" + ) with When("I grant INSERT on the view"): node.query(f"GRANT INSERT ON {view_name} TO {grant_target_name}") with Then("I attempt to insert into the view"): - node.query(f"INSERT INTO {view_name}(d) VALUES ('2020-01-01')", - settings = [("user",f"{user_name}")]) + node.query( + f"INSERT INTO {view_name}(d) VALUES ('2020-01-01')", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario @Requirements( RQ_SRS_006_RBAC_MaterializedView_Insert_TargetTable("1.0"), @@ -2174,19 +3288,23 @@ def insert_on_target_table_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=insert_on_target_table, - name="insert on target table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_on_target_table, + name="insert on target table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=insert_on_target_table, - name="insert on target table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_on_target_table, + name="insert on target table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def insert_on_target_table(self, grant_target_name, user_name, node=None): - """Grant INSERT and check user is able to INSERT into target table. - """ + """Grant INSERT and check user is able to INSERT into target table.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -2197,21 +3315,30 @@ def insert_on_target_table(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} TO {table0_name} AS SELECT * FROM {table1_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} TO {table0_name} AS SELECT * FROM {table1_name}" + ) with When("I grant INSERT on the target table"): node.query(f"GRANT INSERT ON {table0_name} TO {grant_target_name}") with Then("I attempt to insert into the target table"): - node.query(f"INSERT INTO {table0_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) + node.query( + f"INSERT INTO {table0_name}(d) VALUES ('2020-01-01')", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + + @TestScenario @Requirements( RQ_SRS_006_RBAC_MaterializedView_Insert_TargetTable("1.0"), ) -def insert_on_implicit_target_table_privilege_granted_directly_or_via_role(self, node=None): +def insert_on_implicit_target_table_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to INSERT into the implicit target table of a materialized view if and only if they have INSERT privilege for the table, either directly or through a role. """ @@ -2221,22 +3348,26 @@ def insert_on_implicit_target_table_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=insert_on_target_table, - name="insert on implicit target table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=insert_on_target_table, + name="insert on implicit target table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=insert_on_target_table, - name="insert on implicit target table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=insert_on_target_table, + name="insert on implicit target table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def insert_on_target_table(self, grant_target_name, user_name, node=None): - """Grant INSERT and check user is able to INSERT into implicit target table. - """ + """Grant INSERT and check user is able to INSERT into implicit target table.""" view_name = f"view_{getuid()}" table_name = f"table0_{getuid()}" - implicit_table_name = f"\\\".inner.{view_name}\\\"" + implicit_table_name = f'\\".inner.{view_name}\\"' if node is None: node = self.context.node @@ -2244,17 +3375,25 @@ def insert_on_target_table(self, grant_target_name, user_name, node=None): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}") + node.query( + f"CREATE MATERIALIZED VIEW {view_name} ENGINE = Memory AS SELECT * FROM {table_name}" + ) with When("I grant INSERT on the target table"): - node.query(f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}") + node.query( + f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}" + ) with Then("I attempt to insert into the target table"): - node.query(f"INSERT INTO {implicit_table_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) + node.query( + f"INSERT INTO {implicit_table_name}(d) VALUES ('2020-01-01')", + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_MaterializedView("1.0"), diff --git a/tests/testflows/rbac/tests/views/view.py b/tests/testflows/rbac/tests/views/view.py index f4fb4550a75..66788747934 100755 --- a/tests/testflows/rbac/tests/views/view.py +++ b/tests/testflows/rbac/tests/views/view.py @@ -5,28 +5,57 @@ from rbac.requirements import * from rbac.helper.common import * import rbac.helper.errors as errors + @TestSuite @Requirements( RQ_SRS_006_RBAC_View_Create("1.0"), ) def create(self, node=None): - """Test the RBAC functionality of the `CREATE VIEW` command. - """ - Scenario(run=create_without_create_view_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_create_view_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_revoked_create_view_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_union_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_join_union_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=create_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `CREATE VIEW` command.""" + Scenario( + run=create_without_create_view_privilege, setup=instrument_clickhouse_server_log + ) + Scenario( + run=create_with_create_view_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_revoked_create_view_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_union_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_join_union_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=create_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def create_without_create_view_privilege(self, node=None): - """Check that user is unable to create a view without CREATE VIEW privilege. - """ + """Check that user is unable to create a view without CREATE VIEW privilege.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -36,32 +65,40 @@ def create_without_create_view_privilege(self, node=None): with user(node, f"{user_name}"): with When("I try to create a view without CREATE VIEW privilege as the user"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE VIEW {view_name} AS SELECT 1", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE VIEW {view_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_create_view_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to create a view with CREATE VIEW privilege, either granted directly or through a role. - """ + """Check that user is able to create a view with CREATE VIEW privilege, either granted directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_create_view_privilege, - name="create with create view privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_create_view_privilege, + name="create with create view privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_create_view_privilege, - name="create with create view privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_create_view_privilege, + name="create with create view privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_create_view_privilege(self, grant_target_name, user_name, node=None): - """Check that user is able to create a view with the granted privileges. - """ + """Check that user is able to create a view with the granted privileges.""" view_name = f"view_{getuid()}" if node is None: @@ -72,35 +109,46 @@ def create_with_create_view_privilege(self, grant_target_name, user_name, node=N node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I try to create a view without privilege as the user"): - node.query(f"CREATE VIEW {view_name} AS SELECT 1", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE VIEW {view_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def create_with_revoked_create_view_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to create view after the CREATE VIEW privilege is revoked, either directly or from a role. - """ +def create_with_revoked_create_view_privilege_revoked_directly_or_from_role( + self, node=None +): + """Check that user is unable to create view after the CREATE VIEW privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_revoked_create_view_privilege, - name="create with create view privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_view_privilege, + name="create with create view privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_revoked_create_view_privilege, - name="create with create view privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_revoked_create_view_privilege, + name="create with create view privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline -def create_with_revoked_create_view_privilege(self, grant_target_name, user_name, node=None): - """Revoke CREATE VIEW privilege and check the user is unable to create a view. - """ +def create_with_revoked_create_view_privilege( + self, grant_target_name, user_name, node=None +): + """Revoke CREATE VIEW privilege and check the user is unable to create a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -113,8 +161,13 @@ def create_with_revoked_create_view_privilege(self, grant_target_name, user_name node.query(f"REVOKE CREATE VIEW ON {view_name} FROM {grant_target_name}") with Then("I try to create a view on the table as the user"): - node.query(f"CREATE VIEW {view_name} AS SELECT 1", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE VIEW {view_name} AS SELECT 1", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_without_source_table_privilege(self, node=None): @@ -134,8 +187,13 @@ def create_without_source_table_privilege(self, node=None): node.query(f"GRANT CREATE VIEW ON {view_name} TO {user_name}") with Then("I try to create a view without select privilege on the table"): - node.query(f"CREATE VIEW {view_name} AS SELECT * FROM {table_name}", settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"CREATE VIEW {view_name} AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) + @TestScenario def create_with_source_table_privilege_granted_directly_or_via_role(self, node=None): @@ -148,19 +206,23 @@ def create_with_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_source_table_privilege, - name="create with create view and select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create view and select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_source_table_privilege, - name="create with create view and select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_source_table_privilege, + name="create with create view and select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Check that user is unable to create a view without SELECT privilege on the source table. - """ + """Check that user is unable to create a view without SELECT privilege on the source table.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -176,16 +238,20 @@ def create_with_source_table_privilege(self, user_name, grant_target_name, node= with And("I try to create a view on the table as the user"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE VIEW {view_name} AS SELECT * FROM {table_name}", settings = [("user", f"{user_name}")]) + node.query( + f"CREATE VIEW {view_name} AS SELECT * FROM {table_name}", + settings=[("user", f"{user_name}")], + ) with Then("I check the view"): output = node.query(f"SELECT count(*) FROM {view_name}").output - assert output == '0', error() + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view where the stored query has two subqueries @@ -198,14 +264,19 @@ def create_with_subquery_privilege_granted_directly_or_via_role(self, node=None) if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_subquery, - name="create with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_subquery, + name="create with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_subquery(self, user_name, grant_target_name, node=None): @@ -226,29 +297,72 @@ def create_with_subquery(self, user_name, grant_target_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to CREATE VIEW as the user with create privilege"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view where the stored query includes a `JOIN` statement @@ -261,14 +375,19 @@ def create_with_join_query_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_join_query, - name="create with join query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_query, - name="create with join query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_query, + name="create with join query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_query(self, grant_target_name, user_name, node=None): @@ -288,29 +407,63 @@ def create_with_join_query(self, grant_target_name, user_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Then("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_union_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view where the stored query includes a `UNION ALL` statement @@ -323,14 +476,19 @@ def create_with_union_query_privilege_granted_directly_or_via_role(self, node=No if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_union_query, - name="create with union query, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_union_query, + name="create with union query, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_union_query, - name="create with union query, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_union_query, + name="create with union query, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_union_query(self, grant_target_name, user_name, node=None): @@ -350,31 +508,67 @@ def create_with_union_query(self, grant_target_name, user_name, node=None): with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") with Then("I attempt to create view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name), settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): +def create_with_join_union_subquery_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to create a view with a stored query that includes `UNION ALL`, `JOIN` and two subqueries if and only if the user has SELECT privilege on all of the tables, either granted directly or through a role. """ @@ -384,14 +578,19 @@ def create_with_join_union_subquery_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_join_union_subquery, - name="create with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_join_union_subquery, - name="create with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_join_union_subquery, + name="create with join union subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_join_union_subquery(self, grant_target_name, user_name, node=None): @@ -409,36 +608,93 @@ def create_with_join_union_subquery(self, grant_target_name, user_name, node=Non if node is None: node = self.context.node - with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with table( + node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}" + ): with user(node, f"{user_name}"): try: with When("I grant CREATE VIEW privilege"): - node.query(f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}") - with Then("I attempt to create view as the user with CREATE VIEW privilege"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + f"GRANT CREATE VIEW ON {view_name} TO {grant_target_name}" + ) + with Then( + "I attempt to create view as the user with CREATE VIEW privilege" + ): + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=5): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table3_name, table4_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table3_name, + table4_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view_name=view_name, table0_name=table0_name, table1_name=table1_name, table2_name=table2_name, table3_name=table3_name, table4_name=table4_name), - settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view_name=view_name, + table0_name=table0_name, + table1_name=table1_name, + table2_name=table2_name, + table3_name=table3_name, + table4_name=table4_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def create_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to create a view with a stored query that includes other views if and only if @@ -450,14 +706,19 @@ def create_with_nested_views_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=create_with_nested_views, - name="create with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=create_with_nested_views, + name="create with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=create_with_nested_views, - name="create with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=create_with_nested_views, + name="create with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def create_with_nested_views(self, grant_target_name, user_name, node=None): @@ -481,31 +742,86 @@ def create_with_nested_views(self, grant_target_name, user_name, node=None): try: with Given("I have some views"): node.query(f"CREATE VIEW {view0_name} AS SELECT y FROM {table0_name}") - node.query(f"CREATE VIEW {view1_name} AS SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {view0_name} WHERE y<2)") - node.query(f"CREATE VIEW {view2_name} AS SELECT y FROM {table2_name} JOIN {view1_name} USING y") + node.query( + f"CREATE VIEW {view1_name} AS SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {view0_name} WHERE y<2)" + ) + node.query( + f"CREATE VIEW {view2_name} AS SELECT y FROM {table2_name} JOIN {view1_name} USING y" + ) with When("I grant CREATE VIEW privilege"): node.query(f"GRANT CREATE VIEW ON {view3_name} TO {grant_target_name}") - with Then("I attempt to create view as the user with CREATE VIEW privilege"): - node.query(create_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + with Then( + "I attempt to create view as the user with CREATE VIEW privilege" + ): + node.query( + create_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,2,3,7,11,15,31,39,79,95],permutations(table_count=7))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view2_name, table3_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 2, 3, 7, 11, 15, 31, 39, 79, 95], + permutations(table_count=7), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view2_name, + table3_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view3_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + create_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=7))+1, grant_target_name, view0_name, view1_name, view2_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=7)) + 1, + grant_target_name, + view0_name, + view1_name, + view2_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Given("I don't have a view"): node.query(f"DROP VIEW IF EXISTS {view3_name}") with Then("I attempt to create a view as the user"): - node.query(create_view_query.format(view3_name=view3_name, view2_name=view2_name, table3_name=table3_name), - settings = [("user", f"{user_name}")]) + node.query( + create_view_query.format( + view3_name=view3_name, + view2_name=view2_name, + table3_name=table3_name, + ), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -518,28 +834,57 @@ def create_with_nested_views(self, grant_target_name, user_name, node=None): with And("I drop view3", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_View_Select("1.0"), ) def select(self, node=None): - """Test the RBAC functionality of the `SELECT FROM view` command. - """ - Scenario(run=select_without_select_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_select_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_select_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_without_source_table_privilege, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_source_table_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_join_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_union_query_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_join_union_subquery_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=select_with_nested_views_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `SELECT FROM view` command.""" + Scenario( + run=select_without_select_privilege, setup=instrument_clickhouse_server_log + ) + Scenario( + run=select_with_select_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_select_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_without_source_table_privilege, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_source_table_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_join_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_union_query_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_join_union_subquery_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=select_with_nested_views_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario def select_without_select_privilege(self, node=None): - """Check that user is unable to select on a view without view SELECT privilege. - """ + """Check that user is unable to select on a view without view SELECT privilege.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -553,36 +898,44 @@ def select_without_select_privilege(self, node=None): node.query(f"CREATE VIEW {view_name} AS SELECT 1") with Then("I try to select from view without privilege as the user"): - node.query(f"SELECT * FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT * FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_select_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to select from a view if and only if they have select privilege on that view, either directly or from a role. - """ + """Check that user is able to select from a view if and only if they have select privilege on that view, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_select_privilege, - name="select with select privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_select_privilege, - name="select with select privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_select_privilege(self, user_name, grant_target_name, node=None): - """Grant SELECT privilege on a view and check the user is able to SELECT from it. - """ + """Grant SELECT privilege on a view and check the user is able to SELECT from it.""" view_name = f"view_{getuid()}" if node is None: @@ -596,36 +949,42 @@ def select_with_select_privilege(self, user_name, grant_target_name, node=None): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from view with privilege as the user"): - output = node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")]).output - assert output == '1', error() + output = node.query( + f"SELECT count(*) FROM {view_name}", settings=[("user", f"{user_name}")] + ).output + assert output == "1", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_select_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to select from a view if their SELECT privilege is revoked, either directly or from a role. - """ + """Check that user is unable to select from a view if their SELECT privilege is revoked, either directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_select_privilege, - name="select with select privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege revoked directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_select_privilege, - name="select with select privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_select_privilege, + name="select with select privilege revoked from a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_revoked_select_privilege(self, user_name, grant_target_name, node=None): - """Grant and revoke SELECT privilege on a view and check the user is unable to SELECT from it. - """ + """Grant and revoke SELECT privilege on a view and check the user is unable to SELECT from it.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -642,17 +1001,21 @@ def select_with_revoked_select_privilege(self, user_name, grant_target_name, nod node.query(f"REVOKE SELECT ON {view_name} FROM {grant_target_name}") with Then("I attempt to select from view with privilege as the user"): - node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_without_source_table_privilege(self, node=None): - """Check that user is unable to select from a view without SELECT privilege for the source table. - """ + """Check that user is unable to select from a view without SELECT privilege for the source table.""" user_name = f"user_{getuid()}" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -661,7 +1024,7 @@ def select_without_source_table_privilege(self, node=None): if node is None: node = self.context.node with table(node, f"{table_name}"): - with user(node, f"{user_name}"): + with user(node, f"{user_name}"): try: with When("I create a view from the source table"): node.query(f"DROP VIEW IF EXISTS {view_name}") @@ -669,14 +1032,21 @@ def select_without_source_table_privilege(self, node=None): with And("I grant view select privilege to the user"): node.query(f"GRANT SELECT ON {view_name} TO {user_name}") - with Then("I attempt to select from view without privilege on the source table"): - node.query(f"SELECT count(*) FROM {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + with Then( + "I attempt to select from view without privilege on the source table" + ): + node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_source_table_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view, with source table in the stored query, if and only if @@ -688,19 +1058,23 @@ def select_with_source_table_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_source_table_privilege, - name="select with source table, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_source_table_privilege, + name="select with source table, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_source_table_privilege, - name="select with source table, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_source_table_privilege, + name="select with source table, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_source_table_privilege(self, user_name, grant_target_name, node=None): - """Grant SELECT privilege on view and the source table for that view and check the user is able to SELECT from the view. - """ + """Grant SELECT privilege on view and the source table for that view and check the user is able to SELECT from the view.""" view_name = f"view_{getuid()}" table_name = f"table_{getuid()}" @@ -717,13 +1091,17 @@ def select_with_source_table_privilege(self, user_name, grant_target_name, node= node.query(f"GRANT SELECT ON {table_name} TO {grant_target_name}") with Then("I check the user is able to select from the view"): - output = node.query(f"SELECT count(*) FROM {view_name}", settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + f"SELECT count(*) FROM {view_name}", + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_subquery_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query has two subqueries if and only if @@ -735,19 +1113,23 @@ def select_with_subquery_privilege_granted_directly_or_via_role(self, node=None) if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_subquery, - name="select with subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_subquery, + name="select with subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_subquery, - name="select with subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_subquery, + name="select with subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_subquery(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -761,29 +1143,61 @@ def select_with_subquery(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a subquery"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE VIEW {view_name} AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))") + node.query( + f"CREATE VIEW {view_name} AS SELECT * FROM {table0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table2_name} WHERE y<2))" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=3): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=3))+1, grant_target_name, table0_name, table1_name, table2_name): + with grant_select_on_table( + node, + max(permutations(table_count=3)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + ): with Then("I attempt to select from a view as the user"): - output = node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]).output - assert output == '0', error() + output = node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ).output + assert output == "0", error() finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_join_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query includes a `JOIN` statement if and only if @@ -795,19 +1209,23 @@ def select_with_join_query_privilege_granted_directly_or_via_role(self, node=Non if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_join_query, - name="select with join, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_join_query, + name="select with join, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_join_query, - name="select with join, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_join_query, + name="select with join, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_join_query(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -820,28 +1238,54 @@ def select_with_join_query(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a JOIN statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE VIEW {view_name} AS SELECT * FROM {table0_name} JOIN {table1_name} USING d") + node.query( + f"CREATE VIEW {view_name} AS SELECT * FROM {table0_name} JOIN {table1_name} USING d" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_union_query_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view where the stored query includes a `UNION ALL` statement if and only if @@ -853,19 +1297,23 @@ def select_with_union_query_privilege_granted_directly_or_via_role(self, node=No if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_union_query, - name="select with union, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_union_query, + name="select with union, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_union_query, - name="select with union, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_union_query, + name="select with union, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_union_query(self, user_name, grant_target_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -878,30 +1326,58 @@ def select_with_union_query(self, user_name, grant_target_name, node=None): try: with Given("I have a view with a UNION statement"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE VIEW {view_name} AS SELECT * FROM {table0_name} UNION ALL SELECT * FROM {table1_name}") + node.query( + f"CREATE VIEW {view_name} AS SELECT * FROM {table0_name} UNION ALL SELECT * FROM {table1_name}" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=2): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, permutation, grant_target_name, table0_name, table1_name + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=2))+1, grant_target_name, table0_name, table1_name): + with grant_select_on_table( + node, + max(permutations(table_count=2)) + 1, + grant_target_name, + table0_name, + table1_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario -def select_with_join_union_subquery_privilege_granted_directly_or_via_role(self, node=None): +def select_with_join_union_subquery_privilege_granted_directly_or_via_role( + self, node=None +): """Check that user is able to select from a view with a stored query that includes `UNION ALL`, `JOIN` and two subqueries if and only if the user has SELECT privilege on all the tables and the view, either directly or through a role. """ @@ -911,19 +1387,23 @@ def select_with_join_union_subquery_privilege_granted_directly_or_via_role(self, if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_join_union_subquery, - name="select with join union subquery, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_join_union_subquery, + name="select with join union subquery, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_join_union_subquery, - name="select with join union subquery, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_join_union_subquery, + name="select with join union subquery, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_join_union_subquery(self, grant_target_name, user_name, node=None): - """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on the view and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view_name = f"view_{getuid()}" table0_name = f"table0_{getuid()}" table1_name = f"table1_{getuid()}" @@ -935,32 +1415,70 @@ def select_with_join_union_subquery(self, grant_target_name, user_name, node=Non if node is None: node = self.context.node - with table(node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}"): + with table( + node, f"{table0_name},{table1_name},{table2_name},{table3_name},{table4_name}" + ): try: with Given("I have a view"): node.query(f"DROP VIEW IF EXISTS {view_name}") - node.query(f"CREATE VIEW {view_name} AS SELECT y FROM {table0_name} JOIN {table1_name} USING y UNION ALL SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table3_name} WHERE y IN (SELECT y FROM {table4_name} WHERE y<2))") + node.query( + f"CREATE VIEW {view_name} AS SELECT y FROM {table0_name} JOIN {table1_name} USING y UNION ALL SELECT y FROM {table1_name} WHERE y IN (SELECT y FROM {table3_name} WHERE y IN (SELECT y FROM {table4_name} WHERE y<2))" + ) with When("I grant SELECT privilege on view"): node.query(f"GRANT SELECT ON {view_name} TO {grant_target_name}") with Then("I attempt to select from the view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) for permutation in permutations(table_count=5): - with grant_select_on_table(node, permutation, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + with grant_select_on_table( + node, + permutation, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all tables"): - with grant_select_on_table(node, max(permutations(table_count=5))+1, grant_target_name, table0_name, table1_name, table2_name, table3_name, table4_name): + with grant_select_on_table( + node, + max(permutations(table_count=5)) + 1, + grant_target_name, + table0_name, + table1_name, + table2_name, + table3_name, + table4_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view_name=view_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view_name=view_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def select_with_nested_views_privilege_granted_directly_or_via_role(self, node=None): """Check that user is able to select from a view with a stored query that includes other views if and only if @@ -972,19 +1490,23 @@ def select_with_nested_views_privilege_granted_directly_or_via_role(self, node=N if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=select_with_nested_views, - name="select with nested views, privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=select_with_nested_views, + name="select with nested views, privilege granted directly", + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=select_with_nested_views, - name="select with nested views, privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=select_with_nested_views, + name="select with nested views, privilege granted through a role", + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def select_with_nested_views(self, grant_target_name, user_name, node=None): - """Grant SELECT on views and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them. - """ + """Grant SELECT on views and tables in the stored query and check the user is able to SELECT if and only if they have SELECT privilege on all of them.""" view0_name = f"view0_{getuid()}" view1_name = f"view1_{getuid()}" view2_name = f"view2_{getuid()}" @@ -1002,25 +1524,71 @@ def select_with_nested_views(self, grant_target_name, user_name, node=None): try: with Given("I have some views"): node.query(f"CREATE VIEW {view0_name} AS SELECT y FROM {table0_name}") - node.query(f"CREATE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)") - node.query(f"CREATE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y") - node.query(f"CREATE VIEW {view3_name} AS SELECT y FROM {view2_name} UNION ALL SELECT y FROM {table3_name}") + node.query( + f"CREATE VIEW {view1_name} AS SELECT y FROM {view0_name} WHERE y IN (SELECT y FROM {table1_name} WHERE y<2)" + ) + node.query( + f"CREATE VIEW {view2_name} AS SELECT y FROM {view1_name} JOIN {table2_name} USING y" + ) + node.query( + f"CREATE VIEW {view3_name} AS SELECT y FROM {view2_name} UNION ALL SELECT y FROM {table3_name}" + ) with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view3_name=view3_name), - settings = [("user",f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view3_name=view3_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) - for permutation in ([0,1,3,5,7,13,15,23,31,45,63,95,127,173,237,247,253],permutations(table_count=8))[self.context.stress]: - with grant_select_on_table(node, permutation, grant_target_name, view3_name, table3_name, view2_name, view1_name, table2_name, view0_name, table1_name, table0_name) as tables_granted: - with When(f"permutation={permutation}, tables granted = {tables_granted}"): + for permutation in ( + [0, 1, 3, 5, 7, 13, 15, 23, 31, 45, 63, 95, 127, 173, 237, 247, 253], + permutations(table_count=8), + )[self.context.stress]: + with grant_select_on_table( + node, + permutation, + grant_target_name, + view3_name, + table3_name, + view2_name, + view1_name, + table2_name, + view0_name, + table1_name, + table0_name, + ) as tables_granted: + with When( + f"permutation={permutation}, tables granted = {tables_granted}" + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view3_name=view3_name), - settings = [("user", f"{user_name}")], exitcode=exitcode, message=message) + node.query( + select_view_query.format(view3_name=view3_name), + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) with When("I grant select on all views"): - with grant_select_on_table(node, max(permutations(table_count=8))+1, grant_target_name, view0_name, view1_name, view2_name, view3_name, table0_name, table1_name, table2_name, table3_name): + with grant_select_on_table( + node, + max(permutations(table_count=8)) + 1, + grant_target_name, + view0_name, + view1_name, + view2_name, + view3_name, + table0_name, + table1_name, + table2_name, + table3_name, + ): with Then("I attempt to select from a view as the user"): - node.query(select_view_query.format(view3_name=view3_name), settings = [("user", f"{user_name}")]) + node.query( + select_view_query.format(view3_name=view3_name), + settings=[("user", f"{user_name}")], + ) finally: with Finally("I drop the views"): @@ -1033,40 +1601,47 @@ def select_with_nested_views(self, grant_target_name, user_name, node=None): with And("I drop view3", flags=TE): node.query(f"DROP VIEW IF EXISTS {view0_name}") + @TestSuite @Requirements( RQ_SRS_006_RBAC_View_Drop("1.0"), ) def drop(self, node=None): - """Test the RBAC functionality of the `DROP VIEW` command. - """ - Scenario(run=drop_with_privilege_granted_directly_or_via_role, setup=instrument_clickhouse_server_log) - Scenario(run=drop_with_revoked_privilege_revoked_directly_or_from_role, setup=instrument_clickhouse_server_log) + """Test the RBAC functionality of the `DROP VIEW` command.""" + Scenario( + run=drop_with_privilege_granted_directly_or_via_role, + setup=instrument_clickhouse_server_log, + ) + Scenario( + run=drop_with_revoked_privilege_revoked_directly_or_from_role, + setup=instrument_clickhouse_server_log, + ) + @TestScenario - def drop_with_privilege_granted_directly_or_via_role(self, node=None): - """Check that user is able to drop view with DROP VIEW privilege if the user has privilege directly or through a role. - """ + """Check that user is able to drop view with DROP VIEW privilege if the user has privilege directly or through a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=drop_with_privilege, - name="drop privilege granted directly")(grant_target_name=user_name, user_name=user_name) + Scenario(test=drop_with_privilege, name="drop privilege granted directly")( + grant_target_name=user_name, user_name=user_name + ) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=drop_with_privilege, - name="drop privilege granted through a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=drop_with_privilege, name="drop privilege granted through a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def drop_with_privilege(self, grant_target_name, user_name, node=None): - """Grant DROP VIEW privilege and check the user is able to successfully drop a view. - """ + """Grant DROP VIEW privilege and check the user is able to successfully drop a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.table_does_not_exist(name=f"default.{view_name}") @@ -1081,7 +1656,7 @@ def drop_with_privilege(self, grant_target_name, user_name, node=None): node.query(f"GRANT DROP VIEW ON {view_name} TO {grant_target_name}") with And("I drop the view as the user"): - node.query(f"DROP VIEW {view_name}", settings = [("user",f"{user_name}")]) + node.query(f"DROP VIEW {view_name}", settings=[("user", f"{user_name}")]) with Then("I check the table does not exist"): node.query(f"SELECT * FROM {view_name}", exitcode=exitcode, message=message) @@ -1090,29 +1665,31 @@ def drop_with_privilege(self, grant_target_name, user_name, node=None): with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestScenario def drop_with_revoked_privilege_revoked_directly_or_from_role(self, node=None): - """Check that user is unable to drop view with DROP VIEW privilege revoked directly or from a role. - """ + """Check that user is unable to drop view with DROP VIEW privilege revoked directly or from a role.""" user_name = f"user_{getuid()}" role_name = f"role_{getuid()}" if node is None: node = self.context.node with user(node, f"{user_name}"): - Scenario(test=drop_with_revoked_privilege, - name="drop privilege revoked directly")(grant_target_name=user_name, user_name=user_name) + Scenario( + test=drop_with_revoked_privilege, name="drop privilege revoked directly" + )(grant_target_name=user_name, user_name=user_name) with user(node, f"{user_name}"), role(node, f"{role_name}"): with When("I grant the role to the user"): node.query(f"GRANT {role_name} TO {user_name}") - Scenario(test=drop_with_revoked_privilege, - name="drop privilege revoked from a role")(grant_target_name=role_name, user_name=user_name) + Scenario( + test=drop_with_revoked_privilege, name="drop privilege revoked from a role" + )(grant_target_name=role_name, user_name=user_name) + @TestOutline def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): - """Revoke DROP VIEW privilege and check the user is unable to DROP a view. - """ + """Revoke DROP VIEW privilege and check the user is unable to DROP a view.""" view_name = f"view_{getuid()}" exitcode, message = errors.not_enough_privileges(name=f"{user_name}") @@ -1130,13 +1707,18 @@ def drop_with_revoked_privilege(self, grant_target_name, user_name, node=None): node.query(f"REVOKE DROP VIEW ON {view_name} FROM {grant_target_name}") with Then("I drop the view as the user"): - node.query(f"DROP VIEW {view_name}", settings = [("user",f"{user_name}")], - exitcode=exitcode, message=message) + node.query( + f"DROP VIEW {view_name}", + settings=[("user", f"{user_name}")], + exitcode=exitcode, + message=message, + ) finally: with Finally("I drop the view"): node.query(f"DROP VIEW IF EXISTS {view_name}") + @TestFeature @Requirements( RQ_SRS_006_RBAC_View("1.0"), diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index c803d9ef210..bce8274c5cc 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -6,29 +6,72 @@ append_path(sys.path, ".") from helpers.argparser import argparser + @TestModule @Name("clickhouse") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path, stress=None): - """ClickHouse regression. - """ - args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path, "stress": stress} +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """ClickHouse regression.""" + args = { + "local": local, + "clickhouse_binary_path": clickhouse_binary_path, + "clickhouse_version": clickhouse_version, + "stress": stress, + } self.context.stress = stress + self.context.clickhouse_version = clickhouse_version with Pool(8) as pool: try: - Feature(test=load("example.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("ldap.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("rbac.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("aes_encryption.regression", "regression"), parallel=True, executor=pool)(**args) # TODO: fix it! + Feature( + test=load("example.regression", "regression"), + parallel=True, + executor=pool, + )(**args) + Feature( + test=load("ldap.regression", "regression"), parallel=True, executor=pool + )(**args) + Feature( + test=load("rbac.regression", "regression"), parallel=True, executor=pool + )(**args) + Feature( + test=load("aes_encryption.regression", "regression"), + parallel=True, + executor=pool, + )( + **args + ) # TODO: fix it! # Feature(test=load("map_type.regression", "regression"), parallel=True, executor=pool)(**args) # TODO: fix it! - Feature(test=load("window_functions.regression", "regression"), parallel=True, executor=pool)(**args) # TODO: fix it! - Feature(test=load("datetime64_extended_range.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("kerberos.regression", "regression"), parallel=True, executor=pool)(**args) - Feature(test=load("extended_precision_data_types.regression", "regression"), parallel=True, executor=pool)(**args) # TODO: fix it! + Feature( + test=load("window_functions.regression", "regression"), + parallel=True, + executor=pool, + )( + **args + ) # TODO: fix it! + Feature( + test=load("datetime64_extended_range.regression", "regression"), + parallel=True, + executor=pool, + )(**args) + Feature( + test=load("kerberos.regression", "regression"), + parallel=True, + executor=pool, + )(**args) + Feature( + test=load("extended_precision_data_types.regression", "regression"), + parallel=True, + executor=pool, + )( + **args + ) # TODO: fix it! finally: join() + if main(): regression() diff --git a/tests/testflows/window_functions/configs/clickhouse/config.xml b/tests/testflows/window_functions/configs/clickhouse/config.xml deleted file mode 100644 index 842a0573d49..00000000000 --- a/tests/testflows/window_functions/configs/clickhouse/config.xml +++ /dev/null @@ -1,448 +0,0 @@ - - - - - - trace - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - - - 8123 - 9000 - - - - - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - - - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - - - - - - - 9009 - - - - - - - - 0.0.0.0 - - - - - - - - - - - - 4096 - 3 - - - 100 - - - - - - 8589934592 - - - 5368709120 - - - - /var/lib/clickhouse/ - - - /var/lib/clickhouse/tmp/ - - - /var/lib/clickhouse/user_files/ - - - /var/lib/clickhouse/access/ - - - - - - users.xml - - - - /var/lib/clickhouse/access/ - - - - - users.xml - - - default - - - - - - default - - - - - - - - - false - - - - - - - - localhost - 9000 - - - - - - - localhost - 9000 - - - - - localhost - 9000 - - - - - - - localhost - 9440 - 1 - - - - - - - localhost - 9000 - - - - - localhost - 1 - - - - - - - - - - - - - - - - - 3600 - - - - 3600 - - - 60 - - - - - - - - - - system - query_log
- - toYYYYMM(event_date) - - 7500 - - - - - system - trace_log
- - toYYYYMM(event_date) - 7500 - - - - - system - query_thread_log
- toYYYYMM(event_date) - 7500 - - - - - system - part_log
- toYYYYMM(event_date) - 7500 - - - - - - - - - - - - - - - *_dictionary.xml - - - - - - - - - - /clickhouse/task_queue/ddl - - - - - - - - - - - - - - - - click_cost - any - - 0 - 3600 - - - 86400 - 60 - - - - max - - 0 - 60 - - - 3600 - 300 - - - 86400 - 3600 - - - - - - /var/lib/clickhouse/format_schemas/ - - - - diff --git a/tests/testflows/window_functions/configs/clickhouse/users.xml b/tests/testflows/window_functions/configs/clickhouse/users.xml deleted file mode 100644 index c7d0ecae693..00000000000 --- a/tests/testflows/window_functions/configs/clickhouse/users.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - 10000000000 - - - 0 - - - random - - - - - 1 - - - - - - - - - - - - - ::/0 - - - - default - - - default - - - 1 - - - - - - - - - - - - - - - - - 3600 - - - 0 - 0 - 0 - 0 - 0 - - - - diff --git a/tests/testflows/window_functions/regression.py b/tests/testflows/window_functions/regression.py index 2c70fc1d075..f7fa116ead8 100755 --- a/tests/testflows/window_functions/regression.py +++ b/tests/testflows/window_functions/regression.py @@ -8,105 +8,146 @@ append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser -from window_functions.requirements import SRS019_ClickHouse_Window_Functions, RQ_SRS_019_ClickHouse_WindowFunctions +from window_functions.requirements import ( + SRS019_ClickHouse_Window_Functions, + RQ_SRS_019_ClickHouse_WindowFunctions, +) xfails = { - "tests/:/frame clause/range frame/between expr following and expr following without order by error": - [(Fail, "invalid error message")], - "tests/:/frame clause/range frame/between expr following and expr preceding without order by error": - [(Fail, "invalid error message")], - "tests/:/frame clause/range frame/between expr following and current row without order by error": - [(Fail, "invalid error message")], - "tests/:/frame clause/range frame/between expr following and current row zero special case": - [(Fail, "known bug")], - "tests/:/frame clause/range frame/between expr following and expr preceding with order by zero special case": - [(Fail, "known bug")], - "tests/:/funcs/lag/anyOrNull with column value as offset": - [(Fail, "column values are not supported as offset")], - "tests/:/funcs/lead/subquery as offset": - [(Fail, "subquery is not supported as offset")], - "tests/:/frame clause/range frame/between current row and unbounded following modifying named window": - [(Fail, "range with named window is not supported")], - "tests/:/frame clause/range overflow/negative overflow with Int16": - [(Fail, "exception on conversion")], - "tests/:/frame clause/range overflow/positive overflow with Int16": - [(Fail, "exception on conversion")], - "tests/:/misc/subquery expr preceding": - [(Fail, "subquery is not supported as offset")], - "tests/:/frame clause/range errors/error negative preceding offset": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22442")], - "tests/:/frame clause/range errors/error negative following offset": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/22442")], - "tests/:/misc/window functions in select expression": - [(Fail, "not supported, https://github.com/ClickHouse/ClickHouse/issues/19857")], - "tests/:/misc/window functions in subquery": - [(Fail, "not supported, https://github.com/ClickHouse/ClickHouse/issues/19857")], - "tests/:/misc/in view": - [(Fail, "bug, https://github.com/ClickHouse/ClickHouse/issues/26001")], - "tests/:/frame clause/range frame/order by decimal": - [(Fail, "Exception: The RANGE OFFSET frame for 'DB::ColumnDecimal >' ORDER BY column is not implemented")], - "tests/:/frame clause/range frame/with nulls": - [(Fail, "DB::Exception: The RANGE OFFSET frame for 'DB::ColumnNullable' ORDER BY column is not implemented")], - "tests/:/aggregate funcs/aggregate funcs over rows frame/func='mannWhitneyUTest(salary, 1)'": - [(Fail, "need to investigate")], - "tests/:/aggregate funcs/aggregate funcs over rows frame/func='rankCorr(salary, 0.5)'": - [(Fail, "need to investigate")], - "tests/distributed/misc/query with order by and one window": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/distributed/over clause/empty named window": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/distributed/over clause/empty": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/distributed/over clause/adhoc window": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/distributed/frame clause/range datetime/:": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/distributed/frame clause/range frame/between expr preceding and expr following with partition by same column twice": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/:/funcs/leadInFrame/explicit default value": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057")], - "tests/:/funcs/leadInFrame/with nulls": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057")], - "tests/:/funcs/leadInFrame/default offset": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")], - "tests/:/funcs/lagInFrame/explicit default value": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057")], - "tests/:/funcs/lagInFrame/with nulls": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057")], - "tests/:/funcs/lagInFrame/default offset": - [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902")] + "tests/:/frame clause/range frame/between expr following and expr following without order by error": [ + (Fail, "invalid error message") + ], + "tests/:/frame clause/range frame/between expr following and expr preceding without order by error": [ + (Fail, "invalid error message") + ], + "tests/:/frame clause/range frame/between expr following and current row without order by error": [ + (Fail, "invalid error message") + ], + "tests/:/frame clause/range frame/between expr following and current row zero special case": [ + (Fail, "known bug") + ], + "tests/:/frame clause/range frame/between expr following and expr preceding with order by zero special case": [ + (Fail, "known bug") + ], + "tests/:/funcs/lag/anyOrNull with column value as offset": [ + (Fail, "column values are not supported as offset") + ], + "tests/:/funcs/lead/subquery as offset": [ + (Fail, "subquery is not supported as offset") + ], + "tests/:/frame clause/range frame/between current row and unbounded following modifying named window": [ + (Fail, "range with named window is not supported") + ], + "tests/:/frame clause/range overflow/negative overflow with Int16": [ + (Fail, "exception on conversion") + ], + "tests/:/frame clause/range overflow/positive overflow with Int16": [ + (Fail, "exception on conversion") + ], + "tests/:/misc/subquery expr preceding": [ + (Fail, "subquery is not supported as offset") + ], + "tests/:/frame clause/range errors/error negative preceding offset": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22442") + ], + "tests/:/frame clause/range errors/error negative following offset": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/22442") + ], + "tests/:/misc/window functions in select expression": [ + (Fail, "not supported, https://github.com/ClickHouse/ClickHouse/issues/19857") + ], + "tests/:/misc/window functions in subquery": [ + (Fail, "not supported, https://github.com/ClickHouse/ClickHouse/issues/19857") + ], + "tests/:/misc/in view": [ + (Fail, "bug, https://github.com/ClickHouse/ClickHouse/issues/26001") + ], + "tests/:/frame clause/range frame/order by decimal": [ + ( + Fail, + "Exception: The RANGE OFFSET frame for 'DB::ColumnDecimal >' ORDER BY column is not implemented", + ) + ], + "tests/:/frame clause/range frame/with nulls": [ + ( + Fail, + "DB::Exception: The RANGE OFFSET frame for 'DB::ColumnNullable' ORDER BY column is not implemented", + ) + ], + "tests/:/aggregate funcs/aggregate funcs over rows frame/func='mannWhitneyUTest(salary, 1)'": [ + (Fail, "need to investigate") + ], + "tests/:/aggregate funcs/aggregate funcs over rows frame/func='rankCorr(salary, 0.5)'": [ + (Fail, "need to investigate") + ], + "tests/distributed/misc/query with order by and one window": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/distributed/over clause/empty named window": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/distributed/over clause/empty": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/distributed/over clause/adhoc window": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/distributed/frame clause/range datetime/:": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/distributed/frame clause/range frame/between expr preceding and expr following with partition by same column twice": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/:/funcs/leadInFrame/explicit default value": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057") + ], + "tests/:/funcs/leadInFrame/with nulls": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057") + ], + "tests/:/funcs/leadInFrame/default offset": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], + "tests/:/funcs/lagInFrame/explicit default value": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057") + ], + "tests/:/funcs/lagInFrame/with nulls": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/25057") + ], + "tests/:/funcs/lagInFrame/default offset": [ + (Fail, "https://github.com/ClickHouse/ClickHouse/issues/23902") + ], } -xflags = { -} +xflags = {} + @TestModule @ArgumentParser(argparser) @XFails(xfails) @XFlags(xflags) @Name("window functions") -@Specifications( - SRS019_ClickHouse_Window_Functions -) -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions("1.0") -) -def regression(self, local, clickhouse_binary_path, stress=None): - """Window functions regression. - """ - nodes = { - "clickhouse": - ("clickhouse1", "clickhouse2", "clickhouse3") - } +@Specifications(SRS019_ClickHouse_Window_Functions) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions("1.0")) +def regression( + self, local, clickhouse_binary_path, clickhouse_version=None, stress=None +): + """Window functions regression.""" + nodes = {"clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3")} if stress is not None: self.context.stress = stress + self.context.clickhouse_version = clickhouse_version - with Cluster(local, clickhouse_binary_path, nodes=nodes, - docker_compose_project_dir=os.path.join(current_dir(), "window_functions_env")) as cluster: + with Cluster( + local, + clickhouse_binary_path, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), "window_functions_env"), + ) as cluster: self.context.cluster = cluster Feature(run=load("window_functions.tests.feature", "feature"), flags=TE) + if main(): regression() diff --git a/tests/testflows/window_functions/requirements/requirements.py b/tests/testflows/window_functions/requirements/requirements.py index e453b1728e1..3e9a8a46719 100644 --- a/tests/testflows/window_functions/requirements/requirements.py +++ b/tests/testflows/window_functions/requirements/requirements.py @@ -9,3334 +9,3467 @@ from testflows.core import Requirement Heading = Specification.Heading RQ_SRS_019_ClickHouse_WindowFunctions = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [window functions] that produce a result for each row inside the window.\n' - '\n' - ), + "[ClickHouse] SHALL support [window functions] that produce a result for each row inside the window.\n" + "\n" + ), link=None, level=3, - num='3.1.1') + num="3.1.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_NonDistributedTables = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.NonDistributedTables', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.NonDistributedTables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of [window functions] on non-distributed \n' - 'table engines such as `MergeTree`.\n' - '\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of [window functions] on non-distributed \n" + "table engines such as `MergeTree`.\n" + "\n" + "\n" + ), link=None, level=3, - num='3.1.2') + num="3.1.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_DistributedTables = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.DistributedTables', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.DistributedTables", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support correct operation of [window functions] on\n' - '[Distributed](https://clickhouse.com/docs/en/engines/table-engines/special/distributed/) table engine.\n' - '\n' - ), + "[ClickHouse] SHALL support correct operation of [window functions] on\n" + "[Distributed](https://clickhouse.com/docs/en/engines/table-engines/special/distributed/) table engine.\n" + "\n" + ), link=None, level=3, - num='3.1.3') + num="3.1.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_WindowSpec = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowSpec', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowSpec", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support defining a window using window specification clause.\n' - 'The [window_spec] SHALL be defined as\n' - '\n' - '```\n' - 'window_spec:\n' - ' [partition_clause] [order_clause] [frame_clause]\n' - '```\n' - '\n' - 'that SHALL specify how to partition query rows into groups for processing by the window function.\n' - '\n' - ), + "[ClickHouse] SHALL support defining a window using window specification clause.\n" + "The [window_spec] SHALL be defined as\n" + "\n" + "```\n" + "window_spec:\n" + " [partition_clause] [order_clause] [frame_clause]\n" + "```\n" + "\n" + "that SHALL specify how to partition query rows into groups for processing by the window function.\n" + "\n" + ), link=None, level=3, - num='3.2.1') + num="3.2.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [partition_clause] that indicates how to divide the query rows into groups.\n' - 'The [partition_clause] SHALL be defined as\n' - '\n' - '```\n' - 'partition_clause:\n' - ' PARTITION BY expr [, expr] ...\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support [partition_clause] that indicates how to divide the query rows into groups.\n" + "The [partition_clause] SHALL be defined as\n" + "\n" + "```\n" + "partition_clause:\n" + " PARTITION BY expr [, expr] ...\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.3.1') + num="3.3.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_MultipleExpr = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MultipleExpr', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MultipleExpr", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support partitioning by more than one `expr` in the [partition_clause] definition.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL support partitioning by more than one `expr` in the [partition_clause] definition.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT x,s, sum(x) OVER (PARTITION BY x,s) FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))\n" - '```\n' - '\n' - '```bash\n' - '┌─x─┬─s─┬─sum(x) OVER (PARTITION BY x, s)─┐\n' - '│ 1 │ a │ 1 │\n' - '│ 1 │ b │ 1 │\n' - '│ 2 │ b │ 2 │\n' - '└───┴───┴─────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─x─┬─s─┬─sum(x) OVER (PARTITION BY x, s)─┐\n" + "│ 1 │ a │ 1 │\n" + "│ 1 │ b │ 1 │\n" + "│ 2 │ b │ 2 │\n" + "└───┴───┴─────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.3.2') + num="3.3.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_MissingExpr_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MissingExpr.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MissingExpr.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if `expr` is missing in the [partition_clause] definition.\n' - '\n' - '```sql\n' - 'SELECT sum(number) OVER (PARTITION BY) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error if `expr` is missing in the [partition_clause] definition.\n" + "\n" + "```sql\n" + "SELECT sum(number) OVER (PARTITION BY) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.3.3') + num="3.3.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_InvalidExpr_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.InvalidExpr.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.InvalidExpr.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if `expr` is invalid in the [partition_clause] definition.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if `expr` is invalid in the [partition_clause] definition.\n" + "\n" + ), link=None, level=3, - num='3.3.4') + num="3.3.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [order_clause] that indicates how to sort rows in each window.\n' - '\n' - ), + "[ClickHouse] SHALL support [order_clause] that indicates how to sort rows in each window.\n" + "\n" + ), link=None, level=3, - num='3.4.1') + num="3.4.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MultipleExprs', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MultipleExprs", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return support using more than one `expr` in the [order_clause] definition.\n' - '\n' - 'For example, \n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return support using more than one `expr` in the [order_clause] definition.\n" + "\n" + "For example, \n" + "\n" + "```sql\n" "SELECT x,s, sum(x) OVER (ORDER BY x DESC, s DESC) FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))\n" - '```\n' - '\n' - '```bash\n' - '┌─x─┬─s─┬─sum(x) OVER (ORDER BY x DESC, s DESC)─┐\n' - '│ 2 │ b │ 2 │\n' - '│ 1 │ b │ 3 │\n' - '│ 1 │ a │ 4 │\n' - '└───┴───┴───────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─x─┬─s─┬─sum(x) OVER (ORDER BY x DESC, s DESC)─┐\n" + "│ 2 │ b │ 2 │\n" + "│ 1 │ b │ 3 │\n" + "│ 1 │ a │ 4 │\n" + "└───┴───┴───────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.4.2') + num="3.4.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MissingExpr_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MissingExpr.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MissingExpr.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if `expr` is missing in the [order_clause] definition.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if `expr` is missing in the [order_clause] definition.\n" + "\n" + ), link=None, level=3, - num='3.4.3') + num="3.4.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_InvalidExpr_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.InvalidExpr.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.InvalidExpr.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if `expr` is invalid in the [order_clause] definition.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if `expr` is invalid in the [order_clause] definition.\n" + "\n" + ), link=None, level=3, - num='3.4.4') + num="3.4.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_FrameClause = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [frame_clause] that SHALL specify a subset of the current window.\n' - '\n' - 'The `frame_clause` SHALL be defined as\n' - '\n' - '```\n' - 'frame_clause:\n' - ' {ROWS | RANGE } frame_extent\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support [frame_clause] that SHALL specify a subset of the current window.\n" + "\n" + "The `frame_clause` SHALL be defined as\n" + "\n" + "```\n" + "frame_clause:\n" + " {ROWS | RANGE } frame_extent\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.5.1') + num="3.5.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_FrameClause_DefaultFrame = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause.DefaultFrame', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause.DefaultFrame", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the default `frame_clause` to be `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`. \n' - '\n' - 'If the `ORDER BY` clause is specified then this SHALL set the frame to be all rows from \n' - 'the partition start up to and including current row and its peers. \n' - '\n' - 'If the `ORDER BY` clause is not specified then this SHALL set the frame to include all rows\n' - 'in the partition because all the rows are considered to be the peers of the current row.\n' - '\n' - ), + "[ClickHouse] SHALL support the default `frame_clause` to be `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW`. \n" + "\n" + "If the `ORDER BY` clause is specified then this SHALL set the frame to be all rows from \n" + "the partition start up to and including current row and its peers. \n" + "\n" + "If the `ORDER BY` clause is not specified then this SHALL set the frame to include all rows\n" + "in the partition because all the rows are considered to be the peers of the current row.\n" + "\n" + ), link=None, level=3, - num='3.5.2') + num="3.5.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `ROWS` frame to define beginning and ending row positions.\n' - 'Offsets SHALL be differences in row numbers from the current row number.\n' - '\n' - '```sql\n' - 'ROWS frame_extent\n' - '```\n' - '\n' - 'See [frame_extent] definition.\n' - '\n' - ), + "[ClickHouse] SHALL support `ROWS` frame to define beginning and ending row positions.\n" + "Offsets SHALL be differences in row numbers from the current row number.\n" + "\n" + "```sql\n" + "ROWS frame_extent\n" + "```\n" + "\n" + "See [frame_extent] definition.\n" + "\n" + ), link=None, level=4, - num='3.5.3.1') + num="3.5.3.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_MissingFrameExtent_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.MissingFrameExtent.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.MissingFrameExtent.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `ROWS` frame clause is defined without [frame_extent].\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number ROWS) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `ROWS` frame clause is defined without [frame_extent].\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number ROWS) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.3.2') + num="3.5.3.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_InvalidFrameExtent_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.InvalidFrameExtent.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.InvalidFrameExtent.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `ROWS` frame clause has invalid [frame_extent].\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error if the `ROWS` frame clause has invalid [frame_extent].\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number ROWS '1') FROM numbers(1,3)\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=4, - num='3.5.3.3') + num="3.5.3.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include only the current row in the window partition\n' - 'when `ROWS CURRENT ROW` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS CURRENT ROW) FROM numbers(1,2)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 2 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include only the current row in the window partition\n" + "when `ROWS CURRENT ROW` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS CURRENT ROW) FROM numbers(1,2)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 2 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.4.1') + num="3.5.3.4.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_UnboundedPreceding = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedPreceding', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedPreceding", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows before and including the current row in the window partition\n' - 'when `ROWS UNBOUNDED PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 6 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all rows before and including the current row in the window partition\n" + "when `ROWS UNBOUNDED PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 6 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.5.1') + num="3.5.3.5.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_ExprPreceding = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprPreceding', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprPreceding", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include `expr` rows before and including the current row in the window partition \n' - 'when `ROWS expr PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS 1 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 5 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include `expr` rows before and including the current row in the window partition \n" + "when `ROWS expr PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS 1 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 5 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.6.1') + num="3.5.3.6.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_UnboundedFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `ROWS UNBOUNDED FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `ROWS UNBOUNDED FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.7.1') + num="3.5.3.7.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_ExprFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `ROWS expr FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `ROWS expr FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.8.1') + num="3.5.3.8.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include only the current row in the window partition\n' - 'when `ROWS BETWEEN CURRENT ROW AND CURRENT ROW` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN CURRENT ROW AND CURRENT ROW) FROM numbers(1,2)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 2 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include only the current row in the window partition\n" + "when `ROWS BETWEEN CURRENT ROW AND CURRENT ROW` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN CURRENT ROW AND CURRENT ROW) FROM numbers(1,2)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 2 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.9.1') + num="3.5.3.9.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `ROWS BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING` frame is specified.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `ROWS BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING` frame is specified.\n" + "\n" + ), link=None, level=5, - num='3.5.3.9.2') + num="3.5.3.9.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_ExprPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `ROWS BETWEEN CURRENT ROW AND expr PRECEDING` frame is specified.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `ROWS BETWEEN CURRENT ROW AND expr PRECEDING` frame is specified.\n" + "\n" + ), link=None, level=5, - num='3.5.3.9.3') + num="3.5.3.9.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the current row and all the following rows in the window partition\n' - 'when `ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 5 │\n' - '│ 3 │ 3 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the current row and all the following rows in the window partition\n" + "when `ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 5 │\n" + "│ 3 │ 3 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.9.4') + num="3.5.3.9.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_ExprFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the current row and the `expr` rows that are following the current row in the window partition\n' - 'when `ROWS BETWEEN CURRENT ROW AND expr FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING)─┐\n' - '│ 1 │ 3 │\n' - '│ 2 │ 5 │\n' - '│ 3 │ 3 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the current row and the `expr` rows that are following the current row in the window partition\n" + "when `ROWS BETWEEN CURRENT ROW AND expr FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING)─┐\n" + "│ 1 │ 3 │\n" + "│ 2 │ 5 │\n" + "│ 3 │ 3 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.9.5') + num="3.5.3.9.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all the rows before and including the current row in the window partition\n' - 'when `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 6 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all the rows before and including the current row in the window partition\n" + "when `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 6 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.10.1') + num="3.5.3.10.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.10.2') + num="3.5.3.10.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_ExprPreceding = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprPreceding', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprPreceding", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all the rows until and including the current row minus `expr` rows preceding it\n' - 'when `ROWS BETWEEN UNBOUNDED PRECEDING AND expr PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)─┐\n' - '│ 1 │ 0 │\n' - '│ 2 │ 1 │\n' - '│ 3 │ 3 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all the rows until and including the current row minus `expr` rows preceding it\n" + "when `ROWS BETWEEN UNBOUNDED PRECEDING AND expr PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)─┐\n" + "│ 1 │ 0 │\n" + "│ 2 │ 1 │\n" + "│ 3 │ 3 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.10.3') + num="3.5.3.10.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows in the window partition \n' - 'when `ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 6 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all rows in the window partition \n" + "when `ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 6 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.10.4') + num="3.5.3.10.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_ExprFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all the rows until and including the current row plus `expr` rows following it\n' - 'when `ROWS BETWEEN UNBOUNDED PRECEDING AND expr FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)─┐\n' - '│ 1 │ 3 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 6 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all the rows until and including the current row plus `expr` rows following it\n" + "when `ROWS BETWEEN UNBOUNDED PRECEDING AND expr FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)─┐\n" + "│ 1 │ 3 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 6 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.10.5') + num="3.5.3.10.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `UNBOUNDED FOLLOWING` is specified as the start of the frame, including\n' - '\n' - '* `ROWS BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW`\n' - '* `ROWS BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING`\n' - '* `ROWS BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING`\n' - '* `ROWS BETWEEN UNBOUNDED FOLLOWING AND expr PRECEDING`\n' - '* `ROWS BETWEEN UNBOUNDED FOLLOWING AND expr FOLLOWING`\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `UNBOUNDED FOLLOWING` is specified as the start of the frame, including\n" + "\n" + "* `ROWS BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW`\n" + "* `ROWS BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING`\n" + "* `ROWS BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING`\n" + "* `ROWS BETWEEN UNBOUNDED FOLLOWING AND expr PRECEDING`\n" + "* `ROWS BETWEEN UNBOUNDED FOLLOWING AND expr FOLLOWING`\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.11.1') + num="3.5.3.11.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `expr FOLLOWING` is specified as the start of the frame\n' - 'and it points to a row that is after the start of the frame inside the window partition such\n' - 'as the following cases\n' - '\n' - '* `ROWS BETWEEN expr FOLLOWING AND CURRENT ROW`\n' - '* `ROWS BETWEEN expr FOLLOWING AND UNBOUNDED PRECEDING`\n' - '* `ROWS BETWEEN expr FOLLOWING AND expr PRECEDING`\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `expr FOLLOWING` is specified as the start of the frame\n" + "and it points to a row that is after the start of the frame inside the window partition such\n" + "as the following cases\n" + "\n" + "* `ROWS BETWEEN expr FOLLOWING AND CURRENT ROW`\n" + "* `ROWS BETWEEN expr FOLLOWING AND UNBOUNDED PRECEDING`\n" + "* `ROWS BETWEEN expr FOLLOWING AND expr PRECEDING`\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.12.1') + num="3.5.3.12.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `ROWS BETWEEN expr FOLLOWING AND expr FOLLOWING`\n' - 'is specified and the end of the frame specified by the `expr FOLLOWING` is a row that is before the row \n' - 'specified by the frame start.\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `ROWS BETWEEN expr FOLLOWING AND expr FOLLOWING`\n" + "is specified and the end of the frame specified by the `expr FOLLOWING` is a row that is before the row \n" + "specified by the frame start.\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.12.2') + num="3.5.3.12.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all the rows from and including current row plus `expr` rows following it \n' - 'until and including the last row in the window partition\n' - 'when `ROWS BETWEEN expr FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 5 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 0 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all the rows from and including current row plus `expr` rows following it \n" + "until and including the last row in the window partition\n" + "when `ROWS BETWEEN expr FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 5 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 0 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.12.3') + num="3.5.3.12.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the rows from and including current row plus `expr` following it \n' - 'until and including the row specified by the frame end when the frame end \n' - 'is the current row plus `expr` following it is right at or after the start of the frame\n' - 'when `ROWS BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)─┐\n' - '│ 1 │ 5 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 0 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the rows from and including current row plus `expr` following it \n" + "until and including the row specified by the frame end when the frame end \n" + "is the current row plus `expr` following it is right at or after the start of the frame\n" + "when `ROWS BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING)─┐\n" + "│ 1 │ 5 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 0 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.12.4') + num="3.5.3.12.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the rows from and including current row minus `expr` rows\n' - 'preceding it until and including the current row in the window frame\n' - 'when `ROWS BETWEEN expr PRECEDING AND CURRENT ROW` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 5 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the rows from and including current row minus `expr` rows\n" + "preceding it until and including the current row in the window frame\n" + "when `ROWS BETWEEN expr PRECEDING AND CURRENT ROW` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 5 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.13.1') + num="3.5.3.13.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error\n' - 'when `ROWS BETWEEN expr PRECEDING AND UNBOUNDED PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error\n" + "when `ROWS BETWEEN expr PRECEDING AND UNBOUNDED PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.13.2') + num="3.5.3.13.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the rows from and including current row minus `expr` rows\n' - 'preceding it until and including the last row in the window partition\n' - 'when `ROWS BETWEEN expr PRECEDING AND UNBOUNDED FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 5 │\n' - '└────────┴─────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the rows from and including current row minus `expr` rows\n" + "preceding it until and including the last row in the window partition\n" + "when `ROWS BETWEEN expr PRECEDING AND UNBOUNDED FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 5 │\n" + "└────────┴─────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.13.3') + num="3.5.3.13.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the frame end specified by the `expr PRECEDING`\n' - 'evaluates to a row that is before the row specified by the frame start in the window partition\n' - 'when `ROWS BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when the frame end specified by the `expr PRECEDING`\n" + "evaluates to a row that is before the row specified by the frame start in the window partition\n" + "when `ROWS BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.13.4') + num="3.5.3.13.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the rows from and including current row minus `expr` rows preceding it\n' - 'until and including the current row minus `expr` rows preceding it if the end\n' - 'of the frame is after the frame start in the window partition \n' - 'when `ROWS BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 0 PRECEDING)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 5 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the rows from and including current row minus `expr` rows preceding it\n" + "until and including the current row minus `expr` rows preceding it if the end\n" + "of the frame is after the frame start in the window partition \n" + "when `ROWS BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 0 PRECEDING)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 5 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.13.5') + num="3.5.3.13.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include the rows from and including current row minus `expr` rows preceding it\n' - 'until and including the current row plus `expr` rows following it in the window partition\n' - 'when `ROWS BETWEEN expr PRECEDING AND expr FOLLOWING` frame is specified.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)─┐\n' - '│ 1 │ 3 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 5 │\n' - '└────────┴─────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include the rows from and including current row minus `expr` rows preceding it\n" + "until and including the current row plus `expr` rows following it in the window partition\n" + "when `ROWS BETWEEN expr PRECEDING AND expr FOLLOWING` frame is specified.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)─┐\n" + "│ 1 │ 3 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 5 │\n" + "└────────┴─────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.3.13.6') + num="3.5.3.13.6", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `RANGE` frame to define rows within a value range.\n' - 'Offsets SHALL be differences in row values from the current row value.\n' - '\n' - '```sql\n' - 'RANGE frame_extent\n' - '```\n' - '\n' - 'See [frame_extent] definition.\n' - '\n' - ), + "[ClickHouse] SHALL support `RANGE` frame to define rows within a value range.\n" + "Offsets SHALL be differences in row values from the current row value.\n" + "\n" + "```sql\n" + "RANGE frame_extent\n" + "```\n" + "\n" + "See [frame_extent] definition.\n" + "\n" + ), link=None, level=4, - num='3.5.4.1') + num="3.5.4.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_DataTypes_DateAndDateTime = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.DateAndDateTime', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.DateAndDateTime", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `RANGE` frame over columns with `Date` and `DateTime`\n' - 'data types.\n' - '\n' - ), + "[ClickHouse] SHALL support `RANGE` frame over columns with `Date` and `DateTime`\n" + "data types.\n" + "\n" + ), link=None, level=4, - num='3.5.4.2') + num="3.5.4.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_DataTypes_IntAndUInt = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.IntAndUInt', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.IntAndUInt", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `RANGE` frame over columns with numerical data types\n' - 'such `IntX` and `UIntX`.\n' - '\n' - ), + "[ClickHouse] SHALL support `RANGE` frame over columns with numerical data types\n" + "such `IntX` and `UIntX`.\n" + "\n" + ), link=None, level=4, - num='3.5.4.3') + num="3.5.4.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_MultipleColumnsInOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MultipleColumnsInOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MultipleColumnsInOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `RANGE` frame definition is used with `ORDER BY`\n' - 'that uses multiple columns.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `RANGE` frame definition is used with `ORDER BY`\n" + "that uses multiple columns.\n" + "\n" + ), link=None, level=4, - num='3.5.4.4') + num="3.5.4.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_MissingFrameExtent_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MissingFrameExtent.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MissingFrameExtent.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `RANGE` frame definition is missing [frame_extent].\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `RANGE` frame definition is missing [frame_extent].\n" + "\n" + ), link=None, level=4, - num='3.5.4.5') + num="3.5.4.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_InvalidFrameExtent_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.InvalidFrameExtent.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.InvalidFrameExtent.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `RANGE` frame definition has invalid [frame_extent].\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `RANGE` frame definition has invalid [frame_extent].\n" + "\n" + ), link=None, level=4, - num='3.5.4.6') + num="3.5.4.6", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_CurrentRow_Peers = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.CurrentRow.Peers', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.CurrentRow.Peers", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] for the `RANGE` frame SHALL define the `peers` of the `CURRENT ROW` to be all\n' - 'the rows that are inside the same order bucket.\n' - '\n' - ), + "[ClickHouse] for the `RANGE` frame SHALL define the `peers` of the `CURRENT ROW` to be all\n" + "the rows that are inside the same order bucket.\n" + "\n" + ), link=None, level=4, - num='3.5.4.8') + num="3.5.4.8", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_CurrentRow_WithoutOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithoutOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithoutOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows in the window partition\n' - 'when `RANGE CURRENT ROW` frame is specified without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 6 │\n' - '└────────┴──────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all rows in the window partition\n" + "when `RANGE CURRENT ROW` frame is specified without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 6 │\n" + "└────────┴──────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.9.1') + num="3.5.4.9.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_CurrentRow_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows that are [current row peers] in the window partition\n' - 'when `RANGE CURRENT ROW` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows that are [current row peers] in the window partition\n" + "when `RANGE CURRENT ROW` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n' - '│ 1 │ 2 │\n' - '│ 1 │ 2 │\n' - '│ 2 │ 2 │\n' - '│ 3 │ 3 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n" + "│ 1 │ 2 │\n" + "│ 1 │ 2 │\n" + "│ 2 │ 2 │\n" + "│ 3 │ 3 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.9.2') + num="3.5.4.9.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE UNBOUNDED FOLLOWING` frame is specified with or without order by\n' - 'as `UNBOUNDED FOLLOWING` SHALL not be supported as [frame_start].\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE UNBOUNDED FOLLOWING` frame is specified with or without order by\n" + "as `UNBOUNDED FOLLOWING` SHALL not be supported as [frame_start].\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.10.1') + num="3.5.4.10.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedPreceding_WithoutOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithoutOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithoutOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows in the window partition\n' - 'when `RANGE UNBOUNDED PRECEDING` frame is specified without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 6 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all rows in the window partition\n" + "when `RANGE UNBOUNDED PRECEDING` frame is specified without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 6 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.11.1') + num="3.5.4.11.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedPreceding_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include rows with values from and including the first row \n' - 'until and including all [current row peers] in the window partition\n' - 'when `RANGE UNBOUNDED PRECEDING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 6 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include rows with values from and including the first row \n" + "until and including all [current row peers] in the window partition\n" + "when `RANGE UNBOUNDED PRECEDING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 6 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.11.2') + num="3.5.4.11.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE expr PRECEDING` frame is specified without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE 1 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE expr PRECEDING` frame is specified without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE 1 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.12.1') + num="3.5.4.12.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_OrderByNonNumericalColumn_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.OrderByNonNumericalColumn.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.OrderByNonNumericalColumn.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE expr PRECEDING` is used with `ORDER BY` clause\n' - 'over a non-numerical column.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE expr PRECEDING` is used with `ORDER BY` clause\n" + "over a non-numerical column.\n" + "\n" + ), link=None, level=5, - num='3.5.4.12.2') + num="3.5.4.12.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include rows with values from and including current row value minus `expr`\n' - 'until and including the value for the current row \n' - 'when `RANGE expr PRECEDING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE 1 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 5 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include rows with values from and including current row value minus `expr`\n" + "until and including the value for the current row \n" + "when `RANGE expr PRECEDING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE 1 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 5 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.12.3') + num="3.5.4.12.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE expr FOLLOWING` frame is specified without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE expr FOLLOWING` frame is specified without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.13.1') + num="3.5.4.13.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprFollowing_WithOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE expr FOLLOWING` frame is specified wit the `ORDER BY` clause \n' - 'as the value for the frame start cannot be larger than the value for the frame end.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE expr FOLLOWING` frame is specified wit the `ORDER BY` clause \n" + "as the value for the frame start cannot be larger than the value for the frame end.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.13.2') + num="3.5.4.13.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all [current row peers] in the window partition \n' - 'when `RANGE BETWEEN CURRENT ROW AND CURRENT ROW` frame is specified with or without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`** \n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 6 │\n' - '└────────┴──────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - '**With `ORDER BY`** \n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND CURRENT ROW) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n' - '│ 1 │ 1 │\n' - '│ 2 │ 2 │\n' - '│ 3 │ 3 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all [current row peers] in the window partition \n" + "when `RANGE BETWEEN CURRENT ROW AND CURRENT ROW` frame is specified with or without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`** \n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 6 │\n" + "└────────┴──────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + "**With `ORDER BY`** \n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND CURRENT ROW) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND CURRENT ROW)─┐\n" + "│ 1 │ 1 │\n" + "│ 2 │ 2 │\n" + "│ 3 │ 3 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.14.1') + num="3.5.4.14.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING` frame is specified\n' - 'with or without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING` frame is specified\n" + "with or without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.14.2') + num="3.5.4.14.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including [current row peers] until and including\n' - 'the last row in the window partition when `RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING` frame is specified\n' - 'with or without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 6 │\n' - '│ 3 │ 6 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 6 │\n' - '│ 2 │ 5 │\n' - '│ 3 │ 3 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL include all rows with values from and including [current row peers] until and including\n" + "the last row in the window partition when `RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING` frame is specified\n" + "with or without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 6 │\n" + "│ 3 │ 6 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 6 │\n" + "│ 2 │ 5 │\n" + "│ 3 │ 3 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.14.3') + num="3.5.4.14.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN CURRENT ROW AND expr FOLLOWING` frame is specified\n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN CURRENT ROW AND expr FOLLOWING` frame is specified\n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.14.4') + num="3.5.4.14.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprFollowing_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including [current row peers] until and including\n' - 'current row value plus `expr` when `RANGE BETWEEN CURRENT ROW AND expr FOLLOWING` frame is specified\n' - 'with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including [current row peers] until and including\n" + "current row value plus `expr` when `RANGE BETWEEN CURRENT ROW AND expr FOLLOWING` frame is specified\n" + "with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)─┐\n' - '│ 1 │ 4 │\n' - '│ 1 │ 4 │\n' - '│ 2 │ 5 │\n' - '│ 3 │ 3 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)─┐\n" + "│ 1 │ 4 │\n" + "│ 1 │ 4 │\n" + "│ 2 │ 5 │\n" + "│ 3 │ 3 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.14.5') + num="3.5.4.14.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN CURRENT ROW AND expr PRECEDING` frame is specified\n' - 'with or without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN CURRENT ROW AND expr PRECEDING` frame is specified\n" + "with or without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.14.6') + num="3.5.4.14.6", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including the first row until and including\n' - '[current row peers] in the window partition when `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` frame is specified\n' - 'with and without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including the first row until and including\n" + "[current row peers] in the window partition when `RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` frame is specified\n" + "with and without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 7 │\n' - '│ 1 │ 7 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 7 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 7 │\n" + "│ 1 │ 7 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 7 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 2 │\n' - '│ 1 │ 2 │\n' - '│ 2 │ 4 │\n' - '│ 3 │ 7 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 2 │\n" + "│ 1 │ 2 │\n" + "│ 2 │ 4 │\n" + "│ 3 │ 7 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.1') + num="3.5.4.15.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return and error when `RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING` frame is specified\n' - 'with and without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return and error when `RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING` frame is specified\n" + "with and without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.2') + num="3.5.4.15.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows in the window partition when `RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame is specified\n' - 'with and without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows in the window partition when `RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING` frame is specified\n" + "with and without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 7 │\n' - '│ 1 │ 7 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 7 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 7 │\n" + "│ 1 │ 7 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 7 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 7 │\n' - '│ 1 │ 7 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 7 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 7 │\n" + "│ 1 │ 7 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 7 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.3') + num="3.5.4.15.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprPreceding_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr PRECEDING` frame is specified\n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr PRECEDING` frame is specified\n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.4') + num="3.5.4.15.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprPreceding_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including the first row until and including\n' - 'the value of the current row minus `expr` in the window partition\n' - 'when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr PRECEDING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including the first row until and including\n" + "the value of the current row minus `expr` in the window partition\n" + "when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr PRECEDING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)─┐\n' - '│ 1 │ 0 │\n' - '│ 1 │ 0 │\n' - '│ 2 │ 2 │\n' - '│ 3 │ 4 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)─┐\n" + "│ 1 │ 0 │\n" + "│ 1 │ 0 │\n" + "│ 2 │ 2 │\n" + "│ 3 │ 4 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.5') + num="3.5.4.15.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr FOLLOWING` frame is specified\n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr FOLLOWING` frame is specified\n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.6') + num="3.5.4.15.6", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprFollowing_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including the first row until and including\n' - 'the value of the current row plus `expr` in the window partition\n' - 'when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr FOLLOWING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including the first row until and including\n" + "the value of the current row plus `expr` in the window partition\n" + "when `RANGE BETWEEN UNBOUNDED PRECEDING AND expr FOLLOWING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)─┐\n' - '│ 1 │ 4 │\n' - '│ 1 │ 4 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 7 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)─┐\n" + "│ 1 │ 4 │\n" + "│ 1 │ 4 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 7 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.15.7') + num="3.5.4.15.7", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_CurrentRow_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.CurrentRow.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.CurrentRow.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + ), link=None, level=5, - num='3.5.4.16.1') + num="3.5.4.16.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_UnboundedFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + ), link=None, level=5, - num='3.5.4.16.2') + num="3.5.4.16.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + ), link=None, level=5, - num='3.5.4.16.3') + num="3.5.4.16.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_ExprPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND expr PRECEDING` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND expr PRECEDING` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + ), link=None, level=5, - num='3.5.4.16.4') + num="3.5.4.16.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_ExprFollowing_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprFollowing.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprFollowing.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND expr FOLLOWING` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN UNBOUNDED FOLLOWING AND expr FOLLOWING` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + ), link=None, level=5, - num='3.5.4.16.5') + num="3.5.4.16.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_CurrentRow_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including current row minus `expr` \n' - 'until and including [current row peers] in the window partition\n' - 'when `RANGE BETWEEN expr PRECEDING AND CURRENT ROW` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including current row minus `expr` \n" + "until and including [current row peers] in the window partition\n" + "when `RANGE BETWEEN expr PRECEDING AND CURRENT ROW` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n' - '│ 1 │ 2 │\n' - '│ 1 │ 2 │\n' - '│ 2 │ 4 │\n' - '│ 3 │ 5 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND CURRENT ROW)─┐\n" + "│ 1 │ 2 │\n" + "│ 1 │ 2 │\n" + "│ 2 │ 4 │\n" + "│ 3 │ 5 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.1') + num="3.5.4.17.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_CurrentRow_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND CURRENT ROW` frame is specified\n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND CURRENT ROW` frame is specified\n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.2') + num="3.5.4.17.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND UNBOUNDED PRECEDING` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND UNBOUNDED PRECEDING` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.3') + num="3.5.4.17.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND UNBOUNDED FOLLOWING` frame is specified \n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND UNBOUNDED FOLLOWING` frame is specified \n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.4') + num="3.5.4.17.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedFollowing_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including current row minus `expr` \n' - 'until and including the last row in the window partition when `RANGE BETWEEN expr PRECEDING AND UNBOUNDED FOLLOWING` frame\n' - 'is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including current row minus `expr` \n" + "until and including the last row in the window partition when `RANGE BETWEEN expr PRECEDING AND UNBOUNDED FOLLOWING` frame\n" + "is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 7 │\n' - '│ 1 │ 7 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 5 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 7 │\n" + "│ 1 │ 7 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 5 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.5') + num="3.5.4.17.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND expr FOLLOWING` frame is specified \n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND expr FOLLOWING` frame is specified \n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.6') + num="3.5.4.17.6", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprFollowing_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including current row minus preceding `expr` \n' - 'until and including current row plus following `expr` in the window partition \n' - 'when `RANGE BETWEEN expr PRECEDING AND expr FOLLOWING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including current row minus preceding `expr` \n" + "until and including current row plus following `expr` in the window partition \n" + "when `RANGE BETWEEN expr PRECEDING AND expr FOLLOWING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)─┐\n' - '│ 1 │ 4 │\n' - '│ 1 │ 4 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 5 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)─┐\n" + "│ 1 │ 4 │\n" + "│ 1 │ 4 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 5 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.7') + num="3.5.4.17.7", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified \n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM numbers(1,3)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified \n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM numbers(1,3)\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.8') + num="3.5.4.17.8", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when the value of the [frame_end] specified by the \n' - 'current row minus preceding `expr` is greater than the value of the [frame_start] in the window partition\n' - 'when `RANGE BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when the value of the [frame_end] specified by the \n" + "current row minus preceding `expr` is greater than the value of the [frame_start] in the window partition\n" + "when `RANGE BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.9') + num="3.5.4.17.9", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including current row minus preceding `expr` for the [frame_start]\n' - 'until and including current row minus following `expr` for the [frame_end] in the window partition \n' - 'when `RANGE BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified with the `ORDER BY` clause\n' - 'if an only if the [frame_end] value is equal or greater than [frame_start] value.\n' - '\n' - 'For example,\n' - '\n' - '**Greater Than**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including current row minus preceding `expr` for the [frame_start]\n" + "until and including current row minus following `expr` for the [frame_end] in the window partition \n" + "when `RANGE BETWEEN expr PRECEDING AND expr PRECEDING` frame is specified with the `ORDER BY` clause\n" + "if an only if the [frame_end] value is equal or greater than [frame_start] value.\n" + "\n" + "For example,\n" + "\n" + "**Greater Than**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING)─┐\n' - '│ 1 │ 2 │\n' - '│ 1 │ 2 │\n' - '│ 2 │ 4 │\n' - '│ 3 │ 5 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - 'or **Equal**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING)─┐\n" + "│ 1 │ 2 │\n" + "│ 1 │ 2 │\n" + "│ 2 │ 4 │\n" + "│ 3 │ 5 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + "or **Equal**\n" + "\n" + "```sql\n" " SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 PRECEDING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND 1 PRECEDING)─┐\n' - '│ 1 │ 0 │\n' - '│ 1 │ 0 │\n' - '│ 2 │ 2 │\n' - '│ 3 │ 2 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 PRECEDING AND 1 PRECEDING)─┐\n" + "│ 1 │ 0 │\n" + "│ 1 │ 0 │\n" + "│ 2 │ 2 │\n" + "│ 3 │ 2 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.17.10') + num="3.5.4.17.10", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND CURRENT ROW` frame is specified \n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND CURRENT ROW` frame is specified \n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.1') + num="3.5.4.18.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_WithOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND CURRENT ROW` frame is specified \n' - 'with the `ORDER BY` clause and `expr` is greater than `0`.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND CURRENT ROW` frame is specified \n" + "with the `ORDER BY` clause and `expr` is greater than `0`.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.2') + num="3.5.4.18.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_ZeroSpecialCase = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.ZeroSpecialCase', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.ZeroSpecialCase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all [current row peers] in the window partition\n' - 'when `RANGE BETWEEN expr FOLLOWING AND CURRENT ROW` frame is specified \n' - 'with the `ORDER BY` clause if and only if the `expr` equals to `0`.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all [current row peers] in the window partition\n" + "when `RANGE BETWEEN expr FOLLOWING AND CURRENT ROW` frame is specified \n" + "with the `ORDER BY` clause if and only if the `expr` equals to `0`.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW)─┐\n' - '│ 1 │ 7 │\n' - '│ 1 │ 7 │\n' - '│ 2 │ 7 │\n' - '│ 3 │ 7 │\n' - '└────────┴──────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW)─┐\n" + "│ 1 │ 7 │\n" + "│ 1 │ 7 │\n" + "│ 2 │ 7 │\n" + "│ 3 │ 7 │\n" + "└────────┴──────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW)─┐\n' - '│ 1 │ 2 │\n' - '│ 1 │ 2 │\n' - '│ 2 │ 2 │\n' - '│ 3 │ 3 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW)─┐\n" + "│ 1 │ 2 │\n" + "│ 1 │ 2 │\n" + "│ 2 │ 2 │\n" + "│ 3 │ 3 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.3') + num="3.5.4.18.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified \n' - 'without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified \n" + "without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.4') + num="3.5.4.18.4", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedFollowing_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with values from and including current row plus `expr`\n' - 'until and including the last row in the window partition \n' - 'when `RANGE BETWEEN expr FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified with the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with values from and including current row plus `expr`\n" + "until and including the last row in the window partition \n" + "when `RANGE BETWEEN expr FOLLOWING AND UNBOUNDED FOLLOWING` frame is specified with the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)─┐\n' - '│ 1 │ 5 │\n' - '│ 1 │ 5 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 0 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING)─┐\n" + "│ 1 │ 5 │\n" + "│ 1 │ 5 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 0 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.5') + num="3.5.4.18.5", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND UNBOUNDED PRECEDING` frame is specified \n' - 'with or without the `ORDER BY` clause.\n' - '\n' - 'For example,\n' - '\n' - '**Without `ORDER BY`**\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND UNBOUNDED PRECEDING` frame is specified \n" + "with or without the `ORDER BY` clause.\n" + "\n" + "For example,\n" + "\n" + "**Without `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '**With `ORDER BY`**\n' - '\n' - '```sql\n' + "```\n" + "\n" + "**With `ORDER BY`**\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.6') + num="3.5.4.18.6", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr PRECEDING` frame is specified \n' - 'without the `ORDER BY`.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr PRECEDING` frame is specified \n" + "without the `ORDER BY`.\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.7') + num="3.5.4.18.7", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr PRECEDING` frame is specified \n' - 'with the `ORDER BY` clause if the value of both `expr` is not `0`.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr PRECEDING` frame is specified \n" + "with the `ORDER BY` clause if the value of both `expr` is not `0`.\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.8') + num="3.5.4.18.8", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_WithOrderBy_ZeroSpecialCase = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithOrderBy.ZeroSpecialCase', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithOrderBy.ZeroSpecialCase", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with value equal to [current row peers] in the window partition\n' - 'when `RANGE BETWEEN expr FOLLOWING AND expr PRECEDING` frame is specified \n' + "[ClickHouse] SHALL include all rows with value equal to [current row peers] in the window partition\n" + "when `RANGE BETWEEN expr FOLLOWING AND expr PRECEDING` frame is specified \n" "with the `ORDER BY` clause if and only if both `expr`'s are `0`.\n" - '\n' - 'For example,\n' - '\n' - '```sql\n' + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING ─┐\n' - '│ 1 │ 2 │\n' - '│ 1 │ 2 │\n' - '│ 2 │ 2 │\n' - '│ 3 │ 3 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING ─┐\n" + "│ 1 │ 2 │\n" + "│ 1 │ 2 │\n" + "│ 2 │ 2 │\n" + "│ 3 │ 3 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.9') + num="3.5.4.18.9", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithoutOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithoutOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithoutOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified \n' - 'without the `ORDER BY` clause.\n' - '\n' - ), + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified \n" + "without the `ORDER BY` clause.\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.10') + num="3.5.4.18.10", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithOrderBy_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified \n' - 'with the `ORDER BY` clause but the `expr` for the [frame_end] is less than the `expr` for the [frame_start].\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL return an error when `RANGE BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified \n" + "with the `ORDER BY` clause but the `expr` for the [frame_end] is less than the `expr` for the [frame_start].\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - ), + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.11') + num="3.5.4.18.11", +) RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithOrderBy = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL include all rows with value from and including current row plus `expr` for the [frame_start]\n' - 'until and including current row plus `expr` for the [frame_end] in the window partition\n' - 'when `RANGE BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified \n' - 'with the `ORDER BY` clause if and only if the `expr` for the [frame_end] is greater than or equal than the \n' - '`expr` for the [frame_start].\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' + "[ClickHouse] SHALL include all rows with value from and including current row plus `expr` for the [frame_start]\n" + "until and including current row plus `expr` for the [frame_end] in the window partition\n" + "when `RANGE BETWEEN expr FOLLOWING AND expr FOLLOWING` frame is specified \n" + "with the `ORDER BY` clause if and only if the `expr` for the [frame_end] is greater than or equal than the \n" + "`expr` for the [frame_start].\n" + "\n" + "For example,\n" + "\n" + "```sql\n" "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))\n" - '```\n' - '\n' - '```bash\n' - '┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING)─┐\n' - '│ 1 │ 5 │\n' - '│ 1 │ 5 │\n' - '│ 2 │ 3 │\n' - '│ 3 │ 0 │\n' - '└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n' - '```\n' - '\n' - ), + "```\n" + "\n" + "```bash\n" + "┌─number─┬─sum(number) OVER (ORDER BY number ASC RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING)─┐\n" + "│ 1 │ 5 │\n" + "│ 1 │ 5 │\n" + "│ 2 │ 3 │\n" + "│ 3 │ 0 │\n" + "└────────┴──────────────────────────────────────────────────────────────────────────────────┘\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.5.4.18.12') + num="3.5.4.18.12", +) RQ_SRS_019_ClickHouse_WindowFunctions_Frame_Extent = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Extent', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Extent", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [frame_extent] defined as\n' - '\n' - '```\n' - 'frame_extent:\n' - ' {frame_start | frame_between}\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support [frame_extent] defined as\n" + "\n" + "```\n" + "frame_extent:\n" + " {frame_start | frame_between}\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.5.1') + num="3.5.5.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_Frame_Start = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Start', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Start", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [frame_start] defined as\n' - '\n' - '```\n' - 'frame_start: {\n' - ' CURRENT ROW\n' - ' | UNBOUNDED PRECEDING\n' - ' | UNBOUNDED FOLLOWING\n' - ' | expr PRECEDING\n' - ' | expr FOLLOWING\n' - '}\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support [frame_start] defined as\n" + "\n" + "```\n" + "frame_start: {\n" + " CURRENT ROW\n" + " | UNBOUNDED PRECEDING\n" + " | UNBOUNDED FOLLOWING\n" + " | expr PRECEDING\n" + " | expr FOLLOWING\n" + "}\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.6.1') + num="3.5.6.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_Frame_Between = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Between', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Between", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [frame_between] defined as\n' - '\n' - '```\n' - 'frame_between:\n' - ' BETWEEN frame_start AND frame_end\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support [frame_between] defined as\n" + "\n" + "```\n" + "frame_between:\n" + " BETWEEN frame_start AND frame_end\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.7.1') + num="3.5.7.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_Frame_End = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.End', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.End", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support [frame_end] defined as\n' - '\n' - '```\n' - 'frame_end: {\n' - ' CURRENT ROW\n' - ' | UNBOUNDED PRECEDING\n' - ' | UNBOUNDED FOLLOWING\n' - ' | expr PRECEDING\n' - ' | expr FOLLOWING\n' - '}\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support [frame_end] defined as\n" + "\n" + "```\n" + "frame_end: {\n" + " CURRENT ROW\n" + " | UNBOUNDED PRECEDING\n" + " | UNBOUNDED FOLLOWING\n" + " | expr PRECEDING\n" + " | expr FOLLOWING\n" + "}\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.8.1') + num="3.5.8.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_CurrentRow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.CurrentRow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.CurrentRow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `CURRENT ROW` as `frame_start` or `frame_end` value.\n' - '\n' - '* For `ROWS` SHALL define the bound to be the current row\n' - '* For `RANGE` SHALL define the bound to be the peers of the current row\n' - '\n' - ), + "[ClickHouse] SHALL support `CURRENT ROW` as `frame_start` or `frame_end` value.\n" + "\n" + "* For `ROWS` SHALL define the bound to be the current row\n" + "* For `RANGE` SHALL define the bound to be the peers of the current row\n" + "\n" + ), link=None, level=4, - num='3.5.9.1') + num="3.5.9.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_UnboundedPreceding = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedPreceding', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedPreceding", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `UNBOUNDED PRECEDING` as `frame_start` or `frame_end` value\n' - 'and it SHALL define that the bound is the first partition row.\n' - '\n' - ), + "[ClickHouse] SHALL support `UNBOUNDED PRECEDING` as `frame_start` or `frame_end` value\n" + "and it SHALL define that the bound is the first partition row.\n" + "\n" + ), link=None, level=4, - num='3.5.10.1') + num="3.5.10.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_UnboundedFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `UNBOUNDED FOLLOWING` as `frame_start` or `frame_end` value\n' - 'and it SHALL define that the bound is the last partition row.\n' - '\n' - ), + "[ClickHouse] SHALL support `UNBOUNDED FOLLOWING` as `frame_start` or `frame_end` value\n" + "and it SHALL define that the bound is the last partition row.\n" + "\n" + ), link=None, level=4, - num='3.5.11.1') + num="3.5.11.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_ExprPreceding = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `expr PRECEDING` as `frame_start` or `frame_end` value\n' - '\n' - '* For `ROWS` it SHALL define the bound to be the `expr` rows before the current row\n' - '* For `RANGE` it SHALL define the bound to be the rows with values equal to the current row value minus the `expr`.\n' - '\n' - ), + "[ClickHouse] SHALL support `expr PRECEDING` as `frame_start` or `frame_end` value\n" + "\n" + "* For `ROWS` it SHALL define the bound to be the `expr` rows before the current row\n" + "* For `RANGE` it SHALL define the bound to be the rows with values equal to the current row value minus the `expr`.\n" + "\n" + ), link=None, level=4, - num='3.5.12.1') + num="3.5.12.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_ExprPreceding_ExprValue = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding.ExprValue', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding.ExprValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support only non-negative numeric literal as the value for the `expr` in the `expr PRECEDING` frame boundary.\n' - '\n' - 'For example,\n' - '\n' - '```\n' - '5 PRECEDING\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support only non-negative numeric literal as the value for the `expr` in the `expr PRECEDING` frame boundary.\n" + "\n" + "For example,\n" + "\n" + "```\n" + "5 PRECEDING\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.12.2') + num="3.5.12.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `expr FOLLOWING` as `frame_start` or `frame_end` value\n' - '\n' - '* For `ROWS` it SHALL define the bound to be the `expr` rows after the current row\n' - '* For `RANGE` it SHALL define the bound to be the rows with values equal to the current row value plus `expr`\n' - '\n' - ), + "[ClickHouse] SHALL support `expr FOLLOWING` as `frame_start` or `frame_end` value\n" + "\n" + "* For `ROWS` it SHALL define the bound to be the `expr` rows after the current row\n" + "* For `RANGE` it SHALL define the bound to be the rows with values equal to the current row value plus `expr`\n" + "\n" + ), link=None, level=4, - num='3.5.13.1') + num="3.5.13.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing_ExprValue = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing.ExprValue', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing.ExprValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support only non-negative numeric literal as the value for the `expr` in the `expr FOLLOWING` frame boundary.\n' - '\n' - 'For example,\n' - '\n' - '```\n' - '5 FOLLOWING\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support only non-negative numeric literal as the value for the `expr` in the `expr FOLLOWING` frame boundary.\n" + "\n" + "For example,\n" + "\n" + "```\n" + "5 FOLLOWING\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.5.13.2') + num="3.5.13.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `WINDOW` clause to define one or more windows.\n' - '\n' - '```sql\n' - 'WINDOW window_name AS (window_spec)\n' - ' [, window_name AS (window_spec)] ..\n' - '```\n' - '\n' - 'The `window_name` SHALL be the name of a window defined by a `WINDOW` clause.\n' - '\n' - 'The [window_spec] SHALL specify the window.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT ... FROM table WINDOW w AS (partiton by id))\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `WINDOW` clause to define one or more windows.\n" + "\n" + "```sql\n" + "WINDOW window_name AS (window_spec)\n" + " [, window_name AS (window_spec)] ..\n" + "```\n" + "\n" + "The `window_name` SHALL be the name of a window defined by a `WINDOW` clause.\n" + "\n" + "The [window_spec] SHALL specify the window.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT ... FROM table WINDOW w AS (partiton by id))\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.6.1') + num="3.6.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MultipleWindows = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MultipleWindows', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MultipleWindows", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `WINDOW` clause that defines multiple windows.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT ... FROM table WINDOW w1 AS (partition by id), w2 AS (partition by customer)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `WINDOW` clause that defines multiple windows.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT ... FROM table WINDOW w1 AS (partition by id), w2 AS (partition by customer)\n" + "```\n" + "\n" + ), link=None, level=3, - num='3.6.2') + num="3.6.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MissingWindowSpec_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MissingWindowSpec.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MissingWindowSpec.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `WINDOW` clause definition is missing [window_spec].\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `WINDOW` clause definition is missing [window_spec].\n" + "\n" + ), link=None, level=3, - num='3.6.3') + num="3.6.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `OVER` clause to either use named window defined using `WINDOW` clause\n' - 'or adhoc window defined inplace.\n' - '\n' - '\n' - '```\n' - 'OVER ()|(window_spec)|named_window \n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `OVER` clause to either use named window defined using `WINDOW` clause\n" + "or adhoc window defined inplace.\n" + "\n" + "\n" + "```\n" + "OVER ()|(window_spec)|named_window \n" + "```\n" + "\n" + ), link=None, level=3, - num='3.7.1') + num="3.7.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_EmptyOverClause = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.EmptyOverClause', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.EmptyOverClause", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL treat the entire set of query rows as a single partition when `OVER` clause is empty.\n' - 'For example,\n' - '\n' - '```\n' - 'SELECT sum(x) OVER () FROM table\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL treat the entire set of query rows as a single partition when `OVER` clause is empty.\n" + "For example,\n" + "\n" + "```\n" + "SELECT sum(x) OVER () FROM table\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.7.2.1') + num="3.7.2.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_AdHocWindow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support ad hoc window specification in the `OVER` clause.\n' - '\n' - '```\n' - 'OVER [window_spec]\n' - '```\n' - '\n' - 'See [window_spec] definition.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - '(count(*) OVER (partition by id order by time desc))\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support ad hoc window specification in the `OVER` clause.\n" + "\n" + "```\n" + "OVER [window_spec]\n" + "```\n" + "\n" + "See [window_spec] definition.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "(count(*) OVER (partition by id order by time desc))\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.7.3.1') + num="3.7.3.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_AdHocWindow_MissingWindowSpec_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow.MissingWindowSpec.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow.MissingWindowSpec.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `OVER` clause has missing [window_spec].\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `OVER` clause has missing [window_spec].\n" + "\n" + ), link=None, level=4, - num='3.7.3.2') + num="3.7.3.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using a previously defined named window in the `OVER` clause.\n' - '\n' - '```\n' - 'OVER [window_name]\n' - '```\n' - '\n' - 'See [window_name] definition.\n' - '\n' - 'For example,\n' - '\n' - '```sql\n' - 'SELECT count(*) OVER w FROM table WINDOW w AS (partition by id)\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support using a previously defined named window in the `OVER` clause.\n" + "\n" + "```\n" + "OVER [window_name]\n" + "```\n" + "\n" + "See [window_name] definition.\n" + "\n" + "For example,\n" + "\n" + "```sql\n" + "SELECT count(*) OVER w FROM table WINDOW w AS (partition by id)\n" + "```\n" + "\n" + ), link=None, level=4, - num='3.7.4.1') + num="3.7.4.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow_InvalidName_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.InvalidName.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.InvalidName.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `OVER` clause reference invalid window name.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `OVER` clause reference invalid window name.\n" + "\n" + ), link=None, level=4, - num='3.7.4.2') + num="3.7.4.2", +) RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow_MultipleWindows_Error = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.MultipleWindows.Error', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.MultipleWindows.Error", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL return an error if the `OVER` clause references more than one window name.\n' - '\n' - ), + "[ClickHouse] SHALL return an error if the `OVER` clause references more than one window name.\n" + "\n" + ), link=None, level=4, - num='3.7.4.3') + num="3.7.4.3", +) RQ_SRS_019_ClickHouse_WindowFunctions_FirstValue = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.FirstValue', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.FirstValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `first_value` window function that\n' - 'SHALL be synonum for the `any(value)` function\n' - 'that SHALL return the value of `expr` from first row in the window frame.\n' - '\n' - '```\n' - 'first_value(expr) OVER ...\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `first_value` window function that\n" + "SHALL be synonum for the `any(value)` function\n" + "that SHALL return the value of `expr` from first row in the window frame.\n" + "\n" + "```\n" + "first_value(expr) OVER ...\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.8.1.1.1') + num="3.8.1.1.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_LastValue = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.LastValue', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.LastValue", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `last_value` window function that\n' - 'SHALL be synonym for the `anyLast(value)` function\n' - 'that SHALL return the value of `expr` from the last row in the window frame.\n' - '\n' - '```\n' - 'last_value(expr) OVER ...\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `last_value` window function that\n" + "SHALL be synonym for the `anyLast(value)` function\n" + "that SHALL return the value of `expr` from the last row in the window frame.\n" + "\n" + "```\n" + "last_value(expr) OVER ...\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.8.1.2.1') + num="3.8.1.2.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_Lag_Workaround = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Lag.Workaround', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Lag.Workaround", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support a workaround for the `lag(value, offset)` function as\n' - '\n' - '```\n' - 'any(value) OVER (.... ROWS BETWEEN PRECEDING AND PRECEDING)\n' - '```\n' - '\n' - 'The function SHALL returns the value from the row that lags (precedes) the current row\n' - 'by the `N` rows within its partition. Where `N` is the `value` passed to the `any` function.\n' - '\n' - 'If there is no such row, the return value SHALL be default.\n' - '\n' - 'For example, if `N` is 3, the return value is default for the first two rows.\n' - 'If N or default are missing, the defaults are 1 and NULL, respectively.\n' - '\n' - '`N` SHALL be a literal non-negative integer. If N is 0, the value SHALL be\n' - 'returned for the current row.\n' - '\n' - ), + "[ClickHouse] SHALL support a workaround for the `lag(value, offset)` function as\n" + "\n" + "```\n" + "any(value) OVER (.... ROWS BETWEEN PRECEDING AND PRECEDING)\n" + "```\n" + "\n" + "The function SHALL returns the value from the row that lags (precedes) the current row\n" + "by the `N` rows within its partition. Where `N` is the `value` passed to the `any` function.\n" + "\n" + "If there is no such row, the return value SHALL be default.\n" + "\n" + "For example, if `N` is 3, the return value is default for the first two rows.\n" + "If N or default are missing, the defaults are 1 and NULL, respectively.\n" + "\n" + "`N` SHALL be a literal non-negative integer. If N is 0, the value SHALL be\n" + "returned for the current row.\n" + "\n" + ), link=None, level=5, - num='3.8.1.3.1') + num="3.8.1.3.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_Lead_Workaround = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Lead.Workaround', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Lead.Workaround", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support a workaround for the `lead(value, offset)` function as\n' - '\n' - '```\n' - 'any(value) OVER (.... ROWS BETWEEN FOLLOWING AND FOLLOWING)\n' - '```\n' - '\n' - 'The function SHALL returns the value from the row that leads (follows) the current row by\n' - 'the `N` rows within its partition. Where `N` is the `value` passed to the `any` function.\n' - '\n' - 'If there is no such row, the return value SHALL be default.\n' - '\n' - 'For example, if `N` is 3, the return value is default for the last two rows.\n' - 'If `N` or default are missing, the defaults are 1 and NULL, respectively.\n' - '\n' - '`N` SHALL be a literal non-negative integer. If `N` is 0, the value SHALL be\n' - 'returned for the current row.\n' - '\n' - ), + "[ClickHouse] SHALL support a workaround for the `lead(value, offset)` function as\n" + "\n" + "```\n" + "any(value) OVER (.... ROWS BETWEEN FOLLOWING AND FOLLOWING)\n" + "```\n" + "\n" + "The function SHALL returns the value from the row that leads (follows) the current row by\n" + "the `N` rows within its partition. Where `N` is the `value` passed to the `any` function.\n" + "\n" + "If there is no such row, the return value SHALL be default.\n" + "\n" + "For example, if `N` is 3, the return value is default for the last two rows.\n" + "If `N` or default are missing, the defaults are 1 and NULL, respectively.\n" + "\n" + "`N` SHALL be a literal non-negative integer. If `N` is 0, the value SHALL be\n" + "returned for the current row.\n" + "\n" + ), link=None, level=5, - num='3.8.1.4.1') + num="3.8.1.4.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_LeadInFrame = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.LeadInFrame', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.LeadInFrame", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `leadInFrame(expr[, offset, [default]])` function.\n' - '\n' - 'For example,\n' - '```\n' - 'leadInFrame(column) OVER (...)\n' - '```\n' - '\n' - 'The function SHALL return the value from the row that leads (follows) the current row\n' - 'by the `offset` rows within the current frame. If there is no such row,\n' - 'the return value SHALL be the `default` value. If the `default` value is not specified \n' - 'then the default value for the corresponding column data type SHALL be returned.\n' - '\n' - 'The `offset` SHALL be a literal non-negative integer. If the `offset` is set to `0`, then\n' - 'the value SHALL be returned for the current row. If the `offset` is not specified, the default\n' - 'value SHALL be `1`.\n' - '\n' - ), + "[ClickHouse] SHALL support the `leadInFrame(expr[, offset, [default]])` function.\n" + "\n" + "For example,\n" + "```\n" + "leadInFrame(column) OVER (...)\n" + "```\n" + "\n" + "The function SHALL return the value from the row that leads (follows) the current row\n" + "by the `offset` rows within the current frame. If there is no such row,\n" + "the return value SHALL be the `default` value. If the `default` value is not specified \n" + "then the default value for the corresponding column data type SHALL be returned.\n" + "\n" + "The `offset` SHALL be a literal non-negative integer. If the `offset` is set to `0`, then\n" + "the value SHALL be returned for the current row. If the `offset` is not specified, the default\n" + "value SHALL be `1`.\n" + "\n" + ), link=None, level=5, - num='3.8.1.5.1') + num="3.8.1.5.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_LagInFrame = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.LagInFrame', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.LagInFrame", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support the `lagInFrame(expr[, offset, [default]])` function.\n' - '\n' - 'For example,\n' - '```\n' - 'lagInFrame(column) OVER (...)\n' - '```\n' - '\n' - 'The function SHALL return the value from the row that lags (preceds) the current row\n' - 'by the `offset` rows within the current frame. If there is no such row,\n' - 'the return value SHALL be the `default` value. If the `default` value is not specified \n' - 'then the default value for the corresponding column data type SHALL be returned.\n' - '\n' - 'The `offset` SHALL be a literal non-negative integer. If the `offset` is set to `0`, then\n' - 'the value SHALL be returned for the current row. If the `offset` is not specified, the default\n' - 'value SHALL be `1`.\n' - '\n' - ), + "[ClickHouse] SHALL support the `lagInFrame(expr[, offset, [default]])` function.\n" + "\n" + "For example,\n" + "```\n" + "lagInFrame(column) OVER (...)\n" + "```\n" + "\n" + "The function SHALL return the value from the row that lags (preceds) the current row\n" + "by the `offset` rows within the current frame. If there is no such row,\n" + "the return value SHALL be the `default` value. If the `default` value is not specified \n" + "then the default value for the corresponding column data type SHALL be returned.\n" + "\n" + "The `offset` SHALL be a literal non-negative integer. If the `offset` is set to `0`, then\n" + "the value SHALL be returned for the current row. If the `offset` is not specified, the default\n" + "value SHALL be `1`.\n" + "\n" + ), link=None, level=5, - num='3.8.1.6.1') + num="3.8.1.6.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_Rank = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.Rank', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.Rank", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `rank` window function that SHALL\n' - 'return the rank of the current row within its partition with gaps.\n' - '\n' - 'Peers SHALL be considered ties and receive the same rank.\n' - 'The function SHALL not assign consecutive ranks to peer groups if groups of size greater than one exist\n' - 'and the result is noncontiguous rank numbers.\n' - '\n' - 'If the function is used without `ORDER BY` to sort partition rows into the desired order\n' - 'then all rows SHALL be peers.\n' - '\n' - '```\n' - 'rank() OVER ...\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `rank` window function that SHALL\n" + "return the rank of the current row within its partition with gaps.\n" + "\n" + "Peers SHALL be considered ties and receive the same rank.\n" + "The function SHALL not assign consecutive ranks to peer groups if groups of size greater than one exist\n" + "and the result is noncontiguous rank numbers.\n" + "\n" + "If the function is used without `ORDER BY` to sort partition rows into the desired order\n" + "then all rows SHALL be peers.\n" + "\n" + "```\n" + "rank() OVER ...\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.8.1.7.1') + num="3.8.1.7.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_DenseRank = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.DenseRank', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.DenseRank", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `dense_rank` function over a window that SHALL\n' - 'return the rank of the current row within its partition without gaps.\n' - '\n' - 'Peers SHALL be considered ties and receive the same rank.\n' - 'The function SHALL assign consecutive ranks to peer groups and\n' - 'the result is that groups of size greater than one do not produce noncontiguous rank numbers.\n' - '\n' - 'If the function is used without `ORDER BY` to sort partition rows into the desired order\n' - 'then all rows SHALL be peers.\n' - '\n' - '```\n' - 'dense_rank() OVER ...\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `dense_rank` function over a window that SHALL\n" + "return the rank of the current row within its partition without gaps.\n" + "\n" + "Peers SHALL be considered ties and receive the same rank.\n" + "The function SHALL assign consecutive ranks to peer groups and\n" + "the result is that groups of size greater than one do not produce noncontiguous rank numbers.\n" + "\n" + "If the function is used without `ORDER BY` to sort partition rows into the desired order\n" + "then all rows SHALL be peers.\n" + "\n" + "```\n" + "dense_rank() OVER ...\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.8.1.8.1') + num="3.8.1.8.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_RowNumber = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.RowNumber', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowNumber", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support `row_number` function over a window that SHALL\n' - 'returns the number of the current row within its partition.\n' - '\n' - 'Rows numbers SHALL range from 1 to the number of partition rows.\n' - '\n' - 'The `ORDER BY` affects the order in which rows are numbered.\n' - 'Without `ORDER BY`, row numbering MAY be nondeterministic.\n' - '\n' - '```\n' - 'row_number() OVER ...\n' - '```\n' - '\n' - ), + "[ClickHouse] SHALL support `row_number` function over a window that SHALL\n" + "returns the number of the current row within its partition.\n" + "\n" + "Rows numbers SHALL range from 1 to the number of partition rows.\n" + "\n" + "The `ORDER BY` affects the order in which rows are numbered.\n" + "Without `ORDER BY`, row numbering MAY be nondeterministic.\n" + "\n" + "```\n" + "row_number() OVER ...\n" + "```\n" + "\n" + ), link=None, level=5, - num='3.8.1.9.1') + num="3.8.1.9.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support using aggregate functions over windows.\n' - '\n' - '* [count](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/count/)\n' - '* [min](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/min/)\n' - '* [max](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/max/)\n' - '* [sum](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/sum/)\n' - '* [avg](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/avg/)\n' - '* [any](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/any/)\n' - '* [stddevPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stddevpop/)\n' - '* [stddevSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stddevsamp/)\n' - '* [varPop(x)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/varpop/)\n' - '* [varSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/varsamp/)\n' - '* [covarPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/covarpop/)\n' - '* [covarSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/covarsamp/)\n' - '* [anyHeavy](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/anyheavy/)\n' - '* [anyLast](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/anylast/)\n' - '* [argMin](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/argmin/)\n' - '* [argMax](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/argmax/)\n' - '* [avgWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/avgweighted/)\n' - '* [corr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/corr/)\n' - '* [topK](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/topk/)\n' - '* [topKWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/topkweighted/)\n' - '* [groupArray](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparray/)\n' - '* [groupUniqArray](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray/)\n' - '* [groupArrayInsertAt](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat/)\n' - '* [groupArrayMovingSum](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum/)\n' - '* [groupArrayMovingAvg](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg/)\n' - '* [groupArraySample](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparraysample/)\n' - '* [groupBitAnd](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitand/)\n' - '* [groupBitOr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitor/)\n' - '* [groupBitXor](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitxor/)\n' - '* [groupBitmap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmap/)\n' - '* [groupBitmapAnd](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmapand/)\n' - '* [groupBitmapOr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor/)\n' - '* [groupBitmapXor](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor/)\n' - '* [sumWithOverflow](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow/)\n' - '* [deltaSum](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/deltasum/)\n' - '* [sumMap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/summap/)\n' - '* [minMap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/minmap/)\n' - '* [maxMap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/maxmap/)\n' - '* [initializeAggregation](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/initializeAggregation/)\n' - '* [skewPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/skewpop/)\n' - '* [skewSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/skewsamp/)\n' - '* [kurtPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/kurtpop/)\n' - '* [kurtSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/kurtsamp/)\n' - '* [uniq](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniq/)\n' - '* [uniqExact](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqexact/)\n' - '* [uniqCombined](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqcombined/)\n' - '* [uniqCombined64](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64/)\n' - '* [uniqHLL12](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqhll12/)\n' - '* [quantile](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantile/)\n' - '* [quantiles](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiles/)\n' - '* [quantileExact](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantileexact/)\n' - '* [quantileExactWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted/)\n' - '* [quantileTiming](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletiming/)\n' - '* [quantileTimingWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted/)\n' - '* [quantileDeterministic](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiledeterministic/)\n' - '* [quantileTDigest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest/)\n' - '* [quantileTDigestWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted/)\n' - '* [simpleLinearRegression](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression/)\n' - '* [stochasticLinearRegression](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression/)\n' - '* [stochasticLogisticRegression](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression/)\n' - '* [categoricalInformationValue](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression/)\n' - '* [studentTTest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/studentttest/)\n' - '* [welchTTest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/welchttest/)\n' - '* [mannWhitneyUTest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest/)\n' - '* [median](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/median/)\n' - '* [rankCorr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/rankCorr/)\n' - '\n' - ), + "[ClickHouse] SHALL support using aggregate functions over windows.\n" + "\n" + "* [count](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/count/)\n" + "* [min](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/min/)\n" + "* [max](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/max/)\n" + "* [sum](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/sum/)\n" + "* [avg](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/avg/)\n" + "* [any](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/any/)\n" + "* [stddevPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stddevpop/)\n" + "* [stddevSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stddevsamp/)\n" + "* [varPop(x)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/varpop/)\n" + "* [varSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/varsamp/)\n" + "* [covarPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/covarpop/)\n" + "* [covarSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/covarsamp/)\n" + "* [anyHeavy](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/anyheavy/)\n" + "* [anyLast](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/anylast/)\n" + "* [argMin](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/argmin/)\n" + "* [argMax](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/argmax/)\n" + "* [avgWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/avgweighted/)\n" + "* [corr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/corr/)\n" + "* [topK](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/topk/)\n" + "* [topKWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/topkweighted/)\n" + "* [groupArray](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparray/)\n" + "* [groupUniqArray](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray/)\n" + "* [groupArrayInsertAt](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat/)\n" + "* [groupArrayMovingSum](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum/)\n" + "* [groupArrayMovingAvg](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg/)\n" + "* [groupArraySample](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/grouparraysample/)\n" + "* [groupBitAnd](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitand/)\n" + "* [groupBitOr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitor/)\n" + "* [groupBitXor](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitxor/)\n" + "* [groupBitmap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmap/)\n" + "* [groupBitmapAnd](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmapand/)\n" + "* [groupBitmapOr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor/)\n" + "* [groupBitmapXor](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor/)\n" + "* [sumWithOverflow](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow/)\n" + "* [deltaSum](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/deltasum/)\n" + "* [sumMap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/summap/)\n" + "* [minMap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/minmap/)\n" + "* [maxMap](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/maxmap/)\n" + "* [initializeAggregation](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/initializeAggregation/)\n" + "* [skewPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/skewpop/)\n" + "* [skewSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/skewsamp/)\n" + "* [kurtPop](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/kurtpop/)\n" + "* [kurtSamp](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/kurtsamp/)\n" + "* [uniq](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniq/)\n" + "* [uniqExact](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqexact/)\n" + "* [uniqCombined](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqcombined/)\n" + "* [uniqCombined64](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64/)\n" + "* [uniqHLL12](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/uniqhll12/)\n" + "* [quantile](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantile/)\n" + "* [quantiles](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiles/)\n" + "* [quantileExact](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantileexact/)\n" + "* [quantileExactWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantileexactweighted/)\n" + "* [quantileTiming](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletiming/)\n" + "* [quantileTimingWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletimingweighted/)\n" + "* [quantileDeterministic](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiledeterministic/)\n" + "* [quantileTDigest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest/)\n" + "* [quantileTDigestWeighted](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted/)\n" + "* [simpleLinearRegression](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression/)\n" + "* [stochasticLinearRegression](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression/)\n" + "* [stochasticLogisticRegression](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression/)\n" + "* [categoricalInformationValue](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression/)\n" + "* [studentTTest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/studentttest/)\n" + "* [welchTTest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/welchttest/)\n" + "* [mannWhitneyUTest](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest/)\n" + "* [median](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/median/)\n" + "* [rankCorr](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/rankCorr/)\n" + "\n" + ), link=None, level=4, - num='3.8.2.1') + num="3.8.2.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions_Combinators = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Combinators', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Combinators", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support aggregate functions with combinator prefixes over windows.\n' - '\n' - '* [-If](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-if)\n' - '* [-Array](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-array)\n' - '* [-SimpleState](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-simplestate)\n' - '* [-State](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-state)\n' - '* [-Merge](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#aggregate_functions_combinators-merge)\n' - '* [-MergeState](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#aggregate_functions_combinators-mergestate)\n' - '* [-ForEach](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-foreach)\n' - '* [-Distinct](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-distinct)\n' - '* [-OrDefault](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-ordefault)\n' - '* [-OrNull](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-ornull)\n' - '* [-Resample](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-resample)\n' - '\n' - ), + "[ClickHouse] SHALL support aggregate functions with combinator prefixes over windows.\n" + "\n" + "* [-If](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-if)\n" + "* [-Array](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-array)\n" + "* [-SimpleState](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-simplestate)\n" + "* [-State](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-state)\n" + "* [-Merge](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#aggregate_functions_combinators-merge)\n" + "* [-MergeState](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#aggregate_functions_combinators-mergestate)\n" + "* [-ForEach](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-foreach)\n" + "* [-Distinct](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-distinct)\n" + "* [-OrDefault](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-ordefault)\n" + "* [-OrNull](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-ornull)\n" + "* [-Resample](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/combinators/#agg-functions-combinator-resample)\n" + "\n" + ), link=None, level=5, - num='3.8.2.2.1') + num="3.8.2.2.1", +) RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions_Parametric = Requirement( - name='RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Parametric', - version='1.0', + name="RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Parametric", + version="1.0", priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL support parametric aggregate functions over windows.\n' - '\n' - '* [histogram](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#histogram)\n' - '* [sequenceMatch(pattern)(timestamp, cond1, cond2, ...)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#function-sequencematch)\n' - '* [sequenceCount(pattern)(time, cond1, cond2, ...)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#function-sequencecount)\n' - '* [windowFunnel](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#windowfunnel)\n' - '* [retention](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#retention)\n' - '* [uniqUpTo(N)(x)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#uniquptonx)\n' - '* [sumMapFiltered(keys_to_keep)(keys, values)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#summapfilteredkeys-to-keepkeys-values)\n' - '\n' - ), + "[ClickHouse] SHALL support parametric aggregate functions over windows.\n" + "\n" + "* [histogram](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#histogram)\n" + "* [sequenceMatch(pattern)(timestamp, cond1, cond2, ...)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#function-sequencematch)\n" + "* [sequenceCount(pattern)(time, cond1, cond2, ...)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#function-sequencecount)\n" + "* [windowFunnel](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#windowfunnel)\n" + "* [retention](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#retention)\n" + "* [uniqUpTo(N)(x)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#uniquptonx)\n" + "* [sumMapFiltered(keys_to_keep)(keys, values)](https://clickhouse.com/docs/en/sql-reference/aggregate-functions/parametric-functions/#summapfilteredkeys-to-keepkeys-values)\n" + "\n" + ), link=None, level=5, - num='3.8.2.3.1') + num="3.8.2.3.1", +) SRS019_ClickHouse_Window_Functions = Specification( - name='SRS019 ClickHouse Window Functions', + name="SRS019 ClickHouse Window Functions", description=None, author=None, - date=None, - status=None, + date=None, + status=None, approved_by=None, approved_date=None, approved_version=None, @@ -3348,201 +3481,735 @@ SRS019_ClickHouse_Window_Functions = Specification( parent=None, children=None, headings=( - Heading(name='Revision History', level=1, num='1'), - Heading(name='Introduction', level=1, num='2'), - Heading(name='Requirements', level=1, num='3'), - Heading(name='General', level=2, num='3.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions', level=3, num='3.1.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.NonDistributedTables', level=3, num='3.1.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.DistributedTables', level=3, num='3.1.3'), - Heading(name='Window Specification', level=2, num='3.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowSpec', level=3, num='3.2.1'), - Heading(name='PARTITION Clause', level=2, num='3.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause', level=3, num='3.3.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MultipleExpr', level=3, num='3.3.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MissingExpr.Error', level=3, num='3.3.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.InvalidExpr.Error', level=3, num='3.3.4'), - Heading(name='ORDER Clause', level=2, num='3.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause', level=3, num='3.4.1'), - Heading(name='order_clause', level=4, num='3.4.1.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MultipleExprs', level=3, num='3.4.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MissingExpr.Error', level=3, num='3.4.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.InvalidExpr.Error', level=3, num='3.4.4'), - Heading(name='FRAME Clause', level=2, num='3.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause', level=3, num='3.5.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause.DefaultFrame', level=3, num='3.5.2'), - Heading(name='ROWS', level=3, num='3.5.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame', level=4, num='3.5.3.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.MissingFrameExtent.Error', level=4, num='3.5.3.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.InvalidFrameExtent.Error', level=4, num='3.5.3.3'), - Heading(name='ROWS CURRENT ROW', level=4, num='3.5.3.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.CurrentRow', level=5, num='3.5.3.4.1'), - Heading(name='ROWS UNBOUNDED PRECEDING', level=4, num='3.5.3.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedPreceding', level=5, num='3.5.3.5.1'), - Heading(name='ROWS `expr` PRECEDING', level=4, num='3.5.3.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprPreceding', level=5, num='3.5.3.6.1'), - Heading(name='ROWS UNBOUNDED FOLLOWING', level=4, num='3.5.3.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedFollowing.Error', level=5, num='3.5.3.7.1'), - Heading(name='ROWS `expr` FOLLOWING', level=4, num='3.5.3.8'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprFollowing.Error', level=5, num='3.5.3.8.1'), - Heading(name='ROWS BETWEEN CURRENT ROW', level=4, num='3.5.3.9'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.CurrentRow', level=5, num='3.5.3.9.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedPreceding.Error', level=5, num='3.5.3.9.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprPreceding.Error', level=5, num='3.5.3.9.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedFollowing', level=5, num='3.5.3.9.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprFollowing', level=5, num='3.5.3.9.5'), - Heading(name='ROWS BETWEEN UNBOUNDED PRECEDING', level=4, num='3.5.3.10'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.CurrentRow', level=5, num='3.5.3.10.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedPreceding.Error', level=5, num='3.5.3.10.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprPreceding', level=5, num='3.5.3.10.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedFollowing', level=5, num='3.5.3.10.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprFollowing', level=5, num='3.5.3.10.5'), - Heading(name='ROWS BETWEEN UNBOUNDED FOLLOWING', level=4, num='3.5.3.11'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedFollowing.Error', level=5, num='3.5.3.11.1'), - Heading(name='ROWS BETWEEN `expr` FOLLOWING', level=4, num='3.5.3.12'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.Error', level=5, num='3.5.3.12.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing.Error', level=5, num='3.5.3.12.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.UnboundedFollowing', level=5, num='3.5.3.12.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing', level=5, num='3.5.3.12.4'), - Heading(name='ROWS BETWEEN `expr` PRECEDING', level=4, num='3.5.3.13'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.CurrentRow', level=5, num='3.5.3.13.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedPreceding.Error', level=5, num='3.5.3.13.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedFollowing', level=5, num='3.5.3.13.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding.Error', level=5, num='3.5.3.13.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding', level=5, num='3.5.3.13.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprFollowing', level=5, num='3.5.3.13.6'), - Heading(name='RANGE', level=3, num='3.5.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame', level=4, num='3.5.4.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.DateAndDateTime', level=4, num='3.5.4.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.IntAndUInt', level=4, num='3.5.4.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MultipleColumnsInOrderBy.Error', level=4, num='3.5.4.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MissingFrameExtent.Error', level=4, num='3.5.4.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.InvalidFrameExtent.Error', level=4, num='3.5.4.6'), - Heading(name='`CURRENT ROW` Peers', level=4, num='3.5.4.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.CurrentRow.Peers', level=4, num='3.5.4.8'), - Heading(name='RANGE CURRENT ROW', level=4, num='3.5.4.9'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithoutOrderBy', level=5, num='3.5.4.9.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithOrderBy', level=5, num='3.5.4.9.2'), - Heading(name='RANGE UNBOUNDED FOLLOWING', level=4, num='3.5.4.10'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedFollowing.Error', level=5, num='3.5.4.10.1'), - Heading(name='RANGE UNBOUNDED PRECEDING', level=4, num='3.5.4.11'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithoutOrderBy', level=5, num='3.5.4.11.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithOrderBy', level=5, num='3.5.4.11.2'), - Heading(name='RANGE `expr` PRECEDING', level=4, num='3.5.4.12'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithoutOrderBy.Error', level=5, num='3.5.4.12.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.OrderByNonNumericalColumn.Error', level=5, num='3.5.4.12.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithOrderBy', level=5, num='3.5.4.12.3'), - Heading(name='RANGE `expr` FOLLOWING', level=4, num='3.5.4.13'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.13.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithOrderBy.Error', level=5, num='3.5.4.13.2'), - Heading(name='RANGE BETWEEN CURRENT ROW', level=4, num='3.5.4.14'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.CurrentRow', level=5, num='3.5.4.14.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedPreceding.Error', level=5, num='3.5.4.14.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedFollowing', level=5, num='3.5.4.14.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.14.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithOrderBy', level=5, num='3.5.4.14.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprPreceding.Error', level=5, num='3.5.4.14.6'), - Heading(name='RANGE BETWEEN UNBOUNDED PRECEDING', level=4, num='3.5.4.15'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.CurrentRow', level=5, num='3.5.4.15.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedPreceding.Error', level=5, num='3.5.4.15.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedFollowing', level=5, num='3.5.4.15.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithoutOrderBy.Error', level=5, num='3.5.4.15.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithOrderBy', level=5, num='3.5.4.15.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.15.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithOrderBy', level=5, num='3.5.4.15.7'), - Heading(name='RANGE BETWEEN UNBOUNDED FOLLOWING', level=4, num='3.5.4.16'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.CurrentRow.Error', level=5, num='3.5.4.16.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedFollowing.Error', level=5, num='3.5.4.16.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedPreceding.Error', level=5, num='3.5.4.16.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprPreceding.Error', level=5, num='3.5.4.16.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprFollowing.Error', level=5, num='3.5.4.16.5'), - Heading(name='RANGE BETWEEN expr PRECEDING', level=4, num='3.5.4.17'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithOrderBy', level=5, num='3.5.4.17.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithoutOrderBy.Error', level=5, num='3.5.4.17.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedPreceding.Error', level=5, num='3.5.4.17.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.17.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithOrderBy', level=5, num='3.5.4.17.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.17.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithOrderBy', level=5, num='3.5.4.17.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithoutOrderBy.Error', level=5, num='3.5.4.17.8'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy.Error', level=5, num='3.5.4.17.9'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy', level=5, num='3.5.4.17.10'), - Heading(name='RANGE BETWEEN expr FOLLOWING', level=4, num='3.5.4.18'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithoutOrderBy.Error', level=5, num='3.5.4.18.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithOrderBy.Error', level=5, num='3.5.4.18.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.ZeroSpecialCase', level=5, num='3.5.4.18.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.18.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithOrderBy', level=5, num='3.5.4.18.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedPreceding.Error', level=5, num='3.5.4.18.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithoutOrderBy.Error', level=5, num='3.5.4.18.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.Error', level=5, num='3.5.4.18.8'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithOrderBy.ZeroSpecialCase', level=5, num='3.5.4.18.9'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithoutOrderBy.Error', level=5, num='3.5.4.18.10'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy.Error', level=5, num='3.5.4.18.11'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy', level=5, num='3.5.4.18.12'), - Heading(name='Frame Extent', level=3, num='3.5.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Extent', level=4, num='3.5.5.1'), - Heading(name='Frame Start', level=3, num='3.5.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Start', level=4, num='3.5.6.1'), - Heading(name='Frame Between', level=3, num='3.5.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Between', level=4, num='3.5.7.1'), - Heading(name='Frame End', level=3, num='3.5.8'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Frame.End', level=4, num='3.5.8.1'), - Heading(name='`CURRENT ROW`', level=3, num='3.5.9'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.CurrentRow', level=4, num='3.5.9.1'), - Heading(name='`UNBOUNDED PRECEDING`', level=3, num='3.5.10'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedPreceding', level=4, num='3.5.10.1'), - Heading(name='`UNBOUNDED FOLLOWING`', level=3, num='3.5.11'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedFollowing', level=4, num='3.5.11.1'), - Heading(name='`expr PRECEDING`', level=3, num='3.5.12'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding', level=4, num='3.5.12.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding.ExprValue', level=4, num='3.5.12.2'), - Heading(name='`expr FOLLOWING`', level=3, num='3.5.13'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing', level=4, num='3.5.13.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing.ExprValue', level=4, num='3.5.13.2'), - Heading(name='WINDOW Clause', level=2, num='3.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause', level=3, num='3.6.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MultipleWindows', level=3, num='3.6.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MissingWindowSpec.Error', level=3, num='3.6.3'), - Heading(name='`OVER` Clause', level=2, num='3.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause', level=3, num='3.7.1'), - Heading(name='Empty Clause', level=3, num='3.7.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.EmptyOverClause', level=4, num='3.7.2.1'), - Heading(name='Ad-Hoc Window', level=3, num='3.7.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow', level=4, num='3.7.3.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow.MissingWindowSpec.Error', level=4, num='3.7.3.2'), - Heading(name='Named Window', level=3, num='3.7.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow', level=4, num='3.7.4.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.InvalidName.Error', level=4, num='3.7.4.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.MultipleWindows.Error', level=4, num='3.7.4.3'), - Heading(name='Window Functions', level=2, num='3.8'), - Heading(name='Nonaggregate Functions', level=3, num='3.8.1'), - Heading(name='The `first_value(expr)` Function', level=4, num='3.8.1.1'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.FirstValue', level=5, num='3.8.1.1.1'), - Heading(name='The `last_value(expr)` Function', level=4, num='3.8.1.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.LastValue', level=5, num='3.8.1.2.1'), - Heading(name='The `lag(value, offset)` Function Workaround', level=4, num='3.8.1.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Lag.Workaround', level=5, num='3.8.1.3.1'), - Heading(name='The `lead(value, offset)` Function Workaround', level=4, num='3.8.1.4'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Lead.Workaround', level=5, num='3.8.1.4.1'), - Heading(name='The `leadInFrame(expr[, offset, [default]])`', level=4, num='3.8.1.5'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.LeadInFrame', level=5, num='3.8.1.5.1'), - Heading(name='The `lagInFrame(expr[, offset, [default]])`', level=4, num='3.8.1.6'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.LagInFrame', level=5, num='3.8.1.6.1'), - Heading(name='The `rank()` Function', level=4, num='3.8.1.7'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.Rank', level=5, num='3.8.1.7.1'), - Heading(name='The `dense_rank()` Function', level=4, num='3.8.1.8'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.DenseRank', level=5, num='3.8.1.8.1'), - Heading(name='The `row_number()` Function', level=4, num='3.8.1.9'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.RowNumber', level=5, num='3.8.1.9.1'), - Heading(name='Aggregate Functions', level=3, num='3.8.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions', level=4, num='3.8.2.1'), - Heading(name='Combinators', level=4, num='3.8.2.2'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Combinators', level=5, num='3.8.2.2.1'), - Heading(name='Parametric', level=4, num='3.8.2.3'), - Heading(name='RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Parametric', level=5, num='3.8.2.3.1'), - Heading(name='References', level=1, num='4'), + Heading(name="Revision History", level=1, num="1"), + Heading(name="Introduction", level=1, num="2"), + Heading(name="Requirements", level=1, num="3"), + Heading(name="General", level=2, num="3.1"), + Heading(name="RQ.SRS-019.ClickHouse.WindowFunctions", level=3, num="3.1.1"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.NonDistributedTables", + level=3, + num="3.1.2", ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.DistributedTables", + level=3, + num="3.1.3", + ), + Heading(name="Window Specification", level=2, num="3.2"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowSpec", + level=3, + num="3.2.1", + ), + Heading(name="PARTITION Clause", level=2, num="3.3"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause", + level=3, + num="3.3.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MultipleExpr", + level=3, + num="3.3.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.MissingExpr.Error", + level=3, + num="3.3.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.PartitionClause.InvalidExpr.Error", + level=3, + num="3.3.4", + ), + Heading(name="ORDER Clause", level=2, num="3.4"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause", + level=3, + num="3.4.1", + ), + Heading(name="order_clause", level=4, num="3.4.1.1"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MultipleExprs", + level=3, + num="3.4.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.MissingExpr.Error", + level=3, + num="3.4.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OrderClause.InvalidExpr.Error", + level=3, + num="3.4.4", + ), + Heading(name="FRAME Clause", level=2, num="3.5"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause", + level=3, + num="3.5.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.FrameClause.DefaultFrame", + level=3, + num="3.5.2", + ), + Heading(name="ROWS", level=3, num="3.5.3"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame", + level=4, + num="3.5.3.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.MissingFrameExtent.Error", + level=4, + num="3.5.3.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.InvalidFrameExtent.Error", + level=4, + num="3.5.3.3", + ), + Heading(name="ROWS CURRENT ROW", level=4, num="3.5.3.4"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.CurrentRow", + level=5, + num="3.5.3.4.1", + ), + Heading(name="ROWS UNBOUNDED PRECEDING", level=4, num="3.5.3.5"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedPreceding", + level=5, + num="3.5.3.5.1", + ), + Heading(name="ROWS `expr` PRECEDING", level=4, num="3.5.3.6"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprPreceding", + level=5, + num="3.5.3.6.1", + ), + Heading(name="ROWS UNBOUNDED FOLLOWING", level=4, num="3.5.3.7"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.UnboundedFollowing.Error", + level=5, + num="3.5.3.7.1", + ), + Heading(name="ROWS `expr` FOLLOWING", level=4, num="3.5.3.8"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Start.ExprFollowing.Error", + level=5, + num="3.5.3.8.1", + ), + Heading(name="ROWS BETWEEN CURRENT ROW", level=4, num="3.5.3.9"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.CurrentRow", + level=5, + num="3.5.3.9.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedPreceding.Error", + level=5, + num="3.5.3.9.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprPreceding.Error", + level=5, + num="3.5.3.9.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.UnboundedFollowing", + level=5, + num="3.5.3.9.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.CurrentRow.ExprFollowing", + level=5, + num="3.5.3.9.5", + ), + Heading(name="ROWS BETWEEN UNBOUNDED PRECEDING", level=4, num="3.5.3.10"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.CurrentRow", + level=5, + num="3.5.3.10.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedPreceding.Error", + level=5, + num="3.5.3.10.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprPreceding", + level=5, + num="3.5.3.10.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.UnboundedFollowing", + level=5, + num="3.5.3.10.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedPreceding.ExprFollowing", + level=5, + num="3.5.3.10.5", + ), + Heading(name="ROWS BETWEEN UNBOUNDED FOLLOWING", level=4, num="3.5.3.11"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.UnboundedFollowing.Error", + level=5, + num="3.5.3.11.1", + ), + Heading(name="ROWS BETWEEN `expr` FOLLOWING", level=4, num="3.5.3.12"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.Error", + level=5, + num="3.5.3.12.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing.Error", + level=5, + num="3.5.3.12.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.UnboundedFollowing", + level=5, + num="3.5.3.12.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprFollowing.ExprFollowing", + level=5, + num="3.5.3.12.4", + ), + Heading(name="ROWS BETWEEN `expr` PRECEDING", level=4, num="3.5.3.13"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.CurrentRow", + level=5, + num="3.5.3.13.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedPreceding.Error", + level=5, + num="3.5.3.13.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.UnboundedFollowing", + level=5, + num="3.5.3.13.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding.Error", + level=5, + num="3.5.3.13.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprPreceding", + level=5, + num="3.5.3.13.5", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowsFrame.Between.ExprPreceding.ExprFollowing", + level=5, + num="3.5.3.13.6", + ), + Heading(name="RANGE", level=3, num="3.5.4"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame", + level=4, + num="3.5.4.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.DateAndDateTime", + level=4, + num="3.5.4.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.DataTypes.IntAndUInt", + level=4, + num="3.5.4.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MultipleColumnsInOrderBy.Error", + level=4, + num="3.5.4.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.MissingFrameExtent.Error", + level=4, + num="3.5.4.5", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.InvalidFrameExtent.Error", + level=4, + num="3.5.4.6", + ), + Heading(name="`CURRENT ROW` Peers", level=4, num="3.5.4.7"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.CurrentRow.Peers", + level=4, + num="3.5.4.8", + ), + Heading(name="RANGE CURRENT ROW", level=4, num="3.5.4.9"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithoutOrderBy", + level=5, + num="3.5.4.9.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.CurrentRow.WithOrderBy", + level=5, + num="3.5.4.9.2", + ), + Heading(name="RANGE UNBOUNDED FOLLOWING", level=4, num="3.5.4.10"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedFollowing.Error", + level=5, + num="3.5.4.10.1", + ), + Heading(name="RANGE UNBOUNDED PRECEDING", level=4, num="3.5.4.11"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithoutOrderBy", + level=5, + num="3.5.4.11.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.UnboundedPreceding.WithOrderBy", + level=5, + num="3.5.4.11.2", + ), + Heading(name="RANGE `expr` PRECEDING", level=4, num="3.5.4.12"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithoutOrderBy.Error", + level=5, + num="3.5.4.12.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.OrderByNonNumericalColumn.Error", + level=5, + num="3.5.4.12.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprPreceding.WithOrderBy", + level=5, + num="3.5.4.12.3", + ), + Heading(name="RANGE `expr` FOLLOWING", level=4, num="3.5.4.13"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.13.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Start.ExprFollowing.WithOrderBy.Error", + level=5, + num="3.5.4.13.2", + ), + Heading(name="RANGE BETWEEN CURRENT ROW", level=4, num="3.5.4.14"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.CurrentRow", + level=5, + num="3.5.4.14.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedPreceding.Error", + level=5, + num="3.5.4.14.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.UnboundedFollowing", + level=5, + num="3.5.4.14.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.14.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprFollowing.WithOrderBy", + level=5, + num="3.5.4.14.5", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.CurrentRow.ExprPreceding.Error", + level=5, + num="3.5.4.14.6", + ), + Heading(name="RANGE BETWEEN UNBOUNDED PRECEDING", level=4, num="3.5.4.15"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.CurrentRow", + level=5, + num="3.5.4.15.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedPreceding.Error", + level=5, + num="3.5.4.15.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.UnboundedFollowing", + level=5, + num="3.5.4.15.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithoutOrderBy.Error", + level=5, + num="3.5.4.15.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprPreceding.WithOrderBy", + level=5, + num="3.5.4.15.5", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.15.6", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedPreceding.ExprFollowing.WithOrderBy", + level=5, + num="3.5.4.15.7", + ), + Heading(name="RANGE BETWEEN UNBOUNDED FOLLOWING", level=4, num="3.5.4.16"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.CurrentRow.Error", + level=5, + num="3.5.4.16.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedFollowing.Error", + level=5, + num="3.5.4.16.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.UnboundedPreceding.Error", + level=5, + num="3.5.4.16.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprPreceding.Error", + level=5, + num="3.5.4.16.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.UnboundedFollowing.ExprFollowing.Error", + level=5, + num="3.5.4.16.5", + ), + Heading(name="RANGE BETWEEN expr PRECEDING", level=4, num="3.5.4.17"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithOrderBy", + level=5, + num="3.5.4.17.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.CurrentRow.WithoutOrderBy.Error", + level=5, + num="3.5.4.17.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedPreceding.Error", + level=5, + num="3.5.4.17.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.17.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.UnboundedFollowing.WithOrderBy", + level=5, + num="3.5.4.17.5", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.17.6", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprFollowing.WithOrderBy", + level=5, + num="3.5.4.17.7", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithoutOrderBy.Error", + level=5, + num="3.5.4.17.8", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy.Error", + level=5, + num="3.5.4.17.9", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprPreceding.ExprPreceding.WithOrderBy", + level=5, + num="3.5.4.17.10", + ), + Heading(name="RANGE BETWEEN expr FOLLOWING", level=4, num="3.5.4.18"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithoutOrderBy.Error", + level=5, + num="3.5.4.18.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.WithOrderBy.Error", + level=5, + num="3.5.4.18.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.CurrentRow.ZeroSpecialCase", + level=5, + num="3.5.4.18.3", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.18.4", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedFollowing.WithOrderBy", + level=5, + num="3.5.4.18.5", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.UnboundedPreceding.Error", + level=5, + num="3.5.4.18.6", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithoutOrderBy.Error", + level=5, + num="3.5.4.18.7", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.Error", + level=5, + num="3.5.4.18.8", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprPreceding.WithOrderBy.ZeroSpecialCase", + level=5, + num="3.5.4.18.9", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithoutOrderBy.Error", + level=5, + num="3.5.4.18.10", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy.Error", + level=5, + num="3.5.4.18.11", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RangeFrame.Between.ExprFollowing.ExprFollowing.WithOrderBy", + level=5, + num="3.5.4.18.12", + ), + Heading(name="Frame Extent", level=3, num="3.5.5"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Extent", + level=4, + num="3.5.5.1", + ), + Heading(name="Frame Start", level=3, num="3.5.6"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Start", + level=4, + num="3.5.6.1", + ), + Heading(name="Frame Between", level=3, num="3.5.7"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.Between", + level=4, + num="3.5.7.1", + ), + Heading(name="Frame End", level=3, num="3.5.8"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Frame.End", + level=4, + num="3.5.8.1", + ), + Heading(name="`CURRENT ROW`", level=3, num="3.5.9"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.CurrentRow", + level=4, + num="3.5.9.1", + ), + Heading(name="`UNBOUNDED PRECEDING`", level=3, num="3.5.10"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedPreceding", + level=4, + num="3.5.10.1", + ), + Heading(name="`UNBOUNDED FOLLOWING`", level=3, num="3.5.11"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.UnboundedFollowing", + level=4, + num="3.5.11.1", + ), + Heading(name="`expr PRECEDING`", level=3, num="3.5.12"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding", + level=4, + num="3.5.12.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprPreceding.ExprValue", + level=4, + num="3.5.12.2", + ), + Heading(name="`expr FOLLOWING`", level=3, num="3.5.13"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing", + level=4, + num="3.5.13.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.ExprFollowing.ExprValue", + level=4, + num="3.5.13.2", + ), + Heading(name="WINDOW Clause", level=2, num="3.6"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause", + level=3, + num="3.6.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MultipleWindows", + level=3, + num="3.6.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.WindowClause.MissingWindowSpec.Error", + level=3, + num="3.6.3", + ), + Heading(name="`OVER` Clause", level=2, num="3.7"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause", + level=3, + num="3.7.1", + ), + Heading(name="Empty Clause", level=3, num="3.7.2"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.EmptyOverClause", + level=4, + num="3.7.2.1", + ), + Heading(name="Ad-Hoc Window", level=3, num="3.7.3"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow", + level=4, + num="3.7.3.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.AdHocWindow.MissingWindowSpec.Error", + level=4, + num="3.7.3.2", + ), + Heading(name="Named Window", level=3, num="3.7.4"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow", + level=4, + num="3.7.4.1", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.InvalidName.Error", + level=4, + num="3.7.4.2", + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.OverClause.NamedWindow.MultipleWindows.Error", + level=4, + num="3.7.4.3", + ), + Heading(name="Window Functions", level=2, num="3.8"), + Heading(name="Nonaggregate Functions", level=3, num="3.8.1"), + Heading(name="The `first_value(expr)` Function", level=4, num="3.8.1.1"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.FirstValue", + level=5, + num="3.8.1.1.1", + ), + Heading(name="The `last_value(expr)` Function", level=4, num="3.8.1.2"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.LastValue", + level=5, + num="3.8.1.2.1", + ), + Heading( + name="The `lag(value, offset)` Function Workaround", level=4, num="3.8.1.3" + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Lag.Workaround", + level=5, + num="3.8.1.3.1", + ), + Heading( + name="The `lead(value, offset)` Function Workaround", level=4, num="3.8.1.4" + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Lead.Workaround", + level=5, + num="3.8.1.4.1", + ), + Heading( + name="The `leadInFrame(expr[, offset, [default]])`", level=4, num="3.8.1.5" + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.LeadInFrame", + level=5, + num="3.8.1.5.1", + ), + Heading( + name="The `lagInFrame(expr[, offset, [default]])`", level=4, num="3.8.1.6" + ), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.LagInFrame", + level=5, + num="3.8.1.6.1", + ), + Heading(name="The `rank()` Function", level=4, num="3.8.1.7"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.Rank", level=5, num="3.8.1.7.1" + ), + Heading(name="The `dense_rank()` Function", level=4, num="3.8.1.8"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.DenseRank", + level=5, + num="3.8.1.8.1", + ), + Heading(name="The `row_number()` Function", level=4, num="3.8.1.9"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.RowNumber", + level=5, + num="3.8.1.9.1", + ), + Heading(name="Aggregate Functions", level=3, num="3.8.2"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions", + level=4, + num="3.8.2.1", + ), + Heading(name="Combinators", level=4, num="3.8.2.2"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Combinators", + level=5, + num="3.8.2.2.1", + ), + Heading(name="Parametric", level=4, num="3.8.2.3"), + Heading( + name="RQ.SRS-019.ClickHouse.WindowFunctions.AggregateFunctions.Parametric", + level=5, + num="3.8.2.3.1", + ), + Heading(name="References", level=1, num="4"), + ), requirements=( RQ_SRS_019_ClickHouse_WindowFunctions, RQ_SRS_019_ClickHouse_WindowFunctions_NonDistributedTables, @@ -3677,8 +4344,8 @@ SRS019_ClickHouse_Window_Functions = Specification( RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions, RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions_Combinators, RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions_Parametric, - ), - content=''' + ), + content=""" # SRS019 ClickHouse Window Functions # Software Requirements Specification @@ -6037,4 +6704,5 @@ version: 1.0 [GitHub]: https://github.com [PostreSQL]: https://www.postgresql.org/docs/9.2/tutorial-window.html [MySQL]: https://dev.mysql.com/doc/refman/8.0/en/window-functions.html -''') +""", +) diff --git a/tests/testflows/window_functions/tests/aggregate_funcs.py b/tests/testflows/window_functions/tests/aggregate_funcs.py index 67a5f2cfb4f..faac8a84c49 100644 --- a/tests/testflows/window_functions/tests/aggregate_funcs.py +++ b/tests/testflows/window_functions/tests/aggregate_funcs.py @@ -4,109 +4,117 @@ from testflows.asserts import values, error, snapshot from window_functions.requirements import * from window_functions.tests.common import * + @TestOutline(Scenario) -@Examples("func", [ - ("count(salary)",), - ("min(salary)",), - ("max(salary)",), - ("sum(salary)",), - ("avg(salary)",), - ("any(salary)",), - ("stddevPop(salary)",), - ("stddevSamp(salary)",), - ("varPop(salary)",), - ("varSamp(salary)",), - ("covarPop(salary, 2000)",), - ("covarSamp(salary, 2000)",), - ("anyHeavy(salary)",), - ("anyLast(salary)",), - ("argMin(salary, 5000)",), - ("argMax(salary, 5000)",), - ("avgWeighted(salary, 1)",), - ("corr(salary, 0.5)",), - ("topK(salary)",), - ("topKWeighted(salary, 1)",), - ("groupArray(salary)",), - ("groupUniqArray(salary)",), - ("groupArrayInsertAt(salary, 0)",), - ("groupArrayMovingSum(salary)",), - ("groupArrayMovingAvg(salary)",), - ("groupArraySample(3, 1234)(salary)",), - ("groupBitAnd(toUInt8(salary))",), - ("groupBitOr(toUInt8(salary))",), - ("groupBitXor(toUInt8(salary))",), - ("groupBitmap(toUInt8(salary))",), - # #("groupBitmapAnd",), - # #("groupBitmapOr",), - # #("groupBitmapXor",), - ("sumWithOverflow(salary)",), - ("deltaSum(salary)",), - ("sumMap([5000], [salary])",), - ("minMap([5000], [salary])",), - ("maxMap([5000], [salary])",), - # #("initializeAggregation",), - ("skewPop(salary)",), - ("skewSamp(salary)",), - ("kurtPop(salary)",), - ("kurtSamp(salary)",), - ("uniq(salary)",), - ("uniqExact(salary)",), - ("uniqCombined(salary)",), - ("uniqCombined64(salary)",), - ("uniqHLL12(salary)",), - ("quantile(salary)",), - ("quantiles(0.5)(salary)",), - ("quantileExact(salary)",), - ("quantileExactWeighted(salary, 1)",), - ("quantileTiming(salary)",), - ("quantileTimingWeighted(salary, 1)",), - ("quantileDeterministic(salary, 1234)",), - ("quantileTDigest(salary)",), - ("quantileTDigestWeighted(salary, 1)",), - ("simpleLinearRegression(salary, empno)",), - ("stochasticLinearRegression(salary, 1)",), - ("stochasticLogisticRegression(salary, 1)",), - #("categoricalInformationValue(salary, 0)",), - ("studentTTest(salary, 1)",), - ("welchTTest(salary, 1)",), - ("mannWhitneyUTest(salary, 1)",), - ("median(salary)",), - ("rankCorr(salary, 0.5)",), -]) +@Examples( + "func", + [ + ("count(salary)",), + ("min(salary)",), + ("max(salary)",), + ("sum(salary)",), + ("avg(salary)",), + ("any(salary)",), + ("stddevPop(salary)",), + ("stddevSamp(salary)",), + ("varPop(salary)",), + ("varSamp(salary)",), + ("covarPop(salary, 2000)",), + ("covarSamp(salary, 2000)",), + ("anyHeavy(salary)",), + ("anyLast(salary)",), + ("argMin(salary, 5000)",), + ("argMax(salary, 5000)",), + ("avgWeighted(salary, 1)",), + ("corr(salary, 0.5)",), + ("topK(salary)",), + ("topKWeighted(salary, 1)",), + ("groupArray(salary)",), + ("groupUniqArray(salary)",), + ("groupArrayInsertAt(salary, 0)",), + ("groupArrayMovingSum(salary)",), + ("groupArrayMovingAvg(salary)",), + ("groupArraySample(3, 1234)(salary)",), + ("groupBitAnd(toUInt8(salary))",), + ("groupBitOr(toUInt8(salary))",), + ("groupBitXor(toUInt8(salary))",), + ("groupBitmap(toUInt8(salary))",), + # #("groupBitmapAnd",), + # #("groupBitmapOr",), + # #("groupBitmapXor",), + ("sumWithOverflow(salary)",), + ("deltaSum(salary)",), + ("sumMap([5000], [salary])",), + ("minMap([5000], [salary])",), + ("maxMap([5000], [salary])",), + # #("initializeAggregation",), + ("skewPop(salary)",), + ("skewSamp(salary)",), + ("kurtPop(salary)",), + ("kurtSamp(salary)",), + ("uniq(salary)",), + ("uniqExact(salary)",), + ("uniqCombined(salary)",), + ("uniqCombined64(salary)",), + ("uniqHLL12(salary)",), + ("quantile(salary)",), + ("quantiles(0.5)(salary)",), + ("quantileExact(salary)",), + ("quantileExactWeighted(salary, 1)",), + ("quantileTiming(salary)",), + ("quantileTimingWeighted(salary, 1)",), + ("quantileDeterministic(salary, 1234)",), + ("quantileTDigest(salary)",), + ("quantileTDigestWeighted(salary, 1)",), + ("simpleLinearRegression(salary, empno)",), + ("stochasticLinearRegression(salary, 1)",), + ("stochasticLogisticRegression(salary, 1)",), + # ("categoricalInformationValue(salary, 0)",), + ("studentTTest(salary, 1)",), + ("welchTTest(salary, 1)",), + ("mannWhitneyUTest(salary, 1)",), + ("median(salary)",), + ("rankCorr(salary, 0.5)",), + ], +) def aggregate_funcs_over_rows_frame(self, func): - """Checking aggregate funcs over rows frame. - """ - execute_query(f""" + """Checking aggregate funcs over rows frame.""" + execute_query( + f""" SELECT {func} OVER (ORDER BY salary, empno ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS func FROM empsalary """ ) + @TestScenario def avg_with_nulls(self): - """Check `avg` aggregate function using a window that contains NULLs. - """ - expected = convert_output(""" + """Check `avg` aggregate function using a window that contains NULLs.""" + expected = convert_output( + """ i | avg ---+-------------------- 1 | 1.5 2 | 2 3 | \\N 4 | \\N - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT i, avg(v) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS avg FROM values('i Int32, v Nullable(Int32)', (1,1),(2,2),(3,NULL),(4,NULL)) """, - expected=expected + expected=expected, ) + @TestScenario def var_pop(self): - """Check `var_pop` aggregate function ove a window. - """ - expected = convert_output(""" + """Check `var_pop` aggregate function ove a window.""" + expected = convert_output( + """ var_pop ----------------------- 21704 @@ -114,20 +122,23 @@ def var_pop(self): 11266.666666666666 4225 0 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT VAR_POP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS var_pop FROM values('i Int8, n Int32', (1,600),(2,470),(3,170),(4,430),(5,300)) """, - expected=expected + expected=expected, ) + @TestScenario def var_samp(self): - """Check `var_samp` aggregate function ove a window. - """ - expected = convert_output(""" + """Check `var_samp` aggregate function ove a window.""" + expected = convert_output( + """ var_samp ----------------------- 27130 @@ -135,20 +146,23 @@ def var_samp(self): 16900 8450 nan - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT VAR_SAMP(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS var_samp FROM VALUES('i Int8, n Int16',(1,600),(2,470),(3,170),(4,430),(5,300)) """, - expected=expected + expected=expected, ) + @TestScenario def stddevpop(self): - """Check `stddevPop` aggregate function ove a window. - """ - expected = convert_output(""" + """Check `stddevPop` aggregate function ove a window.""" + expected = convert_output( + """ stddev_pop --------------------- 147.32277488562318 @@ -157,20 +171,23 @@ def stddevpop(self): 106.14455552060438 65 0 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT stddevPop(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS stddev_pop FROM VALUES('i Int8, n Nullable(Int16)',(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) """, - expected=expected + expected=expected, ) + @TestScenario def stddevsamp(self): - """Check `stddevSamp` aggregate function ove a window. - """ - expected = convert_output(""" + """Check `stddevSamp` aggregate function ove a window.""" + expected = convert_output( + """ stddev_samp --------------------- 164.7118696390761 @@ -179,20 +196,23 @@ def stddevsamp(self): 130 91.92388155425118 nan - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT stddevSamp(n) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS stddev_samp FROM VALUES('i Int8, n Nullable(Int16)',(1,NULL),(2,600),(3,470),(4,170),(5,430),(6,300)) """, - expected=expected + expected=expected, ) + @TestScenario def aggregate_function_recovers_from_nan(self): - """Check that aggregate function can recover from `nan` value inside a window. - """ - expected = convert_output(""" + """Check that aggregate function can recover from `nan` value inside a window.""" + expected = convert_output( + """ a | b | sum ---+-----+----- 1 | 1 | 1 @@ -200,21 +220,24 @@ def aggregate_function_recovers_from_nan(self): 3 | nan | nan 4 | 3 | nan 5 | 4 | 7 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT a, b, SUM(b) OVER(ORDER BY a ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS sum FROM VALUES('a Int8, b Float64',(1,1),(2,2),(3,nan),(4,3),(5,4)) """, - expected=expected + expected=expected, ) + @TestScenario def bit_functions(self): - """Check trying to use bitwise functions over a window. - """ - expected = convert_output(""" + """Check trying to use bitwise functions over a window.""" + expected = convert_output( + """ i | b | bool_and | bool_or ---+---+----------+--------- 1 | 1 | 1 | 1 @@ -222,21 +245,24 @@ def bit_functions(self): 3 | 0 | 0 | 0 4 | 0 | 0 | 1 5 | 1 | 1 | 1 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT i, b, groupBitAnd(b) OVER w AS bool_and, groupBitOr(b) OVER w AS bool_or FROM VALUES('i Int8, b UInt8', (1,1), (2,1), (3,0), (4,0), (5,1)) WINDOW w AS (ORDER BY i ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) """, - expected=expected + expected=expected, ) + @TestScenario def sum(self): - """Check calculation of sum over a window. - """ - expected = convert_output(""" + """Check calculation of sum over a window.""" + expected = convert_output( + """ sum_1 | ten | four -------+-----+------ 0 | 0 | 0 @@ -249,18 +275,20 @@ def sum(self): 0 | 4 | 0 1 | 7 | 1 1 | 9 | 1 - """) + """ + ) execute_query( "SELECT sum(four) OVER (PARTITION BY ten ORDER BY unique2) AS sum_1, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario def nested_aggregates(self): - """Check using nested aggregates over a window. - """ - expected = convert_output(""" + """Check using nested aggregates over a window.""" + expected = convert_output( + """ ten | two | gsum | wsum -----+-----+-------+-------- 0 | 0 | 45000 | 45000 @@ -273,18 +301,20 @@ def nested_aggregates(self): 5 | 1 | 50000 | 144000 7 | 1 | 52000 | 196000 9 | 1 | 54000 | 250000 - """) + """ + ) execute_query( "SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER (PARTITION BY two ORDER BY ten) AS wsum FROM tenk1 GROUP BY ten, two", - expected=expected + expected=expected, ) + @TestScenario def aggregate_and_window_function_in_the_same_window(self): - """Check using aggregate and window function in the same window. - """ - expected = convert_output(""" + """Check using aggregate and window function in the same window.""" + expected = convert_output( + """ sum | rank -------+------ 6000 | 1 @@ -297,35 +327,36 @@ def aggregate_and_window_function_in_the_same_window(self): 5000 | 1 14600 | 2 14600 | 2 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum, rank() OVER w AS rank FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC)", - expected=expected + expected=expected, ) + @TestScenario def ungrouped_aggregate_over_empty_row_set(self): - """Check using window function with ungrouped aggregate over an empty row set. - """ - expected = convert_output(""" + """Check using window function with ungrouped aggregate over an empty row set.""" + expected = convert_output( + """ sum ----- 0 - """) + """ + ) execute_query( "SELECT SUM(COUNT(number)) OVER () AS sum FROM numbers(10) WHERE number=42", - expected=expected + expected=expected, ) + @TestFeature @Name("aggregate funcs") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_AggregateFunctions("1.0")) def feature(self): - """Check using aggregate functions over windows. - """ + """Check using aggregate functions over windows.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/common.py b/tests/testflows/window_functions/tests/common.py index 4f8b8081bf9..b0bca328e4d 100644 --- a/tests/testflows/window_functions/tests/common.py +++ b/tests/testflows/window_functions/tests/common.py @@ -8,52 +8,77 @@ from testflows.core.name import basename, parentname from testflows._core.testtype import TestSubType from testflows.asserts import values, error, snapshot + def window_frame_error(): return (36, "Exception: Window frame") + def frame_start_error(): return (36, "Exception: Frame start") + def frame_end_error(): return (36, "Exception: Frame end") + def frame_offset_nonnegative_error(): return syntax_error() + def frame_end_unbounded_preceding_error(): return (36, "Exception: Frame end cannot be UNBOUNDED PRECEDING") + def frame_range_offset_error(): return (48, "Exception: The RANGE OFFSET frame") + def frame_requires_order_by_error(): - return (36, "Exception: The RANGE OFFSET window frame requires exactly one ORDER BY column, 0 given") + return ( + 36, + "Exception: The RANGE OFFSET window frame requires exactly one ORDER BY column, 0 given", + ) + def syntax_error(): return (62, "Exception: Syntax error") + def groups_frame_error(): return (48, "Exception: Window frame 'Groups' is not implemented") + def getuid(): if current().subtype == TestSubType.Example: - testname = f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + testname = ( + f"{basename(parentname(current().name)).replace(' ', '_').replace(',','')}" + ) else: testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" - return testname + "_" + str(uuid.uuid1()).replace('-', '_') + return testname + "_" + str(uuid.uuid1()).replace("-", "_") + def convert_output(s): - """Convert expected output to TSV format. - """ - return '\n'.join([l.strip() for i, l in enumerate(re.sub('\s+\|\s+', '\t', s).strip().splitlines()) if i != 1]) + """Convert expected output to TSV format.""" + return "\n".join( + [ + l.strip() + for i, l in enumerate(re.sub("\s+\|\s+", "\t", s).strip().splitlines()) + if i != 1 + ] + ) -def execute_query(sql, expected=None, exitcode=None, message=None, format="TabSeparatedWithNames"): - """Execute SQL query and compare the output to the snapshot. - """ + +def execute_query( + sql, expected=None, exitcode=None, message=None, format="TabSeparatedWithNames" +): + """Execute SQL query and compare the output to the snapshot.""" name = basename(current().name) with When("I execute query", description=sql): - r = current().context.node.query(sql + " FORMAT " + format, exitcode=exitcode, message=message) + r = current().context.node.query( + sql + " FORMAT " + format, exitcode=exitcode, message=message + ) if message is None: if expected is not None: @@ -62,18 +87,21 @@ def execute_query(sql, expected=None, exitcode=None, message=None, format="TabSe else: with Then("I check output against snapshot"): with values() as that: - assert that(snapshot("\n" + r.output.strip() + "\n", "tests", name=name, encoder=str)), error() + assert that( + snapshot( + "\n" + r.output.strip() + "\n", + "tests", + name=name, + encoder=str, + ) + ), error() + @TestStep(Given) def t1_table(self, name="t1", distributed=False): - """Create t1 table. - """ + """Create t1 table.""" table = None - data = [ - "(1, 1)", - "(1, 2)", - "(2, 2)" - ] + data = ["(1, 1)", "(1, 2)", "(2, 2)"] if not distributed: with By("creating table"): @@ -97,10 +125,18 @@ def t1_table(self, name="t1", distributed=False): f2 Int8 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') ORDER BY tuple() """ - create_table(name=name + "_source", statement=sql, on_cluster="sharded_cluster") + create_table( + name=name + "_source", statement=sql, on_cluster="sharded_cluster" + ) with And("a distributed table"): - sql = "CREATE TABLE {name} AS " + name + '_source' + " ENGINE = Distributed(sharded_cluster, default, " + f"{name + '_source'}, f1 % toUInt8(getMacro('shard')))" + sql = ( + "CREATE TABLE {name} AS " + + name + + "_source" + + " ENGINE = Distributed(sharded_cluster, default, " + + f"{name + '_source'}, f1 % toUInt8(getMacro('shard')))" + ) table = create_table(name=name, statement=sql) with And("populating table with data"): @@ -110,10 +146,10 @@ def t1_table(self, name="t1", distributed=False): return table + @TestStep(Given) def datetimes_table(self, name="datetimes", distributed=False): - """Create datetimes table. - """ + """Create datetimes table.""" table = None data = [ "(1, '2000-10-19 10:23:54', '2000-10-19 10:23:54')", @@ -125,7 +161,7 @@ def datetimes_table(self, name="datetimes", distributed=False): "(7, '2005-10-19 10:23:54', '2005-10-19 10:23:54')", "(8, '2006-10-19 10:23:54', '2006-10-19 10:23:54')", "(9, '2007-10-19 10:23:54', '2007-10-19 10:23:54')", - "(10, '2008-10-19 10:23:54', '2008-10-19 10:23:54')" + "(10, '2008-10-19 10:23:54', '2008-10-19 10:23:54')", ] if not distributed: @@ -152,10 +188,18 @@ def datetimes_table(self, name="datetimes", distributed=False): f_timestamp DateTime ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') ORDER BY tuple() """ - create_table(name=name + "_source", statement=sql, on_cluster="sharded_cluster") + create_table( + name=name + "_source", statement=sql, on_cluster="sharded_cluster" + ) with And("a distributed table"): - sql = "CREATE TABLE {name} AS " + name + '_source' + " ENGINE = Distributed(sharded_cluster, default, " + f"{name + '_source'}, id % toUInt8(getMacro('shard')))" + sql = ( + "CREATE TABLE {name} AS " + + name + + "_source" + + " ENGINE = Distributed(sharded_cluster, default, " + + f"{name + '_source'}, id % toUInt8(getMacro('shard')))" + ) table = create_table(name=name, statement=sql) with And("populating table with data"): @@ -165,10 +209,10 @@ def datetimes_table(self, name="datetimes", distributed=False): return table + @TestStep(Given) def numerics_table(self, name="numerics", distributed=False): - """Create numerics tables. - """ + """Create numerics tables.""" table = None data = [ @@ -181,7 +225,7 @@ def numerics_table(self, name="numerics", distributed=False): "(6, 2, 2, 2)", "(7, 100, 100, 100)", "(8, 'infinity', 'infinity', toDecimal64(1000,15))", - "(9, 'NaN', 'NaN', 0)" + "(9, 'NaN', 'NaN', 0)", ] if not distributed: @@ -210,10 +254,18 @@ def numerics_table(self, name="numerics", distributed=False): f_numeric Decimal64(15) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') ORDER BY tuple(); """ - create_table(name=name + "_source", statement=sql, on_cluster="sharded_cluster") + create_table( + name=name + "_source", statement=sql, on_cluster="sharded_cluster" + ) with And("a distributed table"): - sql = "CREATE TABLE {name} AS " + name + '_source' + " ENGINE = Distributed(sharded_cluster, default, " + f"{name + '_source'}, id % toUInt8(getMacro('shard')))" + sql = ( + "CREATE TABLE {name} AS " + + name + + "_source" + + " ENGINE = Distributed(sharded_cluster, default, " + + f"{name + '_source'}, id % toUInt8(getMacro('shard')))" + ) table = create_table(name=name, statement=sql) with And("populating table with data"): @@ -223,10 +275,10 @@ def numerics_table(self, name="numerics", distributed=False): return table + @TestStep(Given) def tenk1_table(self, name="tenk1", distributed=False): - """Create tenk1 table. - """ + """Create tenk1 table.""" table = None if not distributed: @@ -256,7 +308,11 @@ def tenk1_table(self, name="tenk1", distributed=False): with And("populating table with data"): datafile = os.path.join(current_dir(), "tenk.data") debug(datafile) - self.context.cluster.command(None, f"cat \"{datafile}\" | {self.context.node.cluster.docker_compose} exec -T {self.context.node.name} clickhouse client -q \"INSERT INTO {name} FORMAT TSV\"", exitcode=0) + self.context.cluster.command( + None, + f'cat "{datafile}" | {self.context.node.cluster.docker_compose} exec -T {self.context.node.name} clickhouse client -q "INSERT INTO {name} FORMAT TSV"', + exitcode=0, + ) else: with By("creating a table"): sql = """ @@ -279,10 +335,18 @@ def tenk1_table(self, name="tenk1", distributed=False): string4 String ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') ORDER BY tuple() """ - create_table(name=name + '_source', statement=sql, on_cluster="sharded_cluster") + create_table( + name=name + "_source", statement=sql, on_cluster="sharded_cluster" + ) with And("a distributed table"): - sql = "CREATE TABLE {name} AS " + name + '_source' + " ENGINE = Distributed(sharded_cluster, default, " + f"{name + '_source'}, unique1 % toUInt8(getMacro('shard')))" + sql = ( + "CREATE TABLE {name} AS " + + name + + "_source" + + " ENGINE = Distributed(sharded_cluster, default, " + + f"{name + '_source'}, unique1 % toUInt8(getMacro('shard')))" + ) table = create_table(name=name, statement=sql) with And("populating table with data"): @@ -291,22 +355,24 @@ def tenk1_table(self, name="tenk1", distributed=False): with open(datafile, "r") as file: lines = file.readlines() - chunks = [lines[i:i + 1000] for i in range(0, len(lines), 1000)] + chunks = [lines[i : i + 1000] for i in range(0, len(lines), 1000)] for chunk in chunks: with tempfile.NamedTemporaryFile() as file: - file.write(''.join(chunk).encode("utf-8")) + file.write("".join(chunk).encode("utf-8")) file.flush() - self.context.cluster.command(None, - f"cat \"{file.name}\" | {self.context.node.cluster.docker_compose} exec -T {self.context.node.name} clickhouse client -q \"INSERT INTO {table} FORMAT TSV\"", - exitcode=0) + self.context.cluster.command( + None, + f'cat "{file.name}" | {self.context.node.cluster.docker_compose} exec -T {self.context.node.name} clickhouse client -q "INSERT INTO {table} FORMAT TSV"', + exitcode=0, + ) return table + @TestStep(Given) def empsalary_table(self, name="empsalary", distributed=False): - """Create employee salary reference table. - """ + """Create employee salary reference table.""" table = None data = [ @@ -319,7 +385,7 @@ def empsalary_table(self, name="empsalary", distributed=False): "('develop', 9, 4500, '2008-01-01')", "('sales', 3, 4800, '2007-08-01')", "('develop', 8, 6000, '2006-10-01')", - "('develop', 11, 5200, '2007-08-15')" + "('develop', 11, 5200, '2007-08-15')", ] if not distributed: @@ -341,7 +407,7 @@ def empsalary_table(self, name="empsalary", distributed=False): else: with By("creating replicated source tables"): - sql = """ + sql = """ CREATE TABLE {name} ON CLUSTER sharded_cluster ( depname LowCardinality(String), empno UInt64, @@ -350,27 +416,40 @@ def empsalary_table(self, name="empsalary", distributed=False): ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') ORDER BY enroll_date """ - create_table(name=name + "_source", statement=sql, on_cluster="sharded_cluster") + create_table( + name=name + "_source", statement=sql, on_cluster="sharded_cluster" + ) with And("a distributed table"): - sql = "CREATE TABLE {name} AS " + name + '_source' + " ENGINE = Distributed(sharded_cluster, default, " + f"{name + '_source'}, empno % toUInt8(getMacro('shard')))" + sql = ( + "CREATE TABLE {name} AS " + + name + + "_source" + + " ENGINE = Distributed(sharded_cluster, default, " + + f"{name + '_source'}, empno % toUInt8(getMacro('shard')))" + ) table = create_table(name=name, statement=sql) with And("populating distributed table with data"): - with By("inserting one data row at a time", description="so that data is sharded between nodes"): + with By( + "inserting one data row at a time", + description="so that data is sharded between nodes", + ): for row in data: - self.context.node.query(f"INSERT INTO {table} VALUES {row}", - settings=[("insert_distributed_sync", "1")]) + self.context.node.query( + f"INSERT INTO {table} VALUES {row}", + settings=[("insert_distributed_sync", "1")], + ) with And("dumping all the data in the table"): self.context.node.query(f"SELECT * FROM {table}") return table + @TestStep(Given) def create_table(self, name, statement, on_cluster=False): - """Create table. - """ + """Create table.""" node = current().context.node try: with Given(f"I have a {name} table"): diff --git a/tests/testflows/window_functions/tests/errors.py b/tests/testflows/window_functions/tests/errors.py index d7b80ed7cd8..ee9452eecba 100644 --- a/tests/testflows/window_functions/tests/errors.py +++ b/tests/testflows/window_functions/tests/errors.py @@ -3,6 +3,7 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def error_using_non_window_function(self): """Check that trying to use non window or aggregate function over a window @@ -11,87 +12,87 @@ def error_using_non_window_function(self): exitcode = 63 message = "DB::Exception: Unknown aggregate function numbers" - sql = ("SELECT numbers(1, 100) OVER () FROM empsalary") + sql = "SELECT numbers(1, 100) OVER () FROM empsalary" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_order_by_another_window_function(self): - """Check that trying to order by another window function returns an error. - """ + """Check that trying to order by another window function returns an error.""" exitcode = 184 message = "DB::Exception: Window function rank() OVER (ORDER BY random() ASC) is found inside window definition in query" - sql = ("SELECT rank() OVER (ORDER BY rank() OVER (ORDER BY random()))") + sql = "SELECT rank() OVER (ORDER BY rank() OVER (ORDER BY random()))" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_window_function_in_where(self): - """Check that trying to use window function in `WHERE` returns an error. - """ + """Check that trying to use window function in `WHERE` returns an error.""" exitcode = 184 message = "DB::Exception: Window function row_number() OVER (ORDER BY salary ASC) is found in WHERE in query" - sql = ("SELECT * FROM empsalary WHERE row_number() OVER (ORDER BY salary) < 10") + sql = "SELECT * FROM empsalary WHERE row_number() OVER (ORDER BY salary) < 10" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_window_function_in_join(self): - """Check that trying to use window function in `JOIN` returns an error. - """ + """Check that trying to use window function in `JOIN` returns an error.""" exitcode = 147 message = "DB::Exception: Cannot get JOIN keys from JOIN ON section: row_number() OVER (ORDER BY salary ASC) < 10" - sql = ("SELECT * FROM empsalary INNER JOIN tenk1 ON row_number() OVER (ORDER BY salary) < 10") + sql = "SELECT * FROM empsalary INNER JOIN tenk1 ON row_number() OVER (ORDER BY salary) < 10" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_window_function_in_group_by(self): - """Check that trying to use window function in `GROUP BY` returns an error. - """ + """Check that trying to use window function in `GROUP BY` returns an error.""" exitcode = 47 message = "DB::Exception: Unknown identifier: row_number() OVER (ORDER BY salary ASC); there are columns" - sql = ("SELECT rank() OVER (ORDER BY 1), count(*) FROM empsalary GROUP BY row_number() OVER (ORDER BY salary) < 10") + sql = "SELECT rank() OVER (ORDER BY 1), count(*) FROM empsalary GROUP BY row_number() OVER (ORDER BY salary) < 10" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_window_function_in_having(self): - """Check that trying to use window function in `HAVING` returns an error. - """ + """Check that trying to use window function in `HAVING` returns an error.""" exitcode = 184 message = "DB::Exception: Window function row_number() OVER (ORDER BY salary ASC) is found in HAVING in query" - sql = ("SELECT rank() OVER (ORDER BY 1), count(*) FROM empsalary GROUP BY salary HAVING row_number() OVER (ORDER BY salary) < 10") + sql = "SELECT rank() OVER (ORDER BY 1), count(*) FROM empsalary GROUP BY salary HAVING row_number() OVER (ORDER BY salary) < 10" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_select_from_window(self): - """Check that trying to use window function in `FROM` returns an error. - """ + """Check that trying to use window function in `FROM` returns an error.""" exitcode = 46 message = "DB::Exception: Unknown table function rank" - sql = ("SELECT * FROM rank() OVER (ORDER BY random())") + sql = "SELECT * FROM rank() OVER (ORDER BY random())" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_window_function_in_alter_delete_where(self): - """Check that trying to use window function in `ALTER DELETE`'s `WHERE` clause returns an error. - """ + """Check that trying to use window function in `ALTER DELETE`'s `WHERE` clause returns an error.""" if self.context.distributed: exitcode = 48 message = "Exception: Table engine Distributed doesn't support mutations" @@ -99,39 +100,39 @@ def error_window_function_in_alter_delete_where(self): exitcode = 184 message = "DB::Exception: Window function rank() OVER (ORDER BY random() ASC) is found in WHERE in query" - sql = ("ALTER TABLE empsalary DELETE WHERE (rank() OVER (ORDER BY random())) > 10") + sql = "ALTER TABLE empsalary DELETE WHERE (rank() OVER (ORDER BY random())) > 10" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_named_window_defined_twice(self): - """Check that trying to define named window twice. - """ + """Check that trying to define named window twice.""" exitcode = 36 message = "DB::Exception: Window 'w' is defined twice in the WINDOW clause" - sql = ("SELECT count(*) OVER w FROM tenk1 WINDOW w AS (ORDER BY unique1), w AS (ORDER BY unique1)") + sql = "SELECT count(*) OVER w FROM tenk1 WINDOW w AS (ORDER BY unique1), w AS (ORDER BY unique1)" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_coma_between_partition_by_and_order_by_clause(self): - """Check that trying to use a coma between partition by and order by clause. - """ + """Check that trying to use a coma between partition by and order by clause.""" exitcode = 62 message = "DB::Exception: Syntax error" - sql = ("SELECT rank() OVER (PARTITION BY four, ORDER BY ten) FROM tenk1") + sql = "SELECT rank() OVER (PARTITION BY four, ORDER BY ten) FROM tenk1" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestFeature @Name("errors") def feature(self): - """Check different error conditions. - """ + """Check different error conditions.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/feature.py b/tests/testflows/window_functions/tests/feature.py index f6c565d116b..c1454a419e9 100755 --- a/tests/testflows/window_functions/tests/feature.py +++ b/tests/testflows/window_functions/tests/feature.py @@ -6,10 +6,25 @@ from window_functions.requirements import * @TestOutline(Feature) @Name("tests") -@Examples("distributed", [ - (False, Name("non distributed"),Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_NonDistributedTables("1.0"))), - (True, Name("distributed"), Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_DistributedTables("1.0"))) -]) +@Examples( + "distributed", + [ + ( + False, + Name("non distributed"), + Requirements( + RQ_SRS_019_ClickHouse_WindowFunctions_NonDistributedTables("1.0") + ), + ), + ( + True, + Name("distributed"), + Requirements( + RQ_SRS_019_ClickHouse_WindowFunctions_DistributedTables("1.0") + ), + ), + ], +) def feature(self, distributed, node="clickhouse1"): """Check window functions behavior using non-distributed or distributed tables. diff --git a/tests/testflows/window_functions/tests/frame_clause.py b/tests/testflows/window_functions/tests/frame_clause.py index 9c15ace286d..e2ae136ac20 100644 --- a/tests/testflows/window_functions/tests/frame_clause.py +++ b/tests/testflows/window_functions/tests/frame_clause.py @@ -3,15 +3,14 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestFeature -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_FrameClause_DefaultFrame("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_FrameClause_DefaultFrame("1.0")) def default_frame(self): - """Check default frame. - """ + """Check default frame.""" with Scenario("with order by"): - expected = convert_output(""" + expected = convert_output( + """ number | sum ---------+------ 1 | 2 @@ -19,15 +18,17 @@ def default_frame(self): 2 | 4 3 | 10 3 | 10 - """) + """ + ) execute_query( "select number, sum(number) OVER (ORDER BY number) AS sum FROM values('number Int8', (1),(1),(2),(3),(3))", - expected=expected + expected=expected, ) with Scenario("without order by"): - expected = convert_output(""" + expected = convert_output( + """ number | sum ---------+------ 1 | 10 @@ -35,13 +36,15 @@ def default_frame(self): 2 | 10 3 | 10 3 | 10 - """) + """ + ) execute_query( "select number, sum(number) OVER () AS sum FROM values('number Int8', (1),(1),(2),(3),(3))", - expected=expected + expected=expected, ) + @TestFeature @Name("frame clause") @Requirements( @@ -56,11 +59,10 @@ def default_frame(self): RQ_SRS_019_ClickHouse_WindowFunctions_ExprPreceding("1.0"), RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing("1.0"), RQ_SRS_019_ClickHouse_WindowFunctions_ExprPreceding_ExprValue("1.0"), - RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing_ExprValue("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing_ExprValue("1.0"), ) def feature(self): - """Check defining frame clause. - """ + """Check defining frame clause.""" Feature(run=default_frame, flags=TE) Feature(run=load("window_functions.tests.rows_frame", "feature"), flags=TE) Feature(run=load("window_functions.tests.range_frame", "feature"), flags=TE) diff --git a/tests/testflows/window_functions/tests/funcs.py b/tests/testflows/window_functions/tests/funcs.py index 4526e6c9c4a..7060aed9e51 100644 --- a/tests/testflows/window_functions/tests/funcs.py +++ b/tests/testflows/window_functions/tests/funcs.py @@ -3,14 +3,13 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_FirstValue("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_FirstValue("1.0")) def first_value(self): - """Check `first_value` function. - """ - expected = convert_output(""" + """Check `first_value` function.""" + expected = convert_output( + """ first_value | ten | four -------------+-----+------ 0 | 0 | 0 @@ -23,31 +22,34 @@ def first_value(self): 0 | 0 | 2 1 | 1 | 3 1 | 3 | 3 - """) + """ + ) with Example("using first_value"): execute_query( "SELECT first_value(ten) OVER (PARTITION BY four ORDER BY ten) AS first_value, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Example("using any equivalent"): execute_query( "SELECT any(ten) OVER (PARTITION BY four ORDER BY ten) AS first_value, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_LastValue("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_LastValue("1.0")) def last_value(self): - """Check `last_value` function. - """ - with Example("order by window", description=""" + """Check `last_value` function.""" + with Example( + "order by window", + description=""" Check that last_value returns the last row of the frame that is CURRENT ROW in ORDER BY window - """): - expected = convert_output(""" + """, + ): + expected = convert_output( + """ last_value | ten | four ------------+-----+------ 0 | 0 | 0 @@ -60,24 +62,29 @@ def last_value(self): 0 | 4 | 0 1 | 7 | 1 1 | 9 | 1 - """) + """ + ) with Check("using last_value"): execute_query( "SELECT last_value(four) OVER (ORDER BY ten, four) AS last_value, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Check("using anyLast() equivalent"): execute_query( "SELECT anyLast(four) OVER (ORDER BY ten, four) AS last_value, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) - with Example("partition by window", description=""" + with Example( + "partition by window", + description=""" Check that last_value returns the last row of the frame that is CURRENT ROW in ORDER BY window - """): - expected = convert_output(""" + """, + ): + expected = convert_output( + """ last_value | ten | four ------------+-----+------ 4 | 0 | 0 @@ -90,14 +97,15 @@ def last_value(self): 0 | 0 | 2 3 | 1 | 3 3 | 3 | 3 - """) + """ + ) with Check("using last_value"): execute_query( """SELECT last_value(ten) OVER (PARTITION BY four) AS last_value, ten, four FROM (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten) ORDER BY four, ten""", - expected=expected + expected=expected, ) with Check("using anyLast() equivalent"): @@ -105,18 +113,17 @@ def last_value(self): """SELECT anyLast(ten) OVER (PARTITION BY four) AS last_value, ten, four FROM (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten) ORDER BY four, ten""", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_Lag_Workaround("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_Lag_Workaround("1.0")) def lag(self): - """Check `lag` function workaround. - """ + """Check `lag` function workaround.""" with Example("anyOrNull"): - expected = convert_output(""" + expected = convert_output( + """ lag | ten | four -----+-----+------ \\N | 0 | 0 @@ -129,15 +136,17 @@ def lag(self): \\N | 0 | 2 \\N | 1 | 3 1 | 3 | 3 - """) + """ + ) execute_query( "SELECT anyOrNull(ten) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS lag , ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Example("any"): - expected = convert_output(""" + expected = convert_output( + """ lag | ten | four -----+-----+------ 0 | 0 | 0 @@ -150,15 +159,17 @@ def lag(self): 0 | 0 | 2 0 | 1 | 3 1 | 3 | 3 - """) + """ + ) execute_query( "SELECT any(ten) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS lag , ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Example("anyOrNull with column value as offset"): - expected = convert_output(""" + expected = convert_output( + """ lag | ten | four -----+-----+------ 0 | 0 | 0 @@ -171,22 +182,22 @@ def lag(self): \\N | 0 | 2 \\N | 1 | 3 \\N | 3 | 3 - """) + """ + ) execute_query( "SELECT any(ten) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN four PRECEDING AND four PRECEDING) AS lag , ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_Lead_Workaround("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_Lead_Workaround("1.0")) def lead(self): - """Check `lead` function workaround. - """ + """Check `lead` function workaround.""" with Example("anyOrNull"): - expected = convert_output(""" + expected = convert_output( + """ lead | ten | four ------+-----+------ 0 | 0 | 0 @@ -199,15 +210,17 @@ def lead(self): \\N | 0 | 2 3 | 1 | 3 \\N | 3 | 3 - """) + """ + ) execute_query( "SELECT anyOrNull(ten) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) AS lead, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Example("any"): - expected = convert_output(""" + expected = convert_output( + """ lead | ten | four ------+-----+------ 0 | 0 | 0 @@ -220,15 +233,17 @@ def lead(self): 0 | 0 | 2 3 | 1 | 3 0 | 3 | 3 - """) + """ + ) execute_query( "SELECT any(ten) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) AS lead, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Example("any with arithmetic expr"): - expected = convert_output(""" + expected = convert_output( + """ lead | ten | four ------+-----+------ 0 | 0 | 0 @@ -241,15 +256,17 @@ def lead(self): 0 | 0 | 2 6 | 1 | 3 0 | 3 | 3 - """) + """ + ) execute_query( "SELECT any(ten * 2) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) AS lead, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) with Example("subquery as offset"): - expected = convert_output(""" + expected = convert_output( + """ lead ------ 0 @@ -262,22 +279,22 @@ def lead(self): 0 3 \\N - """) + """ + ) execute_query( "SELECT anyNull(ten) OVER (PARTITION BY four ORDER BY ten ROWS BETWEEN (SELECT two FROM tenk1 WHERE unique2 = unique2) FOLLOWING AND (SELECT two FROM tenk1 WHERE unique2 = unique2) FOLLOWING) AS lead " "FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowNumber("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_RowNumber("1.0")) def row_number(self): - """Check `row_number` function. - """ - expected = convert_output(""" + """Check `row_number` function.""" + expected = convert_output( + """ row_number ------------ 1 @@ -290,21 +307,21 @@ def row_number(self): 8 9 10 - """) + """ + ) execute_query( "SELECT row_number() OVER (ORDER BY unique2) AS row_number FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_Rank("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_Rank("1.0")) def rank(self): - """Check `rank` function. - """ - expected = convert_output(""" + """Check `rank` function.""" + expected = convert_output( + """ rank_1 | ten | four --------+-----+------ 1 | 0 | 0 @@ -317,21 +334,21 @@ def rank(self): 1 | 0 | 2 1 | 1 | 3 2 | 3 | 3 - """) + """ + ) execute_query( "SELECT rank() OVER (PARTITION BY four ORDER BY ten) AS rank_1, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_DenseRank("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_DenseRank("1.0")) def dense_rank(self): - """Check `dense_rank` function. - """ - expected = convert_output(""" + """Check `dense_rank` function.""" + expected = convert_output( + """ dense_rank | ten | four ------------+-----+------ 1 | 0 | 0 @@ -344,18 +361,20 @@ def dense_rank(self): 1 | 0 | 2 1 | 1 | 3 2 | 3 | 3 - """) + """ + ) execute_query( "SELECT dense_rank() OVER (PARTITION BY four ORDER BY ten) AS dense_rank, ten, four FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario def last_value_with_no_frame(self): - """Check last_value function with no frame. - """ - expected = convert_output(""" + """Check last_value function with no frame.""" + expected = convert_output( + """ four | ten | sum | last_value ------+-----+-----+------------ 0 | 0 | 0 | 0 @@ -378,24 +397,26 @@ def last_value_with_no_frame(self): 3 | 5 | 9 | 5 3 | 7 | 16 | 7 3 | 9 | 25 | 9 - """) + """ + ) execute_query( "SELECT four, ten, sum(ten) over (partition by four order by ten) AS sum, " "last_value(ten) over (partition by four order by ten) AS last_value " "FROM (select distinct ten, four from tenk1)", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_LastValue("1.0"), RQ_SRS_019_ClickHouse_WindowFunctions_Lag_Workaround("1.0"), ) def last_value_with_lag_workaround(self): - """Check last value with lag workaround. - """ - expected = convert_output(""" + """Check last value with lag workaround.""" + expected = convert_output( + """ last_value | lag | salary ------------+------+-------- 4500 | 0 | 3500 @@ -408,24 +429,26 @@ def last_value_with_lag_workaround(self): 6000 | 5000 | 5200 6000 | 5200 | 5200 6000 | 5200 | 6000 - """) + """ + ) execute_query( "select last_value(salary) over(order by salary range between 1000 preceding and 1000 following) AS last_value, " "any(salary) over(order by salary rows between 1 preceding and 1 preceding) AS lag, " "salary from empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_FirstValue("1.0"), - RQ_SRS_019_ClickHouse_WindowFunctions_Lead_Workaround("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_Lead_Workaround("1.0"), ) def first_value_with_lead_workaround(self): - """Check first value with lead workaround. - """ - expected = convert_output(""" + """Check first value with lead workaround.""" + expected = convert_output( + """ first_value | lead | salary -------------+------+-------- 3500 | 3900 | 3500 @@ -438,24 +461,24 @@ def first_value_with_lead_workaround(self): 4200 | 5200 | 5200 4200 | 6000 | 5200 5000 | 0 | 6000 - """) + """ + ) execute_query( "select first_value(salary) over(order by salary range between 1000 preceding and 1000 following) AS first_value, " "any(salary) over(order by salary rows between 1 following and 1 following) AS lead," "salary from empsalary", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_LeadInFrame("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_LeadInFrame("1.0")) def leadInFrame(self): - """Check `leadInFrame` function. - """ + """Check `leadInFrame` function.""" with Example("non default offset"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lead --------+--------+------- 1 | 5000 | 5000 @@ -468,15 +491,17 @@ def leadInFrame(self): 9 | 4500 | 4500 10 | 5200 | 5200 11 | 5200 | 5200 - """) + """ + ) execute_query( "select empno, salary, leadInFrame(salary,0) OVER (ORDER BY salary) AS lead FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) with Example("default offset"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lead --------+--------+------- 1 | 5000 | 0 @@ -489,15 +514,17 @@ def leadInFrame(self): 9 | 4500 | 0 10 | 5200 | 5200 11 | 5200 | 0 - """) + """ + ) execute_query( "select empno, salary, leadInFrame(salary) OVER (ORDER BY salary) AS lead FROM (SELECT * FROM empsalary ORDER BY empno)", - expected=expected + expected=expected, ) with Example("explicit default value"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lead --------+--------+------- 1 | 5000 | 8 @@ -510,15 +537,17 @@ def leadInFrame(self): 9 | 4500 | 8 10 | 5200 | 5200 11 | 5200 | 8 - """) + """ + ) execute_query( "select empno, salary, leadInFrame(salary,1,8) OVER (ORDER BY salary) AS lead FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) with Example("without order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lead --------+--------+------- 1 | 5000 | 3900 @@ -531,15 +560,17 @@ def leadInFrame(self): 9 | 4500 | 5200 10 | 5200 | 5200 11 | 5200 | 0 - """) + """ + ) execute_query( "select empno, salary, leadInFrame(salary) OVER () AS lead FROM (SELECT * FROM empsalary ORDER BY empno)", - expected=expected + expected=expected, ) with Example("with nulls"): - expected = convert_output(""" + expected = convert_output( + """ number | lead --------+----- 1 | 1 @@ -547,22 +578,22 @@ def leadInFrame(self): 2 | 3 3 | 0 \\N | 0 - """) + """ + ) execute_query( "select number, leadInFrame(number,1,0) OVER () AS lead FROM values('number Nullable(Int8)', (1),(1),(2),(3),(NULL))", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_LagInFrame("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_LagInFrame("1.0")) def lagInFrame(self): - """Check `lagInFrame` function. - """ + """Check `lagInFrame` function.""" with Example("non default offset"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lag --------+--------+------- 1 | 5000 | 5000 @@ -575,15 +606,17 @@ def lagInFrame(self): 9 | 4500 | 4500 10 | 5200 | 5200 11 | 5200 | 5200 - """) + """ + ) execute_query( "select empno, salary, lagInFrame(salary,0) OVER (ORDER BY salary) AS lag FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) with Example("default offset"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lag --------+--------+------- 5 | 3500 | 0 @@ -596,15 +629,17 @@ def lagInFrame(self): 10 | 5200 | 5000 11 | 5200 | 5200 8 | 6000 | 5200 - """) + """ + ) execute_query( "select empno, salary, lagInFrame(salary) OVER (ORDER BY salary) AS lag FROM (SELECT * FROM empsalary ORDER BY empno)", - expected=expected + expected=expected, ) with Example("explicit default value"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lag --------+--------+------- 1 | 5000 | 4800 @@ -617,15 +652,17 @@ def lagInFrame(self): 9 | 4500 | 4200 10 | 5200 | 5000 11 | 5200 | 5200 - """) + """ + ) execute_query( "select empno, salary, lagInFrame(salary,1,8) OVER (ORDER BY salary) AS lag FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) with Example("without order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | salary | lag --------+--------+------- 1 | 5000 | 0 @@ -638,15 +675,17 @@ def lagInFrame(self): 9 | 4500 | 6000 10 | 5200 | 4500 11 | 5200 | 5200 - """) + """ + ) execute_query( "select empno, salary, lagInFrame(salary) OVER () AS lag FROM (SELECT * FROM empsalary ORDER BY empno)", - expected=expected + expected=expected, ) with Example("with nulls"): - expected = convert_output(""" + expected = convert_output( + """ number | lag --------+----- 1 | 0 @@ -654,17 +693,18 @@ def lagInFrame(self): 2 | 1 3 | 2 \\N | 3 - """) + """ + ) execute_query( "select number, lagInFrame(number,1,0) OVER () AS lag FROM values('number Nullable(Int8)', (1),(1),(2),(3),(NULL))", - expected=expected + expected=expected, ) + @TestFeature @Name("funcs") def feature(self): - """Check true window functions. - """ + """Check true window functions.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/misc.py b/tests/testflows/window_functions/tests/misc.py index aca24edfe9c..5cb579c3954 100644 --- a/tests/testflows/window_functions/tests/misc.py +++ b/tests/testflows/window_functions/tests/misc.py @@ -3,11 +3,12 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def subquery_expr_preceding(self): - """Check using subquery expr in preceding. - """ - expected = convert_output(""" + """Check using subquery expr in preceding.""" + expected = convert_output( + """ sum | unique1 -----+--------- 0 | 0 @@ -20,21 +21,23 @@ def subquery_expr_preceding(self): 13 | 7 15 | 8 17 | 9 - """) + """ + ) execute_query( "SELECT sum(unique1) over " "(order by unique1 rows (SELECT unique1 FROM tenk1 ORDER BY unique1 LIMIT 1) + 1 PRECEDING) AS sum, " "unique1 " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestScenario def window_functions_in_select_expression(self): - """Check using multiple window functions in an expression. - """ - expected = convert_output(""" + """Check using multiple window functions in an expression.""" + expected = convert_output( + """ cntsum -------- 22 @@ -47,23 +50,26 @@ def window_functions_in_select_expression(self): 51 92 136 - """) + """ + ) execute_query( "SELECT (count(*) OVER (PARTITION BY four ORDER BY ten) + " "sum(hundred) OVER (PARTITION BY four ORDER BY ten)) AS cntsum " "FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario def window_functions_in_subquery(self): - """Check using window functions in a subquery. - """ - expected = convert_output(""" + """Check using window functions in a subquery.""" + expected = convert_output( + """ total | fourcount | twosum -------+-----------+-------- - """) + """ + ) execute_query( "SELECT * FROM (" @@ -73,14 +79,15 @@ def window_functions_in_subquery(self): " sum(hundred) OVER (PARTITION BY two ORDER BY ten) AS twosum " " FROM tenk1 " ") WHERE total <> fourcount + twosum", - expected=expected + expected=expected, ) + @TestScenario def group_by_and_one_window(self): - """Check running window function with group by and one window. - """ - expected = convert_output(""" + """Check running window function with group by and one window.""" + expected = convert_output( + """ four | ten | sum | avg ------+-----+------+------------------------ 0 | 0 | 0 | 0 @@ -103,24 +110,27 @@ def group_by_and_one_window(self): 3 | 5 | 7500 | 5 3 | 7 | 7500 | 7 3 | 9 | 7500 | 9 - """) + """ + ) execute_query( "SELECT four, ten, SUM(SUM(four)) OVER (PARTITION BY four) AS sum, AVG(ten) AS avg FROM tenk1 GROUP BY four, ten ORDER BY four, ten", expected=expected, ) + @TestScenario def group_by_and_multiple_windows(self): - """Check running window function with group by and multiple windows. - """ - expected = convert_output(""" + """Check running window function with group by and multiple windows.""" + expected = convert_output( + """ sum1 | row_number | sum2 -------+------------+------- 25100 | 1 | 47100 7400 | 2 | 22000 14600 | 3 | 14600 - """) + """ + ) execute_query( "SELECT sum(salary) AS sum1, row_number() OVER (ORDER BY depname) AS row_number, " @@ -129,11 +139,12 @@ def group_by_and_multiple_windows(self): expected=expected, ) + @TestScenario def query_with_order_by_and_one_window(self): - """Check using a window function in the query that has `ORDER BY` clause. - """ - expected = convert_output(""" + """Check using a window function in the query that has `ORDER BY` clause.""" + expected = convert_output( + """ depname | empno | salary | rank ----------+----------+--------+--------- sales | 3 | 4800 | 1 @@ -146,45 +157,51 @@ def query_with_order_by_and_one_window(self): develop | 10 | 5200 | 3 develop | 11 | 5200 | 4 develop | 8 | 6000 | 5 - """) + """ + ) execute_query( "SELECT depname, empno, salary, rank() OVER w AS rank FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary, empno) ORDER BY rank() OVER w, empno", - expected=expected + expected=expected, ) + @TestScenario def with_union_all(self): - """Check using window over rows obtained with `UNION ALL`. - """ - expected = convert_output(""" + """Check using window over rows obtained with `UNION ALL`.""" + expected = convert_output( + """ count ------- - """) + """ + ) execute_query( "SELECT count(*) OVER (PARTITION BY four) AS count FROM (SELECT * FROM tenk1 UNION ALL SELECT * FROM tenk1) LIMIT 0", - expected=expected + expected=expected, ) + @TestScenario def empty_table(self): - """Check using an empty table with a window function. - """ - expected = convert_output(""" + """Check using an empty table with a window function.""" + expected = convert_output( + """ count ------- - """) + """ + ) execute_query( "SELECT count(*) OVER (PARTITION BY four) AS count FROM (SELECT * FROM tenk1 WHERE 0)", - expected=expected + expected=expected, ) + @TestScenario def from_subquery(self): - """Check using a window function over data from subquery. - """ - expected = convert_output(""" + """Check using a window function over data from subquery.""" + expected = convert_output( + """ count | four -------+------ 4 | 1 @@ -193,20 +210,22 @@ def from_subquery(self): 4 | 1 2 | 3 2 | 3 - """) + """ + ) execute_query( "SELECT count(*) OVER (PARTITION BY four) AS count, four FROM (SELECT * FROM tenk1 WHERE two = 1) WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario def groups_frame(self): - """Check using `GROUPS` frame. - """ + """Check using `GROUPS` frame.""" exitcode, message = groups_frame_error() - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 12 | 0 | 0 @@ -219,16 +238,20 @@ def groups_frame(self): 35 | 2 | 2 45 | 3 | 3 45 | 7 | 3 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT sum(unique1) over (order by four groups between unbounded preceding and current row), unique1, four FROM tenk1 WHERE unique1 < 10 """, - exitcode=exitcode, message=message + exitcode=exitcode, + message=message, ) + @TestScenario def count_with_empty_over_clause_without_start(self): """Check that we can use `count()` window function without passing @@ -237,7 +260,7 @@ def count_with_empty_over_clause_without_start(self): exitcode = 0 message = "1" - sql = ("SELECT count() OVER () FROM tenk1 LIMIT 1") + sql = "SELECT count() OVER () FROM tenk1 LIMIT 1" with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) @@ -245,17 +268,19 @@ def count_with_empty_over_clause_without_start(self): @TestScenario def subquery_multiple_window_functions(self): - """Check using multiple window functions is a subquery. - """ - expected = convert_output(""" + """Check using multiple window functions is a subquery.""" + expected = convert_output( + """ depname | depsalary | depminsalary --------+-------------+-------------- sales | 5000 | 5000 sales | 9800 | 4800 sales | 14600 | 4800 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT * FROM (SELECT depname, sum(salary) OVER (PARTITION BY depname order by empno) AS depsalary, @@ -263,15 +288,17 @@ def subquery_multiple_window_functions(self): FROM empsalary) WHERE depname = 'sales' """, - expected=expected + expected=expected, ) + @TestScenario def windows_with_same_partitioning_but_different_ordering(self): """Check using using two windows that use the same partitioning but different ordering. """ - expected = convert_output(""" + expected = convert_output( + """ first | last ------+----- 7 | 7 @@ -284,22 +311,25 @@ def windows_with_same_partitioning_but_different_ordering(self): 3 | 3 3 | 4 3 | 1 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT any(empno) OVER (PARTITION BY depname ORDER BY salary, enroll_date) AS first, anyLast(empno) OVER (PARTITION BY depname ORDER BY salary,enroll_date,empno) AS last FROM empsalary """, - expected=expected + expected=expected, ) + @TestScenario def subquery_with_multiple_windows_filtering(self): - """Check filtering rows from a subquery that uses multiple window functions. - """ - expected = convert_output(""" + """Check filtering rows from a subquery that uses multiple window functions.""" + expected = convert_output( + """ depname | empno | salary | enroll_date | first_emp | last_emp ----------+-------+----------+--------------+-------------+---------- develop | 8 | 6000 | 2006-10-01 | 1 | 5 @@ -308,9 +338,11 @@ def subquery_with_multiple_windows_filtering(self): personnel | 5 | 3500 | 2007-12-10 | 2 | 1 sales | 1 | 5000 | 2006-10-01 | 1 | 3 sales | 4 | 4800 | 2007-08-08 | 3 | 1 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT * FROM (SELECT depname, empno, @@ -321,16 +353,17 @@ def subquery_with_multiple_windows_filtering(self): FROM empsalary) emp WHERE first_emp = 1 OR last_emp = 1 """, - expected=expected + expected=expected, ) + @TestScenario def exclude_clause(self): - """Check if exclude clause is supported. - """ + """Check if exclude clause is supported.""" exitcode, message = syntax_error() - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 7 | 4 | 0 @@ -343,19 +376,21 @@ def exclude_clause(self): 23 | 3 | 3 15 | 7 | 3 10 | 0 | 0 - """) + """ + ) execute_query( "SELECT sum(unique1) over (rows between 2 preceding and 2 following exclude no others) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10", - exitcode=exitcode, message=message + exitcode=exitcode, + message=message, ) + @TestScenario def in_view(self): - """Check using a window function in a view. - """ + """Check using a window function in a view.""" with Given("I create a view"): sql = """ CREATE VIEW v_window AS @@ -364,7 +399,8 @@ def in_view(self): """ create_table(name="v_window", statement=sql) - expected = convert_output(""" + expected = convert_output( + """ number | sum_rows ---------+---------- 1 | 3 @@ -377,20 +413,16 @@ def in_view(self): 8 | 24 9 | 27 10 | 19 - """) - - execute_query( - "SELECT * FROM v_window", - expected=expected + """ ) + execute_query("SELECT * FROM v_window", expected=expected) + + @TestFeature @Name("misc") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_FrameClause("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_FrameClause("1.0")) def feature(self): - """Check misc cases for frame clause. - """ + """Check misc cases for frame clause.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/order_clause.py b/tests/testflows/window_functions/tests/order_clause.py index 2dafe5dafc9..ce8bc3cbd8f 100644 --- a/tests/testflows/window_functions/tests/order_clause.py +++ b/tests/testflows/window_functions/tests/order_clause.py @@ -2,129 +2,142 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def single_expr_asc(self): - """Check defining of order clause with single expr ASC. - """ - expected = convert_output(""" + """Check defining of order clause with single expr ASC.""" + expected = convert_output( + """ x | s | sum ----+---+----- 1 | a | 2 1 | b | 2 2 | b | 4 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (ORDER BY x ASC) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario def single_expr_desc(self): - """Check defining of order clause with single expr DESC. - """ - expected = convert_output(""" + """Check defining of order clause with single expr DESC.""" + expected = convert_output( + """ x | s | sum ----+---+----- 2 | b | 2 1 | a | 4 1 | b | 4 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (ORDER BY x DESC) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs("1.0")) def multiple_expr_desc_desc(self): - """Check defining of order clause with multiple exprs. - """ - expected = convert_output(""" + """Check defining of order clause with multiple exprs.""" + expected = convert_output( + """ x | s | sum --+---+---- 2 | b | 2 1 | b | 3 1 | a | 4 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (ORDER BY x DESC, s DESC) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs("1.0")) def multiple_expr_asc_asc(self): - """Check defining of order clause with multiple exprs. - """ - expected = convert_output(""" + """Check defining of order clause with multiple exprs.""" + expected = convert_output( + """ x | s | sum ----+---+------ 1 | a | 1 1 | b | 2 2 | b | 4 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (ORDER BY x ASC, s ASC) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MultipleExprs("1.0")) def multiple_expr_asc_desc(self): - """Check defining of order clause with multiple exprs. - """ - expected = convert_output(""" + """Check defining of order clause with multiple exprs.""" + expected = convert_output( + """ x | s | sum ----+---+------ 1 | b | 1 1 | a | 2 2 | b | 4 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (ORDER BY x ASC, s DESC) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_MissingExpr_Error("1.0") ) def missing_expr_error(self): - """Check that defining of order clause with missing expr returns an error. - """ + """Check that defining of order clause with missing expr returns an error.""" exitcode = 62 message = "Exception: Syntax error: failed at position" - self.context.node.query("SELECT sum(number) OVER (ORDER BY) FROM numbers(1,3)", exitcode=exitcode, message=message) + self.context.node.query( + "SELECT sum(number) OVER (ORDER BY) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause_InvalidExpr_Error("1.0") ) def invalid_expr_error(self): - """Check that defining of order clause with invalid expr returns an error. - """ + """Check that defining of order clause with invalid expr returns an error.""" exitcode = 47 message = "Exception: Missing columns: 'foo'" - self.context.node.query("SELECT sum(number) OVER (ORDER BY foo) FROM numbers(1,3)", exitcode=exitcode, message=message) + self.context.node.query( + "SELECT sum(number) OVER (ORDER BY foo) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario def by_column(self): - """Check order by using a single column. - """ - expected = convert_output(""" + """Check order by using a single column.""" + expected = convert_output( + """ depname | empno | salary | rank -----------+-------+--------+------ develop | 7 | 4200 | 1 @@ -137,18 +150,20 @@ def by_column(self): sales | 1 | 5000 | 1 sales | 3 | 4800 | 1 sales | 4 | 4800 | 1 - """) + """ + ) execute_query( "SELECT depname, empno, salary, rank() OVER (PARTITION BY depname, empno ORDER BY salary) AS rank FROM empsalary", expected=expected, ) + @TestScenario def by_expr(self): - """Check order by with expression. - """ - expected = convert_output(""" + """Check order by with expression.""" + expected = convert_output( + """ avg ------------------------ 0 @@ -161,16 +176,19 @@ def by_expr(self): 2 3 3 - """) + """ + ) execute_query( "SELECT avg(four) OVER (PARTITION BY four ORDER BY thousand / 100) AS avg FROM tenk1 WHERE unique2 < 10", expected=expected, ) + @TestScenario def by_expr_with_aggregates(self): - expected = convert_output(""" + expected = convert_output( + """ ten | res | rank -----+----------+------ 0 | 9976146 | 4 @@ -183,7 +201,8 @@ def by_expr_with_aggregates(self): 7 | 10120309 | 10 8 | 9991305 | 6 9 | 10040184 | 7 - """) + """ + ) execute_query( "select ten, sum(unique1) + sum(unique2) as res, rank() over (order by sum(unique1) + sum(unique2)) as rank " @@ -191,28 +210,27 @@ def by_expr_with_aggregates(self): expected=expected, ) + @TestScenario def by_a_non_integer_constant(self): - """Check if it is allowed to use a window with ordering by a non integer constant. - """ - expected = convert_output(""" + """Check if it is allowed to use a window with ordering by a non integer constant.""" + expected = convert_output( + """ rank ------ 1 - """) + """ + ) execute_query( - "SELECT rank() OVER (ORDER BY length('abc')) AS rank", - expected=expected + "SELECT rank() OVER (ORDER BY length('abc')) AS rank", expected=expected ) + @TestFeature @Name("order clause") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_OrderClause("1.0")) def feature(self): - """Check defining order clause. - """ + """Check defining order clause.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/over_clause.py b/tests/testflows/window_functions/tests/over_clause.py index d02ddcee656..87fbd7fec0b 100644 --- a/tests/testflows/window_functions/tests/over_clause.py +++ b/tests/testflows/window_functions/tests/over_clause.py @@ -3,14 +3,13 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_EmptyOverClause("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_EmptyOverClause("1.0")) def empty(self): - """Check using empty over clause. - """ - expected = convert_output(""" + """Check using empty over clause.""" + expected = convert_output( + """ count ------- 10 @@ -23,22 +22,24 @@ def empty(self): 10 10 10 - """) + """ + ) execute_query( "SELECT COUNT(*) OVER () AS count FROM tenk1 WHERE unique2 < 10", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_EmptyOverClause("1.0"), - RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow("1.0"), ) def empty_named_window(self): - """Check using over clause with empty window. - """ - expected = convert_output(""" + """Check using over clause with empty window.""" + expected = convert_output( + """ count ------- 10 @@ -51,21 +52,23 @@ def empty_named_window(self): 10 10 10 - """) + """ + ) execute_query( "SELECT COUNT(*) OVER w AS count FROM tenk1 WHERE unique2 < 10 WINDOW w AS ()", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_AdHocWindow("1.0"), ) def adhoc_window(self): - """Check running aggregating `sum` function over an adhoc window. - """ - expected = convert_output(""" + """Check running aggregating `sum` function over an adhoc window.""" + expected = convert_output( + """ depname | empno | salary | sum -----------+-------+--------+------- develop | 7 | 4200 | 25100 @@ -78,60 +81,73 @@ def adhoc_window(self): sales | 3 | 4800 | 14600 sales | 4 | 4800 | 14600 sales | 1 | 5000 | 14600 - """) + """ + ) execute_query( "SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) AS sum FROM empsalary ORDER BY depname, salary, empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_AdHocWindow_MissingWindowSpec_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_AdHocWindow_MissingWindowSpec_Error( + "1.0" + ) ) def missing_window_spec(self): - """Check missing window spec in over clause. - """ + """Check missing window spec in over clause.""" exitcode = 62 message = "Exception: Syntax error" - self.context.node.query("SELECT number,sum(number) OVER FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow_InvalidName_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow_InvalidName_Error( + "1.0" + ) ) def invalid_window_name(self): - """Check invalid window name. - """ + """Check invalid window name.""" exitcode = 47 message = "Exception: Window 'w3' is not defined" - self.context.node.query("SELECT number,sum(number) OVER w3 FROM values('number Int8', (1),(1),(2),(3)) WINDOW w1 AS ()", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER w3 FROM values('number Int8', (1),(1),(2),(3)) WINDOW w1 AS ()", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow_MultipleWindows_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_OverClause_NamedWindow_MultipleWindows_Error( + "1.0" + ) ) def invalid_multiple_windows(self): - """Check invalid multiple window names. - """ + """Check invalid multiple window names.""" exitcode = 47 message = "Exception: Missing columns" - self.context.node.query("SELECT number,sum(number) OVER w1, w2 FROM values('number Int8', (1),(1),(2),(3)) WINDOW w1 AS (), w2 AS (PARTITION BY number)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER w1, w2 FROM values('number Int8', (1),(1),(2),(3)) WINDOW w1 AS (), w2 AS (PARTITION BY number)", + exitcode=exitcode, + message=message, + ) @TestFeature @Name("over clause") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_OverClause("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_OverClause("1.0")) def feature(self): - """Check defining frame clause. - """ + """Check defining frame clause.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/partition_clause.py b/tests/testflows/window_functions/tests/partition_clause.py index 3e9ebefe2ba..e8da74d0603 100644 --- a/tests/testflows/window_functions/tests/partition_clause.py +++ b/tests/testflows/window_functions/tests/partition_clause.py @@ -3,75 +3,82 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def single_expr(self): - """Check defining of partition clause with single expr. - """ - expected = convert_output(""" + """Check defining of partition clause with single expr.""" + expected = convert_output( + """ x | s | sum ----+---+------ 1 | a | 2 1 | b | 2 2 | b | 2 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (PARTITION BY x) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_MultipleExpr("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_MultipleExpr("1.0")) def multiple_expr(self): - """Check defining of partition clause with multiple exprs. - """ - expected = convert_output(""" + """Check defining of partition clause with multiple exprs.""" + expected = convert_output( + """ x | s | sum --+---+---- 1 | a | 1 1 | b | 1 2 | b | 2 - """) + """ + ) execute_query( "SELECT x,s, sum(x) OVER (PARTITION BY x,s) AS sum FROM values('x Int8, s String', (1,'a'),(1,'b'),(2,'b'))", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_MissingExpr_Error("1.0") ) def missing_expr_error(self): - """Check that defining of partition clause with missing expr returns an error. - """ + """Check that defining of partition clause with missing expr returns an error.""" exitcode = 62 message = "Exception: Syntax error: failed at position" - self.context.node.query("SELECT sum(number) OVER (PARTITION BY) FROM numbers(1,3)", exitcode=exitcode, message=message) + self.context.node.query( + "SELECT sum(number) OVER (PARTITION BY) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause_InvalidExpr_Error("1.0") ) def invalid_expr_error(self): - """Check that defining of partition clause with invalid expr returns an error. - """ + """Check that defining of partition clause with invalid expr returns an error.""" exitcode = 47 message = "Exception: Missing columns: 'foo'" - self.context.node.query("SELECT sum(number) OVER (PARTITION BY foo) FROM numbers(1,3)", exitcode=exitcode, message=message) + self.context.node.query( + "SELECT sum(number) OVER (PARTITION BY foo) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) @TestFeature @Name("partition clause") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_PartitionClause("1.0")) def feature(self): - """Check defining partition clause. - """ + """Check defining partition clause.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/range_datetime.py b/tests/testflows/window_functions/tests/range_datetime.py index 0b34fdf43d4..8d335d41345 100644 --- a/tests/testflows/window_functions/tests/range_datetime.py +++ b/tests/testflows/window_functions/tests/range_datetime.py @@ -3,12 +3,14 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def order_by_asc_range_between_days_preceding_and_days_following(self): """Check range between days preceding and days following with ascending order by. """ - expected = convert_output(""" + expected = convert_output( + """ sum | salary | enroll_date -------+--------+------------- 34900 | 5000 | 2006-10-01 @@ -21,19 +23,22 @@ def order_by_asc_range_between_days_preceding_and_days_following(self): 32200 | 4500 | 2008-01-01 47100 | 5200 | 2007-08-01 47100 | 5200 | 2007-08-15 - """) + """ + ) execute_query( "select sum(salary) over (order by enroll_date range between 365 preceding and 365 following) AS sum, " "salary, enroll_date from empsalary order by empno", - expected=expected + expected=expected, ) + @TestScenario def order_by_desc_range_between_days_preceding_and_days_following(self): """Check range between days preceding and days following with descending order by.""" - expected = convert_output(""" + expected = convert_output( + """ sum | salary | enroll_date -------+--------+------------- 34900 | 5000 | 2006-10-01 @@ -46,20 +51,23 @@ def order_by_desc_range_between_days_preceding_and_days_following(self): 32200 | 4500 | 2008-01-01 47100 | 5200 | 2007-08-01 47100 | 5200 | 2007-08-15 - """) + """ + ) execute_query( "select sum(salary) over (order by enroll_date desc range between 365 preceding and 365 following) AS sum, " "salary, enroll_date from empsalary order by empno", - expected=expected + expected=expected, ) + @TestScenario def order_by_desc_range_between_days_following_and_days_following(self): """Check range between days following and days following with descending order by. """ - expected = convert_output(""" + expected = convert_output( + """ sum | salary | enroll_date -------+--------+------------- 0 | 5000 | 2006-10-01 @@ -72,20 +80,23 @@ def order_by_desc_range_between_days_following_and_days_following(self): 0 | 4500 | 2008-01-01 0 | 5200 | 2007-08-01 0 | 5200 | 2007-08-15 - """) + """ + ) execute_query( "select sum(salary) over (order by enroll_date desc range between 365 following and 365 following) AS sum, " "salary, enroll_date from empsalary order by empno", - expected=expected + expected=expected, ) + @TestScenario def order_by_desc_range_between_days_preceding_and_days_preceding(self): """Check range between days preceding and days preceding with descending order by. """ - expected = convert_output(""" + expected = convert_output( + """ sum | salary | enroll_date -------+--------+------------- 0 | 5000 | 2006-10-01 @@ -98,20 +109,23 @@ def order_by_desc_range_between_days_preceding_and_days_preceding(self): 0 | 4500 | 2008-01-01 0 | 5200 | 2007-08-01 0 | 5200 | 2007-08-15 - """) + """ + ) execute_query( "select sum(salary) over (order by enroll_date desc range between 365 preceding and 365 preceding) AS sum, " "salary, enroll_date from empsalary order by empno", - expected=expected + expected=expected, ) + @TestScenario def datetime_with_timezone_order_by_asc_range_between_n_preceding_and_n_following(self): """Check range between preceding and following with DateTime column that has timezone using ascending order by. """ - expected = convert_output(""" + expected = convert_output( + """ id | f_timestamptz | first_value | last_value ----+------------------------------+-------------+------------ 1 | 2000-10-19 10:23:54 | 1 | 3 @@ -124,7 +138,8 @@ def datetime_with_timezone_order_by_asc_range_between_n_preceding_and_n_followin 8 | 2006-10-19 10:23:54 | 7 | 9 9 | 2007-10-19 10:23:54 | 8 | 10 10 | 2008-10-19 10:23:54 | 9 | 10 - """) + """ + ) execute_query( """ @@ -133,15 +148,19 @@ def datetime_with_timezone_order_by_asc_range_between_n_preceding_and_n_followin window w as (order by f_timestamptz range between 31622400 preceding and 31622400 following) order by id """, - expected=expected + expected=expected, ) + @TestScenario -def datetime_with_timezone_order_by_desc_range_between_n_preceding_and_n_following(self): +def datetime_with_timezone_order_by_desc_range_between_n_preceding_and_n_following( + self, +): """Check range between preceding and following with DateTime column that has timezone using descending order by. """ - expected = convert_output(""" + expected = convert_output( + """ id | f_timestamptz | first_value | last_value ----+------------------------------+-------------+------------ 10 | 2008-10-19 10:23:54 | 10 | 9 @@ -154,7 +173,8 @@ def datetime_with_timezone_order_by_desc_range_between_n_preceding_and_n_followi 3 | 2001-10-19 10:23:54 | 4 | 1 2 | 2001-10-19 10:23:54 | 4 | 1 1 | 2000-10-19 10:23:54 | 2 | 1 - """) + """ + ) execute_query( """ @@ -163,15 +183,17 @@ def datetime_with_timezone_order_by_desc_range_between_n_preceding_and_n_followi window w as (order by f_timestamptz desc range between 31622400 preceding and 31622400 following) order by id desc """, - expected=expected + expected=expected, ) + @TestScenario def datetime_order_by_asc_range_between_n_preceding_and_n_following(self): """Check range between preceding and following with DateTime column and ascending order by. """ - expected = convert_output(""" + expected = convert_output( + """ id | f_timestamp | first_value | last_value ----+------------------------------+-------------+------------ 1 | 2000-10-19 10:23:54 | 1 | 3 @@ -184,7 +206,8 @@ def datetime_order_by_asc_range_between_n_preceding_and_n_following(self): 8 | 2006-10-19 10:23:54 | 7 | 9 9 | 2007-10-19 10:23:54 | 8 | 10 10 | 2008-10-19 10:23:54 | 9 | 10 - """) + """ + ) execute_query( """ @@ -193,15 +216,17 @@ def datetime_order_by_asc_range_between_n_preceding_and_n_following(self): window w as (order by f_timestamp range between 31622400 preceding and 31622400 following) ORDER BY id """, - expected=expected + expected=expected, ) + @TestScenario def datetime_order_by_desc_range_between_n_preceding_and_n_following(self): """Check range between preceding and following with DateTime column and descending order by. """ - expected = convert_output(""" + expected = convert_output( + """ id | f_timestamp | first_value | last_value ----+------------------------------+-------------+------------ 10 | 2008-10-19 10:23:54 | 10 | 9 @@ -214,7 +239,8 @@ def datetime_order_by_desc_range_between_n_preceding_and_n_following(self): 2 | 2001-10-19 10:23:54 | 4 | 1 3 | 2001-10-19 10:23:54 | 4 | 1 1 | 2000-10-19 10:23:54 | 2 | 1 - """) + """ + ) execute_query( """ @@ -223,16 +249,16 @@ def datetime_order_by_desc_range_between_n_preceding_and_n_following(self): window w as (order by f_timestamp desc range between 31622400 preceding and 31622400 following) """, - expected=expected + expected=expected, ) + @TestFeature @Name("range datetime") @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_DataTypes_DateAndDateTime("1.0") ) def feature(self): - """Check `Date` and `DateTime` data time with range frames. - """ + """Check `Date` and `DateTime` data time with range frames.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/range_errors.py b/tests/testflows/window_functions/tests/range_errors.py index 67a9cfb14c9..958a4412b4f 100644 --- a/tests/testflows/window_functions/tests/range_errors.py +++ b/tests/testflows/window_functions/tests/range_errors.py @@ -3,9 +3,12 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_MultipleColumnsInOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_MultipleColumnsInOrderBy_Error( + "1.0" + ) ) def error_more_than_one_order_by_column(self): """Check that using more than one column in order by with range frame @@ -14,25 +17,30 @@ def error_more_than_one_order_by_column(self): exitcode = 36 message = "DB::Exception: Received from localhost:9000. DB::Exception: The RANGE OFFSET window frame requires exactly one ORDER BY column, 2 given" - sql = ("select sum(salary) over (order by enroll_date, salary range between 1 preceding and 2 following) AS sum, " - "salary, enroll_date from empsalary") + sql = ( + "select sum(salary) over (order by enroll_date, salary range between 1 preceding and 2 following) AS sum, " + "salary, enroll_date from empsalary" + ) with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_missing_order_by(self): - """Check that using range frame with offsets without order by returns an error. - """ + """Check that using range frame with offsets without order by returns an error.""" exitcode = 36 message = "DB::Exception: The RANGE OFFSET window frame requires exactly one ORDER BY column, 0 given" - sql = ("select sum(salary) over (range between 1 preceding and 2 following) AS sum, " - "salary, enroll_date from empsalary") + sql = ( + "select sum(salary) over (range between 1 preceding and 2 following) AS sum, " + "salary, enroll_date from empsalary" + ) with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_missing_order_by_with_partition_by_clause(self): """Check that range frame with offsets used with partition by but @@ -41,64 +49,66 @@ def error_missing_order_by_with_partition_by_clause(self): exitcode = 36 message = "DB::Exception: The RANGE OFFSET window frame requires exactly one ORDER BY column, 0 given" - sql = ("select f1, sum(f1) over (partition by f1 range between 1 preceding and 1 following) AS sum " - "from t1 where f1 = f2") + sql = ( + "select f1, sum(f1) over (partition by f1 range between 1 preceding and 1 following) AS sum " + "from t1 where f1 = f2" + ) with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario def error_range_over_non_numerical_column(self): - """Check that range over non numerical column returns an error. - """ + """Check that range over non numerical column returns an error.""" exitcode = 48 message = "DB::Exception: The RANGE OFFSET frame for 'DB::ColumnLowCardinality' ORDER BY column is not implemented" - sql = ("select sum(salary) over (order by depname range between 1 preceding and 2 following) as sum, " - "salary, enroll_date from empsalary") + sql = ( + "select sum(salary) over (order by depname range between 1 preceding and 2 following) as sum, " + "salary, enroll_date from empsalary" + ) with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_ExprPreceding_ExprValue("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_ExprPreceding_ExprValue("1.0")) def error_negative_preceding_offset(self): - """Check that non-positive value of preceding offset returns an error. - """ + """Check that non-positive value of preceding offset returns an error.""" exitcode = 36 message = "DB::Exception: Frame start offset must be greater than zero, -1 given" - sql = ("select max(enroll_date) over (order by salary range between -1 preceding and 2 following) AS max, " - "salary, enroll_date from empsalary") + sql = ( + "select max(enroll_date) over (order by salary range between -1 preceding and 2 following) AS max, " + "salary, enroll_date from empsalary" + ) with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing_ExprValue("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_ExprFollowing_ExprValue("1.0")) def error_negative_following_offset(self): - """Check that non-positive value of following offset returns an error. - """ + """Check that non-positive value of following offset returns an error.""" exitcode = 36 message = "DB::Exception: Frame end offset must be greater than zero, -2 given" - sql = ("select max(enroll_date) over (order by salary range between 1 preceding and -2 following) AS max, " - "salary, enroll_date from empsalary") + sql = ( + "select max(enroll_date) over (order by salary range between 1 preceding and -2 following) AS max, " + "salary, enroll_date from empsalary" + ) with When("I execute query", description=sql): r = current().context.node.query(sql, exitcode=exitcode, message=message) + @TestFeature @Name("range errors") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame("1.0")) def feature(self): - """Check different error conditions when usign range frame. - """ + """Check different error conditions when usign range frame.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/range_frame.py b/tests/testflows/window_functions/tests/range_frame.py index 71f00965547..186ca154068 100644 --- a/tests/testflows/window_functions/tests/range_frame.py +++ b/tests/testflows/window_functions/tests/range_frame.py @@ -3,41 +3,51 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_MissingFrameExtent_Error("1.0") ) def missing_frame_extent(self): - """Check that when range frame has missing frame extent then an error is returned. - """ + """Check that when range frame has missing frame extent then an error is returned.""" exitcode, message = syntax_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number RANGE) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number RANGE) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_InvalidFrameExtent_Error("1.0") ) def invalid_frame_extent(self): - """Check that when range frame has invalid frame extent then an error is returned. - """ + """Check that when range frame has invalid frame extent then an error is returned.""" exitcode, message = syntax_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number RANGE '1') FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number RANGE '1') FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_CurrentRow_Peers("1.0"), - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_CurrentRow_WithoutOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_CurrentRow_WithoutOrderBy( + "1.0" + ), ) def start_current_row_without_order_by(self): """Check range current row frame without order by and that the peers of the current row are rows that have values in the same order bucket. In this case without order by clause all rows are the peers of the current row. """ - expected = convert_output(""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 47100 @@ -50,23 +60,28 @@ def start_current_row_without_order_by(self): 9 | 4500 | 47100 10 | 5200 | 47100 11 | 5200 | 47100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, salary, sum(salary) OVER (RANGE CURRENT ROW) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_CurrentRow_Peers("1.0"), - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_CurrentRow_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_CurrentRow_WithOrderBy( + "1.0" + ), ) def start_current_row_with_order_by(self): """Check range current row frame with order by and that the peers of the current row are rows that have values in the same order bucket. """ - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 14600 @@ -79,38 +94,50 @@ def start_current_row_with_order_by(self): 9 | develop | 4500 | 25100 10 | develop | 5200 | 25100 11 | develop | 5200 | 25100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY depname RANGE CURRENT ROW) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedFollowing_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedFollowing_Error( + "1.0" + ) ) def start_unbounded_following_error(self): - """Check range current row frame with or without order by returns an error. - """ + """Check range current row frame with or without order by returns an error.""" exitcode, message = frame_start_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE UNBOUNDED FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE UNBOUNDED FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE UNBOUNDED FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE UNBOUNDED FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedPreceding_WithoutOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedPreceding_WithoutOrderBy( + "1.0" + ) ) def start_unbounded_preceding_without_order_by(self): - """Check range unbounded preceding frame without order by. - """ - expected = convert_output(""" + """Check range unbounded preceding frame without order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 7 | develop | 4200 | 25100 @@ -118,21 +145,25 @@ def start_unbounded_preceding_without_order_by(self): 9 | develop | 4500 | 25100 10 | develop | 5200 | 25100 11 | develop | 5200 | 25100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (RANGE UNBOUNDED PRECEDING) AS sum FROM empsalary WHERE depname = 'develop') ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedPreceding_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_UnboundedPreceding_WithOrderBy( + "1.0" + ) ) def start_unbounded_preceding_with_order_by(self): - """Check range unbounded preceding frame with order by. - """ - expected = convert_output(""" + """Check range unbounded preceding frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 47100 @@ -145,45 +176,59 @@ def start_unbounded_preceding_with_order_by(self): 9 | develop | 4500 | 25100 10 | develop | 5200 | 25100 11 | develop | 5200 | 25100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY depname RANGE UNBOUNDED PRECEDING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def start_expr_following_without_order_by_error(self): - """Check range expr following frame without order by returns an error. - """ + """Check range expr following frame without order by returns an error.""" exitcode, message = window_frame_error() - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE 1 FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE 1 FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprFollowing_WithOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprFollowing_WithOrderBy_Error( + "1.0" + ) ) def start_expr_following_with_order_by_error(self): - """Check range expr following frame with order by returns an error. - """ + """Check range expr following frame with order by returns an error.""" exitcode, message = window_frame_error() - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE 1 FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE 1 FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_WithOrderBy( + "1.0" + ) ) def start_expr_preceding_with_order_by(self): - """Check range expr preceding frame with order by. - """ - expected = convert_output(""" + """Check range expr preceding frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 5000 @@ -196,46 +241,60 @@ def start_expr_preceding_with_order_by(self): 9 | develop | 4500 | 4500 10 | develop | 5200 | 10400 11 | develop | 5200 | 10400 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE 1 PRECEDING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_OrderByNonNumericalColumn_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_OrderByNonNumericalColumn_Error( + "1.0" + ) ) def start_expr_preceding_order_by_non_numerical_column_error(self): - """Check range expr preceding frame with order by non-numerical column returns an error. - """ + """Check range expr preceding frame with order by non-numerical column returns an error.""" exitcode, message = frame_range_offset_error() - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY depname RANGE 1 PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY depname RANGE 1 PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Start_ExprPreceding_WithoutOrderBy_Error( + "1.0" + ) ) def start_expr_preceding_without_order_by_error(self): - """Check range expr preceding frame without order by returns an error. - """ + """Check range expr preceding frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE 1 PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE 1 PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_CurrentRow("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_CurrentRow( + "1.0" + ) ) def between_current_row_and_current_row(self): - """Check range between current row and current row frame with or without order by. - """ + """Check range between current row and current row frame with or without order by.""" with Example("without order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 7 | develop | 4200 | 25100 @@ -243,15 +302,17 @@ def between_current_row_and_current_row(self): 9 | develop | 4500 | 25100 10 | develop | 5200 | 25100 11 | develop | 5200 | 25100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN CURRENT ROW AND CURRENT ROW) AS sum FROM empsalary WHERE depname = 'develop') ORDER BY empno", - expected=expected + expected=expected, ) with Example("with order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+------ 7 | develop | 4200 | 4200 @@ -259,39 +320,51 @@ def between_current_row_and_current_row(self): 9 | develop | 4500 | 4500 10 | develop | 5200 | 5200 11 | develop | 5200 | 5200 - """) + """ + ) execute_query( "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY empno RANGE BETWEEN CURRENT ROW AND CURRENT ROW) AS sum FROM empsalary WHERE depname = 'develop'", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_UnboundedPreceding_Error( + "1.0" + ) ) def between_current_row_and_unbounded_preceding_error(self): - """Check range between current row and unbounded preceding frame with or without order by returns an error. - """ + """Check range between current row and unbounded preceding frame with or without order by returns an error.""" exitcode, message = frame_end_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_UnboundedFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_UnboundedFollowing( + "1.0" + ) ) def between_current_row_and_unbounded_following(self): - """Check range between current row and unbounded following frame with or without order by. - """ + """Check range between current row and unbounded following frame with or without order by.""" with Example("without order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 7 | develop | 4200 | 25100 @@ -299,15 +372,17 @@ def between_current_row_and_unbounded_following(self): 9 | develop | 4500 | 25100 10 | develop | 5200 | 25100 11 | develop | 5200 | 25100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS sum FROM empsalary WHERE depname = 'develop') ORDER BY empno", - expected=expected + expected=expected, ) with Example("with order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 7 | develop | 4200 | 25100 @@ -315,15 +390,17 @@ def between_current_row_and_unbounded_following(self): 9 | develop | 4500 | 14900 10 | develop | 5200 | 10400 11 | develop | 5200 | 5200 - """) + """ + ) execute_query( "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY empno RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS sum FROM empsalary WHERE depname = 'develop'", - expected=expected + expected=expected, ) with Example("with order by from tenk1"): - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 45 | 0 | 0 @@ -336,35 +413,44 @@ def between_current_row_and_unbounded_following(self): 10 | 7 | 3 45 | 8 | 0 33 | 9 | 1 - """) + """ + ) execute_query( "SELECT * FROM (SELECT sum(unique1) over (order by four range between current row and unbounded following) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10) ORDER BY unique1", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def between_current_row_and_expr_following_without_order_by_error(self): - """Check range between current row and expr following frame without order by returns an error. - """ + """Check range between current row and expr following frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprFollowing_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprFollowing_WithOrderBy( + "1.0" + ) ) def between_current_row_and_expr_following_with_order_by(self): - """Check range between current row and expr following frame with order by. - """ - expected = convert_output(""" + """Check range between current row and expr following frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 8900 @@ -377,39 +463,51 @@ def between_current_row_and_expr_following_with_order_by(self): 9 | develop | 4500 | 9700 10 | develop | 5200 | 10400 11 | develop | 5200 | 5200 - """) + """ + ) execute_query( "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY empno RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_CurrentRow_ExprPreceding_Error( + "1.0" + ) ) def between_current_row_and_expr_preceding_error(self): - """Check range between current row and expr preceding frame with or without order by returns an error. - """ + """Check range between current row and expr preceding frame with or without order by returns an error.""" exitcode, message = window_frame_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN CURRENT ROW AND 1 PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_CurrentRow("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_CurrentRow( + "1.0" + ) ) def between_unbounded_preceding_and_current_row(self): - """Check range between unbounded preceding and current row frame with and without order by. - """ + """Check range between unbounded preceding and current row frame with and without order by.""" with Example("with order by"): - expected = convert_output(""" + expected = convert_output( + """ four | ten | sum | last_value ------+-----+-----+------------ 0 | 0 | 0 | 0 @@ -432,18 +530,20 @@ def between_unbounded_preceding_and_current_row(self): 3 | 5 | 9 | 5 3 | 7 | 16 | 7 3 | 9 | 25 | 9 - """) + """ + ) execute_query( "SELECT four, ten," "sum(ten) over (partition by four order by ten range between unbounded preceding and current row) AS sum," "last_value(ten) over (partition by four order by ten range between unbounded preceding and current row) AS last_value " "FROM (select distinct ten, four from tenk1)", - expected=expected + expected=expected, ) with Example("without order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 7 | develop | 4200 | 25100 @@ -451,39 +551,51 @@ def between_unbounded_preceding_and_current_row(self): 9 | develop | 4500 | 25100 10 | develop | 5200 | 25100 11 | develop | 5200 | 25100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS sum FROM empsalary WHERE depname = 'develop') ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_UnboundedPreceding_Error( + "1.0" + ) ) def between_unbounded_preceding_and_unbounded_preceding_error(self): - """Check range between unbounded preceding and unbounded preceding frame with or without order by returns an error. - """ + """Check range between unbounded preceding and unbounded preceding frame with or without order by returns an error.""" exitcode, message = frame_end_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_UnboundedFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_UnboundedFollowing( + "1.0" + ) ) def between_unbounded_preceding_and_unbounded_following(self): - """Check range between unbounded preceding and unbounded following range with and without order by. - """ + """Check range between unbounded preceding and unbounded following range with and without order by.""" with Example("with order by"): - expected = convert_output(""" + expected = convert_output( + """ four | ten | sum | last_value ------+-----+-----+------------ 0 | 0 | 20 | 8 @@ -506,18 +618,20 @@ def between_unbounded_preceding_and_unbounded_following(self): 3 | 5 | 25 | 9 3 | 7 | 25 | 9 3 | 9 | 25 | 9 - """) + """ + ) execute_query( "SELECT four, ten, " "sum(ten) over (partition by four order by ten range between unbounded preceding and unbounded following) AS sum, " "last_value(ten) over (partition by four order by ten range between unbounded preceding and unbounded following) AS last_value " "FROM (select distinct ten, four from tenk1)", - expected=expected + expected=expected, ) with Example("without order by"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 47100 @@ -530,45 +644,59 @@ def between_unbounded_preceding_and_unbounded_following(self): 9 | develop | 4500 | 47100 10 | develop | 5200 | 47100 11 | develop | 5200 | 47100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def between_unbounded_preceding_and_expr_following_without_order_by_error(self): - """Check range between unbounded preceding and expr following frame without order by returns an error. - """ + """Check range between unbounded preceding and expr following frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprPreceding_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprPreceding_WithoutOrderBy_Error( + "1.0" + ) ) def between_unbounded_preceding_and_expr_preceding_without_order_by_error(self): - """Check range between unbounded preceding and expr preceding frame without order by returns an error. - """ + """Check range between unbounded preceding and expr preceding frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprFollowing_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprFollowing_WithOrderBy( + "1.0" + ) ) def between_unbounded_preceding_and_expr_following_with_order_by(self): - """Check range between unbounded preceding and expr following frame with order by. - """ - expected = convert_output(""" + """Check range between unbounded preceding and expr following frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 41100 @@ -581,21 +709,25 @@ def between_unbounded_preceding_and_expr_following_with_order_by(self): 9 | develop | 4500 | 30700 10 | develop | 5200 | 41100 11 | develop | 5200 | 41100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED PRECEDING AND 500 FOLLOWING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprPreceding_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedPreceding_ExprPreceding_WithOrderBy( + "1.0" + ) ) def between_unbounded_preceding_and_expr_preceding_with_order_by(self): - """Check range between unbounded preceding and expr preceding frame with order by. - """ - expected = convert_output(""" + """Check range between unbounded preceding and expr preceding frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 16100 @@ -608,171 +740,243 @@ def between_unbounded_preceding_and_expr_preceding_with_order_by(self): 9 | develop | 4500 | 7400 10 | develop | 5200 | 16100 11 | develop | 5200 | 16100 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED PRECEDING AND 500 PRECEDING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_CurrentRow_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_CurrentRow_Error( + "1.0" + ) ) def between_unbounded_following_and_current_row_error(self): - """Check range between unbounded following and current row frame with or without order by returns an error. - """ + """Check range between unbounded following and current row frame with or without order by returns an error.""" exitcode, message = frame_start_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND CURRENT ROW) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_UnboundedFollowing_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_UnboundedFollowing_Error( + "1.0" + ) ) def between_unbounded_following_and_unbounded_following_error(self): - """Check range between unbounded following and unbounded following frame with or without order by returns an error. - """ + """Check range between unbounded following and unbounded following frame with or without order by returns an error.""" exitcode, message = frame_start_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_UnboundedPreceding_Error( + "1.0" + ) ) def between_unbounded_following_and_unbounded_preceding_error(self): - """Check range between unbounded following and unbounded preceding frame with or without order by returns an error. - """ + """Check range between unbounded following and unbounded preceding frame with or without order by returns an error.""" exitcode, message = frame_start_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_ExprPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_ExprPreceding_Error( + "1.0" + ) ) def between_unbounded_following_and_expr_preceding_error(self): - """Check range between unbounded following and expr preceding frame with or without order by returns an error. - """ + """Check range between unbounded following and expr preceding frame with or without order by returns an error.""" exitcode, message = frame_start_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 PRECEDING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 PRECEDING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_ExprFollowing_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_UnboundedFollowing_ExprFollowing_Error( + "1.0" + ) ) def between_unbounded_following_and_expr_following_error(self): - """Check range between unbounded following and expr following frame with or without order by returns an error. - """ + """Check range between unbounded following and expr following frame with or without order by returns an error.""" exitcode, message = frame_start_error() with Example("without order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 FOLLOWING) AS sum FROM empsalary", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN UNBOUNDED FOLLOWING AND 1 FOLLOWING) AS sum FROM empsalary", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_CurrentRow_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_CurrentRow_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_preceding_and_current_row_without_order_by_error(self): - """Check range between expr preceding and current row frame without order by returns an error. - """ + """Check range between expr preceding and current row frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_preceding_and_unbounded_following_without_order_by_error(self): - """Check range between expr preceding and unbounded following frame without order by returns an error. - """ + """Check range between expr preceding and unbounded following frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_preceding_and_expr_following_without_order_by_error(self): - """Check range between expr preceding and expr following frame without order by returns an error. - """ + """Check range between expr preceding and expr following frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_preceding_and_expr_preceding_without_order_by_error(self): - """Check range between expr preceding and expr preceding frame without order by returns an error. - """ + """Check range between expr preceding and expr preceding frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedPreceding_Error( + "1.0" + ) ) def between_expr_preceding_and_unbounded_preceding_error(self): - """Check range between expr preceding and unbounded preceding frame with or without order by returns an error. - """ + """Check range between expr preceding and unbounded preceding frame with or without order by returns an error.""" exitcode, message = frame_end_unbounded_preceding_error() with Example("without order by"): - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY salary RANGE BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY salary RANGE BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_CurrentRow_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_CurrentRow_WithOrderBy( + "1.0" + ) ) def between_expr_preceding_and_current_row_with_order_by(self): - """Check range between expr preceding and current row frame with order by. - """ - expected = convert_output(""" + """Check range between expr preceding and current row frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 5000 @@ -785,21 +989,25 @@ def between_expr_preceding_and_current_row_with_order_by(self): 9 | develop | 4500 | 36700 10 | develop | 5200 | 41900 11 | develop | 5200 | 47100 - """) + """ + ) execute_query( "SELECT empno, depname, salary, sum(salary) OVER (ORDER BY empno RANGE BETWEEN 500 PRECEDING AND CURRENT ROW) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedFollowing_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_UnboundedFollowing_WithOrderBy( + "1.0" + ) ) def between_expr_preceding_and_unbounded_following_with_order_by(self): - """Check range between expr preceding and unbounded following frame with order by. - """ - expected = convert_output(""" + """Check range between expr preceding and unbounded following frame with order by.""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 35500 @@ -812,22 +1020,26 @@ def between_expr_preceding_and_unbounded_following_with_order_by(self): 9 | develop | 4500 | 39700 10 | develop | 5200 | 31000 11 | develop | 5200 | 31000 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN 500 PRECEDING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprFollowing_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprFollowing_WithOrderBy( + "1.0" + ) ) def between_expr_preceding_and_expr_following_with_order_by(self): - """Check range between expr preceding and expr following frame with order by. - """ + """Check range between expr preceding and expr following frame with order by.""" with Example("empsalary"): - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 29500 @@ -840,15 +1052,17 @@ def between_expr_preceding_and_expr_following_with_order_by(self): 9 | develop | 4500 | 23300 10 | develop | 5200 | 25000 11 | develop | 5200 | 25000 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN 500 PRECEDING AND 500 FOLLOWING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) with Example("tenk1"): - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 4 | 0 | 0 @@ -861,24 +1075,28 @@ def between_expr_preceding_and_expr_following_with_order_by(self): 8 | 6 | 2 10 | 3 | 3 10 | 7 | 3 - """) + """ + ) execute_query( "SELECT sum(unique1) over (partition by four order by unique1 range between 5 preceding and 6 following) AS sum, " "unique1, four " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithOrderBy( + "1.0" + ) ) def between_expr_preceding_and_expr_preceding_with_order_by(self): - """Check range between expr preceding and expr preceding range with order by. - """ + """Check range between expr preceding and expr preceding range with order by.""" with Example("order by asc"): - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 0 | 0 | 0 @@ -891,17 +1109,19 @@ def between_expr_preceding_and_expr_preceding_with_order_by(self): 27 | 6 | 2 23 | 3 | 3 23 | 7 | 3 - """) + """ + ) execute_query( "SELECT * FROM (SELECT sum(unique1) over (order by four range between 2 preceding and 1 preceding) AS sum, " - "unique1, four " + "unique1, four " "FROM tenk1 WHERE unique1 < 10) ORDER BY four, unique1", - expected=expected + expected=expected, ) with Example("order by desc"): - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 23 | 0 | 0 @@ -914,18 +1134,22 @@ def between_expr_preceding_and_expr_preceding_with_order_by(self): 10 | 6 | 2 0 | 3 | 3 0 | 7 | 3 - """) + """ + ) execute_query( "SELECT * FROM (SELECT sum(unique1) over (order by four desc range between 2 preceding and 1 preceding) AS sum, " "unique1, four " "FROM tenk1 WHERE unique1 < 10) ORDER BY four, unique1", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprPreceding_ExprPreceding_WithOrderBy_Error( + "1.0" + ) ) def between_expr_preceding_and_expr_preceding_with_order_by_error(self): """Check range between expr preceding and expr preceding range with order by returns error @@ -933,77 +1157,111 @@ def between_expr_preceding_and_expr_preceding_with_order_by_error(self): """ exitcode, message = frame_start_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_following_and_current_row_without_order_by_error(self): - """Check range between expr following and current row frame without order by returns an error. - """ + """Check range between expr following and current row frame without order by returns an error.""" exitcode, message = window_frame_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_following_and_unbounded_following_without_order_by_error(self): - """Check range between expr following and unbounded following frame without order by returns an error. - """ + """Check range between expr following and unbounded following frame without order by returns an error.""" exitcode, message = frame_requires_order_by_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_following_and_expr_following_without_order_by_error(self): - """Check range between expr following and expr following frame without order by returns an error. - """ + """Check range between expr following and expr following frame without order by returns an error.""" exitcode, message = window_frame_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND 1 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_WithoutOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_WithoutOrderBy_Error( + "1.0" + ) ) def between_expr_following_and_expr_preceding_without_order_by_error(self): - """Check range between expr following and expr preceding frame without order by returns an error. - """ + """Check range between expr following and expr preceding frame without order by returns an error.""" exitcode, message = window_frame_error() - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedPreceding_Error( + "1.0" + ) ) def between_expr_following_and_unbounded_preceding_error(self): - """Check range between expr following and unbounded preceding frame with or without order by returns an error. - """ + """Check range between expr following and unbounded preceding frame with or without order by returns an error.""" exitcode, message = frame_end_unbounded_preceding_error() with Example("without order by"): - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) with Example("with order by"): - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY salary RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY salary RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_WithOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_WithOrderBy_Error( + "1.0" + ) ) def between_expr_following_and_current_row_with_order_by_error(self): """Check range between expr following and current row frame with order by returns an error @@ -1011,12 +1269,18 @@ def between_expr_following_and_current_row_with_order_by_error(self): """ exitcode, message = window_frame_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND CURRENT ROW) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_Error( + "1.0" + ) ) def between_expr_following_and_expr_preceding_error(self): """Check range between expr following and expr preceding frame with order by returns an error @@ -1025,16 +1289,25 @@ def between_expr_following_and_expr_preceding_error(self): exitcode, message = frame_start_error() with Example("1 following 0 preceding"): - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 1 FOLLOWING AND 0 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) with Example("1 following 0 preceding"): - self.context.node.query("SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND 1 PRECEDING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithOrderBy_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithOrderBy_Error( + "1.0" + ) ) def between_expr_following_and_expr_following_with_order_by_error(self): """Check range between expr following and expr following frame with order by returns an error @@ -1042,98 +1315,122 @@ def between_expr_following_and_expr_following_with_order_by_error(self): """ exitcode, message = frame_start_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM values('number Int8', (1),(1),(2),(3))", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_ZeroSpecialCase("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_CurrentRow_ZeroSpecialCase( + "1.0" + ) ) def between_expr_following_and_current_row_zero_special_case(self): """Check range between expr following and current row frame for special case when exp is 0. It is expected to work. """ with When("I use it with order by"): - expected = convert_output(""" + expected = convert_output( + """ number | sum ---------+------ 1 | 2 1 | 2 2 | 2 3 | 3 - """) + """ + ) - execute_query("SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) AS sum FROM values('number Int8', (1),(1),(2),(3))", - expected=expected + execute_query( + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) AS sum FROM values('number Int8', (1),(1),(2),(3))", + expected=expected, ) with And("I use it without order by"): - expected = convert_output(""" + expected = convert_output( + """ number | sum ---------+------ 1 | 7 1 | 7 2 | 7 3 | 7 - """) + """ + ) execute_query( "SELECT number,sum(number) OVER (RANGE BETWEEN 0 FOLLOWING AND CURRENT ROW) AS sum FROM values('number Int8', (1),(1),(2),(3))", - expected=expected - ) + expected=expected, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedFollowing_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_UnboundedFollowing_WithOrderBy( + "1.0" + ) ) def between_expr_following_and_unbounded_following_with_order_by(self): - """Check range between expr following and unbounded following range with order by. - """ - expected = convert_output(""" + """Check range between expr following and unbounded following range with order by.""" + expected = convert_output( + """ number | sum ---------+------ 1 | 5 1 | 5 2 | 3 3 | 0 - """) - + """ + ) execute_query( "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING) AS sum FROM values('number Int8', (1),(1),(2),(3))", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_WithOrderBy_ZeroSpecialCase("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprPreceding_WithOrderBy_ZeroSpecialCase( + "1.0" + ) ) def between_expr_following_and_expr_preceding_with_order_by_zero_special_case(self): """Check range between expr following and expr preceding frame for special case when exp is 0. It is expected to work. """ - expected = convert_output(""" + expected = convert_output( + """ number | sum ---------+------ 1 | 2 1 | 2 2 | 2 3 | 3 - """) - - execute_query("SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING) AS sum FROM values('number Int8', (1),(1),(2),(3))", - expected=expected + """ ) + execute_query( + "SELECT number,sum(number) OVER (ORDER BY number RANGE BETWEEN 0 FOLLOWING AND 0 PRECEDING) AS sum FROM values('number Int8', (1),(1),(2),(3))", + expected=expected, + ) + + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithOrderBy("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_Between_ExprFollowing_ExprFollowing_WithOrderBy( + "1.0" + ) ) def between_expr_following_and_expr_following_with_order_by(self): """Check range between expr following and expr following frame with order by when frame start is before frame end. """ - expected = convert_output(""" + expected = convert_output( + """ empno | depname | salary | sum --------+-----------+--------+--------- 1 | sales | 5000 | 6000 @@ -1146,19 +1443,24 @@ def between_expr_following_and_expr_following_with_order_by(self): 9 | develop | 4500 | 15400 10 | develop | 5200 | 6000 11 | develop | 5200 | 6000 - """) + """ + ) execute_query( "SELECT * FROM (SELECT empno, depname, salary, sum(salary) OVER (ORDER BY salary RANGE BETWEEN 500 FOLLOWING AND 1000 FOLLOWING) AS sum FROM empsalary) ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario -def between_unbounded_preceding_and_current_row_with_expressions_in_order_by_and_aggregate(self): +def between_unbounded_preceding_and_current_row_with_expressions_in_order_by_and_aggregate( + self, +): """Check range between unbounded prceding and current row with expression used in the order by clause and aggregate functions. """ - expected = convert_output(""" + expected = convert_output( + """ four | two | sum | last_value ------+-----+-----+------------ 0 | 0 | 0 | 0 @@ -1181,22 +1483,25 @@ def between_unbounded_preceding_and_current_row_with_expressions_in_order_by_and 3 | 1 | 2 | 1 3 | 1 | 2 | 1 3 | 2 | 4 | 2 - """) + """ + ) execute_query( "SELECT four, toInt8(ten/4) as two, " "sum(toInt8(ten/4)) over (partition by four order by toInt8(ten/4) range between unbounded preceding and current row) AS sum, " "last_value(toInt8(ten/4)) over (partition by four order by toInt8(ten/4) range between unbounded preceding and current row) AS last_value " "FROM (select distinct ten, four from tenk1)", - expected=expected + expected=expected, ) + @TestScenario def between_current_row_and_unbounded_following_modifying_named_window(self): """Check range between current row and unbounded following when modifying named window. """ - expected = convert_output(""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 45 | 0 | 0 @@ -1209,20 +1514,22 @@ def between_current_row_and_unbounded_following_modifying_named_window(self): 18 | 2 | 2 10 | 3 | 3 10 | 7 | 3 - """) + """ + ) execute_query( "SELECT * FROM (SELECT sum(unique1) over (w range between current row and unbounded following) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10 WINDOW w AS (order by four)) ORDER BY unique1", - expected=expected + expected=expected, ) + @TestScenario def between_current_row_and_unbounded_following_in_named_window(self): - """Check range between current row and unbounded following in named window. - """ - expected = convert_output(""" + """Check range between current row and unbounded following in named window.""" + expected = convert_output( + """ first_value | last_value | unique1 | four -------------+------------+---------+------ 0 | 9 | 0 | 0 @@ -1235,27 +1542,31 @@ def between_current_row_and_unbounded_following_in_named_window(self): 7 | 9 | 7 | 3 8 | 9 | 8 | 0 9 | 9 | 9 | 1 - """) + """ + ) execute_query( "SELECT first_value(unique1) over w AS first_value, " "last_value(unique1) over w AS last_value, unique1, four " "FROM tenk1 WHERE unique1 < 10 " "WINDOW w AS (order by unique1 range between current row and unbounded following)", - expected=expected + expected=expected, ) + @TestScenario def between_expr_preceding_and_expr_following_with_partition_by_two_columns(self): """Check range between n preceding and n following frame with partition by two int value columns. """ - expected = convert_output(""" + expected = convert_output( + """ f1 | sum ----+----- 1 | 0 2 | 0 - """) + """ + ) execute_query( """ @@ -1263,20 +1574,23 @@ def between_expr_preceding_and_expr_following_with_partition_by_two_columns(self range between 1 following and 2 following) AS sum from t1 where f1 = f2 """, - expected=expected + expected=expected, ) + @TestScenario def between_expr_preceding_and_expr_following_with_partition_by_same_column_twice(self): """Check range between n preceding and n folowing with partition by the same column twice. """ - expected = convert_output(""" + expected = convert_output( + """ f1 | sum ----+----- 1 | 0 2 | 0 - """) + """ + ) execute_query( """ @@ -1284,20 +1598,23 @@ def between_expr_preceding_and_expr_following_with_partition_by_same_column_twic range between 2 preceding and 1 preceding) AS sum from t1 where f1 = f2) order by f1, sum """, - expected=expected + expected=expected, ) + @TestScenario def between_expr_preceding_and_expr_following_with_partition_and_order_by(self): """Check range between expr preceding and expr following frame used with partition by and order by clauses. """ - expected = convert_output(""" + expected = convert_output( + """ f1 | sum ----+----- 1 | 1 2 | 2 - """) + """ + ) execute_query( """ @@ -1305,14 +1622,15 @@ def between_expr_preceding_and_expr_following_with_partition_and_order_by(self): range between 1 preceding and 1 following) AS sum from t1 where f1 = f2 """, - expected=expected + expected=expected, ) + @TestScenario def order_by_decimal(self): - """Check using range with order by decimal column. - """ - expected = convert_output(""" + """Check using range with order by decimal column.""" + expected = convert_output( + """ id | f_numeric | first_value | last_value ----+-----------+-------------+------------ 0 | -1000 | 0 | 0 @@ -1325,7 +1643,8 @@ def order_by_decimal(self): 7 | 100 | 7 | 7 8 | 1000 | 8 | 8 9 | 0 | 9 | 9 - """) + """ + ) execute_query( """ @@ -1334,14 +1653,15 @@ def order_by_decimal(self): window w as (order by f_numeric range between 1 preceding and 1 following) """, - expected=expected + expected=expected, ) + @TestScenario def order_by_float(self): - """Check using range with order by float column. - """ - expected = convert_output(""" + """Check using range with order by float column.""" + expected = convert_output( + """ id | f_float4 | first_value | last_value ----+-----------+-------------+------------ 0 | -inf | 0 | 0 @@ -1354,7 +1674,8 @@ def order_by_float(self): 7 | 100 | 7 | 7 8 | inf | 8 | 8 9 | nan | 8 | 8 - """) + """ + ) execute_query( """ @@ -1363,14 +1684,15 @@ def order_by_float(self): window w as (order by f_float4 range between 1 preceding and 1 following) """, - expected=expected + expected=expected, ) + @TestScenario def with_nulls(self): - """Check using range frame over window with nulls. - """ - expected = convert_output(""" + """Check using range frame over window with nulls.""" + expected = convert_output( + """ x | y | first_value | last_value ---+----+-------------+------------ \\N | 42 | 42 | 43 @@ -1380,7 +1702,8 @@ def with_nulls(self): 3 | 3 | 1 | 5 4 | 4 | 2 | 5 5 | 5 | 3 | 5 - """) + """ + ) execute_query( """ @@ -1394,17 +1717,17 @@ def with_nulls(self): window w as (order by x asc nulls first range between 2 preceding and 2 following) """, - expected=expected + expected=expected, ) + @TestFeature @Name("range frame") @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame("1.0"), - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_DataTypes_IntAndUInt("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame_DataTypes_IntAndUInt("1.0"), ) def feature(self): - """Check defining range frame. - """ + """Check defining range frame.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/range_overflow.py b/tests/testflows/window_functions/tests/range_overflow.py index 0c66e54c8ee..34a9a9592e5 100644 --- a/tests/testflows/window_functions/tests/range_overflow.py +++ b/tests/testflows/window_functions/tests/range_overflow.py @@ -3,133 +3,143 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def positive_overflow_with_Int16(self): - """Check positive overflow with Int16. - """ - expected = convert_output(""" + """Check positive overflow with Int16.""" + expected = convert_output( + """ x | last_value -------+------------ 32764 | 0 32765 | 0 32766 | 0 - """) + """ + ) execute_query( """ select number as x, last_value(x) over (order by toInt16(x) range between current row and 2147450884 following) AS last_value from numbers(32764, 3) """, - expected=expected + expected=expected, ) + @TestScenario def negative_overflow_with_Int16(self): - """Check negative overflow with Int16. - """ - expected = convert_output(""" + """Check negative overflow with Int16.""" + expected = convert_output( + """ x | last_value --------+------------ -32764 | 0 -32765 | 0 -32766 | 0 - """) + """ + ) execute_query( """ select number as x, last_value(x) over (order by toInt16(x) desc range between current row and 2147450885 following) as last_value from (SELECT -number - 32763 AS number FROM numbers(1, 3)) """, - expected=expected + expected=expected, ) + @TestScenario def positive_overflow_for_Int32(self): - """Check positive overflow for Int32. - """ - expected = convert_output(""" + """Check positive overflow for Int32.""" + expected = convert_output( + """ x | last_value ------------+------------ 2147483644 | 2147483646 2147483645 | 2147483646 2147483646 | 2147483646 - """) + """ + ) execute_query( """ select number as x, last_value(x) over (order by x range between current row and 4 following) as last_value from numbers(2147483644, 3) """, - expected=expected + expected=expected, ) + @TestScenario def negative_overflow_for_Int32(self): - """Check negative overflow for Int32. - """ - expected = convert_output(""" + """Check negative overflow for Int32.""" + expected = convert_output( + """ x | last_value -------------+------------- -2147483644 | -2147483646 -2147483645 | -2147483646 -2147483646 | -2147483646 - """) + """ + ) execute_query( """ select number as x, last_value(x) over (order by x desc range between current row and 5 following) as last_value from (select -number-2147483643 AS number FROM numbers(1,3)) """, - expected=expected + expected=expected, ) + @TestScenario def positive_overflow_for_Int64(self): - """Check positive overflow for Int64. - """ - expected = convert_output(""" + """Check positive overflow for Int64.""" + expected = convert_output( + """ x | last_value ---------------------+--------------------- 9223372036854775804 | 9223372036854775806 9223372036854775805 | 9223372036854775806 9223372036854775806 | 9223372036854775806 - """) + """ + ) execute_query( """ select number as x, last_value(x) over (order by x range between current row and 4 following) as last_value from numbers(9223372036854775804, 3) """, - expected=expected + expected=expected, ) + @TestScenario def negative_overflow_for_Int64(self): - """Check negative overflow for Int64. - """ - expected = convert_output(""" + """Check negative overflow for Int64.""" + expected = convert_output( + """ x | last_value ----------------------+---------------------- -9223372036854775804 | -9223372036854775806 -9223372036854775805 | -9223372036854775806 -9223372036854775806 | -9223372036854775806 - """) + """ + ) execute_query( """ select number as x, last_value(x) over (order by x desc range between current row and 5 following) as last_value from (select -number-9223372036854775803 AS number from numbers(1,3)) """, - expected=expected + expected=expected, ) + @TestFeature @Name("range overflow") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_RangeFrame("1.0")) def feature(self): - """Check using range frame with overflows. - """ + """Check using range frame with overflows.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/rows_frame.py b/tests/testflows/window_functions/tests/rows_frame.py index 07533e8d1ab..f1aed00a9b6 100644 --- a/tests/testflows/window_functions/tests/rows_frame.py +++ b/tests/testflows/window_functions/tests/rows_frame.py @@ -3,38 +3,43 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_MissingFrameExtent_Error("1.0") ) def missing_frame_extent(self): - """Check that when rows frame has missing frame extent then an error is returned. - """ + """Check that when rows frame has missing frame extent then an error is returned.""" exitcode, message = syntax_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number ROWS) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number ROWS) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_InvalidFrameExtent_Error("1.0") ) def invalid_frame_extent(self): - """Check that when rows frame has invalid frame extent then an error is returned. - """ + """Check that when rows frame has invalid frame extent then an error is returned.""" exitcode, message = frame_offset_nonnegative_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number ROWS -1) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number ROWS -1) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_CurrentRow("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_CurrentRow("1.0")) def start_current_row(self): - """Check rows current row frame. - """ - expected = convert_output(""" + """Check rows current row frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+------- 1 | 5000 | 5000 @@ -47,21 +52,23 @@ def start_current_row(self): 9 | 4500 | 4500 10 | 5200 | 5200 11 | 5200 | 5200 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS CURRENT ROW) AS sum FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_UnboundedPreceding("1.0") ) def start_unbounded_preceding(self): - """Check rows unbounded preceding frame. - """ - expected = convert_output(""" + """Check rows unbounded preceding frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+------- 1 | 5000 | 5000 @@ -74,21 +81,23 @@ def start_unbounded_preceding(self): 9 | 4500 | 36700 10 | 5200 | 41900 11 | 5200 | 47100 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS UNBOUNDED PRECEDING) AS sum FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_ExprPreceding("1.0") ) def start_expr_preceding(self): - """Check rows expr preceding frame. - """ - expected = convert_output(""" + """Check rows expr preceding frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 5000 @@ -101,47 +110,55 @@ def start_expr_preceding(self): 9 | 4500 | 10500 10 | 5200 | 9700 11 | 5200 | 10400 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS 1 PRECEDING) AS sum FROM empsalary ORDER BY empno", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_UnboundedFollowing_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_UnboundedFollowing_Error( + "1.0" + ) ) def start_unbounded_following_error(self): - """Check rows unbounded following frame returns an error. - """ + """Check rows unbounded following frame returns an error.""" exitcode, message = frame_start_error() self.context.node.query( "SELECT empno, salary, sum(salary) OVER (ROWS UNBOUNDED FOLLOWING) AS sum FROM empsalary ORDER BY empno", - exitcode=exitcode, message=message) + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Start_ExprFollowing_Error("1.0") ) def start_expr_following_error(self): - """Check rows expr following frame returns an error. - """ + """Check rows expr following frame returns an error.""" exitcode, message = window_frame_error() self.context.node.query( "SELECT empno, salary, sum(salary) OVER (ROWS 1 FOLLOWING) AS sum FROM empsalary ORDER BY empno", - exitcode=exitcode, message=message) + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_CurrentRow("1.0") ) def between_current_row_and_current_row(self): - """Check rows between current row and current row frame. - """ - expected = convert_output(""" + """Check rows between current row and current row frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 5000 @@ -154,45 +171,59 @@ def between_current_row_and_current_row(self): 9 | 4500 | 4500 10 | 5200 | 5200 11 | 5200 | 5200 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN CURRENT ROW AND CURRENT ROW) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_ExprPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_ExprPreceding_Error( + "1.0" + ) ) def between_current_row_and_expr_preceding_error(self): - """Check rows between current row and expr preceding returns an error. - """ + """Check rows between current row and expr preceding returns an error.""" exitcode, message = window_frame_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number ROWS BETWEEN CURRENT ROW AND 1 PRECEDING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number ROWS BETWEEN CURRENT ROW AND 1 PRECEDING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_UnboundedPreceding_Error( + "1.0" + ) ) def between_current_row_and_unbounded_preceding_error(self): - """Check rows between current row and unbounded preceding returns an error. - """ + """Check rows between current row and unbounded preceding returns an error.""" exitcode, message = frame_end_unbounded_preceding_error() - self.context.node.query("SELECT number,sum(number) OVER (ORDER BY number ROWS BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ORDER BY number ROWS BETWEEN CURRENT ROW AND UNBOUNDED PRECEDING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_UnboundedFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_UnboundedFollowing( + "1.0" + ) ) def between_current_row_and_unbounded_following(self): - """Check rows between current row and unbounded following. - """ - expected = convert_output(""" + """Check rows between current row and unbounded following.""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 45 | 0 | 0 @@ -205,23 +236,27 @@ def between_current_row_and_unbounded_following(self): 24 | 7 | 3 17 | 8 | 0 9 | 9 | 1 - """) + """ + ) execute_query( "SELECT sum(unique1) over (order by unique1 rows between current row and unbounded following) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_ExprFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_CurrentRow_ExprFollowing( + "1.0" + ) ) def between_current_row_and_expr_following(self): - """Check rows between current row and expr following. - """ - expected = convert_output(""" + """Check rows between current row and expr following.""" + expected = convert_output( + """ i | b | bool_and | bool_or ---+---+----------+--------- 1 | 1 | 1 | 1 @@ -229,24 +264,29 @@ def between_current_row_and_expr_following(self): 3 | 0 | 0 | 0 4 | 0 | 0 | 1 5 | 1 | 1 | 1 - """) + """ + ) - execute_query(""" + execute_query( + """ SELECT i, b, groupBitAnd(b) OVER w AS bool_and, groupBitOr(b) OVER w AS bool_or FROM VALUES('i Int8, b UInt8', (1,1), (2,1), (3,0), (4,0), (5,1)) WINDOW w AS (ORDER BY i ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) """, - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_CurrentRow("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_CurrentRow( + "1.0" + ) ) def between_unbounded_preceding_and_current_row(self): - """Check rows between unbounded preceding and current row. - """ - expected = convert_output(""" + """Check rows between unbounded preceding and current row.""" + expected = convert_output( + """ four | two | sum | last_value ------+-----+-----+------------ 0 | 0 | 0 | 0 @@ -269,36 +309,45 @@ def between_unbounded_preceding_and_current_row(self): 3 | 1 | 1 | 1 3 | 1 | 2 | 1 3 | 2 | 4 | 2 - """) + """ + ) execute_query( "SELECT four, toInt8(ten/4) as two," "sum(toInt8(ten/4)) over (partition by four order by toInt8(ten/4) rows between unbounded preceding and current row) AS sum," "last_value(toInt8(ten/4)) over (partition by four order by toInt8(ten/4) rows between unbounded preceding and current row) AS last_value " "FROM (select distinct ten, four from tenk1)", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_UnboundedPreceding_Error( + "1.0" + ) ) def between_unbounded_preceding_and_unbounded_preceding_error(self): - """Check rows between unbounded preceding and unbounded preceding returns an error. - """ + """Check rows between unbounded preceding and unbounded preceding returns an error.""" exitcode, message = frame_end_unbounded_preceding_error() - self.context.node.query("SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_ExprPreceding("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_ExprPreceding( + "1.0" + ) ) def between_unbounded_preceding_and_expr_preceding(self): - """Check rows between unbounded preceding and expr preceding frame. - """ - expected = convert_output(""" + """Check rows between unbounded preceding and expr preceding frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 0 @@ -311,21 +360,25 @@ def between_unbounded_preceding_and_expr_preceding(self): 9 | 4500 | 32200 10 | 5200 | 36700 11 | 5200 | 41900 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_UnboundedFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_UnboundedFollowing( + "1.0" + ) ) def between_unbounded_preceding_and_unbounded_following(self): - """Check rows between unbounded preceding and unbounded following frame. - """ - expected = convert_output(""" + """Check rows between unbounded preceding and unbounded following frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 47100 @@ -338,21 +391,25 @@ def between_unbounded_preceding_and_unbounded_following(self): 9 | 4500 | 47100 10 | 5200 | 47100 11 | 5200 | 47100 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_ExprFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedPreceding_ExprFollowing( + "1.0" + ) ) def between_unbounded_preceding_and_expr_following(self): - """Check rows between unbounded preceding and expr following. - """ - expected = convert_output(""" + """Check rows between unbounded preceding and expr following.""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 1 | 0 | 0 @@ -365,52 +422,70 @@ def between_unbounded_preceding_and_expr_following(self): 36 | 7 | 3 45 | 8 | 0 45 | 9 | 1 - """) + """ + ) execute_query( "SELECT sum(unique1) over (order by unique1 rows between unbounded preceding and 1 following) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestOutline(Scenario) @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedFollowing_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_UnboundedFollowing_Error( + "1.0" + ) +) +@Examples( + "range", + [ + ("UNBOUNDED FOLLOWING AND CURRENT ROW",), + ("UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING",), + ("UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING",), + ("UNBOUNDED FOLLOWING AND 1 PRECEDING",), + ("UNBOUNDED FOLLOWING AND 1 FOLLOWING",), + ], ) -@Examples("range", [ - ("UNBOUNDED FOLLOWING AND CURRENT ROW",), - ("UNBOUNDED FOLLOWING AND UNBOUNDED PRECEDING",), - ("UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING",), - ("UNBOUNDED FOLLOWING AND 1 PRECEDING",), - ("UNBOUNDED FOLLOWING AND 1 FOLLOWING",), -]) def between_unbounded_following_error(self, range): - """Check rows between unbounded following and any end frame returns an error. - """ + """Check rows between unbounded following and any end frame returns an error.""" exitcode, message = frame_start_error() - self.context.node.query(f"SELECT number,sum(number) OVER (ROWS BETWEEN {range}) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + f"SELECT number,sum(number) OVER (ROWS BETWEEN {range}) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestOutline(Scenario) @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_Error("1.0") ) -@Examples("range exitcode message", [ - ("1 FOLLOWING AND CURRENT ROW", *window_frame_error()), - ("1 FOLLOWING AND UNBOUNDED PRECEDING", *frame_end_unbounded_preceding_error()), - ("1 FOLLOWING AND 1 PRECEDING", *frame_start_error()) -]) +@Examples( + "range exitcode message", + [ + ("1 FOLLOWING AND CURRENT ROW", *window_frame_error()), + ("1 FOLLOWING AND UNBOUNDED PRECEDING", *frame_end_unbounded_preceding_error()), + ("1 FOLLOWING AND 1 PRECEDING", *frame_start_error()), + ], +) def between_expr_following_error(self, range, exitcode, message): - """Check cases when rows between expr following returns an error. - """ - self.context.node.query(f"SELECT number,sum(number) OVER (ROWS BETWEEN {range}) FROM numbers(1,3)", - exitcode=exitcode, message=message) + """Check cases when rows between expr following returns an error.""" + self.context.node.query( + f"SELECT number,sum(number) OVER (ROWS BETWEEN {range}) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing_Error( + "1.0" + ) ) def between_expr_following_and_expr_following_error(self): """Check rows between expr following and expr following returns an error when frame end index is less @@ -418,17 +493,23 @@ def between_expr_following_and_expr_following_error(self): """ exitcode, message = frame_start_error() - self.context.node.query("SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 FOLLOWING AND 0 FOLLOWING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_UnboundedFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_UnboundedFollowing( + "1.0" + ) ) def between_expr_following_and_unbounded_following(self): - """Check rows between exp following and unbounded following frame. - """ - expected = convert_output(""" + """Check rows between exp following and unbounded following frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 28600 @@ -441,22 +522,27 @@ def between_expr_following_and_unbounded_following(self): 9 | 4500 | 0 10 | 5200 | 0 11 | 5200 | 0 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN 4 FOLLOWING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing( + "1.0" + ) ) def between_expr_following_and_expr_following(self): """Check rows between exp following and expr following frame when end of the frame is greater than the start of the frame. """ - expected = convert_output(""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 17000 @@ -469,68 +555,85 @@ def between_expr_following_and_expr_following(self): 9 | 4500 | 10400 10 | 5200 | 5200 11 | 5200 | 0 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN 1 FOLLOWING AND 4 FOLLOWING) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_CurrentRow("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_CurrentRow( + "1.0" + ) ) def between_expr_preceding_and_current_row(self): - """Check rows between exp preceding and current row frame. - """ - expected = convert_output(""" + """Check rows between exp preceding and current row frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 8 | 6000 | 6000 10 | 5200 | 11200 11 | 5200 | 10400 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS sum FROM empsalary WHERE salary > 5000", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_UnboundedPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_UnboundedPreceding_Error( + "1.0" + ) ) def between_expr_preceding_and_unbounded_preceding_error(self): - """Check rows between expr preceding and unbounded preceding returns an error. - """ + """Check rows between expr preceding and unbounded preceding returns an error.""" exitcode, message = frame_end_error() - self.context.node.query("SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND UNBOUNDED PRECEDING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_UnboundedFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_UnboundedFollowing( + "1.0" + ) ) def between_expr_preceding_and_unbounded_following(self): - """Check rows between exp preceding and unbounded following frame. - """ - expected = convert_output(""" + """Check rows between exp preceding and unbounded following frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 8 | 6000 | 16400 10 | 5200 | 16400 11 | 5200 | 10400 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS sum FROM empsalary WHERE salary > 5000", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding_Error("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding_Error( + "1.0" + ) ) def between_expr_preceding_and_expr_preceding_error(self): """Check rows between expr preceding and expr preceding returns an error when frame end is @@ -538,17 +641,23 @@ def between_expr_preceding_and_expr_preceding_error(self): """ exitcode, message = frame_start_error() - self.context.node.query("SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM numbers(1,3)", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER (ROWS BETWEEN 1 PRECEDING AND 2 PRECEDING) FROM numbers(1,3)", + exitcode=exitcode, + message=message, + ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding( + "1.0" + ) ) def between_expr_preceding_and_expr_preceding(self): - """Check rows between expr preceding and expr preceding frame when frame end is after or at frame start. - """ - expected = convert_output(""" + """Check rows between expr preceding and expr preceding frame when frame end is after or at frame start.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 1 | 5000 | 5000 @@ -561,42 +670,49 @@ def between_expr_preceding_and_expr_preceding(self): 9 | 4500 | 10500 10 | 5200 | 9700 11 | 5200 | 10400 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN 1 PRECEDING AND 0 PRECEDING) AS sum FROM empsalary", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprFollowing( + "1.0" + ) ) def between_expr_preceding_and_expr_following(self): - """Check rows between expr preceding and expr following frame. - """ - expected = convert_output(""" + """Check rows between expr preceding and expr following frame.""" + expected = convert_output( + """ empno | salary | sum --------+--------+-------- 8 | 6000 | 11200 10 | 5200 | 16400 11 | 5200 | 10400 - """) + """ + ) execute_query( "SELECT empno, salary, sum(salary) OVER (ORDER BY empno ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS sum FROM empsalary WHERE salary > 5000", - expected=expected + expected=expected, ) @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprFollowing_ExprFollowing( + "1.0" + ) ) def between_expr_following_and_expr_following_ref(self): - """Check reference result for rows between expr following and expr following range. - """ - expected = convert_output(""" + """Check reference result for rows between expr following and expr following range.""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 6 | 0 | 0 @@ -609,23 +725,27 @@ def between_expr_following_and_expr_following_ref(self): 17 | 7 | 3 9 | 8 | 0 0 | 9 | 1 - """) + """ + ) execute_query( "SELECT sum(unique1) over (order by unique1 rows between 1 following and 3 following) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprPreceding( + "1.0" + ) ) def between_expr_preceding_and_expr_preceding_ref(self): - """Check reference result for rows between expr preceding and expr preceding frame. - """ - expected = convert_output(""" + """Check reference result for rows between expr preceding and expr preceding frame.""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 0 | 0 | 0 @@ -638,23 +758,27 @@ def between_expr_preceding_and_expr_preceding_ref(self): 11 | 7 | 3 13 | 8 | 0 15 | 9 | 1 - """) + """ + ) execute_query( "SELECT sum(unique1) over (order by unique1 rows between 2 preceding and 1 preceding) AS sum," "unique1, four " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestScenario @Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprFollowing("1.0") + RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame_Between_ExprPreceding_ExprFollowing( + "1.0" + ) ) def between_expr_preceding_and_expr_following_ref(self): - """Check reference result for rows between expr preceding and expr following frame. - """ - expected = convert_output(""" + """Check reference result for rows between expr preceding and expr following frame.""" + expected = convert_output( + """ sum | unique1 | four -----+---------+------ 3 | 0 | 0 @@ -667,22 +791,21 @@ def between_expr_preceding_and_expr_following_ref(self): 35 | 7 | 3 30 | 8 | 0 24 | 9 | 1 - """) + """ + ) execute_query( "SELECT sum(unique1) over (order by unique1 rows between 2 preceding and 2 following) AS sum, " "unique1, four " "FROM tenk1 WHERE unique1 < 10", - expected=expected + expected=expected, ) + @TestFeature @Name("rows frame") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_RowsFrame("1.0")) def feature(self): - """Check defining rows frame. - """ + """Check defining rows frame.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/window_clause.py b/tests/testflows/window_functions/tests/window_clause.py index 714fce89895..17ff5a7ddab 100644 --- a/tests/testflows/window_functions/tests/window_clause.py +++ b/tests/testflows/window_functions/tests/window_clause.py @@ -3,11 +3,12 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def single_window(self): - """Check defining a single named window using window clause. - """ - expected = convert_output(""" + """Check defining a single named window using window clause.""" + expected = convert_output( + """ depname | empno | salary | sum -----------+-------+--------+------- develop | 7 | 4200 | 4200 @@ -20,35 +21,37 @@ def single_window(self): sales | 1 | 5000 | 5000 sales | 3 | 4800 | 9800 sales | 4 | 4800 | 14600 - """) + """ + ) execute_query( "SELECT depname, empno, salary, sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY empno)", - expected=expected + expected=expected, ) + @TestScenario def unused_window(self): - """Check unused window. - """ - expected = convert_output(""" + """Check unused window.""" + expected = convert_output( + """ four ------- - """) + """ + ) execute_query( "SELECT four FROM tenk1 WHERE 0 WINDOW w AS (PARTITION BY ten)", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MultipleWindows("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MultipleWindows("1.0")) def multiple_identical_windows(self): - """Check defining multiple windows using window clause. - """ - expected = convert_output(""" + """Check defining multiple windows using window clause.""" + expected = convert_output( + """ sum | count -------+------- 3500 | 1 @@ -61,22 +64,22 @@ def multiple_identical_windows(self): 41100 | 9 41100 | 9 47100 | 10 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w1 AS sum, count(*) OVER w2 AS count " "FROM empsalary WINDOW w1 AS (ORDER BY salary), w2 AS (ORDER BY salary)", - expected=expected + expected=expected, ) + @TestScenario -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MultipleWindows("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MultipleWindows("1.0")) def multiple_windows(self): - """Check defining multiple windows using window clause. - """ - expected = convert_output(""" + """Check defining multiple windows using window clause.""" + expected = convert_output( + """ empno | depname | salary | sum1 | sum2 --------+-----------+--------+-------+-------- 1 | sales | 5000 | 5000 | 5000 @@ -89,33 +92,36 @@ def multiple_windows(self): 9 | develop | 4500 | 14700 | 10500 10 | develop | 5200 | 19900 | 9700 11 | develop | 5200 | 25100 | 10400 - """) - - execute_query("SELECT empno, depname, salary, sum(salary) OVER w1 AS sum1, sum(salary) OVER w2 AS sum2 " - "FROM empsalary WINDOW w1 AS (PARTITION BY depname ORDER BY empno), w2 AS (ORDER BY empno ROWS 1 PRECEDING)", - expected=expected + """ ) + execute_query( + "SELECT empno, depname, salary, sum(salary) OVER w1 AS sum1, sum(salary) OVER w2 AS sum2 " + "FROM empsalary WINDOW w1 AS (PARTITION BY depname ORDER BY empno), w2 AS (ORDER BY empno ROWS 1 PRECEDING)", + expected=expected, + ) + + @TestScenario @Requirements( RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause_MissingWindowSpec_Error("1.0") ) def missing_window_spec(self): - """Check missing window spec in window clause. - """ + """Check missing window spec in window clause.""" exitcode = 62 message = "Exception: Syntax error" - self.context.node.query("SELECT number,sum(number) OVER w1 FROM values('number Int8', (1),(1),(2),(3)) WINDOW w1", - exitcode=exitcode, message=message) + self.context.node.query( + "SELECT number,sum(number) OVER w1 FROM values('number Int8', (1),(1),(2),(3)) WINDOW w1", + exitcode=exitcode, + message=message, + ) + @TestFeature @Name("window clause") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_WindowClause("1.0")) def feature(self): - """Check defining frame clause. - """ + """Check defining frame clause.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/window_functions/tests/window_spec.py b/tests/testflows/window_functions/tests/window_spec.py index aacbc192200..82e3cf0ef47 100644 --- a/tests/testflows/window_functions/tests/window_spec.py +++ b/tests/testflows/window_functions/tests/window_spec.py @@ -2,11 +2,12 @@ from testflows.core import * from window_functions.requirements import * from window_functions.tests.common import * + @TestScenario def partition_clause(self): - """Check window specification that only contains partition clause. - """ - expected = convert_output(""" + """Check window specification that only contains partition clause.""" + expected = convert_output( + """ sum ------- 25100 @@ -19,18 +20,20 @@ def partition_clause(self): 14600 14600 14600 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (PARTITION BY depname)", - expected=expected + expected=expected, ) + @TestScenario def orderby_clause(self): - """Check window specification that only contains order by clause. - """ - expected = convert_output(""" + """Check window specification that only contains order by clause.""" + expected = convert_output( + """ sum ------- 25100 @@ -43,18 +46,20 @@ def orderby_clause(self): 47100 47100 47100 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (ORDER BY depname)", - expected=expected + expected=expected, ) + @TestScenario def frame_clause(self): - """Check window specification that only contains frame clause. - """ - expected = convert_output(""" + """Check window specification that only contains frame clause.""" + expected = convert_output( + """ sum ------- 5000 @@ -67,18 +72,20 @@ def frame_clause(self): 4500 5200 5200 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (ORDER BY empno ROWS CURRENT ROW)", - expected=expected + expected=expected, ) + @TestScenario def partition_with_order_by(self): - """Check window specification that contains partition and order by clauses. - """ - expected = convert_output(""" + """Check window specification that contains partition and order by clauses.""" + expected = convert_output( + """ sum ------- 4200 @@ -91,18 +98,20 @@ def partition_with_order_by(self): 9600 9600 14600 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary)", - expected=expected + expected=expected, ) + @TestScenario def partition_with_frame(self): - """Check window specification that contains partition and frame clauses. - """ - expected = convert_output(""" + """Check window specification that contains partition and frame clauses.""" + expected = convert_output( + """ sum ------- 4200 @@ -115,18 +124,20 @@ def partition_with_frame(self): 5000 4800 4800 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (PARTITION BY depname, empno ROWS 1 PRECEDING)", - expected=expected + expected=expected, ) + @TestScenario def order_by_with_frame(self): - """Check window specification that contains order by and frame clauses. - """ - expected = convert_output(""" + """Check window specification that contains order by and frame clauses.""" + expected = convert_output( + """ sum ------- 4200 @@ -139,18 +150,20 @@ def order_by_with_frame(self): 8500 9800 9600 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (ORDER BY depname, empno ROWS 1 PRECEDING)", - expected=expected + expected=expected, ) + @TestScenario def partition_with_order_by_and_frame(self): - """Check window specification that contains all clauses. - """ - expected = convert_output(""" + """Check window specification that contains all clauses.""" + expected = convert_output( + """ sum ------- 4200 @@ -163,18 +176,20 @@ def partition_with_order_by_and_frame(self): 4800 9600 9800 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary ROWS 1 PRECEDING)", - expected=expected + expected=expected, ) + @TestScenario def empty(self): - """Check defining an empty window specification. - """ - expected = convert_output(""" + """Check defining an empty window specification.""" + expected = convert_output( + """ sum ------- 47100 @@ -187,20 +202,19 @@ def empty(self): 47100 47100 47100 - """) + """ + ) execute_query( "SELECT sum(salary) OVER w AS sum FROM empsalary WINDOW w AS ()", - expected=expected + expected=expected, ) + @TestFeature @Name("window spec") -@Requirements( - RQ_SRS_019_ClickHouse_WindowFunctions_WindowSpec("1.0") -) +@Requirements(RQ_SRS_019_ClickHouse_WindowFunctions_WindowSpec("1.0")) def feature(self): - """Check defining window specifications. - """ + """Check defining window specifications.""" for scenario in loads(current_module(), Scenario): Scenario(run=scenario, flags=TE) diff --git a/utils/changelog/format-changelog.py b/utils/changelog/format-changelog.py index 56fe973eb6f..ef1340d48dd 100755 --- a/utils/changelog/format-changelog.py +++ b/utils/changelog/format-changelog.py @@ -9,25 +9,37 @@ import os import re import sys -parser = argparse.ArgumentParser(description='Format changelog for given PRs.') -parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs='?', default=sys.stdin, help='File with PR numbers, one per line.') +parser = argparse.ArgumentParser(description="Format changelog for given PRs.") +parser.add_argument( + "file", + metavar="FILE", + type=argparse.FileType("r", encoding="utf-8"), + nargs="?", + default=sys.stdin, + help="File with PR numbers, one per line.", +) args = parser.parse_args() # This function mirrors the PR description checks in ClickhousePullRequestTrigger. # Returns False if the PR should not be mentioned changelog. def parse_one_pull_request(item): - description = item['body'] + description = item["body"] # Don't skip empty lines because they delimit parts of description - lines = [line for line in [x.strip() for x in (description.split('\n') if description else [])]] - lines = [re.sub(r'\s+', ' ', l) for l in lines] + lines = [ + line + for line in [ + x.strip() for x in (description.split("\n") if description else []) + ] + ] + lines = [re.sub(r"\s+", " ", l) for l in lines] - category = '' - entry = '' + category = "" + entry = "" if lines: i = 0 while i < len(lines): - if re.match(r'(?i)^[>*_ ]*change\s*log\s*category', lines[i]): + if re.match(r"(?i)^[>*_ ]*change\s*log\s*category", lines[i]): i += 1 if i >= len(lines): break @@ -36,9 +48,11 @@ def parse_one_pull_request(item): i += 1 if i >= len(lines): break - category = re.sub(r'^[-*\s]*', '', lines[i]) + category = re.sub(r"^[-*\s]*", "", lines[i]) i += 1 - elif re.match(r'(?i)^[>*_ ]*(short\s*description|change\s*log\s*entry)', lines[i]): + elif re.match( + r"(?i)^[>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i] + ): i += 1 # Can have one empty line between header and the entry itself. Filter it out. if i < len(lines) and not lines[i]: @@ -48,7 +62,7 @@ def parse_one_pull_request(item): while i < len(lines) and lines[i]: entry_lines.append(lines[i]) i += 1 - entry = ' '.join(entry_lines) + entry = " ".join(entry_lines) else: i += 1 @@ -58,48 +72,59 @@ def parse_one_pull_request(item): category = "NO CL CATEGORY" # Filter out the PR categories that are not for changelog. - if re.match(r'(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)', category): + if re.match( + r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", + category, + ): return False if not entry: # Shouldn't happen, because description check in CI should catch such PRs. category = "NO CL ENTRY" - entry = "NO CL ENTRY: '" + item['title'] + "'" + entry = "NO CL ENTRY: '" + item["title"] + "'" entry = entry.strip() - if entry[-1] != '.': - entry += '.' + if entry[-1] != ".": + entry += "." - item['entry'] = entry - item['category'] = category + item["entry"] = entry + item["category"] = category return True + # This array gives the preferred category order, and is also used to # normalize category names. -categories_preferred_order = ['Backward Incompatible Change', - 'New Feature', 'Performance Improvement', 'Improvement', 'Bug Fix', - 'Build/Testing/Packaging Improvement', 'Other'] +categories_preferred_order = [ + "Backward Incompatible Change", + "New Feature", + "Performance Improvement", + "Improvement", + "Bug Fix", + "Build/Testing/Packaging Improvement", + "Other", +] category_to_pr = collections.defaultdict(lambda: []) users = {} for line in args.file: - pr = json.loads(open(f'pr{line.strip()}.json').read()) - assert(pr['number']) + pr = json.loads(open(f"pr{line.strip()}.json").read()) + assert pr["number"] if not parse_one_pull_request(pr): continue - assert(pr['category']) + assert pr["category"] # Normalize category name for c in categories_preferred_order: - if fuzzywuzzy.fuzz.ratio(pr['category'].lower(), c.lower()) >= 90: - pr['category'] = c + if fuzzywuzzy.fuzz.ratio(pr["category"].lower(), c.lower()) >= 90: + pr["category"] = c break - category_to_pr[pr['category']].append(pr) - user_id = pr['user']['id'] - users[user_id] = json.loads(open(f'user{user_id}.json').read()) + category_to_pr[pr["category"]].append(pr) + user_id = pr["user"]["id"] + users[user_id] = json.loads(open(f"user{user_id}.json").read()) + def print_category(category): print(("#### " + category)) @@ -110,14 +135,25 @@ def print_category(category): # Substitute issue links. # 1) issue number w/o markdown link - pr["entry"] = re.sub(r'([^[])#([0-9]{4,})', r'\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)', pr["entry"]) + pr["entry"] = re.sub( + r"([^[])#([0-9]{4,})", + r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", + pr["entry"], + ) # 2) issue URL w/o markdown link - pr["entry"] = re.sub(r'([^(])https://github.com/ClickHouse/ClickHouse/issues/([0-9]{4,})', r'\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)', pr["entry"]) + pr["entry"] = re.sub( + r"([^(])https://github.com/ClickHouse/ClickHouse/issues/([0-9]{4,})", + r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", + pr["entry"], + ) - print(f'* {pr["entry"]} [#{pr["number"]}]({pr["html_url"]}) ([{user_name}]({user["html_url"]})).') + print( + f'* {pr["entry"]} [#{pr["number"]}]({pr["html_url"]}) ([{user_name}]({user["html_url"]})).' + ) print() + # Print categories in preferred order for category in categories_preferred_order: if category in category_to_pr: diff --git a/utils/check-marks/main.cpp b/utils/check-marks/main.cpp index 36b81509046..df6f6e5267e 100644 --- a/utils/check-marks/main.cpp +++ b/utils/check-marks/main.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include diff --git a/utils/check-mysql-binlog/main.cpp b/utils/check-mysql-binlog/main.cpp index 04dfb56ff08..7dd387ba5be 100644 --- a/utils/check-mysql-binlog/main.cpp +++ b/utils/check-mysql-binlog/main.cpp @@ -61,7 +61,9 @@ static DB::MySQLReplication::BinlogEventPtr parseSingleEventBody( } case DB::MySQLReplication::TABLE_MAP_EVENT: { - event = std::make_shared(std::move(header)); + DB::MySQLReplication::TableMapEventHeader map_event_header; + map_event_header.parse(*event_payload); + event = std::make_shared(std::move(header), map_event_header); event->parseEvent(*event_payload); last_table_map_event = std::static_pointer_cast(event); break; diff --git a/utils/check-style/check-black b/utils/check-style/check-black new file mode 100755 index 00000000000..1f0be9375c2 --- /dev/null +++ b/utils/check-style/check-black @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +# We check only our code, that's why we skip contrib +GIT_ROOT=$(git rev-parse --show-cdup) +GIT_ROOT=${GIT_ROOT:-.} +tmp=$(mktemp) +if ! find "$GIT_ROOT" -name '*.py' -not -path "$GIT_ROOT/contrib/*" -exec black --check {} + 1>"$tmp" 2>&1; then + # Show the result only if some files need formatting + cat "$tmp" +fi +rm "$tmp" diff --git a/utils/check-style/check-style b/utils/check-style/check-style index d178778a410..6ebf53cb932 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -74,6 +74,8 @@ EXTERN_TYPES_EXCLUDES=( ProfileEvents::Type ProfileEvents::TypeEnum ProfileEvents::dumpToMapColumn + ProfileEvents::getProfileEvents + ProfileEvents::ThreadIdToCountersSnapshot ProfileEvents::LOCAL_NAME ProfileEvents::CountersIncrement diff --git a/utils/check-style/check-workflows b/utils/check-style/check-workflows index c0399829c28..6e9cb87ed36 100755 --- a/utils/check-style/check-workflows +++ b/utils/check-style/check-workflows @@ -1,6 +1,9 @@ #!/usr/bin/env bash +set -e + GIT_ROOT=$(git rev-parse --show-cdup) +GIT_ROOT=${GIT_ROOT:-.} act --list --directory="$GIT_ROOT" 1>/dev/null 2>&1 || act --list --directory="$GIT_ROOT" 2>&1 -actionlint +actionlint || : diff --git a/utils/clickhouse-diagnostics/README.md b/utils/clickhouse-diagnostics/README.md index a6f8ed298dd..fcc1be2f244 100644 --- a/utils/clickhouse-diagnostics/README.md +++ b/utils/clickhouse-diagnostics/README.md @@ -22,7 +22,7 @@ python3 -m pip install -r requirements.txt ## Usage ``` -./clickhouse-diagnostics +./clickhouse-diagnostics --host localhost --port 8123 --user default --password xxx ``` Example output: diff --git a/utils/clickhouse-diagnostics/clickhouse-diagnostics b/utils/clickhouse-diagnostics/clickhouse-diagnostics old mode 100644 new mode 100755 index 83c0af9cd11..2fe67071c3c --- a/utils/clickhouse-diagnostics/clickhouse-diagnostics +++ b/utils/clickhouse-diagnostics/clickhouse-diagnostics @@ -12,6 +12,7 @@ from datetime import datetime from typing import MutableMapping import jinja2 +from pandas import describe_option import requests import sqlparse import tenacity @@ -498,10 +499,11 @@ class ClickhouseClient: ClickHouse client. """ - def __init__(self, *, host, port=8123, user=None): + def __init__(self, *, host="localhost", port=8123, user="default", password): self._session = requests.Session() if user: self._session.headers['X-ClickHouse-User'] = user + self._session.headers['X-ClickHouse-Key'] = password self._url = f'http://{host}:{port}' self._timeout = 60 self._ch_version = None @@ -598,9 +600,9 @@ class DiagnosticsData: Diagnostics data. """ - def __init__(self, args, host): + def __init__(self, args): self.args = args - self.host = host + self.host = args.host self._sections = [{'section': None, 'data': {}}] def add_string(self, name, value, section=None): @@ -758,15 +760,13 @@ def main(): Program entry point. """ args = parse_args() - - host = socket.getfqdn() timestamp = datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S') - client = ClickhouseClient(host=host) + client = ClickhouseClient(host=args.host, port=args.port, user=args.user, password=args.password) ch_config = ClickhouseConfig.load() version = client.clickhouse_version system_tables = [row[0] for row in execute_query(client, SELECT_SYSTEM_TABLES, format='JSONCompact')['data']] - diagnostics = DiagnosticsData(args, host) + diagnostics = DiagnosticsData(args) diagnostics.add_string('Version', version) diagnostics.add_string('Timestamp', timestamp) diagnostics.add_string('Uptime', execute_query(client, SELECT_UPTIME)) @@ -895,6 +895,10 @@ def parse_args(): parser.add_argument('--normalize-queries', action='store_true', default=False) + parser.add_argument('--host', dest="host", help="clickhouse host") + parser.add_argument('--port', dest="port", default=8123, help="clickhouse http port") + parser.add_argument('--user', dest="user", default="default", help="clickhouse user") + parser.add_argument('--password', dest="password", help="clickhouse password") return parser.parse_args() diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index 7d71e13a6e9..dec1f6fe60f 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -857,7 +857,7 @@ FuncRet likeFunc(DB::ASTPtr ch, std::map & columns) { std::string value = applyVisitor(DB::FieldVisitorToString(), literal->value); std::string example{}; - for (size_t i = 0; i != value.size(); ++i) + for (size_t i = 0; i != value.size(); ++i) /// NOLINT { if (value[i] == '%') example += randomString(rng() % 10); diff --git a/utils/github-hook/hook.py b/utils/github-hook/hook.py deleted file mode 100644 index 1ea65f3c3ab..00000000000 --- a/utils/github-hook/hook.py +++ /dev/null @@ -1,320 +0,0 @@ -# -*- coding: utf-8 -*- -import json -import requests -import time -import os - -DB = 'gh-data' -RETRIES = 5 - -API_URL = 'https://api.github.com/repos/ClickHouse/ClickHouse/' - - -def _reverse_dict_with_list(source): - result = {} - for key, value in list(source.items()): - for elem in value: - result[elem] = key - return result - - -MARKER_TO_LABEL = { - '- New Feature': ['pr-feature'], - '- Bug Fix': ['pr-bugfix'], - '- Improvement': ['pr-improvement'], - '- Performance Improvement': ['pr-performance'], - '- Backward Incompatible Change': ['pr-backward-incompatible'], - '- Build/Testing/Packaging Improvement': ['pr-build'], - '- Documentation': ['pr-documentation', 'pr-doc-fix'], - '- Other': ['pr-other'], - '- Not for changelog': ['pr-not-for-changelog'] -} - -LABEL_TO_MARKER = _reverse_dict_with_list(MARKER_TO_LABEL) - -DOC_ALERT_LABELS = { - 'pr-feature' -} - - -def set_labels_for_pr(pull_request_number, labels, headers): - data = { - "labels": list(labels) - } - - for i in range(RETRIES): - try: - response = requests.put(API_URL + 'issues/' + str(pull_request_number) + '/labels', json=data, headers=headers) - response.raise_for_status() - break - except Exception as ex: - print(("Exception", ex)) - time.sleep(0.2) - - -def get_required_labels_from_desc(description, current_labels): - result = set([]) - # find first matching category - for marker, labels in list(MARKER_TO_LABEL.items()): - if marker in description: - if not any(label in current_labels for label in labels): - result.add(labels[0]) - break - - # if no category than leave as is - if not result: - return current_labels - - # save all old labels except category label - for label in current_labels: - if label not in result and label not in LABEL_TO_MARKER: - result.add(label) - - # if some of labels require doc alert - if any(label in result for label in DOC_ALERT_LABELS): - result.add('doc-alert') - - return result - - -def label_pull_request_event(response): - pull_request = response['pull_request'] - current_labels = set([label['name'] for label in pull_request['labels']]) - pr_description = pull_request['body'] if pull_request['body'] else '' - required_labels = get_required_labels_from_desc(pr_description, current_labels) - if not required_labels.issubset(current_labels): - token = os.getenv('GITHUB_TOKEN') - auth = {'Authorization': 'token ' + token} - set_labels_for_pr(pull_request['number'], required_labels, auth) - - -def process_issue_event(response): - issue = response['issue'] - return dict( - action=response['action'], - sender=response['sender']['login'], - updated_at=issue['updated_at'], - url=issue['url'], - number=issue['number'], - author=issue['user']['login'], - labels=[label['name'] for label in issue['labels']], - state=issue['state'], - assignees=[assignee['login'] for assignee in issue['assignees']], - created_at=issue['created_at'], - body=issue['body'] if issue['body'] else '', - title=issue['title'], - comments=issue['comments'], - raw_json=json.dumps(response),) - - -def process_issue_comment_event(response): - issue = response['issue'] - comment = response['comment'] - - return dict( - action='comment_' + response['action'], - sender=response['sender']['login'], - updated_at=issue['updated_at'], - url=issue['url'], - number=issue['number'], - author=issue['user']['login'], - labels=[label['name'] for label in issue['labels']], - state=issue['state'], - assignees=[assignee['login'] for assignee in issue['assignees']], - created_at=issue['created_at'], - body=issue['body'] if issue['body'] else '', - title=issue['title'], - comments=issue['comments'], - comment_body=comment['body'], - comment_author=comment['user']['login'], - comment_url=comment['url'], - comment_created_at=comment['created_at'], - comment_updated_at=comment['updated_at'], - raw_json=json.dumps(response),) - - -def process_pull_request_event(response): - pull_request = response['pull_request'] - result = dict( - updated_at=pull_request['updated_at'], - number=pull_request['number'], - action=response['action'], - sender=response['sender']['login'], - url=pull_request['url'], - author=pull_request['user']['login'], - labels=[label['name'] for label in pull_request['labels']], - state=pull_request['state'], - body=pull_request['body'] if pull_request['body'] else '', - title=pull_request['title'], - created_at=pull_request['created_at'], - assignees=[assignee['login'] for assignee in pull_request['assignees']], - requested_reviewers=[reviewer['login'] for reviewer in pull_request['requested_reviewers']], - head_repo=pull_request['head']['repo']['full_name'], - head_ref=pull_request['head']['ref'], - head_clone_url=pull_request['head']['repo']['clone_url'], - head_ssh_url=pull_request['head']['repo']['ssh_url'], - base_repo=pull_request['base']['repo']['full_name'], - base_ref=pull_request['base']['ref'], - base_clone_url=pull_request['base']['repo']['clone_url'], - base_ssh_url=pull_request['base']['repo']['ssh_url'], - raw_json=json.dumps(response), - ) - - if 'mergeable' in pull_request and pull_request['mergeable'] is not None: - result['mergeable'] = 1 if pull_request['mergeable'] else 0 - - if 'merged_by' in pull_request and pull_request['merged_by'] is not None: - result['merged_by'] = pull_request['merged_by']['login'] - - if 'merged_at' in pull_request and pull_request['merged_at'] is not None: - result['merged_at'] = pull_request['merged_at'] - - if 'closed_at' in pull_request and pull_request['closed_at'] is not None: - result['closed_at'] = pull_request['closed_at'] - - if 'merge_commit_sha' in pull_request and pull_request['merge_commit_sha'] is not None: - result['merge_commit_sha'] = pull_request['merge_commit_sha'] - - if 'draft' in pull_request: - result['draft'] = 1 if pull_request['draft'] else 0 - - for field in ['comments', 'review_comments', 'commits', 'additions', 'deletions', 'changed_files']: - if field in pull_request: - result[field] = pull_request[field] - - return result - - -def process_pull_request_review(response): - result = process_pull_request_event(response) - review = response['review'] - result['action'] = 'review_' + result['action'] - result['review_body'] = review['body'] if review['body'] is not None else '' - result['review_id'] = review['id'] - result['review_author'] = review['user']['login'] - result['review_commit_sha'] = review['commit_id'] - result['review_submitted_at'] = review['submitted_at'] - result['review_state'] = review['state'] - return result - - -def process_pull_request_review_comment(response): - result = process_pull_request_event(response) - comment = response['comment'] - result['action'] = 'review_comment_' + result['action'] - result['review_id'] = comment['pull_request_review_id'] - result['review_comment_path'] = comment['path'] - result['review_commit_sha'] = comment['commit_id'] - result['review_comment_body'] = comment['body'] - result['review_comment_author'] = comment['user']['login'] - result['review_comment_created_at'] = comment['created_at'] - result['review_comment_updated_at'] = comment['updated_at'] - return result - - -def process_push(response): - common_part = dict( - before_sha=response['before'], - after_sha=response['after'], - full_ref=response['ref'], - ref=response['ref'].split('/')[-1], - repo=response['repository']['full_name'], - pusher=response['pusher']['name'], - sender=response['sender']['login'], - pushed_at=response['repository']['pushed_at'], - raw_json=json.dumps(response), - ) - commits = response['commits'] - result = [] - for commit in commits: - commit_dict = common_part.copy() - commit_dict['sha'] = commit['id'] - commit_dict['tree_sha'] = commit['tree_id'] - commit_dict['author'] = commit['author']['name'] - commit_dict['committer'] = commit['committer']['name'] - commit_dict['message'] = commit['message'] - commit_dict['commited_at'] = commit['timestamp'] - result.append(commit_dict) - return result - - -def event_processor_dispatcher(headers, body, inserter): - if 'X-Github-Event' in headers: - if headers['X-Github-Event'] == 'issues': - result = process_issue_event(body) - inserter.insert_event_into(DB, 'issues', result) - elif headers['X-Github-Event'] == 'issue_comment': - result = process_issue_comment_event(body) - inserter.insert_event_into(DB, 'issues', result) - elif headers['X-Github-Event'] == 'pull_request': - result = process_pull_request_event(body) - inserter.insert_event_into(DB, 'pull_requests', result) - label_pull_request_event(body) - elif headers['X-Github-Event'] == 'pull_request_review': - result = process_pull_request_review(body) - inserter.insert_event_into(DB, 'pull_requests', result) - elif headers['X-Github-Event'] == 'pull_request_review_comment': - result = process_pull_request_review_comment(body) - inserter.insert_event_into(DB, 'pull_requests', result) - elif headers['X-Github-Event'] == 'push': - result = process_push(body) - inserter.insert_events_into(DB, 'commits', result) - - -class ClickHouseInserter(object): - def __init__(self, url, user, password): - self.url = url - self.auth = { - 'X-ClickHouse-User': user, - 'X-ClickHouse-Key': password - } - - def _insert_json_str_info(self, db, table, json_str): - params = { - 'database': db, - 'query': 'INSERT INTO {table} FORMAT JSONEachRow'.format(table=table), - 'date_time_input_format': 'best_effort' - } - for i in range(RETRIES): - response = None - try: - response = requests.post(self.url, params=params, data=json_str, headers=self.auth, verify=False) - response.raise_for_status() - break - except Exception as ex: - print(("Cannot insert with exception %s", str(ex))) - if response: - print(("Response text %s", response.text)) - time.sleep(0.1) - else: - raise Exception("Cannot insert data into clickhouse") - - def insert_event_into(self, db, table, event): - event_str = json.dumps(event) - self._insert_json_str_info(db, table, event_str) - - def insert_events_into(self, db, table, events): - jsons = [] - for event in events: - jsons.append(json.dumps(event)) - - self._insert_json_str_info(db, table, ','.join(jsons)) - - -def test(event, context): - inserter = ClickHouseInserter( - os.getenv('CLICKHOUSE_URL'), - os.getenv('CLICKHOUSE_USER'), - os.getenv('CLICKHOUSE_PASSWORD')) - - body = json.loads(event['body'], strict=False) - headers = event['headers'] - event_processor_dispatcher(headers, body, inserter) - - return { - 'statusCode': 200, - 'headers': { - 'Content-Type': 'text/plain' - }, - 'isBase64Encoded': False, - } diff --git a/utils/github/backport.py b/utils/github/backport.py index 9227dbf4108..615c0d19ffa 100644 --- a/utils/github/backport.py +++ b/utils/github/backport.py @@ -17,7 +17,9 @@ import sys class Backport: def __init__(self, token, owner, name, team): - self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7) + self._gh = RemoteRepo( + token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7 + ) self._token = token self.default_branch_name = self._gh.default_branch self.ssh_url = self._gh.ssh_url @@ -28,7 +30,7 @@ class Backport: def getBranchesWithRelease(self): branches = set() for pull_request in self._gh.find_pull_requests("release"): - branches.add(pull_request['headRefName']) + branches.add(pull_request["headRefName"]) return branches def execute(self, repo, upstream, until_commit, run_cherrypick): @@ -44,11 +46,11 @@ class Backport: branches.append(branch) if not branches: - logging.info('No release branches found!') + logging.info("No release branches found!") return for branch in branches: - logging.info('Found release branch: %s', branch[0]) + logging.info("Found release branch: %s", branch[0]) if not until_commit: until_commit = branches[0][1] @@ -56,73 +58,128 @@ class Backport: backport_map = {} - RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$') - RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$') - RE_BACKPORTED = re.compile(r'^v(\d+\.\d+)-backported$') + RE_MUST_BACKPORT = re.compile(r"^v(\d+\.\d+)-must-backport$") + RE_NO_BACKPORT = re.compile(r"^v(\d+\.\d+)-no-backport$") + RE_BACKPORTED = re.compile(r"^v(\d+\.\d+)-backported$") # pull-requests are sorted by ancestry from the most recent. for pr in pull_requests: - while repo.comparator(branches[-1][1]) >= repo.comparator(pr['mergeCommit']['oid']): - logging.info("PR #{} is already inside {}. Dropping this branch for further PRs".format(pr['number'], branches[-1][0])) + while repo.comparator(branches[-1][1]) >= repo.comparator( + pr["mergeCommit"]["oid"] + ): + logging.info( + "PR #{} is already inside {}. Dropping this branch for further PRs".format( + pr["number"], branches[-1][0] + ) + ) branches.pop() - logging.info("Processing PR #{}".format(pr['number'])) + logging.info("Processing PR #{}".format(pr["number"])) assert len(branches) branch_set = set([branch[0] for branch in branches]) # First pass. Find all must-backports - for label in pr['labels']['nodes']: - if label['name'] == 'pr-must-backport': - backport_map[pr['number']] = branch_set.copy() + for label in pr["labels"]["nodes"]: + if label["name"] == "pr-must-backport": + backport_map[pr["number"]] = branch_set.copy() continue - matched = RE_MUST_BACKPORT.match(label['name']) + matched = RE_MUST_BACKPORT.match(label["name"]) if matched: - if pr['number'] not in backport_map: - backport_map[pr['number']] = set() - backport_map[pr['number']].add(matched.group(1)) + if pr["number"] not in backport_map: + backport_map[pr["number"]] = set() + backport_map[pr["number"]].add(matched.group(1)) # Second pass. Find all no-backports - for label in pr['labels']['nodes']: - if label['name'] == 'pr-no-backport' and pr['number'] in backport_map: - del backport_map[pr['number']] + for label in pr["labels"]["nodes"]: + if label["name"] == "pr-no-backport" and pr["number"] in backport_map: + del backport_map[pr["number"]] break - matched_no_backport = RE_NO_BACKPORT.match(label['name']) - matched_backported = RE_BACKPORTED.match(label['name']) - if matched_no_backport and pr['number'] in backport_map and matched_no_backport.group(1) in backport_map[pr['number']]: - backport_map[pr['number']].remove(matched_no_backport.group(1)) - logging.info('\tskipping %s because of forced no-backport', matched_no_backport.group(1)) - elif matched_backported and pr['number'] in backport_map and matched_backported.group(1) in backport_map[pr['number']]: - backport_map[pr['number']].remove(matched_backported.group(1)) - logging.info('\tskipping %s because it\'s already backported manually', matched_backported.group(1)) + matched_no_backport = RE_NO_BACKPORT.match(label["name"]) + matched_backported = RE_BACKPORTED.match(label["name"]) + if ( + matched_no_backport + and pr["number"] in backport_map + and matched_no_backport.group(1) in backport_map[pr["number"]] + ): + backport_map[pr["number"]].remove(matched_no_backport.group(1)) + logging.info( + "\tskipping %s because of forced no-backport", + matched_no_backport.group(1), + ) + elif ( + matched_backported + and pr["number"] in backport_map + and matched_backported.group(1) in backport_map[pr["number"]] + ): + backport_map[pr["number"]].remove(matched_backported.group(1)) + logging.info( + "\tskipping %s because it's already backported manually", + matched_backported.group(1), + ) for pr, branches in list(backport_map.items()): - logging.info('PR #%s needs to be backported to:', pr) + logging.info("PR #%s needs to be backported to:", pr) for branch in branches: - logging.info('\t%s, and the status is: %s', branch, run_cherrypick(self._token, pr, branch)) + logging.info( + "\t%s, and the status is: %s", + branch, + run_cherrypick(self._token, pr, branch), + ) # print API costs - logging.info('\nGitHub API total costs per query:') + logging.info("\nGitHub API total costs per query:") for name, value in list(self._gh.api_costs.items()): - logging.info('%s : %s', name, value) + logging.info("%s : %s", name, value) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--token', type=str, required=True, help='token for Github access') - parser.add_argument('--repo', type=str, required=True, help='path to full repository', metavar='PATH') - parser.add_argument('--til', type=str, help='check PRs from HEAD til this commit', metavar='COMMIT') - parser.add_argument('--dry-run', action='store_true', help='do not create or merge any PRs', default=False) - parser.add_argument('--verbose', '-v', action='store_true', help='more verbose output', default=False) - parser.add_argument('--upstream', '-u', type=str, help='remote name of upstream in repository', default='origin') + parser.add_argument( + "--token", type=str, required=True, help="token for Github access" + ) + parser.add_argument( + "--repo", + type=str, + required=True, + help="path to full repository", + metavar="PATH", + ) + parser.add_argument( + "--til", type=str, help="check PRs from HEAD til this commit", metavar="COMMIT" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="do not create or merge any PRs", + default=False, + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="more verbose output", + default=False, + ) + parser.add_argument( + "--upstream", + "-u", + type=str, + help="remote name of upstream in repository", + default="origin", + ) args = parser.parse_args() if args.verbose: - logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.DEBUG) + logging.basicConfig( + format="%(message)s", stream=sys.stdout, level=logging.DEBUG + ) else: - logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.INFO) + logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.INFO) - cherrypick_run = lambda token, pr, branch: CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch).execute(args.repo, args.dry_run) - bp = Backport(args.token, 'ClickHouse', 'ClickHouse', 'core') + cherrypick_run = lambda token, pr, branch: CherryPick( + token, "ClickHouse", "ClickHouse", "core", pr, branch + ).execute(args.repo, args.dry_run) + bp = Backport(args.token, "ClickHouse", "ClickHouse", "core") bp.execute(args.repo, args.upstream, args.til, cherrypick_run) diff --git a/utils/github/cherrypick.py b/utils/github/cherrypick.py index 8bedf54fefa..c6469fa62a9 100644 --- a/utils/github/cherrypick.py +++ b/utils/github/cherrypick.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -''' +""" Backports changes from PR to release branch. Requires multiple separate runs as part of the implementation. @@ -12,7 +12,7 @@ First run should do the following: Second run checks PR from previous run to be merged or at least being mergeable. If it's not merged then try to merge it. Third run creates PR from backport branch (with merged previous PR) to release branch. -''' +""" try: from clickhouse.utils.github.query import Query as RemoteRepo @@ -29,13 +29,13 @@ import sys class CherryPick: class Status(Enum): - DISCARDED = 'discarded' - NOT_INITIATED = 'not started' - FIRST_MERGEABLE = 'waiting for 1st stage' - FIRST_CONFLICTS = 'conflicts on 1st stage' - SECOND_MERGEABLE = 'waiting for 2nd stage' - SECOND_CONFLICTS = 'conflicts on 2nd stage' - MERGED = 'backported' + DISCARDED = "discarded" + NOT_INITIATED = "not started" + FIRST_MERGEABLE = "waiting for 1st stage" + FIRST_CONFLICTS = "conflicts on 1st stage" + SECOND_MERGEABLE = "waiting for 2nd stage" + SECOND_CONFLICTS = "conflicts on 2nd stage" + MERGED = "backported" def _run(self, args): out = subprocess.check_output(args).rstrip() @@ -50,51 +50,90 @@ class CherryPick: # TODO: check if pull-request is merged. - self.merge_commit_oid = self._pr['mergeCommit']['oid'] + self.merge_commit_oid = self._pr["mergeCommit"]["oid"] self.target_branch = target_branch - self.backport_branch = 'backport/{branch}/{pr}'.format(branch=target_branch, pr=pr_number) - self.cherrypick_branch = 'cherrypick/{branch}/{oid}'.format(branch=target_branch, oid=self.merge_commit_oid) + self.backport_branch = "backport/{branch}/{pr}".format( + branch=target_branch, pr=pr_number + ) + self.cherrypick_branch = "cherrypick/{branch}/{oid}".format( + branch=target_branch, oid=self.merge_commit_oid + ) def getCherryPickPullRequest(self): - return self._gh.find_pull_request(base=self.backport_branch, head=self.cherrypick_branch) + return self._gh.find_pull_request( + base=self.backport_branch, head=self.cherrypick_branch + ) def createCherryPickPullRequest(self, repo_path): DESCRIPTION = ( - 'This pull-request is a first step of an automated backporting.\n' - 'It contains changes like after calling a local command `git cherry-pick`.\n' - 'If you intend to continue backporting this changes, then resolve all conflicts if any.\n' - 'Otherwise, if you do not want to backport them, then just close this pull-request.\n' - '\n' - 'The check results does not matter at this step - you can safely ignore them.\n' - 'Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n' + "This pull-request is a first step of an automated backporting.\n" + "It contains changes like after calling a local command `git cherry-pick`.\n" + "If you intend to continue backporting this changes, then resolve all conflicts if any.\n" + "Otherwise, if you do not want to backport them, then just close this pull-request.\n" + "\n" + "The check results does not matter at this step - you can safely ignore them.\n" + "Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n" ) # FIXME: replace with something better than os.system() - git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@yandex-team.ru', '-c', 'user.name=robot-clickhouse'] - base_commit_oid = self._pr['mergeCommit']['parents']['nodes'][0]['oid'] + git_prefix = [ + "git", + "-C", + repo_path, + "-c", + "user.email=robot-clickhouse@yandex-team.ru", + "-c", + "user.name=robot-clickhouse", + ] + base_commit_oid = self._pr["mergeCommit"]["parents"]["nodes"][0]["oid"] # Create separate branch for backporting, and make it look like real cherry-pick. - self._run(git_prefix + ['checkout', '-f', self.target_branch]) - self._run(git_prefix + ['checkout', '-B', self.backport_branch]) - self._run(git_prefix + ['merge', '-s', 'ours', '--no-edit', base_commit_oid]) + self._run(git_prefix + ["checkout", "-f", self.target_branch]) + self._run(git_prefix + ["checkout", "-B", self.backport_branch]) + self._run(git_prefix + ["merge", "-s", "ours", "--no-edit", base_commit_oid]) # Create secondary branch to allow pull request with cherry-picked commit. - self._run(git_prefix + ['branch', '-f', self.cherrypick_branch, self.merge_commit_oid]) + self._run( + git_prefix + ["branch", "-f", self.cherrypick_branch, self.merge_commit_oid] + ) - self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)]) - self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.cherrypick_branch)]) + self._run( + git_prefix + + [ + "push", + "-f", + "origin", + "{branch}:{branch}".format(branch=self.backport_branch), + ] + ) + self._run( + git_prefix + + [ + "push", + "-f", + "origin", + "{branch}:{branch}".format(branch=self.cherrypick_branch), + ] + ) # Create pull-request like a local cherry-pick - pr = self._gh.create_pull_request(source=self.cherrypick_branch, target=self.backport_branch, - title='Cherry pick #{number} to {target}: {title}'.format( - number=self._pr['number'], target=self.target_branch, - title=self._pr['title'].replace('"', '\\"')), - description='Original pull-request #{}\n\n{}'.format(self._pr['number'], DESCRIPTION)) + pr = self._gh.create_pull_request( + source=self.cherrypick_branch, + target=self.backport_branch, + title="Cherry pick #{number} to {target}: {title}".format( + number=self._pr["number"], + target=self.target_branch, + title=self._pr["title"].replace('"', '\\"'), + ), + description="Original pull-request #{}\n\n{}".format( + self._pr["number"], DESCRIPTION + ), + ) # FIXME: use `team` to leave a single eligible assignee. - self._gh.add_assignee(pr, self._pr['author']) - self._gh.add_assignee(pr, self._pr['mergedBy']) + self._gh.add_assignee(pr, self._pr["author"]) + self._gh.add_assignee(pr, self._pr["mergedBy"]) self._gh.set_label(pr, "do not test") self._gh.set_label(pr, "pr-cherrypick") @@ -102,36 +141,76 @@ class CherryPick: return pr def mergeCherryPickPullRequest(self, cherrypick_pr): - return self._gh.merge_pull_request(cherrypick_pr['id']) + return self._gh.merge_pull_request(cherrypick_pr["id"]) def getBackportPullRequest(self): - return self._gh.find_pull_request(base=self.target_branch, head=self.backport_branch) + return self._gh.find_pull_request( + base=self.target_branch, head=self.backport_branch + ) def createBackportPullRequest(self, cherrypick_pr, repo_path): DESCRIPTION = ( - 'This pull-request is a last step of an automated backporting.\n' - 'Treat it as a standard pull-request: look at the checks and resolve conflicts.\n' - 'Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n' + "This pull-request is a last step of an automated backporting.\n" + "Treat it as a standard pull-request: look at the checks and resolve conflicts.\n" + "Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n" ) - git_prefix = ['git', '-C', repo_path, '-c', 'user.email=robot-clickhouse@clickhouse.com', '-c', 'user.name=robot-clickhouse'] + git_prefix = [ + "git", + "-C", + repo_path, + "-c", + "user.email=robot-clickhouse@clickhouse.com", + "-c", + "user.name=robot-clickhouse", + ] - pr_title = 'Backport #{number} to {target}: {title}'.format( - number=self._pr['number'], target=self.target_branch, - title=self._pr['title'].replace('"', '\\"')) + pr_title = "Backport #{number} to {target}: {title}".format( + number=self._pr["number"], + target=self.target_branch, + title=self._pr["title"].replace('"', '\\"'), + ) - self._run(git_prefix + ['checkout', '-f', self.backport_branch]) - self._run(git_prefix + ['pull', '--ff-only', 'origin', self.backport_branch]) - self._run(git_prefix + ['reset', '--soft', self._run(git_prefix + ['merge-base', 'origin/' + self.target_branch, self.backport_branch])]) - self._run(git_prefix + ['commit', '-a', '--allow-empty', '-m', pr_title]) - self._run(git_prefix + ['push', '-f', 'origin', '{branch}:{branch}'.format(branch=self.backport_branch)]) + self._run(git_prefix + ["checkout", "-f", self.backport_branch]) + self._run(git_prefix + ["pull", "--ff-only", "origin", self.backport_branch]) + self._run( + git_prefix + + [ + "reset", + "--soft", + self._run( + git_prefix + + [ + "merge-base", + "origin/" + self.target_branch, + self.backport_branch, + ] + ), + ] + ) + self._run(git_prefix + ["commit", "-a", "--allow-empty", "-m", pr_title]) + self._run( + git_prefix + + [ + "push", + "-f", + "origin", + "{branch}:{branch}".format(branch=self.backport_branch), + ] + ) - pr = self._gh.create_pull_request(source=self.backport_branch, target=self.target_branch, title=pr_title, - description='Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION)) + pr = self._gh.create_pull_request( + source=self.backport_branch, + target=self.target_branch, + title=pr_title, + description="Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}".format( + self._pr["number"], cherrypick_pr["number"], DESCRIPTION + ), + ) # FIXME: use `team` to leave a single eligible assignee. - self._gh.add_assignee(pr, self._pr['author']) - self._gh.add_assignee(pr, self._pr['mergedBy']) + self._gh.add_assignee(pr, self._pr["author"]) + self._gh.add_assignee(pr, self._pr["mergedBy"]) self._gh.set_label(pr, "pr-backport") @@ -142,23 +221,43 @@ class CherryPick: if not pr1: if not dry_run: pr1 = self.createCherryPickPullRequest(repo_path) - logging.debug('Created PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + logging.debug( + "Created PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) else: return CherryPick.Status.NOT_INITIATED else: - logging.debug('Found PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + logging.debug( + "Found PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) - if not pr1['merged'] and pr1['mergeable'] == 'MERGEABLE' and not pr1['closed']: + if not pr1["merged"] and pr1["mergeable"] == "MERGEABLE" and not pr1["closed"]: if not dry_run: pr1 = self.mergeCherryPickPullRequest(pr1) - logging.debug('Merged PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + logging.debug( + "Merged PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) - if not pr1['merged']: - logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) + if not pr1["merged"]: + logging.debug( + "Waiting for PR with cherry-pick of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr1["url"], + ) - if pr1['closed']: + if pr1["closed"]: return CherryPick.Status.DISCARDED - elif pr1['mergeable'] == 'CONFLICTING': + elif pr1["mergeable"] == "CONFLICTING": return CherryPick.Status.FIRST_CONFLICTS else: return CherryPick.Status.FIRST_MERGEABLE @@ -167,31 +266,58 @@ class CherryPick: if not pr2: if not dry_run: pr2 = self.createBackportPullRequest(pr1, repo_path) - logging.debug('Created PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) + logging.debug( + "Created PR with backport of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr2["url"], + ) else: return CherryPick.Status.FIRST_MERGEABLE else: - logging.debug('Found PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) + logging.debug( + "Found PR with backport of %s to %s: %s", + self._pr["number"], + self.target_branch, + pr2["url"], + ) - if pr2['merged']: + if pr2["merged"]: return CherryPick.Status.MERGED - elif pr2['closed']: + elif pr2["closed"]: return CherryPick.Status.DISCARDED - elif pr2['mergeable'] == 'CONFLICTING': + elif pr2["mergeable"] == "CONFLICTING": return CherryPick.Status.SECOND_CONFLICTS else: return CherryPick.Status.SECOND_MERGEABLE if __name__ == "__main__": - logging.basicConfig(format='%(message)s', stream=sys.stdout, level=logging.DEBUG) + logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG) parser = argparse.ArgumentParser() - parser.add_argument('--token', '-t', type=str, required=True, help='token for Github access') - parser.add_argument('--pr', type=str, required=True, help='PR# to cherry-pick') - parser.add_argument('--branch', '-b', type=str, required=True, help='target branch name for cherry-pick') - parser.add_argument('--repo', '-r', type=str, required=True, help='path to full repository', metavar='PATH') + parser.add_argument( + "--token", "-t", type=str, required=True, help="token for Github access" + ) + parser.add_argument("--pr", type=str, required=True, help="PR# to cherry-pick") + parser.add_argument( + "--branch", + "-b", + type=str, + required=True, + help="target branch name for cherry-pick", + ) + parser.add_argument( + "--repo", + "-r", + type=str, + required=True, + help="path to full repository", + metavar="PATH", + ) args = parser.parse_args() - cp = CherryPick(args.token, 'ClickHouse', 'ClickHouse', 'core', args.pr, args.branch) + cp = CherryPick( + args.token, "ClickHouse", "ClickHouse", "core", args.pr, args.branch + ) cp.execute(args.repo) diff --git a/utils/github/local.py b/utils/github/local.py index 2ad8d4b8b71..571c9102ba0 100644 --- a/utils/github/local.py +++ b/utils/github/local.py @@ -20,13 +20,14 @@ class RepositoryBase: return -1 else: return 1 + self.comparator = functools.cmp_to_key(cmp) def get_head_commit(self): return self._repo.commit(self._default) def iterate(self, begin, end): - rev_range = '{}...{}'.format(begin, end) + rev_range = "{}...{}".format(begin, end) for commit in self._repo.iter_commits(rev_range, first_parent=True): yield commit @@ -39,27 +40,35 @@ class Repository(RepositoryBase): self._default = self._remote.refs[default_branch_name] def get_release_branches(self): - ''' + """ Returns sorted list of tuples: * remote branch (git.refs.remote.RemoteReference), * base commit (git.Commit), * head (git.Commit)). List is sorted by commits in ascending order. - ''' + """ release_branches = [] - RE_RELEASE_BRANCH_REF = re.compile(r'^refs/remotes/.+/\d+\.\d+$') + RE_RELEASE_BRANCH_REF = re.compile(r"^refs/remotes/.+/\d+\.\d+$") - for branch in [r for r in self._remote.refs if RE_RELEASE_BRANCH_REF.match(r.path)]: + for branch in [ + r for r in self._remote.refs if RE_RELEASE_BRANCH_REF.match(r.path) + ]: base = self._repo.merge_base(self._default, self._repo.commit(branch)) if not base: - logging.info('Branch %s is not based on branch %s. Ignoring.', branch.path, self._default) + logging.info( + "Branch %s is not based on branch %s. Ignoring.", + branch.path, + self._default, + ) elif len(base) > 1: - logging.info('Branch %s has more than one base commit. Ignoring.', branch.path) + logging.info( + "Branch %s has more than one base commit. Ignoring.", branch.path + ) else: release_branches.append((os.path.basename(branch.name), base[0])) - return sorted(release_branches, key=lambda x : self.comparator(x[1])) + return sorted(release_branches, key=lambda x: self.comparator(x[1])) class BareRepository(RepositoryBase): @@ -68,24 +77,32 @@ class BareRepository(RepositoryBase): self._default = self._repo.branches[default_branch_name] def get_release_branches(self): - ''' + """ Returns sorted list of tuples: * branch (git.refs.head?), * base commit (git.Commit), * head (git.Commit)). List is sorted by commits in ascending order. - ''' + """ release_branches = [] - RE_RELEASE_BRANCH_REF = re.compile(r'^refs/heads/\d+\.\d+$') + RE_RELEASE_BRANCH_REF = re.compile(r"^refs/heads/\d+\.\d+$") - for branch in [r for r in self._repo.branches if RE_RELEASE_BRANCH_REF.match(r.path)]: + for branch in [ + r for r in self._repo.branches if RE_RELEASE_BRANCH_REF.match(r.path) + ]: base = self._repo.merge_base(self._default, self._repo.commit(branch)) if not base: - logging.info('Branch %s is not based on branch %s. Ignoring.', branch.path, self._default) + logging.info( + "Branch %s is not based on branch %s. Ignoring.", + branch.path, + self._default, + ) elif len(base) > 1: - logging.info('Branch %s has more than one base commit. Ignoring.', branch.path) + logging.info( + "Branch %s has more than one base commit. Ignoring.", branch.path + ) else: release_branches.append((os.path.basename(branch.name), base[0])) - return sorted(release_branches, key=lambda x : self.comparator(x[1])) + return sorted(release_branches, key=lambda x: self.comparator(x[1])) diff --git a/utils/github/parser.py b/utils/github/parser.py index 570410ba23d..d8348e6d964 100644 --- a/utils/github/parser.py +++ b/utils/github/parser.py @@ -1,19 +1,20 @@ # -*- coding: utf-8 -*- + class Description: - '''Parsed description representation - ''' + """Parsed description representation""" + MAP_CATEGORY_TO_LABEL = { - 'New Feature': 'pr-feature', - 'Bug Fix': 'pr-bugfix', - 'Improvement': 'pr-improvement', - 'Performance Improvement': 'pr-performance', + "New Feature": "pr-feature", + "Bug Fix": "pr-bugfix", + "Improvement": "pr-improvement", + "Performance Improvement": "pr-performance", # 'Backward Incompatible Change': doesn't match anything - 'Build/Testing/Packaging Improvement': 'pr-build', - 'Non-significant (changelog entry is not needed)': 'pr-non-significant', - 'Non-significant (changelog entry is not required)': 'pr-non-significant', - 'Non-significant': 'pr-non-significant', - 'Documentation (changelog entry is not required)': 'pr-documentation', + "Build/Testing/Packaging Improvement": "pr-build", + "Non-significant (changelog entry is not needed)": "pr-non-significant", + "Non-significant (changelog entry is not required)": "pr-non-significant", + "Non-significant": "pr-non-significant", + "Documentation (changelog entry is not required)": "pr-documentation", # 'Other': doesn't match anything } @@ -21,7 +22,7 @@ class Description: self.label_name = str() self.legal = False - self._parse(pull_request['bodyText']) + self._parse(pull_request["bodyText"]) def _parse(self, text): lines = text.splitlines() @@ -38,14 +39,17 @@ class Description: category = stripped next_category = False - if stripped == 'I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en': + if ( + stripped + == "I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en" + ): self.legal = True category_headers = ( - 'Category (leave one):', - 'Changelog category (leave one):', - 'Changelog category:', - 'Category:' + "Category (leave one):", + "Changelog category (leave one):", + "Changelog category:", + "Category:", ) if stripped in category_headers: @@ -55,6 +59,6 @@ class Description: self.label_name = Description.MAP_CATEGORY_TO_LABEL[category] else: if not category: - print('Cannot find category in pr description') + print("Cannot find category in pr description") else: - print(('Unknown category: ' + category)) + print(("Unknown category: " + category)) diff --git a/utils/github/query.py b/utils/github/query.py index 39b1d0ce003..7afbc57781c 100644 --- a/utils/github/query.py +++ b/utils/github/query.py @@ -4,11 +4,11 @@ import requests class Query: - ''' + """ Implements queries to the Github API using GraphQL - ''' + """ - _PULL_REQUEST = ''' + _PULL_REQUEST = """ author {{ ... on User {{ id @@ -46,7 +46,7 @@ class Query: number title url - ''' + """ def __init__(self, token, owner, name, team, max_page_size=100, min_page_size=10): self._PULL_REQUEST = Query._PULL_REQUEST.format(min_page_size=min_page_size) @@ -62,14 +62,14 @@ class Query: self.api_costs = {} repo = self.get_repository() - self._id = repo['id'] - self.ssh_url = repo['sshUrl'] - self.default_branch = repo['defaultBranchRef']['name'] + self._id = repo["id"] + self.ssh_url = repo["sshUrl"] + self.default_branch = repo["defaultBranchRef"]["name"] self.members = set(self.get_members()) def get_repository(self): - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ defaultBranchRef {{ name @@ -77,19 +77,19 @@ class Query: id sshUrl }} - ''' + """ query = _QUERY.format(owner=self._owner, name=self._name) - return self._run(query)['repository'] + return self._run(query)["repository"] def get_members(self): - '''Get all team members for organization + """Get all team members for organization Returns: members: a map of members' logins to ids - ''' + """ - _QUERY = ''' + _QUERY = """ organization(login: "{organization}") {{ team(slug: "{team}") {{ members(first: {max_page_size} {next}) {{ @@ -104,43 +104,54 @@ class Query: }} }} }} - ''' + """ members = {} not_end = True - query = _QUERY.format(organization=self._owner, team=self._team, - max_page_size=self._max_page_size, - next='') + query = _QUERY.format( + organization=self._owner, + team=self._team, + max_page_size=self._max_page_size, + next="", + ) while not_end: - result = self._run(query)['organization']['team'] + result = self._run(query)["organization"]["team"] if result is None: break - result = result['members'] - not_end = result['pageInfo']['hasNextPage'] - query = _QUERY.format(organization=self._owner, team=self._team, - max_page_size=self._max_page_size, - next='after: "{}"'.format(result["pageInfo"]["endCursor"])) + result = result["members"] + not_end = result["pageInfo"]["hasNextPage"] + query = _QUERY.format( + organization=self._owner, + team=self._team, + max_page_size=self._max_page_size, + next='after: "{}"'.format(result["pageInfo"]["endCursor"]), + ) - members += dict([(node['login'], node['id']) for node in result['nodes']]) + members += dict([(node["login"], node["id"]) for node in result["nodes"]]) return members def get_pull_request(self, number): - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ pullRequest(number: {number}) {{ {pull_request_data} }} }} - ''' + """ - query = _QUERY.format(owner=self._owner, name=self._name, number=number, - pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) - return self._run(query)['repository']['pullRequest'] + query = _QUERY.format( + owner=self._owner, + name=self._name, + number=number, + pull_request_data=self._PULL_REQUEST, + min_page_size=self._min_page_size, + ) + return self._run(query)["repository"]["pullRequest"] def find_pull_request(self, base, head): - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ pullRequests(first: {min_page_size} baseRefName: "{base}" headRefName: "{head}") {{ nodes {{ @@ -149,21 +160,27 @@ class Query: totalCount }} }} - ''' + """ - query = _QUERY.format(owner=self._owner, name=self._name, base=base, head=head, - pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) - result = self._run(query)['repository']['pullRequests'] - if result['totalCount'] > 0: - return result['nodes'][0] + query = _QUERY.format( + owner=self._owner, + name=self._name, + base=base, + head=head, + pull_request_data=self._PULL_REQUEST, + min_page_size=self._min_page_size, + ) + result = self._run(query)["repository"]["pullRequests"] + if result["totalCount"] > 0: + return result["nodes"][0] else: return {} def find_pull_requests(self, label_name): - ''' + """ Get all pull-requests filtered by label name - ''' - _QUERY = ''' + """ + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ pullRequests(first: {min_page_size} labels: "{label_name}" states: OPEN) {{ nodes {{ @@ -171,18 +188,23 @@ class Query: }} }} }} - ''' + """ - query = _QUERY.format(owner=self._owner, name=self._name, label_name=label_name, - pull_request_data=self._PULL_REQUEST, min_page_size=self._min_page_size) - return self._run(query)['repository']['pullRequests']['nodes'] + query = _QUERY.format( + owner=self._owner, + name=self._name, + label_name=label_name, + pull_request_data=self._PULL_REQUEST, + min_page_size=self._min_page_size, + ) + return self._run(query)["repository"]["pullRequests"]["nodes"] def get_pull_requests(self, before_commit): - ''' + """ Get all merged pull-requests from the HEAD of default branch to the last commit (excluding) - ''' + """ - _QUERY = ''' + _QUERY = """ repository(owner: "{owner}" name: "{name}") {{ defaultBranchRef {{ target {{ @@ -220,44 +242,60 @@ class Query: }} }} }} - ''' + """ pull_requests = [] not_end = True - query = _QUERY.format(owner=self._owner, name=self._name, - max_page_size=self._max_page_size, - min_page_size=self._min_page_size, - pull_request_data=self._PULL_REQUEST, - next='') + query = _QUERY.format( + owner=self._owner, + name=self._name, + max_page_size=self._max_page_size, + min_page_size=self._min_page_size, + pull_request_data=self._PULL_REQUEST, + next="", + ) while not_end: - result = self._run(query)['repository']['defaultBranchRef']['target']['history'] - not_end = result['pageInfo']['hasNextPage'] - query = _QUERY.format(owner=self._owner, name=self._name, - max_page_size=self._max_page_size, - min_page_size=self._min_page_size, - pull_request_data=self._PULL_REQUEST, - next='after: "{}"'.format(result["pageInfo"]["endCursor"])) + result = self._run(query)["repository"]["defaultBranchRef"]["target"][ + "history" + ] + not_end = result["pageInfo"]["hasNextPage"] + query = _QUERY.format( + owner=self._owner, + name=self._name, + max_page_size=self._max_page_size, + min_page_size=self._min_page_size, + pull_request_data=self._PULL_REQUEST, + next='after: "{}"'.format(result["pageInfo"]["endCursor"]), + ) - for commit in result['nodes']: + for commit in result["nodes"]: # FIXME: maybe include `before_commit`? - if str(commit['oid']) == str(before_commit): + if str(commit["oid"]) == str(before_commit): not_end = False break # TODO: fetch all pull-requests that were merged in a single commit. - assert commit['associatedPullRequests']['totalCount'] <= self._min_page_size + assert ( + commit["associatedPullRequests"]["totalCount"] + <= self._min_page_size + ) - for pull_request in commit['associatedPullRequests']['nodes']: - if(pull_request['baseRepository']['nameWithOwner'] == '{}/{}'.format(self._owner, self._name) and - pull_request['baseRefName'] == self.default_branch and - pull_request['mergeCommit']['oid'] == commit['oid']): + for pull_request in commit["associatedPullRequests"]["nodes"]: + if ( + pull_request["baseRepository"]["nameWithOwner"] + == "{}/{}".format(self._owner, self._name) + and pull_request["baseRefName"] == self.default_branch + and pull_request["mergeCommit"]["oid"] == commit["oid"] + ): pull_requests.append(pull_request) return pull_requests - def create_pull_request(self, source, target, title, description="", draft=False, can_modify=True): - _QUERY = ''' + def create_pull_request( + self, source, target, title, description="", draft=False, can_modify=True + ): + _QUERY = """ createPullRequest(input: {{ baseRefName: "{target}", headRefName: "{source}", @@ -271,15 +309,22 @@ class Query: {pull_request_data} }} }} - ''' + """ - query = _QUERY.format(target=target, source=source, id=self._id, title=title, body=description, - draft="true" if draft else "false", modify="true" if can_modify else "false", - pull_request_data=self._PULL_REQUEST) - return self._run(query, is_mutation=True)['createPullRequest']['pullRequest'] + query = _QUERY.format( + target=target, + source=source, + id=self._id, + title=title, + body=description, + draft="true" if draft else "false", + modify="true" if can_modify else "false", + pull_request_data=self._PULL_REQUEST, + ) + return self._run(query, is_mutation=True)["createPullRequest"]["pullRequest"] def merge_pull_request(self, id): - _QUERY = ''' + _QUERY = """ mergePullRequest(input: {{ pullRequestId: "{id}" }}) {{ @@ -287,35 +332,35 @@ class Query: {pull_request_data} }} }} - ''' + """ query = _QUERY.format(id=id, pull_request_data=self._PULL_REQUEST) - return self._run(query, is_mutation=True)['mergePullRequest']['pullRequest'] + return self._run(query, is_mutation=True)["mergePullRequest"]["pullRequest"] # FIXME: figure out how to add more assignees at once def add_assignee(self, pr, assignee): - _QUERY = ''' + _QUERY = """ addAssigneesToAssignable(input: {{ assignableId: "{id1}", assigneeIds: "{id2}" }}) {{ clientMutationId }} - ''' + """ - query = _QUERY.format(id1=pr['id'], id2=assignee['id']) + query = _QUERY.format(id1=pr["id"], id2=assignee["id"]) self._run(query, is_mutation=True) def set_label(self, pull_request, label_name): - ''' + """ Set label by name to the pull request Args: pull_request: JSON object returned by `get_pull_requests()` label_name (string): label name - ''' + """ - _GET_LABEL = ''' + _GET_LABEL = """ repository(owner: "{owner}" name: "{name}") {{ labels(first: {max_page_size} {next} query: "{label_name}") {{ pageInfo {{ @@ -329,36 +374,44 @@ class Query: }} }} }} - ''' + """ - _SET_LABEL = ''' + _SET_LABEL = """ addLabelsToLabelable(input: {{ labelableId: "{pr_id}", labelIds: "{label_id}" }}) {{ clientMutationId }} - ''' + """ labels = [] not_end = True - query = _GET_LABEL.format(owner=self._owner, name=self._name, label_name=label_name, - max_page_size=self._max_page_size, - next='') + query = _GET_LABEL.format( + owner=self._owner, + name=self._name, + label_name=label_name, + max_page_size=self._max_page_size, + next="", + ) while not_end: - result = self._run(query)['repository']['labels'] - not_end = result['pageInfo']['hasNextPage'] - query = _GET_LABEL.format(owner=self._owner, name=self._name, label_name=label_name, - max_page_size=self._max_page_size, - next='after: "{}"'.format(result["pageInfo"]["endCursor"])) + result = self._run(query)["repository"]["labels"] + not_end = result["pageInfo"]["hasNextPage"] + query = _GET_LABEL.format( + owner=self._owner, + name=self._name, + label_name=label_name, + max_page_size=self._max_page_size, + next='after: "{}"'.format(result["pageInfo"]["endCursor"]), + ) - labels += [label for label in result['nodes']] + labels += [label for label in result["nodes"]] if not labels: return - query = _SET_LABEL.format(pr_id=pull_request['id'], label_id=labels[0]['id']) + query = _SET_LABEL.format(pr_id=pull_request["id"], label_id=labels[0]["id"]) self._run(query, is_mutation=True) def _run(self, query, is_mutation=False): @@ -380,19 +433,21 @@ class Query: status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) return session - headers = {'Authorization': 'bearer {}'.format(self._token)} + headers = {"Authorization": "bearer {}".format(self._token)} if is_mutation: - query = ''' + query = """ mutation {{ {query} }} - '''.format(query=query) + """.format( + query=query + ) else: - query = ''' + query = """ query {{ {query} rateLimit {{ @@ -400,23 +455,38 @@ class Query: remaining }} }} - '''.format(query=query) + """.format( + query=query + ) while True: - request = requests_retry_session().post('https://api.github.com/graphql', json={'query': query}, headers=headers) + request = requests_retry_session().post( + "https://api.github.com/graphql", json={"query": query}, headers=headers + ) if request.status_code == 200: result = request.json() - if 'errors' in result: - raise Exception('Errors occurred: {}\nOriginal query: {}'.format(result["errors"], query)) + if "errors" in result: + raise Exception( + "Errors occurred: {}\nOriginal query: {}".format( + result["errors"], query + ) + ) if not is_mutation: import inspect + caller = inspect.getouterframes(inspect.currentframe(), 2)[1][3] if caller not in list(self.api_costs.keys()): self.api_costs[caller] = 0 - self.api_costs[caller] += result['data']['rateLimit']['cost'] + self.api_costs[caller] += result["data"]["rateLimit"]["cost"] - return result['data'] + return result["data"] else: import json - raise Exception('Query failed with code {code}:\n{json}'.format(code=request.status_code, json=json.dumps(request.json(), indent=4))) + + raise Exception( + "Query failed with code {code}:\n{json}".format( + code=request.status_code, + json=json.dumps(request.json(), indent=4), + ) + ) diff --git a/utils/graphite-rollup/CMakeLists.txt b/utils/graphite-rollup/CMakeLists.txt index 3cc0d3e756f..d5dca3db32e 100644 --- a/utils/graphite-rollup/CMakeLists.txt +++ b/utils/graphite-rollup/CMakeLists.txt @@ -17,7 +17,7 @@ target_include_directories( ${ClickHouse_SOURCE_DIR}/contrib/double-conversion ${ClickHouse_SOURCE_DIR}/contrib/dragonbox/include ${ClickHouse_SOURCE_DIR}/contrib/fmtlib/include ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include - ${RE2_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/re2_st + ${RE2_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/re2-cmake ) target_compile_definitions(graphite-rollup-bench PRIVATE RULES_DIR="${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/utils/graphite-rollup/graphite-rollup-bench.cpp b/utils/graphite-rollup/graphite-rollup-bench.cpp index dabe0353b0f..4c11f90b3ff 100644 --- a/utils/graphite-rollup/graphite-rollup-bench.cpp +++ b/utils/graphite-rollup/graphite-rollup-bench.cpp @@ -35,7 +35,7 @@ std::vector loadMetrics(const std::string & metrics_file) throw std::runtime_error(strerror(errno)); } - while ((nread = getline(&line, &len, stream)) != -1) + while ((nread = getline(&line, &len, stream)) != -1) /// NOLINT { size_t l = strlen(line); if (l > 0) diff --git a/utils/grpc-client/clickhouse-grpc-client.py b/utils/grpc-client/clickhouse-grpc-client.py index dfaa7ed4e01..0caa9e6fdca 100755 --- a/utils/grpc-client/clickhouse-grpc-client.py +++ b/utils/grpc-client/clickhouse-grpc-client.py @@ -14,14 +14,14 @@ import grpc # pip3 install grpcio import grpc_tools # pip3 install grpcio-tools import argparse, cmd, os, signal, subprocess, sys, threading, time, uuid -DEFAULT_HOST = 'localhost' +DEFAULT_HOST = "localhost" DEFAULT_PORT = 9100 -DEFAULT_USER_NAME = 'default' -DEFAULT_OUTPUT_FORMAT_FOR_INTERACTIVE_MODE = 'PrettyCompact' -HISTORY_FILENAME = '~/.clickhouse_grpc_history' +DEFAULT_USER_NAME = "default" +DEFAULT_OUTPUT_FORMAT_FOR_INTERACTIVE_MODE = "PrettyCompact" +HISTORY_FILENAME = "~/.clickhouse_grpc_history" HISTORY_SIZE = 1000 STDIN_BUFFER_SIZE = 1048576 -DEFAULT_ENCODING = 'utf-8' +DEFAULT_ENCODING = "utf-8" class ClickHouseGRPCError(Exception): @@ -51,10 +51,20 @@ def error_print(*args, **kwargs): class ClickHouseGRPCClient(cmd.Cmd): - prompt="grpc :) " + prompt = "grpc :) " - def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, user_name=DEFAULT_USER_NAME, password='', - database='', output_format='', settings='', verbatim=False, show_debug_info=False): + def __init__( + self, + host=DEFAULT_HOST, + port=DEFAULT_PORT, + user_name=DEFAULT_USER_NAME, + password="", + database="", + output_format="", + settings="", + verbatim=False, + show_debug_info=False, + ): super(ClickHouseGRPCClient, self).__init__(completekey=None) self.host = host self.port = port @@ -80,11 +90,20 @@ class ClickHouseGRPCClient(cmd.Cmd): # Executes a simple query and returns its output. def get_simple_query_output(self, query_text): - result = self.stub.ExecuteQuery(clickhouse_grpc_pb2.QueryInfo(query=query_text, user_name=self.user_name, password=self.password, - database=self.database, output_format='TabSeparated', settings=self.settings, - session_id=self.session_id, query_id=str(uuid.uuid4()))) + result = self.stub.ExecuteQuery( + clickhouse_grpc_pb2.QueryInfo( + query=query_text, + user_name=self.user_name, + password=self.password, + database=self.database, + output_format="TabSeparated", + settings=self.settings, + session_id=self.session_id, + query_id=str(uuid.uuid4()), + ) + ) if self.show_debug_info: - print('\nresult={}'.format(result)) + print("\nresult={}".format(result)) ClickHouseGRPCClient.__check_no_errors(result) return result.output.decode(DEFAULT_ENCODING) @@ -110,11 +129,19 @@ class ClickHouseGRPCClient(cmd.Cmd): with KeyboardInterruptHandlerOverride(keyboard_interrupt_handler): try: + def send_query_info(): # send main query info - info = clickhouse_grpc_pb2.QueryInfo(query=query_text, user_name=self.user_name, password=self.password, - database=self.database, output_format=self.output_format, settings=self.settings, - session_id=self.session_id, query_id=str(uuid.uuid4())) + info = clickhouse_grpc_pb2.QueryInfo( + query=query_text, + user_name=self.user_name, + password=self.password, + database=self.database, + output_format=self.output_format, + settings=self.settings, + session_id=self.session_id, + query_id=str(uuid.uuid4()), + ) # send input data if not sys.stdin.isatty(): while True: @@ -130,10 +157,10 @@ class ClickHouseGRPCClient(cmd.Cmd): cancel_event.wait() if cancel_tries > 0: yield clickhouse_grpc_pb2.QueryInfo(cancel=True) - + for result in self.stub.ExecuteQueryWithStreamIO(send_query_info()): if self.show_debug_info: - print('\nresult={}'.format(result)) + print("\nresult={}".format(result)) ClickHouseGRPCClient.__check_no_errors(result) sys.stdout.buffer.write(result.output) sys.stdout.flush() @@ -144,7 +171,11 @@ class ClickHouseGRPCClient(cmd.Cmd): cancel_event.set() if not cancelled: execution_time = time.time() - start_time - self.verbatim_print('\nElapsed: {execution_time} sec.\n'.format(execution_time=execution_time)) + self.verbatim_print( + "\nElapsed: {execution_time} sec.\n".format( + execution_time=execution_time + ) + ) except Exception as e: if raise_exceptions: @@ -153,24 +184,38 @@ class ClickHouseGRPCClient(cmd.Cmd): # Establish connection. def __connect(self): - self.verbatim_print("Connecting to {host}:{port} as user {user_name}.".format(host=self.host, port=self.port, user_name=self.user_name)) + self.verbatim_print( + "Connecting to {host}:{port} as user {user_name}.".format( + host=self.host, port=self.port, user_name=self.user_name + ) + ) # Secure channels are supported by server but not supported by this client. start_time = time.time() - self.channel = grpc.insecure_channel(self.host + ':' + str(self.port)) + self.channel = grpc.insecure_channel(self.host + ":" + str(self.port)) connection_time = 0 - timeout=5 + timeout = 5 while True: try: grpc.channel_ready_future(self.channel).result(timeout=timeout) - break; + break except grpc.FutureTimeoutError: connection_time += timeout - self.verbatim_print("Couldn't connect to ClickHouse server in {connection_time} seconds.".format(connection_time=connection_time)) + self.verbatim_print( + "Couldn't connect to ClickHouse server in {connection_time} seconds.".format( + connection_time=connection_time + ) + ) self.stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(self.channel) connection_time = time.time() - start_time if self.verbatim: - version = self.get_simple_query_output("SELECT version() FORMAT TabSeparated").rstrip('\n') - self.verbatim_print("Connected to ClickHouse server version {version} via gRPC protocol in {connection_time}.".format(version=version, connection_time=connection_time)) + version = self.get_simple_query_output( + "SELECT version() FORMAT TabSeparated" + ).rstrip("\n") + self.verbatim_print( + "Connected to ClickHouse server version {version} via gRPC protocol in {connection_time}.".format( + version=version, connection_time=connection_time + ) + ) def __disconnect(self): if self.channel: @@ -181,32 +226,39 @@ class ClickHouseGRPCClient(cmd.Cmd): @staticmethod def __check_no_errors(result): - if result.HasField('exception'): + if result.HasField("exception"): raise ClickHouseGRPCError(result.exception.display_text) # Use grpcio-tools to generate *pb2.py files from *.proto. @staticmethod def __generate_pb2(): script_dir = os.path.dirname(os.path.realpath(__file__)) - proto_dir = os.path.join(script_dir, './protos') - gen_dir = os.path.join(script_dir, './_gen') - if os.path.exists(os.path.join(gen_dir, 'clickhouse_grpc_pb2_grpc.py')): + proto_dir = os.path.join(script_dir, "./protos") + gen_dir = os.path.join(script_dir, "./_gen") + if os.path.exists(os.path.join(gen_dir, "clickhouse_grpc_pb2_grpc.py")): return os.makedirs(gen_dir, exist_ok=True) - cmd = ['python3', '-m', 'grpc_tools.protoc', '-I'+proto_dir, '--python_out='+gen_dir, '--grpc_python_out='+gen_dir, - proto_dir+'/clickhouse_grpc.proto'] + cmd = [ + "python3", + "-m", + "grpc_tools.protoc", + "-I" + proto_dir, + "--python_out=" + gen_dir, + "--grpc_python_out=" + gen_dir, + proto_dir + "/clickhouse_grpc.proto", + ] p = subprocess.Popen(cmd, stderr=subprocess.PIPE) # We don't want to show grpc_tools warnings. - errors = p.stderr.read().decode().strip('\n').split('\n') - only_warnings = all(('Warning' in error) for error in errors) + errors = p.stderr.read().decode().strip("\n").split("\n") + only_warnings = all(("Warning" in error) for error in errors) if not only_warnings: - error_print('\n'.join(errors)) + error_print("\n".join(errors)) # Import the generated *pb2.py files. @staticmethod def __import_pb2(): script_dir = os.path.dirname(os.path.realpath(__file__)) - gen_dir = os.path.join(script_dir, './_gen') + gen_dir = os.path.join(script_dir, "./_gen") sys.path.append(gen_dir) global clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc import clickhouse_grpc_pb2, clickhouse_grpc_pb2_grpc @@ -231,9 +283,9 @@ class ClickHouseGRPCClient(cmd.Cmd): # Overrides Cmd.onecmd(). Runs single command. def onecmd(self, line): stripped = line.strip() - if stripped == 'exit' or stripped == 'quit': + if stripped == "exit" or stripped == "quit": return True - if stripped == '': + if stripped == "": return False self.run_query(line, raise_exceptions=False, allow_cancel=True) return False @@ -261,17 +313,61 @@ class ClickHouseGRPCClient(cmd.Cmd): # MAIN + def main(args): - parser = argparse.ArgumentParser(description='ClickHouse client accessing server through gRPC protocol.', add_help=False) - parser.add_argument('--help', help='Show this help message and exit', action='store_true') - parser.add_argument('--host', '-h', help='The server name, ‘localhost’ by default. You can use either the name or the IPv4 or IPv6 address.', default='localhost') - parser.add_argument('--port', help='The port to connect to. This port should be enabled on the ClickHouse server (see grpc_port in the config).', default=9100) - parser.add_argument('--user', '-u', dest='user_name', help='The username. Default value: ‘default’.', default='default') - parser.add_argument('--password', help='The password. Default value: empty string.', default='') - parser.add_argument('--query', '-q', help='The query to process when using non-interactive mode.', default='') - parser.add_argument('--database', '-d', help='Select the current default database. Default value: the current database from the server settings (‘default’ by default).', default='') - parser.add_argument('--format', '-f', dest='output_format', help='Use the specified default format to output the result.', default='') - parser.add_argument('--debug', dest='show_debug_info', help='Enables showing the debug information.', action='store_true') + parser = argparse.ArgumentParser( + description="ClickHouse client accessing server through gRPC protocol.", + add_help=False, + ) + parser.add_argument( + "--help", help="Show this help message and exit", action="store_true" + ) + parser.add_argument( + "--host", + "-h", + help="The server name, ‘localhost’ by default. You can use either the name or the IPv4 or IPv6 address.", + default="localhost", + ) + parser.add_argument( + "--port", + help="The port to connect to. This port should be enabled on the ClickHouse server (see grpc_port in the config).", + default=9100, + ) + parser.add_argument( + "--user", + "-u", + dest="user_name", + help="The username. Default value: ‘default’.", + default="default", + ) + parser.add_argument( + "--password", help="The password. Default value: empty string.", default="" + ) + parser.add_argument( + "--query", + "-q", + help="The query to process when using non-interactive mode.", + default="", + ) + parser.add_argument( + "--database", + "-d", + help="Select the current default database. Default value: the current database from the server settings (‘default’ by default).", + default="", + ) + parser.add_argument( + "--format", + "-f", + dest="output_format", + help="Use the specified default format to output the result.", + default="", + ) + parser.add_argument( + "--debug", + dest="show_debug_info", + help="Enables showing the debug information.", + action="store_true", + ) args = parser.parse_args(args) if args.help: @@ -284,11 +380,18 @@ def main(args): output_format = args.output_format if not output_format and interactive_mode: output_format = DEFAULT_OUTPUT_FORMAT_FOR_INTERACTIVE_MODE - + try: - with ClickHouseGRPCClient(host=args.host, port=args.port, user_name=args.user_name, password=args.password, - database=args.database, output_format=output_format, verbatim=verbatim, - show_debug_info=args.show_debug_info) as client: + with ClickHouseGRPCClient( + host=args.host, + port=args.port, + user_name=args.user_name, + password=args.password, + database=args.database, + output_format=output_format, + verbatim=verbatim, + show_debug_info=args.show_debug_info, + ) as client: if interactive_mode: client.cmdloop() else: @@ -301,5 +404,6 @@ def main(args): if verbatim: print("\nBye") -if __name__ == '__main__': + +if __name__ == "__main__": main(sys.argv[1:]) diff --git a/utils/kafka/consume.py b/utils/kafka/consume.py index c82901f9e0e..74542baf218 100755 --- a/utils/kafka/consume.py +++ b/utils/kafka/consume.py @@ -9,24 +9,40 @@ from pprint import pprint def main(): - parser = argparse.ArgumentParser(description='Kafka Producer client') - parser.add_argument('--server', type=str, metavar='HOST', default='localhost', - help='Kafka bootstrap-server address') - parser.add_argument('--port', type=int, metavar='PORT', default=9092, - help='Kafka bootstrap-server port') - parser.add_argument('--client', type=str, default='ch-kafka-python', - help='custom client id for this producer') - parser.add_argument('--topic', type=str, required=True, - help='name of Kafka topic to store in') - parser.add_argument('--group', type=str, required=True, - help='name of the consumer group') + parser = argparse.ArgumentParser(description="Kafka Producer client") + parser.add_argument( + "--server", + type=str, + metavar="HOST", + default="localhost", + help="Kafka bootstrap-server address", + ) + parser.add_argument( + "--port", + type=int, + metavar="PORT", + default=9092, + help="Kafka bootstrap-server port", + ) + parser.add_argument( + "--client", + type=str, + default="ch-kafka-python", + help="custom client id for this producer", + ) + parser.add_argument( + "--topic", type=str, required=True, help="name of Kafka topic to store in" + ) + parser.add_argument( + "--group", type=str, required=True, help="name of the consumer group" + ) args = parser.parse_args() config = { - 'bootstrap_servers': f'{args.server}:{args.port}', - 'client_id': args.client, - 'group_id': args.group, - 'auto_offset_reset': 'earliest', + "bootstrap_servers": f"{args.server}:{args.port}", + "client_id": args.client, + "group_id": args.group, + "auto_offset_reset": "earliest", } client = kafka.KafkaConsumer(**config) diff --git a/utils/kafka/manage.py b/utils/kafka/manage.py index 7458bdceb74..578a7df7310 100755 --- a/utils/kafka/manage.py +++ b/utils/kafka/manage.py @@ -8,24 +8,48 @@ import argparse def main(): - parser = argparse.ArgumentParser(description='Kafka Topic manager') - parser.add_argument('--server', type=str, metavar='HOST', default='localhost', - help='Kafka bootstrap-server address') - parser.add_argument('--port', type=int, metavar='PORT', default=9092, - help='Kafka bootstrap-server port') - parser.add_argument('--client', type=str, default='ch-kafka-python', - help='custom client id for this producer') + parser = argparse.ArgumentParser(description="Kafka Topic manager") + parser.add_argument( + "--server", + type=str, + metavar="HOST", + default="localhost", + help="Kafka bootstrap-server address", + ) + parser.add_argument( + "--port", + type=int, + metavar="PORT", + default=9092, + help="Kafka bootstrap-server port", + ) + parser.add_argument( + "--client", + type=str, + default="ch-kafka-python", + help="custom client id for this producer", + ) commands = parser.add_mutually_exclusive_group() - commands.add_argument('--create', type=str, metavar='TOPIC', nargs='+', - help='create new topic(s) in the cluster') - commands.add_argument('--delete', type=str, metavar='TOPIC', nargs='+', - help='delete existing topic(s) from the cluster') + commands.add_argument( + "--create", + type=str, + metavar="TOPIC", + nargs="+", + help="create new topic(s) in the cluster", + ) + commands.add_argument( + "--delete", + type=str, + metavar="TOPIC", + nargs="+", + help="delete existing topic(s) from the cluster", + ) args = parser.parse_args() config = { - 'bootstrap_servers': f'{args.server}:{args.port}', - 'client_id': args.client, + "bootstrap_servers": f"{args.server}:{args.port}", + "client_id": args.client, } client = kafka.KafkaAdminClient(**config) diff --git a/utils/kafka/produce.py b/utils/kafka/produce.py index 97e2e6b7705..f82e56d8478 100755 --- a/utils/kafka/produce.py +++ b/utils/kafka/produce.py @@ -13,50 +13,82 @@ import time class Sync(enum.Enum): - NONE = 'none' - LEAD = 'leader' - ALL = 'all' + NONE = "none" + LEAD = "leader" + ALL = "all" def __str__(self): return self.value def convert(self): values = { - str(Sync.NONE): '0', - str(Sync.LEAD): '1', - str(Sync.ALL): 'all', + str(Sync.NONE): "0", + str(Sync.LEAD): "1", + str(Sync.ALL): "all", } return values[self.value] def main(): - parser = argparse.ArgumentParser(description='Produce a single message taken from input') - parser.add_argument('--server', type=str, metavar='HOST', default='localhost', - help='Kafka bootstrap-server address') - parser.add_argument('--port', type=int, metavar='PORT', default=9092, - help='Kafka bootstrap-server port') - parser.add_argument('--client', type=str, default='ch-kafka-python', - help='custom client id for this producer') - parser.add_argument('--topic', type=str, required=True, - help='name of Kafka topic to store in') - parser.add_argument('--retries', type=int, default=0, - help='number of retries to send on failure') - parser.add_argument('--multiply', type=int, default=1, - help='multiplies incoming string many times') - parser.add_argument('--repeat', type=int, default=1, - help='send same (multiplied) message many times') + parser = argparse.ArgumentParser( + description="Produce a single message taken from input" + ) + parser.add_argument( + "--server", + type=str, + metavar="HOST", + default="localhost", + help="Kafka bootstrap-server address", + ) + parser.add_argument( + "--port", + type=int, + metavar="PORT", + default=9092, + help="Kafka bootstrap-server port", + ) + parser.add_argument( + "--client", + type=str, + default="ch-kafka-python", + help="custom client id for this producer", + ) + parser.add_argument( + "--topic", type=str, required=True, help="name of Kafka topic to store in" + ) + parser.add_argument( + "--retries", type=int, default=0, help="number of retries to send on failure" + ) + parser.add_argument( + "--multiply", type=int, default=1, help="multiplies incoming string many times" + ) + parser.add_argument( + "--repeat", + type=int, + default=1, + help="send same (multiplied) message many times", + ) mode_group = parser.add_mutually_exclusive_group() - mode_group.add_argument('--jobs', type=int, default=multiprocessing.cpu_count(), - help='number of concurrent jobs') - mode_group.add_argument('--delay', type=int, metavar='SECONDS', default=0, - help='delay before sending next message') + mode_group.add_argument( + "--jobs", + type=int, + default=multiprocessing.cpu_count(), + help="number of concurrent jobs", + ) + mode_group.add_argument( + "--delay", + type=int, + metavar="SECONDS", + default=0, + help="delay before sending next message", + ) args = parser.parse_args() config = { - 'bootstrap_servers': f'{args.server}:{args.port}', - 'client_id': args.client, - 'retries': args.retries, + "bootstrap_servers": f"{args.server}:{args.port}", + "client_id": args.client, + "retries": args.retries, } client = kafka.KafkaProducer(**config) @@ -66,7 +98,7 @@ def main(): if args.delay > 0: time.sleep(args.delay) client.send(topic=args.topic, value=message) - print(f'iteration {num}: sent a message multiplied {args.multiply} times') + print(f"iteration {num}: sent a message multiplied {args.multiply} times") if args.delay > 0: args.jobs = 1 diff --git a/utils/kafka/status.py b/utils/kafka/status.py index 28ba3c9c36f..12ea3d23bdf 100755 --- a/utils/kafka/status.py +++ b/utils/kafka/status.py @@ -8,18 +8,34 @@ import argparse def main(): - parser = argparse.ArgumentParser(description='Kafka client to get groups and topics status') - parser.add_argument('--server', type=str, metavar='HOST', default='localhost', - help='Kafka bootstrap-server address') - parser.add_argument('--port', type=int, metavar='PORT', default=9092, - help='Kafka bootstrap-server port') - parser.add_argument('--client', type=str, default='ch-kafka-python', - help='custom client id for this producer') + parser = argparse.ArgumentParser( + description="Kafka client to get groups and topics status" + ) + parser.add_argument( + "--server", + type=str, + metavar="HOST", + default="localhost", + help="Kafka bootstrap-server address", + ) + parser.add_argument( + "--port", + type=int, + metavar="PORT", + default=9092, + help="Kafka bootstrap-server port", + ) + parser.add_argument( + "--client", + type=str, + default="ch-kafka-python", + help="custom client id for this producer", + ) args = parser.parse_args() config = { - 'bootstrap_servers': f'{args.server}:{args.port}', - 'client_id': args.client, + "bootstrap_servers": f"{args.server}:{args.port}", + "client_id": args.client, } client = kafka.KafkaAdminClient(**config) @@ -28,10 +44,13 @@ def main(): topics = cluster.topics() for topic in topics: - print(f'Topic "{topic}":', end='') + print(f'Topic "{topic}":', end="") for partition in cluster.partitions_for_topic(topic): tp = kafka.TopicPartition(topic, partition) - print(f' {partition} (begin: {consumer.beginning_offsets([tp])[tp]}, end: {consumer.end_offsets([tp])[tp]})', end='') + print( + f" {partition} (begin: {consumer.beginning_offsets([tp])[tp]}, end: {consumer.end_offsets([tp])[tp]})", + end="", + ) print() groups = client.list_consumer_groups() @@ -41,7 +60,9 @@ def main(): consumer = kafka.KafkaConsumer(**config, group_id=group[0]) offsets = client.list_consumer_group_offsets(group[0]) for topic, offset in offsets.items(): - print(f'\t{topic.topic}[{topic.partition}]: {consumer.beginning_offsets([topic])[topic]}, {offset.offset}, {consumer.end_offsets([topic])[topic]}') + print( + f"\t{topic.topic}[{topic.partition}]: {consumer.beginning_offsets([topic])[topic]}, {offset.offset}, {consumer.end_offsets([topic])[topic]}" + ) consumer.close() client.close() diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index cff412d9fd8..e87c4ea2b46 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,7 @@ +v22.3.2.2-lts 2022-03-17 +v22.2.3.5-stable 2022-02-25 +v22.2.2.1-stable 2022-02-17 +v22.1.4.30-stable 2022-02-25 v22.1.3.7-stable 2022-01-23 v22.1.2.2-stable 2022-01-19 v21.12.4.1-stable 2022-01-23 @@ -23,6 +27,7 @@ v21.9.5.16-stable 2021-10-19 v21.9.4.35-stable 2021-09-24 v21.9.3.30-stable 2021-09-17 v21.9.2.17-stable 2021-09-09 +v21.8.15.7-lts 2022-02-25 v21.8.14.5-lts 2022-01-26 v21.8.13.6-lts 2021-12-27 v21.8.12.29-lts 2021-12-02 diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 7f8e89b8355..8b75164eb60 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -673,7 +673,7 @@ static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict size -= padding; } - while (size >= 512) + while (size >= 512) /// NOLINT { __asm__( "vmovups (%[s]), %%ymm0\n" @@ -794,19 +794,19 @@ static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict return ret; } -extern "C" void * __memcpy_erms(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_sse2_unaligned(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_ssse3(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_ssse3_back(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_avx_unaligned(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_avx_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_avx512_unaligned(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_avx512_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size); -extern "C" void * __memcpy_avx512_no_vzeroupper(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_erms(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_sse2_unaligned(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_ssse3(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_ssse3_back(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_avx_unaligned(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_avx_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_avx512_unaligned(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_avx512_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT +extern "C" void * __memcpy_avx512_no_vzeroupper(void * __restrict destination, const void * __restrict source, size_t size); /// NOLINT #define VARIANT(N, NAME) \ - if (memcpy_variant == N) \ + if (memcpy_variant == (N)) \ return test(dst, src, size, iterations, num_threads, std::forward(generator), NAME, #NAME); template diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 9f6c2285d93..538d596d263 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -243,16 +243,6 @@ function make_rpm { cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec rpm_pack - PACKAGE=clickhouse-test - ARCH=all - TARGET=noarch - deb_unpack - mv ${PACKAGE}-$VERSION_FULL-2.spec ${PACKAGE}-$VERSION_FULL-2.spec_tmp - echo "Requires: python3" >> ${PACKAGE}-$VERSION_FULL-2.spec - #echo "Requires: python3-termcolor" >> ${PACKAGE}-$VERSION-2.spec - cat ${PACKAGE}-$VERSION_FULL-2.spec_tmp >> ${PACKAGE}-$VERSION_FULL-2.spec - rpm_pack - PACKAGE=clickhouse-common-static ARCH=amd64 TARGET=x86_64 @@ -271,7 +261,7 @@ function make_tgz { VERSION_FULL="${VERSION_STRING}" PACKAGE_DIR=${PACKAGE_DIR=../} - for PACKAGE in clickhouse-server clickhouse-client clickhouse-test clickhouse-common-static clickhouse-common-static-dbg; do + for PACKAGE in clickhouse-server clickhouse-client clickhouse-common-static clickhouse-common-static-dbg; do alien --verbose --scripts --generate --to-tgz ${PACKAGE_DIR}${PACKAGE}_${VERSION_FULL}_*.deb PKGDIR="./${PACKAGE}-${VERSION_FULL}" if [ ! -d "$PKGDIR/install" ]; then diff --git a/utils/zero_copy/zero_copy_schema_converter.py b/utils/zero_copy/zero_copy_schema_converter.py index 6fdd03add5a..6103ac69c6e 100755 --- a/utils/zero_copy/zero_copy_schema_converter.py +++ b/utils/zero_copy/zero_copy_schema_converter.py @@ -4,61 +4,96 @@ import socket import uuid from kazoo.client import KazooClient + def parse_args(): """ Parse command-line arguments. """ parser = argparse.ArgumentParser() - parser.add_argument('--hosts', default=socket.getfqdn() + ':2181', help='ZooKeeper hosts (host:port,host:port,...)') - parser.add_argument('-s', '--secure', default=False, action='store_true', help='Use secure connection') - parser.add_argument('--cert', default='', help='Client TLS certificate file') - parser.add_argument('--key', default='', help='Client TLS key file') - parser.add_argument('--ca', default='', help='Client TLS ca file') - parser.add_argument('-u', '--user', default='', help='ZooKeeper ACL user') - parser.add_argument('-p', '--password', default='', help='ZooKeeper ACL password') - parser.add_argument('-r', '--root', default='/clickhouse', help='ZooKeeper root path for ClickHouse') - parser.add_argument('-z', '--zcroot', default='zero_copy', help='ZooKeeper node for new zero-copy data') - parser.add_argument('--dryrun', default=False, action='store_true', help='Do not perform any actions') - parser.add_argument('--cleanup', default=False, action='store_true', help='Clean old nodes') - parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose mode') + parser.add_argument( + "--hosts", + default=socket.getfqdn() + ":2181", + help="ZooKeeper hosts (host:port,host:port,...)", + ) + parser.add_argument( + "-s", + "--secure", + default=False, + action="store_true", + help="Use secure connection", + ) + parser.add_argument("--cert", default="", help="Client TLS certificate file") + parser.add_argument("--key", default="", help="Client TLS key file") + parser.add_argument("--ca", default="", help="Client TLS ca file") + parser.add_argument("-u", "--user", default="", help="ZooKeeper ACL user") + parser.add_argument("-p", "--password", default="", help="ZooKeeper ACL password") + parser.add_argument( + "-r", "--root", default="/clickhouse", help="ZooKeeper root path for ClickHouse" + ) + parser.add_argument( + "-z", + "--zcroot", + default="clickhouse/zero_copy", + help="ZooKeeper node for new zero-copy data", + ) + parser.add_argument( + "--dryrun", + default=False, + action="store_true", + help="Do not perform any actions", + ) + parser.add_argument( + "--cleanup", default=False, action="store_true", help="Clean old nodes" + ) + parser.add_argument( + "-v", "--verbose", action="store_true", default=False, help="Verbose mode" + ) return parser.parse_args() # Several folders to heuristic that zookeepr node is folder node # May be false positive when someone creates set of tables with same paths -table_nodes = ['alter_partition_version', 'block_numbers', 'blocks', 'columns', 'leader_election'] -zc_nodes = ['zero_copy_s3', 'zero_copy_hdfs'] +table_nodes = [ + "alter_partition_version", + "block_numbers", + "blocks", + "columns", + "leader_election", +] +zc_nodes = ["zero_copy_s3", "zero_copy_hdfs"] def convert_node(client, args, path, zc_node): - base_path = f'{path}/{zc_node}/shared' + base_path = f"{path}/{zc_node}/shared" parts = client.get_children(base_path) - table_id_path = f'{path}/table_id' - table_id = '' + table_id_path = f"{path}/table_shared_id" + table_id = "" if client.exists(table_id_path): - table_id = client.get(table_id_path)[0].decode('UTF-8') + table_id = client.get(table_id_path)[0].decode("UTF-8") else: table_id = str(uuid.uuid4()) if args.verbose: print(f'Make table_id "{table_id_path}" = "{table_id}"') if not args.dryrun: - client.create(table_id_path, bytes(table_id, 'UTF-8')) + client.create(table_id_path, bytes(table_id, "UTF-8")) for part in parts: - part_path = f'{base_path}/{part}' + part_path = f"{base_path}/{part}" uniq_ids = client.get_children(part_path) for uniq_id in uniq_ids: - uniq_path = f'{part_path}/{uniq_id}' + uniq_path = f"{part_path}/{uniq_id}" replicas = client.get_children(uniq_path) for replica in replicas: - replica_path = f'{uniq_path}/{replica}' - new_path = f'{args.root}/{args.zcroot}/{zc_node}/{table_id}/{part}/{uniq_id}/{replica}' + replica_path = f"{uniq_path}/{replica}" + new_path = f"{args.root}/{args.zcroot}/{zc_node}/{table_id}/{part}/{uniq_id}/{replica}" if not client.exists(new_path): if args.verbose: print(f'Make node "{new_path}"') if not args.dryrun: - client.ensure_path(f'{args.root}/{args.zcroot}/{zc_node}/{table_id}/{part}/{uniq_id}') - client.create(new_path, value=b'lock') + client.ensure_path( + f"{args.root}/{args.zcroot}/{zc_node}/{table_id}/{part}/{uniq_id}" + ) + client.create(new_path, value=b"lock") if args.cleanup: if args.verbose: print(f'Remove node "{replica_path}"') @@ -70,7 +105,7 @@ def convert_node(client, args, path, zc_node): client.delete(part_path) if args.cleanup and not args.dryrun: client.delete(base_path) - client.delete(f'{path}/{zc_node}') + client.delete(f"{path}/{zc_node}") def convert_table(client, args, path, nodes): @@ -93,29 +128,30 @@ def scan_recursive(client, args, path): convert_table(client, args, path, nodes) else: for node in nodes: - scan_recursive(client, args, f'{path}/{node}') + scan_recursive(client, args, f"{path}/{node}") def scan(client, args): nodes = client.get_children(args.root) for node in nodes: if node != args.zcroot: - scan_recursive(client, args, f'{args.root}/{node}') + scan_recursive(client, args, f"{args.root}/{node}") def get_client(args): - client = KazooClient(connection_retry=3, - command_retry=3, - timeout=1, - hosts=args.hosts, - use_ssl=args.secure, - certfile=args.cert, - keyfile=args.key, - ca=args.ca - ) + client = KazooClient( + connection_retry=3, + command_retry=3, + timeout=1, + hosts=args.hosts, + use_ssl=args.secure, + certfile=args.cert, + keyfile=args.key, + ca=args.ca, + ) client.start() - if (args.user and args.password): - client.add_auth('digest', f'{args.user}:{args.password}') + if args.user and args.password: + client.add_auth("digest", f"{args.user}:{args.password}") return client @@ -125,5 +161,5 @@ def main(): scan(client, args) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/website/benchmark/dbms/index.html b/website/benchmark/dbms/index.html index b4e29098ead..a856bbb0502 100644 --- a/website/benchmark/dbms/index.html +++ b/website/benchmark/dbms/index.html @@ -15,7 +15,7 @@
- ClickHouse + ClickHouse

Performance comparison of analytical DBMS

diff --git a/website/benchmark/hardware/index.html b/website/benchmark/hardware/index.html index c6b1e2be275..42c87c334c0 100644 --- a/website/benchmark/hardware/index.html +++ b/website/benchmark/hardware/index.html @@ -15,7 +15,7 @@
- ClickHouse + ClickHouse

{{ title }}

@@ -85,6 +85,16 @@ Results for ThinkPad P15 are from Mikhail Shiryaev.
Results for RockPi4 are from Kirill Zholnay.
Results for Xeon 6266C are from David in Shanghai.
Results for SSDNodes and Cavium are from Lorenzo QXIP.
+Results for AMD EPYC 7662 64-Core Processor are from Evgeniy Kuts.
+Results for scaleway GP1-S 8x x86 64bit 32GB ram 300gb NVMe are from Dag Vilmar Tveit.
+Results for scaleway GP1-M 16x x86 64bit 64GB ram 600gb NVMe are from Dag Vilmar Tveit.
+Results for Intel(R) Core(TM) i5-4440 CPU @ 3.10GHz are from Peter, Chun-Sheng, Li.
+Results for MacBook Pro M1 are from Filatenkov Arthur.
+Results for AWS instance type im4gn.4xlarge are from Ananth Gundabattula (Darwinium).
+Results for AWS instance type im4gn.8xlarge are from Ananth Gundabattula (Darwinium).
+Results for AWS instance type im4gn.16xlarge are from Ananth Gundabattula (Darwinium).
+Results for AWS instance type i3.2xlarge are from Ananth Gundabattula (Darwinium).
+Results for 2x EPYC 7702 on ZFS mirror NVME are from Alibek A.

diff --git a/website/benchmark/hardware/results/amd_epyc_7662.json b/website/benchmark/hardware/results/amd_epyc_7662.json new file mode 100644 index 00000000000..436c0099992 --- /dev/null +++ b/website/benchmark/hardware/results/amd_epyc_7662.json @@ -0,0 +1,54 @@ +[ + { + "system": "AMD EPYC 7662", + "system_full": "AMD EPYC 7662 64-Core Processor", + "time": "2022-01-26 11:28:55", + "kind": "server", + "result": + [ + [0.001, 0.001, 0.001], + [0.037, 0.019, 0.020], + [0.082, 0.034, 0.026], + [0.298, 0.045, 0.038], + [0.424, 0.188, 0.178], + [0.594, 0.229, 0.227], + [0.037, 0.028, 0.032], + [0.060, 0.028, 0.027], + [0.496, 0.185, 0.192], + [0.611, 0.210, 0.214], + [0.400, 0.148, 0.137], + [0.424, 0.155, 0.144], + [0.639, 0.256, 0.239], + [0.944, 0.404, 0.309], + [0.699, 0.326, 0.288], + [0.461, 0.221, 0.216], + [1.176, 0.539, 0.561], + [1.070, 0.410, 0.426], + [2.080, 0.950, 0.866], + [0.351, 0.066, 0.130], + [3.248, 0.461, 0.313], + [3.612, 0.261, 0.231], + [6.720, 0.682, 0.671], + [6.300, 0.517, 0.488], + [0.982, 0.136, 0.125], + [0.531, 0.112, 0.109], + [1.006, 0.133, 0.118], + [3.184, 0.324, 0.310], + [2.799, 0.327, 0.308], + [0.569, 0.492, 0.493], + [0.900, 0.212, 0.221], + [1.925, 0.353, 0.326], + [2.489, 1.173, 1.248], + [3.626, 0.990, 0.897], + [3.743, 0.935, 0.915], + [0.419, 0.311, 0.339], + [0.278, 0.244, 0.236], + [0.111, 0.099, 0.098], + [0.139, 0.086, 0.084], + [0.664, 0.520, 0.552], + [0.072, 0.028, 0.036], + [0.050, 0.031, 0.022], + [0.005, 0.005, 0.011] + ] + } +] diff --git a/website/benchmark/hardware/results/amd_epyc_7702_zfs.json b/website/benchmark/hardware/results/amd_epyc_7702_zfs.json new file mode 100644 index 00000000000..9e7c15f579f --- /dev/null +++ b/website/benchmark/hardware/results/amd_epyc_7702_zfs.json @@ -0,0 +1,54 @@ +[ + { + "system": "2x EPYC 7702 on ZFS mirror NVME", + "system_full": "2x EPYC 7702 on ZFS mirror NVME, AMD EPYC 7702 64-Core Processor", + "time": "2022-01-14 21:07:13", + "kind": "server", + "result": + [ + [0.001, 0.002, 0.001], + [0.033, 0.021, 0.022], + [0.026, 0.022, 0.024], + [0.032, 0.024, 0.027], + [0.114, 0.115, 0.116], + [0.156, 0.150, 0.156], + [0.035, 0.023, 0.022], + [0.035, 0.023, 0.023], + [0.134, 0.148, 0.133], + [0.165, 0.150, 0.156], + [0.132, 0.087, 0.083], + [0.103, 0.124, 0.094], + [0.273, 0.221, 0.229], + [0.305, 0.263, 0.267], + [0.273, 0.267, 0.239], + [0.210, 0.228, 0.241], + [0.641, 0.518, 0.498], + [0.413, 0.423, 0.485], + [1.044, 0.991, 0.999], + [0.091, 0.144, 0.071], + [0.203, 0.190, 0.203], + [0.199, 0.210, 0.189], + [0.662, 0.753, 0.705], + [0.636, 0.461, 0.445], + [0.093, 0.079, 0.082], + [0.066, 0.070, 0.072], + [0.086, 0.080, 0.091], + [0.293, 0.280, 0.298], + [0.301, 0.258, 0.268], + [0.624, 0.611, 0.613], + [0.170, 0.168, 0.170], + [0.317, 0.269, 0.273], + [1.801, 1.071, 1.183], + [1.049, 1.080, 0.957], + [0.904, 0.892, 0.898], + [0.293, 0.288, 0.291], + [0.176, 0.173, 0.176], + [0.068, 0.068, 0.070], + [0.060, 0.060, 0.061], + [0.412, 0.388, 0.382], + [0.021, 0.019, 0.019], + [0.019, 0.022, 0.015], + [0.004, 0.010, 0.009] + ] + } +] diff --git a/website/benchmark/hardware/results/gp1_s_16x.json b/website/benchmark/hardware/results/gp1_s_16x.json new file mode 100644 index 00000000000..1353fc87d00 --- /dev/null +++ b/website/benchmark/hardware/results/gp1_s_16x.json @@ -0,0 +1,54 @@ +[ + { + "system": "scaleway GP1-S 8x x86", + "system_full": "scaleway GP1-M 16x x86 64bit 64GB ram 600gb NVMe", + "time": "2022-02-16 00:00:00", + "kind": "cloud", + "result": + [ + [0.005, 0.005, 0.036], + [0.039, 0.026, 0.026], + [0.092, 0.046, 0.046], + [0.172, 0.056, 0.055], + [0.166, 0.126, 0.123], + [0.364, 0.272, 0.265], + [0.005, 0.006, 0.005], + [0.028, 0.027, 0.029], + [0.581, 0.49, 0.486], + [0.69, 0.549, 0.553], + [0.248, 0.178, 0.175], + [0.266, 0.208, 0.208], + [1.584, 1.017, 0.868], + [1.717, 1.113, 1.145], + [1.144, 1.084, 1.048], + [0.991, 0.92, 0.895], + [4.121, 2.639, 2.621], + [1.447, 1.348, 1.354], + [6.802, 6.466, 6.433], + [0.142, 0.057, 0.052], + [1.252, 0.743, 0.715], + [1.389, 0.823, 0.791], + [3.143, 2.225, 2.159], + [1.795, 0.871, 0.837], + [0.361, 0.236, 0.229], + [0.264, 0.211, 0.214], + [0.37, 0.24, 0.225], + [1.449, 0.967, 0.876], + [1.605, 1.206, 1.16 ], + [3.412, 3.388, 3.397], + [0.783, 0.628, 0.65 ], + [1.419, 1.134, 1.112], + [6.983, 6.843, 6.852], + [5.466, 5.082, 4.955], + [5.632, 4.972, 5.22 ], + [1.639, 1.604, 1.571], + [0.285, 0.298, 0.269], + [0.115, 0.115, 0.101], + [0.098, 0.1, 0.092], + [0.563, 0.562, 0.512], + [0.058, 0.039, 0.042], + [0.039, 0.039, 0.025], + [0.029, 0.012, 0.012] + ] + } +] diff --git a/website/benchmark/hardware/results/gp1_s_8x.json b/website/benchmark/hardware/results/gp1_s_8x.json new file mode 100644 index 00000000000..2bc008af54c --- /dev/null +++ b/website/benchmark/hardware/results/gp1_s_8x.json @@ -0,0 +1,54 @@ +[ + { + "system": "scaleway GP1-S 8x x86", + "system_full": "scaleway GP1-S 8x x86 64bit 32GB ram 300gb NVMe", + "time": "2022-02-16 00:00:00", + "kind": "cloud", + "result": + [ + [0.026, 0.004, 0.004], + [0.038, 0.026, 0.026], + [0.071, 0.058, 0.059], + [0.118, 0.072, 0.069], + [0.190, 0.151, 0.155], + [0.465, 0.438, 0.401], + [0.002, 0.004, 0.004], + [0.028, 0.029, 0.026], + [0.751, 0.672, 0.676], + [0.897, 0.845, 0.798], + [0.291, 0.234, 0.254], + [0.371, 0.297, 0.296], + [1.208, 1.041, 1.005], + [1.445, 1.400, 1.414], + [1.406, 1.317, 1.342], + [1.414, 1.242, 1.244], + [4.179, 3.849, 3.878], + [2.320, 2.275, 2.201], + [7.499, 7.424, 7.196], + [0.135, 0.077, 0.068], + [1.465, 1.075, 1.063], + [1.700, 1.221, 1.198], + [3.731, 2.959, 2.905], + [2.283, 1.401, 1.342], + [0.474, 0.377, 0.367], + [0.371, 0.314, 0.337], + [0.483, 0.357, 0.356], + [1.565, 1.194, 1.181], + [2.226, 1.815, 1.746], + [2.990, 2.971, 2.947], + [1.003, 0.815, 0.842], + [1.386, 1.127, 1.108], + [8.174, 7.690, 7.735], + [6.171, 5.802, 5.933], + [6.201, 5.774, 5.972], + [1.758, 1.642, 1.639], + [0.288, 0.273, 0.253], + [0.121, 0.125, 0.107], + [0.096, 0.082, 0.088], + [0.490, 0.461, 0.476], + [0.041, 0.037, 0.035], + [0.035, 0.031, 0.025], + [0.008, 0.011, 0.015] + ] + } +] diff --git a/website/benchmark/hardware/results/i3_2xlarge.json b/website/benchmark/hardware/results/i3_2xlarge.json new file mode 100644 index 00000000000..e716b99e8a2 --- /dev/null +++ b/website/benchmark/hardware/results/i3_2xlarge.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS i3.2xlarge", + "system_full": "AWS i3.2xlarge Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz", + "time": "2022-01-02 03:16:35", + "kind": "cloud", + "result": + [ + [0.002, 0.002, 0.002], + [0.040, 0.023, 0.027], + [0.153, 0.084, 0.090], + [0.682, 0.113, 0.120], + [1.218, 0.227, 0.225], + [1.972, 0.708, 0.700], + [0.066, 0.052, 0.052], + [0.086, 0.037, 0.030], + [1.609, 1.123, 1.119], + [1.784, 1.231, 1.241], + [0.782, 0.444, 0.392], + [0.929, 0.504, 0.476], + [2.273, 1.649, 1.633], + [4.022, 2.181, 2.214], + [2.459, 2.022, 1.925], + [2.015, 1.621, 1.677], + [6.344, 5.439, 5.625], + [4.450, 3.724, 3.678], + [12.221, 10.922, 10.933], + [0.674, 0.139, 0.132], + [18.758, 2.164, 2.152], + [20.902, 2.440, 2.367], + [39.396, 5.476, 5.427], + [31.640, 2.759, 2.755], + [4.498, 0.647, 0.646], + [1.709, 0.627, 0.540], + [4.488, 0.665, 0.656], + [18.286, 2.023, 2.013], + [15.375, 2.896, 2.959], + [2.962, 2.899, 2.974], + [3.663, 1.299, 1.304], + [9.731, 1.922, 1.915], + [11.575, 10.394, 10.514], + [20.617, 8.121, 8.097], + [20.558, 8.088, 8.049], + [3.059, 2.780, 2.678], + [0.322, 0.244, 0.217], + [0.122, 0.082, 0.092], + [0.146, 0.073, 0.072], + [0.652, 0.473, 0.502], + [0.097, 0.025, 0.034], + [0.052, 0.025, 0.019], + [0.007, 0.004, 0.005] + ] + } +] diff --git a/website/benchmark/hardware/results/im4gn_16xlarge.json b/website/benchmark/hardware/results/im4gn_16xlarge.json new file mode 100644 index 00000000000..6db4f08021f --- /dev/null +++ b/website/benchmark/hardware/results/im4gn_16xlarge.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS im4gn.16xlarge", + "system_full": "AWS im4gn.16xlarge Neoverse-N1 4x7,500 NVMe SSD", + "time": "2022-01-04 01:04:37", + "kind": "cloud", + "result": + [ + [0.002, 0.001, 0.001], + [0.046, 0.017, 0.021], + [0.044, 0.021, 0.022], + [0.850, 0.064, 0.066], + [1.423, 0.076, 0.075], + [2.368, 0.141, 0.139], + [0.022, 0.013, 0.013], + [0.037, 0.038, 0.036], + [1.434, 0.138, 0.138], + [2.173, 0.159, 0.158], + [1.253, 0.089, 0.091], + [1.481, 0.102, 0.093], + [2.377, 0.211, 0.206], + [3.850, 0.272, 0.253], + [2.180, 0.276, 0.239], + [1.030, 0.242, 0.228], + [3.966, 0.564, 0.526], + [3.549, 0.404, 0.377], + [6.940, 1.389, 1.267], + [0.741, 0.225, 0.126], + [19.135, 0.398, 0.371], + [21.322, 0.330, 0.322], + [40.018, 0.727, 0.697], + [33.059, 1.592, 1.565], + [4.599, 0.098, 0.092], + [2.270, 0.089, 0.088], + [5.238, 0.098, 0.095], + [19.201, 0.358, 0.349], + [15.661, 0.430, 0.412], + [0.896, 0.876, 0.863], + [3.579, 0.223, 0.200], + [9.826, 0.344, 0.314], + [7.844, 2.085, 2.183], + [19.018, 1.143, 1.036], + [19.009, 1.203, 1.046], + [0.531, 0.325, 0.331], + [0.262, 0.221, 0.218], + [0.137, 0.101, 0.090], + [0.116, 0.099, 0.079], + [0.531, 0.468, 0.468], + [0.070, 0.025, 0.043], + [0.034, 0.020, 0.020], + [0.007, 0.004, 0.018] + ] + } +] diff --git a/website/benchmark/hardware/results/im4gn_4xlarge.json b/website/benchmark/hardware/results/im4gn_4xlarge.json new file mode 100644 index 00000000000..c3024c8dff2 --- /dev/null +++ b/website/benchmark/hardware/results/im4gn_4xlarge.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS im4gn.4xlarge", + "system_full": "AWS im4gn.4xlarge Neoverse-N1 1x7,500 NVMe SSD", + "time": "2022-01-02 06:59:48", + "kind": "cloud", + "result": + [ + [0.002, 0.002, 0.002], + [0.023, 0.013, 0.013], + [0.061, 0.026, 0.025], + [0.841, 0.033, 0.032], + [1.530, 0.086, 0.084], + [2.362, 0.291, 0.292], + [0.038, 0.029, 0.028], + [0.016, 0.015, 0.014], + [1.341, 0.302, 0.301], + [1.845, 0.376, 0.360], + [0.888, 0.184, 0.181], + [1.343, 0.215, 0.210], + [2.185, 0.469, 0.459], + [3.662, 0.603, 0.580], + [2.150, 0.587, 0.561], + [0.875, 0.458, 0.449], + [4.079, 1.425, 1.343], + [3.451, 0.927, 0.859], + [7.646, 2.890, 2.877], + [0.710, 0.107, 0.042], + [19.321, 0.696, 0.677], + [21.321, 0.740, 0.726], + [40.051, 1.625, 1.598], + [32.154, 0.842, 0.819], + [4.681, 0.240, 0.221], + [1.976, 0.197, 0.195], + [5.062, 0.241, 0.223], + [18.972, 0.643, 0.628], + [15.676, 0.978, 0.957], + [0.524, 0.505, 0.518], + [3.589, 0.460, 0.461], + [9.647, 0.674, 0.642], + [8.330, 3.414, 3.354], + [19.314, 2.296, 2.286], + [19.278, 2.311, 2.273], + [0.799, 0.753, 0.717], + [0.288, 0.222, 0.222], + [0.118, 0.101, 0.099], + [0.126, 0.085, 0.084], + [0.542, 0.480, 0.446], + [0.065, 0.025, 0.031], + [0.046, 0.021, 0.020], + [0.006, 0.010, 0.017] + ] + } +] diff --git a/website/benchmark/hardware/results/im4gn_8xlarge.json b/website/benchmark/hardware/results/im4gn_8xlarge.json new file mode 100644 index 00000000000..117812b0162 --- /dev/null +++ b/website/benchmark/hardware/results/im4gn_8xlarge.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS im4gn.8xlarge", + "system_full": "AWS im4gn.8xlarge Neoverse-N1 2x7,500 NVMe SSD", + "time": "2022-01-03 22:23:27", + "kind": "cloud", + "result": + [ + [0.002, 0.001, 0.001], + [0.034, 0.010, 0.010], + [0.044, 0.016, 0.016], + [0.862, 0.020, 0.020], + [1.500, 0.069, 0.071], + [2.454, 0.174, 0.172], + [0.025, 0.017, 0.017], + [0.023, 0.023, 0.023], + [1.329, 0.182, 0.181], + [2.167, 0.216, 0.212], + [1.159, 0.125, 0.119], + [1.483, 0.127, 0.122], + [2.313, 0.268, 0.260], + [3.788, 0.361, 0.329], + [2.043, 0.343, 0.308], + [0.872, 0.321, 0.309], + [3.921, 0.879, 0.840], + [3.460, 0.587, 0.543], + [7.272, 1.517, 1.447], + [0.707, 0.078, 0.064], + [19.314, 0.425, 0.385], + [21.332, 0.414, 0.405], + [40.030, 0.945, 0.921], + [32.867, 0.513, 0.477], + [4.640, 0.130, 0.124], + [2.227, 0.115, 0.107], + [5.223, 0.134, 0.126], + [19.179, 0.371, 0.367], + [15.658, 0.557, 0.545], + [0.541, 0.558, 0.552], + [3.548, 0.273, 0.250], + [9.772, 0.384, 0.357], + [7.896, 2.431, 2.661], + [19.149, 1.389, 1.268], + [19.103, 1.342, 1.282], + [0.583, 0.530, 0.541], + [0.238, 0.233, 0.243], + [0.114, 0.098, 0.102], + [0.124, 0.092, 0.089], + [0.552, 0.471, 0.481], + [0.053, 0.025, 0.025], + [0.047, 0.057, 0.020], + [0.022, 0.032, 0.004] + ] + } +] diff --git a/website/benchmark/hardware/results/intel_core_i5_4440.json b/website/benchmark/hardware/results/intel_core_i5_4440.json new file mode 100644 index 00000000000..b70b9e08fd4 --- /dev/null +++ b/website/benchmark/hardware/results/intel_core_i5_4440.json @@ -0,0 +1,54 @@ +[ + { + "system": "Intel(R) Core(TM) i5-4440 CPU @ 3.10GHz", + "system_full": "Intel(R) Core(TM) i5-4440 CPU @ 3.10GHz", + "time": "2022-01-06 08:48:45", + "kind": "server", + "result": + [ + [0.002, 0.001, 0.001], + [0.136, 0.021, 0.020], + [1.102, 0.061, 0.055], + [2.669, 0.089, 0.084], + [2.646, 0.198, 0.192], + [4.018, 0.606, 0.600], + [0.115, 0.034, 0.044], + [0.210, 0.018, 0.018], + [4.655, 1.002, 1.004], + [6.715, 1.139, 1.150], + [3.235, 0.351, 0.352], + [3.850, 0.410, 0.408], + [4.446, 1.579, 1.570], + [7.112, 2.031, 2.061], + [5.658, 1.812, 1.804], + [3.528, 1.600, 1.599], + [9.216, 5.029, 5.031], + [7.023, 2.968, 3.362], + [17.412, 9.705, 9.695], + [2.717, 0.110, 0.100], + [28.586, 1.907, 1.870], + [34.064, 2.178, 2.172], + [67.172, 5.105, 5.101], + [79.885, 2.579, 2.540], + [9.176, 0.572, 0.560], + [4.050, 0.496, 0.492], + [8.918, 0.575, 0.568], + [28.731, 2.089, 2.058], + [24.174, 2.956, 3.043], + [5.103, 5.010, 5.007], + [10.075, 1.188, 1.197], + [18.485, 1.966, 1.954], + [19.455, 10.855, 10.917], + [31.320, 7.848, 7.831], + [30.794, 7.871, 7.877], + [3.360, 2.777, 2.778], + [0.371, 0.166, 0.180], + [0.259, 0.064, 0.083], + [0.275, 0.060, 0.058], + [1.024, 0.380, 0.378], + [0.198, 0.025, 0.025], + [0.162, 0.023, 0.015], + [0.059, 0.006, 0.007] + ] + } +] diff --git a/website/benchmark/hardware/results/macbook_pro_m1_2021.json b/website/benchmark/hardware/results/macbook_pro_m1_2021.json new file mode 100644 index 00000000000..516940e1ef2 --- /dev/null +++ b/website/benchmark/hardware/results/macbook_pro_m1_2021.json @@ -0,0 +1,54 @@ +[ + { + "system": "MacBook Pro M1", + "system_full": "MacBook Pro M1 Max 16\" 2022, 64 GiB RAM, 1 TB SSD", + "time": "2022-02-27 00:00:00", + "kind": "laptop", + "result": + [ + [0.012, 0.001, 0.001], + [0.096, 0.012, 0.010], + [0.043, 0.022, 0.023], + [0.063, 0.031, 0.030], + [0.099, 0.070, 0.070], + [0.229, 0.197, 0.195], + [0.012, 0.001, 0.001], + [0.027, 0.012, 0.011], + [0.340, 0.301, 0.306], + [0.439, 0.383, 0.386], + [0.169, 0.134, 0.136], + [0.197, 0.160, 0.162], + [0.475, 0.435, 0.432], + [0.615, 0.557, 0.553], + [0.553, 0.502, 0.507], + [0.490, 0.445, 0.439], + [1.392, 1.260, 1.254], + [0.865, 0.833, 0.835], + [2.285, 2.180, 2.194], + [0.064, 0.035, 0.033], + [0.761, 0.650, 0.651], + [0.867, 0.715, 0.718], + [1.753, 1.478, 1.499], + [1.037, 0.737, 0.735], + [0.251, 0.201, 0.202], + [0.208, 0.172, 0.174], + [0.254, 0.202, 0.201], + [0.733, 0.598, 0.603], + [0.995, 0.882, 0.879], + [0.562, 0.545, 0.545], + [0.431, 0.371, 0.371], + [0.586, 0.490, 0.490], + [2.882, 2.664, 2.656], + [2.255, 2.147, 2.146], + [2.248, 2.137, 2.154], + [0.659, 0.638, 0.631], + [0.125, 0.108, 0.108], + [0.070, 0.052, 0.052], + [0.060, 0.042, 0.042], + [0.250, 0.229, 0.228], + [0.030, 0.013, 0.012], + [0.026, 0.011, 0.010], + [0.017, 0.003, 0.003] + ] + } +] diff --git a/website/blog/en/2022/clickhouse-v22.2-released.md b/website/blog/en/2022/clickhouse-v22.2-released.md new file mode 100644 index 00000000000..d55b0e6bcf0 --- /dev/null +++ b/website/blog/en/2022/clickhouse-v22.2-released.md @@ -0,0 +1,90 @@ +--- +title: 'ClickHouse 22.2 Released' +image: 'https://blog-images.clickhouse.com/en/2022/clickhouse-v22-2/featured.jpg' +date: '2022-02-23' +author: 'Alexey Milovidov' +tags: ['company', 'community'] +--- + +We prepared a new ClickHouse release 22.2, so it's nice if you have tried it on 2022-02-22. If not, you can try it today. This latest release includes 2,140 new commits from 118 contributors, including 41 new contributors: + +> Aaron Katz, Andre Marianiello, Andrew, Andrii Buriachevskyi, Brian Hunter, CoolT2, Federico Rodriguez, Filippov Denis, Gaurav Kumar, Geoff Genz, HarryLeeIBM, Heena Bansal, ILya Limarenko, Igor Nikonov, IlyaTsoi, Jake Liu, JaySon-Huang, Lemore, Leonid Krylov, Michail Safronov, Mikhail Fursov, Nikita, RogerYK, Roy Bellingan, Saad Ur Rahman, W, Yakov Olkhovskiy, alexeypavlenko, cnmade, grantovsky, hanqf-git, liuneng1994, mlkui, s-kat, tesw yew isal, vahid-sohrabloo, yakov-olkhovskiy, zhifeng, zkun, zxealous, 박동철. + +Let me tell you what is most interesting in 22.2... + +## Projections are production ready + +Projections allow you to have multiple data representations in the same table. For example, you can have data aggregations along with the raw data. There are no restrictions on which aggregate functions can be used - you can have count distinct, quantiles, or whatever you want. You can have data in multiple different sorting orders. ClickHouse will automatically select the most suitable projection for your query, so the query will be automatically optimized. + +Projections are somewhat similar to Materialized Views, which also allow you to have incremental aggregation and multiple sorting orders. But unlike Materialized Views, projections are updated atomically and consistently with the main table. The data for projections is being stored in the same "data parts" of the table and is being merged in the same way as the main data. + +The feature was developed by **Amos Bird**, a prominent ClickHouse contributor. The [prototype](https://github.com/ClickHouse/ClickHouse/pull/20202) has been available since Feb 2021, it has been merged in the main codebase by **Nikolai Kochetov** in May 2021 under experimental flag, and after 21 follow-up pull requests we ensured that it passed the full set of test suites and enabled it by default. + +Read an example of how to optimize queries with projections [in our docs](https://clickhouse.com/docs/en/getting-started/example-datasets/uk-price-paid/#speedup-with-projections). + +## Control of file creation and rewriting on data export + +When you export your data with an `INSERT INTO TABLE FUNCTION` statement into `file`, `s3` or `hdfs` and the target file already exists, you can now control how to deal with it: you can append new data into the file if it is possible, rewrite it with new data, or create another file with a similar name like 'data.1.parquet.gz'. + +Some storage systems like `s3` and some formats like `Parquet` don't support data appending. In previous ClickHouse versions, if you insert multiple times into a file with Parquet data format, you will end up with a file that is not recognized by other systems. Now you can choose between throwing exceptions on subsequent inserts or creating more files. + +So, new settings were introduced: `s3_truncate_on_insert`, `s3_create_new_file_on_insert`, `hdfs_truncate_on_insert`, `hdfs_create_new_file_on_insert`, `engine_file_allow_create_multiple_files`. + +This feature [was developed](https://github.com/ClickHouse/ClickHouse/pull/33302) by **Pavel Kruglov**. + +## Custom deduplication token + +`ReplicatedMergeTree` and `MergeTree` types of tables implement block-level deduplication. When a block of data is inserted, its cryptographic hash is calculated and if the same block was already inserted before, then the duplicate is skipped and the insert query succeeds. This makes it possible to implement exactly-once semantics for inserts. + +In ClickHouse version 22.2 you can provide your own deduplication token instead of an automatically calculated hash. This makes sense if you already have batch identifiers from some other system and you want to reuse them. It also makes sense when blocks can be identical but they should actually be inserted multiple times. Or the opposite - when blocks contain some random data and you want to deduplicate only by significant columns. + +This is implemented by adding the setting `insert_deduplication_token`. The feature was contributed by **Igor Nikonov**. + +## DEFAULT keyword for INSERT + +A small addition for SQL compatibility - now we allow using the `DEFAULT` keyword instead of a value in `INSERT INTO ... VALUES` statement. It looks like this: + +`INSERT INTO test VALUES (1, 'Hello', DEFAULT)` + +Thanks to **Andrii Buriachevskyi** for this feature. + +## EPHEMERAL columns + +A column in a table can have a `DEFAULT` expression like `c INT DEFAULT a + b`. In ClickHouse you can also use `MATERIALIZED` instead of `DEFAULT` if you want the column to be always calculated with the provided expression instead of allowing a user to insert data. And you can use `ALIAS` if you don't want the column to be stored at all but instead to be calculated on the fly if referenced. + +Since version 22.2 a new type of column is added: `EPHEMERAL` column. The user can insert data into this column but the column is not stored in a table, it's ephemeral. The purpose of this column is to provide data to calculate other columns that can reference it with `DEFAULT` or `MATERIALIZED` expressions. + +This feature was made by **Yakov Olkhovskiy**. + +## Improvements for multi-disk configuration + +You can configure multiple disks to store ClickHouse data instead of managing RAID and ClickHouse will automatically manage the data placement. + +Since version 22.2 ClickHouse can automatically repair broken disks without server restart by downloading the missing parts from replicas and placing them on the healthy disks. + +This feature was implemented by **Amos Bird** and is already being used for more than 1.5 years in production at Kuaishou. + +Another improvement is the option to specify TTL MOVE TO DISK/VOLUME **IF EXISTS**. It allows replicas with non-uniform disk configuration and to have one replica to move old data to cold storage while another replica has all the data on hot storage. Data will be moved only on replicas that have the specified disk or volume, hence *if exists*. This was developed by **Anton Popov**. + +## Flexible memory limits + +We split per-query and per-user memory limits into a pair of hard and soft limits. The settings `max_memory_usage` and `max_memory_usage_for_user` act as hard limits. When memory consumption is approaching the hard limit, an exception will be thrown. Two other settings: `max_guaranteed_memory_usage` and `max_guaranteed_memory_usage_for_user` act as soft limits. + +A query will be allowed to use more memory than a soft limit if there is available memory. But if there will be memory shortage (relative to the per-user hard limit or total per-server memory consumption), we calculate the "overcommit ratio" - how much more memory every query is consuming relative to the soft limit - and we will kill the most overcommitted query to let other queries run. + +In short, your query will not be limited to a few gigabytes of RAM if you have hundreds of gigabytes available. + +This experimental feature was implemented by **Dmitry Novik** and is continuing to be developed. + +## Shell-style comments in SQL + +Now we allow comments starting with `# ` or `#!`, similar to MySQL. The variant with `#!` allows using shell scripts with "shebang" interpreted by `clickhouse-local`. + +This feature was contributed by **Aaron Katz**. Very nice. + + +## And many more... + +Maxim Kita, Danila Kutenin, Anton Popov, zhanglistar, Federico Rodriguez, Raúl Marín, Amos Bird and Alexey Milovidov have contributed a ton of performance optimizations for this release. We are obsessed with high performance, as usual. :) + +Read the [full changelog](https://github.com/ClickHouse/ClickHouse/blob/master/CHANGELOG.md) for the 22.2 release and follow [the roadmap](https://github.com/ClickHouse/ClickHouse/issues/32513). diff --git a/website/blog/en/2022/opensee-analyzing-terabytes-of-financial-data-a-day-with-clickhouse.md b/website/blog/en/2022/opensee-analyzing-terabytes-of-financial-data-a-day-with-clickhouse.md new file mode 100644 index 00000000000..25d9fd3e965 --- /dev/null +++ b/website/blog/en/2022/opensee-analyzing-terabytes-of-financial-data-a-day-with-clickhouse.md @@ -0,0 +1,75 @@ +--- +title: 'Opensee: Analyzing Terabytes of Financial Data a Day With ClickHouse' +image: 'https://blog-images.clickhouse.com/en/2022/opensee/featured.png' +date: '2022-02-22' +author: 'Christophe Rivoire, Elena Bessis' +tags: ['company', 'community'] +--- + +We’d like to welcome Christophe Rivoire (UK Country Manager) and Elena Bessis (Product Marketing Assistant) from Opensee as guests to our blog. Today, they’re telling us how their product, powered by ClickHouse, allows financial institutions’ business users to directly harness 100% of their vast quantities of data instantly and on demand, with no size limitations. + +Opensee is a financial technology company providing real time self-service analytics solutions to financial institutions, which help them turn their big data challenges into a competitive advantage — unlocking vital opportunities led by business users. Opensee, formerly ICA, was started by a team of financial industry and technology experts frustrated that no simple big data analytics solution enabled them to dive deeper into all their data easily and efficiently, or perform what-if analysis on the hundreds of terabytes of data they were handling. + +So they built their own. + + +## ClickHouse For Trillions Of Financial Data Points + +Financial institutions have always been storing a lot of data (customer data, risk data, transaction data...) for their own decision processes and for regulatory reasons. Since the financial crisis, regulators all around the world have been significantly increasing the reporting requirements, insisting on longer historical ranges and deeper granularity. This combination has generated an exponential amount of data, which has forced financial institutions to review and upgrade their infrastructure. Opensee offers a solution to navigate all these very large data cubes, based on millions, billions or even trillions of data points. In order to build it, a data storage system capable of scaling horizontally with data and with fast OLAP query response time was required. In 2016, after thorough evaluation, Opensee concluded ClickHouse was the obvious solution. + +There are many use cases that involve storing and leveraging massive amounts of data on a daily basis, but Opensee built from the strength of their own expertise, evaluating risk linked to activities in the financial market. There are various types of risks (market risk, credit risk, liquidity risk…) and all of them need to aggregate a lot of data in order to calculate linear or non-linear indicators, both business and regulatory, and analyze all those numbers on the fly. + +!["Dashboard in Opensee for a Market Risk use case"](https://blog-images.clickhouse.com/en/2022/opensee/dashboard.png) +_Dashboard in Opensee for a Market Risk use case_ + + +## ClickHouse for Scalability, Granularity, Speed and Cost Control + +Financial institutions have sometimes believed that their ability to craft efficient storage solutions like data lakes for their vast amounts of data, typically built on a Hadoop stack, would make real-time analytics available. Unfortunately, many of these systems are too slow for at-scale analytics. + +Running a query on a Hadoop data lake is just not an option for users with real-time needs! Banks experimented with different types of analytical layers between the data lakes and the users, in order to allow access to their stored data and to run analytics, but ran into new challenges: in-memory computing solutions have a lack of scalability and high hardware costs. Others tried query accelerators but were forced to analyze only prepared data (pre-aggregated or specifically indexed data), losing the granularity which is always required to understand things like daily changes. More recently, financial institutions have been contemplating cloud database management systems, but for very large datasets and calculations the speed of these services is far from what ClickHouse can achieve for their specific use cases. + +Ultimately, none of these technologies could simultaneously combine scalability, granularity, speed and cost control, forcing financial institutions into a series of compromises. With Opensee, there is no need to compromise: the platform leverages ClickHouse's capacity to handle the huge volume that data lakes require and the fast response that in-memory databases can give, without the need to pre-aggregate the data. + + + +!["Dashboard in Opensee for a Market Risk use case"](https://blog-images.clickhouse.com/en/2022/opensee/pivot-table.png) +_Pivot table from the Opensee UI on a liquidity use case_ + + +## Opensee Architecture + +Opensee provides a series of APIs which allows users to fully abstract all the complexity and in particular the physical data model. These APIs are typically used for data ingestion, data query, model management, etc. Thanks to Opensee’s low-code API, users don’t need to access data through complex quasi-SQL queries, but rather through simple business queries that are optimized by Opensee to deliver performance. Opensee’s back end, which provides indirect access to Clickhouse, is written in Scala, while PostgreSQL contains all the configuration and context data that must be managed transactionally. Opensee also provides various options for front ends (dedicated Opensee web or rich user interface, Excel, others…) to interact with the data, navigate through the cube and leverage functionality like data versioning — built for the financial institution’s use. + + + +!["Dashboard in Opensee for a Market Risk use case"](https://blog-images.clickhouse.com/en/2022/opensee/architecture-chart.png) +_Opensee architecture chart_ + + +## Advantages of ClickHouse + +For Opensee, the most valuable feature is horizontal scalability, the capability to shard the data. Next comes the very fast dictionary lookup, rapid calculations with vectorization and the capability to manage array values. In the financial industry, where time series or historical data is everywhere, this capacity to calculate vectors and manage array values is critical. + +On top of being a solution that is extremely fast and efficient, other advantages include: + + +- distributed and replicated, with high availability and a performant map/reduce system +- wide range of features fit for analytics +- really good and extensive format support (csv, json, parquet, orc, protobuf ....) +- very rapid evolutions through the high contributions of a wide community to a very popular Open Source technology + +On top of these native ClickHouse strengths and functionalities, Opensee has developed a lot of other functionalities dedicated to financial institutions. To name only a few, a data versioning mechanism has been created allowing business users to either correct on the fly inaccurate data or simulate new values. This ‘What If’ simulation feature can be used to add, amend or delete transactions,with full auditability and traceability, without deleting any data. + +Another key feature is a Python processor which is available to define more complex calculations. Furthermore, the abstraction model layer has been built to remove the complexity of the physical data model for the users and optimize the queries. And, last but not least, in terms of visualization, a UI dedicated to financial institutions has been developed with and for its users. + + +## Dividing Hardware Costs By 10+ + +The cost efficiency factor is a key improvement for large financial institutions typically using in-memory computing technology. Dividing by ten (and sometimes more) the hardware cost is no small achievement! Being able to use very large datasets on standard servers on premise or in the cloud is a big achievement. With Opensee powered by ClickHouse, financial institutions are able to alleviate critical limitations of their existing solutions, avoiding legacy compromises and a lack of flexibility. Finally, these organizations are able to provide their users a turn-key solution to analyze all their data sets, which used to be siloed, in one single place, one single data model, one single infrastructure, and all of that in real time, combining very granular and very long historical ranges. + +## About Opensee + +Opensee empowers financial data divers to analyze deeper and faster. Headquartered in Paris, with offices in London and New York, Opensee is working with a trusted client base across global Tier 1 banks, asset managers, hedge funds and trading platforms. + +For more information please visit [www.opensee.io](http://www.opensee.io) or follow them on [LinkedIn](https://www.linkedin.com/company/opensee-company) and [Twitter](https://twitter.com/opensee_io). diff --git a/website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md b/website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md deleted file mode 100644 index 2c0463687b4..00000000000 --- a/website/blog/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: 'ClickHouse Meetup в Москве, 21 ноября 2016' -image: 'https://blog-images.clickhouse.com/ru/2016/clickhouse-meetup-v-moskve-21-noyabrya-2016/main.jpg' -date: '2016-11-22' -tags: ['мероприятия', 'meetup', 'Москва'] ---- - -[Посмотреть видео](https://events.yandex.ru/lib/talks/4351/) diff --git a/website/blog/ru/2016/clickhouse-na-highload-2016.md b/website/blog/ru/2016/clickhouse-na-highload-2016.md deleted file mode 100644 index 7dacbde140a..00000000000 --- a/website/blog/ru/2016/clickhouse-na-highload-2016.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: 'ClickHouse на HighLoad++ 2016' -image: 'https://blog-images.clickhouse.com/ru/2016/clickhouse-na-highload-2016/main.jpg' -date: '2016-12-10' -tags: ['мероприятия', 'конференции', 'Москва', 'HighLoad++'] ---- - -![iframe](https://www.youtube.com/embed/TAiCXHgZn50) - -[Расшифровка доклада](https://habrahabr.ru/post/322724/) - -![iframe](https://www.youtube.com/embed/tf38TPvwjJ4) - -[Расшифровка доклада](https://habrahabr.ru/post/322620/) diff --git a/website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md b/website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md deleted file mode 100644 index d90a7b9c4bb..00000000000 --- a/website/blog/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: 'ClickHouse на встрече про инфраструктуру хранения и обработки данных в Яндексе' -image: 'https://blog-images.clickhouse.com/ru/2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse/main.jpg' -date: '2016-10-16' -tags: ['мероприятия', 'инфраструктура'] ---- - -![iframe](https://www.youtube.com/embed/Ho4_dQk7dAg) - -[Страница мероприятия «Яндекс изнутри: инфраструктура хранения и обработки данных»](https://events.yandex.ru/events/meetings/15-oct-2016/), прошедшего 15 октября 2016 года. diff --git a/website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md b/website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md deleted file mode 100644 index e7216f47408..00000000000 --- a/website/blog/ru/2016/yandeks-otkryvaet-clickhouse.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: 'Яндекс открывает ClickHouse' -image: 'https://blog-images.clickhouse.com/ru/2016/yandeks-otkryvaet-clickhouse/main.jpg' -date: '2016-06-15' -tags: ['анонс', 'GitHub', 'лицензия'] ---- - -Сегодня внутренняя разработка компании Яндекс — [аналитическая СУБД ClickHouse](https://clickhouse.com/), стала доступна каждому. Исходники опубликованы на [GitHub](https://github.com/ClickHouse/ClickHouse) под лицензией Apache 2.0. - -ClickHouse позволяет выполнять аналитические запросы в интерактивном режиме по данным, обновляемым в реальном времени. Система способна масштабироваться до десятков триллионов записей и петабайт хранимых данных. Использование ClickHouse открывает возможности, которые раньше было даже трудно представить: вы можете сохранять весь поток данных без предварительной агрегации и быстро получать отчёты в любых разрезах. ClickHouse разработан в Яндексе для задач [Яндекс.Метрики](https://metrika.yandex.ru/) — второй по величине системы веб-аналитики в мире. diff --git a/website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md b/website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md deleted file mode 100644 index adab2fd7676..00000000000 --- a/website/blog/ru/2017/clickhouse-meetup-edet-v-minsk.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: 'ClickHouse MeetUp едет в Минск!' -image: 'https://blog-images.clickhouse.com/ru/2017/clickhouse-meetup-edet-v-minsk/main.jpg' -date: '2017-06-13' -tags: ['мероприятия', 'meetup', 'Минск', 'Беларусь', 'анонс'] ---- - -29 июня в Минске впервые выступят с докладами создатели СУБД ClickHоuse и те, кто ежедневно использует её для решения аналитических задач. Докладчики расскажут о последних изменениях и предстоящих обновлениях СУБД, а также о нюансах работы с ней. - -Встреча будет интересна администраторам ClickHouse и тем, кто пока только присматривается к системе. Мы приглашаем белорусских пользователей также поделиться своим опытом использования ClickHоuse и выступить на встрече с блиц-докладами: при регистрации мы предложим вам такую возможность! - -Участие в мероприятии бесплатное, но необходимо заранее зарегистрироваться: количество мест в зале ограничено. - -Посмотреть программу и подать заявку на участие можно на [странице встречи](https://events.yandex.ru/events/meetings/29-june-2017). diff --git a/website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md b/website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md deleted file mode 100644 index b7441b7ac30..00000000000 --- a/website/blog/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: 'ClickHouse Meetup в Екатеринбурге, 16 мая 2017' -image: 'https://blog-images.clickhouse.com/ru/2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017/main.jpg' -date: '2017-05-17' -tags: ['мероприятия', 'meetup', 'Екатеринбург'] ---- - -[Посмотреть презентацию](https://presentations.clickhouse.com/meetup6/) diff --git a/website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md b/website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md deleted file mode 100644 index 8cd3375abe9..00000000000 --- a/website/blog/ru/2017/clickhouse-meetup-v-minske-itogi.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: 'ClickHouse MeetUp в Минске: итоги' -image: 'https://blog-images.clickhouse.com/ru/2017/clickhouse-meetup-v-minske-itogi/main.jpg' -date: '2017-06-19' -tags: ['мероприятия', 'meetup', 'Минск', 'Беларусь'] ---- - -Недавно в Минске мы встретились с пользователями ClickHouse и техническими специалистами, кто только знакомится с СУБД. - -Мы делимся с вами презентациями докладчиков и будем рады ответить на вопросы в [чате ClickHouse в Телеграме](https://t.me/clickhouse_ru). - -[История создания ClickHouse, новости и планы по развитию](https://presentations.clickhouse.com/meetup7/), Алексей Миловидов - -[Использование ClickHouse для мониторинга связности сети](https://presentations.clickhouse.com/meetup7/netmon.pdf), Дмитрий Липин - -[Разбираемся во внутреннем устройстве ClickHouse](https://presentations.clickhouse.com/meetup7/internals.pdf), Виталий Людвиченко diff --git a/website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md b/website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md deleted file mode 100644 index e8bbf23c2c4..00000000000 --- a/website/blog/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: 'ClickHouse Meetup в Новосибирске, 3 апреля 2017' -image: 'https://blog-images.clickhouse.com/ru/2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017/main.jpg' -date: '2017-04-04' -tags: ['мероприятия', 'meetup', 'Новосибирск'] ---- - -[Презентация Алексея Миловидова](https://presentations.clickhouse.com/meetup4/) - -[Презентация Марии Мансуровой](https://presentations.clickhouse.com/meetup4/clickhouse_for_analysts.pdf) diff --git a/website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md b/website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md deleted file mode 100644 index 16bf2822746..00000000000 --- a/website/blog/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: 'ClickHouse Meetup в Санкт-Петербурге, 28 февраля 2017' -image: 'https://blog-images.clickhouse.com/ru/2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017/main.jpg' -date: '2017-03-01' -tags: ['мероприятия', 'meetup', 'Санкт-Петербург'] ---- - -![iframe](https://www.youtube.com/embed/CVrwp4Zoex4) diff --git a/website/blog/ru/2017/clickhouse-na-uwdc-2017.md b/website/blog/ru/2017/clickhouse-na-uwdc-2017.md deleted file mode 100644 index 1806f5fb6ba..00000000000 --- a/website/blog/ru/2017/clickhouse-na-uwdc-2017.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: 'ClickHouse на UWDC 2017' -image: 'https://blog-images.clickhouse.com/ru/2017/clickhouse-na-uwdc-2017/main.jpg' -date: '2017-05-20' -tags: ['мероприятия', 'конференции', 'Челябинск'] ---- - -![iframe](https://www.youtube.com/embed/isYA4e5zg1M?t=2h8m15s) - -[Посмотреть презентацию](https://presentations.clickhouse.com/uwdc/) diff --git a/website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md deleted file mode 100644 index a4dbff081ff..00000000000 --- a/website/blog/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: 'ClickHouse Meetup в Лимассоле, 7 мая 2019' -image: 'https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/main.jpg' -date: '2019-05-14' -tags: ['мероприятия', 'meetup', 'Лимассол', 'Кипр', 'Европа'] ---- - -Первый ClickHouse Meetup под открытым небом прошел в сердце Лимассола, второго по размеру города Кипра, на крыше, любезно предоставленной Exness Group. С крыши открывались сногсшибательные виды, но докладчики отлично справлялись с конкуренцией с ними за внимание аудитории. Более ста человек присоединилось к мероприятие, что в очередной раз подтверждает высокий интерес к ClickHouse по всему земному шару. Контент мероприятия также доступен в формате [видеозаписи](https://www.youtube.com/watch?v=_rpU-TvSfZ8). - -[Кирилл Шваков](https://github.com/kshvakov) сыграл ключевую роль в том, чтобы данное мероприятие стало возможным: наладил коммуникацию с ClickHouse сообществом на Кипре, нашел отличную площадку и докладчиков. Большинство ClickHouse митапов по всему миру происходят благодаря активным участникам сообщества таким как Кирилл. Если вы хотите помочь нам организовать ClickHouse митап в своём регионе, пожалуйста свяжитесь с командой ClickHouse в Яндексе через [эту форму](https://clickhouse.com/#meet) или любым другим удобным способом. - -![Кирилл Шваков](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/1.jpg) - -Кирилл широко известен благодаря его замечательногму [ClickHouse Go Driver](https://github.com/clickhouse/clickhouse-go), работающему по нативному протоколу, а его открывающий доклад был о его опыте оптимизации ClickHouse запросов и решению реальных прикладных задач в Integros и Wisebits. [Слайды](https://presentations.clickhouse.com/meetup22/strategies.pdf). [Полные тексты запросов](https://github.com/kshvakov/ClickHouse-Meetup-Exness). - -Мероприятие началось ранним вечером… -![Вечер в Лимассоле](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/2.jpg) - -…но природе потребовалось всего около часа, чтобы включить «ночной режим». Зато проецируемые слайды стало заметно легче читать. -![Ночь в Лимассоле](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/3.jpg) - -Сергей Томилов с его коллегами из Exness Platform Team поделились деталями об эволюции их систем для анализа логов и метрик, а также как они в итоге стали использовать ClickHouse для долгосрочного хранения и анализа данных([слайды](https://presentations.clickhouse.com/meetup22/exness.pdf)): -![Сергей Томилов](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/4.jpg) - -Алексей Миловидов из команды ClickHouse в Яндексе продемонстрировал функциональность из недавних релизов ClickHouse, а также рассказал о том, что стоит ждать в ближайшем будущем([слайды](https://presentations.clickhouse.com/meetup22/new_features/)): -![Алексей Миловидов](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/5.jpg) - -Александр Зайцев, технический директор Altinity, показал обзор того, как можно интегрировать ClickHouse в окружения, работающие на Kubernetes([слайды](https://presentations.clickhouse.com/meetup22/kubernetes.pdf)): -![Александр Зайцев](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/6.jpg) - -Владимир Гончаров, бекенд разработчик из Aloha Browser, закрывал ClickHouse Limassol Meetup демонстрацией нескольких проектов для интеграции других opensource продуктов для анализа логов с ClickHouse ([слайды](https://presentations.clickhouse.com/meetup22/aloha.pdf)): -![Владимир Гончаров](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/7.jpg) - -К сожалению, приближалась полнось и только самые «морозостойкие» любители ClickHouse продержались всё мероприятие, так стало заметно холодать. - -![Лимассол](https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-limassole-7-maya-2019/8.jpg) - -Больше фотографий с мероприятия доступно в [коротком послесловии от Exness](https://www.facebook.com/events/386638262181785/permalink/402167077295570/). diff --git a/website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md deleted file mode 100644 index 7e82fd653d7..00000000000 --- a/website/blog/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: 'ClickHouse Meetup в Москве, 5 сентября 2019' -image: 'https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019/main.jpg' -date: '2019-09-06' -tags: ['мероприятия', 'meetup', 'Москва'] ---- - -![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3gYSwohnKFUozYy9QdUpcT_) - -[Слайды опубликованы на GitHub](https://github.com/clickhouse/clickhouse-presentations/tree/master/meetup28). diff --git a/website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md deleted file mode 100644 index a90efdca645..00000000000 --- a/website/blog/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -title: 'ClickHouse Meetup в Новосибирске, 26 июня 2019' -image: 'https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019/main.jpg' -date: '2019-06-05' -tags: ['мероприятия', 'meetup', 'Новосибирск'] ---- - -Изюминкой второго ClickHouse митапа в Новосибирске были два низкоуровневых доклада с погружением во внутренности ClickHouse, а остальная часть контента была очень прикладной с конкретными сценариями. Любезно предоставленный S7 зал на сто человек был полон до самого завершения последнего доклада где-то ближе к полуночи. - -![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3ionSVt-NYC9Vu_83xxhb4J) - -Как обычно, [все слайды опубликованы на GitHub](https://presentations.clickhouse.com/meetup25). diff --git a/website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md b/website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md deleted file mode 100644 index bef157ade4e..00000000000 --- a/website/blog/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: 'ClickHouse Meetup в Санкт-Петербурге, 27 июля 2019' -image: 'https://blog-images.clickhouse.com/ru/2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019/main.jpg' -date: '2019-08-01' -tags: ['мероприятия', 'meetup', 'Санкт-Петербург'] ---- - -![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3j3X7TWrKmnEPcfEG901W-T) - -[Слайды опубликованы на GitHub](https://github.com/ClickHouse/clickhouse-presentations/tree/master/meetup27). diff --git a/website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md b/website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md deleted file mode 100644 index e6897f17156..00000000000 --- a/website/blog/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -title: 'ClickHouse Meetup в Минске, 11 июля 2019' -image: 'https://blog-images.clickhouse.com/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019/main.jpg' -date: '2019-07-12' -tags: ['мероприятия', 'meetup', 'Минск', 'Беларусь'] ---- - -![iframe](https://www.youtube.com/embed/videoseries?list=PL0Z2YDlm0b3hLz6dmyu6gM_X871FG9eCc) - -[Все слайды опубликованы на GitHub](https://github.com/ClickHouse/clickhouse-presentations/tree/master/meetup26). - -![Минск](https://blog-images.clickhouse.com/ru/2019/clickrouse-meetup-v-minske-11-iyulya-2019/1.jpg) diff --git a/website/blog/ru/index.md b/website/blog/ru/index.md deleted file mode 100644 index 227a69408dc..00000000000 --- a/website/blog/ru/index.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -is_index: true ---- diff --git a/website/blog/ru/redirects.txt b/website/blog/ru/redirects.txt deleted file mode 100644 index 4e34d53af3d..00000000000 --- a/website/blog/ru/redirects.txt +++ /dev/null @@ -1,15 +0,0 @@ -yandeks-otkryvaet-clickhouse.md 2016/yandeks-otkryvaet-clickhouse.md -clickhouse-meetup-v-moskve-21-noyabrya-2016.md 2016/clickhouse-meetup-v-moskve-21-noyabrya-2016.md -clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md 2016/clickhouse-na-vstreche-pro-infrastrukturu-khraneniya-i-obrabotki-dannykh-v-yandekse.md -clickhouse-na-highload-2016.md 2016/clickhouse-na-highload-2016.md -clickhouse-meetup-v-novosibirske-3-aprelya-2017.md 2017/clickhouse-meetup-v-novosibirske-3-aprelya-2017.md -clickhouse-meetup-v-minske-itogi.md 2017/clickhouse-meetup-v-minske-itogi.md -clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md 2017/clickhouse-meetup-v-sankt-peterburge-28-fevralya-2017.md -clickhouse-meetup-v-ekaterinburge-16-maya-2017.md 2017/clickhouse-meetup-v-ekaterinburge-16-maya-2017.md -clickhouse-na-uwdc-2017.md 2017/clickhouse-na-uwdc-2017.md -clickhouse-meetup-edet-v-minsk.md 2017/clickhouse-meetup-edet-v-minsk.md -clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md 2019/clickhouse-meetup-v-sankt-peterburge-27-iyulya-2019.md -clickhouse-meetup-v-moskve-5-sentyabrya-2019.md 2019/clickhouse-meetup-v-moskve-5-sentyabrya-2019.md -clickhouse-meetup-v-novosibirske-26-iyunya-2019.md 2019/clickhouse-meetup-v-novosibirske-26-iyunya-2019.md -clickrouse-meetup-v-minske-11-iyulya-2019.md 2019/clickrouse-meetup-v-minske-11-iyulya-2019.md -clickhouse-meetup-v-limassole-7-maya-2019.md 2019/clickhouse-meetup-v-limassole-7-maya-2019.md diff --git a/website/css/greenhouse.css b/website/css/greenhouse.css deleted file mode 100644 index 76812a169e8..00000000000 --- a/website/css/greenhouse.css +++ /dev/null @@ -1 +0,0 @@ -#main{padding-bottom:0;padding-top:0}#wrapper{max-width:1078px;padding:0}body>#wrapper>#main>#wrapper>#content,body>#wrapper>#main>#wrapper>#logo,body>#wrapper>#main>#wrapper>h1{display:none}body>#wrapper>#main>#wrapper>#board_title{margin-top:0}body>#wrapper>#main>#logo{margin-top:80px}body>#wrapper>#main>:last-child{margin-bottom:120px} \ No newline at end of file diff --git a/website/images/backgrounds/bg-card-pattern-red.png b/website/images/backgrounds/bg-card-pattern-red.png index a944ec45bd0..84f781590e6 100644 Binary files a/website/images/backgrounds/bg-card-pattern-red.png and b/website/images/backgrounds/bg-card-pattern-red.png differ diff --git a/website/images/backgrounds/bg-quotes.svg b/website/images/backgrounds/bg-quotes.svg index 651351a0812..2bab82c0d75 100644 --- a/website/images/backgrounds/bg-quotes.svg +++ b/website/images/backgrounds/bg-quotes.svg @@ -1,12 +1 @@ - - - Artboard - - - - - - - - - \ No newline at end of file +Artboard \ No newline at end of file diff --git a/website/images/icons/icon-arrow.svg b/website/images/icons/icon-arrow.svg index 3a03b2da23b..22f67781c6e 100644 --- a/website/images/icons/icon-arrow.svg +++ b/website/images/icons/icon-arrow.svg @@ -1,17 +1 @@ - - - icon-arrow - - - - - - - - - - - - - - \ No newline at end of file +icon-arrow \ No newline at end of file diff --git a/website/images/icons/icon-blog-black.svg b/website/images/icons/icon-blog-black.svg index 2e448c5c0da..42d5c1a3010 100644 --- a/website/images/icons/icon-blog-black.svg +++ b/website/images/icons/icon-blog-black.svg @@ -1,14 +1 @@ - - - icon-blog - - - - - - - - - - - \ No newline at end of file +icon-blog \ No newline at end of file diff --git a/website/images/icons/icon-facebook-gray.svg b/website/images/icons/icon-facebook-gray.svg index fc42770a749..6c0e3190e0a 100644 --- a/website/images/icons/icon-facebook-gray.svg +++ b/website/images/icons/icon-facebook-gray.svg @@ -1,15 +1 @@ - - - icon-facebook-gray - - - - - - - \ No newline at end of file +icon-facebook-gray \ No newline at end of file diff --git a/website/images/icons/icon-facebook.svg b/website/images/icons/icon-facebook.svg index 910235c7ed8..c7c66a5cb5d 100644 --- a/website/images/icons/icon-facebook.svg +++ b/website/images/icons/icon-facebook.svg @@ -1,19 +1 @@ - - - Group - - - - - - - - - - - - - - - - \ No newline at end of file +Group \ No newline at end of file diff --git a/website/images/icons/icon-github.svg b/website/images/icons/icon-github.svg index c22c8563b21..79f936ad51b 100644 --- a/website/images/icons/icon-github.svg +++ b/website/images/icons/icon-github.svg @@ -1,9 +1 @@ - - - icon-github - - - - - - \ No newline at end of file +icon-github \ No newline at end of file diff --git a/website/images/icons/icon-google.svg b/website/images/icons/icon-google.svg index 4fa5d1eb118..732c960a052 100644 --- a/website/images/icons/icon-google.svg +++ b/website/images/icons/icon-google.svg @@ -1,12 +1 @@ - - - icon-google - - - - - - - - - \ No newline at end of file +icon-google \ No newline at end of file diff --git a/website/images/icons/icon-linkedin-alt-gray.svg b/website/images/icons/icon-linkedin-alt-gray.svg index e286486ab9b..eb5931c3a65 100644 --- a/website/images/icons/icon-linkedin-alt-gray.svg +++ b/website/images/icons/icon-linkedin-alt-gray.svg @@ -1,13 +1 @@ - - - icon-linkedin-gray - - - - - - - - - - \ No newline at end of file +icon-linkedin-gray \ No newline at end of file diff --git a/website/images/icons/icon-linkedin-gray.svg b/website/images/icons/icon-linkedin-gray.svg index ff19bbbe686..684f35e8f47 100644 --- a/website/images/icons/icon-linkedin-gray.svg +++ b/website/images/icons/icon-linkedin-gray.svg @@ -1,15 +1 @@ - - - icon-linkedin-gray - - - - - - - \ No newline at end of file +icon-linkedin-gray \ No newline at end of file diff --git a/website/images/icons/icon-linkedin.png b/website/images/icons/icon-linkedin.png index e50cf591487..1facc9e76df 100644 Binary files a/website/images/icons/icon-linkedin.png and b/website/images/icons/icon-linkedin.png differ diff --git a/website/images/icons/icon-menu.svg b/website/images/icons/icon-menu.svg index 49aa5b52d37..08d3edb0dfa 100644 --- a/website/images/icons/icon-menu.svg +++ b/website/images/icons/icon-menu.svg @@ -1,10 +1 @@ - - - - - - - - - - + \ No newline at end of file diff --git a/website/images/icons/icon-performance.svg b/website/images/icons/icon-performance.svg index 758149fc872..9fcd2a6558a 100644 --- a/website/images/icons/icon-performance.svg +++ b/website/images/icons/icon-performance.svg @@ -1,16 +1 @@ - - - icon-performance - - - - - - - - - - - - - \ No newline at end of file +icon-performance \ No newline at end of file diff --git a/website/images/icons/icon-reliability.svg b/website/images/icons/icon-reliability.svg index 0b18d1451c0..add18fe0e76 100644 --- a/website/images/icons/icon-reliability.svg +++ b/website/images/icons/icon-reliability.svg @@ -1,16 +1 @@ - - - icon-reliability - - - - - - - - - - - - - \ No newline at end of file +icon-reliability \ No newline at end of file diff --git a/website/images/icons/icon-scalability.svg b/website/images/icons/icon-scalability.svg index 4b0a99dc233..9b015300f99 100644 --- a/website/images/icons/icon-scalability.svg +++ b/website/images/icons/icon-scalability.svg @@ -1,16 +1 @@ - - - icon-scalability - - - - - - - - - - - - - \ No newline at end of file +icon-scalability \ No newline at end of file diff --git a/website/images/icons/icon-security.svg b/website/images/icons/icon-security.svg index 716f7cc5789..da341224ed3 100644 --- a/website/images/icons/icon-security.svg +++ b/website/images/icons/icon-security.svg @@ -1,16 +1 @@ - - - icon-security - - - - - - - - - - - - - \ No newline at end of file +icon-security \ No newline at end of file diff --git a/website/images/icons/icon-slack-black.svg b/website/images/icons/icon-slack-black.svg index 89b6ddcbada..03420c85534 100644 --- a/website/images/icons/icon-slack-black.svg +++ b/website/images/icons/icon-slack-black.svg @@ -1,14 +1 @@ - - - icon-slack - - - - - - - - - - - \ No newline at end of file +icon-slack \ No newline at end of file diff --git a/website/images/icons/icon-slack.svg b/website/images/icons/icon-slack.svg index cb74d47cb04..42e33c4d65f 100644 --- a/website/images/icons/icon-slack.svg +++ b/website/images/icons/icon-slack.svg @@ -1,12 +1 @@ - - - icon-slack - - - - - - - - - \ No newline at end of file +icon-slack \ No newline at end of file diff --git a/website/images/icons/icon-stack-overflow.svg b/website/images/icons/icon-stack-overflow.svg index 5c9a2d57d2b..c8244ac9e4c 100644 --- a/website/images/icons/icon-stack-overflow.svg +++ b/website/images/icons/icon-stack-overflow.svg @@ -1,16 +1 @@ - - - icon-stack-overflow - - - - - - - - - - - - - \ No newline at end of file +icon-stack-overflow \ No newline at end of file diff --git a/website/images/icons/icon-telegram.svg b/website/images/icons/icon-telegram.svg index 5ab06341383..3bf90eea1a4 100644 --- a/website/images/icons/icon-telegram.svg +++ b/website/images/icons/icon-telegram.svg @@ -1,12 +1 @@ - - - icon-telegram - - - - - - - - - \ No newline at end of file +icon-telegram \ No newline at end of file diff --git a/website/images/icons/icon-twitter-gray.svg b/website/images/icons/icon-twitter-gray.svg index 115fe5ff390..b209f7b07cb 100644 --- a/website/images/icons/icon-twitter-gray.svg +++ b/website/images/icons/icon-twitter-gray.svg @@ -1,11 +1 @@ - - - icon-twitter-gray - - - - - - - - \ No newline at end of file +icon-twitter-gray \ No newline at end of file diff --git a/website/images/icons/icon-twitter.svg b/website/images/icons/icon-twitter.svg index b25ac9353ac..ef11bb2f0ba 100644 --- a/website/images/icons/icon-twitter.svg +++ b/website/images/icons/icon-twitter.svg @@ -1,16 +1 @@ - - - + \ No newline at end of file diff --git a/website/images/icons/icon-youtube-black.svg b/website/images/icons/icon-youtube-black.svg index 0cb017d64a4..bb20f823a06 100644 --- a/website/images/icons/icon-youtube-black.svg +++ b/website/images/icons/icon-youtube-black.svg @@ -1,15 +1 @@ - - - icon-youtube - - - - - - - - - - - - +icon-youtube \ No newline at end of file diff --git a/website/images/icons/icon-youtube.svg b/website/images/icons/icon-youtube.svg index ee676649770..224fc1c49cf 100644 --- a/website/images/icons/icon-youtube.svg +++ b/website/images/icons/icon-youtube.svg @@ -1,10 +1 @@ - - - icon-youtube - - - - - - - \ No newline at end of file +icon-youtube \ No newline at end of file diff --git a/website/images/logo-clickhouse.svg b/website/images/logo-clickhouse.svg index 560e15572d9..b6bb0cd2d07 100644 --- a/website/images/logo-clickhouse.svg +++ b/website/images/logo-clickhouse.svg @@ -1,28 +1 @@ - - - ClickHouse Logo - - - - \ No newline at end of file +ClickHouse Logo \ No newline at end of file diff --git a/website/images/logos/logo-almaz-capital.svg b/website/images/logos/logo-almaz-capital.svg index ed2357083ed..20cf10bd205 100644 --- a/website/images/logos/logo-almaz-capital.svg +++ b/website/images/logos/logo-almaz-capital.svg @@ -1 +1 @@ -AC Logo \ No newline at end of file +AC Logo \ No newline at end of file diff --git a/website/images/logos/logo-cloudflare.svg b/website/images/logos/logo-cloudflare.svg index e4bf4f2f803..9c6b408c388 100644 --- a/website/images/logos/logo-cloudflare.svg +++ b/website/images/logos/logo-cloudflare.svg @@ -1,25 +1 @@ - - - cf-logo-h - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file +cf-logo-h \ No newline at end of file diff --git a/website/images/logos/logo-firstmark.svg b/website/images/logos/logo-firstmark.svg index e529766bae4..9f372958295 100644 --- a/website/images/logos/logo-firstmark.svg +++ b/website/images/logos/logo-firstmark.svg @@ -1,579 +1 @@ - - - logo-firstmark - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file +logo-firstmark \ No newline at end of file diff --git a/website/images/logos/logo-yandex.png b/website/images/logos/logo-yandex.png index 92349036b3f..02ea52f2c7f 100644 Binary files a/website/images/logos/logo-yandex.png and b/website/images/logos/logo-yandex.png differ diff --git a/website/images/photos/aaron-katz.jpg b/website/images/photos/aaron-katz.jpg index 02aff426b33..3e03047a247 100644 Binary files a/website/images/photos/aaron-katz.jpg and b/website/images/photos/aaron-katz.jpg differ diff --git a/website/images/photos/alexander-sapin.jpg b/website/images/photos/alexander-sapin.jpg index 83121a980e7..8f132c6f406 100644 Binary files a/website/images/photos/alexander-sapin.jpg and b/website/images/photos/alexander-sapin.jpg differ diff --git a/website/images/photos/alexander-tokmakov.jpg b/website/images/photos/alexander-tokmakov.jpg index 3215788443f..dcee6b9d551 100644 Binary files a/website/images/photos/alexander-tokmakov.jpg and b/website/images/photos/alexander-tokmakov.jpg differ diff --git a/website/images/photos/alexey-milovidov.jpg b/website/images/photos/alexey-milovidov.jpg index 31a4ab5311d..9b5b5a12e88 100644 Binary files a/website/images/photos/alexey-milovidov.jpg and b/website/images/photos/alexey-milovidov.jpg differ diff --git a/website/images/photos/andy-james.jpg b/website/images/photos/andy-james.jpg new file mode 100644 index 00000000000..d1f5b47f39c Binary files /dev/null and b/website/images/photos/andy-james.jpg differ diff --git a/website/images/photos/anton-popov.jpg b/website/images/photos/anton-popov.jpg index 663fc9255e9..86fefbba27a 100644 Binary files a/website/images/photos/anton-popov.jpg and b/website/images/photos/anton-popov.jpg differ diff --git a/website/images/photos/arno-van-driel.jpg b/website/images/photos/arno-van-driel.jpg index 9a7855491aa..878ab2c336d 100644 Binary files a/website/images/photos/arno-van-driel.jpg and b/website/images/photos/arno-van-driel.jpg differ diff --git a/website/images/photos/caryn-marooney.jpg b/website/images/photos/caryn-marooney.jpg index 446e890ebe1..fa0dd29ad1f 100644 Binary files a/website/images/photos/caryn-marooney.jpg and b/website/images/photos/caryn-marooney.jpg differ diff --git a/website/images/photos/dmitry-novik.jpg b/website/images/photos/dmitry-novik.jpg index c2ce965e48d..d688407e5e4 100644 Binary files a/website/images/photos/dmitry-novik.jpg and b/website/images/photos/dmitry-novik.jpg differ diff --git a/website/images/photos/dorota-szeremeta.jpg b/website/images/photos/dorota-szeremeta.jpg index 70e427a1154..1e7c0aa2471 100644 Binary files a/website/images/photos/dorota-szeremeta.jpg and b/website/images/photos/dorota-szeremeta.jpg differ diff --git a/website/images/photos/ilya-yatsishin.jpg b/website/images/photos/ilya-yatsishin.jpg index b8d9d0866a1..23d10a86c7b 100644 Binary files a/website/images/photos/ilya-yatsishin.jpg and b/website/images/photos/ilya-yatsishin.jpg differ diff --git a/website/images/photos/ivan-blinkov.jpg b/website/images/photos/ivan-blinkov.jpg index 0e70ff6f773..b11d4504bde 100644 Binary files a/website/images/photos/ivan-blinkov.jpg and b/website/images/photos/ivan-blinkov.jpg differ diff --git a/website/images/photos/jason-chan.jpg b/website/images/photos/jason-chan.jpg index 6ce0b524cb2..2f69200a0b4 100644 Binary files a/website/images/photos/jason-chan.jpg and b/website/images/photos/jason-chan.jpg differ diff --git a/website/images/photos/kseniia-sumarokova.jpg b/website/images/photos/kseniia-sumarokova.jpg index 0fd4efeee25..8125d39869a 100644 Binary files a/website/images/photos/kseniia-sumarokova.jpg and b/website/images/photos/kseniia-sumarokova.jpg differ diff --git a/website/images/photos/maksim-kita.jpg b/website/images/photos/maksim-kita.jpg index e6aac5737cb..d740059b71a 100644 Binary files a/website/images/photos/maksim-kita.jpg and b/website/images/photos/maksim-kita.jpg differ diff --git a/website/images/photos/manas-alekar.jpg b/website/images/photos/manas-alekar.jpg index 307a860455f..a5463a527a9 100644 Binary files a/website/images/photos/manas-alekar.jpg and b/website/images/photos/manas-alekar.jpg differ diff --git a/website/images/photos/mark-zitnik.jpg b/website/images/photos/mark-zitnik.jpg new file mode 100644 index 00000000000..7ffc5105258 Binary files /dev/null and b/website/images/photos/mark-zitnik.jpg differ diff --git a/website/images/photos/melvyn-peignon.jpg b/website/images/photos/melvyn-peignon.jpg index 532c1759c65..3c4d2a212a0 100644 Binary files a/website/images/photos/melvyn-peignon.jpg and b/website/images/photos/melvyn-peignon.jpg differ diff --git a/website/images/photos/miel-donkers.jpg b/website/images/photos/miel-donkers.jpg new file mode 100644 index 00000000000..8d67c2440b1 Binary files /dev/null and b/website/images/photos/miel-donkers.jpg differ diff --git a/website/images/photos/mike-hayes.jpg b/website/images/photos/mike-hayes.jpg index 13dfbe4cefd..df0380063fa 100644 Binary files a/website/images/photos/mike-hayes.jpg and b/website/images/photos/mike-hayes.jpg differ diff --git a/website/images/photos/nihat-hosgur.jpg b/website/images/photos/nihat-hosgur.jpg index ad47b4aba50..fb3dd451713 100644 Binary files a/website/images/photos/nihat-hosgur.jpg and b/website/images/photos/nihat-hosgur.jpg differ diff --git a/website/images/photos/nikita-mikhailov.jpg b/website/images/photos/nikita-mikhailov.jpg index 549e5bc06e8..d956d334798 100644 Binary files a/website/images/photos/nikita-mikhailov.jpg and b/website/images/photos/nikita-mikhailov.jpg differ diff --git a/website/images/photos/nikolai-kochetov.jpg b/website/images/photos/nikolai-kochetov.jpg index 924df4e59cb..135852aa078 100644 Binary files a/website/images/photos/nikolai-kochetov.jpg and b/website/images/photos/nikolai-kochetov.jpg differ diff --git a/website/images/photos/pavel-kruglov.jpg b/website/images/photos/pavel-kruglov.jpg index 6287ef1bfe3..e1699785c9b 100644 Binary files a/website/images/photos/pavel-kruglov.jpg and b/website/images/photos/pavel-kruglov.jpg differ diff --git a/website/images/photos/rich-raposa.jpg b/website/images/photos/rich-raposa.jpg index f8338d6e3df..2d155356e1f 100644 Binary files a/website/images/photos/rich-raposa.jpg and b/website/images/photos/rich-raposa.jpg differ diff --git a/website/images/photos/thom-oconnor.jpg b/website/images/photos/thom-oconnor.jpg index 23d2d10382b..e44ece64548 100644 Binary files a/website/images/photos/thom-oconnor.jpg and b/website/images/photos/thom-oconnor.jpg differ diff --git a/website/images/photos/tyler-hannan.jpg b/website/images/photos/tyler-hannan.jpg new file mode 100644 index 00000000000..e14d9544bfe Binary files /dev/null and b/website/images/photos/tyler-hannan.jpg differ diff --git a/website/images/photos/vitaly-baranov.jpg b/website/images/photos/vitaly-baranov.jpg index 516b73073c4..2f0df629d5b 100644 Binary files a/website/images/photos/vitaly-baranov.jpg and b/website/images/photos/vitaly-baranov.jpg differ diff --git a/website/images/photos/vladimir-cherkasov.jpg b/website/images/photos/vladimir-cherkasov.jpg index 9d8246be2ae..31c2cb42eee 100644 Binary files a/website/images/photos/vladimir-cherkasov.jpg and b/website/images/photos/vladimir-cherkasov.jpg differ diff --git a/website/images/photos/yossi-kahlon.jpg b/website/images/photos/yossi-kahlon.jpg index dbcff5185c4..4a9672f99bc 100644 Binary files a/website/images/photos/yossi-kahlon.jpg and b/website/images/photos/yossi-kahlon.jpg differ diff --git a/website/images/photos/yury-izrailevsky.jpg b/website/images/photos/yury-izrailevsky.jpg index 785479d4ae3..0115a248e1e 100644 Binary files a/website/images/photos/yury-izrailevsky.jpg and b/website/images/photos/yury-izrailevsky.jpg differ diff --git a/website/js/base.js b/website/js/base.js index 52b801eb98f..6704231c69d 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -55,7 +55,7 @@ $('pre').each(function(_, element) { $(element).prepend( - 'Copy' + 'Copy' ); }); @@ -85,6 +85,9 @@ $(element).append( '' ); + $(element).append( + '' + ); }); } }); diff --git a/website/sitemap-static.xml b/website/sitemap-static.xml index b5b5f3aa0d5..88888e31b3b 100644 --- a/website/sitemap-static.xml +++ b/website/sitemap-static.xml @@ -17,7 +17,7 @@ weekly - https://clickhouse.com/codebrowser/html_report/ClickHouse/index.html + https://clickhouse.com/codebrowser/ClickHouse/index.html daily diff --git a/website/src/scss/greenhouse.scss b/website/src/scss/greenhouse.scss deleted file mode 100644 index 710b606fa15..00000000000 --- a/website/src/scss/greenhouse.scss +++ /dev/null @@ -1,27 +0,0 @@ -#main { - padding-bottom: 0; - padding-top: 0; -} - -#wrapper { - max-width: 1078px; - padding: 0; -} - -body > #wrapper > #main > #wrapper > #logo, -body > #wrapper > #main > #wrapper > h1, -body > #wrapper > #main > #wrapper > #content { - display: none; -} - -body > #wrapper > #main > #wrapper > #board_title { - margin-top: 0; -} - -body > #wrapper > #main > #logo { - margin-top: 80px; -} - -body > #wrapper > #main > :last-child { - margin-bottom: 120px; -} diff --git a/website/templates/common_css.html b/website/templates/common_css.html index ac10b233f25..b26b2bf973e 100644 --- a/website/templates/common_css.html +++ b/website/templates/common_css.html @@ -1,4 +1,4 @@ - + {% for src in extra_css %} diff --git a/website/templates/common_js.html b/website/templates/common_js.html index 72421f00562..93e35d37918 100644 --- a/website/templates/common_js.html +++ b/website/templates/common_js.html @@ -1,4 +1,4 @@ - + {% for src in extra_js %} diff --git a/website/templates/common_meta.html b/website/templates/common_meta.html index 018d533e893..07aa05d28b1 100644 --- a/website/templates/common_meta.html +++ b/website/templates/common_meta.html @@ -7,7 +7,7 @@ {% if title %}{{ title }}{% else %}{{ _('ClickHouse - fast open-source OLAP DBMS') }}{% endif %} - + @@ -15,7 +15,7 @@ {% if page and page.meta.image %} {% else %} - + {% endif %} diff --git a/website/templates/company/team.html b/website/templates/company/team.html index c2e6bfe496d..01341fb4a5f 100644 --- a/website/templates/company/team.html +++ b/website/templates/company/team.html @@ -154,6 +154,19 @@
+ + + +

+ {{ _('Miel Donkers') }} +

+

+ {{ _('Senior Cloud Software Engineer') }} +

+ +
+
+ @@ -229,6 +242,19 @@ {{ _('Associate, Business Strategy & Ops') }}

+
+
+ + + + +

+ {{ _('Tyler Hannan') }} +

+

+ {{ _('Senior Director, Developer Advocacy') }} +

+
@@ -268,6 +294,19 @@ {{ _('Account Executive, AMER') }}

+
+
+ + + + +

+ {{ _('Andy James') }} +

+

+ {{ _('Business Technology Full Stack Lead') }} +

+
@@ -697,6 +736,19 @@ {{ _('Senior Technical Project Manager') }}

+
+
+ + + + +

+ {{ _('Mark Zitnik') }} +

+

+ {{ _('Senior Cloud Software Engineer') }} +

+
diff --git a/website/templates/docs/amp.html b/website/templates/docs/amp.html index 5d2777af188..dc7dd7acb49 100644 --- a/website/templates/docs/amp.html +++ b/website/templates/docs/amp.html @@ -20,7 +20,7 @@
- +
diff --git a/website/templates/docs/nav.html b/website/templates/docs/nav.html index 4d57d282796..afac39c2fab 100644 --- a/website/templates/docs/nav.html +++ b/website/templates/docs/nav.html @@ -1,7 +1,7 @@